diff --git a/config/config.go b/config/config.go index 2bb282a..c332b18 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config import ( "database/sql" "os" + "sync" "time" "github.com/advantageous/go-logback/logging" @@ -14,32 +15,35 @@ const ( ) var ( - DB *sql.DB + DB *sql.DB + SqlText map[string]map[string]string + SqlTextMutex sync.RWMutex ) type Config struct { - Debug bool `hcl:"debug"` - Env string `hcl:"env"` - Hostname string `hcl:"hostname"` - ApiKey string `hcl:"apikey"` - MetricsPeriod time.Duration `hcl:"interval_seconds"` - ReadConfigPeriod time.Duration `hcl:"interval_read_config_seconds"` - GenerateConfigPeriod time.Duration `hcl:"interval_generate_config_seconds"` - QueryOptimizationPeriod time.Duration `hcl:"interval_query_optimization_seconds"` - MysqlPassword string `hcl:"mysql_password"` - MysqlUser string `hcl:"mysql_user"` - MysqlHost string `hcl:"mysql_host"` - MysqlPort string `hcl:"mysql_port"` - MysqlSslMode bool `hcl:"mysql_ssl_mode"` - CommandRestartService string `hcl:"mysql_restart_service"` - MysqlConfDir string `hcl:"mysql_cnf_dir"` - ReleemConfDir string `hcl:"releem_cnf_dir"` - ReleemDir string `hcl:"releem_dir"` - MemoryLimit int `hcl:"memory_limit"` - InstanceType string `hcl:"instance_type"` - AwsRegion string `hcl:"aws_region"` - AwsRDSDB string `hcl:"aws_rds_db"` - QueryOptimization bool `hcl:"query_optimization"` + Debug bool `hcl:"debug"` + Env string `hcl:"env"` + Hostname string `hcl:"hostname"` + ApiKey string `hcl:"apikey"` + MetricsPeriod time.Duration `hcl:"interval_seconds"` + ReadConfigPeriod time.Duration `hcl:"interval_read_config_seconds"` + GenerateConfigPeriod time.Duration `hcl:"interval_generate_config_seconds"` + QueryOptimizationPeriod time.Duration `hcl:"interval_query_optimization_seconds"` + QueryOptimizationCollectSqlTextPeriod time.Duration `hcl:"interval_query_optimization_collect_sqltext_seconds"` + MysqlPassword string `hcl:"mysql_password"` + MysqlUser string `hcl:"mysql_user"` + MysqlHost string `hcl:"mysql_host"` + MysqlPort string `hcl:"mysql_port"` + MysqlSslMode bool `hcl:"mysql_ssl_mode"` + CommandRestartService string `hcl:"mysql_restart_service"` + MysqlConfDir string `hcl:"mysql_cnf_dir"` + ReleemConfDir string `hcl:"releem_cnf_dir"` + ReleemDir string `hcl:"releem_dir"` + MemoryLimit int `hcl:"memory_limit"` + InstanceType string `hcl:"instance_type"` + AwsRegion string `hcl:"aws_region"` + AwsRDSDB string `hcl:"aws_rds_db"` + QueryOptimization bool `hcl:"query_optimization"` } func LoadConfig(filename string, logger logging.Logger) (*Config, error) { @@ -72,6 +76,9 @@ func LoadConfigFromString(data string, logger logging.Logger) (*Config, error) { if config.QueryOptimizationPeriod == 0 { config.QueryOptimizationPeriod = 3600 } + if config.QueryOptimizationCollectSqlTextPeriod == 0 { + config.QueryOptimizationCollectSqlTextPeriod = 1 + } if config.MysqlHost == "" { config.MysqlHost = "127.0.0.1" } diff --git a/errors/releemErrors.go b/errors/releemErrors.go index ca7f8dc..8850630 100644 --- a/errors/releemErrors.go +++ b/errors/releemErrors.go @@ -27,13 +27,13 @@ func (repeater ReleemErrorsRepeater) ProcessErrors(message string) interface{} { env = "prod" } if env == "dev2" { - api_domain = "https://api.dev2.releem.com/v2/events/saving_agent_errors_log" + api_domain = "https://api.dev2.releem.com/v2/events/agent_errors_log" } else if env == "dev" { - api_domain = "https://api.dev.releem.com/v2/events/saving_agent_errors_log" + api_domain = "https://api.dev.releem.com/v2/events/agent_errors_log" } else if env == "stage" { - api_domain = "https://api.stage.releem.com/v2/events/saving_agent_errors_log" + api_domain = "https://api.stage.releem.com/v2/events/agent_errors_log" } else { - api_domain = "https://api.releem.com/v2/events/saving_agent_errors_log" + api_domain = "https://api.releem.com/v2/events/agent_errors_log" } req, err := http.NewRequest(http.MethodPost, api_domain, bodyReader) if err != nil { diff --git a/install.sh b/install.sh index 2a6d589..84efcc1 100644 --- a/install.sh +++ b/install.sh @@ -10,7 +10,7 @@ export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ set -e -E install_script_version=1.19.2 -logfile="releem-install.log" +logfile="/var/log/releem-install.log" WORKDIR="/opt/releem" CONF="$WORKDIR/releem.conf" diff --git a/metrics/Metrics.go b/metrics/Metrics.go index 310e0b2..8fdb6c2 100644 --- a/metrics/Metrics.go +++ b/metrics/Metrics.go @@ -96,3 +96,9 @@ func HandlePanic(configuration *config.Config, logger logging.Logger) { sender.ProcessErrors(fmt.Sprintf("%+v", err)) } } + +type SqlTextType struct { + CURRENT_SCHEMA string + DIGEST string + SQL_TEXT string +} diff --git a/metrics/dbCollectQueries.go b/metrics/dbCollectQueries.go index b4307e0..c71e480 100644 --- a/metrics/dbCollectQueries.go +++ b/metrics/dbCollectQueries.go @@ -60,7 +60,21 @@ func (DbCollectQueriesOptimization *DbCollectQueriesOptimization) GetMetrics(met DbCollectQueriesOptimization.logger.Error(err) return err } - output_digest[query_id] = MetricGroupValue{"schema_name": schema_name, "query_id": query_id, "query": query, "calls": calls, "avg_time_us": avg_time_us, "sum_time_us": sum_time_us, "SUM_LOCK_TIME": SUM_LOCK_TIME, "SUM_ERRORS": SUM_ERRORS, "SUM_WARNINGS": SUM_WARNINGS, "SUM_ROWS_AFFECTED": SUM_ROWS_AFFECTED, "SUM_ROWS_SENT": SUM_ROWS_SENT, "SUM_ROWS_EXAMINED": SUM_ROWS_EXAMINED, "SUM_CREATED_TMP_DISK_TABLES": SUM_CREATED_TMP_DISK_TABLES, "SUM_CREATED_TMP_TABLES": SUM_CREATED_TMP_TABLES, "SUM_SELECT_FULL_JOIN": SUM_SELECT_FULL_JOIN, "SUM_SELECT_FULL_RANGE_JOIN": SUM_SELECT_FULL_RANGE_JOIN, "SUM_SELECT_RANGE": SUM_SELECT_RANGE, "SUM_SELECT_RANGE_CHECK": SUM_SELECT_RANGE_CHECK, "SUM_SELECT_SCAN": SUM_SELECT_SCAN, "SUM_SORT_MERGE_PASSES": SUM_SORT_MERGE_PASSES, "SUM_SORT_RANGE": SUM_SORT_RANGE, "SUM_SORT_ROWS": SUM_SORT_ROWS, "SUM_SORT_SCAN": SUM_SORT_SCAN, "SUM_NO_INDEX_USED": SUM_NO_INDEX_USED, "SUM_NO_GOOD_INDEX_USED": SUM_NO_GOOD_INDEX_USED} + config.SqlTextMutex.RLock() + _, ok_schema_name := config.SqlText[schema_name] + _, ok_query_id := config.SqlText[schema_name][query_id] + + if ok_schema_name && ok_query_id { + query_text = config.SqlText[schema_name][query_id] + } else { + query_text = "" + } + config.SqlTextMutex.RUnlock() + output_digest[query_id] = MetricGroupValue{"schema_name": schema_name, "query_id": query_id, "query": query, "query_text": query_text, "calls": calls, "avg_time_us": avg_time_us, "sum_time_us": sum_time_us, "SUM_LOCK_TIME": SUM_LOCK_TIME, "SUM_ERRORS": SUM_ERRORS, "SUM_WARNINGS": SUM_WARNINGS, "SUM_ROWS_AFFECTED": SUM_ROWS_AFFECTED, "SUM_ROWS_SENT": SUM_ROWS_SENT, "SUM_ROWS_EXAMINED": SUM_ROWS_EXAMINED, "SUM_CREATED_TMP_DISK_TABLES": SUM_CREATED_TMP_DISK_TABLES, "SUM_CREATED_TMP_TABLES": SUM_CREATED_TMP_TABLES, "SUM_SELECT_FULL_JOIN": SUM_SELECT_FULL_JOIN, "SUM_SELECT_FULL_RANGE_JOIN": SUM_SELECT_FULL_RANGE_JOIN, "SUM_SELECT_RANGE": SUM_SELECT_RANGE, "SUM_SELECT_RANGE_CHECK": SUM_SELECT_RANGE_CHECK, "SUM_SELECT_SCAN": SUM_SELECT_SCAN, "SUM_SORT_MERGE_PASSES": SUM_SORT_MERGE_PASSES, "SUM_SORT_RANGE": SUM_SORT_RANGE, "SUM_SORT_ROWS": SUM_SORT_ROWS, "SUM_SORT_SCAN": SUM_SORT_SCAN, "SUM_NO_INDEX_USED": SUM_NO_INDEX_USED, "SUM_NO_GOOD_INDEX_USED": SUM_NO_GOOD_INDEX_USED} + } + if DbCollectQueriesOptimization.configuration.QueryOptimization { + CollectionExplain(output_digest, "sum_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration, true) + CollectionExplain(output_digest, "avg_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration, true) } } @@ -74,8 +88,8 @@ func (DbCollectQueriesOptimization *DbCollectQueriesOptimization) GetMetrics(met output_digest[query_id] = MetricGroupValue{"schema_name": schema_name, "query_id": query_id, "query": query, "query_text": query_text, "calls": calls, "avg_time_us": avg_time_us, "sum_time_us": sum_time_us, "SUM_LOCK_TIME": SUM_LOCK_TIME, "SUM_ERRORS": SUM_ERRORS, "SUM_WARNINGS": SUM_WARNINGS, "SUM_ROWS_AFFECTED": SUM_ROWS_AFFECTED, "SUM_ROWS_SENT": SUM_ROWS_SENT, "SUM_ROWS_EXAMINED": SUM_ROWS_EXAMINED, "SUM_CREATED_TMP_DISK_TABLES": SUM_CREATED_TMP_DISK_TABLES, "SUM_CREATED_TMP_TABLES": SUM_CREATED_TMP_TABLES, "SUM_SELECT_FULL_JOIN": SUM_SELECT_FULL_JOIN, "SUM_SELECT_FULL_RANGE_JOIN": SUM_SELECT_FULL_RANGE_JOIN, "SUM_SELECT_RANGE": SUM_SELECT_RANGE, "SUM_SELECT_RANGE_CHECK": SUM_SELECT_RANGE_CHECK, "SUM_SELECT_SCAN": SUM_SELECT_SCAN, "SUM_SORT_MERGE_PASSES": SUM_SORT_MERGE_PASSES, "SUM_SORT_RANGE": SUM_SORT_RANGE, "SUM_SORT_ROWS": SUM_SORT_ROWS, "SUM_SORT_SCAN": SUM_SORT_SCAN, "SUM_NO_INDEX_USED": SUM_NO_INDEX_USED, "SUM_NO_GOOD_INDEX_USED": SUM_NO_GOOD_INDEX_USED} } if DbCollectQueriesOptimization.configuration.QueryOptimization { - CollectionExplain(output_digest, "sum_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration) - CollectionExplain(output_digest, "avg_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration) + CollectionExplain(output_digest, "sum_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration, false) + CollectionExplain(output_digest, "avg_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration, false) } } for _, value := range output_digest { @@ -362,8 +376,8 @@ func (DbCollectQueriesOptimization *DbCollectQueriesOptimization) GetMetrics(met } -func CollectionExplain(digests map[string]MetricGroupValue, field_sorting string, logger logging.Logger, configuration *config.Config) { - var explain, schema_name_conn string +func CollectionExplain(digests map[string]MetricGroupValue, field_sorting string, logger logging.Logger, configuration *config.Config, is_mysql57 bool) { + var explain, schema_name_conn, query_text string var i int var db *sql.DB @@ -385,6 +399,13 @@ func CollectionExplain(digests map[string]MetricGroupValue, field_sorting string (strings.Contains(digests[k]["query_text"].(string), "SELECT") || strings.Contains(digests[k]["query_text"].(string), "select")) && digests[k]["explain"] == nil { + if digests[k]["query_text"].(string) == "" { + continue + } + if strings.Contains(digests[k]["query_text"].(string), "EXPLAIN FORMAT=JSON") { + continue + } + if (strings.Contains(digests[k]["query_text"].(string), "SELECT") || strings.Contains(digests[k]["query_text"].(string), "select")) && strings.Contains(digests[k]["query_text"].(string), "SQL_NO_CACHE") && !(strings.Contains(digests[k]["query_text"].(string), "WHERE") || strings.Contains(digests[k]["query_text"].(string), "where")) { @@ -392,7 +413,7 @@ func CollectionExplain(digests map[string]MetricGroupValue, field_sorting string continue } - if strings.HasSuffix(digests[k]["query_text"].(string), "...") { + if strings.HasSuffix(digests[k]["query"].(string), "...") || strings.HasSuffix(digests[k]["query_text"].(string), "...") { digests[k]["explain"] = "need_full_query" logger.Debug("need_full_query") //, digests[k]["query_text"].(string)) continue @@ -405,11 +426,14 @@ func CollectionExplain(digests map[string]MetricGroupValue, field_sorting string defer db.Close() schema_name_conn = digests[k]["schema_name"].(string) } - err := db.QueryRow("EXPLAIN FORMAT=JSON " + strings.Replace(digests[k]["query_text"].(string), "\"", "'", -1)).Scan(&explain) + if is_mysql57 { + query_text = strings.Replace(digests[k]["query_text"].(string), "\"", "`", -1) + } else { + query_text = strings.Replace(digests[k]["query_text"].(string), "\"", "'", -1) + } + err := db.QueryRow("EXPLAIN FORMAT=JSON " + query_text).Scan(&explain) if err != nil { - logger.DebugError("Explain Error", err) - //logger.Println(digests[k]["query_text"].(string)) - //digests[k]["explain"] = err.Error() + logger.PrintError("Explain Error", err) } else { logger.Debug(i, "OK") digests[k]["explain"] = explain diff --git a/metrics/runner.go b/metrics/runner.go index 2ab9ba2..4f7080a 100644 --- a/metrics/runner.go +++ b/metrics/runner.go @@ -7,6 +7,7 @@ import ( "os/exec" "os/signal" "strings" + "sync" "syscall" "time" @@ -37,7 +38,6 @@ func RunWorker(gatherers []MetricsGatherer, gatherers_configuration []MetricsGat } } - logger.Debug(configuration) if (Mode.Name == "Configurations" && Mode.ModeType != "default") || Mode.Name == "Event" || Mode.Name == "TaskSet" { GenerateTimer = time.NewTimer(0 * time.Second) timer = time.NewTimer(3600 * time.Second) @@ -45,9 +45,14 @@ func RunWorker(gatherers []MetricsGatherer, gatherers_configuration []MetricsGat GenerateTimer = time.NewTimer(configuration.GenerateConfigPeriod * time.Second) timer = time.NewTimer(1 * time.Second) } - QueryOptimizationTimer = time.NewTimer(10 * time.Second) + QueryOptimizationTimer = time.NewTimer(60 * time.Second) + QueryOptimizationCollectSqlText := time.NewTimer(1 * time.Second) + config.SqlText = make(map[string]map[string]string) + config.SqlTextMutex = sync.RWMutex{} + if !configuration.QueryOptimization { QueryOptimizationTimer.Stop() + QueryOptimizationCollectSqlText.Stop() } terminator := makeTerminateChannel() for { @@ -113,8 +118,33 @@ func RunWorker(gatherers []MetricsGatherer, gatherers_configuration []MetricsGat } logger.Println("Saved a queries...") }() + case <-QueryOptimizationCollectSqlText.C: + QueryOptimizationCollectSqlText.Reset(configuration.QueryOptimizationCollectSqlTextPeriod * time.Second) + go func() { + defer HandlePanic(configuration, logger) + Ready = false + var SqlText_elem SqlTextType + rows, err := config.DB.Query("SELECT CURRENT_SCHEMA, DIGEST, SQL_TEXT FROM performance_schema.events_statements_history WHERE DIGEST IS NOT NULL AND CURRENT_SCHEMA IS NOT NULL GROUP BY current_schema, digest") + if err != nil { + logger.Error(err) + } else { + for rows.Next() { + err := rows.Scan(&SqlText_elem.CURRENT_SCHEMA, &SqlText_elem.DIGEST, &SqlText_elem.SQL_TEXT) + if err != nil { + logger.Error(err) + } else { + config.SqlTextMutex.Lock() + if config.SqlText[SqlText_elem.CURRENT_SCHEMA] == nil { + config.SqlText[SqlText_elem.CURRENT_SCHEMA] = make(map[string]string) + } + config.SqlText[SqlText_elem.CURRENT_SCHEMA][SqlText_elem.DIGEST] = SqlText_elem.SQL_TEXT + config.SqlTextMutex.Unlock() + } + } + } + + }() } - logger.Info("LOOP") } } diff --git a/mysqlconfigurer.sh b/mysqlconfigurer.sh index dafb185..a33275d 100755 --- a/mysqlconfigurer.sh +++ b/mysqlconfigurer.sh @@ -4,6 +4,7 @@ # All rights reserved export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin +export EXTEND_TIMEOUT_USEC=18000000000 # Variables MYSQLCONFIGURER_PATH="/opt/releem/conf/" @@ -16,7 +17,7 @@ MYSQLCONFIGURER_CONFIGFILE="${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}" MYSQL_MEMORY_LIMIT=0 VERSION="1.19.2" RELEEM_INSTALL_PATH=$MYSQLCONFIGURER_PATH"install.sh" -logfile="releem-mysqlconfigurer.log" +logfile="/var/log/releem-mysqlconfigurer.log" # Set up a named pipe for logging npipe=/tmp/$$.mysqlconfigurer.tmp @@ -132,10 +133,6 @@ function releem_rollback_config() { printf "\033[37m\n * Try to reinstall Releem Agent, and please set the my.cnf location.\033[0m" exit 3; fi - if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then - printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m" - exit 4; - fi FLAG_RESTART_SERVICE=1 if [ -z "$RELEEM_RESTART_SERVICE" ]; then @@ -161,6 +158,10 @@ function releem_rollback_config() { cp -f "${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}.bkp" "${RELEEM_MYSQL_CONFIG_DIR}/${MYSQLCONFIGURER_FILE_NAME}" fi + if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then + printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m" + exit 4; + fi printf "\033[31m\n * Restarting with command '$RELEEM_MYSQL_RESTART_SERVICE'...\033[0m\n" eval "$RELEEM_MYSQL_RESTART_SERVICE" & wait_restart $! @@ -295,10 +296,6 @@ function releem_apply_manual() { printf "\033[37m\n * Try to reinstall Releem Agent, and please set the my.cnf location.\033[0m" exit 3; fi - if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then - printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m" - exit 4; - fi diff_cmd=$(which diff || true) if [ -n "$diff_cmd" ];then diff "${RELEEM_MYSQL_CONFIG_DIR}/${MYSQLCONFIGURER_FILE_NAME}" "$MYSQLCONFIGURER_CONFIGFILE" > /dev/null 2>&1 @@ -332,6 +329,10 @@ function releem_apply_manual() { fi yes | cp -fr $MYSQLCONFIGURER_CONFIGFILE $RELEEM_MYSQL_CONFIG_DIR/ chmod 644 $RELEEM_MYSQL_CONFIG_DIR/* + if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then + printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m" + exit 4; + fi #echo "-------Test config-------" printf "\n`date +%Y%m%d-%H:%M:%S`\033[37m Restarting MySQL with the command '$RELEEM_MYSQL_RESTART_SERVICE'...\033[0m\n" @@ -384,10 +385,6 @@ function releem_apply_automatic() { printf "\033[37m\n * Try to reinstall Releem Agent, and please set the my.cnf location.\033[0m" exit 3; fi - if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then - printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m" - exit 4; - fi FLAG_RESTART_SERVICE=1 if [ -z "$RELEEM_RESTART_SERVICE" ]; then @@ -411,6 +408,10 @@ function releem_apply_automatic() { chmod 644 $RELEEM_MYSQL_CONFIG_DIR/* if [ "$FLAG_RESTART_SERVICE" -ne 0 ]; then + if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then + printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m" + exit 4; + fi #echo "-------Test config-------" printf "\n`date +%Y%m%d-%H:%M:%S`\033[37m Restarting MySQL with the command '$RELEEM_MYSQL_RESTART_SERVICE'...\033[0m\n" eval "$RELEEM_MYSQL_RESTART_SERVICE" & diff --git a/releem.conf b/releem.conf index 657356b..a3ff857 100644 --- a/releem.conf +++ b/releem.conf @@ -33,6 +33,10 @@ interval_generate_config_seconds=43200 # Defaults to 3600 seconds, how often query metrics are collected. interval_query_optimization_seconds=3600 +# QueryOptimizationCollectSqlTextPeriod time.Duration `hcl:"interval_query_optimization_collect_sqltext_seconds"` +# Defaults to 1 seconds, how often query sql text are collected. +interval_query_optimization_collect_sqltext_seconds=1 + # MysqlUser string`hcl:"mysql_user"` # Mysql user name for collection metrics. mysql_user="releem"