Skip to content

Commit

Permalink
sql_text and explain collection, fixed bugs (#351)
Browse files Browse the repository at this point in the history 
* Added collect aql_text and explain for all versions db

* #337: Changed path  the log location

* #350: fixed issue

* #330: Increased timeout to service restart
  • Loading branch information
@kochetovd
kochetovd authored Sep 26, 2024
1 parent 158f02f commit aa248ff
Show file tree
Hide file tree
Showing 8 changed files with 126 additions and 54 deletions.
53 changes: 30 additions & 23 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package config
import (
"database/sql"
"os"
"sync"
"time"

"github.com/advantageous/go-logback/logging"
Expand All @@ -14,32 +15,35 @@ const (
)

var (
DB *sql.DB
DB *sql.DB
SqlText map[string]map[string]string
SqlTextMutex sync.RWMutex
)

type Config struct {
Debug bool `hcl:"debug"`
Env string `hcl:"env"`
Hostname string `hcl:"hostname"`
ApiKey string `hcl:"apikey"`
MetricsPeriod time.Duration `hcl:"interval_seconds"`
ReadConfigPeriod time.Duration `hcl:"interval_read_config_seconds"`
GenerateConfigPeriod time.Duration `hcl:"interval_generate_config_seconds"`
QueryOptimizationPeriod time.Duration `hcl:"interval_query_optimization_seconds"`
MysqlPassword string `hcl:"mysql_password"`
MysqlUser string `hcl:"mysql_user"`
MysqlHost string `hcl:"mysql_host"`
MysqlPort string `hcl:"mysql_port"`
MysqlSslMode bool `hcl:"mysql_ssl_mode"`
CommandRestartService string `hcl:"mysql_restart_service"`
MysqlConfDir string `hcl:"mysql_cnf_dir"`
ReleemConfDir string `hcl:"releem_cnf_dir"`
ReleemDir string `hcl:"releem_dir"`
MemoryLimit int `hcl:"memory_limit"`
InstanceType string `hcl:"instance_type"`
AwsRegion string `hcl:"aws_region"`
AwsRDSDB string `hcl:"aws_rds_db"`
QueryOptimization bool `hcl:"query_optimization"`
Debug bool `hcl:"debug"`
Env string `hcl:"env"`
Hostname string `hcl:"hostname"`
ApiKey string `hcl:"apikey"`
MetricsPeriod time.Duration `hcl:"interval_seconds"`
ReadConfigPeriod time.Duration `hcl:"interval_read_config_seconds"`
GenerateConfigPeriod time.Duration `hcl:"interval_generate_config_seconds"`
QueryOptimizationPeriod time.Duration `hcl:"interval_query_optimization_seconds"`
QueryOptimizationCollectSqlTextPeriod time.Duration `hcl:"interval_query_optimization_collect_sqltext_seconds"`
MysqlPassword string `hcl:"mysql_password"`
MysqlUser string `hcl:"mysql_user"`
MysqlHost string `hcl:"mysql_host"`
MysqlPort string `hcl:"mysql_port"`
MysqlSslMode bool `hcl:"mysql_ssl_mode"`
CommandRestartService string `hcl:"mysql_restart_service"`
MysqlConfDir string `hcl:"mysql_cnf_dir"`
ReleemConfDir string `hcl:"releem_cnf_dir"`
ReleemDir string `hcl:"releem_dir"`
MemoryLimit int `hcl:"memory_limit"`
InstanceType string `hcl:"instance_type"`
AwsRegion string `hcl:"aws_region"`
AwsRDSDB string `hcl:"aws_rds_db"`
QueryOptimization bool `hcl:"query_optimization"`
}

func LoadConfig(filename string, logger logging.Logger) (*Config, error) {
Expand Down Expand Up @@ -72,6 +76,9 @@ func LoadConfigFromString(data string, logger logging.Logger) (*Config, error) {
if config.QueryOptimizationPeriod == 0 {
config.QueryOptimizationPeriod = 3600
}
if config.QueryOptimizationCollectSqlTextPeriod == 0 {
config.QueryOptimizationCollectSqlTextPeriod = 1
}
if config.MysqlHost == "" {
config.MysqlHost = "127.0.0.1"
}
Expand Down
8 changes: 4 additions & 4 deletions errors/releemErrors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,13 @@ func (repeater ReleemErrorsRepeater) ProcessErrors(message string) interface{} {
env = "prod"
}
if env == "dev2" {
api_domain = "https://api.dev2.releem.com/v2/events/saving_agent_errors_log"
api_domain = "https://api.dev2.releem.com/v2/events/agent_errors_log"
} else if env == "dev" {
api_domain = "https://api.dev.releem.com/v2/events/saving_agent_errors_log"
api_domain = "https://api.dev.releem.com/v2/events/agent_errors_log"
} else if env == "stage" {
api_domain = "https://api.stage.releem.com/v2/events/saving_agent_errors_log"
api_domain = "https://api.stage.releem.com/v2/events/agent_errors_log"
} else {
api_domain = "https://api.releem.com/v2/events/saving_agent_errors_log"
api_domain = "https://api.releem.com/v2/events/agent_errors_log"
}
req, err := http.NewRequest(http.MethodPost, api_domain, bodyReader)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/

set -e -E
install_script_version=1.19.2
logfile="releem-install.log"
logfile="/var/log/releem-install.log"

WORKDIR="/opt/releem"
CONF="$WORKDIR/releem.conf"
Expand Down
6 changes: 6 additions & 0 deletions metrics/Metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,3 +96,9 @@ func HandlePanic(configuration *config.Config, logger logging.Logger) {
sender.ProcessErrors(fmt.Sprintf("%+v", err))
}
}

type SqlTextType struct {
CURRENT_SCHEMA string
DIGEST string
SQL_TEXT string
}
44 changes: 34 additions & 10 deletions metrics/dbCollectQueries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,21 @@ func (DbCollectQueriesOptimization *DbCollectQueriesOptimization) GetMetrics(met
DbCollectQueriesOptimization.logger.Error(err)
return err
}
output_digest[query_id] = MetricGroupValue{"schema_name": schema_name, "query_id": query_id, "query": query, "calls": calls, "avg_time_us": avg_time_us, "sum_time_us": sum_time_us, "SUM_LOCK_TIME": SUM_LOCK_TIME, "SUM_ERRORS": SUM_ERRORS, "SUM_WARNINGS": SUM_WARNINGS, "SUM_ROWS_AFFECTED": SUM_ROWS_AFFECTED, "SUM_ROWS_SENT": SUM_ROWS_SENT, "SUM_ROWS_EXAMINED": SUM_ROWS_EXAMINED, "SUM_CREATED_TMP_DISK_TABLES": SUM_CREATED_TMP_DISK_TABLES, "SUM_CREATED_TMP_TABLES": SUM_CREATED_TMP_TABLES, "SUM_SELECT_FULL_JOIN": SUM_SELECT_FULL_JOIN, "SUM_SELECT_FULL_RANGE_JOIN": SUM_SELECT_FULL_RANGE_JOIN, "SUM_SELECT_RANGE": SUM_SELECT_RANGE, "SUM_SELECT_RANGE_CHECK": SUM_SELECT_RANGE_CHECK, "SUM_SELECT_SCAN": SUM_SELECT_SCAN, "SUM_SORT_MERGE_PASSES": SUM_SORT_MERGE_PASSES, "SUM_SORT_RANGE": SUM_SORT_RANGE, "SUM_SORT_ROWS": SUM_SORT_ROWS, "SUM_SORT_SCAN": SUM_SORT_SCAN, "SUM_NO_INDEX_USED": SUM_NO_INDEX_USED, "SUM_NO_GOOD_INDEX_USED": SUM_NO_GOOD_INDEX_USED}
config.SqlTextMutex.RLock()
_, ok_schema_name := config.SqlText[schema_name]
_, ok_query_id := config.SqlText[schema_name][query_id]

if ok_schema_name && ok_query_id {
query_text = config.SqlText[schema_name][query_id]
} else {
query_text = ""
}
config.SqlTextMutex.RUnlock()
output_digest[query_id] = MetricGroupValue{"schema_name": schema_name, "query_id": query_id, "query": query, "query_text": query_text, "calls": calls, "avg_time_us": avg_time_us, "sum_time_us": sum_time_us, "SUM_LOCK_TIME": SUM_LOCK_TIME, "SUM_ERRORS": SUM_ERRORS, "SUM_WARNINGS": SUM_WARNINGS, "SUM_ROWS_AFFECTED": SUM_ROWS_AFFECTED, "SUM_ROWS_SENT": SUM_ROWS_SENT, "SUM_ROWS_EXAMINED": SUM_ROWS_EXAMINED, "SUM_CREATED_TMP_DISK_TABLES": SUM_CREATED_TMP_DISK_TABLES, "SUM_CREATED_TMP_TABLES": SUM_CREATED_TMP_TABLES, "SUM_SELECT_FULL_JOIN": SUM_SELECT_FULL_JOIN, "SUM_SELECT_FULL_RANGE_JOIN": SUM_SELECT_FULL_RANGE_JOIN, "SUM_SELECT_RANGE": SUM_SELECT_RANGE, "SUM_SELECT_RANGE_CHECK": SUM_SELECT_RANGE_CHECK, "SUM_SELECT_SCAN": SUM_SELECT_SCAN, "SUM_SORT_MERGE_PASSES": SUM_SORT_MERGE_PASSES, "SUM_SORT_RANGE": SUM_SORT_RANGE, "SUM_SORT_ROWS": SUM_SORT_ROWS, "SUM_SORT_SCAN": SUM_SORT_SCAN, "SUM_NO_INDEX_USED": SUM_NO_INDEX_USED, "SUM_NO_GOOD_INDEX_USED": SUM_NO_GOOD_INDEX_USED}
}
if DbCollectQueriesOptimization.configuration.QueryOptimization {
CollectionExplain(output_digest, "sum_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration, true)
CollectionExplain(output_digest, "avg_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration, true)
}
}

Expand All @@ -74,8 +88,8 @@ func (DbCollectQueriesOptimization *DbCollectQueriesOptimization) GetMetrics(met
output_digest[query_id] = MetricGroupValue{"schema_name": schema_name, "query_id": query_id, "query": query, "query_text": query_text, "calls": calls, "avg_time_us": avg_time_us, "sum_time_us": sum_time_us, "SUM_LOCK_TIME": SUM_LOCK_TIME, "SUM_ERRORS": SUM_ERRORS, "SUM_WARNINGS": SUM_WARNINGS, "SUM_ROWS_AFFECTED": SUM_ROWS_AFFECTED, "SUM_ROWS_SENT": SUM_ROWS_SENT, "SUM_ROWS_EXAMINED": SUM_ROWS_EXAMINED, "SUM_CREATED_TMP_DISK_TABLES": SUM_CREATED_TMP_DISK_TABLES, "SUM_CREATED_TMP_TABLES": SUM_CREATED_TMP_TABLES, "SUM_SELECT_FULL_JOIN": SUM_SELECT_FULL_JOIN, "SUM_SELECT_FULL_RANGE_JOIN": SUM_SELECT_FULL_RANGE_JOIN, "SUM_SELECT_RANGE": SUM_SELECT_RANGE, "SUM_SELECT_RANGE_CHECK": SUM_SELECT_RANGE_CHECK, "SUM_SELECT_SCAN": SUM_SELECT_SCAN, "SUM_SORT_MERGE_PASSES": SUM_SORT_MERGE_PASSES, "SUM_SORT_RANGE": SUM_SORT_RANGE, "SUM_SORT_ROWS": SUM_SORT_ROWS, "SUM_SORT_SCAN": SUM_SORT_SCAN, "SUM_NO_INDEX_USED": SUM_NO_INDEX_USED, "SUM_NO_GOOD_INDEX_USED": SUM_NO_GOOD_INDEX_USED}
}
if DbCollectQueriesOptimization.configuration.QueryOptimization {
CollectionExplain(output_digest, "sum_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration)
CollectionExplain(output_digest, "avg_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration)
CollectionExplain(output_digest, "sum_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration, false)
CollectionExplain(output_digest, "avg_time_us", DbCollectQueriesOptimization.logger, DbCollectQueriesOptimization.configuration, false)
}
}
for _, value := range output_digest {
Expand Down Expand Up @@ -362,8 +376,8 @@ func (DbCollectQueriesOptimization *DbCollectQueriesOptimization) GetMetrics(met

}

func CollectionExplain(digests map[string]MetricGroupValue, field_sorting string, logger logging.Logger, configuration *config.Config) {
var explain, schema_name_conn string
func CollectionExplain(digests map[string]MetricGroupValue, field_sorting string, logger logging.Logger, configuration *config.Config, is_mysql57 bool) {
var explain, schema_name_conn, query_text string
var i int
var db *sql.DB

Expand All @@ -385,14 +399,21 @@ func CollectionExplain(digests map[string]MetricGroupValue, field_sorting string
(strings.Contains(digests[k]["query_text"].(string), "SELECT") || strings.Contains(digests[k]["query_text"].(string), "select")) &&
digests[k]["explain"] == nil {

if digests[k]["query_text"].(string) == "" {
continue
}
if strings.Contains(digests[k]["query_text"].(string), "EXPLAIN FORMAT=JSON") {
continue
}

if (strings.Contains(digests[k]["query_text"].(string), "SELECT") || strings.Contains(digests[k]["query_text"].(string), "select")) &&
strings.Contains(digests[k]["query_text"].(string), "SQL_NO_CACHE") &&
!(strings.Contains(digests[k]["query_text"].(string), "WHERE") || strings.Contains(digests[k]["query_text"].(string), "where")) {
logger.Debug("Query From mysqldump", digests[k]["query_text"].(string))
continue
}

if strings.HasSuffix(digests[k]["query_text"].(string), "...") {
if strings.HasSuffix(digests[k]["query"].(string), "...") || strings.HasSuffix(digests[k]["query_text"].(string), "...") {
digests[k]["explain"] = "need_full_query"
logger.Debug("need_full_query") //, digests[k]["query_text"].(string))
continue
Expand All @@ -405,11 +426,14 @@ func CollectionExplain(digests map[string]MetricGroupValue, field_sorting string
defer db.Close()
schema_name_conn = digests[k]["schema_name"].(string)
}
err := db.QueryRow("EXPLAIN FORMAT=JSON " + strings.Replace(digests[k]["query_text"].(string), "\"", "'", -1)).Scan(&explain)
if is_mysql57 {
query_text = strings.Replace(digests[k]["query_text"].(string), "\"", "`", -1)
} else {
query_text = strings.Replace(digests[k]["query_text"].(string), "\"", "'", -1)
}
err := db.QueryRow("EXPLAIN FORMAT=JSON " + query_text).Scan(&explain)
if err != nil {
logger.DebugError("Explain Error", err)
//logger.Println(digests[k]["query_text"].(string))
//digests[k]["explain"] = err.Error()
logger.PrintError("Explain Error", err)
} else {
logger.Debug(i, "OK")
digests[k]["explain"] = explain
Expand Down
36 changes: 33 additions & 3 deletions metrics/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os/exec"
"os/signal"
"strings"
"sync"
"syscall"
"time"

Expand Down Expand Up @@ -37,17 +38,21 @@ func RunWorker(gatherers []MetricsGatherer, gatherers_configuration []MetricsGat
}
}

logger.Debug(configuration)
if (Mode.Name == "Configurations" && Mode.ModeType != "default") || Mode.Name == "Event" || Mode.Name == "TaskSet" {
GenerateTimer = time.NewTimer(0 * time.Second)
timer = time.NewTimer(3600 * time.Second)
} else {
GenerateTimer = time.NewTimer(configuration.GenerateConfigPeriod * time.Second)
timer = time.NewTimer(1 * time.Second)
}
QueryOptimizationTimer = time.NewTimer(10 * time.Second)
QueryOptimizationTimer = time.NewTimer(60 * time.Second)
QueryOptimizationCollectSqlText := time.NewTimer(1 * time.Second)
config.SqlText = make(map[string]map[string]string)
config.SqlTextMutex = sync.RWMutex{}

if !configuration.QueryOptimization {
QueryOptimizationTimer.Stop()
QueryOptimizationCollectSqlText.Stop()
}
terminator := makeTerminateChannel()
for {
Expand Down Expand Up @@ -113,8 +118,33 @@ func RunWorker(gatherers []MetricsGatherer, gatherers_configuration []MetricsGat
}
logger.Println("Saved a queries...")
}()
case <-QueryOptimizationCollectSqlText.C:
QueryOptimizationCollectSqlText.Reset(configuration.QueryOptimizationCollectSqlTextPeriod * time.Second)
go func() {
defer HandlePanic(configuration, logger)
Ready = false
var SqlText_elem SqlTextType
rows, err := config.DB.Query("SELECT CURRENT_SCHEMA, DIGEST, SQL_TEXT FROM performance_schema.events_statements_history WHERE DIGEST IS NOT NULL AND CURRENT_SCHEMA IS NOT NULL GROUP BY current_schema, digest")
if err != nil {
logger.Error(err)
} else {
for rows.Next() {
err := rows.Scan(&SqlText_elem.CURRENT_SCHEMA, &SqlText_elem.DIGEST, &SqlText_elem.SQL_TEXT)
if err != nil {
logger.Error(err)
} else {
config.SqlTextMutex.Lock()
if config.SqlText[SqlText_elem.CURRENT_SCHEMA] == nil {
config.SqlText[SqlText_elem.CURRENT_SCHEMA] = make(map[string]string)
}
config.SqlText[SqlText_elem.CURRENT_SCHEMA][SqlText_elem.DIGEST] = SqlText_elem.SQL_TEXT
config.SqlTextMutex.Unlock()
}
}
}

}()
}
logger.Info("LOOP")
}
}

Expand Down
27 changes: 14 additions & 13 deletions mysqlconfigurer.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
# All rights reserved

export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
export EXTEND_TIMEOUT_USEC=18000000000

# Variables
MYSQLCONFIGURER_PATH="/opt/releem/conf/"
Expand All @@ -16,7 +17,7 @@ MYSQLCONFIGURER_CONFIGFILE="${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}"
MYSQL_MEMORY_LIMIT=0
VERSION="1.19.2"
RELEEM_INSTALL_PATH=$MYSQLCONFIGURER_PATH"install.sh"
logfile="releem-mysqlconfigurer.log"
logfile="/var/log/releem-mysqlconfigurer.log"

# Set up a named pipe for logging
npipe=/tmp/$$.mysqlconfigurer.tmp
Expand Down Expand Up @@ -132,10 +133,6 @@ function releem_rollback_config() {
printf "\033[37m\n * Try to reinstall Releem Agent, and please set the my.cnf location.\033[0m"
exit 3;
fi
if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then
printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m"
exit 4;
fi

FLAG_RESTART_SERVICE=1
if [ -z "$RELEEM_RESTART_SERVICE" ]; then
Expand All @@ -161,6 +158,10 @@ function releem_rollback_config() {
cp -f "${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}.bkp" "${RELEEM_MYSQL_CONFIG_DIR}/${MYSQLCONFIGURER_FILE_NAME}"
fi

if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then
printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m"
exit 4;
fi
printf "\033[31m\n * Restarting with command '$RELEEM_MYSQL_RESTART_SERVICE'...\033[0m\n"
eval "$RELEEM_MYSQL_RESTART_SERVICE" &
wait_restart $!
Expand Down Expand Up @@ -295,10 +296,6 @@ function releem_apply_manual() {
printf "\033[37m\n * Try to reinstall Releem Agent, and please set the my.cnf location.\033[0m"
exit 3;
fi
if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then
printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m"
exit 4;
fi
diff_cmd=$(which diff || true)
if [ -n "$diff_cmd" ];then
diff "${RELEEM_MYSQL_CONFIG_DIR}/${MYSQLCONFIGURER_FILE_NAME}" "$MYSQLCONFIGURER_CONFIGFILE" > /dev/null 2>&1
Expand Down Expand Up @@ -332,6 +329,10 @@ function releem_apply_manual() {
fi
yes | cp -fr $MYSQLCONFIGURER_CONFIGFILE $RELEEM_MYSQL_CONFIG_DIR/
chmod 644 $RELEEM_MYSQL_CONFIG_DIR/*
if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then
printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m"
exit 4;
fi

#echo "-------Test config-------"
printf "\n`date +%Y%m%d-%H:%M:%S`\033[37m Restarting MySQL with the command '$RELEEM_MYSQL_RESTART_SERVICE'...\033[0m\n"
Expand Down Expand Up @@ -384,10 +385,6 @@ function releem_apply_automatic() {
printf "\033[37m\n * Try to reinstall Releem Agent, and please set the my.cnf location.\033[0m"
exit 3;
fi
if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then
printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m"
exit 4;
fi

FLAG_RESTART_SERVICE=1
if [ -z "$RELEEM_RESTART_SERVICE" ]; then
Expand All @@ -411,6 +408,10 @@ function releem_apply_automatic() {
chmod 644 $RELEEM_MYSQL_CONFIG_DIR/*

if [ "$FLAG_RESTART_SERVICE" -ne 0 ]; then
if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then
printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m"
exit 4;
fi
#echo "-------Test config-------"
printf "\n`date +%Y%m%d-%H:%M:%S`\033[37m Restarting MySQL with the command '$RELEEM_MYSQL_RESTART_SERVICE'...\033[0m\n"
eval "$RELEEM_MYSQL_RESTART_SERVICE" &
Expand Down
4 changes: 4 additions & 0 deletions releem.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ interval_generate_config_seconds=43200
# Defaults to 3600 seconds, how often query metrics are collected.
interval_query_optimization_seconds=3600

# QueryOptimizationCollectSqlTextPeriod time.Duration `hcl:"interval_query_optimization_collect_sqltext_seconds"`
# Defaults to 1 seconds, how often query sql text are collected.
interval_query_optimization_collect_sqltext_seconds=1

# MysqlUser string`hcl:"mysql_user"`
# Mysql user name for collection metrics.
mysql_user="releem"
Expand Down

0 comments on commit aa248ff

Please sign in to comment.