Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@systay
systay committed Dec 5, 2024
1 parent b5e74f3 commit 8986648
Show file tree
Hide file tree
Showing 3 changed files with 194 additions and 1 deletion.
15 changes: 14 additions & 1 deletion go/cmd/keys.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

func keysCmd() *cobra.Command {
var inputType string
var csvConfig data.CSVConfig

cmd := &cobra.Command{
Use: "keys ",
Expand All @@ -49,17 +50,29 @@ func keysCmd() *cobra.Command {
}

addInputTypeFlag(cmd, &inputType)
addCSVConfigFlag(cmd, &csvConfig)

return cmd
}

const allowedInputTypes = "'sql', 'mysql-log' or 'vtgate-log'"
const allowedInputTypes = "'sql', 'mysql-log', 'vtgate-log', 'csv'"

func addInputTypeFlag(cmd *cobra.Command, s *string) {
*s = "sql"
cmd.Flags().StringVar(s, "input-type", "sql", fmt.Sprintf("Specifies the type of input file: %s", allowedInputTypes))
}

func addCSVConfigFlag(cmd *cobra.Command, c *data.CSVConfig) {
cmd.Flags().BoolVar(&c.Header, "csv-header", false, "Indicates that the CSV file has a header row")
cmd.Flags().StringVar(&c.QueryField, "csv-query-field", "", "Column index or name for the query field (required)")
cmd.Flags().StringVar(&c.ConnectionIDField, "csv-connection-id-field", "", "Column index or name for the connection ID field")
cmd.Flags().StringVar(&c.QueryTimeField, "csv-query-time-field", "", "Column index or name for the query time field")
cmd.Flags().StringVar(&c.LockTimeField, "csv-lock-time-field", "", "Column index or name for the lock time field")
cmd.Flags().StringVar(&c.RowsSentField, "csv-rows-sent-field", "", "Column index or name for the rows sent field")
cmd.Flags().StringVar(&c.RowsExaminedField, "csv-rows-examined-field", "", "Column index or name for the rows examined field")
cmd.Flags().StringVar(&c.TimestampField, "csv-timestamp-field", "", "Column index or name for the timestamp field")
}

func configureLoader(inputType string, needsBindVars bool) (data.Loader, error) {
switch inputType {
case "sql":
Expand Down
80 changes: 80 additions & 0 deletions go/data/csv_loader.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
Copyright 2024 The Vitess Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package data

import (
"encoding/csv"
"os"
)

type (
CSVLogLoader struct {
Config CSVConfig
}

CSVConfig struct {
Header bool

QueryField string
ConnectionIDField string
QueryTimeField string
LockTimeField string
RowsSentField string
RowsExaminedField string
TimestampField string
}

csvLogReaderState struct {
file *os.File
reader *csv.Reader

QueryField int
ConnectionIDField int
QueryTimeField int
LockTimeField int
RowsSentField int
RowsExaminedField int
TimestampField int
}
)

func (c CSVLogLoader) Load(fileName string) IteratorLoader {
fd, err := os.OpenFile(fileName, os.O_RDONLY, 0)
if err != nil {
return &errLoader{err}
}

reader := csv.NewReader(fd)
logReader := &csvLogReaderState{
file: fd,
reader: reader,
}

if c.Config.Header {
header := reader.Read()

Check failure on line 68 in go/data/csv_loader.go

View workflow job for this annotation

GitHub Actions / test 

declared and not used: header

Check failure on line 68 in go/data/csv_loader.go

View workflow job for this annotation

GitHub Actions / test 

assignment mismatch: 1 variable but reader.Read returns 2 values
}

return logReader
}

func (c *csvLogReaderState) Next() (Query, bool) {
c.reader.Read()
}

Check failure on line 76 in go/data/csv_loader.go

View workflow job for this annotation

GitHub Actions / test 

missing return

func (c *csvLogReaderState) Close() error {
return c.file.Close()
}
100 changes: 100 additions & 0 deletions go/testdata/rds.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
2023-12-19T18:01:34.672Z # Time: 2023-12-19T18:01:34.672281Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637665
# Query_time: 0.000012 Lock_time: 0.000000 Rows_sent: 413 Rows_examined: 413
SET timestamp=1703008894;
# administrator command: Ping;
2023-12-19T18:01:34.672Z # Time: 2023-12-19T18:01:34.672807Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637665
# Query_time: 0.000042 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 1
SET timestamp=1703008894;
SELECT @@version;
2023-12-19T18:01:34.673Z # Time: 2023-12-19T18:01:34.673964Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637665
# Query_time: 0.000661 Lock_time: 0.000002 Rows_sent: 1 Rows_examined: 16
SET timestamp=1703008894;
SELECT
column_name
FROM information_schema.columns
WHERE table_schema = 'information_schema'
AND table_name = 'INNODB_METRICS'
AND column_name IN ('status', 'enabled')
LIMIT 1;
2023-12-19T18:01:34.674Z # Time: 2023-12-19T18:01:34.674198Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637664
# Query_time: 0.000815 Lock_time: 0.000001 Rows_sent: 413 Rows_examined: 413
SET timestamp=1703008894;
SHOW GLOBAL STATUS;
2023-12-19T18:01:34.674Z # Time: 2023-12-19T18:01:34.674354Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637662
# Query_time: 0.000213 Lock_time: 0.000001 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008894;
/*!80000 SELECT conn_status.channel_name as channel_name, conn_status.service_state as IO_thread, applier_status.service_state as SQL_thread, LAST_APPLIED_TRANSACTION_END_APPLY_TIMESTAMP - LAST_APPLIED_TRANSACTION_ORIGINAL_COMMIT_TIMESTAMP 'rep_delay_seconds', LAST_QUEUED_TRANSACTION_START_QUEUE_TIMESTAMP - LAST_QUEUED_TRANSACTION_ORIGINAL_COMMIT_TIMESTAMP 'transport_time_seconds', LAST_QUEUED_TRANSACTION_END_QUEUE_TIMESTAMP - LAST_QUEUED_TRANSACTION_START_QUEUE_TIMESTAMP 'time_RL_seconds', LAST_APPLIED_TRANSACTION_END_APPLY_TIMESTAMP - LAST_APPLIED_TRANSACTION_START_APPLY_TIMESTAMP 'apply_time_seconds', if(GTID_SUBTRACT(LAST_QUEUED_TRANSACTION, LAST_APPLIED_TRANSACTION) = '','0' , abs(time_to_sec(if(time_to_sec(APPLYING_TRANSACTION_ORIGINAL_COMMIT_TIMESTAMP)=0,0,timediff(APPLYING_TRANSACTION_ORIGINAL_COMMIT_TIMESTAMP,now()))))) `lag_in_seconds` FROM performance_schema.replication_connection_status AS conn_status JOIN performance_schema.replication_applier_status_by_worker AS applier_status ON applier_status.channel_name = conn_status.channel_name WHERE conn_status.service_state = 'ON' ORDER BY lag_in_seconds, lag_in_seconds desc */;
2023-12-19T18:01:34.675Z # Time: 2023-12-19T18:01:34.675003Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637665
# Query_time: 0.000576 Lock_time: 0.000000 Rows_sent: 74 Rows_examined: 314
SET timestamp=1703008894;
SELECT
name, subsystem, type, comment,
count
FROM information_schema.innodb_metrics
WHERE `status` = 'enabled';
2023-12-19T18:01:34.675Z # Time: 2023-12-19T18:01:34.675027Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637662
# Query_time: 0.000150 Lock_time: 0.000001 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008894;
/*!50700 SELECT conn_status.channel_name as channel_name, conn_status.service_state as IO_thread, applier_status.service_state as SQL_thread, 1 as info FROM performance_schema.replication_connection_status AS conn_status JOIN performance_schema.replication_applier_status_by_worker AS applier_status ON applier_status.channel_name = conn_status.channel_name WHERE conn_status.service_state = 'ON' and (SELECT SUBSTRING(@@VERSION,1,1) = 5) */;
2023-12-19T18:01:34.675Z # Time: 2023-12-19T18:01:34.675614Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637662
# Query_time: 0.000088 Lock_time: 0.000001 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008894;
/*!80000 SELECT CHANNEL_NAME as channel_name, MEMBER_ID as member_id, MEMBER_HOST as member_host, MEMBER_PORT as member_port, MEMBER_STATE as member_state, MEMBER_ROLE as member_role, MEMBER_VERSION as member_version, CASE WHEN MEMBER_STATE = 'ONLINE' THEN 1 WHEN MEMBER_STATE = 'RECOVERING' THEN 2 WHEN MEMBER_STATE = 'OFFLINE' THEN 3 WHEN MEMBER_STATE = 'ERROR' THEN 4 WHEN MEMBER_STATE = 'UNREACHABLE' THEN 5 END as member_info FROM performance_schema.replication_group_members WHERE MEMBER_ID=@@server_uuid */;
2023-12-19T18:01:34.676Z # Time: 2023-12-19T18:01:34.676129Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637662
# Query_time: 0.000068 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008894;
/*!50700 SELECT CHANNEL_NAME as channel_name, MEMBER_ID as member_id, MEMBER_HOST as member_host, MEMBER_PORT as member_port, MEMBER_STATE as member_state, CASE WHEN MEMBER_STATE = 'ONLINE' THEN 1 WHEN MEMBER_STATE = 'RECOVERING' THEN 2 WHEN MEMBER_STATE = 'OFFLINE' THEN 3 WHEN MEMBER_STATE = 'ERROR' THEN 4 WHEN MEMBER_STATE = 'UNREACHABLE' THEN 5 END as member_info FROM performance_schema.replication_group_members WHERE MEMBER_ID=@@server_uuid and (SELECT SUBSTRING(@@VERSION,1,1) = 5) */;
2023-12-19T18:01:34.676Z # Time: 2023-12-19T18:01:34.676650Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637662
# Query_time: 0.000060 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008894;
/*!80000 SELECT COUNT_TRANSACTIONS_IN_QUEUE as transactions_in_queue, COUNT_TRANSACTIONS_CHECKED as transactions_checked_total, COUNT_CONFLICTS_DETECTED as conflicts_detected_total, COUNT_TRANSACTIONS_ROWS_VALIDATING as transactions_rows_validating_total, COUNT_TRANSACTIONS_REMOTE_IN_APPLIER_QUEUE as transactions_remote_in_applier_queue, COUNT_TRANSACTIONS_REMOTE_APPLIED as transactions_remote_applied_total, COUNT_TRANSACTIONS_LOCAL_PROPOSED as transactions_local_proposed_total, COUNT_TRANSACTIONS_LOCAL_ROLLBACK as transactions_local_rollback_total FROM performance_schema.replication_group_member_stats WHERE MEMBER_ID=@@server_uuid */;
2023-12-19T18:01:34.677Z # Time: 2023-12-19T18:01:34.677162Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637662
# Query_time: 0.000056 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008894;
/*!50700 SELECT COUNT_TRANSACTIONS_IN_QUEUE as transactions_in_queue, COUNT_TRANSACTIONS_CHECKED as transactions_checked_total, COUNT_CONFLICTS_DETECTED as conflicts_detected_total, COUNT_TRANSACTIONS_ROWS_VALIDATING as transactions_rows_validating_total FROM performance_schema.replication_group_member_stats WHERE MEMBER_ID=@@server_uuid and (SELECT SUBSTRING(@@VERSION,1,1) = 5) */;
2023-12-19T18:01:55.401Z # Time: 2023-12-19T18:01:55.401592Z
# User@Host: esfeed_server[esfeed_server] @ [172.19.251.8] Id: 7637670
# Query_time: 0.000120 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 1
SET timestamp=1703008915;
/* mysql-connector-java-8.0.30 (Revision: 1de2fe873fe26189564c030a343885011412976a) */SELECT @@session.auto_increment_increment AS auto_increment_increment, @@character_set_client AS character_set_client, @@character_set_connection AS character_set_connection, @@character_set_results AS character_set_results, @@character_set_server AS character_set_server, @@collation_server AS collation_server, @@collation_connection AS collation_connection, @@init_connect AS init_connect, @@interactive_timeout AS interactive_timeout, @@license AS license, @@lower_case_table_names AS lower_case_table_names, @@max_allowed_packet AS max_allowed_packet, @@net_write_timeout AS net_write_timeout, @@performance_schema AS performance_schema, @@sql_mode AS sql_mode, @@system_time_zone AS system_time_zone, @@time_zone AS time_zone, @@transaction_isolation AS transaction_isolation, @@wait_timeout AS wait_timeout;
2023-12-19T18:01:55.402Z # Time: 2023-12-19T18:01:55.402020Z
# User@Host: esfeed_server[esfeed_server] @ [172.19.251.8] Id: 7637670
# Query_time: 0.000034 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008915;
SET character_set_results = NULL;
2023-12-19T18:01:55.402Z # Time: 2023-12-19T18:01:55.402401Z
# User@Host: esfeed_server[esfeed_server] @ [172.19.251.8] Id: 7637670
# Query_time: 0.000020 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008915;
SET autocommit=1;
2023-12-19T18:01:55.402Z # Time: 2023-12-19T18:01:55.402791Z
# User@Host: esfeed_server[esfeed_server] @ [172.19.251.8] Id: 7637670
# Query_time: 0.000035 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008915;
SET sql_mode='NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES';
2023-12-19T18:01:55.403Z # Time: 2023-12-19T18:01:55.403156Z
# User@Host: esfeed_server[esfeed_server] @ [172.19.251.8] Id: 7637670
# Query_time: 0.000021 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008915;
SHOW WARNINGS;
2023-12-19T18:01:55.403Z # Time: 2023-12-19T18:01:55.403576Z
# User@Host: esfeed_server[esfeed_server] @ [172.19.251.8] Id: 7637670
# Query_time: 0.000034 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 1
SET timestamp=1703008915;
SELECT @@session.transaction_read_only;
2023-12-19T18:01:55.429Z # Time: 2023-12-19T18:01:55.429029Z
# User@Host: pmm[pmm] @ [172.19.227.210] Id: 7637669
# Query_time: 0.000008 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
SET timestamp=1703008915;
# administrator command: Ping;

0 comments on commit 8986648

Please sign in to comment.