From 34901db16813f8c50c1943ceb63601b3f82be97b Mon Sep 17 00:00:00 2001 From: Robert Cowham Date: Mon, 25 Mar 2024 14:57:38 +0000 Subject: [PATCH] Provide option to specify no completion records expected in log. --- cmd/log2sql/main.go | 16 +++++++++++++--- metrics/metrics.go | 5 +++++ p4dlog.go | 13 +++++++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/cmd/log2sql/main.go b/cmd/log2sql/main.go index e57766e..7ef9d36 100644 --- a/cmd/log2sql/main.go +++ b/cmd/log2sql/main.go @@ -488,6 +488,10 @@ func main() { "case.insensitive.server", "Set if server is case insensitive and usernames may occur in either case.", ).Default("false").Bool() + noCompletionRecords = kingpin.Flag( + "no.completion.records", + "Set if log was generated with server=1 and thus no completion records expected.", + ).Default("false").Bool() debugPID = kingpin.Flag( "debug.pid", "Set for debug output for specified PID - requires debug.cmd to be also specified.", @@ -499,7 +503,7 @@ func main() { ) kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(version.Print("log2sql")).Author("Robert Cowham") kingpin.CommandLine.Help = "Parses one or more p4d text log files (which may be gzipped) into a Sqlite3 database and/or JSON or SQL format.\n" + - "The output of historical Prometheus compatible metrics is also by default." + + "The output of historical Prometheus compatible metrics is also on by default." + "These can be viewed using VictoriaMetrics which is a Prometheus compatible data store, and viewed in Grafana. " + "Where referred to in help is the first logfile specified with any .gz or .log suffix removed." kingpin.HelpFlag.Short('h') @@ -528,8 +532,8 @@ func main() { logger.Infof("Starting %s, Logfiles: %v", startTime, *logfiles) logger.Infof("Flags: debug %v, json/file %v/%v, sql/file %v/%v, dbName %s, noMetrics/file %v/%v", *debug, *jsonOutput, *jsonOutputFile, *sqlOutput, *sqlOutputFile, *dbName, *noMetrics, *metricsOutputFile) - logger.Infof(" serverID %v, sdpInstance %v, updateInterval %v, noOutputCmdsByUser %v, outputCmdsByUserRegex %s caseInsensitve %v, debugPID/cmd %v/%s", - *serverID, *sdpInstance, *updateInterval, *noOutputCmdsByUser, *outputCmdsByUserRegex, *caseInsensitiveServer, *debugPID, *debugCmd) + logger.Infof(" serverID %v, sdpInstance %v, updateInterval %v, noOutputCmdsByUser %v, outputCmdsByUserRegex %s caseInsensitve %v, noCompletionRecords %v, debugPID/cmd %v/%s", + *serverID, *sdpInstance, *updateInterval, *noOutputCmdsByUser, *outputCmdsByUserRegex, *caseInsensitiveServer, *noCompletionRecords, *debugPID, *debugCmd) linesChan := make(chan string, 10000) @@ -613,6 +617,9 @@ func main() { if *debugPID != 0 && *debugCmd != "" { mp.SetDebugPID(*debugPID, *debugCmd) } + if *noCompletionRecords { + mp.SetNoCompletionRecords() + } cmdChan, metricsChan = mp.ProcessEvents(ctx, linesChan, needCmdChan) // Process all metrics - need to consume them even if we ignore them (overhead is minimal) @@ -632,6 +639,9 @@ func main() { if *debug > 0 { fp.SetDebugMode(*debug) } + if *noCompletionRecords { + fp.SetNoCompletionRecords() + } cmdChan = fp.LogParser(ctx, linesChan, nil) } diff --git a/metrics/metrics.go b/metrics/metrics.go index 88927cd..4100452 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -173,6 +173,11 @@ func (p4m *P4DMetrics) SetDebugMode(level int) { p4m.fp.SetDebugMode(level) } +// SetNoCompletionRecords() - can be set if log was generated with server=1 +func (p4m *P4DMetrics) SetNoCompletionRecords() { + p4m.fp.SetNoCompletionRecords() +} + // defines metrics label type labelStruct struct { name string diff --git a/p4dlog.go b/p4dlog.go index fe6815b..1c1de69 100644 --- a/p4dlog.go +++ b/p4dlog.go @@ -47,6 +47,7 @@ const ( DebugDatabase DebugJSON DebugCommands + DebugAddCommands DebugTrackRunning DebugUnrecognised DebugPending @@ -1014,6 +1015,7 @@ type P4dFileParser struct { blockChan chan *Block currTime time.Time debug int + noCompletionRecords bool // Can be set if completion records not expected - e.g. configurable server=1 currStartTime time.Time timeLastCmdProcessed time.Time pidsSeenThisSecond map[int64]bool @@ -1050,6 +1052,11 @@ func (fp *P4dFileParser) SetDebugPID(pid int64, cmdName string) { fp.debugCmd = cmdName } +// SetNoCompletionRecords - don't expect completion records +func (fp *P4dFileParser) SetNoCompletionRecords() { + fp.noCompletionRecords = true +} + func (fp *P4dFileParser) debugLog(cmd *Command) bool { return cmd.Pid == fp.debugPID && cmd.Cmd == fp.debugCmd } @@ -1105,7 +1112,7 @@ func (fp *P4dFileParser) trackRunning(msg string, cmd *Command, delta int) { } func (fp *P4dFileParser) addCommand(newCmd *Command, hasTrackInfo bool) { - debugLog := fp.debugLog(newCmd) + debugLog := fp.debugLog(newCmd) || FlagSet(fp.debug, DebugAddCommands) if debugLog { fp.logger.Infof("addCommand: hasTrack %v, pid %d lineNo %d cmd %s dup %v", hasTrackInfo, newCmd.Pid, newCmd.LineNo, newCmd.Cmd, newCmd.duplicateKey) } @@ -1186,7 +1193,9 @@ func (fp *P4dFileParser) addCommand(newCmd *Command, hasTrackInfo bool) { } // Special commands which only have start records not completion records +// This was a thing with older p4d versions but now all commands have them func cmdHasNoCompletionRecord(cmdName string) bool { + // return false return cmdName == "rmt-FileFetch" || cmdName == "rmt-FileFetchMulti" || cmdName == "rmt-Journal" || @@ -1658,7 +1667,7 @@ func (fp *P4dFileParser) outputCompletedCommands() { completed = true } // Handle the special commands which don't receive a completed time - we use StartTime - if !completed && fp.currStartTime.Sub(cmd.StartTime) >= timeWindow && cmdHasNoCompletionRecord(cmd.Cmd) { + if !completed && fp.currStartTime.Sub(cmd.StartTime) >= timeWindow && (cmdHasNoCompletionRecord(cmd.Cmd) || fp.noCompletionRecords) { if debugLog { fp.logger.Infof("output: r5 pid %d lineNo %d cmd %s", cmd.Pid, cmd.LineNo, cmd.Cmd) }