Provide option to specify no completion records expected in log.

cmd/log2sql/main.go

@@ -488,6 +488,10 @@ func main() {
 			"case.insensitive.server",
 			"Set if server is case insensitive and usernames may occur in either case.",
 		).Default("false").Bool()
+		noCompletionRecords = kingpin.Flag(
+			"no.completion.records",
+			"Set if log was generated with server=1 and thus no completion records expected.",
+		).Default("false").Bool()
 		debugPID = kingpin.Flag(
 			"debug.pid",
 			"Set for debug output for specified PID - requires debug.cmd to be also specified.",
@@ -499,7 +503,7 @@ func main() {
 	)
 	kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(version.Print("log2sql")).Author("Robert Cowham")
 	kingpin.CommandLine.Help = "Parses one or more p4d text log files (which may be gzipped) into a Sqlite3 database and/or JSON or SQL format.\n" +
-		"The output of historical Prometheus compatible metrics is also by default." +
+		"The output of historical Prometheus compatible metrics is also on by default." +
 		"These can be viewed using VictoriaMetrics which is a Prometheus compatible data store, and viewed in Grafana. " +
 		"Where referred to in help <logfile-prefix> is the first logfile specified with any .gz or .log suffix removed."
 	kingpin.HelpFlag.Short('h')
@@ -528,8 +532,8 @@ func main() {
 	logger.Infof("Starting %s, Logfiles: %v", startTime, *logfiles)
 	logger.Infof("Flags: debug %v, json/file %v/%v, sql/file %v/%v, dbName %s, noMetrics/file %v/%v",
 		*debug, *jsonOutput, *jsonOutputFile, *sqlOutput, *sqlOutputFile, *dbName, *noMetrics, *metricsOutputFile)
-	logger.Infof("       serverID %v, sdpInstance %v, updateInterval %v, noOutputCmdsByUser %v, outputCmdsByUserRegex %s caseInsensitve %v, debugPID/cmd %v/%s",
-		*serverID, *sdpInstance, *updateInterval, *noOutputCmdsByUser, *outputCmdsByUserRegex, *caseInsensitiveServer, *debugPID, *debugCmd)
+	logger.Infof("       serverID %v, sdpInstance %v, updateInterval %v, noOutputCmdsByUser %v, outputCmdsByUserRegex %s caseInsensitve %v, noCompletionRecords %v, debugPID/cmd %v/%s",
+		*serverID, *sdpInstance, *updateInterval, *noOutputCmdsByUser, *outputCmdsByUserRegex, *caseInsensitiveServer, *noCompletionRecords, *debugPID, *debugCmd)
 
 	linesChan := make(chan string, 10000)
 
@@ -613,6 +617,9 @@ func main() {
 		if *debugPID != 0 && *debugCmd != "" {
 			mp.SetDebugPID(*debugPID, *debugCmd)
 		}
+		if *noCompletionRecords {
+			mp.SetNoCompletionRecords()
+		}
 		cmdChan, metricsChan = mp.ProcessEvents(ctx, linesChan, needCmdChan)
 
 		// Process all metrics - need to consume them even if we ignore them (overhead is minimal)
@@ -632,6 +639,9 @@ func main() {
 		if *debug > 0 {
 			fp.SetDebugMode(*debug)
 		}
+		if *noCompletionRecords {
+			fp.SetNoCompletionRecords()
+		}
 		cmdChan = fp.LogParser(ctx, linesChan, nil)
 	}
 

metrics/metrics.go

@@ -173,6 +173,11 @@ func (p4m *P4DMetrics) SetDebugMode(level int) {
 	p4m.fp.SetDebugMode(level)
 }
 
+// SetNoCompletionRecords() - can be set if log was generated with server=1
+func (p4m *P4DMetrics) SetNoCompletionRecords() {
+	p4m.fp.SetNoCompletionRecords()
+}
+
 // defines metrics label
 type labelStruct struct {
 	name  string

p4dlog.go

@@ -47,6 +47,7 @@ const (
 	DebugDatabase
 	DebugJSON
 	DebugCommands
+	DebugAddCommands
 	DebugTrackRunning
 	DebugUnrecognised
 	DebugPending
@@ -1014,6 +1015,7 @@ type P4dFileParser struct {
 	blockChan            chan *Block
 	currTime             time.Time
 	debug                int
+	noCompletionRecords  bool // Can be set if completion records not expected - e.g. configurable server=1
 	currStartTime        time.Time
 	timeLastCmdProcessed time.Time
 	pidsSeenThisSecond   map[int64]bool
@@ -1050,6 +1052,11 @@ func (fp *P4dFileParser) SetDebugPID(pid int64, cmdName string) {
 	fp.debugCmd = cmdName
 }
 
+// SetNoCompletionRecords - don't expect completion records
+func (fp *P4dFileParser) SetNoCompletionRecords() {
+	fp.noCompletionRecords = true
+}
+
 func (fp *P4dFileParser) debugLog(cmd *Command) bool {
 	return cmd.Pid == fp.debugPID && cmd.Cmd == fp.debugCmd
 }
@@ -1105,7 +1112,7 @@ func (fp *P4dFileParser) trackRunning(msg string, cmd *Command, delta int) {
 }
 
 func (fp *P4dFileParser) addCommand(newCmd *Command, hasTrackInfo bool) {
-	debugLog := fp.debugLog(newCmd)
+	debugLog := fp.debugLog(newCmd) || FlagSet(fp.debug, DebugAddCommands)
 	if debugLog {
 		fp.logger.Infof("addCommand: hasTrack %v, pid %d lineNo %d cmd %s dup %v", hasTrackInfo, newCmd.Pid, newCmd.LineNo, newCmd.Cmd, newCmd.duplicateKey)
 	}
@@ -1186,7 +1193,9 @@ func (fp *P4dFileParser) addCommand(newCmd *Command, hasTrackInfo bool) {
 }
 
 // Special commands which only have start records not completion records
+// This was a thing with older p4d versions but now all commands have them
 func cmdHasNoCompletionRecord(cmdName string) bool {
+	// return false
 	return cmdName == "rmt-FileFetch" ||
 		cmdName == "rmt-FileFetchMulti" ||
 		cmdName == "rmt-Journal" ||
@@ -1658,7 +1667,7 @@ func (fp *P4dFileParser) outputCompletedCommands() {
 			completed = true
 		}
 		// Handle the special commands which don't receive a completed time - we use StartTime
-		if !completed && fp.currStartTime.Sub(cmd.StartTime) >= timeWindow && cmdHasNoCompletionRecord(cmd.Cmd) {
+		if !completed && fp.currStartTime.Sub(cmd.StartTime) >= timeWindow && (cmdHasNoCompletionRecord(cmd.Cmd) || fp.noCompletionRecords) {
 			if debugLog {
 				fp.logger.Infof("output: r5 pid %d lineNo %d cmd %s", cmd.Pid, cmd.LineNo, cmd.Cmd)
 			}