From 2e33eccfb5af5f4d131fab65a0dda8f7a111d0e3 Mon Sep 17 00:00:00 2001 From: James McMullan Date: Tue, 14 Jan 2025 09:43:38 -0500 Subject: [PATCH] HPCC-32248 Add tracing to rowservice - Added opentelemetry tracing to rowservice Signed-off-by: James McMullan James.McMullan@lexisnexis.com --- fs/dafilesrv/dafilesrv.cpp | 18 ++++++-- fs/dafsserver/dafsserver.cpp | 81 ++++++++++++++++++++++++++++++++++-- 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/fs/dafilesrv/dafilesrv.cpp b/fs/dafilesrv/dafilesrv.cpp index ffe2e689aea..6a5b4d449c1 100644 --- a/fs/dafilesrv/dafilesrv.cpp +++ b/fs/dafilesrv/dafilesrv.cpp @@ -366,7 +366,6 @@ version: 1.0 detail: 100 )!!"; - int main(int argc, const char* argv[]) { InitModuleObjects(); @@ -386,7 +385,21 @@ int main(int argc, const char* argv[]) StringBuffer componentName; // NB: bare-metal dafilesrv does not have a component specific xml - Owned config = loadConfiguration(defaultYaml, argv, "dafilesrv", "DAFILESRV", nullptr, nullptr); + Owned extractedGlobalConfig = createPTree("dafilesrv"); + +#ifndef _CONTAINERIZED + Owned env = getHPCCEnvironment(); + IPropertyTree* globalTracing = env->getPropTree("Software/tracing"); + if (globalTracing != nullptr) + extractedGlobalConfig->addPropTree("tracing", globalTracing); +#endif + + const char* componentTag = "dafilesrv"; + Owned defaultConfig = createPTreeFromYAMLString(defaultYaml, 0, ptr_ignoreWhiteSpace, nullptr); + Owned componentDefault(defaultConfig->getPropTree(componentTag)); + + // NB: bare-metal dafilesrv does not have a component specific xml, extracting relevant global configuration instead + Owned config = loadConfiguration(componentDefault, extractedGlobalConfig, argv, componentTag, "DAFILESRV", nullptr, nullptr); Owned keyPairInfo; // NB: not used in containerized mode // Get SSL Settings @@ -513,7 +526,6 @@ int main(int argc, const char* argv[]) IPropertyTree *dafileSrvInstance = nullptr; #ifndef _CONTAINERIZED - Owned env = getHPCCEnvironment(); Owned _dafileSrvInstance; if (env) { diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index a2311f52be4..f3ffc8fd5df 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -839,6 +839,9 @@ class CRemoteRequest : public CSimpleInterfaceOf MemoryBuffer expandMb; Owned responseWriter; // for xml or json response + OwnedSpanLifetime requestSpan; + std::string requestTraceParent; + bool handleFull(MemoryBuffer &inMb, size32_t inPos, MemoryBuffer &compressMb, ICompressor *compressor, size32_t replyLimit, size32_t &totalSz) { size32_t sz = inMb.length()-inPos; @@ -1092,6 +1095,15 @@ class CRemoteRequest : public CSimpleInterfaceOf responseWriter->outputUInt(cursorHandle, sizeof(cursorHandle), "handle"); } } + + ~CRemoteRequest() + { + if (requestSpan != nullptr) + { + requestSpan->setSpanStatusSuccess(true); + } + } + OutputFormat queryFormat() const { return format; } unsigned __int64 queryReplyLimit() const { return replyLimit; } IRemoteActivity *queryActivity() const { return activity; } @@ -1099,6 +1111,42 @@ class CRemoteRequest : public CSimpleInterfaceOf void process(IPropertyTree *requestTree, MemoryBuffer &restMb, MemoryBuffer &responseMb, CClientStats &stats) { + bool traceParentChanged = false; + const char* fullTraceContext = requestTree->queryProp("_trace/traceparent"); + if (fullTraceContext != nullptr) + { + // We only want to compare the trace-id & span-id, so ignore the last sampling group after the '-' + const char* lastHyphen = strchr(fullTraceContext, '-'); + if (lastHyphen != nullptr) + { + size_t lastHyphenIdx = lastHyphen - fullTraceContext; + traceParentChanged = strncmp(fullTraceContext, requestTraceParent.c_str(), lastHyphenIdx) != 0; + } + } + + if (traceParentChanged) + { + // Check to see if we have an existing span that needs to be marked successful before close + if (requestSpan != nullptr) + { + requestSpan->setSpanStatusSuccess(true); + } + + Owned traceHeaders = createProperties(); + traceHeaders->setProp("traceparent", fullTraceContext); + + const char* requestSpanName = nullptr; + if (activity->queryIsReadActivity()) + requestSpanName = "ReadRequest"; + else + requestSpanName = "WriteRequest"; + + requestSpan.setown(queryTraceManager().createServerSpan(requestSpanName, traceHeaders)); + requestTraceParent = fullTraceContext; + } + + ActiveSpanScope activeSpan(requestSpan.query()); + if (requestTree->hasProp("replyLimit")) replyLimit = requestTree->getPropInt64("replyLimit", defaultDaFSReplyLimitKB) * 1024; @@ -3027,12 +3075,12 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface else { if (gc) - THROWJSOCKEXCEPTION(JSOCKERR_graceful_close); + THROWJSOCKEXCEPTION(JSOCKERR_graceful_close); break; // wait for rest via subsequent notifySelected's } } else if (gc) - THROWJSOCKEXCEPTION(JSOCKERR_graceful_close); + THROWJSOCKEXCEPTION(JSOCKERR_graceful_close); // to be here, implies handled full message, loop around to see if more on the wire. // will break out if nothing/partial. } @@ -4818,7 +4866,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface * } * } * } - * + * * fetch continuation: * { * "format" : "binary", @@ -4960,8 +5008,23 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface } case StreamCmd::CLOSE: { + OwnedActiveSpanScope closeSpan; + const char* traceParent = requestTree->queryProp("_trace/traceparent"); + if (traceParent != nullptr) + { + Owned traceHeaders = createProperties(); + traceHeaders->setProp("traceparent", traceParent); + + closeSpan.setown(queryTraceManager().createServerSpan("CloseRequest", traceHeaders)); + } + if (0 == cursorHandle) - throw createDafsException(DAFSERR_cmdstream_protocol_failure, "cursor handle not supplied to 'close' command"); + { + IDAFS_Exception* exception = createDafsException(DAFSERR_cmdstream_protocol_failure, "cursor handle not supplied to 'close' command"); + closeSpan->recordException(exception); + throw exception; + } + IFileIO *dummy; checkFileIOHandle(cursorHandle, dummy, true); break; @@ -4990,6 +5053,16 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface { case StreamCmd::VERSION: { + OwnedActiveSpanScope versionSpan; + const char* traceParent = requestTree->queryProp("_trace/traceparent"); + if (traceParent != nullptr) + { + Owned traceHeaders = createProperties(); + traceHeaders->setProp("traceparent", traceParent); + + versionSpan.setown(queryTraceManager().createServerSpan("VersionRequest", traceHeaders)); + } + if (outFmt_Binary == outputFormat) reply.append(DAFILESRV_VERSIONSTRING); else