From fc6094af84a01e565d394e9d2aeaacc395e6ca56 Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Tue, 24 Sep 2024 20:02:58 +0530 Subject: [PATCH 01/15] feat: Adding confluent cloud cluster entity --- entity-types/ext-service/definition.yml | 2 + .../confluent_dashboard.json | 575 ++++++++++++++++++ .../definition.yml | 61 ++ .../golden_metrics.yml | 28 + .../opentelemetry_dashboard.json | 575 ++++++++++++++++++ .../summary_metrics.yml | 13 + .../tests/Metric.json | 114 ++++ 7 files changed, 1368 insertions(+) create mode 100644 entity-types/infra-confluentcloudcluster/confluent_dashboard.json create mode 100644 entity-types/infra-confluentcloudcluster/definition.yml create mode 100644 entity-types/infra-confluentcloudcluster/golden_metrics.yml create mode 100644 entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json create mode 100644 entity-types/infra-confluentcloudcluster/summary_metrics.yml create mode 100644 entity-types/infra-confluentcloudcluster/tests/Metric.json diff --git a/entity-types/ext-service/definition.yml b/entity-types/ext-service/definition.yml index 62cff04c2..a4d932fc1 100644 --- a/entity-types/ext-service/definition.yml +++ b/entity-types/ext-service/definition.yml @@ -77,6 +77,8 @@ synthesis: value: opentelemetry - attribute: newrelic.entity.type present: false + - attribute: kafka_id # to skip confluent kafka server metrics to create confluentcloudcluster entities instead of service entities + present: false tags: k8s.cluster.name: ttl: P1D diff --git a/entity-types/infra-confluentcloudcluster/confluent_dashboard.json b/entity-types/infra-confluentcloudcluster/confluent_dashboard.json new file mode 100644 index 000000000..b56993714 --- /dev/null +++ b/entity-types/infra-confluentcloudcluster/confluent_dashboard.json @@ -0,0 +1,575 @@ +{ + "name": "Confluent Cloud", + "description": null, + "pages": [ + { + "name": "Confluent Cloud", + "description": null, + "widgets": [ + { + "title": "Confluent Cloud", + "layout": { + "column": 1, + "row": 1, + "width": 2, + "height": 4 + }, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "![ConfluentLogo](https://cdn.confluent.io/wp-content/uploads/seo-logo-meadow.png)\n# Confluent Cloud Kafka\n#### [Confluent API docs](https://api.telemetry.confluent.cloud/docs/descriptors/datasets/cloud?_ga=2.214885143.1264186867.1705940186-6520871.1686857317&_gl=1*xevtc5*_ga*NjUyMDg3MS4xNjg2ODU3MzE3*_ga_D2D3EGKSGD*MTcwNTk0MDA5MC41My4xLjE3MDU5NDAxODUuNjAuMC4w)" + } + }, + { + "title": "Cluster Specs", + "layout": { + "column": 3, + "row": 1, + "width": 5, + "height": 4 + }, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "dataFormatters": [ + { + "name": "Last Update", + "type": "date" + } + ], + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT uniqueCount(confluent.kafka.server.metric.topic) as 'Topics', latest(confluent.kafka.server.partition_count) as 'Partitions', uniqueCount(confluent.kafka.server.metric.consumer_group_id) as 'Consumer Groups' FROM Metric FACET `confluent.clusterId`)" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Request Bytes (bytes/second) Per Type", + "layout": { + "column": 8, + "row": 1, + "width": 5, + "height": 4 + }, + "visualization": { + "id": "viz.pie" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": true + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 second) FROM Metric FACET confluent.clusterId, confluent.kafka.server.metric.type LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Retained Bytes Per Topic", + "layout": { + "column": 1, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(confluent.kafka.server.retained_bytes) FROM Metric TIMESERIES FACET confluent.clusterId, confluent.kafka.server.metric.topic LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Records Sent Per Minute", + "layout": { + "column": 5, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent.kafka.server.sent_records),1 minute ) FROM Metric TIMESERIES FACET `confluent.kafka.server.metric.topic`, confluent.clusterId LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Active Connections", + "layout": { + "column": 9, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(confluent.kafka.server.active_connection_count) AS 'Active Connections' FROM Metric facet confluent.clusterId LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Partitions ", + "layout": { + "column": 1, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(confluent.kafka.server.partition_count) AS partitions FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Recieved Bytes by Topic", + "layout": { + "column": 5, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(confluent.kafka.server.received_bytes) FROM Metric TIMESERIES FACET `confluent.kafka.server.metric.topic`, confluent.clusterId LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": false + } + } + }, + { + "title": "Records Recieved Per Minute", + "layout": { + "column": 9, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent.kafka.server.received_records),1 minute ) FROM Metric TIMESERIES FACET `confluent.kafka.server.metric.topic`, confluent.clusterId LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Successful Auth Attempts", + "layout": { + "column": 1, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent.kafka.server.successful_authentication_count), 1 minute) AS 'Auths Per Minute' FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "COUNT" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Egres rate (bytes/minute)", + "layout": { + "column": 5, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(l) as egress from (SELECT rate(sum(confluent.kafka.server.response_bytes), 1 minute) as l FROM Metric FACET confluent.clusterId TIMESERIES) LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Sent Bytes by Topic", + "layout": { + "column": 9, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(confluent.kafka.server.sent_bytes) FROM Metric TIMESERIES FACET `confluent.kafka.server.metric.topic`, confluent.clusterId LIMIT MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Response Bytes (bytes/second) Per Kafka Protocol Request Type", + "layout": { + "column": 1, + "row": 14, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 second) FROM Metric FACET confluent.clusterId) FACET confluent.kafka.server.metric.type LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES_PER_SECOND" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Request Rate", + "layout": { + "column": 5, + "row": 14, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent.kafka.server.request_count), 1 minute) as 'Requests Per Minute' FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "REQUESTS_PER_SECOND" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Ingress rate (bytes/minute)", + "layout": { + "column": 9, + "row": 14, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 minute) as 'Ingress' FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Request Bytes (bytes/second) Per Kafka Protocol Request Type", + "layout": { + "column": 1, + "row": 17, + "width": 6, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 second) FROM Metric FACET confluent.kafka.server.metric.type, confluent.clusterId LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES_PER_SECOND" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Response Rate ", + "layout": { + "column": 7, + "row": 17, + "width": 6, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 minute) as 'Bytes Per Minute' FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "REQUESTS_PER_SECOND" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + } + ] + } + ] +} diff --git a/entity-types/infra-confluentcloudcluster/definition.yml b/entity-types/infra-confluentcloudcluster/definition.yml new file mode 100644 index 000000000..a0f16a6f4 --- /dev/null +++ b/entity-types/infra-confluentcloudcluster/definition.yml @@ -0,0 +1,61 @@ +domain: INFRA +type: CONFLUENTCLOUDCLUSTER +goldenTags: + - confluent.clusterName + - confluent.metricNamespace + - collector.name + - instrumentation.provider + - newrelic.source + - kafka.cluster_name + - confluent.resourceType +configuration: + entityExpirationTime: DAILY + alertable: true + +dashboardTemplates: + opentelemetry: + template: opentelemetry_dashboard.json + confluent: + template: confluent_dashboard.json + +synthesis: + rules: + # telemetry from opentelemetry provider + - identifier: kafka_id + name: kafka.cluster_name + encodeIdentifierInGUID: true + tags: + instrumentation.provider: + newrelic.source: + conditions: + - attribute: eventType + value: Metric + - attribute: metricName + prefix: confluent_kafka_server_ + - attribute: instrumentation.provider + value: opentelemetry + - attribute: kafka.cluster_name + present: true + + # telemetry from confluent-cloud provider + - identifier: confluent.clusterId + name: confluent.clusterName + encodeIdentifierInGUID: true + tags: + collector.name: + confluent.metricNamespace: + instrumentation.provider: + newrelic.source: + conditions: + - attribute: eventType + value: Metric + - attribute: collector.name + value: confluent-monitor + - attribute: metricName + prefix: confluent.kafka.server. + - attribute: instrumentation.provider + value: confluent + - attribute: confluent.resourceType + value: kafka.server + - attribute: confluent.clusterName + present: true \ No newline at end of file diff --git a/entity-types/infra-confluentcloudcluster/golden_metrics.yml b/entity-types/infra-confluentcloudcluster/golden_metrics.yml new file mode 100644 index 000000000..4ef91a3bc --- /dev/null +++ b/entity-types/infra-confluentcloudcluster/golden_metrics.yml @@ -0,0 +1,28 @@ +clusterLoadPercent: + title: Cluster load percent + unit: PERCENTAGE + queries: + opentelemetry: + select: min(confluent_kafka_server_cluster_load_percent) * 100 + from: Metric + where: kafka_id is not null + eventName: kafka_id + confluent: + select: min(confluent.kafka.server.cluster_load_percent) * 100 + from: Metric + where: confluent.clusterId is not null + eventName: confluent.clusterId +partitionsCount: + title: Partitions count + unit: COUNT + queries: + opentelemetry: + select: min(confluent_kafka_server_partition_count) + from: Metric + where: kafka_id is not null + eventName: kafka_id + confluent: + select: min(confluent.kafka.server.partition_count) + from: Metric + where: confluent.clusterId is not null + eventName: confluent.clusterId diff --git a/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json b/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json new file mode 100644 index 000000000..25d280044 --- /dev/null +++ b/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json @@ -0,0 +1,575 @@ +{ + "name": "Confluent Cloud", + "description": null, + "pages": [ + { + "name": "Confluent Cloud", + "description": null, + "widgets": [ + { + "title": "Confluent Cloud", + "layout": { + "column": 1, + "row": 1, + "width": 2, + "height": 4 + }, + "visualization": { + "id": "viz.markdown" + }, + "rawConfiguration": { + "text": "![ConfluentLogo](https://cdn.confluent.io/wp-content/uploads/seo-logo-meadow.png)\n# Confluent Cloud Kafka\n#### [Confluent API docs](https://api.telemetry.confluent.cloud/docs/descriptors/datasets/cloud?_ga=2.214885143.1264186867.1705940186-6520871.1686857317&_gl=1*xevtc5*_ga*NjUyMDg3MS4xNjg2ODU3MzE3*_ga_D2D3EGKSGD*MTcwNTk0MDA5MC41My4xLjE3MDU5NDAxODUuNjAuMC4w)" + } + }, + { + "title": "Cluster Specs", + "layout": { + "column": 3, + "row": 1, + "width": 5, + "height": 4 + }, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "dataFormatters": [ + { + "name": "Last Update", + "type": "date" + } + ], + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT uniqueCount(topic) as 'Topics', latest(confluent_kafka_server_partition_count) as 'Partitions', uniqueCount(consumer_group_id) as 'Consumer Groups', uniqueCount(connector_id) AS 'Connectors' FROM Metric FACET `kafka_id`)" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Request Bytes (bytes/second) Per Type", + "layout": { + "column": 8, + "row": 1, + "width": 5, + "height": 4 + }, + "visualization": { + "id": "viz.pie" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": true + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric FACET kafka_id, type LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Retained Bytes Per Topic", + "layout": { + "column": 1, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(confluent_kafka_server_retained_bytes) FROM Metric TIMESERIES FACET kafka_id, topic LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Records Sent Per Minute", + "layout": { + "column": 5, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent_kafka_server_sent_records),1 minute ) FROM Metric TIMESERIES FACET `topic`, kafka_id LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Active Connections", + "layout": { + "column": 9, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT max(confluent_kafka_server_active_connection_count) AS 'Active Connections' FROM Metric facet kafka_id LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Partitions ", + "layout": { + "column": 1, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT latest(confluent_kafka_server_partition_count) AS partitions FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Recieved Bytes by Topic", + "layout": { + "column": 5, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(confluent_kafka_server_received_bytes) FROM Metric TIMESERIES FACET `topic`, kafka_id LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": false + } + } + }, + { + "title": "Records Recieved Per Minute", + "layout": { + "column": 9, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent_kafka_server_received_records),1 minute ) FROM Metric TIMESERIES FACET `topic`, kafka_id LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Successful Auth Attempts", + "layout": { + "column": 1, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent_kafka_server_successful_authentication_count), 1 minute) AS 'Auths Per Minute' FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "COUNT" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Egres rate (bytes/minute)", + "layout": { + "column": 5, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(l) as egress from (SELECT rate(sum(confluent_kafka_server_response_bytes), 1 minute) as l FROM Metric FACET kafka_id TIMESERIES) LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Sent Bytes by Topic", + "layout": { + "column": 9, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT sum(confluent_kafka_server_sent_bytes) FROM Metric TIMESERIES FACET `topic`, kafka_id LIMIT MAX " + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Response Bytes (bytes/second) Per Kafka Protocol Request Type", + "layout": { + "column": 1, + "row": 14, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 second) FROM Metric FACET kafka_id) FACET type LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES_PER_SECOND" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Request Rate", + "layout": { + "column": 5, + "row": 14, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent_kafka_server_request_count), 1 minute) as 'Requests Per Minute' FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "REQUESTS_PER_SECOND" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Ingress rate (bytes/minute)", + "layout": { + "column": 9, + "row": 14, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 minute) as 'Ingress' FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Request Bytes (bytes/second) Per Kafka Protocol Request Type", + "layout": { + "column": 1, + "row": 17, + "width": 6, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric FACET type, kafka_id LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "BYTES_PER_SECOND" + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Response Rate ", + "layout": { + "column": 7, + "row": 17, + "width": 6, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 minute) as 'Bytes Per Minute' FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "units": { + "unit": "REQUESTS_PER_SECOND" + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + } + ] + } + ] +} diff --git a/entity-types/infra-confluentcloudcluster/summary_metrics.yml b/entity-types/infra-confluentcloudcluster/summary_metrics.yml new file mode 100644 index 000000000..b4d681687 --- /dev/null +++ b/entity-types/infra-confluentcloudcluster/summary_metrics.yml @@ -0,0 +1,13 @@ +providerName: + tag: + key: instrumentation.provider + title: Provider + unit: STRING +clusterLoadPercent: + goldenMetric: clusterLoadPercent + unit: PERCENTAGE + title: Cluster load percent +partitionsCount: + goldenMetric: partitionsCount + unit: COUNT + title: Partitions count \ No newline at end of file diff --git a/entity-types/infra-confluentcloudcluster/tests/Metric.json b/entity-types/infra-confluentcloudcluster/tests/Metric.json new file mode 100644 index 000000000..6b9f35587 --- /dev/null +++ b/entity-types/infra-confluentcloudcluster/tests/Metric.json @@ -0,0 +1,114 @@ +[ + { + "confluent_kafka_server_consumer_lag_offsets": { + "type": "gauge", + "count": 352, + "sum": 0, + "min": 0, + "max": 0, + "latest": 0 + }, + "consumer_group_id": "agent_actions_vortex_vortex-3_23456", + "description": "The lag between a group member's committed offset and the partition's high watermark.", + "endTimestamp": 1719554400000, + "http.scheme": "https", + "instrumentation.provider": "opentelemetry", + "kafka.cluster_name": "us-jurassic-dust", + "kafka_id": "lkc-knjjjv", + "metricName": "confluent_kafka_server_consumer_lag_offsets", + "net.host.name": "api.telemetry.confluent.cloud", + "net.host.port": "443", + "newrelic.source": "api.metrics.otlp", + "nr.dataPointCount": 352, + "otel.library.name": "", + "otel.library.version": "", + "service.instance.id": "api.telemetry.confluent.cloud:443", + "service.name": "confluent", + "timestamp": 1719532800000, + "topic": "agent_actions" + }, + { + "confluent_kafka_server_received_bytes": { + "type": "gauge", + "count": 312, + "sum": 293255, + "min": 0, + "max": 74103, + "latest": 2741 + }, + "description": "The delta count of bytes of the customer's data received from the network. Each sample is the number of bytes received since the previous data sample. The count is sampled every 60 seconds.", + "endTimestamp": 1719554400000, + "http.scheme": "https", + "instrumentation.provider": "opentelemetry", + "kafka.cluster_name": "stg-rat-hole", + "kafka_id": "lkc-gx5zg1", + "metricName": "confluent_kafka_server_received_bytes", + "net.host.name": "api.telemetry.confluent.cloud", + "net.host.port": "443", + "newrelic.source": "api.metrics.otlp", + "nr.dataPointCount": 312, + "otel.library.name": "", + "otel.library.version": "", + "service.instance.id": "api.telemetry.confluent.cloud:443", + "service.name": "confluent", + "timestamp": 1719532800000, + "topic": "connect_account_data_2", + "unit": "bytes" + }, + { + "collector.name": "confluent-monitor", + "collector.version": "release-11", + "confluent.clusterId": "lkc-jr15qm", + "confluent.clusterName": "cluster_0", + "confluent.kafka.server.active_connection_count": { + "type": "summary", + "count": 1, + "sum": 24, + "min": 24, + "max": 24 + }, + "confluent.kafka.server.metric.principal_id": "u-896ov5", + "confluent.metricNamespace": "confluent.kafka.server", + "confluent.region": "ap-south-2", + "confluent.resourceGroup": "io.confluent.kafka.server", + "confluent.resourceId": "lkc-jr15qm", + "confluent.resourceType": "kafka.server", + "endTimestamp": 1727184120000, + "instrumentation.provider": "confluent", + "metricName": "confluent.kafka.server.active_connection_count", + "newrelic.cloudIntegrations.integrationId": "12", + "newrelic.cloudIntegrations.integrationName": "Confluent Cloud", + "newrelic.cloudIntegrations.providerAccountId": "0", + "newrelic.cloudIntegrations.providerAccountName": "sa-6v61z6", + "newrelic.source": "metricAPI", + "timestamp": 1727183640000 + }, + { + "collector.name": "confluent-monitor", + "collector.version": "release-11", + "confluent.clusterId": "lkc-jr15qm", + "confluent.clusterName": "cluster_0", + "confluent.kafka.server.metric.principal_id": "u-896ov5", + "confluent.kafka.server.successful_authentication_count": { + "type": "summary", + "count": 1, + "sum": 29, + "min": 29, + "max": 29 + }, + "confluent.metricNamespace": "confluent.kafka.server", + "confluent.region": "ap-south-2", + "confluent.resourceGroup": "io.confluent.kafka.server", + "confluent.resourceId": "lkc-jr15qm", + "confluent.resourceType": "kafka.server", + "endTimestamp": 1727184000000, + "instrumentation.provider": "confluent", + "metricName": "confluent.kafka.server.successful_authentication_count", + "newrelic.cloudIntegrations.integrationId": "12", + "newrelic.cloudIntegrations.integrationName": "Confluent Cloud", + "newrelic.cloudIntegrations.providerAccountId": "0", + "newrelic.cloudIntegrations.providerAccountName": "sa-6v61z6", + "newrelic.source": "metricAPI", + "timestamp": 1727183640000 + } +] From 61012db8f4a25df3f620850cb03af94434884ef9 Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Fri, 27 Sep 2024 19:47:15 +0530 Subject: [PATCH 02/15] feat: Adding confluent cloud cluster entity metrics --- .../confluent_dashboard.json | 59 +++++++------------ .../definition.yml | 10 +++- .../golden_metrics.yml | 26 ++++++-- .../opentelemetry_dashboard.json | 57 +++++++----------- .../summary_metrics.yml | 10 +++- 5 files changed, 78 insertions(+), 84 deletions(-) diff --git a/entity-types/infra-confluentcloudcluster/confluent_dashboard.json b/entity-types/infra-confluentcloudcluster/confluent_dashboard.json index b56993714..91d58a46d 100644 --- a/entity-types/infra-confluentcloudcluster/confluent_dashboard.json +++ b/entity-types/infra-confluentcloudcluster/confluent_dashboard.json @@ -6,27 +6,12 @@ "name": "Confluent Cloud", "description": null, "widgets": [ - { - "title": "Confluent Cloud", - "layout": { - "column": 1, - "row": 1, - "width": 2, - "height": 4 - }, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "![ConfluentLogo](https://cdn.confluent.io/wp-content/uploads/seo-logo-meadow.png)\n# Confluent Cloud Kafka\n#### [Confluent API docs](https://api.telemetry.confluent.cloud/docs/descriptors/datasets/cloud?_ga=2.214885143.1264186867.1705940186-6520871.1686857317&_gl=1*xevtc5*_ga*NjUyMDg3MS4xNjg2ODU3MzE3*_ga_D2D3EGKSGD*MTcwNTk0MDA5MC41My4xLjE3MDU5NDAxODUuNjAuMC4w)" - } - }, { "title": "Cluster Specs", "layout": { - "column": 3, + "column": 1, "row": 1, - "width": 5, + "width": 6, "height": 4 }, "visualization": { @@ -45,7 +30,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT uniqueCount(confluent.kafka.server.metric.topic) as 'Topics', latest(confluent.kafka.server.partition_count) as 'Partitions', uniqueCount(confluent.kafka.server.metric.consumer_group_id) as 'Consumer Groups' FROM Metric FACET `confluent.clusterId`)" + "query": "SELECT uniqueCount(confluent.kafka.server.metric.topic) as 'Topics', average(confluent.kafka.server.partition_count) as 'Partitions', uniqueCount(confluent.kafka.server.metric.consumer_group_id) as 'Consumer Groups' FROM Metric" } ], "platformOptions": { @@ -56,9 +41,9 @@ { "title": "Request Bytes (bytes/second) Per Type", "layout": { - "column": 8, + "column": 7, "row": 1, - "width": 5, + "width": 6, "height": 4 }, "visualization": { @@ -74,7 +59,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 second) FROM Metric FACET confluent.clusterId, confluent.kafka.server.metric.type LIMIT MAX" + "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 second) FROM Metric FACET confluent.kafka.server.metric.type LIMIT MAX" } ], "platformOptions": { @@ -103,7 +88,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(confluent.kafka.server.retained_bytes) FROM Metric TIMESERIES FACET confluent.clusterId, confluent.kafka.server.metric.topic LIMIT MAX" + "query": "SELECT average(confluent.kafka.server.retained_bytes) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic LIMIT MAX" } ], "platformOptions": { @@ -141,7 +126,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.sent_records),1 minute ) FROM Metric TIMESERIES FACET `confluent.kafka.server.metric.topic`, confluent.clusterId LIMIT MAX" + "query": "SELECT rate(sum(confluent.kafka.server.sent_records),1 minute) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic LIMIT MAX" } ], "platformOptions": { @@ -173,7 +158,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT max(confluent.kafka.server.active_connection_count) AS 'Active Connections' FROM Metric facet confluent.clusterId LIMIT MAX TIMESERIES" + "query": "SELECT max(confluent.kafka.server.active_connection_count) AS 'Active Connections' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -188,7 +173,7 @@ } }, { - "title": "Partitions ", + "title": "Partitions", "layout": { "column": 1, "row": 8, @@ -208,7 +193,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(confluent.kafka.server.partition_count) AS partitions FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + "query": "SELECT average(confluent.kafka.server.partition_count) AS partitions FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -240,7 +225,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(confluent.kafka.server.received_bytes) FROM Metric TIMESERIES FACET `confluent.kafka.server.metric.topic`, confluent.clusterId LIMIT MAX" + "query": "SELECT sum(confluent.kafka.server.received_bytes) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic" } ], "platformOptions": { @@ -275,7 +260,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.received_records),1 minute ) FROM Metric TIMESERIES FACET `confluent.kafka.server.metric.topic`, confluent.clusterId LIMIT MAX" + "query": "SELECT rate(sum(confluent.kafka.server.received_records),1 minute) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic LIMIT MAX" } ], "platformOptions": { @@ -307,7 +292,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.successful_authentication_count), 1 minute) AS 'Auths Per Minute' FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent.kafka.server.successful_authentication_count), 1 minute) AS 'Auths Per Minute' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -342,7 +327,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(l) as egress from (SELECT rate(sum(confluent.kafka.server.response_bytes), 1 minute) as l FROM Metric FACET confluent.clusterId TIMESERIES) LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 minute) as 'Egres' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -377,7 +362,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(confluent.kafka.server.sent_bytes) FROM Metric TIMESERIES FACET `confluent.kafka.server.metric.topic`, confluent.clusterId LIMIT MAX " + "query": "SELECT sum(confluent.kafka.server.sent_bytes) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic LIMIT MAX " } ], "platformOptions": { @@ -412,7 +397,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 second) FROM Metric FACET confluent.clusterId) FACET confluent.kafka.server.metric.type LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 second) FROM Metric FACET confluent.kafka.server.metric.type LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -447,7 +432,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.request_count), 1 minute) as 'Requests Per Minute' FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent.kafka.server.request_count), 1 minute) as 'Requests Per Minute' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -482,7 +467,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 minute) as 'Ingress' FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 minute) as 'Ingress' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -517,7 +502,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 second) FROM Metric FACET confluent.kafka.server.metric.type, confluent.clusterId LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 second) FROM Metric FACET confluent.kafka.server.metric.type LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -532,7 +517,7 @@ } }, { - "title": "Response Rate ", + "title": "Response Rate", "layout": { "column": 7, "row": 17, @@ -552,7 +537,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 minute) as 'Bytes Per Minute' FROM Metric FACET confluent.clusterId LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 minute) as 'Bytes Per Minute' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { diff --git a/entity-types/infra-confluentcloudcluster/definition.yml b/entity-types/infra-confluentcloudcluster/definition.yml index a0f16a6f4..cc34e1b77 100644 --- a/entity-types/infra-confluentcloudcluster/definition.yml +++ b/entity-types/infra-confluentcloudcluster/definition.yml @@ -2,6 +2,8 @@ domain: INFRA type: CONFLUENTCLOUDCLUSTER goldenTags: - confluent.clusterName + - kafka_id + - confluent.clusterId - confluent.metricNamespace - collector.name - instrumentation.provider @@ -25,6 +27,8 @@ synthesis: name: kafka.cluster_name encodeIdentifierInGUID: true tags: + kafka_id: + kafka.cluster_name: instrumentation.provider: newrelic.source: conditions: @@ -34,7 +38,7 @@ synthesis: prefix: confluent_kafka_server_ - attribute: instrumentation.provider value: opentelemetry - - attribute: kafka.cluster_name + - attribute: kafka_id present: true # telemetry from confluent-cloud provider @@ -42,6 +46,8 @@ synthesis: name: confluent.clusterName encodeIdentifierInGUID: true tags: + confluent.clusterId: + confluent.clusterName: collector.name: confluent.metricNamespace: instrumentation.provider: @@ -57,5 +63,5 @@ synthesis: value: confluent - attribute: confluent.resourceType value: kafka.server - - attribute: confluent.clusterName + - attribute: confluent.clusterId present: true \ No newline at end of file diff --git a/entity-types/infra-confluentcloudcluster/golden_metrics.yml b/entity-types/infra-confluentcloudcluster/golden_metrics.yml index 4ef91a3bc..0c5c52a52 100644 --- a/entity-types/infra-confluentcloudcluster/golden_metrics.yml +++ b/entity-types/infra-confluentcloudcluster/golden_metrics.yml @@ -3,26 +3,40 @@ clusterLoadPercent: unit: PERCENTAGE queries: opentelemetry: - select: min(confluent_kafka_server_cluster_load_percent) * 100 + select: average(confluent_kafka_server_cluster_load_percent) * 100 from: Metric where: kafka_id is not null eventName: kafka_id confluent: - select: min(confluent.kafka.server.cluster_load_percent) * 100 + select: average(confluent.kafka.server.cluster_load_percent) * 100 from: Metric where: confluent.clusterId is not null eventName: confluent.clusterId -partitionsCount: - title: Partitions count +hotPartitionIngress: + title: Hot Partition Ingress unit: COUNT queries: opentelemetry: - select: min(confluent_kafka_server_partition_count) + select: filter(uniqueCount(topic), WHERE metricName='confluent_kafka_server_hot_partition_ingress') from: Metric where: kafka_id is not null eventName: kafka_id confluent: - select: min(confluent.kafka.server.partition_count) + select: filter(uniqueCount(confluent.kafka.server.metric.topic), WHERE metricName='confluent.kafka.server.hot_partition_ingress') + from: Metric + where: confluent.clusterId is not null + eventName: confluent.clusterId +hotPartitionEgress: + title: Hot Partition Egress + unit: COUNT + queries: + opentelemetry: + select: filter(uniqueCount(topic), WHERE metricName='confluent_kafka_server_hot_partition_egress') + from: Metric + where: kafka_id is not null + eventName: kafka_id + confluent: + select: filter(uniqueCount(confluent.kafka.server.metric.topic), WHERE metricName='confluent.kafka.server.hot_partition_egress') from: Metric where: confluent.clusterId is not null eventName: confluent.clusterId diff --git a/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json b/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json index 25d280044..7c3ee085e 100644 --- a/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json +++ b/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json @@ -6,27 +6,12 @@ "name": "Confluent Cloud", "description": null, "widgets": [ - { - "title": "Confluent Cloud", - "layout": { - "column": 1, - "row": 1, - "width": 2, - "height": 4 - }, - "visualization": { - "id": "viz.markdown" - }, - "rawConfiguration": { - "text": "![ConfluentLogo](https://cdn.confluent.io/wp-content/uploads/seo-logo-meadow.png)\n# Confluent Cloud Kafka\n#### [Confluent API docs](https://api.telemetry.confluent.cloud/docs/descriptors/datasets/cloud?_ga=2.214885143.1264186867.1705940186-6520871.1686857317&_gl=1*xevtc5*_ga*NjUyMDg3MS4xNjg2ODU3MzE3*_ga_D2D3EGKSGD*MTcwNTk0MDA5MC41My4xLjE3MDU5NDAxODUuNjAuMC4w)" - } - }, { "title": "Cluster Specs", "layout": { - "column": 3, + "column": 1, "row": 1, - "width": 5, + "width": 6, "height": 4 }, "visualization": { @@ -45,7 +30,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT uniqueCount(topic) as 'Topics', latest(confluent_kafka_server_partition_count) as 'Partitions', uniqueCount(consumer_group_id) as 'Consumer Groups', uniqueCount(connector_id) AS 'Connectors' FROM Metric FACET `kafka_id`)" + "query": "SELECT uniqueCount(topic) as 'Topics', latest(confluent_kafka_server_partition_count) as 'Partitions', uniqueCount(consumer_group_id) as 'Consumer Groups', uniqueCount(connector_id) AS 'Connectors' FROM Metric" } ], "platformOptions": { @@ -56,9 +41,9 @@ { "title": "Request Bytes (bytes/second) Per Type", "layout": { - "column": 8, + "column": 7, "row": 1, - "width": 5, + "width": 6, "height": 4 }, "visualization": { @@ -74,7 +59,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric FACET kafka_id, type LIMIT MAX" + "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric WHERE FACET type LIMIT MAX" } ], "platformOptions": { @@ -103,7 +88,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(confluent_kafka_server_retained_bytes) FROM Metric TIMESERIES FACET kafka_id, topic LIMIT MAX" + "query": "SELECT average(confluent_kafka_server_retained_bytes) FROM Metric TIMESERIES FACET topic LIMIT MAX" } ], "platformOptions": { @@ -141,7 +126,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_sent_records),1 minute ) FROM Metric TIMESERIES FACET `topic`, kafka_id LIMIT MAX" + "query": "SELECT rate(sum(confluent_kafka_server_sent_records),1 minute ) FROM Metric TIMESERIES FACET `topic` LIMIT MAX" } ], "platformOptions": { @@ -173,7 +158,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT max(confluent_kafka_server_active_connection_count) AS 'Active Connections' FROM Metric facet kafka_id LIMIT MAX TIMESERIES" + "query": "SELECT max(confluent_kafka_server_active_connection_count) AS 'Active Connections' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -188,7 +173,7 @@ } }, { - "title": "Partitions ", + "title": "Partitions", "layout": { "column": 1, "row": 8, @@ -208,7 +193,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(confluent_kafka_server_partition_count) AS partitions FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + "query": "SELECT latest(confluent_kafka_server_partition_count) AS partitions FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -240,7 +225,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(confluent_kafka_server_received_bytes) FROM Metric TIMESERIES FACET `topic`, kafka_id LIMIT MAX" + "query": "SELECT sum(confluent_kafka_server_received_bytes) FROM Metric TIMESERIES FACET `topic` LIMIT MAX" } ], "platformOptions": { @@ -275,7 +260,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_received_records),1 minute ) FROM Metric TIMESERIES FACET `topic`, kafka_id LIMIT MAX" + "query": "SELECT rate(sum(confluent_kafka_server_received_records),1 minute) FROM Metric TIMESERIES FACET `topic` LIMIT MAX" } ], "platformOptions": { @@ -307,7 +292,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_successful_authentication_count), 1 minute) AS 'Auths Per Minute' FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent_kafka_server_successful_authentication_count), 1 minute) AS 'Auths Per Minute' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -342,7 +327,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(l) as egress from (SELECT rate(sum(confluent_kafka_server_response_bytes), 1 minute) as l FROM Metric FACET kafka_id TIMESERIES) LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 minute) FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -377,7 +362,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(confluent_kafka_server_sent_bytes) FROM Metric TIMESERIES FACET `topic`, kafka_id LIMIT MAX " + "query": "SELECT sum(confluent_kafka_server_sent_bytes) FROM Metric TIMESERIES FACET `topic` LIMIT MAX " } ], "platformOptions": { @@ -412,7 +397,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 second) FROM Metric FACET kafka_id) FACET type LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 second) FROM Metric FACET type LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -447,7 +432,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_request_count), 1 minute) as 'Requests Per Minute' FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent_kafka_server_request_count), 1 minute) as 'Requests Per Minute' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -482,7 +467,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 minute) as 'Ingress' FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 minute) as 'Ingress' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -517,7 +502,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric FACET type, kafka_id LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric FACET type LIMIT MAX TIMESERIES" } ], "platformOptions": { @@ -552,7 +537,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 minute) as 'Bytes Per Minute' FROM Metric FACET kafka_id LIMIT MAX TIMESERIES" + "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 minute) as 'Bytes Per Minute' FROM Metric LIMIT MAX TIMESERIES" } ], "platformOptions": { diff --git a/entity-types/infra-confluentcloudcluster/summary_metrics.yml b/entity-types/infra-confluentcloudcluster/summary_metrics.yml index b4d681687..ef8a08b54 100644 --- a/entity-types/infra-confluentcloudcluster/summary_metrics.yml +++ b/entity-types/infra-confluentcloudcluster/summary_metrics.yml @@ -7,7 +7,11 @@ clusterLoadPercent: goldenMetric: clusterLoadPercent unit: PERCENTAGE title: Cluster load percent -partitionsCount: - goldenMetric: partitionsCount +hotPartitionIngress: + goldenMetric: hotPartitionIngress unit: COUNT - title: Partitions count \ No newline at end of file + title: Hot Partition Ingress +hotPartitionEgress: + goldenMetric: hotPartitionEgress + unit: COUNT + title: Hot Partition Egress \ No newline at end of file From b836e50462337b57167823be6ee0c9267656c52c Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Mon, 30 Sep 2024 13:35:45 +0530 Subject: [PATCH 03/15] feat: NR-258723 dashboard metric query syntax issue fix --- .../infra-confluentcloudcluster/opentelemetry_dashboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json b/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json index 7c3ee085e..ac2f54276 100644 --- a/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json +++ b/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json @@ -59,7 +59,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric WHERE FACET type LIMIT MAX" + "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric FACET type LIMIT MAX" } ], "platformOptions": { From eade0d7229cf57f0ae08204522667e237f545d6f Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Mon, 16 Dec 2024 18:09:44 +0530 Subject: [PATCH 04/15] feat: NR-346401 adding new confluent cloud kafka topic entity --- .../confluent_dashboard.json | 153 ++------ .../definition.yml | 17 +- .../golden_metrics.yml | 28 -- .../opentelemetry_dashboard.json | 153 ++------ .../summary_metrics.yml | 10 +- .../tests/Metric.json | 56 --- .../confluent_dashboard.json | 338 ++++++++++++++++++ .../infra-confluentcloudtopic/definition.yml | 92 +++++ .../golden_metrics.yml | 28 ++ .../opentelemetry_dashboard.json | 338 ++++++++++++++++++ .../summary_metrics.yml | 13 + .../tests/Metric.json | 58 +++ 12 files changed, 946 insertions(+), 338 deletions(-) create mode 100644 entity-types/infra-confluentcloudtopic/confluent_dashboard.json create mode 100644 entity-types/infra-confluentcloudtopic/definition.yml create mode 100644 entity-types/infra-confluentcloudtopic/golden_metrics.yml create mode 100644 entity-types/infra-confluentcloudtopic/opentelemetry_dashboard.json create mode 100644 entity-types/infra-confluentcloudtopic/summary_metrics.yml create mode 100644 entity-types/infra-confluentcloudtopic/tests/Metric.json diff --git a/entity-types/infra-confluentcloudcluster/confluent_dashboard.json b/entity-types/infra-confluentcloudcluster/confluent_dashboard.json index 91d58a46d..d1f3f423d 100644 --- a/entity-types/infra-confluentcloudcluster/confluent_dashboard.json +++ b/entity-types/infra-confluentcloudcluster/confluent_dashboard.json @@ -1,13 +1,13 @@ { - "name": "Confluent Cloud", + "name": "Confluent Cloud Cluster", "description": null, "pages": [ { - "name": "Confluent Cloud", + "name": "Confluent Cloud Cluster", "description": null, "widgets": [ { - "title": "Cluster Specs", + "title": "Cluster load percent", "layout": { "column": 1, "row": 1, @@ -15,7 +15,7 @@ "height": 4 }, "visualization": { - "id": "viz.billboard" + "id": "viz.line" }, "rawConfiguration": { "dataFormatters": [ @@ -30,7 +30,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT uniqueCount(confluent.kafka.server.metric.topic) as 'Topics', average(confluent.kafka.server.partition_count) as 'Partitions', uniqueCount(confluent.kafka.server.metric.consumer_group_id) as 'Consumer Groups' FROM Metric" + "query": "select average(confluent.kafka.server.cluster_load_percent) * 100 AS 'Cluster load percent' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -39,7 +39,7 @@ } }, { - "title": "Request Bytes (bytes/second) Per Type", + "title": "Request throughtput (bytes/second)", "layout": { "column": 7, "row": 1, @@ -47,7 +47,7 @@ "height": 4 }, "visualization": { - "id": "viz.pie" + "id": "viz.line" }, "rawConfiguration": { "facet": { @@ -59,7 +59,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 second) FROM Metric FACET confluent.kafka.server.metric.type LIMIT MAX" + "query": "select rate(sum(confluent.kafka.server.request_bytes), 1 second) AS 'Request throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -68,7 +68,7 @@ } }, { - "title": "Retained Bytes Per Topic", + "title": "Response throughtput (bytes/second)", "layout": { "column": 1, "row": 5, @@ -88,15 +88,12 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT average(confluent.kafka.server.retained_bytes) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic LIMIT MAX" + "query": "select rate(sum(confluent.kafka.server.response_bytes), 1 second) AS 'Response throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "BYTES" - }, "yAxisLeft": { "zero": true }, @@ -106,7 +103,7 @@ } }, { - "title": "Records Sent Per Minute", + "title": "Active connections", "layout": { "column": 5, "row": 5, @@ -126,7 +123,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.sent_records),1 minute) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic LIMIT MAX" + "query": "select max(confluent.kafka.server.active_connection_count) AS 'Active connections' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -138,7 +135,7 @@ } }, { - "title": "Active Connections", + "title": "Requests", "layout": { "column": 9, "row": 5, @@ -158,7 +155,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT max(confluent.kafka.server.active_connection_count) AS 'Active Connections' FROM Metric LIMIT MAX TIMESERIES" + "query": "select max(confluent.kafka.server.request_count) AS 'Requests' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -193,7 +190,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT average(confluent.kafka.server.partition_count) AS partitions FROM Metric LIMIT MAX TIMESERIES" + "query": "select latest(confluent.kafka.server.partition_count) AS 'Partitions' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -205,7 +202,7 @@ } }, { - "title": "Recieved Bytes by Topic", + "title": "Successful Auth Attempts", "layout": { "column": 5, "row": 8, @@ -225,22 +222,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(confluent.kafka.server.received_bytes) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic" + "query": "select sum(confluent.kafka.server.successful_authentication_count) AS 'Successful Authentications' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "BYTES" - }, "yAxisLeft": { "zero": false } } }, { - "title": "Records Recieved Per Minute", + "title": "Rest produced request throughtput (bytes/second)", "layout": { "column": 9, "row": 8, @@ -260,7 +254,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.received_records),1 minute) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic LIMIT MAX" + "query": "select rate(sum(confluent.kafka.server.rest_produce_request_bytes), 1 second) AS 'Rest produced request throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -272,7 +266,7 @@ } }, { - "title": "Successful Auth Attempts", + "title": "Link destination response throughtput (bytes/second)", "layout": { "column": 1, "row": 11, @@ -292,22 +286,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.successful_authentication_count), 1 minute) AS 'Auths Per Minute' FROM Metric LIMIT MAX TIMESERIES" + "query": "select rate(sum(confluent.kafka.server.cluster_link_destination_response_bytes), 1 second) AS 'Cluster link destination response throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "COUNT" - }, "yAxisLeft": { "zero": true } } }, { - "title": "Egres rate (bytes/minute)", + "title": "Link source response throughtput (bytes/second)", "layout": { "column": 5, "row": 11, @@ -327,7 +318,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 minute) as 'Egres' FROM Metric LIMIT MAX TIMESERIES" + "query": "select rate(sum(confluent.kafka.server.cluster_link_source_response_bytes), 1 second) AS 'Cluster link source response throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -342,7 +333,7 @@ } }, { - "title": "Sent Bytes by Topic", + "title": "Cluster links", "layout": { "column": 9, "row": 11, @@ -362,22 +353,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(confluent.kafka.server.sent_bytes) FROM Metric TIMESERIES FACET confluent.kafka.server.metric.topic LIMIT MAX " + "query": "select max(confluent.kafka.server.cluster_link_count) AS 'Cluster links' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "BYTES" - }, "yAxisLeft": { "zero": true } } }, { - "title": "Response Bytes (bytes/second) Per Kafka Protocol Request Type", + "title": "Cluster link tasks", "layout": { "column": 1, "row": 14, @@ -397,22 +385,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 second) FROM Metric FACET confluent.kafka.server.metric.type LIMIT MAX TIMESERIES" + "query": "select max(confluent.kafka.server.cluster_link_task_count) AS 'Cluster link tasks' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "BYTES_PER_SECOND" - }, "yAxisLeft": { "zero": true } } }, { - "title": "Request Rate", + "title": "Link mirror topics error count", "layout": { "column": 5, "row": 14, @@ -432,22 +417,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.request_count), 1 minute) as 'Requests Per Minute' FROM Metric LIMIT MAX TIMESERIES" + "query": "select sum(confluent.kafka.server.cluster_link_mirror_transition_in_error) AS 'cluster link mirror topic errors' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "REQUESTS_PER_SECOND" - }, "yAxisLeft": { "zero": true } } }, { - "title": "Ingress rate (bytes/minute)", + "title": "Link mirror topics count", "layout": { "column": 9, "row": 14, @@ -467,7 +449,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 minute) as 'Ingress' FROM Metric LIMIT MAX TIMESERIES" + "query": "select sum(confluent.kafka.server.cluster_link_mirror_topic_count) AS 'Cluster link mirror topics' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -480,79 +462,6 @@ "zero": true } } - }, - { - "title": "Request Bytes (bytes/second) Per Kafka Protocol Request Type", - "layout": { - "column": 1, - "row": 17, - "width": 6, - "height": 3 - }, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.request_bytes), 1 second) FROM Metric FACET confluent.kafka.server.metric.type LIMIT MAX TIMESERIES" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "BYTES_PER_SECOND" - }, - "yAxisLeft": { - "zero": true - } - } - }, - { - "title": "Response Rate", - "layout": { - "column": 7, - "row": 17, - "width": 6, - "height": 3 - }, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(confluent.kafka.server.response_bytes), 1 minute) as 'Bytes Per Minute' FROM Metric LIMIT MAX TIMESERIES" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "REQUESTS_PER_SECOND" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } } ] } diff --git a/entity-types/infra-confluentcloudcluster/definition.yml b/entity-types/infra-confluentcloudcluster/definition.yml index cc34e1b77..b13b67133 100644 --- a/entity-types/infra-confluentcloudcluster/definition.yml +++ b/entity-types/infra-confluentcloudcluster/definition.yml @@ -10,6 +10,17 @@ goldenTags: - newrelic.source - kafka.cluster_name - confluent.resourceType + - collector.version + - confluent.region + - confluent.resourceGroup + - confluent.resourceId + - confluent.resourceType + - http.scheme + - instrumentation.provider + - net.host.name + - net.host.port + - service.instance.id + - service.name configuration: entityExpirationTime: DAILY alertable: true @@ -40,6 +51,8 @@ synthesis: value: opentelemetry - attribute: kafka_id present: true + - attribute: topic + present: false # telemetry from confluent-cloud provider - identifier: confluent.clusterId @@ -64,4 +77,6 @@ synthesis: - attribute: confluent.resourceType value: kafka.server - attribute: confluent.clusterId - present: true \ No newline at end of file + present: true + - attribute: confluent.kafka.server.metric.topic + present: false \ No newline at end of file diff --git a/entity-types/infra-confluentcloudcluster/golden_metrics.yml b/entity-types/infra-confluentcloudcluster/golden_metrics.yml index 0c5c52a52..eeb2a0180 100644 --- a/entity-types/infra-confluentcloudcluster/golden_metrics.yml +++ b/entity-types/infra-confluentcloudcluster/golden_metrics.yml @@ -12,31 +12,3 @@ clusterLoadPercent: from: Metric where: confluent.clusterId is not null eventName: confluent.clusterId -hotPartitionIngress: - title: Hot Partition Ingress - unit: COUNT - queries: - opentelemetry: - select: filter(uniqueCount(topic), WHERE metricName='confluent_kafka_server_hot_partition_ingress') - from: Metric - where: kafka_id is not null - eventName: kafka_id - confluent: - select: filter(uniqueCount(confluent.kafka.server.metric.topic), WHERE metricName='confluent.kafka.server.hot_partition_ingress') - from: Metric - where: confluent.clusterId is not null - eventName: confluent.clusterId -hotPartitionEgress: - title: Hot Partition Egress - unit: COUNT - queries: - opentelemetry: - select: filter(uniqueCount(topic), WHERE metricName='confluent_kafka_server_hot_partition_egress') - from: Metric - where: kafka_id is not null - eventName: kafka_id - confluent: - select: filter(uniqueCount(confluent.kafka.server.metric.topic), WHERE metricName='confluent.kafka.server.hot_partition_egress') - from: Metric - where: confluent.clusterId is not null - eventName: confluent.clusterId diff --git a/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json b/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json index ac2f54276..04382f891 100644 --- a/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json +++ b/entity-types/infra-confluentcloudcluster/opentelemetry_dashboard.json @@ -1,13 +1,13 @@ { - "name": "Confluent Cloud", + "name": "Confluent Cloud Cluster", "description": null, "pages": [ { - "name": "Confluent Cloud", + "name": "Confluent Cloud Cluster", "description": null, "widgets": [ { - "title": "Cluster Specs", + "title": "Cluster load percent", "layout": { "column": 1, "row": 1, @@ -15,7 +15,7 @@ "height": 4 }, "visualization": { - "id": "viz.billboard" + "id": "viz.line" }, "rawConfiguration": { "dataFormatters": [ @@ -30,7 +30,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT uniqueCount(topic) as 'Topics', latest(confluent_kafka_server_partition_count) as 'Partitions', uniqueCount(consumer_group_id) as 'Consumer Groups', uniqueCount(connector_id) AS 'Connectors' FROM Metric" + "query": "select average(confluent_kafka_server_cluster_load_percent) * 100 AS 'Cluster load percent' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -39,7 +39,7 @@ } }, { - "title": "Request Bytes (bytes/second) Per Type", + "title": "Request throughtput (bytes/second)", "layout": { "column": 7, "row": 1, @@ -47,7 +47,7 @@ "height": 4 }, "visualization": { - "id": "viz.pie" + "id": "viz.line" }, "rawConfiguration": { "facet": { @@ -59,7 +59,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric FACET type LIMIT MAX" + "query": "select rate(sum(confluent_kafka_server_request_bytes), 1 second) AS 'Request throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -68,7 +68,7 @@ } }, { - "title": "Retained Bytes Per Topic", + "title": "Response throughtput (bytes/second)", "layout": { "column": 1, "row": 5, @@ -88,15 +88,12 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT average(confluent_kafka_server_retained_bytes) FROM Metric TIMESERIES FACET topic LIMIT MAX" + "query": "select rate(sum(confluent_kafka_server_response_bytes), 1 second) AS 'Response throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "BYTES" - }, "yAxisLeft": { "zero": true }, @@ -106,7 +103,7 @@ } }, { - "title": "Records Sent Per Minute", + "title": "Active connections", "layout": { "column": 5, "row": 5, @@ -126,7 +123,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_sent_records),1 minute ) FROM Metric TIMESERIES FACET `topic` LIMIT MAX" + "query": "select max(confluent_kafka_server_active_connection_count) AS 'Active connections' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -138,7 +135,7 @@ } }, { - "title": "Active Connections", + "title": "Requests", "layout": { "column": 9, "row": 5, @@ -158,7 +155,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT max(confluent_kafka_server_active_connection_count) AS 'Active Connections' FROM Metric LIMIT MAX TIMESERIES" + "query": "select max(confluent_kafka_server_request_count) AS 'Requests' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -193,7 +190,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(confluent_kafka_server_partition_count) AS partitions FROM Metric LIMIT MAX TIMESERIES" + "query": "select latest(confluent_kafka_server_partition_count) AS 'Partitions' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -205,7 +202,7 @@ } }, { - "title": "Recieved Bytes by Topic", + "title": "Successful Auth Attempts", "layout": { "column": 5, "row": 8, @@ -225,22 +222,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(confluent_kafka_server_received_bytes) FROM Metric TIMESERIES FACET `topic` LIMIT MAX" + "query": "select sum(confluent_kafka_server_successful_authentication_count) AS 'Successful Authentications' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "BYTES" - }, "yAxisLeft": { "zero": false } } }, { - "title": "Records Recieved Per Minute", + "title": "Rest produced request throughtput (bytes/second)", "layout": { "column": 9, "row": 8, @@ -260,7 +254,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_received_records),1 minute) FROM Metric TIMESERIES FACET `topic` LIMIT MAX" + "query": "select rate(sum(confluent_kafka_server_rest_produce_request_bytes), 1 second) AS 'Rest produced request throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -272,7 +266,7 @@ } }, { - "title": "Successful Auth Attempts", + "title": "Link destination response throughtput (bytes/second)", "layout": { "column": 1, "row": 11, @@ -292,22 +286,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_successful_authentication_count), 1 minute) AS 'Auths Per Minute' FROM Metric LIMIT MAX TIMESERIES" + "query": "select rate(sum(confluent_kafka_server_cluster_link_destination_response_bytes), 1 second) AS 'Cluster link destination response throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "COUNT" - }, "yAxisLeft": { "zero": true } } }, { - "title": "Egres rate (bytes/minute)", + "title": "Link source response throughtput (bytes/second)", "layout": { "column": 5, "row": 11, @@ -327,7 +318,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 minute) FROM Metric LIMIT MAX TIMESERIES" + "query": "select rate(sum(confluent_kafka_server_cluster_link_source_response_bytes), 1 second) AS 'Cluster link source response throughtput' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -342,7 +333,7 @@ } }, { - "title": "Sent Bytes by Topic", + "title": "Cluster links", "layout": { "column": 9, "row": 11, @@ -362,22 +353,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(confluent_kafka_server_sent_bytes) FROM Metric TIMESERIES FACET `topic` LIMIT MAX " + "query": "select max(confluent_kafka_server_cluster_link_count) AS 'Cluster links' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "BYTES" - }, "yAxisLeft": { "zero": true } } }, { - "title": "Response Bytes (bytes/second) Per Kafka Protocol Request Type", + "title": "Cluster link tasks", "layout": { "column": 1, "row": 14, @@ -397,22 +385,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 second) FROM Metric FACET type LIMIT MAX TIMESERIES" + "query": "select max(confluent_kafka_server_cluster_link_task_count) AS 'Cluster link tasks' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "BYTES_PER_SECOND" - }, "yAxisLeft": { "zero": true } } }, { - "title": "Request Rate", + "title": "Link mirror topics error count", "layout": { "column": 5, "row": 14, @@ -432,22 +417,19 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_request_count), 1 minute) as 'Requests Per Minute' FROM Metric LIMIT MAX TIMESERIES" + "query": "select sum(confluent_kafka_server_cluster_link_mirror_transition_in_error) AS 'cluster link mirror topic errors' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { "ignoreTimeRange": false }, - "units": { - "unit": "REQUESTS_PER_SECOND" - }, "yAxisLeft": { "zero": true } } }, { - "title": "Ingress rate (bytes/minute)", + "title": "Link mirror topics count", "layout": { "column": 9, "row": 14, @@ -467,7 +449,7 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 minute) as 'Ingress' FROM Metric LIMIT MAX TIMESERIES" + "query": "select sum(confluent_kafka_server_cluster_link_mirror_topic_count) AS 'Cluster link mirror topics' from Metric TIMESERIES LIMIT MAX" } ], "platformOptions": { @@ -480,79 +462,6 @@ "zero": true } } - }, - { - "title": "Request Bytes (bytes/second) Per Kafka Protocol Request Type", - "layout": { - "column": 1, - "row": 17, - "width": 6, - "height": 3 - }, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_request_bytes), 1 second) FROM Metric FACET type LIMIT MAX TIMESERIES" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "BYTES_PER_SECOND" - }, - "yAxisLeft": { - "zero": true - } - } - }, - { - "title": "Response Rate ", - "layout": { - "column": 7, - "row": 17, - "width": 6, - "height": 3 - }, - "visualization": { - "id": "viz.line" - }, - "rawConfiguration": { - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "nrqlQueries": [ - { - "accountId": 0, - "query": "SELECT rate(sum(confluent_kafka_server_response_bytes), 1 minute) as 'Bytes Per Minute' FROM Metric LIMIT MAX TIMESERIES" - } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "units": { - "unit": "REQUESTS_PER_SECOND" - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } - } } ] } diff --git a/entity-types/infra-confluentcloudcluster/summary_metrics.yml b/entity-types/infra-confluentcloudcluster/summary_metrics.yml index ef8a08b54..609c3f05e 100644 --- a/entity-types/infra-confluentcloudcluster/summary_metrics.yml +++ b/entity-types/infra-confluentcloudcluster/summary_metrics.yml @@ -6,12 +6,4 @@ providerName: clusterLoadPercent: goldenMetric: clusterLoadPercent unit: PERCENTAGE - title: Cluster load percent -hotPartitionIngress: - goldenMetric: hotPartitionIngress - unit: COUNT - title: Hot Partition Ingress -hotPartitionEgress: - goldenMetric: hotPartitionEgress - unit: COUNT - title: Hot Partition Egress \ No newline at end of file + title: Cluster load percent \ No newline at end of file diff --git a/entity-types/infra-confluentcloudcluster/tests/Metric.json b/entity-types/infra-confluentcloudcluster/tests/Metric.json index 6b9f35587..7d45e84ff 100644 --- a/entity-types/infra-confluentcloudcluster/tests/Metric.json +++ b/entity-types/infra-confluentcloudcluster/tests/Metric.json @@ -1,60 +1,4 @@ [ - { - "confluent_kafka_server_consumer_lag_offsets": { - "type": "gauge", - "count": 352, - "sum": 0, - "min": 0, - "max": 0, - "latest": 0 - }, - "consumer_group_id": "agent_actions_vortex_vortex-3_23456", - "description": "The lag between a group member's committed offset and the partition's high watermark.", - "endTimestamp": 1719554400000, - "http.scheme": "https", - "instrumentation.provider": "opentelemetry", - "kafka.cluster_name": "us-jurassic-dust", - "kafka_id": "lkc-knjjjv", - "metricName": "confluent_kafka_server_consumer_lag_offsets", - "net.host.name": "api.telemetry.confluent.cloud", - "net.host.port": "443", - "newrelic.source": "api.metrics.otlp", - "nr.dataPointCount": 352, - "otel.library.name": "", - "otel.library.version": "", - "service.instance.id": "api.telemetry.confluent.cloud:443", - "service.name": "confluent", - "timestamp": 1719532800000, - "topic": "agent_actions" - }, - { - "confluent_kafka_server_received_bytes": { - "type": "gauge", - "count": 312, - "sum": 293255, - "min": 0, - "max": 74103, - "latest": 2741 - }, - "description": "The delta count of bytes of the customer's data received from the network. Each sample is the number of bytes received since the previous data sample. The count is sampled every 60 seconds.", - "endTimestamp": 1719554400000, - "http.scheme": "https", - "instrumentation.provider": "opentelemetry", - "kafka.cluster_name": "stg-rat-hole", - "kafka_id": "lkc-gx5zg1", - "metricName": "confluent_kafka_server_received_bytes", - "net.host.name": "api.telemetry.confluent.cloud", - "net.host.port": "443", - "newrelic.source": "api.metrics.otlp", - "nr.dataPointCount": 312, - "otel.library.name": "", - "otel.library.version": "", - "service.instance.id": "api.telemetry.confluent.cloud:443", - "service.name": "confluent", - "timestamp": 1719532800000, - "topic": "connect_account_data_2", - "unit": "bytes" - }, { "collector.name": "confluent-monitor", "collector.version": "release-11", diff --git a/entity-types/infra-confluentcloudtopic/confluent_dashboard.json b/entity-types/infra-confluentcloudtopic/confluent_dashboard.json new file mode 100644 index 000000000..0a23a1d26 --- /dev/null +++ b/entity-types/infra-confluentcloudtopic/confluent_dashboard.json @@ -0,0 +1,338 @@ +{ + "name": "Confluent Cloud Topic", + "description": null, + "pages": [ + { + "name": "Confluent Cloud Topic", + "description": null, + "widgets": [ + { + "title": "Received throughput (bytes/seconds)", + "layout": { + "column": 1, + "row": 1, + "width": 6, + "height": 4 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "dataFormatters": [ + { + "name": "Last Update", + "type": "date" + } + ], + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent.kafka.server.received_bytes), 1 second) AS 'Received throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Sent throughput (bytes/seconds)", + "layout": { + "column": 7, + "row": 1, + "width": 6, + "height": 4 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": true + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent.kafka.server.sent_bytes), 1 second) AS 'Sent throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Received records throughput (records/seconds)", + "layout": { + "column": 1, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent.kafka.server.received_records), 1 second) AS 'Received records throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Sent records throughput (records/seconds)", + "layout": { + "column": 5, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent.kafka.server.sent_records), 1 second) AS 'Sent records throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Retained throughput (bytes/seconds)", + "layout": { + "column": 9, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent.kafka.server.retained_bytes), 1 second) AS 'Retained throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Consumer lag offsets", + "layout": { + "column": 1, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select latest(confluent.kafka.server.consumer_lag_offsets) from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Link mirror topics offset lag", + "layout": { + "column": 5, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select latest(confluent.kafka.server.cluster_link_mirror_topic_offset_lag) from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": false + } + } + }, + { + "title": "Link mirror topics throughput (bytes/seconds)", + "layout": { + "column": 9, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent.kafka.server.cluster_link_mirror_topic_bytes), 1 second) AS 'Link mirror topics throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Hot partition ingress", + "layout": { + "column": 1, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select latest(confluent.kafka.server.hot_partition_ingress) from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Hot partition egress", + "layout": { + "column": 5, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select latest(confluent.kafka.server.hot_partition_egress) from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + } + ] + } + ] +} diff --git a/entity-types/infra-confluentcloudtopic/definition.yml b/entity-types/infra-confluentcloudtopic/definition.yml new file mode 100644 index 000000000..29e041506 --- /dev/null +++ b/entity-types/infra-confluentcloudtopic/definition.yml @@ -0,0 +1,92 @@ +domain: INFRA +type: CONFLUENTCLOUDKAFKATOPIC +goldenTags: + - confluent.clusterName + - kafka_id + - topic + - confluent.kafka.server.metric.topic + - confluent.clusterId + - confluent.metricNamespace + - collector.name + - instrumentation.provider + - newrelic.source + - kafka.cluster_name + - confluent.resourceType + - collector.version + - confluent.region + - confluent.resourceGroup + - confluent.resourceId + - confluent.resourceType + - http.scheme + - instrumentation.provider + - net.host.name + - net.host.port + - service.instance.id + - service.name +configuration: + entityExpirationTime: DAILY + alertable: true + +dashboardTemplates: + opentelemetry: + template: opentelemetry_dashboard.json + confluent: + template: confluent_dashboard.json + +synthesis: + rules: + # telemetry from opentelemetry provider + - compositeIdentifier: + separator: ":" + attributes: + - kafka_id + - topic + name: kafka.cluster_name || ":" || topic + encodeIdentifierInGUID: true + tags: + kafka_id: + kafka.cluster_name: + instrumentation.provider: + newrelic.source: + conditions: + - attribute: eventType + value: Metric + - attribute: metricName + prefix: confluent_kafka_server_ + - attribute: instrumentation.provider + value: opentelemetry + - attribute: kafka_id + present: true + - attribute: topic + present: true + + # telemetry from confluent-cloud provider + - compositeIdentifier: + separator: ":" + attributes: + - confluent.clusterId + - confluent.kafka.server.metric.topic + name: confluent.clusterName || ":" || confluent.kafka.server.metric.topic + encodeIdentifierInGUID: true + tags: + confluent.clusterId: + confluent.clusterName: + collector.name: + confluent.metricNamespace: + instrumentation.provider: + newrelic.source: + conditions: + - attribute: eventType + value: Metric + - attribute: collector.name + value: confluent-monitor + - attribute: metricName + prefix: confluent.kafka.server. + - attribute: instrumentation.provider + value: confluent + - attribute: confluent.resourceType + value: kafka.server + - attribute: confluent.clusterId + present: true + - attribute: confluent.kafka.server.metric.topic + present: true \ No newline at end of file diff --git a/entity-types/infra-confluentcloudtopic/golden_metrics.yml b/entity-types/infra-confluentcloudtopic/golden_metrics.yml new file mode 100644 index 000000000..d886102d5 --- /dev/null +++ b/entity-types/infra-confluentcloudtopic/golden_metrics.yml @@ -0,0 +1,28 @@ +hotPartitionIngress: + title: Hot Partition Ingress + unit: COUNT + queries: + opentelemetry: + select: filter(uniqueCount(topic), WHERE metricName='confluent_kafka_server_hot_partition_ingress') + from: Metric + where: kafka_id is not null + eventName: kafka_id + confluent: + select: filter(uniqueCount(confluent.kafka.server.metric.topic), WHERE metricName='confluent.kafka.server.hot_partition_ingress') + from: Metric + where: confluent.clusterId is not null + eventName: confluent.clusterId +hotPartitionEgress: + title: Hot Partition Egress + unit: COUNT + queries: + opentelemetry: + select: filter(uniqueCount(topic), WHERE metricName='confluent_kafka_server_hot_partition_egress') + from: Metric + where: kafka_id is not null + eventName: kafka_id + confluent: + select: filter(uniqueCount(confluent.kafka.server.metric.topic), WHERE metricName='confluent.kafka.server.hot_partition_egress') + from: Metric + where: confluent.clusterId is not null + eventName: confluent.clusterId diff --git a/entity-types/infra-confluentcloudtopic/opentelemetry_dashboard.json b/entity-types/infra-confluentcloudtopic/opentelemetry_dashboard.json new file mode 100644 index 000000000..424c2f77a --- /dev/null +++ b/entity-types/infra-confluentcloudtopic/opentelemetry_dashboard.json @@ -0,0 +1,338 @@ +{ + "name": "Confluent Cloud Topic", + "description": null, + "pages": [ + { + "name": "Confluent Cloud Topic", + "description": null, + "widgets": [ + { + "title": "Received throughput (bytes/seconds)", + "layout": { + "column": 1, + "row": 1, + "width": 6, + "height": 4 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "dataFormatters": [ + { + "name": "Last Update", + "type": "date" + } + ], + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent_kafka_server_received_bytes), 1 second) AS 'Received throughtput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Sent throughput (bytes/seconds)", + "layout": { + "column": 7, + "row": 1, + "width": 6, + "height": 4 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": true + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent_kafka_server_sent_bytes), 1 second) AS 'Sent throughtput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + } + } + }, + { + "title": "Received records throughput (records/seconds)", + "layout": { + "column": 1, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent_kafka_server_received_records), 1 second) AS 'Received records throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Sent records throughput (records/seconds)", + "layout": { + "column": 5, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent_kafka_server_sent_records), 1 second) AS 'Sent records throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Retained throughput (bytes/seconds)", + "layout": { + "column": 9, + "row": 5, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent_kafka_server_retained_bytes), 1 second) AS 'Retained throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + }, + { + "title": "Consumer lag offsets", + "layout": { + "column": 1, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select latest(confluent_kafka_server_consumer_lag_offsets) from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Link mirror topics offset lag", + "layout": { + "column": 5, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select latest(confluent_kafka_server_cluster_link_mirror_topic_offset_lag) from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": false + } + } + }, + { + "title": "Link mirror topics throughput (bytes/seconds)", + "layout": { + "column": 9, + "row": 8, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select rate(sum(confluent_kafka_server_cluster_link_mirror_topic_bytes), 1 second) AS 'Link mirror topics throughput' from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Hot partition ingress", + "layout": { + "column": 1, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select latest(confluent_kafka_server_hot_partition_ingress) from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + } + } + }, + { + "title": "Hot partition egress", + "layout": { + "column": 5, + "row": 11, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "select latest(confluent_kafka_server_hot_partition_egress) from Metric TIMESERIES LIMIT MAX" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } + } + } + ] + } + ] +} diff --git a/entity-types/infra-confluentcloudtopic/summary_metrics.yml b/entity-types/infra-confluentcloudtopic/summary_metrics.yml new file mode 100644 index 000000000..bfd04e865 --- /dev/null +++ b/entity-types/infra-confluentcloudtopic/summary_metrics.yml @@ -0,0 +1,13 @@ +providerName: + tag: + key: instrumentation.provider + title: Provider + unit: STRING +hotPartitionIngress: + goldenMetric: hotPartitionIngress + unit: COUNT + title: Hot Partition Ingress +hotPartitionEgress: + goldenMetric: hotPartitionEgress + unit: COUNT + title: Hot Partition Egress \ No newline at end of file diff --git a/entity-types/infra-confluentcloudtopic/tests/Metric.json b/entity-types/infra-confluentcloudtopic/tests/Metric.json new file mode 100644 index 000000000..457b27246 --- /dev/null +++ b/entity-types/infra-confluentcloudtopic/tests/Metric.json @@ -0,0 +1,58 @@ +[ + { + "confluent_kafka_server_consumer_lag_offsets": { + "type": "gauge", + "count": 352, + "sum": 0, + "min": 0, + "max": 0, + "latest": 0 + }, + "consumer_group_id": "agent_actions_vortex_vortex-3_23456", + "description": "The lag between a group member's committed offset and the partition's high watermark.", + "endTimestamp": 1719554400000, + "http.scheme": "https", + "instrumentation.provider": "opentelemetry", + "kafka.cluster_name": "us-jurassic-dust", + "kafka_id": "lkc-knjjjv", + "metricName": "confluent_kafka_server_consumer_lag_offsets", + "net.host.name": "api.telemetry.confluent.cloud", + "net.host.port": "443", + "newrelic.source": "api.metrics.otlp", + "nr.dataPointCount": 352, + "otel.library.name": "", + "otel.library.version": "", + "service.instance.id": "api.telemetry.confluent.cloud:443", + "service.name": "confluent", + "timestamp": 1719532800000, + "topic": "agent_actions" + }, + { + "confluent_kafka_server_received_bytes": { + "type": "gauge", + "count": 312, + "sum": 293255, + "min": 0, + "max": 74103, + "latest": 2741 + }, + "description": "The delta count of bytes of the customer's data received from the network. Each sample is the number of bytes received since the previous data sample. The count is sampled every 60 seconds.", + "endTimestamp": 1719554400000, + "http.scheme": "https", + "instrumentation.provider": "opentelemetry", + "kafka.cluster_name": "stg-rat-hole", + "kafka_id": "lkc-gx5zg1", + "metricName": "confluent_kafka_server_received_bytes", + "net.host.name": "api.telemetry.confluent.cloud", + "net.host.port": "443", + "newrelic.source": "api.metrics.otlp", + "nr.dataPointCount": 312, + "otel.library.name": "", + "otel.library.version": "", + "service.instance.id": "api.telemetry.confluent.cloud:443", + "service.name": "confluent", + "timestamp": 1719532800000, + "topic": "connect_account_data_2", + "unit": "bytes" + } +] From ea4576ff7971abbb85dd59c9797c9a388809e4f5 Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Mon, 16 Dec 2024 18:57:37 +0530 Subject: [PATCH 05/15] feat: Changed folder name --- .../confluent_dashboard.json | 0 .../definition.yml | 0 .../golden_metrics.yml | 0 .../opentelemetry_dashboard.json | 0 .../summary_metrics.yml | 0 .../tests/Metric.json | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename entity-types/{infra-confluentcloudtopic => infra-confluentcloudkafkatopic}/confluent_dashboard.json (100%) rename entity-types/{infra-confluentcloudtopic => infra-confluentcloudkafkatopic}/definition.yml (100%) rename entity-types/{infra-confluentcloudtopic => infra-confluentcloudkafkatopic}/golden_metrics.yml (100%) rename entity-types/{infra-confluentcloudtopic => infra-confluentcloudkafkatopic}/opentelemetry_dashboard.json (100%) rename entity-types/{infra-confluentcloudtopic => infra-confluentcloudkafkatopic}/summary_metrics.yml (100%) rename entity-types/{infra-confluentcloudtopic => infra-confluentcloudkafkatopic}/tests/Metric.json (100%) diff --git a/entity-types/infra-confluentcloudtopic/confluent_dashboard.json b/entity-types/infra-confluentcloudkafkatopic/confluent_dashboard.json similarity index 100% rename from entity-types/infra-confluentcloudtopic/confluent_dashboard.json rename to entity-types/infra-confluentcloudkafkatopic/confluent_dashboard.json diff --git a/entity-types/infra-confluentcloudtopic/definition.yml b/entity-types/infra-confluentcloudkafkatopic/definition.yml similarity index 100% rename from entity-types/infra-confluentcloudtopic/definition.yml rename to entity-types/infra-confluentcloudkafkatopic/definition.yml diff --git a/entity-types/infra-confluentcloudtopic/golden_metrics.yml b/entity-types/infra-confluentcloudkafkatopic/golden_metrics.yml similarity index 100% rename from entity-types/infra-confluentcloudtopic/golden_metrics.yml rename to entity-types/infra-confluentcloudkafkatopic/golden_metrics.yml diff --git a/entity-types/infra-confluentcloudtopic/opentelemetry_dashboard.json b/entity-types/infra-confluentcloudkafkatopic/opentelemetry_dashboard.json similarity index 100% rename from entity-types/infra-confluentcloudtopic/opentelemetry_dashboard.json rename to entity-types/infra-confluentcloudkafkatopic/opentelemetry_dashboard.json diff --git a/entity-types/infra-confluentcloudtopic/summary_metrics.yml b/entity-types/infra-confluentcloudkafkatopic/summary_metrics.yml similarity index 100% rename from entity-types/infra-confluentcloudtopic/summary_metrics.yml rename to entity-types/infra-confluentcloudkafkatopic/summary_metrics.yml diff --git a/entity-types/infra-confluentcloudtopic/tests/Metric.json b/entity-types/infra-confluentcloudkafkatopic/tests/Metric.json similarity index 100% rename from entity-types/infra-confluentcloudtopic/tests/Metric.json rename to entity-types/infra-confluentcloudkafkatopic/tests/Metric.json From 65c00e39010d40412b6dc2fb070cc12e93324de6 Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Thu, 26 Dec 2024 15:40:11 +0530 Subject: [PATCH 06/15] Adding APM entity to ConfluentCloudKafkaTopic entity relationship --- .../candidates/CONFLUENTCLOUDKAFKATOPIC.yml | 18 ++++++++ ...TION-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml | 46 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 relationships/candidates/CONFLUENTCLOUDKAFKATOPIC.yml create mode 100644 relationships/synthesis/APM-APPLICATION-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml diff --git a/relationships/candidates/CONFLUENTCLOUDKAFKATOPIC.yml b/relationships/candidates/CONFLUENTCLOUDKAFKATOPIC.yml new file mode 100644 index 000000000..9971db6fa --- /dev/null +++ b/relationships/candidates/CONFLUENTCLOUDKAFKATOPIC.yml @@ -0,0 +1,18 @@ +category: CONFLUENTCLOUDKAFKATOPIC +lookups: + - entityTypes: + - domain: INFRA + type: CONFLUENTCLOUDKAFKATOPIC + tags: + matchingMode: ALL + predicates: + - tagKeys: ["bootstrap_servers"] + field: bootstrapServers + - tagKeys: ["confluent.kafka.server.metric.topic", "topic"] + field: topic + onMatch: + onMultipleMatches: RELATE_ALL + onMiss: + action: CREATE_UNINSTRUMENTED + uninstrumented: + type: KAFKATOPIC diff --git a/relationships/synthesis/APM-APPLICATION-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml b/relationships/synthesis/APM-APPLICATION-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml new file mode 100644 index 000000000..ea8959c76 --- /dev/null +++ b/relationships/synthesis/APM-APPLICATION-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml @@ -0,0 +1,46 @@ +relationships: + - name: apmProducesConfluentCloudKafkaTopic + version: "1" + origins: + - APM Metrics + conditions: + - attribute: metricName + regex: "^MessageBroker/Kafka/Nodes/([^.]*).([^.]*).([^.]*).confluent.cloud:9[0-9]+/Produce/.*" + relationship: + expires: P75M + relationshipType: PRODUCES + source: + extractGuid: + attribute: entity.guid + target: + lookupGuid: + candidateCategory: CONFLUENTCLOUDKAFKATOPIC + fields: + - field: bootstrapServers + capture: + attribute: metricName__4 + - field: topic + attribute: metricName__6 + + - name: apmConsumesConfluentCloudKafkaTopic + version: "1" + origins: + - APM Metrics + conditions: + - attribute: metricName + regex: "^MessageBroker/Kafka/Nodes/([^.]*).([^.]*).([^.]*).confluent.cloud:9[0-9]+/Consume/.*" + relationship: + expires: P75M + relationshipType: CONSUMES + source: + extractGuid: + attribute: entity.guid + target: + lookupGuid: + candidateCategory: CONFLUENTCLOUDKAFKATOPIC + fields: + - field: bootstrapServers + capture: + attribute: metricName__4 + - field: topic + attribute: metricName__6 \ No newline at end of file From 6345886916bfe8325455ec9a1de63d29238acff3 Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Thu, 26 Dec 2024 16:05:04 +0530 Subject: [PATCH 07/15] Adding APM entity to ConfluentCloudKafkaTopic entity relationship --- ...ATOPIC.yml => APM-APPLICATION-to-CONFLUENTCLOUDKAFKATOPIC.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename relationships/synthesis/{APM-APPLICATION-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml => APM-APPLICATION-to-CONFLUENTCLOUDKAFKATOPIC.yml} (100%) diff --git a/relationships/synthesis/APM-APPLICATION-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml b/relationships/synthesis/APM-APPLICATION-to-CONFLUENTCLOUDKAFKATOPIC.yml similarity index 100% rename from relationships/synthesis/APM-APPLICATION-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml rename to relationships/synthesis/APM-APPLICATION-to-CONFLUENTCLOUDKAFKATOPIC.yml From df0ba5fc050cb457ae5070f65ab53f5535a0e3c2 Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Fri, 3 Jan 2025 16:03:22 +0530 Subject: [PATCH 08/15] feat: NR-346406 adding new relationship CONFLUENTCLOUDCLUSTER MANAGES CONFLUENTCLOUDKAFKATOPIC --- .../definition.yml | 68 ++++++++++++++++++- .../candidates/CONFLUENTCLOUDCLUSTER.yml | 16 +++++ ...STER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml | 26 +++++++ 3 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 relationships/candidates/CONFLUENTCLOUDCLUSTER.yml create mode 100644 relationships/synthesis/INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml diff --git a/entity-types/infra-confluentcloudkafkatopic/definition.yml b/entity-types/infra-confluentcloudkafkatopic/definition.yml index 29e041506..2c2403a28 100644 --- a/entity-types/infra-confluentcloudkafkatopic/definition.yml +++ b/entity-types/infra-confluentcloudkafkatopic/definition.yml @@ -41,7 +41,7 @@ synthesis: attributes: - kafka_id - topic - name: kafka.cluster_name || ":" || topic + name: displayName encodeIdentifierInGUID: true tags: kafka_id: @@ -59,6 +59,34 @@ synthesis: present: true - attribute: topic present: true + - attribute: displayName + present: true + + - compositeIdentifier: + separator: ":" + attributes: + - kafka_id + - topic + name: topic + encodeIdentifierInGUID: true + tags: + kafka_id: + kafka.cluster_name: + instrumentation.provider: + newrelic.source: + conditions: + - attribute: eventType + value: Metric + - attribute: metricName + prefix: confluent_kafka_server_ + - attribute: instrumentation.provider + value: opentelemetry + - attribute: kafka_id + present: true + - attribute: topic + present: true + - attribute: displayName + present: false # telemetry from confluent-cloud provider - compositeIdentifier: @@ -66,7 +94,7 @@ synthesis: attributes: - confluent.clusterId - confluent.kafka.server.metric.topic - name: confluent.clusterName || ":" || confluent.kafka.server.metric.topic + name: displayName encodeIdentifierInGUID: true tags: confluent.clusterId: @@ -89,4 +117,38 @@ synthesis: - attribute: confluent.clusterId present: true - attribute: confluent.kafka.server.metric.topic - present: true \ No newline at end of file + present: true + - attribute: displayName + present: true + + - compositeIdentifier: + separator: ":" + attributes: + - confluent.clusterId + - confluent.kafka.server.metric.topic + name: confluent.kafka.server.metric.topic + encodeIdentifierInGUID: true + tags: + confluent.clusterId: + confluent.clusterName: + collector.name: + confluent.metricNamespace: + instrumentation.provider: + newrelic.source: + conditions: + - attribute: eventType + value: Metric + - attribute: collector.name + value: confluent-monitor + - attribute: metricName + prefix: confluent.kafka.server. + - attribute: instrumentation.provider + value: confluent + - attribute: confluent.resourceType + value: kafka.server + - attribute: confluent.clusterId + present: true + - attribute: confluent.kafka.server.metric.topic + present: true + - attribute: displayName + present: false \ No newline at end of file diff --git a/relationships/candidates/CONFLUENTCLOUDCLUSTER.yml b/relationships/candidates/CONFLUENTCLOUDCLUSTER.yml new file mode 100644 index 000000000..a04751c89 --- /dev/null +++ b/relationships/candidates/CONFLUENTCLOUDCLUSTER.yml @@ -0,0 +1,16 @@ +category: CONFLUENTCLOUDCLUSTER +lookups: + - entityTypes: + - domain: INFRA + type: CONFLUENTCLOUDCLUSTER + tags: + matchingMode: ALL + predicates: + - tagKeys: ["kafka_id", "confluent.clusterId"] + field: confluentCloudClusterId + - tagKeys: ["kafka.cluster_name", "confluent.clusterName"] + field: confluentCloudClusterName + onMatch: + onMultipleMatches: RELATE_ALL + onMiss: + action: NO_OP \ No newline at end of file diff --git a/relationships/synthesis/INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml b/relationships/synthesis/INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml new file mode 100644 index 000000000..d2f23cac0 --- /dev/null +++ b/relationships/synthesis/INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml @@ -0,0 +1,26 @@ +relationships: + - name: confluentCloudClusterManagesConfluentCloudKafkaTopic + version: "1" + origins: + - AWS Integration + conditions: + - attribute: eventType + anyOf: [ "Metric" ] + - attribute: entity.type + anyOf: [ "CONFLUENTCLOUDKAFKATOPIC" ] + relationship: + expires: P75M + relationshipType: MANAGES + source: + lookupGuid: + candidateCategory: CONFLUENTCLOUDCLUSTER + fields: + - field: confluentCloudClusterId + attribute: confluent.clusterId + - field: confluentCloudClusterName + attribute: confluent.clusterName + target: + extractGuid: + attribute: entity.guid + entityType: + value: CONFLUENTCLOUDKAFKATOPIC From 5e8be020c23a009dbf248a700b9eb06c68857ff1 Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Fri, 3 Jan 2025 16:10:58 +0530 Subject: [PATCH 09/15] feat: NR-346406 using existing category and added new lookup for kafka topic --- .../candidates/CONFLUENTCLOUDKAFKATOPIC.yml | 18 -------- relationships/candidates/KAFKATOPIC.yml | 17 +++++++ ...PPLICATION-to-CONFLUENTCLOUDKAFKATOPIC.yml | 46 ------------------- .../APM-APPLICATION-to-KAFKATOPIC.yml | 46 +++++++++++++++++++ 4 files changed, 63 insertions(+), 64 deletions(-) delete mode 100644 relationships/candidates/CONFLUENTCLOUDKAFKATOPIC.yml delete mode 100644 relationships/synthesis/APM-APPLICATION-to-CONFLUENTCLOUDKAFKATOPIC.yml diff --git a/relationships/candidates/CONFLUENTCLOUDKAFKATOPIC.yml b/relationships/candidates/CONFLUENTCLOUDKAFKATOPIC.yml deleted file mode 100644 index 9971db6fa..000000000 --- a/relationships/candidates/CONFLUENTCLOUDKAFKATOPIC.yml +++ /dev/null @@ -1,18 +0,0 @@ -category: CONFLUENTCLOUDKAFKATOPIC -lookups: - - entityTypes: - - domain: INFRA - type: CONFLUENTCLOUDKAFKATOPIC - tags: - matchingMode: ALL - predicates: - - tagKeys: ["bootstrap_servers"] - field: bootstrapServers - - tagKeys: ["confluent.kafka.server.metric.topic", "topic"] - field: topic - onMatch: - onMultipleMatches: RELATE_ALL - onMiss: - action: CREATE_UNINSTRUMENTED - uninstrumented: - type: KAFKATOPIC diff --git a/relationships/candidates/KAFKATOPIC.yml b/relationships/candidates/KAFKATOPIC.yml index bc422b89c..fd3f06863 100644 --- a/relationships/candidates/KAFKATOPIC.yml +++ b/relationships/candidates/KAFKATOPIC.yml @@ -18,3 +18,20 @@ lookups: action: CREATE_UNINSTRUMENTED uninstrumented: type: KAFKATOPIC + + - entityTypes: + - domain: INFRA + type: CONFLUENTCLOUDKAFKATOPIC + tags: + matchingMode: ALL + predicates: + - tagKeys: ["bootstrap_servers"] + field: bootstrapServers + - tagKeys: ["confluent.kafka.server.metric.topic", "topic"] + field: topic + onMatch: + onMultipleMatches: RELATE_ALL + onMiss: + action: CREATE_UNINSTRUMENTED + uninstrumented: + type: KAFKATOPIC diff --git a/relationships/synthesis/APM-APPLICATION-to-CONFLUENTCLOUDKAFKATOPIC.yml b/relationships/synthesis/APM-APPLICATION-to-CONFLUENTCLOUDKAFKATOPIC.yml deleted file mode 100644 index ea8959c76..000000000 --- a/relationships/synthesis/APM-APPLICATION-to-CONFLUENTCLOUDKAFKATOPIC.yml +++ /dev/null @@ -1,46 +0,0 @@ -relationships: - - name: apmProducesConfluentCloudKafkaTopic - version: "1" - origins: - - APM Metrics - conditions: - - attribute: metricName - regex: "^MessageBroker/Kafka/Nodes/([^.]*).([^.]*).([^.]*).confluent.cloud:9[0-9]+/Produce/.*" - relationship: - expires: P75M - relationshipType: PRODUCES - source: - extractGuid: - attribute: entity.guid - target: - lookupGuid: - candidateCategory: CONFLUENTCLOUDKAFKATOPIC - fields: - - field: bootstrapServers - capture: - attribute: metricName__4 - - field: topic - attribute: metricName__6 - - - name: apmConsumesConfluentCloudKafkaTopic - version: "1" - origins: - - APM Metrics - conditions: - - attribute: metricName - regex: "^MessageBroker/Kafka/Nodes/([^.]*).([^.]*).([^.]*).confluent.cloud:9[0-9]+/Consume/.*" - relationship: - expires: P75M - relationshipType: CONSUMES - source: - extractGuid: - attribute: entity.guid - target: - lookupGuid: - candidateCategory: CONFLUENTCLOUDKAFKATOPIC - fields: - - field: bootstrapServers - capture: - attribute: metricName__4 - - field: topic - attribute: metricName__6 \ No newline at end of file diff --git a/relationships/synthesis/APM-APPLICATION-to-KAFKATOPIC.yml b/relationships/synthesis/APM-APPLICATION-to-KAFKATOPIC.yml index 7084f947e..3ac6104fb 100644 --- a/relationships/synthesis/APM-APPLICATION-to-KAFKATOPIC.yml +++ b/relationships/synthesis/APM-APPLICATION-to-KAFKATOPIC.yml @@ -54,3 +54,49 @@ relationships: regex: "^[^\\.]+\\.[^\\.]+\\.[^\\.]+\\.[^\\.]+\\.kafka\\.([^\\.]+)\\.amazonaws\\.com:9[0-9]+" - field: topic attribute: metricName__6 + + - name: apmProducesConfluentCloudKafkaTopic + version: "1" + origins: + - APM Metrics + conditions: + - attribute: metricName + regex: "^MessageBroker/Kafka/Nodes/([^.]*).([^.]*).([^.]*).confluent.cloud:9[0-9]+/Produce/.*" + relationship: + expires: P75M + relationshipType: PRODUCES + source: + extractGuid: + attribute: entity.guid + target: + lookupGuid: + candidateCategory: KAFKATOPIC + fields: + - field: bootstrapServers + capture: + attribute: metricName__4 + - field: topic + attribute: metricName__6 + + - name: apmConsumesConfluentCloudKafkaTopic + version: "1" + origins: + - APM Metrics + conditions: + - attribute: metricName + regex: "^MessageBroker/Kafka/Nodes/([^.]*).([^.]*).([^.]*).confluent.cloud:9[0-9]+/Consume/.*" + relationship: + expires: P75M + relationshipType: CONSUMES + source: + extractGuid: + attribute: entity.guid + target: + lookupGuid: + candidateCategory: KAFKATOPIC + fields: + - field: bootstrapServers + capture: + attribute: metricName__4 + - field: topic + attribute: metricName__6 \ No newline at end of file From fc10a60b817e0686776c9d5f7b7eff0bcebc704e Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Wed, 8 Jan 2025 14:56:51 +0530 Subject: [PATCH 10/15] feat: Adding tags for confluent kafka topic --- entity-types/infra-confluentcloudkafkatopic/definition.yml | 6 +++++- relationships/candidates/KAFKATOPIC.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/entity-types/infra-confluentcloudkafkatopic/definition.yml b/entity-types/infra-confluentcloudkafkatopic/definition.yml index 2c2403a28..37eda6390 100644 --- a/entity-types/infra-confluentcloudkafkatopic/definition.yml +++ b/entity-types/infra-confluentcloudkafkatopic/definition.yml @@ -7,11 +7,11 @@ goldenTags: - confluent.kafka.server.metric.topic - confluent.clusterId - confluent.metricNamespace + - confluent.clusterBootstrapServers - collector.name - instrumentation.provider - newrelic.source - kafka.cluster_name - - confluent.resourceType - collector.version - confluent.region - confluent.resourceGroup @@ -47,6 +47,7 @@ synthesis: kafka_id: kafka.cluster_name: instrumentation.provider: + bootstrap.servers: newrelic.source: conditions: - attribute: eventType @@ -73,6 +74,7 @@ synthesis: kafka_id: kafka.cluster_name: instrumentation.provider: + bootstrap.servers: newrelic.source: conditions: - attribute: eventType @@ -100,6 +102,7 @@ synthesis: confluent.clusterId: confluent.clusterName: collector.name: + confluent.clusterBootstrapServers: confluent.metricNamespace: instrumentation.provider: newrelic.source: @@ -133,6 +136,7 @@ synthesis: confluent.clusterName: collector.name: confluent.metricNamespace: + confluent.clusterBootstrapServers: instrumentation.provider: newrelic.source: conditions: diff --git a/relationships/candidates/KAFKATOPIC.yml b/relationships/candidates/KAFKATOPIC.yml index fd3f06863..be361233a 100644 --- a/relationships/candidates/KAFKATOPIC.yml +++ b/relationships/candidates/KAFKATOPIC.yml @@ -25,7 +25,7 @@ lookups: tags: matchingMode: ALL predicates: - - tagKeys: ["bootstrap_servers"] + - tagKeys: ["confluent.clusterBootstrapServers", "bootstrap.servers"] field: bootstrapServers - tagKeys: ["confluent.kafka.server.metric.topic", "topic"] field: topic From 91a8005d7fcb5b108e6ecd4a4c5cc7fc7c6a781a Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Mon, 13 Jan 2025 13:32:52 +0530 Subject: [PATCH 11/15] feat: Adding test jsons --- .../tests/Metric.json | 143 ++++++++++++------ 1 file changed, 93 insertions(+), 50 deletions(-) diff --git a/entity-types/infra-confluentcloudkafkatopic/tests/Metric.json b/entity-types/infra-confluentcloudkafkatopic/tests/Metric.json index 457b27246..c1cb8a834 100644 --- a/entity-types/infra-confluentcloudkafkatopic/tests/Metric.json +++ b/entity-types/infra-confluentcloudkafkatopic/tests/Metric.json @@ -1,58 +1,101 @@ [ { - "confluent_kafka_server_consumer_lag_offsets": { - "type": "gauge", - "count": 352, - "sum": 0, - "min": 0, - "max": 0, - "latest": 0 + "collector.name": "confluent-monitor", + "collector.version": "release-39", + "confluent.cloudProvider": "AWS", + "confluent.clusterBootstrapServers": "pkc-921jm.us-east-2.aws.confluent.cloud:9092", + "confluent.clusterId": "lkc-3kkndw", + "confluent.clusterName": "test_cluster", + "confluent.environment": "env-1nrgd5", + "confluent.kafka.server.metric.topic": "test-topic-conf-1", + "confluent.kafka.server.retained_bytes": { + "type": "summary", + "count": 1, + "sum": 1942860, + "min": 1942860, + "max": 1942860 }, - "consumer_group_id": "agent_actions_vortex_vortex-3_23456", - "description": "The lag between a group member's committed offset and the partition's high watermark.", - "endTimestamp": 1719554400000, - "http.scheme": "https", - "instrumentation.provider": "opentelemetry", - "kafka.cluster_name": "us-jurassic-dust", - "kafka_id": "lkc-knjjjv", - "metricName": "confluent_kafka_server_consumer_lag_offsets", - "net.host.name": "api.telemetry.confluent.cloud", - "net.host.port": "443", - "newrelic.source": "api.metrics.otlp", - "nr.dataPointCount": 352, - "otel.library.name": "", - "otel.library.version": "", - "service.instance.id": "api.telemetry.confluent.cloud:443", - "service.name": "confluent", - "timestamp": 1719532800000, - "topic": "agent_actions" + "confluent.metricNamespace": "confluent.kafka.server", + "confluent.region": "us-east-2", + "confluent.resourceGroup": "io.confluent.kafka.server", + "confluent.resourceId": "lkc-3kkndw", + "confluent.resourceType": "kafka.server", + "displayName": "test-topic-conf-1 (test_cluster)", + "endTimestamp": 1736440020000, + "instrumentation.provider": "confluent", + "metricName": "confluent.kafka.server.retained_bytes", + "newrelic.cloudIntegrations.integrationId": "107879", + "newrelic.cloudIntegrations.integrationName": "Confluent Kafka Resource", + "newrelic.cloudIntegrations.providerAccountId": "61124", + "newrelic.cloudIntegrations.providerAccountName": "sa-6v61z6", + "newrelic.cloudIntegrations.providerExternalId": "sa-6v61z6", + "newrelic.source": "metricAPI", + "timestamp": 1736439720000 }, { - "confluent_kafka_server_received_bytes": { - "type": "gauge", - "count": 312, - "sum": 293255, - "min": 0, - "max": 74103, - "latest": 2741 + "collector.name": "confluent-monitor", + "collector.version": "release-39", + "confluent.cloudProvider": "AWS", + "confluent.clusterBootstrapServers": "pkc-921jm.us-east-2.aws.confluent.cloud:9092", + "confluent.clusterId": "lkc-3kkndw", + "confluent.clusterName": "test_cluster", + "confluent.environment": "env-1nrgd5", + "confluent.kafka.server.metric.topic": "test-topic-conf-1", + "confluent.kafka.server.retained_bytes": { + "type": "summary", + "count": 1, + "sum": 1942860, + "min": 1942860, + "max": 1942860 }, - "description": "The delta count of bytes of the customer's data received from the network. Each sample is the number of bytes received since the previous data sample. The count is sampled every 60 seconds.", - "endTimestamp": 1719554400000, - "http.scheme": "https", - "instrumentation.provider": "opentelemetry", - "kafka.cluster_name": "stg-rat-hole", - "kafka_id": "lkc-gx5zg1", - "metricName": "confluent_kafka_server_received_bytes", - "net.host.name": "api.telemetry.confluent.cloud", - "net.host.port": "443", - "newrelic.source": "api.metrics.otlp", - "nr.dataPointCount": 312, - "otel.library.name": "", - "otel.library.version": "", - "service.instance.id": "api.telemetry.confluent.cloud:443", - "service.name": "confluent", - "timestamp": 1719532800000, - "topic": "connect_account_data_2", - "unit": "bytes" + "confluent.metricNamespace": "confluent.kafka.server", + "confluent.region": "us-east-2", + "confluent.resourceGroup": "io.confluent.kafka.server", + "confluent.resourceId": "lkc-3kkndw", + "confluent.resourceType": "kafka.server", + "displayName": "test-topic-conf-1 (test_cluster)", + "endTimestamp": 1736439960000, + "instrumentation.provider": "confluent", + "metricName": "confluent.kafka.server.retained_bytes", + "newrelic.cloudIntegrations.integrationId": "107879", + "newrelic.cloudIntegrations.integrationName": "Confluent Kafka Resource", + "newrelic.cloudIntegrations.providerAccountId": "61124", + "newrelic.cloudIntegrations.providerAccountName": "sa-6v61z6", + "newrelic.cloudIntegrations.providerExternalId": "sa-6v61z6", + "newrelic.source": "metricAPI", + "timestamp": 1736439660000 + }, + { + "collector.name": "confluent-monitor", + "collector.version": "release-39", + "confluent.cloudProvider": "AWS", + "confluent.clusterBootstrapServers": "pkc-921jm.us-east-2.aws.confluent.cloud:9092", + "confluent.clusterId": "lkc-3kkndw", + "confluent.clusterName": "test_cluster", + "confluent.environment": "env-1nrgd5", + "confluent.kafka.server.metric.topic": "test-topic-conf-1", + "confluent.kafka.server.retained_bytes": { + "type": "summary", + "count": 1, + "sum": 1942860, + "min": 1942860, + "max": 1942860 + }, + "confluent.metricNamespace": "confluent.kafka.server", + "confluent.region": "us-east-2", + "confluent.resourceGroup": "io.confluent.kafka.server", + "confluent.resourceId": "lkc-3kkndw", + "confluent.resourceType": "kafka.server", + "displayName": "test-topic-conf-1 (test_cluster)", + "endTimestamp": 1736439900000, + "instrumentation.provider": "confluent", + "metricName": "confluent.kafka.server.retained_bytes", + "newrelic.cloudIntegrations.integrationId": "107879", + "newrelic.cloudIntegrations.integrationName": "Confluent Kafka Resource", + "newrelic.cloudIntegrations.providerAccountId": "61124", + "newrelic.cloudIntegrations.providerAccountName": "sa-6v61z6", + "newrelic.cloudIntegrations.providerExternalId": "sa-6v61z6", + "newrelic.source": "metricAPI", + "timestamp": 1736439600000 } ] From 7f3a7fbfc2816eac4827cb5c347cd7b6bdd33462 Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Thu, 16 Jan 2025 15:50:55 +0530 Subject: [PATCH 12/15] feat: Adding topic name in tags for relationships --- entity-types/infra-confluentcloudkafkatopic/definition.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/entity-types/infra-confluentcloudkafkatopic/definition.yml b/entity-types/infra-confluentcloudkafkatopic/definition.yml index 37eda6390..3f120d5ce 100644 --- a/entity-types/infra-confluentcloudkafkatopic/definition.yml +++ b/entity-types/infra-confluentcloudkafkatopic/definition.yml @@ -47,6 +47,7 @@ synthesis: kafka_id: kafka.cluster_name: instrumentation.provider: + topic: bootstrap.servers: newrelic.source: conditions: @@ -73,6 +74,7 @@ synthesis: tags: kafka_id: kafka.cluster_name: + topic: instrumentation.provider: bootstrap.servers: newrelic.source: @@ -102,6 +104,7 @@ synthesis: confluent.clusterId: confluent.clusterName: collector.name: + confluent.kafka.server.metric.topic: confluent.clusterBootstrapServers: confluent.metricNamespace: instrumentation.provider: @@ -136,6 +139,7 @@ synthesis: confluent.clusterName: collector.name: confluent.metricNamespace: + confluent.kafka.server.metric.topic: confluent.clusterBootstrapServers: instrumentation.provider: newrelic.source: From 62b4eb3ed834e1367b571eca2f0a143b855b51bb Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Thu, 16 Jan 2025 15:57:56 +0530 Subject: [PATCH 13/15] feat: Renaming relationship file --- ...> CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename relationships/synthesis/{INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml => CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml} (100%) diff --git a/relationships/synthesis/INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml b/relationships/synthesis/CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml similarity index 100% rename from relationships/synthesis/INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml rename to relationships/synthesis/CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml From b0fcdb1cffc7c09a0c0dd2f2ae0394c7652647ca Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Thu, 16 Jan 2025 16:06:09 +0530 Subject: [PATCH 14/15] feat: Revert renaming relationship file --- ...A-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename relationships/synthesis/{CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml => INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml} (100%) diff --git a/relationships/synthesis/CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml b/relationships/synthesis/INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml similarity index 100% rename from relationships/synthesis/CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml rename to relationships/synthesis/INFRA-CONFLUENTCLOUDCLUSTER-to-INFRA-CONFLUENTCLOUDKAFKATOPIC.yml From 896c83bdce8f16e4a7c6e29b3fa2f863535c916b Mon Sep 17 00:00:00 2001 From: dsankaraganti Date: Thu, 16 Jan 2025 16:52:10 +0530 Subject: [PATCH 15/15] feat: Removed capture using for relationships --- relationships/synthesis/APM-APPLICATION-to-KAFKATOPIC.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/relationships/synthesis/APM-APPLICATION-to-KAFKATOPIC.yml b/relationships/synthesis/APM-APPLICATION-to-KAFKATOPIC.yml index 3ac6104fb..a2d066074 100644 --- a/relationships/synthesis/APM-APPLICATION-to-KAFKATOPIC.yml +++ b/relationships/synthesis/APM-APPLICATION-to-KAFKATOPIC.yml @@ -73,8 +73,7 @@ relationships: candidateCategory: KAFKATOPIC fields: - field: bootstrapServers - capture: - attribute: metricName__4 + attribute: metricName__4 - field: topic attribute: metricName__6 @@ -96,7 +95,6 @@ relationships: candidateCategory: KAFKATOPIC fields: - field: bootstrapServers - capture: - attribute: metricName__4 + attribute: metricName__4 - field: topic attribute: metricName__6 \ No newline at end of file