fix: loki schema config update

pepr.ts

@@ -8,6 +8,7 @@ import { PeprModule } from "pepr";
 import cfg from "./package.json";
 
 import { Component, setupLogger } from "./src/pepr/logger";
+import { loki } from "./src/pepr/loki";
 import { operator } from "./src/pepr/operator";
 import { setupAuthserviceSecret } from "./src/pepr/operator/controllers/keycloak/authservice/config";
 import { registerCRDs } from "./src/pepr/operator/crd/register";
@@ -33,6 +34,9 @@ const log = setupLogger(Component.STARTUP);
     // Prometheus monitoring stack
     prometheus,
 
+    // Loki logging stack
+    loki,
+
     // Patches for specific components
     patches,
   ]);

src/loki/chart/templates/loki-dashboards.yaml

@@ -4,7 +4,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: loki-grafana-dashboards 
+  name: loki-grafana-dashboards
   namespace: grafana
   labels:
     grafana_dashboard: "1"

src/loki/values/values.yaml

@@ -46,6 +46,13 @@ loki:
         index:
           prefix: loki_index_
           period: 24h
+      - from: "2020-01-11" # Updated dynamically by Pepr to be default retention store
+        store: tsdb
+        object_store: "{{ .Values.loki.storage.type }}"
+        schema: v13
+        index:
+          prefix: loki_index_
+          period: 24h
   limits_config:
     split_queries_by_interval: "30m"
     allow_structured_metadata: false

src/pepr/config.ts

@@ -44,6 +44,9 @@ export const UDSConfig = {
 
   // Track if UDS Core identity-authorization layer is deployed
   isIdentityDeployed: false,
+
+  //Loki Default Store Type
+  lokiDefaultStore: "tsdb",
 };
 
 // configure subproject logger

src/pepr/logger.ts

@@ -23,6 +23,7 @@ export enum Component {
   POLICIES_EXEMPTIONS = "policies.exemptions",
   PROMETHEUS = "prometheus",
   PATCHES = "patches",
+  LOKI = "LOKI",
 }
 
 export function setupLogger(component: Component) {

src/pepr/loki/index.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright 2024 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { Capability, kind } from "pepr";
+import { UDSConfig } from "../config";
+import { Component, setupLogger } from "../logger";
+import {
+  calculateFutureDate,
+  encodeConfig,
+  isConfigUpdateRequired,
+  parseLokiConfig,
+  updateConfigDate,
+} from "./utils";
+
+const log = setupLogger(Component.LOKI);
+
+export const loki = new Capability({
+  name: "loki",
+  description: "UDS Core Capability for the Loki stack.",
+});
+
+const { When } = loki;
+
+When(kind.Secret)
+  .IsCreatedOrUpdated()
+  .InNamespace("loki")
+  .WithName("loki")
+  .Mutate(async secret => {
+    if (!secret.Raw.data || !secret.Raw.data["config.yaml"]) {
+      log.error(`Missing 'data' field or 'config.yaml' key in ${secret.Raw.metadata?.name}`);
+      return;
+    }
+
+    const lokiConfig = parseLokiConfig(secret.Raw.data["config.yaml"]);
+    if (!lokiConfig) {
+      log.error("Failed to parse Loki configuration.");
+      return;
+    }
+
+    if (isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)) {
+      const futureDate = calculateFutureDate(2);
+      if (
+        updateConfigDate(
+          lokiConfig.schema_config?.configs || [],
+          UDSConfig.lokiDefaultStore,
+          futureDate,
+        )
+      ) {
+        secret.Raw.data["config.yaml"] = encodeConfig(lokiConfig);
+        log.info(
+          `Loki schemaConfig configuration updated and saved for ${secret.Raw.metadata?.name}`,
+        );
+      } else {
+        log.error(
+          `Failed to update Loki schemaConfig configuration for ${secret.Raw.metadata?.name}`,
+        );
+      }
+    } else {
+      log.info(
+        `No update required for Loki schemaConfig configuration for ${secret.Raw.metadata?.name}`,
+      );
+    }
+  });

src/pepr/loki/utils.spec.ts

@@ -0,0 +1,161 @@
+/**
+ * Copyright 2024 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import { afterAll, beforeAll, describe, expect, it, jest } from "@jest/globals";
+import {
+  calculateFutureDate,
+  encodeConfig,
+  isConfigUpdateRequired,
+  parseLokiConfig,
+  updateConfigDate,
+} from "./utils";
+
+import { UDSConfig } from "../config";
+
+describe("calculateFutureDate", () => {
+  beforeAll(() => {
+    // Mocking global date to ensure consistent results in testing future date calculations.
+    jest.spyOn(Date, "now").mockImplementation(() => new Date("2023-01-01T00:00:00Z").getTime());
+    const originalDate = Date;
+    jest.spyOn(global, "Date").mockImplementation((...args) => {
+      return args.length ? new originalDate(...args) : new originalDate("2023-01-01T00:00:00Z");
+    });
+  });
+
+  afterAll(() => {
+    // Restoring all mocks to their original implementations after tests are done.
+    jest.restoreAllMocks();
+  });
+
+  it("should return a date string two days in the future", () => {
+    // Ensuring the calculateFutureDate correctly adds days to the current date.
+    const result = calculateFutureDate(2);
+    expect(result).toBe("2023-01-03");
+  });
+});
+
+describe("parseLokiConfig", () => {
+  it("should parse valid YAML string into an object", () => {
+    // Testing the function's ability to correctly parse a well-formed YAML into a config object.
+    const yamlString = `
+      schema_config:
+        configs:
+          - from: "2023-01-01"
+            store: "${UDSConfig.lokiDefaultStore}"
+            index:
+              prefix: "loki_"
+              period: "24h"
+    `;
+    const result = parseLokiConfig(yamlString);
+    expect(result).toEqual({
+      schema_config: {
+        configs: [
+          {
+            from: "2023-01-01",
+            store: UDSConfig.lokiDefaultStore,
+            index: {
+              prefix: "loki_",
+              period: "24h",
+            },
+          },
+        ],
+      },
+    });
+  });
+
+  it("should return null on invalid YAML", () => {
+    // Testing the function's error handling on receiving bad YAML format.
+    const badYaml = `: I am not YAML!`;
+    expect(parseLokiConfig(badYaml)).toBeNull();
+  });
+});
+
+describe("updateConfigDate", () => {
+  it(`should update the from date in the ${UDSConfig.lokiDefaultStore} config`, () => {
+    // Verifying that the function updates the 'from' date for a specified store type.
+    const configs = [
+      {
+        from: "2023-01-01",
+        store: UDSConfig.lokiDefaultStore,
+      },
+    ];
+    const result = updateConfigDate(configs, UDSConfig.lokiDefaultStore, "2023-01-10");
+    expect(result).toBeTruthy();
+    expect(configs[0].from).toBe("2023-01-10");
+  });
+
+  it(`should return false if no ${UDSConfig.lokiDefaultStore} config is found`, () => {
+    // Testing the function's response when no matching store type configuration is found.
+    const configs = [{ from: "2023-01-01", store: "other" }];
+    expect(updateConfigDate(configs, "2023-01-10", UDSConfig.lokiDefaultStore)).toBeFalsy();
+  });
+});
+
+describe("encodeConfig", () => {
+  it("should encode a config object to a YAML string", () => {
+    // Ensuring that the encodeConfig function can serialize a config object back to YAML format correctly.
+    const config = {
+      schema_config: {
+        configs: [
+          {
+            from: "2023-01-01",
+            store: UDSConfig.lokiDefaultStore,
+            index: {
+              prefix: "loki_",
+              period: "24h",
+            },
+          },
+        ],
+      },
+    };
+    const yamlString = encodeConfig(config);
+    expect(yamlString).toMatch(new RegExp(UDSConfig.lokiDefaultStore));
+    expect(yamlString).toMatch(/loki_/);
+  });
+});
+
+describe("isConfigUpdateRequired", () => {
+  it(`should return false if ${UDSConfig.lokiDefaultStore} is set correctly in the future and is the latest configuration`, () => {
+    // Validating that no update is required if the default store type is already correctly set.
+    const configs = [
+      { from: "2023-01-01", store: "boltdb-shipper" },
+      { from: "2025-01-01", store: UDSConfig.lokiDefaultStore },
+    ];
+    const lokiConfig = { schema_config: { configs } };
+    expect(isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)).toBe(false);
+  });
+
+  it(`should return true if ${UDSConfig.lokiDefaultStore} is set in the past`, () => {
+    // Checking that an update is required if the default store type's date is set in the past.
+    const configs = [
+      { from: "2021-01-01", store: "boltdb-shipper" },
+      { from: "2020-01-01", store: UDSConfig.lokiDefaultStore },
+    ];
+    const lokiConfig = { schema_config: { configs } };
+    expect(isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)).toBe(true);
+  });
+
+  it(`should return true if ${UDSConfig.lokiDefaultStore} is not the latest configuration`, () => {
+    // Ensuring that an update is needed if there is a newer configuration than the default store type.
+    const configs = [
+      { from: "2025-01-02", store: "boltdb-shipper" },
+      { from: "2025-01-01", store: UDSConfig.lokiDefaultStore },
+    ];
+    const lokiConfig = { schema_config: { configs } };
+    expect(isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)).toBe(true);
+  });
+
+  it(`should return true if ${UDSConfig.lokiDefaultStore} configuration is missing`, () => {
+    // Testing that an update is required if the default store type configuration is completely missing.
+    const configs = [{ from: "2023-01-01", store: "boltdb-shipper" }];
+    const lokiConfig = { schema_config: { configs } };
+    expect(isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)).toBe(true);
+  });
+
+  it(`should return true when config is empty`, () => {
+    // Testing that an update is required if the config is completely missing.
+    expect(isConfigUpdateRequired({}, UDSConfig.lokiDefaultStore)).toBe(true);
+  });
+});