diff --git a/pepr.ts b/pepr.ts index 64646428c..1bba50b4d 100644 --- a/pepr.ts +++ b/pepr.ts @@ -8,6 +8,7 @@ import { PeprModule } from "pepr"; import cfg from "./package.json"; import { Component, setupLogger } from "./src/pepr/logger"; +import { loki } from "./src/pepr/loki"; import { operator } from "./src/pepr/operator"; import { setupAuthserviceSecret } from "./src/pepr/operator/controllers/keycloak/authservice/config"; import { registerCRDs } from "./src/pepr/operator/crd/register"; @@ -33,6 +34,9 @@ const log = setupLogger(Component.STARTUP); // Prometheus monitoring stack prometheus, + // Loki logging stack + loki, + // Patches for specific components patches, ]); diff --git a/src/loki/chart/templates/loki-dashboards.yaml b/src/loki/chart/templates/loki-dashboards.yaml index 92cc07c13..3b4ffc141 100644 --- a/src/loki/chart/templates/loki-dashboards.yaml +++ b/src/loki/chart/templates/loki-dashboards.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: loki-grafana-dashboards + name: loki-grafana-dashboards namespace: grafana labels: grafana_dashboard: "1" diff --git a/src/loki/values/values.yaml b/src/loki/values/values.yaml index 0c9827d51..de52c517c 100644 --- a/src/loki/values/values.yaml +++ b/src/loki/values/values.yaml @@ -46,6 +46,13 @@ loki: index: prefix: loki_index_ period: 24h + - from: "2020-01-11" # Updated dynamically by Pepr to be default retention store + store: tsdb + object_store: "{{ .Values.loki.storage.type }}" + schema: v13 + index: + prefix: loki_index_ + period: 24h limits_config: split_queries_by_interval: "30m" allow_structured_metadata: false diff --git a/src/pepr/config.ts b/src/pepr/config.ts index 9ae7d4bbe..71ede7538 100644 --- a/src/pepr/config.ts +++ b/src/pepr/config.ts @@ -44,6 +44,9 @@ export const UDSConfig = { // Track if UDS Core identity-authorization layer is deployed isIdentityDeployed: false, + + //Loki Default Store Type + lokiDefaultStore: "tsdb", }; // configure subproject logger diff --git a/src/pepr/logger.ts b/src/pepr/logger.ts index fd39a81f4..203b025e1 100644 --- a/src/pepr/logger.ts +++ b/src/pepr/logger.ts @@ -23,6 +23,7 @@ export enum Component { POLICIES_EXEMPTIONS = "policies.exemptions", PROMETHEUS = "prometheus", PATCHES = "patches", + LOKI = "LOKI", } export function setupLogger(component: Component) { diff --git a/src/pepr/loki/index.ts b/src/pepr/loki/index.ts new file mode 100644 index 000000000..22ac6236c --- /dev/null +++ b/src/pepr/loki/index.ts @@ -0,0 +1,65 @@ +/** + * Copyright 2024 Defense Unicorns + * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + */ + +import { Capability, kind } from "pepr"; +import { UDSConfig } from "../config"; +import { Component, setupLogger } from "../logger"; +import { + calculateFutureDate, + encodeConfig, + isConfigUpdateRequired, + parseLokiConfig, + updateConfigDate, +} from "./utils"; + +const log = setupLogger(Component.LOKI); + +export const loki = new Capability({ + name: "loki", + description: "UDS Core Capability for the Loki stack.", +}); + +const { When } = loki; + +When(kind.Secret) + .IsCreatedOrUpdated() + .InNamespace("loki") + .WithName("loki") + .Mutate(async secret => { + if (!secret.Raw.data || !secret.Raw.data["config.yaml"]) { + log.error(`Missing 'data' field or 'config.yaml' key in ${secret.Raw.metadata?.name}`); + return; + } + + const lokiConfig = parseLokiConfig(secret.Raw.data["config.yaml"]); + if (!lokiConfig) { + log.error("Failed to parse Loki configuration."); + return; + } + + if (isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)) { + const futureDate = calculateFutureDate(2); + if ( + updateConfigDate( + lokiConfig.schema_config?.configs || [], + UDSConfig.lokiDefaultStore, + futureDate, + ) + ) { + secret.Raw.data["config.yaml"] = encodeConfig(lokiConfig); + log.info( + `Loki schemaConfig configuration updated and saved for ${secret.Raw.metadata?.name}`, + ); + } else { + log.error( + `Failed to update Loki schemaConfig configuration for ${secret.Raw.metadata?.name}`, + ); + } + } else { + log.info( + `No update required for Loki schemaConfig configuration for ${secret.Raw.metadata?.name}`, + ); + } + }); diff --git a/src/pepr/loki/utils.spec.ts b/src/pepr/loki/utils.spec.ts new file mode 100644 index 000000000..4fdf7edd2 --- /dev/null +++ b/src/pepr/loki/utils.spec.ts @@ -0,0 +1,161 @@ +/** + * Copyright 2024 Defense Unicorns + * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + */ + +import { afterAll, beforeAll, describe, expect, it, jest } from "@jest/globals"; +import { + calculateFutureDate, + encodeConfig, + isConfigUpdateRequired, + parseLokiConfig, + updateConfigDate, +} from "./utils"; + +import { UDSConfig } from "../config"; + +describe("calculateFutureDate", () => { + beforeAll(() => { + // Mocking global date to ensure consistent results in testing future date calculations. + jest.spyOn(Date, "now").mockImplementation(() => new Date("2023-01-01T00:00:00Z").getTime()); + const originalDate = Date; + jest.spyOn(global, "Date").mockImplementation((...args) => { + return args.length ? new originalDate(...args) : new originalDate("2023-01-01T00:00:00Z"); + }); + }); + + afterAll(() => { + // Restoring all mocks to their original implementations after tests are done. + jest.restoreAllMocks(); + }); + + it("should return a date string two days in the future", () => { + // Ensuring the calculateFutureDate correctly adds days to the current date. + const result = calculateFutureDate(2); + expect(result).toBe("2023-01-03"); + }); +}); + +describe("parseLokiConfig", () => { + it("should parse valid YAML string into an object", () => { + // Testing the function's ability to correctly parse a well-formed YAML into a config object. + const yamlString = ` + schema_config: + configs: + - from: "2023-01-01" + store: "${UDSConfig.lokiDefaultStore}" + index: + prefix: "loki_" + period: "24h" + `; + const result = parseLokiConfig(yamlString); + expect(result).toEqual({ + schema_config: { + configs: [ + { + from: "2023-01-01", + store: UDSConfig.lokiDefaultStore, + index: { + prefix: "loki_", + period: "24h", + }, + }, + ], + }, + }); + }); + + it("should return null on invalid YAML", () => { + // Testing the function's error handling on receiving bad YAML format. + const badYaml = `: I am not YAML!`; + expect(parseLokiConfig(badYaml)).toBeNull(); + }); +}); + +describe("updateConfigDate", () => { + it(`should update the from date in the ${UDSConfig.lokiDefaultStore} config`, () => { + // Verifying that the function updates the 'from' date for a specified store type. + const configs = [ + { + from: "2023-01-01", + store: UDSConfig.lokiDefaultStore, + }, + ]; + const result = updateConfigDate(configs, UDSConfig.lokiDefaultStore, "2023-01-10"); + expect(result).toBeTruthy(); + expect(configs[0].from).toBe("2023-01-10"); + }); + + it(`should return false if no ${UDSConfig.lokiDefaultStore} config is found`, () => { + // Testing the function's response when no matching store type configuration is found. + const configs = [{ from: "2023-01-01", store: "other" }]; + expect(updateConfigDate(configs, "2023-01-10", UDSConfig.lokiDefaultStore)).toBeFalsy(); + }); +}); + +describe("encodeConfig", () => { + it("should encode a config object to a YAML string", () => { + // Ensuring that the encodeConfig function can serialize a config object back to YAML format correctly. + const config = { + schema_config: { + configs: [ + { + from: "2023-01-01", + store: UDSConfig.lokiDefaultStore, + index: { + prefix: "loki_", + period: "24h", + }, + }, + ], + }, + }; + const yamlString = encodeConfig(config); + expect(yamlString).toMatch(new RegExp(UDSConfig.lokiDefaultStore)); + expect(yamlString).toMatch(/loki_/); + }); +}); + +describe("isConfigUpdateRequired", () => { + it(`should return false if ${UDSConfig.lokiDefaultStore} is set correctly in the future and is the latest configuration`, () => { + // Validating that no update is required if the default store type is already correctly set. + const configs = [ + { from: "2023-01-01", store: "boltdb-shipper" }, + { from: "2025-01-01", store: UDSConfig.lokiDefaultStore }, + ]; + const lokiConfig = { schema_config: { configs } }; + expect(isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)).toBe(false); + }); + + it(`should return true if ${UDSConfig.lokiDefaultStore} is set in the past`, () => { + // Checking that an update is required if the default store type's date is set in the past. + const configs = [ + { from: "2021-01-01", store: "boltdb-shipper" }, + { from: "2020-01-01", store: UDSConfig.lokiDefaultStore }, + ]; + const lokiConfig = { schema_config: { configs } }; + expect(isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)).toBe(true); + }); + + it(`should return true if ${UDSConfig.lokiDefaultStore} is not the latest configuration`, () => { + // Ensuring that an update is needed if there is a newer configuration than the default store type. + const configs = [ + { from: "2025-01-02", store: "boltdb-shipper" }, + { from: "2025-01-01", store: UDSConfig.lokiDefaultStore }, + ]; + const lokiConfig = { schema_config: { configs } }; + expect(isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)).toBe(true); + }); + + it(`should return true if ${UDSConfig.lokiDefaultStore} configuration is missing`, () => { + // Testing that an update is required if the default store type configuration is completely missing. + const configs = [{ from: "2023-01-01", store: "boltdb-shipper" }]; + const lokiConfig = { schema_config: { configs } }; + expect(isConfigUpdateRequired(lokiConfig, UDSConfig.lokiDefaultStore)).toBe(true); + }); + + it(`should return true when config is empty`, () => { + // Testing that an update is required if the config is completely missing. + expect(isConfigUpdateRequired({}, UDSConfig.lokiDefaultStore)).toBe(true); + }); +}); diff --git a/src/pepr/loki/utils.ts b/src/pepr/loki/utils.ts new file mode 100644 index 000000000..4721ef91d --- /dev/null +++ b/src/pepr/loki/utils.ts @@ -0,0 +1,114 @@ +/** + * Copyright 2024 Defense Unicorns + * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + */ + +import yaml from "js-yaml"; +import { Component, setupLogger } from "../logger"; +const log = setupLogger(Component.LOKI); + +export interface IndexConfig { + prefix?: string; + period?: string; +} + +export interface ConfigEntry { + from: string; + store: string; + object_store?: string; + schema?: string; + index?: IndexConfig; +} + +export interface LokiConfig { + schema_config?: { + configs: ConfigEntry[]; + }; +} + +export interface Secret { + Raw: { + metadata?: { + name?: string; + annotations?: Record; + }; + }; +} + +/** + * Calculates a future date by adding a specified number of days to the current date. + * @param {number} days - The number of days to add to the current date. + * @return {string} - The ISO string representation of the future date. + */ +export function calculateFutureDate(days: number): string { + const now = new Date(); + const futureDate = new Date(now.setDate(now.getDate() + days)); + return futureDate.toISOString().split("T")[0]; +} + +/** + * Parses a YAML string into a LokiConfig object. + * @param {string} data - The YAML string to be parsed. + * @return {LokiConfig | null} - The parsed configuration object or null if parsing fails. + */ +export function parseLokiConfig(data: string): LokiConfig | null { + try { + return yaml.load(data) as LokiConfig; + } catch (error) { + log.error(`Failed to parse config: ${error.message}`); + return null; + } +} + +/** + * Updates the 'from' date in a configuration entry for the given store type. + * @param {ConfigEntry[]} configs - Array of configuration entries. + * @param {string} newDate - The new 'from' date to be set in the configuration. + * @return {boolean} - True if update is successful, false otherwise. + */ +export function updateConfigDate( + configs: ConfigEntry[], + storeType: string, + newDate: string, +): boolean { + const config = configs.find(c => c.store === storeType); + if (!config) { + log.warn("No schemaConfig entry found"); + return false; + } + config.from = newDate; + return true; +} + +/** + * Encodes a LokiConfig object into a YAML string. + * @param {LokiConfig} config - The configuration object to encode. + * @return {string} - The YAML string representation of the configuration. + */ +export function encodeConfig(config: LokiConfig): string { + return yaml.dump(config); +} + +/** + * Determines if the store type configuration needs to be updated based on the current configuration. + * This checks if storeType's 'from' date is set properly for the future and after all current schemas. + */ +export function isConfigUpdateRequired(lokiConfig: LokiConfig, storeType: string): boolean { + const configs = lokiConfig.schema_config?.configs || []; + const config = configs.find(c => c.store === storeType); + + // Check if storeType in config is missing + if (!config) { + return true; + } + + // Ensure storeType 'from' date is the latest among all configurations + for (const c of configs) { + if (c.store !== storeType && new Date(c.from) >= new Date(config.from)) { + return true; + } + } + + // loki schemaConfig is properly configured, no update necessary + return false; +}