diff --git a/digiwf-engine/digiwf-engine-service/pom.xml b/digiwf-engine/digiwf-engine-service/pom.xml index 350be47c4c..69578ef3be 100644 --- a/digiwf-engine/digiwf-engine-service/pom.xml +++ b/digiwf-engine/digiwf-engine-service/pom.xml @@ -83,6 +83,12 @@ ${project.version} + + de.muenchen.oss.digiwf + digiwf-camunda-prometheus-starter + ${project.version} + + diff --git a/digiwf-engine/digiwf-engine-service/src/main/resources/application-local.yml b/digiwf-engine/digiwf-engine-service/src/main/resources/application-local.yml index 47433347e6..3034d16319 100644 --- a/digiwf-engine/digiwf-engine-service/src/main/resources/application-local.yml +++ b/digiwf-engine/digiwf-engine-service/src/main/resources/application-local.yml @@ -12,11 +12,6 @@ # logging: # requests: all -spring: - sleuth: - web: - enabled: false - server: error: include-exception: true diff --git a/digiwf-engine/digiwf-engine-service/src/main/resources/application.yml b/digiwf-engine/digiwf-engine-service/src/main/resources/application.yml index 6e914ad040..d2e5a7584b 100644 --- a/digiwf-engine/digiwf-engine-service/src/main/resources/application.yml +++ b/digiwf-engine/digiwf-engine-service/src/main/resources/application.yml @@ -185,6 +185,16 @@ digiwf: s3service: topic: 'dwf-s3-${DIGIWF_ENV}' httpAPI: ${DIGIWF_S3_HTTPAPI:http://localhost:8086} + prometheus: + process-engine: + update-interval: 30000 + providers: + fniAndEde: true + incident: true + job: true + process: false # deactivate the process definition metrics + task: true + io: muenchendigital: digiwf: diff --git a/digiwf-integrations/digiwf-dms-integration/digiwf-dms-integration-fabasoft/digiwf-dms-integration-fabasoft-mock-service/src/test/resources/application-itest.yml b/digiwf-integrations/digiwf-dms-integration/digiwf-dms-integration-fabasoft/digiwf-dms-integration-fabasoft-mock-service/src/test/resources/application-itest.yml new file mode 100644 index 0000000000..c3e2e1d98d --- /dev/null +++ b/digiwf-integrations/digiwf-dms-integration/digiwf-dms-integration-fabasoft/digiwf-dms-integration-fabasoft-mock-service/src/test/resources/application-itest.yml @@ -0,0 +1,11 @@ +logging: + level: + org.apache.kafka.clients.admin: WARN + org.apache.kafka.clients.consumer: WARN + org.apache.kafka.clients.producer: WARN + +# overwrite vars +DIGIWF_ENV: itest +#KAFKA_BOOTSTRAP_SERVER: localhost +#KAFKA_BOOTSTRAP_SERVER_PORT: 19999 +#KAFKA_SECURITY_PROTOCOL: PLAINTEXT diff --git a/digiwf-integrations/digiwf-example-integration/digiwf-example-integration-service/src/test/resources/application-itest.yml b/digiwf-integrations/digiwf-example-integration/digiwf-example-integration-service/src/test/resources/application-itest.yml new file mode 100644 index 0000000000..c3e2e1d98d --- /dev/null +++ b/digiwf-integrations/digiwf-example-integration/digiwf-example-integration-service/src/test/resources/application-itest.yml @@ -0,0 +1,11 @@ +logging: + level: + org.apache.kafka.clients.admin: WARN + org.apache.kafka.clients.consumer: WARN + org.apache.kafka.clients.producer: WARN + +# overwrite vars +DIGIWF_ENV: itest +#KAFKA_BOOTSTRAP_SERVER: localhost +#KAFKA_BOOTSTRAP_SERVER_PORT: 19999 +#KAFKA_SECURITY_PROTOCOL: PLAINTEXT diff --git a/digiwf-integrations/digiwf-s3-integration/digiwf-s3-integration-client-starter/src/main/java/de/muenchen/oss/digiwf/s3/integration/client/configuration/S3IntegrationClientAutoConfiguration.java b/digiwf-integrations/digiwf-s3-integration/digiwf-s3-integration-client-starter/src/main/java/de/muenchen/oss/digiwf/s3/integration/client/configuration/S3IntegrationClientAutoConfiguration.java index d314b2cb79..cccf55ff90 100644 --- a/digiwf-integrations/digiwf-s3-integration/digiwf-s3-integration-client-starter/src/main/java/de/muenchen/oss/digiwf/s3/integration/client/configuration/S3IntegrationClientAutoConfiguration.java +++ b/digiwf-integrations/digiwf-s3-integration/digiwf-s3-integration-client-starter/src/main/java/de/muenchen/oss/digiwf/s3/integration/client/configuration/S3IntegrationClientAutoConfiguration.java @@ -5,7 +5,9 @@ import de.muenchen.oss.digiwf.s3.integration.client.api.FolderApiApi; import de.muenchen.oss.digiwf.s3.integration.client.properties.S3IntegrationClientProperties; import de.muenchen.oss.digiwf.s3.integration.client.service.ApiClientFactory; +import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -39,10 +41,16 @@ }) @RequiredArgsConstructor @EnableConfigurationProperties(S3IntegrationClientProperties.class) +@Slf4j public class S3IntegrationClientAutoConfiguration { public final S3IntegrationClientProperties s3IntegrationClientProperties; + @PostConstruct + public void init() { + log.info("[DIGIWF-S3-INTEGRATION-CLIENT]: Staring integration client, security is {}.", s3IntegrationClientProperties.isEnableSecurity() ? "enabled" : "disabled" ); + } + @Bean @ConditionalOnProperty(prefix = "io.muenchendigital.digiwf.s3.client", name = "securityEnabled", havingValue = "true") public ApiClientFactory securedApiClientFactory(final ClientRegistrationRepository clientRegistrationRepository, diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/pom.xml b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/pom.xml index 4f45753d83..263663c730 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/pom.xml +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/pom.xml @@ -18,6 +18,11 @@ camunda-engine provided + + org.springframework + spring-context + provided + io.micrometer @@ -31,7 +36,6 @@ ${project.version} test - diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/ExecutionEventReporter.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/ExecutionEventReporter.java new file mode 100644 index 0000000000..e43310fc33 --- /dev/null +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/ExecutionEventReporter.java @@ -0,0 +1,54 @@ +package de.muenchen.oss.digiwf.camunda.prometheus; + +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Counter; +import lombok.RequiredArgsConstructor; +import org.camunda.bpm.engine.RepositoryService; +import org.camunda.bpm.engine.delegate.DelegateExecution; +import org.springframework.context.event.EventListener; + +@RequiredArgsConstructor +public class ExecutionEventReporter implements MetricsReporter { + + private final RepositoryService repositoryService; + private Counter startExecutionEvents; + private Counter endExecutionEvents; + + @Override + public void registerMetrics(CollectorRegistry collectorRegistry) { + this.startExecutionEvents = Counter.build() + .name("camunda_start_process_events_count") + .help("Start process events by process definition.") + .labelNames("processDefinitionKey") + .register(collectorRegistry); + this.endExecutionEvents = Counter.build() + .name("camunda_end_process_events_count") + .help("End process events by process definition.") + .labelNames("processDefinitionKey") + .register(collectorRegistry); + } + + @EventListener(condition = "#execution.eventName.equals('start')") + public void countStartEvent(DelegateExecution execution) { + if (isModelElementOfType("startEvent", execution)) { + startExecutionEvents.labels(getProcessDefinitionKey(execution)).inc(); + } + } + + @EventListener(condition = "#execution.eventName.equals('end')") + public void countEndEvent(DelegateExecution execution) { + if (isModelElementOfType("endEvent", execution)) { + endExecutionEvents.labels(getProcessDefinitionKey(execution)).inc(); + } + } + + private static boolean isModelElementOfType(String typeName, DelegateExecution execution) { + return execution.getBpmnModelElementInstance() != null + && typeName.equals(execution.getBpmnModelElementInstance().getElementType().getTypeName()); + } + + private String getProcessDefinitionKey(DelegateExecution execution) { + return repositoryService.getProcessDefinition(execution.getProcessDefinitionId()).getKey(); + } + +} diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/FniAndEdeMetricsProvider.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/FniAndEdeMetricsProvider.java index ff391faba0..1945902f2d 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/FniAndEdeMetricsProvider.java +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/FniAndEdeMetricsProvider.java @@ -3,12 +3,19 @@ import io.prometheus.client.CollectorRegistry; import io.prometheus.client.Gauge; import lombok.RequiredArgsConstructor; +import org.camunda.bpm.engine.HistoryService; import org.camunda.bpm.engine.ManagementService; +import org.camunda.bpm.engine.RepositoryService; +import org.camunda.bpm.engine.repository.ProcessDefinition; + +import java.util.List; @RequiredArgsConstructor public class FniAndEdeMetricsProvider implements MetricsProvider { private final ManagementService managementService; + private final RepositoryService repositoryService; + private final HistoryService historyService; private Gauge fniCount; private Gauge edeCount; @@ -16,8 +23,16 @@ public class FniAndEdeMetricsProvider implements MetricsProvider { @Override public void updateMetrics() { // Get total number of FNIs - this.fniCount.labels("total").set(this.managementService.createMetricsQuery().name("activity-instance-start").sum()); - + this.fniCount + .labels("total", "total") + .set(this.managementService.createMetricsQuery().name("activity-instance-start").sum()); + + // Get FNIs by process definition (this only includes the currently deployed definitions and may be lower than the total count) + repositoryService.createProcessDefinitionQuery().list().forEach(processDefinition -> + this.fniCount + .labels(processDefinition.getKey(), processDefinition.getId()) + .set(historyService.createHistoricActivityInstanceQuery().processDefinitionId(processDefinition.getId()).count()) + ); // Get total number of EDEs this.edeCount.set(this.managementService.createMetricsQuery().name("executed-decision-elements").sum()); } @@ -27,7 +42,7 @@ public void registerMetrics(final CollectorRegistry collectorRegistry) { this.fniCount = Gauge.build() .name("camunda_activity_instances") .help("Number of activity instances (BPMN FNI) in total and by deployed process definition.") - .labelNames("processDefinitionId") + .labelNames("processDefinitionKey", "processDefinitionId") .register(collectorRegistry); this.edeCount = Gauge.build() diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/HistoryEventReporter.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/HistoryEventReporter.java new file mode 100644 index 0000000000..992f8a0f7b --- /dev/null +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/HistoryEventReporter.java @@ -0,0 +1,44 @@ +package de.muenchen.oss.digiwf.camunda.prometheus; + +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Summary; +import org.camunda.bpm.engine.impl.history.event.HistoricActivityInstanceEventEntity; +import org.springframework.context.event.EventListener; + +public class HistoryEventReporter implements MetricsReporter { + + private Summary camundaActivityTime; + + @Override + public void registerMetrics(CollectorRegistry collectorRegistry) { + camundaActivityTime = Summary.build() + .quantile(0.5, 0.05) + .quantile(0.9, 0.01) + .quantile(0.99, 0.001) + .name("camunda_activity_execution_time_milliseconds") + .labelNames("processDefinitionKey", "activityType", "activityName") + .help("Duration of activities in milliseconds.") + .register(collectorRegistry); + } + + /** + * Observes the duration of any ended camunda activity + * + * @param historyEvent the caught event + */ + @EventListener(condition = "#historyEvent != null && #historyEvent.eventType.equals('end')") + public void handleEvent(HistoricActivityInstanceEventEntity historyEvent) { + + String activityName = historyEvent.getActivityName(); + if (activityName == null || activityName.isEmpty()) { + activityName = historyEvent.getActivityId(); + } + + if (historyEvent.getDurationInMillis() != null) { + camundaActivityTime + .labels(historyEvent.getProcessDefinitionKey(), historyEvent.getActivityType(), activityName) + .observe(historyEvent.getDurationInMillis()); + } + } + +} diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/IncidentMetricsProvider.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/IncidentMetricsProvider.java index 94aff9008c..a3d063bc48 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/IncidentMetricsProvider.java +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/IncidentMetricsProvider.java @@ -3,26 +3,43 @@ import io.prometheus.client.CollectorRegistry; import io.prometheus.client.Gauge; import lombok.RequiredArgsConstructor; +import org.camunda.bpm.engine.RepositoryService; import org.camunda.bpm.engine.RuntimeService; +import org.camunda.bpm.engine.repository.ProcessDefinition; + +import static java.util.stream.Collectors.toMap; @RequiredArgsConstructor public class IncidentMetricsProvider implements MetricsProvider { private final RuntimeService runtimeService; + private final RepositoryService repositoryService; + private static final String NOT_PROCESS_INCIDENT = "__no_process_definition_key"; private Gauge openIncidents; @Override public void updateMetrics() { - this.openIncidents.set(this.runtimeService.createIncidentQuery().count()); + var processDefinitions = repositoryService.createProcessDefinitionQuery().list(); + var incidentCounts = processDefinitions.stream().collect(toMap(ProcessDefinition::getKey, ignored -> 0L, Long::sum)); + incidentCounts.put(NOT_PROCESS_INCIDENT, 0L); + var processDefinitionKeys = processDefinitions.stream().collect(toMap(ProcessDefinition::getId, ProcessDefinition::getKey)); + + runtimeService.createIncidentQuery().list().forEach(incident -> { + var processDefinitionKey = incident.getProcessDefinitionId() == null ? NOT_PROCESS_INCIDENT : processDefinitionKeys.get(incident.getProcessDefinitionId()); + incidentCounts.merge(processDefinitionKey, 1L, Long::sum); + }); + incidentCounts.forEach( + (processDefinitionKey, incidents) -> this.openIncidents.labels(processDefinitionKey).set(incidents)); } @Override public void registerMetrics(final CollectorRegistry collectorRegistry) { this.openIncidents = Gauge.build() - .name("camunda_incidents_open") - .help("Number of open incidents.") - .register(collectorRegistry); + .name("camunda_incidents_open") + .labelNames("processDefinitionKey") + .help("Number of open incidents by process definition key.") + .register(collectorRegistry); } } diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/JobMetricsProvider.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/JobMetricsProvider.java index 9810875aa8..16dd9d23fc 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/JobMetricsProvider.java +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/JobMetricsProvider.java @@ -4,6 +4,7 @@ import io.prometheus.client.Gauge; import lombok.RequiredArgsConstructor; import org.camunda.bpm.engine.ManagementService; +import org.camunda.bpm.engine.management.Metrics; import java.util.Date; @@ -16,6 +17,9 @@ public class JobMetricsProvider implements MetricsProvider { private Gauge jobsFuture; private Gauge jobsNoRetries; private Gauge jobsSuspended; + private Gauge jobsSuccess; + private Gauge jobsFailed; + @Override public void updateMetrics() { @@ -23,29 +27,38 @@ public void updateMetrics() { this.jobsFuture.set(this.managementService.createJobQuery().duedateHigherThan(new Date()).count()); this.jobsNoRetries.set(this.managementService.createJobQuery().noRetriesLeft().count()); this.jobsSuspended.set(this.managementService.createJobQuery().suspended().count()); + this.jobsSuccess.set(managementService.createMetricsQuery().name(Metrics.JOB_SUCCESSFUL).sum()); + this.jobsFailed.set(managementService.createMetricsQuery().name(Metrics.JOB_FAILED).sum()); + } @Override public void registerMetrics(final CollectorRegistry collectorRegistry) { this.jobsExecutable = Gauge.build() - .name("camunda_jobs_executable") - .help("Number of jobs which are executable, ie. retries > 0 and due date is null or due date is in the past") - .register(collectorRegistry); - + .name("camunda_jobs_executable") + .help("Number of jobs which are executable, ie. retries > 0 and due date is null or due date is in the past") + .register(collectorRegistry); this.jobsFuture = Gauge.build() - .name("camunda_jobs_future") - .help("Number of jobs where the due date is in the future") - .register(collectorRegistry); - + .name("camunda_jobs_future") + .help("Number of jobs where the due date is in the future") + .register(collectorRegistry); this.jobsNoRetries = Gauge.build() - .name("camunda_jobs_out_of_retries") - .help("Number of jobs with no retries left") - .register(collectorRegistry); - + .name("camunda_jobs_out_of_retries") + .help("Number of jobs with no retries left") + .register(collectorRegistry); this.jobsSuspended = Gauge.build() - .name("camunda_jobs_suspended") - .help("Number of suspended jobs") - .register(collectorRegistry); + .name("camunda_jobs_suspended") + .help("Number of suspended jobs") + .register(collectorRegistry); + this.jobsSuccess = Gauge.build() + .name("camunda_jobs_successfully_executed") + .help("The number of jobs successfully executed.") + .register(collectorRegistry); + this.jobsFailed = Gauge.build() + .name("camunda_jobs_failed") + .help("The number of jobs that failed to executed and were submitted for retry.") + .register(collectorRegistry); + } } diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsProvider.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsProvider.java index 58fe487f12..57cf4a81ee 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsProvider.java +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsProvider.java @@ -2,10 +2,20 @@ import io.prometheus.client.CollectorRegistry; +/** + * Metrics provider delivers metrics on external trigger. + */ public interface MetricsProvider { + /** + * Triggers metrics update. + */ void updateMetrics(); + /** + * Registers collector registry. + * @param collectorRegistry registry. + */ void registerMetrics(CollectorRegistry collectorRegistry); } diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsReporter.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsReporter.java new file mode 100644 index 0000000000..b9bc4540a0 --- /dev/null +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsReporter.java @@ -0,0 +1,14 @@ +package de.muenchen.oss.digiwf.camunda.prometheus; + +import io.prometheus.client.CollectorRegistry; + +/** + * Metric reporter is an active component, pushing metrics without explicit trigger. + */ +public interface MetricsReporter { + /** + * Registers collector registry. + * @param collectorRegistry registry. + */ + void registerMetrics(CollectorRegistry collectorRegistry); +} diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/ProcessMetricsProvider.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/ProcessMetricsProvider.java index 7678890d09..20b67e4525 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/ProcessMetricsProvider.java +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/ProcessMetricsProvider.java @@ -5,9 +5,14 @@ import lombok.RequiredArgsConstructor; import org.camunda.bpm.engine.RepositoryService; import org.camunda.bpm.engine.RuntimeService; +import org.camunda.bpm.engine.repository.ProcessDefinition; import java.util.ArrayList; import java.util.List; +import java.util.Map; + +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.groupingBy; @RequiredArgsConstructor public class ProcessMetricsProvider implements MetricsProvider { @@ -21,21 +26,27 @@ public class ProcessMetricsProvider implements MetricsProvider { @Override public void updateMetrics() { - final List processDefinitionKeys = new ArrayList<>(); - this.repositoryService.createProcessDefinitionQuery().list().forEach(processDefinition -> processDefinitionKeys.add(processDefinition.getKey())); - this.processDefinitionCount.set(processDefinitionKeys.size()); - this.processDefinitionCountUnique.set(processDefinitionKeys.stream().distinct().count()); - - if (processDefinitionKeys.size() > 0) { - processDefinitionKeys.forEach(processDefinitionKey -> this.processInstanceCount.labels(processDefinitionKey) - .set(this.runtimeService.createProcessInstanceQuery() - .processDefinitionKey(processDefinitionKey) - .count() - ) - ); - } else { - this.processInstanceCount.labels("NA").set(0); - } + + Map> maxVersionByProcessDefinitionKey = repositoryService.createProcessDefinitionQuery().list().stream() + .sorted(comparing(ProcessDefinition::getVersion).reversed()) + .collect(groupingBy(ProcessDefinition::getKey)); + + processDefinitionCount.set(maxVersionByProcessDefinitionKey.values().stream().mapToLong(List::size).sum()); + processDefinitionCountUnique.set(maxVersionByProcessDefinitionKey.size()); + + maxVersionByProcessDefinitionKey.forEach((processDefinitionKey, processDefinitions) -> { + long instanceCountByKey = runtimeService.createProcessInstanceQuery() + .processDefinitionKey(processDefinitionKey) + .count(); + long instanceCountLatestVersion = runtimeService.createProcessInstanceQuery() + .processDefinitionId(processDefinitions.get(0).getId()) + .count(); + + processInstanceCount.labels(processDefinitionKey, "true") + .set(instanceCountLatestVersion); + processInstanceCount.labels(processDefinitionKey, "false") + .set(instanceCountByKey - instanceCountLatestVersion); + }); } @Override @@ -53,7 +64,7 @@ public void registerMetrics(final CollectorRegistry collectorRegistry) { this.processInstanceCount = Gauge.build() .name("camunda_running_process_instances") .help("Running process instances by process definition key.") - .labelNames("processDefinitionKey") + .labelNames("processDefinitionKey", "latest") .register(collectorRegistry); } } diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/TaskEventReporter.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/TaskEventReporter.java new file mode 100644 index 0000000000..c600f23eb9 --- /dev/null +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/TaskEventReporter.java @@ -0,0 +1,96 @@ +package de.muenchen.oss.digiwf.camunda.prometheus; + +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Counter; +import lombok.RequiredArgsConstructor; +import org.camunda.bpm.engine.RepositoryService; +import org.camunda.bpm.engine.delegate.DelegateTask; +import org.camunda.bpm.engine.delegate.TaskListener; +import org.springframework.context.event.EventListener; + +/** + * Reports task events. + * @see TaskListener + */ +@RequiredArgsConstructor +public class TaskEventReporter implements MetricsReporter { + + private static final String PROCESS_DEFINITION_KEY = "processDefinitionKey"; + private static final String TASK_DEFINITION_KEY = "taskDefinitionKey"; + + private final RepositoryService repositoryService; + private Counter createTaskEvents; + private Counter completeTaskEvents; + private Counter assignTaskEvents; + private Counter deleteTaskEvents; + private Counter updateTaskEvents; + private Counter timeoutTaskEvents; + + @Override + public void registerMetrics(CollectorRegistry collectorRegistry) { + createTaskEvents = Counter.build() + .name("camunda_create_task_events_count") + .help("Create task events by deployed process definition.") + .labelNames(PROCESS_DEFINITION_KEY, TASK_DEFINITION_KEY) + .register(collectorRegistry); + completeTaskEvents = Counter.build() + .name("camunda_complete_task_events_count") + .help("Complete task events by deployed process definition.") + .labelNames(PROCESS_DEFINITION_KEY, TASK_DEFINITION_KEY) + .register(collectorRegistry); + assignTaskEvents = Counter.build() + .name("camunda_assign_task_events_count") + .help("Assign and un-assign task events by deployed process definition.") + .labelNames(PROCESS_DEFINITION_KEY, TASK_DEFINITION_KEY) + .register(collectorRegistry); + deleteTaskEvents = Counter.build() + .name("camunda_delete_task_events_count") + .help("Delete task events by deployed process definition.") + .labelNames(PROCESS_DEFINITION_KEY, TASK_DEFINITION_KEY) + .register(collectorRegistry); + updateTaskEvents = Counter.build() + .name("camunda_update_task_events_count") + .help("Update task events by deployed process definition.") + .labelNames(PROCESS_DEFINITION_KEY, TASK_DEFINITION_KEY) + .register(collectorRegistry); + timeoutTaskEvents = Counter.build() + .name("camunda_timeout_task_events_count") + .help("Timout task events by deployed process definition.") + .labelNames(PROCESS_DEFINITION_KEY, TASK_DEFINITION_KEY) + .register(collectorRegistry); + } + + @EventListener(condition = "#task.eventName.equals('create')") + public void countCreateEvent(DelegateTask task) { + createTaskEvents.labels(getProcessDefinitionKey(task), task.getTaskDefinitionKey()).inc(); + } + + @EventListener(condition = "#task.eventName.equals('complete')") + public void countCompleteEvent(DelegateTask task) { + completeTaskEvents.labels(getProcessDefinitionKey(task), task.getTaskDefinitionKey()).inc(); + } + + @EventListener(condition = "#task.eventName.equals('assignment')") + public void countAssignmentEvent(DelegateTask task) { + assignTaskEvents.labels(getProcessDefinitionKey(task), task.getTaskDefinitionKey()).inc(); + } + + @EventListener(condition = "#task.eventName.equals('delete')") + public void countDeleteEvent(DelegateTask task) { + deleteTaskEvents.labels(getProcessDefinitionKey(task), task.getTaskDefinitionKey()).inc(); + } + + @EventListener(condition = "#task.eventName.equals('timeout')") + public void countTimeoutEvent(DelegateTask task) { + timeoutTaskEvents.labels(getProcessDefinitionKey(task), task.getTaskDefinitionKey()).inc(); + } + @EventListener(condition = "#task.eventName.equals('update')") + public void countUpdateEvent(DelegateTask task) { + updateTaskEvents.labels(getProcessDefinitionKey(task), task.getTaskDefinitionKey()).inc(); + } + + private String getProcessDefinitionKey(DelegateTask task) { + return repositoryService.getProcessDefinition(task.getProcessDefinitionId()).getKey(); + } + +} diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/TaskMetricsProvider.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/TaskMetricsProvider.java index dfaa65340d..6c52bfda3d 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/TaskMetricsProvider.java +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-core/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/TaskMetricsProvider.java @@ -3,7 +3,9 @@ import io.prometheus.client.CollectorRegistry; import io.prometheus.client.Gauge; import lombok.RequiredArgsConstructor; +import org.camunda.bpm.engine.RepositoryService; import org.camunda.bpm.engine.TaskService; +import org.camunda.bpm.engine.repository.ProcessDefinition; import java.util.Optional; @@ -11,6 +13,7 @@ public class TaskMetricsProvider implements MetricsProvider { private final TaskService taskService; + private final RepositoryService repositoryService; private Gauge openTasksByGroup; private Gauge openTasks; @@ -27,12 +30,17 @@ public void updateMetrics() { return false; })); - this.openTasks.labels("assigned").set(this.taskService.createTaskQuery().taskAssigned().count()); - this.openTasks.labels("unassigned").set(this.taskService.createTaskQuery().taskUnassigned().count()); - this.openTasks.labels("hasCandidateGroups").set(this.taskService.createTaskQuery().withCandidateGroups().count()); - this.openTasks.labels("hasCandidateUsers").set(this.taskService.createTaskQuery().withCandidateUsers().count()); - this.openTasks.labels("unassignedWithNoCandidates").set(this.taskService.createTaskQuery().taskUnassigned().withoutCandidateGroups().withoutCandidateUsers().count()); - this.openTasks.labels("total").set(this.taskService.createTaskQuery().count()); + this.repositoryService.createProcessDefinitionQuery().list().stream() + .map(ProcessDefinition::getKey) + .distinct() + .forEach( processDefinitionKey -> { + this.openTasks.labels(processDefinitionKey, "assigned").set(this.taskService.createTaskQuery().taskAssigned().count()); + this.openTasks.labels(processDefinitionKey, "unassigned").set(this.taskService.createTaskQuery().taskUnassigned().count()); + this.openTasks.labels(processDefinitionKey, "hasCandidateGroups").set(this.taskService.createTaskQuery().withCandidateGroups().count()); + this.openTasks.labels(processDefinitionKey, "hasCandidateUsers").set(this.taskService.createTaskQuery().withCandidateUsers().count()); + this.openTasks.labels(processDefinitionKey, "unassignedWithNoCandidates").set(this.taskService.createTaskQuery().taskUnassigned().withoutCandidateGroups().withoutCandidateUsers().count()); + this.openTasks.labels(processDefinitionKey, "total").set(this.taskService.createTaskQuery().count()); + } ); } @Override @@ -46,7 +54,7 @@ public void registerMetrics(final CollectorRegistry collectorRegistry) { this.openTasks = Gauge.build() .name("camunda_open_tasks") .help("Number of open tasks.") - .labelNames("status") + .labelNames("processDefinitionKey", "status") .register(collectorRegistry); } diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/pom.xml b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/pom.xml index 8167118991..323f786937 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/pom.xml +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/pom.xml @@ -21,16 +21,20 @@ org.springframework.boot spring-boot-starter-actuator - de.muenchen.oss.digiwf digiwf-camunda-prometheus-starter ${project.version} - org.camunda.bpm.springboot camunda-bpm-spring-boot-starter-webapp + + + junit + junit + + com.h2database @@ -42,7 +46,6 @@ 4.0.4 - org.springframework.boot @@ -53,12 +56,6 @@ org.springframework.boot spring-boot-starter-test test - - - junit - junit - - org.springframework.security diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/src/main/resources/application.properties b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/src/main/resources/application.properties deleted file mode 100644 index 3e714d3344..0000000000 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/src/main/resources/application.properties +++ /dev/null @@ -1,8 +0,0 @@ -io.muenchendigital.camunda.prometheus.update-interval=3000 -management.server.port=8080 -management.endpoints.enabled-by-default=false -management.endpoints.web.exposure.include=health, info, prometheus -management.endpoints.web.path-mapping.prometheus=metrics -management.endpoint.health.enabled=true -management.endpoint.info.enabled=true -management.endpoint.prometheus.enabled=true diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/src/main/resources/application.yaml b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/src/main/resources/application.yaml new file mode 100644 index 0000000000..50fbaeb126 --- /dev/null +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-example/src/main/resources/application.yaml @@ -0,0 +1,47 @@ +digiwf: + prometheus: + process-engine: + update-interval: 30000 + providers: + fniAndEde: true + incident: true + job: true + process: true + task: true + +management: + server: + port: 8080 + endpoint: + health: + enabled: true + info: + enabled: true + prometheus: + enabled: true + + endpoints: + enabled-by-default: false + web: + exposure: + include: 'health,info,prometheus' + path-mapping: + prometheus: metrics + + metrics: + enable: + jvm: false + tomcat: false + cache: false + logback: false + process: false + + prometheus: + metrics: + export: + enabled: true + +logging: + level: + org.springframework.web: DEBUG # DEBUG + org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping: TRACE # set to TRACE for request logging diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/pom.xml b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/pom.xml index 24e74c12ab..1c6a9c9e70 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/pom.xml +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/pom.xml @@ -34,6 +34,12 @@ org.springframework.boot spring-boot-starter + + + org.springframework.boot + spring-boot-configuration-processor + true + org.camunda.bpm diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/CamundaPrometheusAutoConfiguration.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/CamundaPrometheusAutoConfiguration.java index 05e3db10b6..5c890c52e2 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/CamundaPrometheusAutoConfiguration.java +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/CamundaPrometheusAutoConfiguration.java @@ -1,30 +1,11 @@ package de.muenchen.oss.digiwf.camunda.prometheus; -import io.prometheus.client.CollectorRegistry; -import jakarta.annotation.PostConstruct; -import lombok.RequiredArgsConstructor; -import org.springframework.context.annotation.Import; -import org.springframework.scheduling.annotation.EnableScheduling; -import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; -import java.util.List; -@EnableScheduling -@Import(MetricsConfiguration.class) -@RequiredArgsConstructor +@Configuration +@EnableConfigurationProperties(CamundaPrometheusProperties.class) public class CamundaPrometheusAutoConfiguration { - private final List metricsProviders; - private final CollectorRegistry collectorRegistry; - - @Scheduled(fixedDelayString = "${io.muenchendigital.camunda.prometheus.update-interval}") - public void updateMetrics() { - metricsProviders.forEach(MetricsProvider::updateMetrics); - } - - @PostConstruct - public void initalizeMetrics() { - metricsProviders.forEach(provider -> provider.registerMetrics(collectorRegistry)); - } - } diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/CamundaPrometheusProperties.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/CamundaPrometheusProperties.java new file mode 100644 index 0000000000..870b6b3753 --- /dev/null +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/CamundaPrometheusProperties.java @@ -0,0 +1,38 @@ +package de.muenchen.oss.digiwf.camunda.prometheus; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "digiwf.prometheus.process-engine") +public class CamundaPrometheusProperties { + /** + * Update interval for running metrics providers, which actively execute queries. + */ + private int updateInterval = 30000; + /** + * Flags controlling metrics. + */ + private Providers providers = new Providers(); + + public static class Providers { + /** + * Flag to activate FNI and EDE metric. + */ + private boolean fniAndEde; + /** + * Flag to activate incident metric. + */ + private boolean incident; + /** + * Flag to activate job metric. + */ + private boolean job; + /** + * Flag to activate process metric. + */ + private boolean process; + /** + * Flag to activate task metric. + */ + private boolean task; + } +} diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsAutoConfiguration.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsAutoConfiguration.java new file mode 100644 index 0000000000..4a937e32c5 --- /dev/null +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsAutoConfiguration.java @@ -0,0 +1,55 @@ +package de.muenchen.oss.digiwf.camunda.prometheus; + +import lombok.RequiredArgsConstructor; +import org.camunda.bpm.engine.*; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +@RequiredArgsConstructor +public class MetricsAutoConfiguration { + + @Bean + public MetricsReporter executionEventReporter(RepositoryService repositoryService) { + return new ExecutionEventReporter(repositoryService); + } + @Bean + public MetricsReporter historyEventReporter() { + return new HistoryEventReporter(); + } + + @Bean + public MetricsReporter taskEventReporter(RepositoryService repositoryService) { + return new TaskEventReporter(repositoryService); + } + + @Bean + @ConditionalOnProperty(prefix = "digiwf.prometheus.process-engine.providers", name = "fniAndEde") + public MetricsProvider fniAndEdeMetricsProvider(ManagementService managementService, RepositoryService repositoryService, HistoryService historyService) { + return new FniAndEdeMetricsProvider(managementService, repositoryService, historyService); + } + + @Bean + @ConditionalOnProperty(prefix = "digiwf.prometheus.process-engine.providers", name = "incident") + public MetricsProvider incidentMetricsProvider(RuntimeService runtimeService, RepositoryService repositoryService) { + return new IncidentMetricsProvider(runtimeService, repositoryService); + } + @Bean + @ConditionalOnProperty(prefix = "digiwf.prometheus.process-engine.providers", name = "job") + public MetricsProvider jobMetricsProvider(ManagementService managementService) { + return new JobMetricsProvider(managementService); + } + @Bean + @ConditionalOnProperty(prefix = "digiwf.prometheus.process-engine.providers", name = "process") + public MetricsProvider processMetricsProvider(RuntimeService runtimeService, RepositoryService repositoryService) { + return new ProcessMetricsProvider(runtimeService, repositoryService); + } + @Bean + @ConditionalOnProperty(prefix = "digiwf.prometheus.process-engine.providers", name = "task") + public MetricsProvider taskMetricsProvider(TaskService taskService, RepositoryService repositoryService) { + return new TaskMetricsProvider(taskService, repositoryService); + } +} diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsConfiguration.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsConfiguration.java deleted file mode 100644 index a3ac48140d..0000000000 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsConfiguration.java +++ /dev/null @@ -1,37 +0,0 @@ -package de.muenchen.oss.digiwf.camunda.prometheus; - -import org.camunda.bpm.engine.ManagementService; -import org.camunda.bpm.engine.RepositoryService; -import org.camunda.bpm.engine.RuntimeService; -import org.camunda.bpm.engine.TaskService; -import org.springframework.context.annotation.Bean; - - -public class MetricsConfiguration { - - @Bean - public MetricsProvider taskMetricsProvider(final TaskService taskService) { - return new TaskMetricsProvider(taskService); - } - - @Bean - public MetricsProvider processMetricsProvider(final RuntimeService runtimeService, final RepositoryService repositoryService) { - return new ProcessMetricsProvider(runtimeService, repositoryService); - } - - @Bean - public MetricsProvider jobMetricsProvider(final ManagementService managementService) { - return new JobMetricsProvider(managementService); - } - - @Bean - public MetricsProvider incidentMetricsProvider(final RuntimeService runtimeService) { - return new IncidentMetricsProvider(runtimeService); - } - - @Bean - public MetricsProvider fniAndEdeMetricsProvider(final ManagementService managementService) { - return new FniAndEdeMetricsProvider(managementService); - } - -} diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsProviderSchedulerAutoConfiguration.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsProviderSchedulerAutoConfiguration.java new file mode 100644 index 0000000000..1c162a61fa --- /dev/null +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsProviderSchedulerAutoConfiguration.java @@ -0,0 +1,36 @@ +package de.muenchen.oss.digiwf.camunda.prometheus; + +import io.prometheus.client.CollectorRegistry; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; + +import java.util.Map; + +@EnableScheduling +@Configuration +@AutoConfigureAfter(MetricsAutoConfiguration.class) +@RequiredArgsConstructor +@Slf4j +public class MetricsProviderSchedulerAutoConfiguration { + + private final Map metricsProviders; + private final CollectorRegistry collectorRegistry; + + @PostConstruct + public void initializeProviderMetrics() { + // providers + log.info("[DIGIWF METRICS]: Registered {} metrics providers: {}", metricsProviders.keySet().size(), String.join(", ", metricsProviders.keySet())); + metricsProviders.forEach((key, value) -> value.registerMetrics(collectorRegistry)); + } + + @Scheduled(fixedDelayString = "${digiwf.prometheus.process-engine.update-interval}") + public void updateProviderMetrics() { + metricsProviders.forEach((key, value) -> value.updateMetrics()); + } + +} diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsReporterAutoConfiguration.java b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsReporterAutoConfiguration.java new file mode 100644 index 0000000000..88340a290c --- /dev/null +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/java/de/muenchen/oss/digiwf/camunda/prometheus/MetricsReporterAutoConfiguration.java @@ -0,0 +1,19 @@ +package de.muenchen.oss.digiwf.camunda.prometheus; + +import io.prometheus.client.CollectorRegistry; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + +@Configuration +@AutoConfigureAfter(MetricsAutoConfiguration.class) +public class MetricsReporterAutoConfiguration { + + @Autowired + public void configureReporters(List metricsReporters, CollectorRegistry collectorRegistry) { + // reporters + metricsReporters.forEach(provider -> provider.registerMetrics(collectorRegistry)); + } +} diff --git a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 5550b79cd3..0b054f1f15 100644 --- a/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/digiwf-libs/digiwf-camunda-prometheus/digiwf-camunda-prometheus-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1 +1,4 @@ de.muenchen.oss.digiwf.camunda.prometheus.CamundaPrometheusAutoConfiguration +de.muenchen.oss.digiwf.camunda.prometheus.MetricsAutoConfiguration +de.muenchen.oss.digiwf.camunda.prometheus.MetricsProviderSchedulerAutoConfiguration +de.muenchen.oss.digiwf.camunda.prometheus.MetricsReporterAutoConfiguration diff --git a/digiwf-libs/digiwf-spring-security/digiwf-spring-security-starter/src/main/resources/digiwf-security-application.yaml b/digiwf-libs/digiwf-spring-security/digiwf-spring-security-starter/src/main/resources/digiwf-security-application.yaml index e258677fe2..a070d2aee1 100644 --- a/digiwf-libs/digiwf-spring-security/digiwf-spring-security-starter/src/main/resources/digiwf-security-application.yaml +++ b/digiwf-libs/digiwf-spring-security/digiwf-spring-security-starter/src/main/resources/digiwf-security-application.yaml @@ -6,6 +6,7 @@ digiwf: - /actuator/info # allow access to /actuator/info - /actuator/health # allow access to /actuator/health for OpenShift Health Check - /actuator/metrics # allow access to /actuator/metrics for Prometheus monitoring in OpenShift + - /actuator/prometheus # allow access to /actuator/prometheus for Prometheus monitoring in OpenShift - /swagger-ui/index.html # allow access to swagger - /swagger-ui*/*swagger-initializer.js # allow access to swagger - /swagger-ui*/** # allow access to swagger diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/TaskListApplication.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/TaskListApplication.java index 0b423f6ca5..138c7a465c 100644 --- a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/TaskListApplication.java +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/TaskListApplication.java @@ -2,8 +2,10 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication +@EnableScheduling public class TaskListApplication { public static void main(String... args) { SpringApplication.run(TaskListApplication.class, args); diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/application/usecase/WorkOnTaskFileUseCase.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/application/usecase/WorkOnTaskFileUseCase.java index 1e9b4e72f2..f0d90d0c43 100644 --- a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/application/usecase/WorkOnTaskFileUseCase.java +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/application/usecase/WorkOnTaskFileUseCase.java @@ -12,6 +12,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.springframework.context.annotation.Lazy; import org.springframework.http.HttpStatus; import org.springframework.lang.NonNull; import org.springframework.stereotype.Component; @@ -26,7 +27,7 @@ @RequiredArgsConstructor public class WorkOnTaskFileUseCase implements WorkOnTaskFile { - protected final DocumentStorageFolderRepository documentStorageFolderRepository; + private final DocumentStorageFolderRepository documentStorageFolderRepository; private final PresignedUrlPort presignedUrlPort; diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/axon/AxonGeneralConfiguration.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/axon/AxonGeneralConfiguration.java new file mode 100644 index 0000000000..b0db2097d1 --- /dev/null +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/axon/AxonGeneralConfiguration.java @@ -0,0 +1,53 @@ +package de.muenchen.oss.digiwf.task.service.infra.axon; + +import com.fasterxml.jackson.databind.ObjectMapper; +import de.muenchen.oss.digiwf.task.PolyflowObjectMapper; +import org.axonframework.eventhandling.deadletter.jpa.DeadLetterEntry; +import org.axonframework.eventhandling.tokenstore.jpa.TokenEntry; +import org.axonframework.eventsourcing.eventstore.EventStorageEngine; +import org.axonframework.eventsourcing.eventstore.inmemory.InMemoryEventStorageEngine; +import org.axonframework.modelling.saga.repository.SagaStore; +import org.axonframework.modelling.saga.repository.inmemory.InMemorySagaStore; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EntityScan(basePackageClasses = { + TokenEntry.class, + DeadLetterEntry.class +}) +public class AxonGeneralConfiguration { + /** + * Provides an object mapper for Axon message serialization. + * + * @return object mapper. + */ + @Bean("defaultAxonObjectMapper") + @Qualifier("defaultAxonObjectMapper") + public ObjectMapper defaultAxonObjectMapper() { + return PolyflowObjectMapper.DEFAULT; + } + + /** + * We will receive events via Kafka, so no event storage is available in this component. + * + * @return in-memory storage engine, to make Axon Framework happy. + */ + @Bean + public EventStorageEngine inMemoryStorageEngine() { + return new InMemoryEventStorageEngine(); + } + + /** + * No sagas should be handled in this component. + * + * @return in-memory saga-store to make Axon Framework happy. + */ + @Bean + public SagaStore inMemorySagaStore() { + return new InMemorySagaStore(); + } + +} diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/axon/AxonMetricsConfiguration.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/axon/AxonMetricsConfiguration.java new file mode 100644 index 0000000000..cc7a3fcea9 --- /dev/null +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/axon/AxonMetricsConfiguration.java @@ -0,0 +1,42 @@ +package de.muenchen.oss.digiwf.task.service.infra.axon; + +import de.muenchen.oss.digiwf.task.service.infra.metrics.ApplicationNameCompositeMessageMonitorWrapper; +import de.muenchen.oss.digiwf.task.service.infra.metrics.EventMessageCountingMonitor; +import io.micrometer.core.instrument.MeterRegistry; +import org.axonframework.config.ConfigurerModule; +import org.axonframework.config.MessageMonitorFactory; +import org.axonframework.eventhandling.TrackingEventProcessor; +import org.axonframework.messaging.Message; +import org.axonframework.monitoring.MessageMonitor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class AxonMetricsConfiguration { + + /** + * Configure message monitor. + * @param meterRegistry meter registry. + * @return module with monitoring. + */ + @Bean + public ConfigurerModule metricsConfigurer(MeterRegistry meterRegistry) { + return configurer -> { + var messageMonitorFactory = new MessageMonitorFactory() { + @Override + public MessageMonitor> create(org.axonframework.config.Configuration configuration, + Class componentType, + String componentName + ) { + return new ApplicationNameCompositeMessageMonitorWrapper<>( + (applicationName) -> new EventMessageCountingMonitor( + meterRegistry, "polyflow_axon_kafka_events_received" + ) + ); + } + }; + configurer.configureMessageMonitor(TrackingEventProcessor.class, messageMonitorFactory); + }; + } + +} diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/ingress/AxonKafkaIngressConfiguration.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/ingress/AxonKafkaIngressConfiguration.java index 80efbf02c4..9c9f47ba44 100644 --- a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/ingress/AxonKafkaIngressConfiguration.java +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/ingress/AxonKafkaIngressConfiguration.java @@ -1,5 +1,6 @@ package de.muenchen.oss.digiwf.task.service.infra.ingress; +import de.muenchen.oss.digiwf.task.service.infra.metrics.MetricsBindingConsumerFactory; import io.micrometer.core.instrument.MeterRegistry; import org.axonframework.extensions.kafka.KafkaProperties; import org.axonframework.extensions.kafka.eventhandling.KafkaMessageConverter; @@ -79,7 +80,6 @@ public StreamableKafkaMessageSource kafkaMessageSourcePolyflowDa .builder() .topics(Collections.singletonList(extendedProperties.getTopicDataEntries())) .consumerFactory(new MetricsBindingConsumerFactory<>(meterRegistry, kafkaConsumerFactory)) - .consumerFactory(kafkaConsumerFactory) .serializer(serializer) .fetcher(kafkaFetcher) .messageConverter(messageConverter) @@ -114,7 +114,6 @@ public StreamableKafkaMessageSource kafkaMessageSourcePolyflowTa .builder() .topics(Collections.singletonList(extendedProperties.getTopicTasks())) .consumerFactory(new MetricsBindingConsumerFactory<>(meterRegistry, kafkaConsumerFactory)) - .consumerFactory(kafkaConsumerFactory) .serializer(serializer) .fetcher(kafkaFetcher) .messageConverter(messageConverter) diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/ApplicationNameCompositeMessageMonitorWrapper.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/ApplicationNameCompositeMessageMonitorWrapper.java new file mode 100644 index 0000000000..0632978963 --- /dev/null +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/ApplicationNameCompositeMessageMonitorWrapper.java @@ -0,0 +1,45 @@ +package de.muenchen.oss.digiwf.task.service.infra.metrics; + +import io.holunda.camunda.taskpool.api.business.DataEntryCreatedEvent; +import io.holunda.camunda.taskpool.api.business.DataEntryUpdatedEvent; +import io.holunda.camunda.taskpool.api.task.TaskIdentity; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.axonframework.messaging.Message; +import org.axonframework.monitoring.MessageMonitor; +import org.jetbrains.annotations.NotNull; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +@RequiredArgsConstructor +@Slf4j +public class ApplicationNameCompositeMessageMonitorWrapper>> implements MessageMonitor> { + + private final Function monitorSupplier; + + private final Map applicationNameMonitors = new ConcurrentHashMap<>(); + + @Override + public MonitorCallback onMessageIngested(@NotNull Message message) { + var applicationName = determineApplicationName(message); + var messageMonitorForApplicationName = applicationNameMonitors.computeIfAbsent(applicationName, monitorSupplier); + return messageMonitorForApplicationName.onMessageIngested(message); + } + + private static String determineApplicationName(Message message) { + if (TaskIdentity.class.isAssignableFrom(message.getPayloadType())) { + return ((TaskIdentity) message.getPayload()).getSourceReference().getApplicationName(); + } + if (DataEntryCreatedEvent.class.isAssignableFrom(message.getPayloadType())) { + return ((DataEntryCreatedEvent) message.getPayload()).getApplicationName(); + } + if (DataEntryUpdatedEvent.class.isAssignableFrom(message.getPayloadType())) { + return ((DataEntryUpdatedEvent) message.getPayload()).getApplicationName(); + } + log.debug("Cannot determine application name for payload type {}", message.getPayloadType()); + return "unknown"; + } + +} diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/EventMessageCountingMonitor.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/EventMessageCountingMonitor.java new file mode 100644 index 0000000000..f7a302f266 --- /dev/null +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/EventMessageCountingMonitor.java @@ -0,0 +1,49 @@ +package de.muenchen.oss.digiwf.task.service.infra.metrics; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import org.axonframework.messaging.Message; +import org.axonframework.monitoring.MessageMonitor; +import org.jetbrains.annotations.NotNull; + +public class EventMessageCountingMonitor implements MessageMonitor> { + + private final Counter ingestedCounter; + private final Counter successCounter; + private final Counter failureCounter; + private final Counter processedCounter; + private final Counter ignoredCounter; + + public EventMessageCountingMonitor(MeterRegistry meterRegistry, String meterName) { + ingestedCounter = meterRegistry.counter(meterName, "status", "ingested"); + successCounter = meterRegistry.counter(meterName, "status", "success"); + failureCounter = meterRegistry.counter(meterName, "status", "failure"); + processedCounter = meterRegistry.counter(meterName, "status", "processed"); + ignoredCounter = meterRegistry.counter(meterName, "status", "ignored"); + } + + @Override + public MonitorCallback onMessageIngested(@NotNull Message message) { + ingestedCounter.increment(); + return new MessageMonitor.MonitorCallback() { + @Override + public void reportSuccess() { + processedCounter.increment(); + successCounter.increment(); + } + + @Override + public void reportFailure(Throwable cause) { + processedCounter.increment(); + failureCounter.increment(); + } + + @Override + public void reportIgnored() { + ignoredCounter.increment(); + } + }; + } +} diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/GaugeTaskCount.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/GaugeTaskCount.java new file mode 100644 index 0000000000..d35919090c --- /dev/null +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/GaugeTaskCount.java @@ -0,0 +1,38 @@ +package de.muenchen.oss.digiwf.task.service.infra.metrics; + +import io.holunda.polyflow.view.TaskQueryClient; +import io.holunda.polyflow.view.query.task.TaskCountByApplicationQuery; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Gauge; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class GaugeTaskCount { + + private final TaskQueryClient taskQueryClient; + private final CollectorRegistry collectorRegistry; + private Gauge taskCountGauge; + + @PostConstruct + public void init() { + this.taskCountGauge = Gauge.build() + .name("polyflow_tasklist_task_count") + .help("Current task count per application") + .labelNames("applicationName") + .register(collectorRegistry); + } + + @Scheduled(fixedDelayString = "${polyflow.metrics.delay.taskcount}") + @SneakyThrows + public void countTasks() { + var counts = taskQueryClient.query(new TaskCountByApplicationQuery()).get(); + counts.forEach(count -> + taskCountGauge.labels(count.getApplication()).set(count.getTaskCount()) + ); + } +} diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/ingress/MetricsBindingConsumerFactory.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/MetricsBindingConsumerFactory.java similarity index 83% rename from digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/ingress/MetricsBindingConsumerFactory.java rename to digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/MetricsBindingConsumerFactory.java index 1eca37bfa8..0a2e7f4e06 100644 --- a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/ingress/MetricsBindingConsumerFactory.java +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/metrics/MetricsBindingConsumerFactory.java @@ -1,4 +1,4 @@ -package de.muenchen.oss.digiwf.task.service.infra.ingress; +package de.muenchen.oss.digiwf.task.service.infra.metrics; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.kafka.KafkaClientMetrics; @@ -9,6 +9,11 @@ import java.util.UUID; +/** + * Metrics-aware consumer factory binding Kafka metrics ti meter registry. + * @param message partition key. + * @param message encoding. + */ @RequiredArgsConstructor @Slf4j public class MetricsBindingConsumerFactory implements ConsumerFactory { @@ -21,7 +26,7 @@ public Consumer createConsumer(String groupId) { var consumer = delegateConsumerFactory.createConsumer(groupId); var metrics = new KafkaClientMetrics(consumer); metrics.bindTo(meterRegistry); - return new MetricsAwareConsumer(consumer, metrics); + return new MetricsAwareConsumer<>(consumer, metrics); } @RequiredArgsConstructor diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/persistence/PolyflowPersistenceConfiguration.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/persistence/PolyflowPersistenceConfiguration.java deleted file mode 100644 index 67edcfcfad..0000000000 --- a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/persistence/PolyflowPersistenceConfiguration.java +++ /dev/null @@ -1,16 +0,0 @@ -package de.muenchen.oss.digiwf.task.service.infra.persistence; - -import io.holunda.polyflow.view.jpa.EnablePolyflowJpaView; -import org.axonframework.eventhandling.deadletter.jpa.DeadLetterEntry; -import org.axonframework.eventhandling.tokenstore.jpa.TokenEntry; -import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.context.annotation.Configuration; - -@Configuration -@EnablePolyflowJpaView -@EntityScan(basePackageClasses = { - TokenEntry.class, - DeadLetterEntry.class -}) -public class PolyflowPersistenceConfiguration { -} diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/polyflow/PolyflowGeneralConfiguration.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/polyflow/PolyflowGeneralConfiguration.java index 0721f3d934..cc614f4095 100644 --- a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/polyflow/PolyflowGeneralConfiguration.java +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/polyflow/PolyflowGeneralConfiguration.java @@ -5,6 +5,7 @@ import de.muenchen.oss.digiwf.task.PolyflowObjectMapper; import io.holunda.polyflow.bus.jackson.config.FallbackPayloadObjectMapperAutoConfiguration; import io.holunda.polyflow.view.TaskQueryClient; +import io.holunda.polyflow.view.jpa.EnablePolyflowJpaView; import org.axonframework.eventsourcing.eventstore.EventStorageEngine; import org.axonframework.eventsourcing.eventstore.inmemory.InMemoryEventStorageEngine; import org.axonframework.modelling.saga.repository.SagaStore; @@ -16,13 +17,9 @@ import org.springframework.context.annotation.Primary; @Configuration +@EnablePolyflowJpaView public class PolyflowGeneralConfiguration { - @Bean - @Primary - public ObjectMapper primaryObjectMapper() { - return PolyflowObjectMapper.DEFAULT; - } @Bean @Qualifier(FallbackPayloadObjectMapperAutoConfiguration.PAYLOAD_OBJECT_MAPPER) @@ -30,36 +27,6 @@ public ObjectMapper payloadObjectMapper() { return PolyflowObjectMapper.DEFAULT; } - /** - * Provides an objectmapper for Axon message serialization. - * - * @return objectmapper. - */ - @Bean("defaultAxonObjectMapper") - @Qualifier("defaultAxonObjectMapper") - public ObjectMapper defaultAxonObjectMapper() { - return PolyflowObjectMapper.DEFAULT; - } - - /** - * We will receive events via Kafka, so no event storage is available in this component. - * - * @return in-memory storage engine, to make Axon Framework happy. - */ - @Bean - public EventStorageEngine inMemoryStorageEngine() { - return new InMemoryEventStorageEngine(); - } - - /** - * No sagas should be handled in this component. - * - * @return in-memory saga-store to make Axon Framework happy. - */ - @Bean - public SagaStore inMemorySagaStore() { - return new InMemorySagaStore(); - } /** diff --git a/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/spring/JacksonConfiguration.java b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/spring/JacksonConfiguration.java new file mode 100644 index 0000000000..52ecb2681d --- /dev/null +++ b/digiwf-task/digiwf-tasklist-service/src/main/java/de/muenchen/oss/digiwf/task/service/infra/spring/JacksonConfiguration.java @@ -0,0 +1,17 @@ +package de.muenchen.oss.digiwf.task.service.infra.spring; + +import com.fasterxml.jackson.databind.ObjectMapper; +import de.muenchen.oss.digiwf.task.PolyflowObjectMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +@Configuration +public class JacksonConfiguration { + @Bean + @Primary + public ObjectMapper primaryObjectMapper() { + return PolyflowObjectMapper.DEFAULT; + } + +} diff --git a/digiwf-task/digiwf-tasklist-service/src/main/resources/application.yml b/digiwf-task/digiwf-tasklist-service/src/main/resources/application.yml index eda639f7bc..1b6731319e 100644 --- a/digiwf-task/digiwf-tasklist-service/src/main/resources/application.yml +++ b/digiwf-task/digiwf-tasklist-service/src/main/resources/application.yml @@ -73,7 +73,11 @@ axon: properties: security.protocol: ${KAFKA_SECURITY_PROTOCOL} + polyflow: + metrics: + delay: + taskcount: 30000 axon: kafka: enabled: true @@ -144,5 +148,5 @@ management: web: exposure: include: health, info, prometheus, livenessstate, readinessstate - path-mapping: - prometheus: metrics \ No newline at end of file + path-mapping: + prometheus: metrics diff --git a/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/FileOperationsIT.java b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/FileOperationsIT.java index 66b1ad0abd..e0199bdeff 100644 --- a/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/FileOperationsIT.java +++ b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/FileOperationsIT.java @@ -7,6 +7,7 @@ import de.muenchen.oss.digiwf.task.TaskVariables; import de.muenchen.oss.digiwf.task.service.TaskListApplication; import de.muenchen.oss.digiwf.task.service.infra.file.S3MockConfiguration; +import de.muenchen.oss.digiwf.task.service.infra.metrics.CollectorRegistryMockingConfiguration; import de.muenchen.oss.digiwf.task.service.infra.security.TestUser; import de.muenchen.oss.digiwf.task.service.infra.security.WithKeycloakUser; import io.holunda.camunda.bpm.data.CamundaBpmData; @@ -38,7 +39,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; @SpringBootTest(classes = TaskListApplication.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ContextConfiguration(classes = {S3MockConfiguration.class}) +@ContextConfiguration(classes = {S3MockConfiguration.class, CollectorRegistryMockingConfiguration.class}) @ActiveProfiles({"itest", "embedded-kafka", "no-security"}) @AutoConfigureMockMvc(addFilters = true) @EmbeddedKafka( diff --git a/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/RetrieveTasksIT.java b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/RetrieveTasksIT.java index d4c2c01bf1..dbbbef99d8 100644 --- a/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/RetrieveTasksIT.java +++ b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/RetrieveTasksIT.java @@ -3,6 +3,7 @@ import com.google.common.collect.Sets; import de.muenchen.oss.digiwf.task.service.TaskListApplication; import de.muenchen.oss.digiwf.task.service.adapter.out.user.MockUserGroupResolverAdapter; +import de.muenchen.oss.digiwf.task.service.infra.metrics.CollectorRegistryMockingConfiguration; import de.muenchen.oss.digiwf.task.service.infra.security.TestUser; import de.muenchen.oss.digiwf.task.service.infra.security.WithKeycloakUser; import io.holunda.polyflow.view.Task; @@ -21,6 +22,7 @@ import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; import org.springframework.test.web.servlet.MockMvc; import java.util.Arrays; @@ -51,6 +53,7 @@ ) @Slf4j @DirtiesContext +@ContextConfiguration(classes = {CollectorRegistryMockingConfiguration.class}) public class RetrieveTasksIT { @Autowired diff --git a/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/TaskExternalLinksIT.java b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/TaskExternalLinksIT.java index 6dceb86db9..5173b6ce36 100644 --- a/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/TaskExternalLinksIT.java +++ b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/TaskExternalLinksIT.java @@ -9,6 +9,7 @@ import de.muenchen.oss.digiwf.task.service.adapter.out.user.MockUserGroupResolverAdapter; import de.muenchen.oss.digiwf.task.service.application.port.out.engine.TaskCommandPort; import de.muenchen.oss.digiwf.task.service.application.usecase.TestFixtures; +import de.muenchen.oss.digiwf.task.service.infra.metrics.CollectorRegistryMockingConfiguration; import de.muenchen.oss.digiwf.task.service.infra.security.TestUser; import de.muenchen.oss.digiwf.task.service.infra.security.WithKeycloakUser; import io.holunda.camunda.bpm.data.CamundaBpmData; @@ -30,6 +31,7 @@ import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; import org.springframework.test.web.servlet.MockMvc; import java.time.Instant; @@ -59,6 +61,7 @@ @Slf4j @WireMockTest(httpPort = 7080) @DirtiesContext +@ContextConfiguration(classes = {CollectorRegistryMockingConfiguration.class}) public class TaskExternalLinksIT { private final Instant followUpDate = Instant.now().plus(2, ChronoUnit.DAYS); diff --git a/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/TaskOperationsIT.java b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/TaskOperationsIT.java index af74032551..bfa746e7d0 100644 --- a/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/TaskOperationsIT.java +++ b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/adapter/in/rest/TaskOperationsIT.java @@ -6,6 +6,7 @@ import de.muenchen.oss.digiwf.task.service.TaskListApplication; import de.muenchen.oss.digiwf.task.service.adapter.out.user.MockUserGroupResolverAdapter; import de.muenchen.oss.digiwf.task.service.application.port.out.engine.TaskCommandPort; +import de.muenchen.oss.digiwf.task.service.infra.metrics.CollectorRegistryMockingConfiguration; import de.muenchen.oss.digiwf.task.service.infra.security.TestUser; import de.muenchen.oss.digiwf.task.service.infra.security.WithKeycloakUser; import io.holunda.polyflow.view.Task; @@ -25,6 +26,7 @@ import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; import org.springframework.test.web.servlet.MockMvc; import java.time.Instant; @@ -55,6 +57,7 @@ @Slf4j @WireMockTest(httpPort = 7080) @DirtiesContext +@ContextConfiguration(classes = {CollectorRegistryMockingConfiguration.class}) public class TaskOperationsIT { diff --git a/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/infra/metrics/CollectorRegistryMockingConfiguration.java b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/infra/metrics/CollectorRegistryMockingConfiguration.java new file mode 100644 index 0000000000..d60cd60bab --- /dev/null +++ b/digiwf-task/digiwf-tasklist-service/src/test/java/de/muenchen/oss/digiwf/task/service/infra/metrics/CollectorRegistryMockingConfiguration.java @@ -0,0 +1,15 @@ +package de.muenchen.oss.digiwf.task.service.infra.metrics; + +import io.prometheus.client.CollectorRegistry; +import org.mockito.Mockito; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class CollectorRegistryMockingConfiguration { + + @Bean + public CollectorRegistry mockRegistry() { + return Mockito.mock(CollectorRegistry.class); + } +} diff --git a/digiwf-task/digiwf-tasklist-service/src/test/resources/application-embedded-kafka.yml b/digiwf-task/digiwf-tasklist-service/src/test/resources/application-embedded-kafka.yml index 163a49a051..e2331dc8e0 100644 --- a/digiwf-task/digiwf-tasklist-service/src/test/resources/application-embedded-kafka.yml +++ b/digiwf-task/digiwf-tasklist-service/src/test/resources/application-embedded-kafka.yml @@ -23,6 +23,7 @@ polyflow: logging: level: + kafka.server: WARN org.apache.kafka.clients.admin: WARN org.apache.kafka.clients.consumer: WARN org.apache.kafka.clients.producer: WARN diff --git a/digiwf-task/pom.xml b/digiwf-task/pom.xml index 9d9e03713e..7b988dfbea 100644 --- a/digiwf-task/pom.xml +++ b/digiwf-task/pom.xml @@ -18,7 +18,7 @@ 1.9.22 7.20.0 - 4.1.1 + 4.1.2 1.5.0 diff --git a/stack/docker-compose.yml b/stack/docker-compose.yml index a7439c7ffb..960421d862 100644 --- a/stack/docker-compose.yml +++ b/stack/docker-compose.yml @@ -1,6 +1,35 @@ # Use this only in dev environments. It's not intended for production usage. version: '3.9' services: + + digiwf-gateway: + image: itatm/digiwf-gateway:dev + depends_on: + init-keycloak: + condition: service_completed_successfully + env_file: + - local-docker.env + ports: + - "8083:8083" + environment: + # + # profile specific configuration + # + SPRING_PROFILES_ACTIVE: "local, docker" #possible values: local or (local, docker) # add profile 'docker' if you want to run the engine and frontend in docker, add local when you want to run the engine and frontend outside of docker + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - internal + # enable the tasklist with --profile tasklist if you don't want to run it locally. Then access via http://localhost:8083 + digiwf-tasklist: + image: itatm/digiwf-tasklist:dev + ports: + - "8091:8080" + profiles: + - tasklist-frontend + networks: + - internal + zookeeper: image: confluentinc/cp-zookeeper:latest environment: @@ -34,6 +63,23 @@ services: networks: - internal + kafka-ui: + image: provectuslabs/kafka-ui + container_name: kafka-ui + ports: + - '8089:8080' + depends_on: + - kafka + - zookeeper + environment: + KAFKA_CLUSTERS_0_NAME: local + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092 + KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181 + DYNAMIC_CONFIG_ENABLED: 'true' + networks: + - internal + + init-kafka: image: confluentinc/cp-kafka:latest depends_on: @@ -52,10 +98,10 @@ services: echo -e 'Creating kafka topics...' kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic $${KAFKA_TOPIC_TASKS} --replication-factor 1 --partitions 1 kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic $${KAFKA_TOPIC_DATA_ENTRIES} --replication-factor 1 --partitions 1 - + kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic $${KAFKA_TOPIC_COCREATION_DEPLOY} --replication-factor 1 --partitions 1 kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic $${KAFKA_TOPIC_COCREATION} --replication-factor 1 --partitions 1 - + kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic $${KAFKA_TOPIC_ENGINE} --replication-factor 1 --partitions 1 kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic $${KAFKA_TOPIC_ENGINE_DLQ} --replication-factor 1 --partitions 1 @@ -71,7 +117,7 @@ services: kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic $${KAFKA_TOPICS_COSYS_INTEGRATION} --replication-factor 1 --partitions 1 kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic $${KAFKA_TOPICS_ALW_INTEGRATION} --replication-factor 1 --partitions 1 kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic $${KAFKA_TOPICS_DMS_INTEGRATION} --replication-factor 1 --partitions 1 - + echo -e 'Resulting topics:' kafka-topics --bootstrap-server kafka:9092 --list " @@ -111,33 +157,7 @@ services: networks: - internal - digiwf-gateway: - image: itatm/digiwf-gateway:dev - depends_on: - init-keycloak: - condition: service_completed_successfully - env_file: - - local-docker.env - ports: - - "8083:8083" - environment: - # - # profile specific configuration - # - SPRING_PROFILES_ACTIVE: "local, docker" #possible values: local or (local, docker) # add profile 'docker' if you want to run the engine and frontend in docker, add local when you want to run the engine and frontend outside of docker - extra_hosts: - - "host.docker.internal:host-gateway" - networks: - - internal - # enable the tasklist with --profile tasklist if you don't want to run it locally. Then access via http://localhost:8083 - digiwf-tasklist: - image: itatm/digiwf-tasklist:dev - ports: - - "8091:8080" - profiles: - - tasklist-frontend - networks: - - internal + # # Local keycloak. To work properly, you need to change your local hosts file and add an alias to your # `127.0.0.1 localhost` line to look like this: `127.0.0.1 localhost keycloak`.