This is an automated email from the ASF dual-hosted git repository. fmariani pushed a commit to branch smart-log-analyzer in repository https://gitbox.apache.org/repos/asf/camel-jbang-examples.git
commit 3129d478323f3bf6a0aacd33e77ea92424a21e57 Author: Croway <[email protected]> AuthorDate: Wed Jan 28 13:14:18 2026 +0100 Smart log analyzer example --- .gitignore | 1 + smart-log-analyzer/README.adoc | 215 +++++++++++++ .../analyzer/application-dev.properties | 31 ++ .../analyzer/error-analyzer.camel.yaml | 74 +++++ .../caches/infinispan-events-config.json | 14 + .../infinispan-events-to-process-config.json | 14 + smart-log-analyzer/containers/docker-compose.yaml | 98 ++++++ .../containers/otel-collector-config.yaml | 32 ++ .../correlator/application-dev.properties | 28 ++ .../correlator/correlated-log-schema.json | 38 +++ .../correlator/correlated-trace-schema.json | 58 ++++ .../correlator/infinispan.camel.yaml | 104 +++++++ smart-log-analyzer/correlator/kafka-ca-cert.pem | 70 +++++ .../correlator/kaoto-datamapper-4a94acc3.xsl | 38 +++ .../correlator/kaoto-datamapper-8f5bb2dd.xsl | 64 ++++ .../correlator/logs-mapper.camel.yaml | 45 +++ .../correlator/otel-log-record-schema.json | 137 +++++++++ .../correlator/otel-logs-schema.json | 227 ++++++++++++++ .../correlator/otel-span-schema.json | 244 +++++++++++++++ .../correlator/otel-traces-schema.json | 334 +++++++++++++++++++++ .../correlator/traces-mapper.camel.yaml | 45 +++ .../first-iteration/analyzer.camel.yaml | 62 ++++ .../first-iteration/application.properties | 2 + .../first-iteration/load-generator.camel.yaml | 48 +++ .../test/first-iteration.camel.it.yaml | 46 +++ .../first-iteration/test/jbang.properties | 3 + .../first-iteration/test/payload-error.json | 1 + .../first-iteration/test/payload-info.json | 1 + smart-log-analyzer/log-generator/agent.properties | 6 + .../log-generator/application-dev.properties | 3 + .../log-generator/log-generator.camel.yaml | 87 ++++++ .../log-generator/opentelemetry-javaagent.jar | Bin 0 -> 24026362 bytes .../ui-console/application-dev.properties | 31 ++ smart-log-analyzer/ui-console/index.html | 287 ++++++++++++++++++ .../ui-console/jms-file-storage.camel.yaml | 14 + smart-log-analyzer/ui-console/rest-api.camel.yaml | 84 ++++++ 36 files changed, 2586 insertions(+) diff --git a/.gitignore b/.gitignore index aeaba61..7bda6c8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ target/ camel-runner.jar *.i?? .citrus-jbang/ +.kaoto \ No newline at end of file diff --git a/smart-log-analyzer/README.adoc b/smart-log-analyzer/README.adoc new file mode 100644 index 0000000..c96d25b --- /dev/null +++ b/smart-log-analyzer/README.adoc @@ -0,0 +1,215 @@ += Smart Log Analyzer + +An intelligent observability system that correlates OpenTelemetry logs and traces, then uses LLM for automated root cause analysis. + +== Architecture + +[source,mermaid] +---- +sequenceDiagram + participant App as Application + participant OTel as OTEL Collector + participant Kafka as Kafka + participant Correlator as Correlator + participant Events as Infinispan (events) + participant ToProcess as Infinispan (events-to-process) + participant JMS as JMS Queue + participant Analyzer as Analyzer + participant LLM as Ollama LLM + participant UIConsole as UI Console + + App->>OTel: Send traces & logs (OTLP) + OTel->>Kafka: Export to otlp_spans & otlp_logs topics + + Kafka->>Correlator: Consume spans + Kafka->>Correlator: Consume logs + Correlator->>Correlator: Transform via XSLT + Correlator->>Events: Store by traceId (TTL: 5min) + + alt Error detected (severityText = ERROR) + Correlator->>ToProcess: Add traceId (TTL: 20s) + Note over ToProcess: Wait for related events + ToProcess-->>Correlator: Cache entry expired event + Correlator->>JMS: Send traceId to queue + JMS->>Analyzer: Consume traceId + Analyzer->>Events: Fetch all events for traceId + Analyzer->>LLM: Send events for analysis + LLM-->>Analyzer: Root cause analysis result + Analyzer->>JMS: Send analysis result + JMS->>UIConsole: Consume analysis result + UIConsole->>UIConsole: Save and display result + end +---- + +== First Iteration (Prototype) + +The `first-iteration/` folder contains a simpler prototype using in-memory aggregation. Ideal for learning and local testing, but won't work in distributed deployments. To run it: + +[source,bash] +---- +# Terminal 1: Start Kafka +camel infra run kafka + +# Terminal 2: Start the load generator +cd first-iteration +camel run load-generator.camel.yaml + +# Terminal 3: Start the analyzer (requires Ollama with granite4:3b) +cd first-iteration +camel run analyzer.camel.yaml --dep=org.apache.camel:camel-openai +---- + +=== Running the Test + +The first-iteration includes a Citrus integration test that uses `--stub=openai` to mock the LLM endpoint. + +First, install Citrus as a JBang app (one-time setup): + +[source,bash] +---- +jbang app install citrus@citrusframework/citrus +---- + +Then run the test: + +[source,bash] +---- +cd first-iteration/test +citrus run first-iteration.camel.it.yaml +---- + +The test starts Kafka via testcontainers, sends log events, and verifies the aggregation triggers correctly. + +== Components + +=== Correlator + +The correlator is the central event processing component: + +* Consumes OTEL logs from `otlp_logs` Kafka topic +* Consumes OTEL traces from `otlp_spans` Kafka topic +* Transforms complex OTEL format to simplified JSON via XSLT +* Correlates all events by traceId in Infinispan `events` cache (TTL: 5min) +* Uses `events-to-process` cache for deferred error analysis: +** When an ERROR log is detected, adds the traceId to this cache with a 20s TTL +** Listens for cache expiration events to trigger analysis +** This delay ensures all related spans and logs are collected before analysis +* Sends expired traceIds to JMS queue for the Analyzer + +=== Analyzer + +The analyzer performs intelligent root cause analysis: + +* Listens on JMS `error-logs` queue for traceIds from the Correlator +* Retrieves full trace context from Infinispan `events` cache +* Sends events to Ollama LLM (granite4:3b) for intelligent analysis +* Outputs analysis results to logs + +=== Log Generator + +A synthetic log generator for demonstration purposes that shows how to add OpenTelemetry observability to Apache Camel applications. It simulates order processing with intentional errors (30% failure rate). + +*Adding OpenTelemetry to your Camel application requires:* + +. *opentelemetry-javaagent.jar* - The OTEL Java agent for automatic instrumentation. Download from https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases[OpenTelemetry releases]. + +. *agent.properties* - Configures the OTEL agent: ++ +[source,properties] +---- +otel.service.name=log-generator +otel.traces.exporter=otlp +otel.logs.exporter=otlp +otel.metrics.exporter=none +---- + +. *application-dev.properties* - Enables Camel OpenTelemetry integration: ++ +[source,properties] +---- +camel.opentelemetry2.enabled = true +camel.jbang.dependencies=org.apache.camel:camel-opentelemetry2 +---- + +Run with the javaagent flag pointing to the agent jar and its configuration file. With this setup, Camel will automatically expose custom traces for route executions and exchanges. + +=== UI Console + +The UI Console provides visualization and storage for analysis results: + +* Listens on JMS queue for analysis results from the Analyzer +* Stores analysis results to files for persistence +* Exposes REST API to display and access the results +* Provides a user interface to view root cause analysis + +== Prerequisites + +* Docker & Docker Compose +* https://www.jbang.dev/download/[JBang] +* https://ollama.ai/[Ollama] with `granite4:3b` model: ++ +[source,bash] +---- +ollama pull granite4:3b +---- + +== Running the Application + +=== 1. Start Infrastructure + +[source,bash] +---- +cd containers +docker-compose up +---- + +=== 2. Start Correlator + +[source,bash] +---- +cd correlator +camel run \ + traces-mapper.camel.yaml \ + logs-mapper.camel.yaml \ + infinispan.camel.yaml \ + kaoto-datamapper-4a94acc3.xsl \ + kaoto-datamapper-8f5bb2dd.xsl +---- + +=== 3. Start Analyzer + +[source,bash] +---- +cd analyzer +camel run error-analyzer.camel.yaml +---- + +=== 4. Start Log Generator + +[source,bash] +---- +cd log-generator +jbang --javaagent=./opentelemetry-javaagent.jar \ + -Dotel.javaagent.configuration-file=./agent.properties \ + camel@apache/camel run log-generator.camel.yaml +---- + +=== 5. Start UI Console + +[source,bash] +---- +cd ui-console +camel run * +---- + +== Infrastructure Services + +[cols="1,1,2"] +|=== +| Service | Port | Purpose + +| Kafka | 9092 | Message broker for OTEL data +| Infinispan | 11222 | Event cache (admin:password) +| OTEL Collector | 4317, 4318 | Telemetry receiver (gRPC, HTTP) +| ActiveMQ Artemis | 61616, 8161 | JMS broker (artemis:artemis) +|=== diff --git a/smart-log-analyzer/analyzer/application-dev.properties b/smart-log-analyzer/analyzer/application-dev.properties new file mode 100644 index 0000000..caef75f --- /dev/null +++ b/smart-log-analyzer/analyzer/application-dev.properties @@ -0,0 +1,31 @@ +# Infinispan configuration +camel.component.infinispan.hosts=localhost:11222 +camel.component.infinispan.username=admin +camel.component.infinispan.password=password +camel.component.infinispan.sasl-mechanism=DIGEST-MD5 +camel.component.infinispan.security-realm=default +camel.component.infinispan.security-server-name=infinispan +camel.component.infinispan.secure=true + +# Artemis JMS configuration +camel.beans.artemisCF = #class:org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory +camel.beans.artemisCF.brokerURL = tcp://localhost:61616 +camel.beans.artemisCF.user = artemis +camel.beans.artemisCF.password = artemis +camel.beans.poolCF = #class:org.messaginghub.pooled.jms.JmsPoolConnectionFactory +camel.beans.poolCF.connectionFactory = #bean:artemisCF +camel.beans.poolCF.maxSessionsPerConnection = 500 +camel.beans.poolCF.connectionIdleTimeout = 20000 +camel.component.jms.connection-factory = #bean:poolCF +camel.jms.queue.error-logs=error-logs +camel.jms.queue.analysis-result=analysis-result + +# Ollama configuration (OpenAI-compatible API) +camel.component.openai.apiKey=ollama +camel.component.openai.baseUrl=http://localhost:11434/v1 +camel.component.openai.model=granite4:3b + +# Camel configuration +camel.main.name=analyzer + +camel.jbang.dependencies=org.apache.camel:camel-openai diff --git a/smart-log-analyzer/analyzer/error-analyzer.camel.yaml b/smart-log-analyzer/analyzer/error-analyzer.camel.yaml new file mode 100644 index 0000000..406f570 --- /dev/null +++ b/smart-log-analyzer/analyzer/error-analyzer.camel.yaml @@ -0,0 +1,74 @@ +# Analyzes error logs by retrieving OpenTelemetry events from Infinispan +# and sending them to an LLM for root cause analysis and recommendations +- route: + id: error-log-analyzer + from: + uri: jms:{{camel.jms.queue.error-logs}} + steps: + - log: + loggingLevel: INFO + message: Received ERROR log for analysis + - setVariable: + name: traceId + simple: ${body} + - log: + loggingLevel: INFO + message: "Retrieving all events for traceId: ${variable.traceId}" + - setHeader: + name: CamelInfinispanKey + simple: otel-${variable.traceId} + - to: + uri: infinispan:events + parameters: + operation: GET + - choice: + otherwise: + steps: + - log: + loggingLevel: WARN + message: "No events found in Infinispan for traceId: ${variable.traceId}" + when: + - steps: + - setVariable: + name: recordCount + jq: + expression: length + resultType: java.lang.String + - log: + message: Sending ${variable.recordCount} records to LLM for analysis + - setVariable: + name: eventsJson + simple: ${body} + - setBody: + simple: > + Analyze the following OpenTelemetry logs and traces for + traceId ${variable.traceId}. + + Identify the root cause of the error, explain what + happened, and suggest possible fixes. + + + Events (sorted by timestamp): + + ${variable.eventsJson} + - to: + uri: openai:chat-completion + parameters: + systemMessage: > + You are an expert DevOps engineer and log analyst. + Analyze OpenTelemetry logs and traces to identify + errors, their root causes, and provide actionable + recommendations. Be concise and focus on the error. + You want to build a tree of calls by associating each + span with its parent, where parentSpanId corresponds + to the parent caller’s spanId. + - log: + message: "LLM Analysis for traceId ${variable.traceId}: ${body}" + - setHeader: + name: traceId + simple: "${variable.traceId}" + - to: + uri: jms:{{camel.jms.queue.analysis-result}} + - log: + message: "Sent analysis result to queue for traceId: ${variable.traceId}" + simple: ${body} != null && ${body} != '' diff --git a/smart-log-analyzer/containers/caches/infinispan-events-config.json b/smart-log-analyzer/containers/caches/infinispan-events-config.json new file mode 100644 index 0000000..da20578 --- /dev/null +++ b/smart-log-analyzer/containers/caches/infinispan-events-config.json @@ -0,0 +1,14 @@ +{ + "events": { + "distributed-cache": { + "mode": "SYNC", + "statistics": true, + "encoding": { + "media-type": "text/plain" + }, + "expiration": { + "lifespan": "600000ms" + } + } + } +} diff --git a/smart-log-analyzer/containers/caches/infinispan-events-to-process-config.json b/smart-log-analyzer/containers/caches/infinispan-events-to-process-config.json new file mode 100644 index 0000000..d34d9a4 --- /dev/null +++ b/smart-log-analyzer/containers/caches/infinispan-events-to-process-config.json @@ -0,0 +1,14 @@ +{ + "events-to-process": { + "distributed-cache": { + "mode": "SYNC", + "statistics": true, + "encoding": { + "media-type": "text/plain" + }, + "expiration": { + "lifespan": "20000ms" + } + } + } +} diff --git a/smart-log-analyzer/containers/docker-compose.yaml b/smart-log-analyzer/containers/docker-compose.yaml new file mode 100644 index 0000000..afc84d9 --- /dev/null +++ b/smart-log-analyzer/containers/docker-compose.yaml @@ -0,0 +1,98 @@ +services: + kafka: + image: apache/kafka:latest + container_name: kafka + hostname: kafka + environment: + # KRaft mode configuration (no Zookeeper) + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + # Multiple listeners: PLAINTEXT for Docker network, PLAINTEXT_HOST for localhost access + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:19092,PLAINTEXT_HOST://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:19092,PLAINTEXT_HOST://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_DEFAULT_REPLICATION_FACTOR: 1 + KAFKA_MIN_INSYNC_REPLICAS: 1 + # Storage (ephemeral - no persistence) + KAFKA_LOG_DIRS: /tmp/kraft-combined-logs + ports: + - "9092:9092" # External access from localhost + networks: + - otel-network + + infinispan: + image: quay.io/infinispan/server:16.0.5 + container_name: infinispan + hostname: infinispan + environment: + USER: admin + PASS: password + ports: + - "11222:11222" + networks: + - otel-network + healthcheck: + test: ["CMD", "curl", "-u", "admin:password", "--digest", "http://localhost:11222/rest/v2/caches";] + interval: 5s + timeout: 3s + retries: 8 + + infinispan-init: + image: curlimages/curl:latest + container_name: infinispan-init + depends_on: + infinispan: + condition: service_healthy + networks: + - otel-network + volumes: + - ./caches:/etc/caches:ro + command: > + sh -c " + sleep 2 && + echo 'Creating caches...' && + curl -f -u admin:password --digest -X POST 'http://infinispan:11222/rest/v2/caches/events' -H 'Content-Type: application/json' -d @/etc/caches/infinispan-events-config.json && + curl -f -u admin:password --digest -X POST 'http://infinispan:11222/rest/v2/caches/events-to-process' -H 'Content-Type: application/json' -d @/etc/caches/infinispan-events-to-process-config.json && + echo 'Caches created successfully' + " + deploy: + restart_policy: + condition: on-failure + max_attempts: 5 + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: otel-collector + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver + networks: + - otel-network + depends_on: + - kafka + + artemis: + image: apache/activemq-artemis:latest + container_name: artemis + hostname: artemis + environment: + ARTEMIS_USER: artemis + ARTEMIS_PASSWORD: artemis + ports: + - "8161:8161" # Web console + - "61616:61616" # Core protocol + networks: + - otel-network + +networks: + otel-network: + driver: bridge diff --git a/smart-log-analyzer/containers/otel-collector-config.yaml b/smart-log-analyzer/containers/otel-collector-config.yaml new file mode 100644 index 0000000..876bbf6 --- /dev/null +++ b/smart-log-analyzer/containers/otel-collector-config.yaml @@ -0,0 +1,32 @@ +exporters: + debug: + verbosity: detailed + kafka: + brokers: + - kafka:19092 + logs: + encoding: otlp_json + traces: + encoding: otlp_json +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +service: + pipelines: + logs: + receivers: + - otlp + exporters: + - debug + - kafka + traces: + receivers: + - otlp + exporters: + - debug + - kafka diff --git a/smart-log-analyzer/correlator/application-dev.properties b/smart-log-analyzer/correlator/application-dev.properties new file mode 100644 index 0000000..3f126e6 --- /dev/null +++ b/smart-log-analyzer/correlator/application-dev.properties @@ -0,0 +1,28 @@ +# Kafka configuration +camel.component.kafka.brokers=localhost:9092 +camel.kafka.topic.logs=otlp_logs +camel.kafka.topic.spans=otlp_spans + +# Infinispan configuration +camel.component.infinispan.hosts=localhost:11222 +camel.component.infinispan.username=admin +camel.component.infinispan.password=password +camel.component.infinispan.sasl-mechanism=DIGEST-MD5 +camel.component.infinispan.security-realm=default +camel.component.infinispan.security-server-name=infinispan +camel.component.infinispan.secure=true + +# Artemis JMS configuration +camel.beans.artemisCF = #class:org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory +camel.beans.artemisCF.brokerURL = tcp://localhost:61616 +camel.beans.artemisCF.user = artemis +camel.beans.artemisCF.password = artemis +camel.beans.poolCF = #class:org.messaginghub.pooled.jms.JmsPoolConnectionFactory +camel.beans.poolCF.connectionFactory = #bean:artemisCF +camel.beans.poolCF.maxSessionsPerConnection = 500 +camel.beans.poolCF.connectionIdleTimeout = 20000 +camel.component.jms.connection-factory = #bean:poolCF +camel.jms.queue.error-logs=error-logs + +# Camel configuration +camel.main.name=correlator diff --git a/smart-log-analyzer/correlator/correlated-log-schema.json b/smart-log-analyzer/correlator/correlated-log-schema.json new file mode 100644 index 0000000..a49f953 --- /dev/null +++ b/smart-log-analyzer/correlator/correlated-log-schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema";, + "$id": "correlated-log-entry", + "title": "Correlated Log Entry", + "description": "Simplified log entry for storage in Infinispan", + "type": "object", + "required": ["timeUnixNano", "severityText", "message", "traceId", "spanId"], + "properties": { + "timeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Time when the event occurred (nanoseconds since Unix epoch)" + }, + "observedTimeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Time when the event was observed (nanoseconds since Unix epoch)" + }, + "severityText": { + "type": "string", + "description": "Severity level (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)" + }, + "message": { + "type": "string", + "description": "Combined message from body and attributes" + }, + "traceId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{32}$", + "description": "Trace ID for correlation" + }, + "spanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Span ID" + } + } +} diff --git a/smart-log-analyzer/correlator/correlated-trace-schema.json b/smart-log-analyzer/correlator/correlated-trace-schema.json new file mode 100644 index 0000000..7cdec55 --- /dev/null +++ b/smart-log-analyzer/correlator/correlated-trace-schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema";, + "$id": "correlated-trace-entry", + "title": "Correlated Trace Entry", + "description": "Simplified span entry for storage in Infinispan, designed for LLM error analysis", + "type": "object", + "required": ["type", "timeUnixNano", "traceId", "spanId", "name", "status"], + "properties": { + "type": { + "type": "string", + "const": "span", + "description": "Entry type to distinguish from logs in combined timeline" + }, + "timeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Start time of the span (nanoseconds since Unix epoch), used for ordering with logs" + }, + "durationNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Duration of the span in nanoseconds (endTimeUnixNano - startTimeUnixNano)" + }, + "traceId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{32}$", + "description": "Trace ID for correlation with logs" + }, + "spanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Span ID" + }, + "parentSpanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Parent span ID for understanding call hierarchy" + }, + "name": { + "type": "string", + "description": "Operation name (e.g., timer, http.request)" + }, + "kind": { + "type": "string", + "enum": ["UNSPECIFIED", "INTERNAL", "SERVER", "CLIENT", "PRODUCER", "CONSUMER"], + "description": "Span kind indicating the type of operation" + }, + "status": { + "type": "string", + "enum": ["UNSET", "OK", "ERROR"], + "description": "Span status" + }, + "message": { + "type": "string", + "description": "Combined message from key attributes and error event messages" + } + } +} diff --git a/smart-log-analyzer/correlator/infinispan.camel.yaml b/smart-log-analyzer/correlator/infinispan.camel.yaml new file mode 100644 index 0000000..c4e0cde --- /dev/null +++ b/smart-log-analyzer/correlator/infinispan.camel.yaml @@ -0,0 +1,104 @@ +# Stores OpenTelemetry records in Infinispan cache, grouped by traceId +# and marks ERROR events for deferred processing +- route: + id: route-1200 + from: + id: from-9847 + uri: direct + parameters: + name: store + steps: + - log: + id: log-record + loggingLevel: DEBUG + message: "Processing record: ${body}" + - setVariable: + jq: + expression: .traceId + resultType: java.lang.String + name: traceId + - filter: + expression: + simple: ${variable.traceId} != null && ${variable.traceId} != '' + steps: + - claimCheck: + key: currentRecord + operation: Set + - setHeader: + name: CamelInfinispanKey + simple: otel-${variable.traceId} + - to: + uri: infinispan:events + parameters: + operation: GET + - choice: + otherwise: + steps: + - claimCheck: + key: currentRecord + operation: Get + - setBody: + jq: + expression: "[.]" + resultType: java.lang.String + when: + - steps: + - setVariable: + name: existingRecords + simple: ${body} + - claimCheck: + key: currentRecord + operation: Get + - setBody: + jq: + expression: (variable("existingRecords") | fromjson) + [.] | + sort_by(.timeUnixNano) + resultType: java.lang.String + simple: ${body} != null && ${body} != '' + - setHeader: + name: CamelInfinispanKey + simple: otel-${variable.traceId} + - setHeader: + name: CamelInfinispanValue + simple: ${body} + - to: + uri: infinispan:events + parameters: + operation: PUT + - claimCheck: + key: currentRecord + operation: GetAndRemove + - log: + id: log-stored + message: "Stored record for traceId: ${variable.traceId}" + - choice: + when: + - steps: + - log: + message: "Adding ERROR log to events-to-process cache for traceId: + ${variable.traceId}" + - setHeader: + name: CamelInfinispanKey + simple: ${variable.traceId} + - setHeader: + name: CamelInfinispanValue + simple: ${variable.traceId} + - to: + uri: infinispan:events-to-process + parameters: + operation: PUTIFABSENT + jq: + expression: .severityText == "ERROR" +- route: + id: route-expired-events + from: + uri: infinispan:events-to-process + parameters: + eventTypes: CLIENT_CACHE_ENTRY_EXPIRED + steps: + - setBody: + simple: ${header.CamelInfinispanKey} + - log: + message: "Expired cache entry received, sending to JMS queue: ${body}" + - to: + uri: jms:{{camel.jms.queue.error-logs}} diff --git a/smart-log-analyzer/correlator/kafka-ca-cert.pem b/smart-log-analyzer/correlator/kafka-ca-cert.pem new file mode 100644 index 0000000..3ea54c6 --- /dev/null +++ b/smart-log-analyzer/correlator/kafka-ca-cert.pem @@ -0,0 +1,70 @@ +-----BEGIN CERTIFICATE----- +MIIHFTCCBP2gAwIBAgIUddBSUC646dJUTeT+vKy9E8WclvAwDQYJKoZIhvcNAQEN +BQAwLTETMBEGA1UECgwKaW8uc3RyaW16aTEWMBQGA1UEAwwNY2x1c3Rlci1jYSB2 +MDAeFw0yNjAxMTYxNDI4NTZaFw0yNzAxMTYxNDI4NTZaMDMxEzARBgNVBAoMCmlv +LnN0cmltemkxHDAaBgNVBAMME2NhbWVsLWNsdXN0ZXIta2Fma2EwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCPWjgPFIDyK0i7bns4j28WZhD5xUgIol/0 +ns0awE6MZGFerQEOQXcbwrFmAn/LoTnAnlzcROOx096KJVNXj6JnaSBbGvrnCcYF +2d6pBz7WxfHG9Pn/I6qMYTgc3XiJ4KLO+CZ/GKzo9fc6dl5kkciVS61thq26jkV1 +4aTA6oz9RComXdFjlzuSBoXULrSijWRYrGaT618kSYE2MBfJjOdAv08WoLE5UVf8 +F+bLEInJ7j1Z+/fmCxj0UB8j2UH3Wh/oAaQUBP1IGOPYfcfhiRzKgy7RWTGKMOOc +pTmtHm5wRDvSm+1OYSZlFyMxtmmqK4ZstWf0OJhrORF2Kb6KJTTrAgMBAAGjggMl +MIIDITCCAt0GA1UdEQSCAtQwggLQgj5jYW1lbC1jbHVzdGVyLWthZmthLWJyb2tl +cnMuY2FtZWwtb3RlbC1pbmZyYS5zdmMuY2x1c3Rlci5sb2NhbIJYY2FtZWwtY2x1 +c3Rlci1tdWx0aXJvbGUtMC5jYW1lbC1jbHVzdGVyLWthZmthLWJyb2tlcnMuY2Ft +ZWwtb3RlbC1pbmZyYS5zdmMuY2x1c3Rlci5sb2NhbIIwY2FtZWwtY2x1c3Rlci1r +YWZrYS1icm9rZXJzLmNhbWVsLW90ZWwtaW5mcmEuc3Zjgh1jYW1lbC1jbHVzdGVy +LWthZmthLWJvb3RzdHJhcIJUY2FtZWwtY2x1c3Rlci1rYWZrYS1ib290c3RyYXAt +Y2FtZWwtb3RlbC1pbmZyYS5hcHBzLmNzYi1sYXN0LmZ1c2UuaW50ZWdyYXRpb24t +cWUuY29tgjJjYW1lbC1jbHVzdGVyLWthZmthLWJvb3RzdHJhcC5jYW1lbC1vdGVs +LWluZnJhLnN2Y4JKY2FtZWwtY2x1c3Rlci1tdWx0aXJvbGUtMC5jYW1lbC1jbHVz +dGVyLWthZmthLWJyb2tlcnMuY2FtZWwtb3RlbC1pbmZyYS5zdmOCQGNhbWVsLWNs +dXN0ZXIta2Fma2EtYm9vdHN0cmFwLmNhbWVsLW90ZWwtaW5mcmEuc3ZjLmNsdXN0 +ZXIubG9jYWyCLmNhbWVsLWNsdXN0ZXIta2Fma2EtYm9vdHN0cmFwLmNhbWVsLW90 +ZWwtaW5mcmGCLGNhbWVsLWNsdXN0ZXIta2Fma2EtYnJva2Vycy5jYW1lbC1vdGVs +LWluZnJhghtjYW1lbC1jbHVzdGVyLWthZmthLWJyb2tlcnOCUGNhbWVsLWNsdXN0 +ZXItbXVsdGlyb2xlLTAtY2FtZWwtb3RlbC1pbmZyYS5hcHBzLmNzYi1sYXN0LmZ1 +c2UuaW50ZWdyYXRpb24tcWUuY29tMB0GA1UdDgQWBBQTlA+mmTCgeDSUKAwSpkIB +TX+2+DAfBgNVHSMEGDAWgBTQeJU9RNTnrxXBxi8RaI3kQ6Y3RDANBgkqhkiG9w0B +AQ0FAAOCAgEAIpbu83u9leM4dMqOLJWX1ZYAR31DxkhI+jMOe1JLxatD7UV/m/sh +bdOaT1kV/ddynpuBxryXmrsm63kcCmUxYtI8bs/hLoxhXRAAy4CM9fezx7C5zVqW +rqJ8Xfy8tTlDyHW9Ar3mxffQN3Sgnuvb+yqeW+OGj3nnXosrPL6cNZ2s19uRMjdl +j5jfZu8tOT74UJvnQVuESfAxXvK0X/5IoTOo8vCayWFlvMnctus2VzbCCnNtFnlM +R58qIbw+u1uESiQx2+YB8de/O9DhWLXjxqURV4Qb4OBhVuxVnACJkXnOdV70l+yN +Am6R5p1mDkPqzkklA2Nl1xcjd1IFMqkgFBmP/JeLLI/3oH3nzVu6WvA8oxHuYF4C +odojpPrnISwYYVCw0MkcYbGICbAPJHD2TZKFQ5QHolJgs+rn7c2e6vZj6nAspZU8 +4ebHXWf8jX7+T2eI2S7T8LmQV2lhIwJeKhvevrBQQtecqdQn+nFPRjsrwmeZ48Zw +S3UL33dRhAmImny3vzgPDYCLCF0WiylmJfUT+sDTxfa1KW473f7S3KKLtx184vFO +zjAUwHuSAmeO17I8Eyzdv0RcFx6YWKaVcGNvYzuFHtGQ3R0dYN0fPZ01sy53Nh42 +amfW9wQBD0MstgsxDWDfS4N6zwKSHAduAYsRHsSOwa06A+5hVdLworU= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFLTCCAxWgAwIBAgIUARYnemLQ0i/Ml55sf+x1BmUme1gwDQYJKoZIhvcNAQEN +BQAwLTETMBEGA1UECgwKaW8uc3RyaW16aTEWMBQGA1UEAwwNY2x1c3Rlci1jYSB2 +MDAeFw0yNjAxMTYxNDEzMzhaFw0yNzAxMTYxNDEzMzhaMC0xEzARBgNVBAoMCmlv +LnN0cmltemkxFjAUBgNVBAMMDWNsdXN0ZXItY2EgdjAwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCzmMknBFpQuFkvgyxKOq97A0/elsyiplC6BSi818Z/ +0AKhYKf91EDUyfG2HhUsjrnpE3yv1B3DhE8mae4JKKeN+5M7/NJsJe5nfrkxQ2pL +AQtOpxiD0J4EAl5TINQS8sU4dnllHCbiegepPO22+jsoLqt9l4wJmETpgmaVftCC +/XbKrXng8wGr8xFYIUsJ/JgB1mn+w7oIgib3RQsk3nclHkn7IUs0mXxznzdj49nU +nMCgtEg9CA9K8r/WQjij1AY3auaGAQN4i1uvzk2/mOjcTRjXdWe30asRlfvPoxZn +ouIAJSokync5dP630jLWEXvGZSmqiXwAffCOpfdPyTazWhT+IQ/p/mk6dFZnQhIT +tLEVJCHji3Y7JmuKEXgfKHuqI4CNX6VD/NSlTFSrShodNqbtKJp0ecIg+jX8tKvd +u9dNgziCcCJAKOfTNO2gs3EZ2TPH6kxxOCeLCJ60QFifrN0trs63HRuY86CS3XNd +10jpDHpOVSCWa09+g8Zk/jf8VyybzUtu1APxJMlhze8BrOx5mIQWBQds7EqCa7vs +0PaptOYHJ8JMl69pLFzYDWRWdNVSc41GHMeLX5Hg9ENRjGq4eXJQ18SKw6vZMEtV +PTG+mKGibA0kgKyf1ftSjC8urkfqYRVViHw/MPtjAWwer2QXXWHkjXklfh1OVhFZ +BwIDAQABo0UwQzAdBgNVHQ4EFgQU0HiVPUTU568VwcYvEWiN5EOmN0QwEgYDVR0T +AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQENBQADggIB +ACkXOjCMFtEZL2BSMYi/rWbHAht2K67yj8kiT3Xru+tZ0grS6KNxb3xtpXqyxH/z +j1jDW7vfW0vopTZRd0fZP7re+qHwVcEzaQfLisJ8IVmmMKOlhDXC0uD8zWH56g0/ +7gw9hDjJeNNsZLxkNd4e+rv9wJxVPm0DZ6h17rv3gMz5c24pqsv/5qO+HofE+Y46 +2jMS5t+ZzmpbYVoY0rzvkkACGjwcQ/Cr5WPEDIh1ndLXn7iIPnhNUwwLM1+97y00 +tGrEIXuL5r1p2ZZYT6KX/1yOVV6jPZXxbtD2mGWsVcmrWoaJfwkrAqaRjonPSQGB +sEnLDYdyl0hkZivd7ti+UIDxy9hAD9DA2lM1+0MTG8a2u6slAZDBvkbLOeywK+g4 +WNcGLz0zxulN1x3rGL5WOPaf7lA3hSO9p9kDT7RXkawmw/mQLCl5tX4qSpyVvIZz +6kdnxss+vsgb6qjqc8QMV+P6cXBWEaxcd2KexSmBBCg6FrjomFBMusw5tjcvdqwi +jAGY1S8vodQDAbDMmidOgYdaR+hNdPPM+U1wlR0wPJx0BiVzJA+GUaVqsu+hCcw/ +c75SgDNerU7oO9bPxNeE+0lyUm813IhhAJWjkMgSGWyys53Gs6y28cBEFlA962Rs +vSSSsYPkKrwL+sDvG8Ut+xQ4cshAFCyShOfV3MRwoEDL +-----END CERTIFICATE----- diff --git a/smart-log-analyzer/correlator/kaoto-datamapper-4a94acc3.xsl b/smart-log-analyzer/correlator/kaoto-datamapper-4a94acc3.xsl new file mode 100644 index 0000000..1e9e7f4 --- /dev/null +++ b/smart-log-analyzer/correlator/kaoto-datamapper-4a94acc3.xsl @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- This file is generated by Kaoto DataMapper. Do not edit. --> +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; version="3.0" xmlns:xs="http://www.w3.org/2001/XMLSchema"; xmlns:fn="http://www.w3.org/2005/xpath-functions";> + <xsl:output method="text" indent="yes"/> + <xsl:param name="logs"/> + <xsl:variable name="logs-x" select="json-to-xml($logs)"/> + <xsl:variable name="mapped-xml"> + <map xmlns="http://www.w3.org/2005/xpath-functions";> + <string key="timeUnixNano"> + <xsl:value-of select="$logs-x/fn:map/fn:string[@key='timeUnixNano']"/> + </string> + <string key="observedTimeUnixNano"> + <xsl:value-of select="$logs-x/fn:map/fn:string[@key='observedTimeUnixNano']"/> + </string> + <string key="severityText"> + <xsl:value-of select="$logs-x/fn:map/fn:string[@key='severityText']"/> + </string> + <string key="message"> + <xsl:value-of select="$logs-x/fn:map/fn:map[@key='body']/fn:string[@key='stringValue']"/> + <xsl:for-each select="$logs-x/fn:map/fn:array[@key='attributes']/fn:map"> + <xsl:value-of select="concat(' ', fn:string[@key='key'], ': ', fn:map[@key='value']/fn:string[@key='stringValue'])"/> + </xsl:for-each> + </string> + <string key="traceId"> + <xsl:value-of select="$logs-x/fn:map/fn:string[@key='traceId']"/> + </string> + <string key="spanId"> + <xsl:value-of select="$logs-x/fn:map/fn:string[@key='spanId']"/> + </string> + <string key="type"> + <xsl:value-of select="'log'"/> + </string> + </map> + </xsl:variable> + <xsl:template match="/"> + <xsl:value-of select="xml-to-json($mapped-xml)"/> + </xsl:template> +</xsl:stylesheet> \ No newline at end of file diff --git a/smart-log-analyzer/correlator/kaoto-datamapper-8f5bb2dd.xsl b/smart-log-analyzer/correlator/kaoto-datamapper-8f5bb2dd.xsl new file mode 100644 index 0000000..c818b32 --- /dev/null +++ b/smart-log-analyzer/correlator/kaoto-datamapper-8f5bb2dd.xsl @@ -0,0 +1,64 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- This file is generated by Kaoto DataMapper. Do not edit. --> +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; version="3.0" xmlns:xs="http://www.w3.org/2001/XMLSchema"; xmlns:fn="http://www.w3.org/2005/xpath-functions";> + <xsl:output method="text" indent="yes"/> + <xsl:param name="traces"/> + <xsl:variable name="traces-x" select="json-to-xml($traces)"/> + <xsl:variable name="mapped-xml"> + <map xmlns="http://www.w3.org/2005/xpath-functions";> + <string key="type"> + <xsl:value-of select="'trace'"/> + </string> + <string key="timeUnixNano"> + <xsl:value-of select="$traces-x/fn:map/fn:string[@key='startTimeUnixNano']"/> + </string> + <string key="durationNano"> + <xsl:value-of select="xs:integer($traces-x/fn:map/fn:string[@key='endTimeUnixNano']) - xs:integer($traces-x/fn:map/fn:string[@key='startTimeUnixNano'])"/> + </string> + <string key="traceId"> + <xsl:value-of select="$traces-x/fn:map/fn:string[@key='traceId']"/> + </string> + <string key="spanId"> + <xsl:value-of select="$traces-x/fn:map/fn:string[@key='spanId']"/> + </string> + <xsl:if test="$traces-x/fn:map/fn:string[@key='parentSpanId']"> + <string key="parentSpanId"> + <xsl:value-of select="$traces-x/fn:map/fn:string[@key='parentSpanId']"/> + </string> + </xsl:if> + <string key="name"> + <xsl:value-of select="$traces-x/fn:map/fn:string[@key='name']"/> + </string> + <string key="kind"> + <xsl:value-of select="$traces-x/fn:map/fn:number[@key='kind']"/> + </string> + <string key="status"> + <xsl:value-of select="$traces-x/fn:map/fn:map[@key='status']/fn:number[@key='code']"/> + </string> + <string key="message"> + <xsl:for-each select="$traces-x/fn:map/fn:array[@key='attributes']/fn:map"> + <xsl:choose> + <xsl:when test="fn:map[@key='value']/fn:string[@key='stringValue']"> + <xsl:value-of select="concat(' ', fn:string[@key='key'], ': ', fn:map[@key='value']/fn:string[@key='stringValue'])"/> + </xsl:when> + <xsl:when test="fn:map[@key='value']/fn:string[@key='intValue']"> + <xsl:value-of select="concat(' ', fn:string[@key='key'], ': ', fn:map[@key='value']/fn:string[@key='intValue'])"/> + </xsl:when> + <xsl:when test="fn:map[@key='value']/fn:number[@key='intValue']"> + <xsl:value-of select="concat(' ', fn:string[@key='key'], ': ', fn:map[@key='value']/fn:number[@key='intValue'])"/> + </xsl:when> + </xsl:choose> + </xsl:for-each> + <xsl:for-each select="$traces-x/fn:map/fn:array[@key='events']/fn:map"> + <xsl:value-of select="concat(' [Event] ', fn:string[@key='name'])"/> + <xsl:for-each select="fn:array[@key='attributes']/fn:map"> + <xsl:value-of select="concat(' ', fn:string[@key='key'], ': ', fn:map[@key='value']/fn:string[@key='stringValue'])"/> + </xsl:for-each> + </xsl:for-each> + </string> + </map> + </xsl:variable> + <xsl:template match="/"> + <xsl:value-of select="xml-to-json($mapped-xml)"/> + </xsl:template> +</xsl:stylesheet> \ No newline at end of file diff --git a/smart-log-analyzer/correlator/logs-mapper.camel.yaml b/smart-log-analyzer/correlator/logs-mapper.camel.yaml new file mode 100644 index 0000000..f4f85df --- /dev/null +++ b/smart-log-analyzer/correlator/logs-mapper.camel.yaml @@ -0,0 +1,45 @@ +- route: + id: log-consumer + from: + uri: kafka:{{camel.kafka.topic.logs}} + parameters: + autoOffsetReset: earliest + groupId: correlator + steps: + - log: + loggingLevel: DEBUG + message: Received log from Kafka \n ${body} + - split: + id: split-logs + expression: + jsonpath: + expression: $.resourceLogs[*].scopeLogs[*].logRecords[*] + writeAsString: true + steps: + - setHeader: + id: set-logs-header + expression: + simple: + expression: ${body} + name: logs + - setBody: + id: set-dummy-xml + expression: + constant: + expression: <dummy/> + - step: + id: kaoto-datamapper-4a94acc3 + steps: + - to: + id: kaoto-datamapper-xslt + uri: xslt-saxon:kaoto-datamapper-4a94acc3.xsl + parameters: + allowStAX: true + - log: + loggingLevel: DEBUG + message: "Mapped output: ${body}" + - to: + id: to-2266 + uri: direct + parameters: + name: store diff --git a/smart-log-analyzer/correlator/otel-log-record-schema.json b/smart-log-analyzer/correlator/otel-log-record-schema.json new file mode 100644 index 0000000..afc34fa --- /dev/null +++ b/smart-log-analyzer/correlator/otel-log-record-schema.json @@ -0,0 +1,137 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema";, + "$id": "https://opentelemetry.io/schemas/logs/1.24.0/LogRecord";, + "title": "OpenTelemetry Log Record", + "description": "JSON Schema for a single OpenTelemetry log record (after split from OTLP batch)", + "type": "object", + "properties": { + "timeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Time when the event occurred (nanoseconds since Unix epoch)" + }, + "observedTimeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Time when the event was observed (nanoseconds since Unix epoch)" + }, + "severityNumber": { + "type": "integer", + "minimum": 0, + "maximum": 24, + "description": "Numerical severity value (1-24)" + }, + "severityText": { + "type": "string", + "description": "Severity text (e.g., TRACE, DEBUG, INFO, WARN, ERROR, FATAL)" + }, + "body": { + "$ref": "#/$defs/AnyValue", + "description": "The body of the log record" + }, + "attributes": { + "type": "array", + "description": "Additional attributes for the log record", + "items": { + "$ref": "#/$defs/KeyValue" + } + }, + "droppedAttributesCount": { + "type": "integer", + "description": "Number of attributes that were dropped due to limits" + }, + "flags": { + "type": "integer", + "description": "Flags for the log record" + }, + "traceId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{32}$", + "description": "Trace ID (32 hex characters)" + }, + "spanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Span ID (16 hex characters)" + } + }, + "$defs": { + "KeyValue": { + "type": "object", + "description": "A key-value pair", + "required": ["key", "value"], + "properties": { + "key": { + "type": "string", + "description": "The attribute key" + }, + "value": { + "$ref": "#/$defs/AnyValue", + "description": "The attribute value" + } + } + }, + "AnyValue": { + "type": "object", + "description": "A value of any type", + "properties": { + "stringValue": { + "type": "string" + }, + "boolValue": { + "type": "boolean" + }, + "intValue": { + "type": "string", + "pattern": "^-?[0-9]+$" + }, + "doubleValue": { + "type": "number" + }, + "arrayValue": { + "$ref": "#/$defs/ArrayValue" + }, + "kvlistValue": { + "$ref": "#/$defs/KeyValueList" + }, + "bytesValue": { + "type": "string", + "contentEncoding": "base64" + } + }, + "oneOf": [ + { "required": ["stringValue"] }, + { "required": ["boolValue"] }, + { "required": ["intValue"] }, + { "required": ["doubleValue"] }, + { "required": ["arrayValue"] }, + { "required": ["kvlistValue"] }, + { "required": ["bytesValue"] } + ] + }, + "ArrayValue": { + "type": "object", + "description": "An array of values", + "properties": { + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/AnyValue" + } + } + } + }, + "KeyValueList": { + "type": "object", + "description": "A list of key-value pairs", + "properties": { + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/KeyValue" + } + } + } + } + } +} diff --git a/smart-log-analyzer/correlator/otel-logs-schema.json b/smart-log-analyzer/correlator/otel-logs-schema.json new file mode 100644 index 0000000..ffa7674 --- /dev/null +++ b/smart-log-analyzer/correlator/otel-logs-schema.json @@ -0,0 +1,227 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema";, + "$id": "https://opentelemetry.io/schemas/logs/1.24.0";, + "title": "OpenTelemetry Log Data", + "description": "JSON Schema for OpenTelemetry Protocol (OTLP) log data", + "type": "object", + "required": ["resourceLogs"], + "properties": { + "resourceLogs": { + "type": "array", + "description": "Array of ResourceLogs messages", + "items": { + "$ref": "#/$defs/ResourceLogs" + } + } + }, + "$defs": { + "ResourceLogs": { + "type": "object", + "description": "A collection of ScopeLogs from a Resource", + "properties": { + "resource": { + "$ref": "#/$defs/Resource" + }, + "scopeLogs": { + "type": "array", + "description": "A list of ScopeLogs that originate from a resource", + "items": { + "$ref": "#/$defs/ScopeLogs" + } + }, + "schemaUrl": { + "type": "string", + "format": "uri", + "description": "Schema URL for the resource" + } + } + }, + "Resource": { + "type": "object", + "description": "Resource information", + "properties": { + "attributes": { + "type": "array", + "description": "Set of attributes that describe the resource", + "items": { + "$ref": "#/$defs/KeyValue" + } + } + } + }, + "ScopeLogs": { + "type": "object", + "description": "A collection of log records from an instrumentation scope", + "properties": { + "scope": { + "$ref": "#/$defs/InstrumentationScope" + }, + "logRecords": { + "type": "array", + "description": "A list of log records", + "items": { + "$ref": "#/$defs/LogRecord" + } + }, + "schemaUrl": { + "type": "string", + "format": "uri", + "description": "Schema URL for the scope" + } + } + }, + "InstrumentationScope": { + "type": "object", + "description": "Instrumentation scope information", + "properties": { + "name": { + "type": "string", + "description": "Name of the instrumentation scope" + }, + "version": { + "type": "string", + "description": "Version of the instrumentation scope" + }, + "attributes": { + "type": "array", + "description": "Attributes of the instrumentation scope", + "items": { + "$ref": "#/$defs/KeyValue" + } + } + } + }, + "LogRecord": { + "type": "object", + "description": "A log record", + "properties": { + "timeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Time when the event occurred (nanoseconds since Unix epoch)" + }, + "observedTimeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Time when the event was observed (nanoseconds since Unix epoch)" + }, + "severityNumber": { + "type": "integer", + "minimum": 0, + "maximum": 24, + "description": "Numerical severity value (1-24)" + }, + "severityText": { + "type": "string", + "description": "Severity text (e.g., TRACE, DEBUG, INFO, WARN, ERROR, FATAL)" + }, + "body": { + "$ref": "#/$defs/AnyValue", + "description": "The body of the log record" + }, + "attributes": { + "type": "array", + "description": "Additional attributes for the log record", + "items": { + "$ref": "#/$defs/KeyValue" + } + }, + "droppedAttributesCount": { + "type": "integer", + "description": "Number of attributes that were dropped due to limits" + }, + "flags": { + "type": "integer", + "description": "Flags for the log record" + }, + "traceId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{32}$", + "description": "Trace ID (32 hex characters)" + }, + "spanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Span ID (16 hex characters)" + } + } + }, + "KeyValue": { + "type": "object", + "description": "A key-value pair", + "required": ["key", "value"], + "properties": { + "key": { + "type": "string", + "description": "The attribute key" + }, + "value": { + "$ref": "#/$defs/AnyValue", + "description": "The attribute value" + } + } + }, + "AnyValue": { + "type": "object", + "description": "A value of any type", + "properties": { + "stringValue": { + "type": "string" + }, + "boolValue": { + "type": "boolean" + }, + "intValue": { + "type": "string", + "pattern": "^-?[0-9]+$" + }, + "doubleValue": { + "type": "number" + }, + "arrayValue": { + "$ref": "#/$defs/ArrayValue" + }, + "kvlistValue": { + "$ref": "#/$defs/KeyValueList" + }, + "bytesValue": { + "type": "string", + "contentEncoding": "base64" + } + }, + "oneOf": [ + { "required": ["stringValue"] }, + { "required": ["boolValue"] }, + { "required": ["intValue"] }, + { "required": ["doubleValue"] }, + { "required": ["arrayValue"] }, + { "required": ["kvlistValue"] }, + { "required": ["bytesValue"] } + ] + }, + "ArrayValue": { + "type": "object", + "description": "An array of values", + "properties": { + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/AnyValue" + } + } + } + }, + "KeyValueList": { + "type": "object", + "description": "A list of key-value pairs", + "properties": { + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/KeyValue" + } + } + } + } + } +} diff --git a/smart-log-analyzer/correlator/otel-span-schema.json b/smart-log-analyzer/correlator/otel-span-schema.json new file mode 100644 index 0000000..9b12c69 --- /dev/null +++ b/smart-log-analyzer/correlator/otel-span-schema.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema";, + "$id": "https://opentelemetry.io/schemas/traces/1.24.0/Span";, + "title": "OpenTelemetry Span", + "description": "JSON Schema for a single OpenTelemetry span (after split from OTLP batch)", + "type": "object", + "properties": { + "traceId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{32}$", + "description": "Trace ID (32 hex characters)" + }, + "spanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Span ID (16 hex characters)" + }, + "traceState": { + "type": "string", + "description": "W3C trace state" + }, + "parentSpanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Parent span ID (16 hex characters)" + }, + "flags": { + "type": "integer", + "description": "Trace flags (W3C format)" + }, + "name": { + "type": "string", + "description": "Name of the span" + }, + "kind": { + "type": "integer", + "minimum": 0, + "maximum": 5, + "description": "Span kind (0=UNSPECIFIED, 1=INTERNAL, 2=SERVER, 3=CLIENT, 4=PRODUCER, 5=CONSUMER)" + }, + "startTimeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Start time (nanoseconds since Unix epoch)" + }, + "endTimeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "End time (nanoseconds since Unix epoch)" + }, + "attributes": { + "type": "array", + "description": "Attributes for the span", + "items": { + "$ref": "#/$defs/KeyValue" + } + }, + "droppedAttributesCount": { + "type": "integer", + "description": "Number of attributes that were dropped due to limits" + }, + "events": { + "type": "array", + "description": "Timed events within the span", + "items": { + "$ref": "#/$defs/SpanEvent" + } + }, + "droppedEventsCount": { + "type": "integer", + "description": "Number of events that were dropped due to limits" + }, + "links": { + "type": "array", + "description": "Links to other spans", + "items": { + "$ref": "#/$defs/SpanLink" + } + }, + "droppedLinksCount": { + "type": "integer", + "description": "Number of links that were dropped due to limits" + }, + "status": { + "$ref": "#/$defs/Status" + } + }, + "$defs": { + "SpanEvent": { + "type": "object", + "description": "A timed event within a span", + "properties": { + "timeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Time of the event (nanoseconds since Unix epoch)" + }, + "name": { + "type": "string", + "description": "Name of the event" + }, + "attributes": { + "type": "array", + "description": "Attributes for the event", + "items": { + "$ref": "#/$defs/KeyValue" + } + }, + "droppedAttributesCount": { + "type": "integer", + "description": "Number of attributes that were dropped due to limits" + } + } + }, + "SpanLink": { + "type": "object", + "description": "A link to another span", + "properties": { + "traceId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{32}$", + "description": "Trace ID of the linked span" + }, + "spanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Span ID of the linked span" + }, + "traceState": { + "type": "string", + "description": "W3C trace state" + }, + "attributes": { + "type": "array", + "description": "Attributes for the link", + "items": { + "$ref": "#/$defs/KeyValue" + } + }, + "droppedAttributesCount": { + "type": "integer", + "description": "Number of attributes that were dropped due to limits" + }, + "flags": { + "type": "integer", + "description": "Trace flags" + } + } + }, + "Status": { + "type": "object", + "description": "Status of the span", + "properties": { + "message": { + "type": "string", + "description": "Human-readable status message" + }, + "code": { + "type": "integer", + "minimum": 0, + "maximum": 2, + "description": "Status code (0=UNSET, 1=OK, 2=ERROR)" + } + } + }, + "KeyValue": { + "type": "object", + "description": "A key-value pair", + "required": ["key", "value"], + "properties": { + "key": { + "type": "string", + "description": "The attribute key" + }, + "value": { + "$ref": "#/$defs/AnyValue", + "description": "The attribute value" + } + } + }, + "AnyValue": { + "type": "object", + "description": "A value of any type", + "properties": { + "stringValue": { + "type": "string" + }, + "boolValue": { + "type": "boolean" + }, + "intValue": { + "type": "string", + "pattern": "^-?[0-9]+$" + }, + "doubleValue": { + "type": "number" + }, + "arrayValue": { + "$ref": "#/$defs/ArrayValue" + }, + "kvlistValue": { + "$ref": "#/$defs/KeyValueList" + }, + "bytesValue": { + "type": "string", + "contentEncoding": "base64" + } + }, + "oneOf": [ + { "required": ["stringValue"] }, + { "required": ["boolValue"] }, + { "required": ["intValue"] }, + { "required": ["doubleValue"] }, + { "required": ["arrayValue"] }, + { "required": ["kvlistValue"] }, + { "required": ["bytesValue"] } + ] + }, + "ArrayValue": { + "type": "object", + "description": "An array of values", + "properties": { + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/AnyValue" + } + } + } + }, + "KeyValueList": { + "type": "object", + "description": "A list of key-value pairs", + "properties": { + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/KeyValue" + } + } + } + } + } +} diff --git a/smart-log-analyzer/correlator/otel-traces-schema.json b/smart-log-analyzer/correlator/otel-traces-schema.json new file mode 100644 index 0000000..6d05e12 --- /dev/null +++ b/smart-log-analyzer/correlator/otel-traces-schema.json @@ -0,0 +1,334 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema";, + "$id": "https://opentelemetry.io/schemas/traces/1.24.0";, + "title": "OpenTelemetry Trace Data", + "description": "JSON Schema for OpenTelemetry Protocol (OTLP) trace data", + "type": "object", + "required": ["resourceSpans"], + "properties": { + "resourceSpans": { + "type": "array", + "description": "Array of ResourceSpans messages", + "items": { + "$ref": "#/$defs/ResourceSpans" + } + } + }, + "$defs": { + "ResourceSpans": { + "type": "object", + "description": "A collection of ScopeSpans from a Resource", + "properties": { + "resource": { + "$ref": "#/$defs/Resource" + }, + "scopeSpans": { + "type": "array", + "description": "A list of ScopeSpans that originate from a resource", + "items": { + "$ref": "#/$defs/ScopeSpans" + } + }, + "schemaUrl": { + "type": "string", + "format": "uri", + "description": "Schema URL for the resource" + } + } + }, + "Resource": { + "type": "object", + "description": "Resource information", + "properties": { + "attributes": { + "type": "array", + "description": "Set of attributes that describe the resource", + "items": { + "$ref": "#/$defs/KeyValue" + } + } + } + }, + "ScopeSpans": { + "type": "object", + "description": "A collection of spans from an instrumentation scope", + "properties": { + "scope": { + "$ref": "#/$defs/InstrumentationScope" + }, + "spans": { + "type": "array", + "description": "A list of spans", + "items": { + "$ref": "#/$defs/Span" + } + }, + "schemaUrl": { + "type": "string", + "format": "uri", + "description": "Schema URL for the scope" + } + } + }, + "InstrumentationScope": { + "type": "object", + "description": "Instrumentation scope information", + "properties": { + "name": { + "type": "string", + "description": "Name of the instrumentation scope" + }, + "version": { + "type": "string", + "description": "Version of the instrumentation scope" + }, + "attributes": { + "type": "array", + "description": "Attributes of the instrumentation scope", + "items": { + "$ref": "#/$defs/KeyValue" + } + } + } + }, + "Span": { + "type": "object", + "description": "A span representing a trace segment", + "properties": { + "traceId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{32}$", + "description": "Trace ID (32 hex characters)" + }, + "spanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Span ID (16 hex characters)" + }, + "traceState": { + "type": "string", + "description": "W3C trace state" + }, + "parentSpanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Parent span ID (16 hex characters)" + }, + "flags": { + "type": "integer", + "description": "Trace flags (W3C format)" + }, + "name": { + "type": "string", + "description": "Name of the span" + }, + "kind": { + "type": "integer", + "minimum": 0, + "maximum": 5, + "description": "Span kind (0=UNSPECIFIED, 1=INTERNAL, 2=SERVER, 3=CLIENT, 4=PRODUCER, 5=CONSUMER)" + }, + "startTimeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Start time (nanoseconds since Unix epoch)" + }, + "endTimeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "End time (nanoseconds since Unix epoch)" + }, + "attributes": { + "type": "array", + "description": "Attributes for the span", + "items": { + "$ref": "#/$defs/KeyValue" + } + }, + "droppedAttributesCount": { + "type": "integer", + "description": "Number of attributes that were dropped due to limits" + }, + "events": { + "type": "array", + "description": "Timed events within the span", + "items": { + "$ref": "#/$defs/SpanEvent" + } + }, + "droppedEventsCount": { + "type": "integer", + "description": "Number of events that were dropped due to limits" + }, + "links": { + "type": "array", + "description": "Links to other spans", + "items": { + "$ref": "#/$defs/SpanLink" + } + }, + "droppedLinksCount": { + "type": "integer", + "description": "Number of links that were dropped due to limits" + }, + "status": { + "$ref": "#/$defs/Status" + } + } + }, + "SpanEvent": { + "type": "object", + "description": "A timed event within a span", + "properties": { + "timeUnixNano": { + "type": "string", + "pattern": "^[0-9]+$", + "description": "Time of the event (nanoseconds since Unix epoch)" + }, + "name": { + "type": "string", + "description": "Name of the event" + }, + "attributes": { + "type": "array", + "description": "Attributes for the event", + "items": { + "$ref": "#/$defs/KeyValue" + } + }, + "droppedAttributesCount": { + "type": "integer", + "description": "Number of attributes that were dropped due to limits" + } + } + }, + "SpanLink": { + "type": "object", + "description": "A link to another span", + "properties": { + "traceId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{32}$", + "description": "Trace ID of the linked span" + }, + "spanId": { + "type": "string", + "pattern": "^[a-fA-F0-9]{16}$", + "description": "Span ID of the linked span" + }, + "traceState": { + "type": "string", + "description": "W3C trace state" + }, + "attributes": { + "type": "array", + "description": "Attributes for the link", + "items": { + "$ref": "#/$defs/KeyValue" + } + }, + "droppedAttributesCount": { + "type": "integer", + "description": "Number of attributes that were dropped due to limits" + }, + "flags": { + "type": "integer", + "description": "Trace flags" + } + } + }, + "Status": { + "type": "object", + "description": "Status of the span", + "properties": { + "message": { + "type": "string", + "description": "Human-readable status message" + }, + "code": { + "type": "integer", + "minimum": 0, + "maximum": 2, + "description": "Status code (0=UNSET, 1=OK, 2=ERROR)" + } + } + }, + "KeyValue": { + "type": "object", + "description": "A key-value pair", + "required": ["key", "value"], + "properties": { + "key": { + "type": "string", + "description": "The attribute key" + }, + "value": { + "$ref": "#/$defs/AnyValue", + "description": "The attribute value" + } + } + }, + "AnyValue": { + "type": "object", + "description": "A value of any type", + "properties": { + "stringValue": { + "type": "string" + }, + "boolValue": { + "type": "boolean" + }, + "intValue": { + "type": "string", + "pattern": "^-?[0-9]+$" + }, + "doubleValue": { + "type": "number" + }, + "arrayValue": { + "$ref": "#/$defs/ArrayValue" + }, + "kvlistValue": { + "$ref": "#/$defs/KeyValueList" + }, + "bytesValue": { + "type": "string", + "contentEncoding": "base64" + } + }, + "oneOf": [ + { "required": ["stringValue"] }, + { "required": ["boolValue"] }, + { "required": ["intValue"] }, + { "required": ["doubleValue"] }, + { "required": ["arrayValue"] }, + { "required": ["kvlistValue"] }, + { "required": ["bytesValue"] } + ] + }, + "ArrayValue": { + "type": "object", + "description": "An array of values", + "properties": { + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/AnyValue" + } + } + } + }, + "KeyValueList": { + "type": "object", + "description": "A list of key-value pairs", + "properties": { + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/KeyValue" + } + } + } + } + } +} diff --git a/smart-log-analyzer/correlator/traces-mapper.camel.yaml b/smart-log-analyzer/correlator/traces-mapper.camel.yaml new file mode 100644 index 0000000..c75721a --- /dev/null +++ b/smart-log-analyzer/correlator/traces-mapper.camel.yaml @@ -0,0 +1,45 @@ +- route: + id: trace-consumer + from: + uri: kafka:{{camel.kafka.topic.spans}} + parameters: + autoOffsetReset: earliest + groupId: correlator + steps: + - log: + loggingLevel: DEBUG + message: Received trace from Kafka + - split: + id: split-spans + expression: + jsonpath: + expression: $.resourceSpans[*].scopeSpans[*].spans[*] + writeAsString: true + steps: + - setHeader: + id: set-traces-header + expression: + simple: + expression: ${body} + name: traces + - setBody: + id: set-dummy-xml-1 + expression: + constant: + expression: <dummy/> + - step: + id: kaoto-datamapper-8f5bb2dd + steps: + - to: + id: kaoto-datamapper-xslt-1526 + uri: xslt-saxon:kaoto-datamapper-8f5bb2dd.xsl + parameters: + allowStAX: true + - log: + loggingLevel: DEBUG + message: "Mapped output: ${body}" + - to: + id: to-22661 + uri: direct + parameters: + name: store diff --git a/smart-log-analyzer/first-iteration/analyzer.camel.yaml b/smart-log-analyzer/first-iteration/analyzer.camel.yaml new file mode 100644 index 0000000..01cc80d --- /dev/null +++ b/smart-log-analyzer/first-iteration/analyzer.camel.yaml @@ -0,0 +1,62 @@ +# Consumes log events from Kafka, aggregates by traceId, and sends +# to LLM for analysis when an ERROR level event is detected +- route: + id: log-event-consumer + from: + uri: kafka:log-events + parameters: + autoOffsetReset: earliest + brokers: localhost:9092 + groupId: log-analyzer + steps: + - log: + loggingLevel: DEBUG + message: "Received event from Kafka: ${body}" + - setVariable: + name: traceId + jq: + expression: .traceId + resultType: java.lang.String + - setVariable: + name: level + jq: + expression: .level + resultType: java.lang.String + - aggregate: + correlationExpression: + simple: + expression: ${variable.traceId} + aggregationStrategy: "#stringAggregationStrategy" + completionPredicate: + simple: + expression: ${variable.level} == 'ERROR' + completionTimeout: 30000 + steps: + - log: + message: "Aggregation completed for traceId: ${variable.traceId} - sending to LLM" + - setVariable: + name: eventsJson + simple: ${body} + - setBody: + simple: > + Analyze the following log events for traceId ${variable.traceId}. + + An ERROR was detected. Identify the root cause, explain what + happened in the sequence of events, and suggest possible fixes. + + Events: ${variable.eventsJson} + - to: + uri: openai:chat-completion + parameters: + apiKey: not-needed + baseUrl: http://localhost:11434/v1 + model: granite4:3b + systemMessage: > + Analyze log events to identify errors, their root causes, and + provide actionable recommendations. Be concise and focus + on the error chain and sequence of events. + - log: + message: "LLM Analysis for traceId ${variable.traceId}: ${body}" +- beans: + - name: stringAggregationStrategy + type: org.apache.camel.processor.aggregate.StringAggregationStrategy \ No newline at end of file diff --git a/smart-log-analyzer/first-iteration/application.properties b/smart-log-analyzer/first-iteration/application.properties new file mode 100644 index 0000000..caa88c5 --- /dev/null +++ b/smart-log-analyzer/first-iteration/application.properties @@ -0,0 +1,2 @@ +# Dependencies for the first-iteration analyzer +camel.jbang.dependencies=org.apache.camel:camel-openai diff --git a/smart-log-analyzer/first-iteration/load-generator.camel.yaml b/smart-log-analyzer/first-iteration/load-generator.camel.yaml new file mode 100644 index 0000000..3a8f248 --- /dev/null +++ b/smart-log-analyzer/first-iteration/load-generator.camel.yaml @@ -0,0 +1,48 @@ +- route: + id: event-generator + from: + uri: timer:eventGenerator + parameters: + period: 5000 + steps: + # Generate a new traceId for each timer event + - setVariable: + name: traceId + simple: "${random(10000,99999)}-${date:now:yyyyMMddHHmmssSSS}" + + # Send first INFO event + - setBody: + simple: | + {"traceId": "${variable.traceId}", "level": "INFO", "message": "Application started processing request", "timestamp": "${date:now:yyyy-MM-dd'T'HH:mm:ss.SSSZ}"} + - to: + uri: kafka:log-events + parameters: + brokers: localhost:9092 + + # Send second INFO event + - setBody: + simple: | + {"traceId": "${variable.traceId}", "level": "INFO", "message": "Request validation completed successfully", "timestamp": "${date:now:yyyy-MM-dd'T'HH:mm:ss.SSSZ}"} + - to: + uri: kafka:log-events + parameters: + brokers: localhost:9092 + + # 30% chance of generating an ERROR event + - choice: + when: + - simple: "${random(1,10)} <= 3" + steps: + - setBody: + simple: | + {"traceId": "${variable.traceId}", "level": "ERROR", "message": "Unexpected error occurred during processing", "timestamp": "${date:now:yyyy-MM-dd'T'HH:mm:ss.SSSZ}"} + - to: + uri: kafka:log-events + parameters: + brokers: localhost:9092 + - log: + message: "ERROR event sent for traceId: ${variable.traceId}" + otherwise: + steps: + - log: + message: "No error generated for traceId: ${variable.traceId}" \ No newline at end of file diff --git a/smart-log-analyzer/first-iteration/test/first-iteration.camel.it.yaml b/smart-log-analyzer/first-iteration/test/first-iteration.camel.it.yaml new file mode 100644 index 0000000..900b164 --- /dev/null +++ b/smart-log-analyzer/first-iteration/test/first-iteration.camel.it.yaml @@ -0,0 +1,46 @@ +name: first-iteration-test +description: Integration test for the first-iteration log analyzer prototype +variables: + - name: kafka.broker + value: localhost:9092 + - name: kafka.topic + value: log-events +actions: + - camel: + infra: + run: + service: kafka + - camel: + jbang: + run: + stub: + - "openai:*" + integration: + name: "log-analyzer" + file: "../analyzer.camel.yaml" + systemProperties: + file: "../application.properties" + - send: + endpoint: | + camel:kafka:${kafka.topic}?brokers=${kafka.broker} + message: + body: + resource: + file: "payload-info.json" + - send: + endpoint: | + camel:kafka:${kafka.topic}?brokers=${kafka.broker} + message: + body: + resource: + file: "payload-error.json" + - camel: + jbang: + verify: + integration: "log-analyzer" + logMessage: "Aggregation completed for traceId" + - camel: + jbang: + verify: + integration: "log-analyzer" + logMessage: "LLM Analysis for traceId test-12345" diff --git a/smart-log-analyzer/first-iteration/test/jbang.properties b/smart-log-analyzer/first-iteration/test/jbang.properties new file mode 100644 index 0000000..8be759f --- /dev/null +++ b/smart-log-analyzer/first-iteration/test/jbang.properties @@ -0,0 +1,3 @@ +run.deps=org.citrusframework:citrus-kafka:4.9.1,\ + org.apache.camel:camel-test-infra-kafka:4.17.0,\ + org.apache.camel:camel-kafka:4.17.0 diff --git a/smart-log-analyzer/first-iteration/test/payload-error.json b/smart-log-analyzer/first-iteration/test/payload-error.json new file mode 100644 index 0000000..2a7dd86 --- /dev/null +++ b/smart-log-analyzer/first-iteration/test/payload-error.json @@ -0,0 +1 @@ +{"traceId": "test-12345", "level": "ERROR", "message": "Unexpected error occurred during processing", "timestamp": "2026-01-28T10:00:01.000+0000"} diff --git a/smart-log-analyzer/first-iteration/test/payload-info.json b/smart-log-analyzer/first-iteration/test/payload-info.json new file mode 100644 index 0000000..c2604b1 --- /dev/null +++ b/smart-log-analyzer/first-iteration/test/payload-info.json @@ -0,0 +1 @@ +{"traceId": "test-12345", "level": "INFO", "message": "Application started processing request", "timestamp": "2026-01-28T10:00:00.000+0000"} diff --git a/smart-log-analyzer/log-generator/agent.properties b/smart-log-analyzer/log-generator/agent.properties new file mode 100644 index 0000000..6019ebc --- /dev/null +++ b/smart-log-analyzer/log-generator/agent.properties @@ -0,0 +1,6 @@ +otel.javaagent.debug=false + +otel.service.name=log-generator +otel.traces.exporter=otlp +otel.metrics.exporter=none +otel.logs.exporter=otlp diff --git a/smart-log-analyzer/log-generator/application-dev.properties b/smart-log-analyzer/log-generator/application-dev.properties new file mode 100644 index 0000000..0950ceb --- /dev/null +++ b/smart-log-analyzer/log-generator/application-dev.properties @@ -0,0 +1,3 @@ +camel.opentelemetry2.enabled = true + +camel.jbang.dependencies=org.apache.camel:camel-opentelemetry2 diff --git a/smart-log-analyzer/log-generator/log-generator.camel.yaml b/smart-log-analyzer/log-generator/log-generator.camel.yaml new file mode 100644 index 0000000..dcd316f --- /dev/null +++ b/smart-log-analyzer/log-generator/log-generator.camel.yaml @@ -0,0 +1,87 @@ +# Simulates order processing with random success/failure outcomes (30% failure rate) +# to generate realistic log data for testing the error analyzer +- route: + id: order-processor + from: + uri: timer:orderProcessor + parameters: + period: "{{timer.order.period:2000}}" + fixedRate: true + steps: + + # Generate random order ID + - setVariable: + name: orderId + simple: "ORD-${random(10000,99999)}" + + # Log: Order received + - log: + message: "Received new order ${variable.orderId} from customer ${random(1000,9999)}" + loggingLevel: INFO + logName: com.example.order.OrderService + + # Log: Validation + - log: + message: "Validating order ${variable.orderId}: checking inventory and pricing" + loggingLevel: INFO + logName: com.example.order.ValidationService + + # Simulate random processing outcomes (30% failure rate) + - choice: + when: + - simple: "${random(0,100)} < 30" + steps: + # Simulate different error types + - choice: + when: + - simple: "${random(0,100)} < 33" + steps: + - log: + message: "Connecting to database for order ${variable.orderId}" + loggingLevel: INFO + logName: com.example.order.DatabaseService + - throwException: + message: "Database connection failed: Connection refused to postgres:5432 while processing order ${variable.orderId}" + exceptionType: java.lang.RuntimeException + - simple: "${random(0,100)} < 66" + steps: + - log: + message: "Initiating payment for order ${variable.orderId}" + loggingLevel: INFO + logName: com.example.payment.PaymentGateway + - throwException: + message: "Network timeout while connecting to payment gateway api.stripe.com for order ${variable.orderId}" + exceptionType: java.lang.RuntimeException + otherwise: + steps: + - log: + message: "Verifying user authentication for order ${variable.orderId}" + loggingLevel: INFO + logName: com.example.auth.AuthService + - throwException: + message: "Authentication failed: Invalid JWT token signature for order ${variable.orderId}" + exceptionType: java.lang.RuntimeException + otherwise: + steps: + - log: + message: "Payment processed successfully for order ${variable.orderId}" + loggingLevel: INFO + logName: com.example.payment.PaymentGateway + - log: + message: "Order ${variable.orderId} completed successfully" + loggingLevel: INFO + logName: com.example.order.OrderService + +# Periodic health check logging for system monitoring +- route: + id: health-checker + from: + uri: timer:healthCheck + parameters: + period: 5000 + fixedRate: true + steps: + - log: + message: "Health check: All systems operational" + loggingLevel: DEBUG + logName: com.example.health.HealthChecker diff --git a/smart-log-analyzer/log-generator/opentelemetry-javaagent.jar b/smart-log-analyzer/log-generator/opentelemetry-javaagent.jar new file mode 100644 index 0000000..926195a Binary files /dev/null and b/smart-log-analyzer/log-generator/opentelemetry-javaagent.jar differ diff --git a/smart-log-analyzer/ui-console/application-dev.properties b/smart-log-analyzer/ui-console/application-dev.properties new file mode 100644 index 0000000..7f32b42 --- /dev/null +++ b/smart-log-analyzer/ui-console/application-dev.properties @@ -0,0 +1,31 @@ +# Storage configuration +analyzer.storage.root=/tmp/analyzer-store + +# JMS Queue configuration +camel.jms.queue.analysis-result=analysis-result + +# Artemis JMS configuration +camel.beans.artemisCF = #class:org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory +camel.beans.artemisCF.brokerURL = tcp://localhost:61616 +camel.beans.artemisCF.user = artemis +camel.beans.artemisCF.password = artemis +camel.beans.poolCF = #class:org.messaginghub.pooled.jms.JmsPoolConnectionFactory +camel.beans.poolCF.connectionFactory = #bean:artemisCF +camel.beans.poolCF.maxSessionsPerConnection = 500 +camel.beans.poolCF.connectionIdleTimeout = 20000 +camel.component.jms.connection-factory = #bean:poolCF + +# REST configuration +camel.rest.component=platform-http +camel.rest.binding-mode=off +camel.rest.port=8080 +camel.rest.host=0.0.0.0 + +# Camel configuration +camel.main.name=ui-console + +camel.server.enabled=true +camel.server.staticEnabled=true + +# Dependencies +camel.jbang.dependencies=org.apache.camel:camel-groovy diff --git a/smart-log-analyzer/ui-console/index.html b/smart-log-analyzer/ui-console/index.html new file mode 100644 index 0000000..3474b82 --- /dev/null +++ b/smart-log-analyzer/ui-console/index.html @@ -0,0 +1,287 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>Smart Log Analyzer - Console</title> + <link rel="stylesheet" href="https://unpkg.com/[email protected]/css/carbon-components.min.css";> + <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;600&family=IBM+Plex+Sans:wght@400;600&display=swap";> + <style> + body { + margin: 0; + padding: 0; + font-family: 'IBM Plex Sans', 'Helvetica Neue', Arial, sans-serif; + } + + .bx--header { + position: fixed; + top: 0; + left: 0; + right: 0; + z-index: 8000; + } + + .camel-logo { + width: 32px; + height: 32px; + margin-right: 0.75rem; + vertical-align: middle; + } + + .container { + display: flex; + height: 100vh; + padding-top: 48px; + } + + .sidebar { + width: 320px; + background-color: #161616; + color: #f4f4f4; + padding: 1rem; + overflow-y: auto; + border-right: 1px solid #393939; + } + + .main-content { + flex: 1; + padding: 2rem; + overflow-y: auto; + background-color: #f4f4f4; + } + + .trace-list { + list-style: none; + padding: 0; + margin-top: 1rem; + } + + .trace-item { + padding: 0.75rem 1rem; + margin-bottom: 0.5rem; + background-color: #262626; + cursor: pointer; + transition: background-color 0.11s; + border-left: 3px solid transparent; + } + + .trace-item:hover { + background-color: #333333; + } + + .trace-item.active { + background-color: #0f62fe; + border-left-color: #78a9ff; + } + + .trace-id { + font-size: 0.875rem; + word-break: break-all; + font-family: 'IBM Plex Mono', monospace; + } + + .content-body { + background-color: #ffffff; + padding: 1rem; + white-space: pre-wrap; + word-wrap: break-word; + font-family: 'IBM Plex Mono', monospace; + font-size: 0.875rem; + line-height: 1.6; + max-height: calc(100vh - 250px); + overflow-y: auto; + border: 1px solid #e0e0e0; + } + + .empty-state { + text-align: center; + color: #525252; + padding: 4rem 2rem; + } + + .empty-state-icon { + width: 80px; + height: 80px; + margin: 0 auto 1rem; + display: block; + } + + .count-badge { + display: inline-block; + background-color: #0f62fe; + color: white; + padding: 0.25rem 0.75rem; + border-radius: 1rem; + font-size: 0.75rem; + margin-left: 0.5rem; + margin-bottom: 1rem; + } + + .sidebar-header { + margin-bottom: 1rem; + padding-bottom: 1rem; + border-bottom: 1px solid #393939; + } + + .sidebar-title { + font-size: 1.25rem; + font-weight: 600; + margin-bottom: 0.5rem; + } + </style> +</head> +<body class="bx--body"> + <header class="bx--header" role="banner"> + <a class="bx--header__name" href="javascript:void(0)"> + <img src="https://raw.githubusercontent.com/apache/camel/refs/heads/main/docs/img/logo-d.svg"; alt="Apache Camel" class="camel-logo"> + <span class="bx--header__name--prefix">Smart Log</span> + Analyzer + </a> + </header> + + <div class="container"> + <div class="sidebar"> + <div class="sidebar-header"> + <div class="sidebar-title">Trace Analysis</div> + <div id="traceCount"></div> + </div> + <div class="bx--form-item"> + <input + type="text" + class="bx--text-input bx--text-input--light" + id="searchBox" + placeholder="Search trace IDs..."> + </div> + <ul class="trace-list" id="traceList"></ul> + </div> + <div class="main-content"> + <div id="contentPanel"></div> + </div> + </div> + + <script> + const API_BASE = '/api/traces'; + let allTraces = []; + let selectedTraceId = null; + + async function loadTraces() { + try { + const response = await fetch(API_BASE); + if (!response.ok) throw new Error('Failed to fetch traces'); + const data = await response.json(); + allTraces = data || []; + renderTraceList(allTraces); + } catch (error) { + console.error('Error loading traces:', error); + document.getElementById('traceList').innerHTML = ` + <div class="bx--inline-notification bx--inline-notification--error" role="alert" style="margin-top: 1rem;"> + <div class="bx--inline-notification__details"> + <div class="bx--inline-notification__text-wrapper"> + <p class="bx--inline-notification__subtitle">Failed to load traces. Make sure the API server is running.</p> + </div> + </div> + </div> + `; + } + } + + function renderTraceList(traces) { + const traceList = document.getElementById('traceList'); + const traceCount = document.getElementById('traceCount'); + + traceCount.innerHTML = `<span class="count-badge">${traces.length} trace${traces.length !== 1 ? 's' : ''}</span>`; + + if (traces.length === 0) { + traceList.innerHTML = '<div style="padding: 1rem; color: #525252; text-align: center;">No traces found</div>'; + return; + } + + traceList.innerHTML = traces.map(trace => ` + <li class="trace-item ${trace.id === selectedTraceId ? 'active' : ''}" + onclick="loadTraceContent('${trace.id}')"> + <div class="trace-id">${escapeHtml(trace.id)}</div> + </li> + `).join(''); + } + + async function loadTraceContent(traceId) { + selectedTraceId = traceId; + renderTraceList(filterTraces(document.getElementById('searchBox').value)); + + const contentPanel = document.getElementById('contentPanel'); + contentPanel.innerHTML = ` + <div class="bx--loading bx--loading--small"> + <svg class="bx--loading__svg" viewBox="0 0 100 100"> + <circle class="bx--loading__stroke" cx="50%" cy="50%" r="44"></circle> + </svg> + </div> + `; + + try { + const response = await fetch(`${API_BASE}/${traceId}`); + if (!response.ok) throw new Error('Trace not found'); + const data = await response.json(); + + contentPanel.innerHTML = ` + <div class="bx--tile" style="padding: 0;"> + <div style="display: flex; justify-content: space-between; align-items: center; padding: 1.5rem; border-bottom: 1px solid #e0e0e0;"> + <h3 class="bx--type-productive-heading-03">Trace: ${escapeHtml(data.id)}</h3> + <button class="bx--btn bx--btn--primary bx--btn--sm" onclick="loadTraceContent('${escapeHtml(traceId)}')"> + Refresh + </button> + </div> + <div style="padding: 1.5rem;"> + <div class="content-body">${escapeHtml(data.content)}</div> + </div> + </div> + `; + } catch (error) { + console.error('Error loading trace content:', error); + contentPanel.innerHTML = ` + <div class="bx--inline-notification bx--inline-notification--error" role="alert"> + <div class="bx--inline-notification__details"> + <div class="bx--inline-notification__text-wrapper"> + <p class="bx--inline-notification__title">Error</p> + <p class="bx--inline-notification__subtitle">Failed to load trace content</p> + </div> + </div> + </div> + `; + } + } + + function filterTraces(searchTerm) { + if (!searchTerm) return allTraces; + const term = searchTerm.toLowerCase(); + return allTraces.filter(trace => + trace.id.toLowerCase().includes(term) + ); + } + + function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + } + + document.getElementById('searchBox').addEventListener('input', (e) => { + const filtered = filterTraces(e.target.value); + renderTraceList(filtered); + }); + + // Initial load + document.getElementById('contentPanel').innerHTML = ` + <div class="empty-state"> + <img src="https://raw.githubusercontent.com/apache/camel/refs/heads/main/docs/img/logo-d.svg"; alt="Apache Camel" class="empty-state-icon"> + <h3 class="bx--type-productive-heading-03" style="margin-bottom: 0.5rem;">Welcome to Smart Log Analyzer</h3> + <p class="bx--type-body-short-01">Select a trace from the sidebar to view its analysis</p> + </div> + `; + + loadTraces(); + + // Auto-refresh every 15 seconds + setInterval(loadTraces, 15000); + </script> +</body> +</html> diff --git a/smart-log-analyzer/ui-console/jms-file-storage.camel.yaml b/smart-log-analyzer/ui-console/jms-file-storage.camel.yaml new file mode 100644 index 0000000..b426959 --- /dev/null +++ b/smart-log-analyzer/ui-console/jms-file-storage.camel.yaml @@ -0,0 +1,14 @@ +- route: + id: jms-to-file-storage + from: + uri: jms:{{camel.jms.queue.analysis-result}} + steps: + - log: + message: "Received message from JMS queue: ${body}" + - setHeader: + name: CamelFileName + simple: ${header.traceId}.txt + - to: + uri: file:{{analyzer.storage.root}} + - log: + message: "Saved content for traceId: ${header.traceId}" diff --git a/smart-log-analyzer/ui-console/rest-api.camel.yaml b/smart-log-analyzer/ui-console/rest-api.camel.yaml new file mode 100644 index 0000000..e166719 --- /dev/null +++ b/smart-log-analyzer/ui-console/rest-api.camel.yaml @@ -0,0 +1,84 @@ +- rest: + path: /api/traces + get: + - id: list-traces + produces: application/json + to: direct:list-traces + - id: get-trace-by-id + path: /{id} + produces: application/json + to: direct:get-trace + +- route: + id: route-list-traces + from: + uri: direct:list-traces + steps: + - setVariable: + name: storagePath + simple: "{{analyzer.storage.root}}" + - setBody: + groovy: | + import java.nio.file.* + try { + def path = Paths.get(exchange.getVariable('storagePath')) + if (!Files.exists(path)) { + return [] + } + return Files.list(path) + .filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().endsWith('.txt')) + .map(p -> { + def filename = p.getFileName().toString() + def traceId = filename.substring(0, filename.lastIndexOf('.txt')) + return [id: traceId] + }) + .collect() + } catch (Exception e) { + return [] + } + - marshal: + json: + library: Jackson + - setHeader: + name: Content-Type + constant: application/json + +- route: + id: route-get-trace + from: + uri: direct:get-trace + steps: + - setVariable: + name: traceId + simple: ${header.id} + - pollEnrich: + expression: + simple: file:{{analyzer.storage.root}}?fileName=${variable.traceId}.txt&noop=true&idempotent=false + timeout: 1000 + - choice: + when: + - simple: ${body} != null + steps: + - convertBodyTo: + type: String + - setVariable: + name: fileContent + simple: ${body} + - marshal: + json: {} + - setBody: + simple: '{"id":"${variable.traceId}","content":${body}}' + - setHeader: + name: Content-Type + constant: application/json + otherwise: + steps: + - setHeader: + name: CamelHttpResponseCode + constant: 404 + - setBody: + constant: '{"error":"Trace not found"}' + - setHeader: + name: Content-Type + constant: application/json \ No newline at end of file
![https://raw.githubusercontent.com/apache/camel/refs/heads/main/docs/img/logo-d.svg"](https://raw.githubusercontent.com/apache/camel/refs/heads/main/docs/img/logo-d.svg"){kind=link}