fix: extract snapshot data from chunks, reduce ClickHouse log noise
- ChunkAccumulator now extracts inputBody/outputBody/inputHeaders/outputHeaders from ExecutionChunk.inputSnapshot/outputSnapshot instead of storing empty strings - Set ClickHouse server log level to warning (was trace by default) - Update CLAUDE.md to document Ed25519 key derivation Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -40,7 +40,7 @@ java -jar cameleer3-server-app/target/cameleer3-server-app-1.0-SNAPSHOT.jar
|
|||||||
- Maintains agent instance registry (in-memory) with states: LIVE → STALE → DEAD. Auto-heals from JWT claims on heartbeat/SSE after server restart. Route catalog falls back to ClickHouse stats for route discovery when registry has incomplete data.
|
- Maintains agent instance registry (in-memory) with states: LIVE → STALE → DEAD. Auto-heals from JWT claims on heartbeat/SSE after server restart. Route catalog falls back to ClickHouse stats for route discovery when registry has incomplete data.
|
||||||
- Storage: PostgreSQL for RBAC, config, and audit; ClickHouse for all observability data (executions, search, logs, metrics, stats, diagrams). ClickHouse schema migrations in `clickhouse/*.sql`, run idempotently on startup by `ClickHouseSchemaInitializer`. Use `IF NOT EXISTS` for CREATE and ADD PROJECTION.
|
- Storage: PostgreSQL for RBAC, config, and audit; ClickHouse for all observability data (executions, search, logs, metrics, stats, diagrams). ClickHouse schema migrations in `clickhouse/*.sql`, run idempotently on startup by `ClickHouseSchemaInitializer`. Use `IF NOT EXISTS` for CREATE and ADD PROJECTION.
|
||||||
- Logging: ClickHouse JDBC set to INFO (`com.clickhouse`), HTTP client to WARN (`org.apache.hc.client5`) in application.yml
|
- Logging: ClickHouse JDBC set to INFO (`com.clickhouse`), HTTP client to WARN (`org.apache.hc.client5`) in application.yml
|
||||||
- Security: JWT auth with RBAC (AGENT/VIEWER/OPERATOR/ADMIN roles), Ed25519 config signing, bootstrap token for registration
|
- Security: JWT auth with RBAC (AGENT/VIEWER/OPERATOR/ADMIN roles), Ed25519 config signing (key derived deterministically from JWT secret via HMAC-SHA256), bootstrap token for registration
|
||||||
- OIDC: Optional external identity provider support (token exchange pattern). Configured via admin API, stored in database (`server_config` table)
|
- OIDC: Optional external identity provider support (token exchange pattern). Configured via admin API, stored in database (`server_config` table)
|
||||||
- User persistence: PostgreSQL `users` table, admin CRUD at `/api/v1/admin/users`
|
- User persistence: PostgreSQL `users` table, admin CRUD at `/api/v1/admin/users`
|
||||||
- Usage analytics: ClickHouse `usage_events` table tracks authenticated UI requests, flushed every 5s
|
- Usage analytics: ClickHouse `usage_events` table tracks authenticated UI requests, flushed every 5s
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package com.cameleer3.server.core.ingestion;
|
package com.cameleer3.server.core.ingestion;
|
||||||
|
|
||||||
|
import com.cameleer3.common.model.ExchangeSnapshot;
|
||||||
import com.cameleer3.common.model.ExecutionChunk;
|
import com.cameleer3.common.model.ExecutionChunk;
|
||||||
import com.cameleer3.common.model.FlatProcessorRecord;
|
import com.cameleer3.common.model.FlatProcessorRecord;
|
||||||
import com.cameleer3.server.core.storage.DiagramStore;
|
import com.cameleer3.server.core.storage.DiagramStore;
|
||||||
@@ -140,6 +141,8 @@ public class ChunkAccumulator {
|
|||||||
merged.setSpanId(coalesce(newer.getSpanId(), older.getSpanId()));
|
merged.setSpanId(coalesce(newer.getSpanId(), older.getSpanId()));
|
||||||
merged.setOriginalExchangeId(coalesce(newer.getOriginalExchangeId(), older.getOriginalExchangeId()));
|
merged.setOriginalExchangeId(coalesce(newer.getOriginalExchangeId(), older.getOriginalExchangeId()));
|
||||||
merged.setReplayExchangeId(coalesce(newer.getReplayExchangeId(), older.getReplayExchangeId()));
|
merged.setReplayExchangeId(coalesce(newer.getReplayExchangeId(), older.getReplayExchangeId()));
|
||||||
|
merged.setInputSnapshot(coalesce(newer.getInputSnapshot(), older.getInputSnapshot()));
|
||||||
|
merged.setOutputSnapshot(coalesce(newer.getOutputSnapshot(), older.getOutputSnapshot()));
|
||||||
merged.setChunkSeq(Math.max(newer.getChunkSeq(), older.getChunkSeq()));
|
merged.setChunkSeq(Math.max(newer.getChunkSeq(), older.getChunkSeq()));
|
||||||
merged.setFinal(newer.isFinal() || older.isFinal());
|
merged.setFinal(newer.isFinal() || older.isFinal());
|
||||||
merged.setProcessors(List.of()); // processors are handled separately
|
merged.setProcessors(List.of()); // processors are handled separately
|
||||||
@@ -182,10 +185,10 @@ public class ChunkAccumulator {
|
|||||||
envelope.getRootCauseMessage(),
|
envelope.getRootCauseMessage(),
|
||||||
diagramHash,
|
diagramHash,
|
||||||
envelope.getEngineLevel(),
|
envelope.getEngineLevel(),
|
||||||
"", // inputBody — on processor records now
|
extractBody(envelope.getInputSnapshot()),
|
||||||
"", // outputBody
|
extractBody(envelope.getOutputSnapshot()),
|
||||||
"", // inputHeaders
|
extractHeaders(envelope.getInputSnapshot()),
|
||||||
"", // outputHeaders
|
extractHeaders(envelope.getOutputSnapshot()),
|
||||||
serializeAttributes(envelope.getAttributes()),
|
serializeAttributes(envelope.getAttributes()),
|
||||||
envelope.getTraceId(),
|
envelope.getTraceId(),
|
||||||
envelope.getSpanId(),
|
envelope.getSpanId(),
|
||||||
@@ -196,6 +199,21 @@ public class ChunkAccumulator {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static String extractBody(ExchangeSnapshot snapshot) {
|
||||||
|
if (snapshot == null || snapshot.getBody() == null) return "";
|
||||||
|
return snapshot.getBody();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String extractHeaders(ExchangeSnapshot snapshot) {
|
||||||
|
if (snapshot == null || snapshot.getHeaders() == null) return "";
|
||||||
|
try {
|
||||||
|
return MAPPER.writeValueAsString(snapshot.getHeaders());
|
||||||
|
} catch (JsonProcessingException e) {
|
||||||
|
log.warn("Failed to serialize snapshot headers", e);
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static String serializeAttributes(Map<String, String> attributes) {
|
private static String serializeAttributes(Map<String, String> attributes) {
|
||||||
if (attributes == null || attributes.isEmpty()) {
|
if (attributes == null || attributes.isEmpty()) {
|
||||||
return "{}";
|
return "{}";
|
||||||
|
|||||||
@@ -114,6 +114,11 @@ data:
|
|||||||
# Based on https://kb.altinity.com/altinity-kb-setup-and-maintenance/configure_clickhouse_for_low_mem_envs/
|
# Based on https://kb.altinity.com/altinity-kb-setup-and-maintenance/configure_clickhouse_for_low_mem_envs/
|
||||||
low-memory.xml: |
|
low-memory.xml: |
|
||||||
<clickhouse>
|
<clickhouse>
|
||||||
|
<!-- Reduce log verbosity (default is trace) -->
|
||||||
|
<logger>
|
||||||
|
<level>warning</level>
|
||||||
|
</logger>
|
||||||
|
|
||||||
<!-- Let OS page out binary segments instead of pinning them in RAM -->
|
<!-- Let OS page out binary segments instead of pinning them in RAM -->
|
||||||
<mlock_executable>false</mlock_executable>
|
<mlock_executable>false</mlock_executable>
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user