From aa2d203f4e7fdb9cf973c663c52d359e11265ee8 Mon Sep 17 00:00:00 2001 From: hsiegeln <37154749+hsiegeln@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:53:32 +0200 Subject: [PATCH] feat: add UI usage analytics tracking Tracks authenticated UI user requests to understand usage patterns: - New ClickHouse usage_events table with 90-day TTL - UsageTrackingInterceptor captures method, path, duration, user - Path normalization groups dynamic segments ({id}, {hash}) - Buffered writes via WriteBuffer + periodic flush - Admin endpoint GET /api/v1/admin/usage with groupBy=endpoint|user|hour - Skips agent requests, health checks, and data ingestion Co-Authored-By: Claude Opus 4.6 (1M context) --- .../app/analytics/UsageFlushScheduler.java | 30 +++++ .../analytics/UsageTrackingInterceptor.java | 88 ++++++++++++++ .../server/app/config/StorageBeanConfig.java | 18 +++ .../server/app/config/WebConfig.java | 18 ++- .../controller/UsageAnalyticsController.java | 50 ++++++++ .../app/storage/ClickHouseUsageTracker.java | 108 ++++++++++++++++++ .../clickhouse/V10__usage_events.sql | 13 +++ .../server/core/analytics/UsageEvent.java | 14 +++ .../server/core/analytics/UsageStats.java | 7 ++ .../server/core/analytics/UsageTracker.java | 6 + 10 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 cameleer3-server-app/src/main/java/com/cameleer3/server/app/analytics/UsageFlushScheduler.java create mode 100644 cameleer3-server-app/src/main/java/com/cameleer3/server/app/analytics/UsageTrackingInterceptor.java create mode 100644 cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/UsageAnalyticsController.java create mode 100644 cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseUsageTracker.java create mode 100644 cameleer3-server-app/src/main/resources/clickhouse/V10__usage_events.sql create mode 100644 cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageEvent.java create mode 100644 cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageStats.java create mode 100644 cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageTracker.java diff --git a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/analytics/UsageFlushScheduler.java b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/analytics/UsageFlushScheduler.java new file mode 100644 index 00000000..3388a81a --- /dev/null +++ b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/analytics/UsageFlushScheduler.java @@ -0,0 +1,30 @@ +package com.cameleer3.server.app.analytics; + +import com.cameleer3.server.app.storage.ClickHouseUsageTracker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnBean(ClickHouseUsageTracker.class) +public class UsageFlushScheduler { + + private static final Logger log = LoggerFactory.getLogger(UsageFlushScheduler.class); + + private final ClickHouseUsageTracker tracker; + + public UsageFlushScheduler(ClickHouseUsageTracker tracker) { + this.tracker = tracker; + } + + @Scheduled(fixedDelayString = "${cameleer.usage.flush-interval-ms:5000}") + public void flush() { + try { + tracker.flush(); + } catch (Exception e) { + log.warn("Usage event flush failed: {}", e.getMessage()); + } + } +} diff --git a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/analytics/UsageTrackingInterceptor.java b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/analytics/UsageTrackingInterceptor.java new file mode 100644 index 00000000..894b1f1d --- /dev/null +++ b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/analytics/UsageTrackingInterceptor.java @@ -0,0 +1,88 @@ +package com.cameleer3.server.app.analytics; + +import com.cameleer3.server.core.analytics.UsageEvent; +import com.cameleer3.server.core.analytics.UsageTracker; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.web.servlet.HandlerInterceptor; + +import java.time.Instant; +import java.util.regex.Pattern; + +/** + * Tracks authenticated UI user requests for usage analytics. + * Skips agent requests, health checks, data ingestion, and static assets. + */ +public class UsageTrackingInterceptor implements HandlerInterceptor { + + private static final String START_ATTR = "usage.startNanos"; + + // Patterns for normalizing dynamic path segments + private static final Pattern EXCHANGE_ID = Pattern.compile( + "/[A-F0-9]{15,}-[A-F0-9]{16}(?=/|$)", Pattern.CASE_INSENSITIVE); + private static final Pattern UUID = Pattern.compile( + "/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?=/|$)", Pattern.CASE_INSENSITIVE); + private static final Pattern HEX_HASH = Pattern.compile( + "/[0-9a-f]{32,64}(?=/|$)", Pattern.CASE_INSENSITIVE); + private static final Pattern NUMERIC_ID = Pattern.compile( + "(?<=/)(\\d{2,})(?=/|$)"); + // Agent instance IDs like "cameleer3-sample-598867949d-g7nt4-1" + private static final Pattern INSTANCE_ID = Pattern.compile( + "(?<=/agents/)[^/]+(?=/)", Pattern.CASE_INSENSITIVE); + + private final UsageTracker usageTracker; + + public UsageTrackingInterceptor(UsageTracker usageTracker) { + this.usageTracker = usageTracker; + } + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { + request.setAttribute(START_ATTR, System.nanoTime()); + return true; + } + + @Override + public void afterCompletion(HttpServletRequest request, HttpServletResponse response, + Object handler, Exception ex) { + String username = extractUsername(); + if (username == null) return; // unauthenticated or agent request + + Long startNanos = (Long) request.getAttribute(START_ATTR); + long durationMs = startNanos != null ? (System.nanoTime() - startNanos) / 1_000_000 : 0; + + String path = request.getRequestURI(); + String queryString = request.getQueryString(); + + usageTracker.track(new UsageEvent( + Instant.now(), + username, + request.getMethod(), + path, + normalizePath(path), + response.getStatus(), + durationMs, + queryString + )); + } + + private String extractUsername() { + Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + if (auth == null || auth.getName() == null) return null; + String name = auth.getName(); + // Only track UI users (user:admin), not agents + if (!name.startsWith("user:")) return null; + return name; + } + + static String normalizePath(String path) { + String normalized = EXCHANGE_ID.matcher(path).replaceAll("/{id}"); + normalized = UUID.matcher(normalized).replaceAll("/{id}"); + normalized = HEX_HASH.matcher(normalized).replaceAll("/{hash}"); + normalized = INSTANCE_ID.matcher(normalized).replaceAll("{id}"); + normalized = NUMERIC_ID.matcher(normalized).replaceAll("{id}"); + return normalized; + } +} diff --git a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/StorageBeanConfig.java b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/StorageBeanConfig.java index 06def6ba..1fc8254e 100644 --- a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/StorageBeanConfig.java +++ b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/StorageBeanConfig.java @@ -2,6 +2,7 @@ package com.cameleer3.server.app.config; import com.cameleer3.server.app.search.ClickHouseLogStore; import com.cameleer3.server.app.storage.ClickHouseAgentEventRepository; +import com.cameleer3.server.app.storage.ClickHouseUsageTracker; import com.cameleer3.server.app.storage.ClickHouseDiagramStore; import com.cameleer3.server.app.storage.ClickHouseMetricsQueryStore; import com.cameleer3.server.app.storage.ClickHouseMetricsStore; @@ -170,4 +171,21 @@ public class StorageBeanConfig { @Qualifier("clickHouseJdbcTemplate") JdbcTemplate clickHouseJdbc) { return new ClickHouseLogStore(clickHouseJdbc); } + + // ── Usage Analytics ────────────────────────────────────────────── + + @Bean + @ConditionalOnProperty(name = "clickhouse.enabled", havingValue = "true") + public ClickHouseUsageTracker clickHouseUsageTracker( + @Qualifier("clickHouseJdbcTemplate") JdbcTemplate clickHouseJdbc) { + return new ClickHouseUsageTracker(clickHouseJdbc, + new com.cameleer3.server.core.ingestion.WriteBuffer<>(5000)); + } + + @Bean + @ConditionalOnProperty(name = "clickhouse.enabled", havingValue = "true") + public com.cameleer3.server.app.analytics.UsageTrackingInterceptor usageTrackingInterceptor( + ClickHouseUsageTracker usageTracker) { + return new com.cameleer3.server.app.analytics.UsageTrackingInterceptor(usageTracker); + } } diff --git a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/WebConfig.java b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/WebConfig.java index a5132e69..160dbe54 100644 --- a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/WebConfig.java +++ b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/WebConfig.java @@ -1,5 +1,6 @@ package com.cameleer3.server.app.config; +import com.cameleer3.server.app.analytics.UsageTrackingInterceptor; import com.cameleer3.server.app.interceptor.AuditInterceptor; import com.cameleer3.server.app.interceptor.ProtocolVersionInterceptor; import org.springframework.context.annotation.Configuration; @@ -14,11 +15,14 @@ public class WebConfig implements WebMvcConfigurer { private final ProtocolVersionInterceptor protocolVersionInterceptor; private final AuditInterceptor auditInterceptor; + private final UsageTrackingInterceptor usageTrackingInterceptor; public WebConfig(ProtocolVersionInterceptor protocolVersionInterceptor, - AuditInterceptor auditInterceptor) { + AuditInterceptor auditInterceptor, + @org.springframework.lang.Nullable UsageTrackingInterceptor usageTrackingInterceptor) { this.protocolVersionInterceptor = protocolVersionInterceptor; this.auditInterceptor = auditInterceptor; + this.usageTrackingInterceptor = usageTrackingInterceptor; } @Override @@ -35,6 +39,18 @@ public class WebConfig implements WebMvcConfigurer { "/api/v1/agents/*/refresh" ); + // Usage analytics: tracks authenticated UI user requests + if (usageTrackingInterceptor != null) { + registry.addInterceptor(usageTrackingInterceptor) + .addPathPatterns("/api/v1/**") + .excludePathPatterns( + "/api/v1/data/**", + "/api/v1/agents/*/heartbeat", + "/api/v1/agents/*/events", + "/api/v1/health" + ); + } + // Safety-net audit: catches any unaudited POST/PUT/DELETE registry.addInterceptor(auditInterceptor) .addPathPatterns("/api/v1/**") diff --git a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/UsageAnalyticsController.java b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/UsageAnalyticsController.java new file mode 100644 index 00000000..104cdb52 --- /dev/null +++ b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/UsageAnalyticsController.java @@ -0,0 +1,50 @@ +package com.cameleer3.server.app.controller; + +import com.cameleer3.server.app.storage.ClickHouseUsageTracker; +import com.cameleer3.server.core.analytics.UsageStats; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +@RestController +@RequestMapping("/api/v1/admin/usage") +@ConditionalOnBean(ClickHouseUsageTracker.class) +@Tag(name = "Usage Analytics", description = "UI usage pattern analytics") +public class UsageAnalyticsController { + + private final ClickHouseUsageTracker tracker; + + public UsageAnalyticsController(ClickHouseUsageTracker tracker) { + this.tracker = tracker; + } + + @GetMapping + @Operation(summary = "Query usage statistics", + description = "Returns aggregated API usage stats grouped by endpoint, user, or hour") + public ResponseEntity> getUsage( + @RequestParam(required = false) String from, + @RequestParam(required = false) String to, + @RequestParam(required = false) String username, + @RequestParam(defaultValue = "endpoint") String groupBy) { + + Instant fromInstant = from != null ? Instant.parse(from) : Instant.now().minus(7, ChronoUnit.DAYS); + Instant toInstant = to != null ? Instant.parse(to) : Instant.now(); + + List stats = switch (groupBy) { + case "user" -> tracker.queryByUser(fromInstant, toInstant); + case "hour" -> tracker.queryByHour(fromInstant, toInstant, username); + default -> tracker.queryByEndpoint(fromInstant, toInstant, username); + }; + + return ResponseEntity.ok(stats); + } +} diff --git a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseUsageTracker.java b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseUsageTracker.java new file mode 100644 index 00000000..3b7aba75 --- /dev/null +++ b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseUsageTracker.java @@ -0,0 +1,108 @@ +package com.cameleer3.server.app.storage; + +import com.cameleer3.server.core.analytics.UsageEvent; +import com.cameleer3.server.core.analytics.UsageStats; +import com.cameleer3.server.core.analytics.UsageTracker; +import com.cameleer3.server.core.ingestion.WriteBuffer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; + +import java.sql.Timestamp; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; + +public class ClickHouseUsageTracker implements UsageTracker { + + private static final Logger log = LoggerFactory.getLogger(ClickHouseUsageTracker.class); + private static final DateTimeFormatter CH_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") + .withZone(ZoneOffset.UTC); + + private final JdbcTemplate jdbc; + private final WriteBuffer buffer; + + public ClickHouseUsageTracker(JdbcTemplate jdbc, WriteBuffer buffer) { + this.jdbc = jdbc; + this.buffer = buffer; + } + + @Override + public void track(UsageEvent event) { + buffer.offerOrWarn(event); + } + + public void flush() { + List batch = buffer.drain(200); + if (batch.isEmpty()) return; + + jdbc.batchUpdate(""" + INSERT INTO usage_events (timestamp, username, method, path, normalized, + status_code, duration_ms, query_params) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + batch.stream().map(e -> new Object[]{ + Timestamp.from(e.timestamp()), + e.username(), + e.method(), + e.path(), + e.normalized(), + e.statusCode(), + e.durationMs(), + e.queryParams() != null ? e.queryParams() : "" + }).toList()); + + log.debug("Flushed {} usage events to ClickHouse", batch.size()); + } + + public List queryByEndpoint(Instant from, Instant to, String username) { + StringBuilder sql = new StringBuilder(""" + SELECT concat(method, ' ', normalized) AS key, + count() AS cnt, + avg(duration_ms) AS avg_dur + FROM usage_events + WHERE timestamp >= '%s' AND timestamp < '%s' + """.formatted(CH_FMT.format(from), CH_FMT.format(to))); + + if (username != null && !username.isBlank()) { + sql.append(" AND username = '").append(username.replace("'", "\\'")).append("'"); + } + sql.append(" GROUP BY key ORDER BY cnt DESC LIMIT 100"); + + return jdbc.query(sql.toString(), (rs, i) -> new UsageStats( + rs.getString("key"), rs.getLong("cnt"), rs.getLong("avg_dur"))); + } + + public List queryByUser(Instant from, Instant to) { + String sql = """ + SELECT username AS key, count() AS cnt, avg(duration_ms) AS avg_dur + FROM usage_events + WHERE timestamp >= '%s' AND timestamp < '%s' + GROUP BY key ORDER BY cnt DESC LIMIT 100 + """.formatted(CH_FMT.format(from), CH_FMT.format(to)); + + return jdbc.query(sql, (rs, i) -> new UsageStats( + rs.getString("key"), rs.getLong("cnt"), rs.getLong("avg_dur"))); + } + + public List queryByHour(Instant from, Instant to, String username) { + StringBuilder sql = new StringBuilder(""" + SELECT formatDateTime(toStartOfHour(timestamp), '%%Y-%%m-%%d %%H:00') AS key, + count() AS cnt, + avg(duration_ms) AS avg_dur + FROM usage_events + WHERE timestamp >= '%s' AND timestamp < '%s' + """.formatted(CH_FMT.format(from), CH_FMT.format(to))); + + if (username != null && !username.isBlank()) { + sql.append(" AND username = '").append(username.replace("'", "\\'")).append("'"); + } + sql.append(" GROUP BY key ORDER BY key LIMIT 720"); + + return jdbc.query(sql.toString(), (rs, i) -> new UsageStats( + rs.getString("key"), rs.getLong("cnt"), rs.getLong("avg_dur"))); + } +} diff --git a/cameleer3-server-app/src/main/resources/clickhouse/V10__usage_events.sql b/cameleer3-server-app/src/main/resources/clickhouse/V10__usage_events.sql new file mode 100644 index 00000000..561f84fb --- /dev/null +++ b/cameleer3-server-app/src/main/resources/clickhouse/V10__usage_events.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS usage_events ( + timestamp DateTime64(3) DEFAULT now64(3), + username LowCardinality(String), + method LowCardinality(String), + path String, + normalized LowCardinality(String), + status_code UInt16, + duration_ms UInt32, + query_params String DEFAULT '' +) +ENGINE = MergeTree() +ORDER BY (username, timestamp) +TTL toDateTime(timestamp) + INTERVAL 90 DAY; diff --git a/cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageEvent.java b/cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageEvent.java new file mode 100644 index 00000000..d99de3f7 --- /dev/null +++ b/cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageEvent.java @@ -0,0 +1,14 @@ +package com.cameleer3.server.core.analytics; + +import java.time.Instant; + +public record UsageEvent( + Instant timestamp, + String username, + String method, + String path, + String normalized, + int statusCode, + long durationMs, + String queryParams +) {} diff --git a/cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageStats.java b/cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageStats.java new file mode 100644 index 00000000..d27182c7 --- /dev/null +++ b/cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageStats.java @@ -0,0 +1,7 @@ +package com.cameleer3.server.core.analytics; + +public record UsageStats( + String key, + long count, + long avgDurationMs +) {} diff --git a/cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageTracker.java b/cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageTracker.java new file mode 100644 index 00000000..26495899 --- /dev/null +++ b/cameleer3-server-core/src/main/java/com/cameleer3/server/core/analytics/UsageTracker.java @@ -0,0 +1,6 @@ +package com.cameleer3.server.core.analytics; + +public interface UsageTracker { + + void track(UsageEvent event); +}