fix(stats): close 8 ClickHouseStatsStoreIT TZ failures (bucket DateTime('UTC') + JVM UTC pin)

Two-layer fix for the TZ drift that caused stats reads to miss every row
when the JVM default TZ and CH session TZ disagreed:

- Insert side: ClickHouse JDBC 0.9.7 formats java.sql.Timestamp via
  Timestamp.toString(), which uses JVM default TZ. A CEST JVM shipping
  to a UTC CH server stored Unix timestamps off by the TZ offset (the
  triage report's original symptom). Pinned JVM default to UTC in
  CameleerServerApplication.main() — standard practice for observability
  servers that push to time-series stores.
- Read side: stats_1m_* tables now declare bucket as DateTime('UTC'),
  MV SELECTs wrap toStartOfMinute(start_time) in toDateTime(..., 'UTC')
  so projections match column type, and ClickHouseStatsStore.lit(Instant)
  emits toDateTime('...', 'UTC') rather than a bare literal — defence
  in depth against future refactors.

Test class pins its own JVM TZ (the store IT builds its own
HikariDataSource, bypassing the main() path). Debug scaffolding from
the triage investigation removed.

Greenfield CH — no migration needed.

Verified: 14/14 ClickHouseStatsStoreIT green, plus 84/84 across all
ClickHouse IT classes (no regression from the JVM TZ default change).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-04-21 23:25:22 +02:00
parent d32208d403
commit a9a6b465d4
4 changed files with 37 additions and 42 deletions

View File

@@ -5,6 +5,7 @@ import com.cameleer.server.core.search.StatsTimeseries;
import com.cameleer.server.core.search.TopError;
import com.cameleer.server.core.storage.StatsStore.PunchcardCell;
import com.zaxxer.hikari.HikariDataSource;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import com.cameleer.server.app.ClickHouseTestHelper;
@@ -13,7 +14,6 @@ import org.testcontainers.clickhouse.ClickHouseContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import java.nio.charset.StandardCharsets;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.List;
@@ -34,10 +34,22 @@ class ClickHouseStatsStoreIT {
// base time: 2026-03-31T10:00:00Z (a Tuesday)
private static final Instant BASE = Instant.parse("2026-03-31T10:00:00Z");
@BeforeAll
static void pinJvmUtc() {
// ClickHouse JDBC driver 0.9.x formats java.sql.Timestamp via its
// toString(), which uses JVM default TZ. On a non-UTC dev JVM
// (e.g. CEST), timestamps were being sent to CH off by the TZ offset
// even though the CH server TZ is UTC. Pinning JVM default to UTC
// for this test class makes inserts round-trip to the UTC-typed
// bucket column predictably.
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
}
@BeforeEach
void setUp() throws Exception {
HikariDataSource ds = new HikariDataSource();
ds.setJdbcUrl(clickhouse.getJdbcUrl());
// Pin driver to UTC so Timestamp binding doesn't depend on JVM default TZ.
ds.setJdbcUrl(clickhouse.getJdbcUrl() + "?use_server_time_zone=false&use_time_zone=UTC");
ds.setUsername(clickhouse.getUsername());
ds.setPassword(clickhouse.getPassword());
@@ -51,30 +63,6 @@ class ClickHouseStatsStoreIT {
seedTestData();
// Try the failing query to capture it in query_log, then check
try {
jdbc.queryForMap(
"SELECT countMerge(total_count) AS tc, countIfMerge(failed_count) AS fc, " +
"sumMerge(duration_sum) / greatest(countMerge(total_count), 1) AS avg, " +
"quantileMerge(0.99)(p99_duration) AS p99, " +
"countIfMerge(running_count) AS rc " +
"FROM stats_1m_all WHERE tenant_id = 'default' " +
"AND bucket >= '2026-03-31 09:59:00' AND bucket < '2026-03-31 10:05:00'");
} catch (Exception e) {
System.out.println("Expected error: " + e.getMessage().substring(0, 80));
}
jdbc.execute("SYSTEM FLUSH LOGS");
// Get ALL recent queries to see what the driver sends
var queryLog = jdbc.queryForList(
"SELECT type, substring(query, 1, 200) AS q " +
"FROM system.query_log WHERE event_time > now() - 30 " +
"AND query NOT LIKE '%system.query_log%' AND query NOT LIKE '%FLUSH%' " +
"ORDER BY event_time DESC LIMIT 20");
for (var entry : queryLog) {
System.out.println("LOG: " + entry.get("type") + " | " + entry.get("q"));
}
store = new ClickHouseStatsStore("default", jdbc);
}