cameleer/cameleer-server
1
0
Fork 0
Code Issues 41 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(logs): add instanceIds multi-value filter to /logs endpoint

Browse Source 
Adds List<String> instanceIds to LogSearchRequest (null-normalized to
List.of() in compact ctor) and generates an IN clause in both
ClickHouseLogStore.search() and countLogs(), mirroring the existing
sources pattern. LogQueryController parses ?instanceIds= as a
comma-split list. All existing LogSearchRequest call sites updated.
New ClickHouseLogStoreInstanceIdsIT covers: multi-value filter, empty
filter (all rows), null filter (all rows), single-value filter, and
coexistence with the singular instanceId field.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-04-23 12:41:09 +02:00
parent 2312a7304d
commit 382e1801a7
7 changed files with 253 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cameleer-server-app/src/main/java/com/cameleer/server/app/alerting/eval/LogPatternEvaluator.java
View File

@@ -61,7 +61,8 @@ public class LogPatternEvaluator implements ConditionEvaluator<LogPatternConditi
to,
null, // cursor
1, // limit (count query; value irrelevant)
"desc" // sort
"desc", // sort
null // instanceIds
);
return logStore.countLogs(req);
});

12
cameleer-server-app/src/main/java/com/cameleer/server/app/controller/LogQueryController.java
View File

@@ -44,6 +44,7 @@ public class LogQueryController {
@RequestParam(required = false) String exchangeId,
@RequestParam(required = false) String logger,
@RequestParam(required = false) String source,
@RequestParam(required = false) String instanceIds,
@RequestParam(required = false) String from,
@RequestParam(required = false) String to,
@RequestParam(required = false) String cursor,
@@ -69,12 +70,21 @@ public class LogQueryController {
.toList();
}
List<String> instanceIdList = List.of();
if (instanceIds != null && !instanceIds.isEmpty()) {
instanceIdList = Arrays.stream(instanceIds.split(","))
.map(String::trim)
.filter(s -> !s.isEmpty())
.toList();
}
Instant fromInstant = from != null ? Instant.parse(from) : null;
Instant toInstant = to != null ? Instant.parse(to) : null;
LogSearchRequest request = new LogSearchRequest(
searchText, levels, application, instanceId, exchangeId,
logger, env.slug(), sources, fromInstant, toInstant, cursor, limit, sort);
logger, env.slug(), sources, fromInstant, toInstant, cursor, limit, sort,
instanceIdList);
LogSearchResponse result = logIndex.search(request);

16
cameleer-server-app/src/main/java/com/cameleer/server/app/search/ClickHouseLogStore.java
View File

@@ -122,6 +122,14 @@ public class ClickHouseLogStore implements LogIndex {
baseParams.add(request.instanceId());
}
if (request.instanceIds() != null && !request.instanceIds().isEmpty()) {
String placeholders = String.join(", ", Collections.nCopies(request.instanceIds().size(), "?"));
baseConditions.add("instance_id IN (" + placeholders + ")");
for (String id : request.instanceIds()) {
baseParams.add(id);
}
}
if (request.exchangeId() != null && !request.exchangeId().isEmpty()) {
baseConditions.add("(exchange_id = ?" +
" OR (mapContains(mdc, 'cameleer.exchangeId') AND mdc['cameleer.exchangeId'] = ?)" +
@@ -281,6 +289,14 @@ public class ClickHouseLogStore implements LogIndex {
params.add(request.instanceId());
}
if (request.instanceIds() != null && !request.instanceIds().isEmpty()) {
String placeholders = String.join(", ", Collections.nCopies(request.instanceIds().size(), "?"));
conditions.add("instance_id IN (" + placeholders + ")");
for (String id : request.instanceIds()) {
params.add(id);
}
}
if (request.exchangeId() != null && !request.exchangeId().isEmpty()) {
conditions.add("(exchange_id = ?" +
" OR (mapContains(mdc, 'cameleer.exchangeId') AND mdc['cameleer.exchangeId'] = ?)" +

8
cameleer-server-app/src/test/java/com/cameleer/server/app/search/ClickHouseLogStoreCountIT.java
View File

@@ -79,7 +79,8 @@ class ClickHouseLogStoreCountIT {
base.plusSeconds(30),
null,
100,
"desc"));
"desc",
null));
assertThat(count).isEqualTo(3);
}
@@ -102,7 +103,8 @@ class ClickHouseLogStoreCountIT {
base.plusSeconds(30),
null,
100,
"desc"));
"desc",
null));
assertThat(count).isZero();
}
@@ -120,7 +122,7 @@ class ClickHouseLogStoreCountIT {
null, List.of("ERROR"), "orders", null, null, null,
"dev", List.of(),
base.minusSeconds(1), base.plusSeconds(60),
null, 100, "desc"));
null, 100, "desc", null));
assertThat(devCount).isEqualTo(2);
}

32
cameleer-server-app/src/test/java/com/cameleer/server/app/search/ClickHouseLogStoreIT.java
View File

@@ -53,7 +53,7 @@ class ClickHouseLogStoreIT {
}
private LogSearchRequest req(String application) {
return new LogSearchRequest(null, null, application, null, null, null, null, null, null, null, null, 100, "desc");
return new LogSearchRequest(null, null, application, null, null, null, null, null, null, null, null, 100, "desc", null);
}
// ── Tests ─────────────────────────────────────────────────────────────
@@ -99,7 +99,7 @@ class ClickHouseLogStoreIT {
));
LogSearchResponse result = store.search(new LogSearchRequest(
null, List.of("ERROR"), "my-app", null, null, null, null, null, null, null, null, 100, "desc"));
null, List.of("ERROR"), "my-app", null, null, null, null, null, null, null, null, 100, "desc", null));
assertThat(result.data()).hasSize(1);
assertThat(result.data().get(0).level()).isEqualTo("ERROR");
@@ -116,7 +116,7 @@ class ClickHouseLogStoreIT {
));
LogSearchResponse result = store.search(new LogSearchRequest(
null, List.of("WARN", "ERROR"), "my-app", null, null, null, null, null, null, null, null, 100, "desc"));
null, List.of("WARN", "ERROR"), "my-app", null, null, null, null, null, null, null, null, 100, "desc", null));
assertThat(result.data()).hasSize(2);
}
@@ -130,7 +130,7 @@ class ClickHouseLogStoreIT {
));
LogSearchResponse result = store.search(new LogSearchRequest(
"order #12345", null, "my-app", null, null, null, null, null, null, null, null, 100, "desc"));
"order #12345", null, "my-app", null, null, null, null, null, null, null, null, 100, "desc", null));
assertThat(result.data()).hasSize(1);
assertThat(result.data().get(0).message()).contains("order #12345");
@@ -147,7 +147,7 @@ class ClickHouseLogStoreIT {
));
LogSearchResponse result = store.search(new LogSearchRequest(
null, null, "my-app", null, "exchange-abc", null, null, null, null, null, null, 100, "desc"));
null, null, "my-app", null, "exchange-abc", null, null, null, null, null, null, 100, "desc", null));
assertThat(result.data()).hasSize(1);
assertThat(result.data().get(0).message()).isEqualTo("msg with exchange");
@@ -170,7 +170,7 @@ class ClickHouseLogStoreIT {
Instant to = Instant.parse("2026-03-31T13:00:00Z");
LogSearchResponse result = store.search(new LogSearchRequest(
null, null, "my-app", null, null, null, null, null, from, to, null, 100, "desc"));
null, null, "my-app", null, null, null, null, null, from, to, null, 100, "desc", null));
assertThat(result.data()).hasSize(1);
assertThat(result.data().get(0).message()).isEqualTo("noon");
@@ -188,7 +188,7 @@ class ClickHouseLogStoreIT {
// No application filter — should return both
LogSearchResponse result = store.search(new LogSearchRequest(
null, null, null, null, null, null, null, null, null, null, null, 100, "desc"));
null, null, null, null, null, null, null, null, null, null, null, 100, "desc", null));
assertThat(result.data()).hasSize(2);
}
@@ -202,7 +202,7 @@ class ClickHouseLogStoreIT {
));
LogSearchResponse result = store.search(new LogSearchRequest(
null, null, "my-app", null, null, "OrderProcessor", null, null, null, null, null, 100, "desc"));
null, null, "my-app", null, null, "OrderProcessor", null, null, null, null, null, 100, "desc", null));
assertThat(result.data()).hasSize(1);
assertThat(result.data().get(0).loggerName()).contains("OrderProcessor");
@@ -221,7 +221,7 @@ class ClickHouseLogStoreIT {
// Page 1: limit 2
LogSearchResponse page1 = store.search(new LogSearchRequest(
null, null, "my-app", null, null, null, null, null, null, null, null, 2, "desc"));
null, null, "my-app", null, null, null, null, null, null, null, null, 2, "desc", null));
assertThat(page1.data()).hasSize(2);
assertThat(page1.hasMore()).isTrue();
@@ -230,7 +230,7 @@ class ClickHouseLogStoreIT {
// Page 2: use cursor
LogSearchResponse page2 = store.search(new LogSearchRequest(
null, null, "my-app", null, null, null, null, null, null, null, page1.nextCursor(), 2, "desc"));
null, null, "my-app", null, null, null, null, null, null, null, page1.nextCursor(), 2, "desc", null));
assertThat(page2.data()).hasSize(2);
assertThat(page2.hasMore()).isTrue();
@@ -238,7 +238,7 @@ class ClickHouseLogStoreIT {
// Page 3: last page
LogSearchResponse page3 = store.search(new LogSearchRequest(
null, null, "my-app", null, null, null, null, null, null, null, page2.nextCursor(), 2, "desc"));
null, null, "my-app", null, null, null, null, null, null, null, page2.nextCursor(), 2, "desc", null));
assertThat(page3.data()).hasSize(1);
assertThat(page3.hasMore()).isFalse();
@@ -257,7 +257,7 @@ class ClickHouseLogStoreIT {
// Filter for ERROR only, but counts should include all levels
LogSearchResponse result = store.search(new LogSearchRequest(
null, List.of("ERROR"), "my-app", null, null, null, null, null, null, null, null, 100, "desc"));
null, List.of("ERROR"), "my-app", null, null, null, null, null, null, null, null, 100, "desc", null));
assertThat(result.data()).hasSize(1);
assertThat(result.levelCounts()).containsEntry("INFO", 2L);
@@ -275,7 +275,7 @@ class ClickHouseLogStoreIT {
));
LogSearchResponse result = store.search(new LogSearchRequest(
null, null, "my-app", null, null, null, null, null, null, null, null, 100, "asc"));
null, null, "my-app", null, null, null, null, null, null, null, null, 100, "asc", null));
assertThat(result.data()).hasSize(3);
assertThat(result.data().get(0).message()).isEqualTo("msg-1");
@@ -340,7 +340,7 @@ class ClickHouseLogStoreIT {
LogSearchResponse result = store.search(new LogSearchRequest(
null, null, "my-app", null, null, null, null,
List.of("container"), null, null, null, 100, "desc"));
List.of("container"), null, null, null, 100, "desc", null));
assertThat(result.data()).hasSize(1);
assertThat(result.data().get(0).message()).isEqualTo("container msg");
@@ -365,7 +365,7 @@ class ClickHouseLogStoreIT {
LogSearchResponse result = store.search(new LogSearchRequest(
null, null, "my-app", null, null, null, null,
List.of("app", "container"), null, null, null, 100, "desc"));
List.of("app", "container"), null, null, null, 100, "desc", null));
assertThat(result.data()).hasSize(2);
assertThat(result.data()).extracting(LogEntryResult::message)
@@ -388,7 +388,7 @@ class ClickHouseLogStoreIT {
for (int page = 0; page < 10; page++) {
LogSearchResponse resp = store.search(new LogSearchRequest(
null, null, "my-app", null, null, null, null, null,
null, null, cursor, 2, "desc"));
null, null, cursor, 2, "desc", null));
for (LogEntryResult r : resp.data()) {
assertThat(seen.add(r.message())).as("duplicate row returned: " + r.message()).isTrue();
}

196
cameleer-server-app/src/test/java/com/cameleer/server/app/search/ClickHouseLogStoreInstanceIdsIT.java Normal file
View File

@@ -0,0 +1,196 @@
package com.cameleer.server.app.search;
import com.cameleer.server.core.ingestion.BufferedLogEntry;
import com.cameleer.server.core.search.LogSearchRequest;
import com.cameleer.server.core.search.LogSearchResponse;
import com.cameleer.common.model.LogEntry;
import com.cameleer.server.app.ClickHouseTestHelper;
import com.zaxxer.hikari.HikariDataSource;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.testcontainers.clickhouse.ClickHouseContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import java.time.Instant;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
/**
* Integration test for the {@code instanceIds} multi-value filter on
* {@link ClickHouseLogStore#search(LogSearchRequest)}.
*
* <p>Three rows are seeded with distinct {@code instance_id} values:
* <ul>
* <li>{@code prod-app1-0-aaa11111} — included in filter</li>
* <li>{@code prod-app1-1-aaa11111} — included in filter</li>
* <li>{@code prod-app1-0-bbb22222} — excluded from filter</li>
* </ul>
*/
@Testcontainers
class ClickHouseLogStoreInstanceIdsIT {
@Container
static final ClickHouseContainer clickhouse =
new ClickHouseContainer("clickhouse/clickhouse-server:24.12");
private JdbcTemplate jdbc;
private ClickHouseLogStore store;
private static final String TENANT = "default";
private static final String ENV = "prod";
private static final String APP = "app1";
private static final String INST_A = "prod-app1-0-aaa11111";
private static final String INST_B = "prod-app1-1-aaa11111";
private static final String INST_C = "prod-app1-0-bbb22222";
@BeforeEach
void setUp() throws Exception {
HikariDataSource ds = new HikariDataSource();
ds.setJdbcUrl(clickhouse.getJdbcUrl());
ds.setUsername(clickhouse.getUsername());
ds.setPassword(clickhouse.getPassword());
jdbc = new JdbcTemplate(ds);
ClickHouseTestHelper.executeInitSql(jdbc);
jdbc.execute("TRUNCATE TABLE logs");
store = new ClickHouseLogStore(TENANT, jdbc);
Instant base = Instant.parse("2026-04-23T09:00:00Z");
seedLog(INST_A, base, "msg-from-replica-0-gen-aaa");
seedLog(INST_B, base.plusSeconds(1), "msg-from-replica-1-gen-aaa");
seedLog(INST_C, base.plusSeconds(2), "msg-from-replica-0-gen-bbb");
}
@AfterEach
void tearDown() {
jdbc.execute("TRUNCATE TABLE logs");
}
private void seedLog(String instanceId, Instant ts, String message) {
LogEntry entry = new LogEntry(ts, "INFO", "com.example.Svc", message, "main", null, null);
store.insertBufferedBatch(List.of(
new BufferedLogEntry(TENANT, ENV, instanceId, APP, entry)));
}
// ── Tests ─────────────────────────────────────────────────────────────
@Test
void search_instanceIds_returnsOnlyMatchingInstances() {
LogSearchResponse result = store.search(new LogSearchRequest(
null,
List.of(),
APP,
null,
null,
null,
ENV,
List.of(),
null,
null,
null,
100,
"desc",
List.of(INST_A, INST_B)));
assertThat(result.data()).hasSize(2);
assertThat(result.data())
.extracting(r -> r.instanceId())
.containsExactlyInAnyOrder(INST_A, INST_B);
assertThat(result.data())
.extracting(r -> r.instanceId())
.doesNotContain(INST_C);
}
@Test
void search_emptyInstanceIds_returnsAllRows() {
LogSearchResponse result = store.search(new LogSearchRequest(
null,
List.of(),
APP,
null,
null,
null,
ENV,
List.of(),
null,
null,
null,
100,
"desc",
List.of()));
assertThat(result.data()).hasSize(3);
}
@Test
void search_nullInstanceIds_returnsAllRows() {
LogSearchResponse result = store.search(new LogSearchRequest(
null,
List.of(),
APP,
null,
null,
null,
ENV,
List.of(),
null,
null,
null,
100,
"desc",
null));
assertThat(result.data()).hasSize(3);
}
@Test
void search_instanceIds_singleValue_filtersToOneReplica() {
LogSearchResponse result = store.search(new LogSearchRequest(
null,
List.of(),
APP,
null,
null,
null,
ENV,
List.of(),
null,
null,
null,
100,
"desc",
List.of(INST_C)));
assertThat(result.data()).hasSize(1);
assertThat(result.data().get(0).instanceId()).isEqualTo(INST_C);
assertThat(result.data().get(0).message()).isEqualTo("msg-from-replica-0-gen-bbb");
}
@Test
void search_instanceIds_doesNotConflictWithSingularInstanceId() {
// Singular instanceId=INST_A AND instanceIds=[INST_B] → intersection = empty
// (both conditions apply: instance_id = A AND instance_id IN (B))
LogSearchResponse result = store.search(new LogSearchRequest(
null,
List.of(),
APP,
INST_A, // singular
null,
null,
ENV,
List.of(),
null,
null,
null,
100,
"desc",
List.of(INST_B))); // plural — no overlap
assertThat(result.data()).isEmpty();
}
}

9
cameleer-server-core/src/main/java/com/cameleer/server/core/search/LogSearchRequest.java
View File

@@ -9,7 +9,7 @@ import java.util.List;
* @param q free-text search across message and stack trace
* @param levels log level filter (e.g. ["WARN","ERROR"]), OR-joined
* @param application application ID filter (nullable = all apps)
* @param instanceId agent instance ID filter
* @param instanceId agent instance ID filter (single value; coexists with instanceIds)
* @param exchangeId Camel exchange ID filter
* @param logger logger name substring filter
* @param environment optional environment filter (e.g. "dev", "staging", "prod")
@@ -19,6 +19,9 @@ import java.util.List;
* @param cursor ISO timestamp cursor for keyset pagination
* @param limit page size (1-500, default 100)
* @param sort sort direction: "asc" or "desc" (default "desc")
* @param instanceIds multi-value instance ID filter (IN clause); scopes logs to one deployment's
* replicas when provided. Both instanceId and instanceIds may coexist — both
* conditions apply (AND). Empty/null means no additional filtering.
*/
public record LogSearchRequest(
String q,
@@ -33,7 +36,8 @@ public record LogSearchRequest(
Instant to,
String cursor,
int limit,
String sort
String sort,
List<String> instanceIds
) {
private static final int DEFAULT_LIMIT = 100;
@@ -45,5 +49,6 @@ public record LogSearchRequest(
if (sort == null || !"asc".equalsIgnoreCase(sort)) sort = "desc";
if (levels == null) levels = List.of();
if (sources == null) sources = List.of();
if (instanceIds == null) instanceIds = List.of();
}
}