cameleer/cameleer-server
1
0
Fork 0
Code Issues 47 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: remove all ClickHouse code, old interfaces, and SQL migrations

Browse Source 
- Delete all ClickHouse storage implementations and config
- Delete old core interfaces (ExecutionRepository, DiagramRepository, MetricsRepository, SearchEngine, RawExecutionRow)
- Delete ClickHouse SQL migration files
- Delete AbstractClickHouseIT
- Update controllers to use new store interfaces (DiagramStore, ExecutionStore)
- Fix IngestionService calls in controllers for new synchronous API
- Migrate all ITs from AbstractClickHouseIT to AbstractPostgresIT
- Fix count() syntax and remove ClickHouse-specific test assertions
- Update TreeReconstructionTest for new buildTree() method

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-03-16 18:56:13 +01:00
parent 7dbfaf0932
commit 565b548ac1
68 changed files with 226 additions and 2238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
cameleer3-server-app/src/main/java/com/cameleer3/server/app/ingestion/ClickHouseFlushScheduler.java
View File

@@ -1,159 +0,0 @@
package com.cameleer3.server.app.ingestion;
import com.cameleer3.server.app.config.IngestionConfig;
import com.cameleer3.server.core.ingestion.TaggedDiagram;
import com.cameleer3.server.core.ingestion.TaggedExecution;
import com.cameleer3.server.core.ingestion.WriteBuffer;
import com.cameleer3.server.core.storage.DiagramRepository;
import com.cameleer3.server.core.storage.ExecutionRepository;
import com.cameleer3.server.core.storage.MetricsRepository;
import com.cameleer3.server.core.storage.model.MetricsSnapshot;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.SmartLifecycle;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* Scheduled task that drains the write buffers and batch-inserts into ClickHouse.
* <p>
* Implements {@link SmartLifecycle} to ensure all remaining buffered data is
* flushed on application shutdown.
*/
@Component
public class ClickHouseFlushScheduler implements SmartLifecycle {
private static final Logger log = LoggerFactory.getLogger(ClickHouseFlushScheduler.class);
private final WriteBuffer<TaggedExecution> executionBuffer;
private final WriteBuffer<TaggedDiagram> diagramBuffer;
private final WriteBuffer<MetricsSnapshot> metricsBuffer;
private final ExecutionRepository executionRepository;
private final DiagramRepository diagramRepository;
private final MetricsRepository metricsRepository;
private final int batchSize;
private volatile boolean running = false;
public ClickHouseFlushScheduler(WriteBuffer<TaggedExecution> executionBuffer,
WriteBuffer<TaggedDiagram> diagramBuffer,
WriteBuffer<MetricsSnapshot> metricsBuffer,
ExecutionRepository executionRepository,
DiagramRepository diagramRepository,
MetricsRepository metricsRepository,
IngestionConfig config) {
this.executionBuffer = executionBuffer;
this.diagramBuffer = diagramBuffer;
this.metricsBuffer = metricsBuffer;
this.executionRepository = executionRepository;
this.diagramRepository = diagramRepository;
this.metricsRepository = metricsRepository;
this.batchSize = config.getBatchSize();
}
@Scheduled(fixedDelayString = "${ingestion.flush-interval-ms:1000}")
public void flushAll() {
flushExecutions();
flushDiagrams();
flushMetrics();
}
private void flushExecutions() {
try {
List<TaggedExecution> batch = executionBuffer.drain(batchSize);
if (!batch.isEmpty()) {
executionRepository.insertBatch(batch);
log.debug("Flushed {} executions to ClickHouse", batch.size());
}
} catch (Exception e) {
log.error("Failed to flush executions to ClickHouse", e);
}
}
private void flushDiagrams() {
try {
List<TaggedDiagram> batch = diagramBuffer.drain(batchSize);
for (TaggedDiagram diagram : batch) {
diagramRepository.store(diagram);
}
if (!batch.isEmpty()) {
log.debug("Flushed {} diagrams to ClickHouse", batch.size());
}
} catch (Exception e) {
log.error("Failed to flush diagrams to ClickHouse", e);
}
}
private void flushMetrics() {
try {
List<MetricsSnapshot> batch = metricsBuffer.drain(batchSize);
if (!batch.isEmpty()) {
metricsRepository.insertBatch(batch);
log.debug("Flushed {} metrics to ClickHouse", batch.size());
}
} catch (Exception e) {
log.error("Failed to flush metrics to ClickHouse", e);
}
}
// SmartLifecycle -- flush remaining data on shutdown
@Override
public void start() {
running = true;
log.info("ClickHouseFlushScheduler started");
}
@Override
public void stop() {
log.info("ClickHouseFlushScheduler stopping -- flushing remaining data");
drainAll();
running = false;
}
@Override
public boolean isRunning() {
return running;
}
@Override
public int getPhase() {
// Run after most beans but before DataSource shutdown
return Integer.MAX_VALUE - 1;
}
/**
* Drain all buffers completely (loop until empty).
*/
private void drainAll() {
drainBufferCompletely("executions", executionBuffer, batch -> executionRepository.insertBatch(batch));
drainBufferCompletely("diagrams", diagramBuffer, batch -> {
for (TaggedDiagram d : batch) {
diagramRepository.store(d);
}
});
drainBufferCompletely("metrics", metricsBuffer, batch -> metricsRepository.insertBatch(batch));
}
private <T> void drainBufferCompletely(String name, WriteBuffer<T> buffer, java.util.function.Consumer<List<T>> inserter) {
int total = 0;
while (buffer.size() > 0) {
List<T> batch = buffer.drain(batchSize);
if (batch.isEmpty()) {
break;
}
try {
inserter.accept(batch);
total += batch.size();
} catch (Exception e) {
log.error("Failed to flush remaining {} during shutdown", name, e);
break;
}
}
if (total > 0) {
log.info("Flushed {} remaining {} during shutdown", total, name);
}
}
}