Skip to content

Commit 02c4ca3

Browse files
author
Ziggy
committed
[BUGFIX] Fix ClickHouse time range interpolation and time series rendering
Signed-off-by: Kostiantyn R <k-bx@k-bx.com>
1 parent 2cf76b6 commit 02c4ca3

7 files changed

Lines changed: 284 additions & 36 deletions

File tree

‎clickhouse/src/model/click-house-client.test.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright The Perses Authors
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
import { formatClickHouseDateTime, query, replaceTimeRangePlaceholders } from './click-house-client';
15+
16+
describe('ClickHouse client', () => {
17+
afterEach(() => {
18+
jest.restoreAllMocks();
19+
});
20+
21+
it('should replace time range placeholders', () => {
22+
expect(
23+
replaceTimeRangePlaceholders(
24+
"SELECT * FROM logs WHERE timestamp BETWEEN '{start}' AND '{end}'",
25+
'2025-01-01 00:00:00',
26+
'2025-01-02 00:00:00'
27+
)
28+
).toEqual("SELECT * FROM logs WHERE timestamp BETWEEN '2025-01-01 00:00:00' AND '2025-01-02 00:00:00'");
29+
});
30+
31+
it('should format time range dates for ClickHouse SQL literals', () => {
32+
expect(formatClickHouseDateTime(new Date('2025-01-01T00:00:00.000Z'))).toBe('2025-01-01 00:00:00');
33+
});
34+
35+
it('should send interpolated query to ClickHouse', async () => {
36+
const fetchMock = jest.fn<Promise<Pick<Response, 'ok' | 'json'>>, [string, RequestInit?]>(async () => ({
37+
ok: true,
38+
json: jest.fn(async () => ({ data: [] })),
39+
}));
40+
global.fetch = fetchMock as unknown as typeof fetch;
41+
42+
await query(
43+
{
44+
query: "SELECT * FROM logs WHERE timestamp BETWEEN '{start}' AND '{end}'",
45+
start: '2025-01-01 00:00:00',
46+
end: '2025-01-02 00:00:00',
47+
},
48+
{
49+
datasourceUrl: 'http://clickhouse.example.com',
50+
}
51+
);
52+
53+
expect(fetchMock).toHaveBeenCalledTimes(1);
54+
const [calledUrl] = fetchMock.mock.calls[0]!;
55+
const url = new URL(calledUrl);
56+
expect(url.searchParams.get('query')).toBe(
57+
"SELECT * FROM logs WHERE timestamp BETWEEN '2025-01-01 00:00:00' AND '2025-01-02 00:00:00' FORMAT JSON"
58+
);
59+
});
60+
});

‎clickhouse/src/model/click-house-client.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import { RequestHeaders } from '@perses-dev/core';
1515

1616
export interface ClickHouseQueryParams {
1717
query: string;
18+
start?: string;
19+
end?: string;
1820
database?: string;
1921
}
2022

@@ -32,6 +34,17 @@ export interface ClickHouseClient {
3234
query: (params: { start: string; end: string; query: string }) => Promise<ClickHouseQueryResponse>;
3335
}
3436

37+
export function replaceTimeRangePlaceholders(query: string, start?: string, end?: string): string {
38+
return query.replaceAll('{start}', start ?? '{start}').replaceAll('{end}', end ?? '{end}');
39+
}
40+
41+
export function formatClickHouseDateTime(date: Date): string {
42+
return date
43+
.toISOString()
44+
.replace('T', ' ')
45+
.replace(/\.\d{3}Z$/, '');
46+
}
47+
3548
export async function query(
3649
params: ClickHouseQueryParams,
3750
queryOptions: ClickHouseQueryOptions
@@ -43,7 +56,7 @@ export async function query(
4356
throw new Error('No query provided in params');
4457
}
4558

46-
let finalQuery = params.query.trim();
59+
let finalQuery = replaceTimeRangePlaceholders(params.query.trim(), params.start, params.end);
4760
if (!finalQuery.toUpperCase().includes('FORMAT')) {
4861
finalQuery += ' FORMAT JSON';
4962
}

‎clickhouse/src/model/click-house-data-types.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@ export interface ClickHouseTimeSeriesData extends TimeSeriesData {
1919

2020
export interface TimeSeriesEntry {
2121
time: string;
22-
log_count: number | string;
22+
[key: string]: number | string | null | undefined;
2323
}

‎clickhouse/src/queries/click-house-log-query/click-house-log-query-types.test.ts‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ const createStubContext = (): ClickHouseQueryContext => {
4747
setSavedDatasources: jest.fn(),
4848
},
4949
timeRange: {
50-
end: new Date('01-01-2025'),
51-
start: new Date('01-02-2025'),
50+
end: new Date('2025-01-02T00:00:00.000Z'),
51+
start: new Date('2025-01-01T00:00:00.000Z'),
5252
},
5353
variableState: {},
5454
};
@@ -71,4 +71,23 @@ describe('ClickHouseLogQuery', () => {
7171
const initialOptions = ClickHouseLogQuery.createInitialOptions();
7272
expect(initialOptions).toEqual({ query: '' });
7373
});
74+
75+
it('should run query with interpolated time range', async () => {
76+
const response = await ClickHouseLogQuery.getLogData(
77+
{
78+
query: "SELECT * FROM application_logs WHERE timestamp >= '{start}' AND timestamp <= '{end}'",
79+
},
80+
createStubContext()
81+
);
82+
83+
expect(clickhouseStubClient.query).toHaveBeenCalledWith({
84+
start: '2025-01-01 00:00:00',
85+
end: '2025-01-02 00:00:00',
86+
query:
87+
"SELECT * FROM application_logs WHERE timestamp >= '2025-01-01 00:00:00' AND timestamp <= '2025-01-02 00:00:00'",
88+
});
89+
expect(response.metadata?.executedQueryString).toBe(
90+
"SELECT * FROM application_logs WHERE timestamp >= '2025-01-01 00:00:00' AND timestamp <= '2025-01-02 00:00:00'"
91+
);
92+
});
7493
});

‎clickhouse/src/queries/click-house-log-query/get-click-house-log-data.ts‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,12 @@
1313

1414
import { replaceVariables } from '@perses-dev/plugin-system';
1515
import { LogEntry, LogData } from '@perses-dev/core';
16-
import { ClickHouseClient, ClickHouseQueryResponse } from '../../model/click-house-client';
16+
import {
17+
ClickHouseClient,
18+
ClickHouseQueryResponse,
19+
formatClickHouseDateTime,
20+
replaceTimeRangePlaceholders,
21+
} from '../../model/click-house-client';
1722
import { DEFAULT_DATASOURCE } from '../constants';
1823
import { ClickHouseLogQuerySpec } from './click-house-log-query-types';
1924
import { LogQueryPlugin } from './log-query-plugin-interface';
@@ -83,18 +88,21 @@ export const getClickHouseLogData: LogQueryPlugin<ClickHouseLogQuerySpec>['getLo
8388
)) as ClickHouseClient;
8489

8590
const { start, end } = context.timeRange;
91+
const startTime = formatClickHouseDateTime(start);
92+
const endTime = formatClickHouseDateTime(end);
93+
const executedQueryString = replaceTimeRangePlaceholders(query, startTime, endTime);
8694

8795
const response: ClickHouseQueryResponse = await client.query({
88-
start: start.getTime().toString(),
89-
end: end.getTime().toString(),
90-
query,
96+
start: startTime,
97+
end: endTime,
98+
query: executedQueryString,
9199
});
92100

93101
return {
94102
timeRange: { start, end },
95103
logs: convertStreamsToLogs(response.data as LogEntry[]),
96104
metadata: {
97-
executedQueryString: query,
105+
executedQueryString,
98106
},
99107
};
100108
};

‎clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts‎

Lines changed: 91 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ clickhouseStubClient.query = jest.fn(async () => {
3232
const stubResponse: ClickHouseQueryResponse = {
3333
status: 'success',
3434
data: [
35-
{ time: '2025-09-09 05:18:00', log_count: '277' },
36-
{ time: '2025-09-09 05:19:00', log_count: '156102' },
35+
{ time: '2025-09-09 05:18:00', avg_cpu: '2.5', max_memory: 277, service: 'api' },
36+
{ time: '2025-09-09 05:19:00', avg_cpu: '3.5', max_memory: 156102, service: 'api' },
3737
],
3838
};
3939
return stubResponse as ClickHouseQueryResponse;
@@ -65,8 +65,8 @@ const createStubContext = (): TimeSeriesQueryContext => {
6565
setSavedDatasources: jest.fn(),
6666
},
6767
timeRange: {
68-
end: new Date('01-01-2025'),
69-
start: new Date('01-02-2025'),
68+
end: new Date('2025-01-02T00:00:00.000Z'),
69+
start: new Date('2025-01-01T00:00:00.000Z'),
7070
},
7171
variableState: {},
7272
};
@@ -90,11 +90,92 @@ describe('ClickHouseTimeSeriesQuery', () => {
9090
expect(initialOptions).toEqual({ query: '' });
9191
});
9292

93-
it('should run query and return ClickHouse data only', async () => {
94-
const client = getDatasourceClient();
95-
const resp = await client.query('SELECT count(*) FROM otel_logs');
96-
expect(resp.data.length).toBeGreaterThan(0);
97-
expect(resp.data[0]).toHaveProperty('time');
98-
expect(resp.data[0]).toHaveProperty('log_count');
93+
it('should run query with interpolated time range and return one series per numeric column', async () => {
94+
const response = await ClickHouseTimeSeriesQuery.getTimeSeriesData(
95+
{
96+
query:
97+
"SELECT toStartOfMinute(timestamp) as time, avg(cpu_usage) as avg_cpu, max(memory_usage) as max_memory FROM system_metrics WHERE timestamp BETWEEN '{start}' AND '{end}' GROUP BY time ORDER BY time",
98+
},
99+
createStubContext()
100+
);
101+
102+
expect(clickhouseStubClient.query).toHaveBeenCalledWith({
103+
start: '2025-01-01 00:00:00',
104+
end: '2025-01-02 00:00:00',
105+
query:
106+
"SELECT toStartOfMinute(timestamp) as time, avg(cpu_usage) as avg_cpu, max(memory_usage) as max_memory FROM system_metrics WHERE timestamp BETWEEN '2025-01-01 00:00:00' AND '2025-01-02 00:00:00' GROUP BY time ORDER BY time",
107+
});
108+
expect(response.series).toEqual([
109+
{
110+
name: 'avg_cpu',
111+
values: [
112+
[new Date('2025-09-09 05:18:00').getTime(), 2.5],
113+
[new Date('2025-09-09 05:19:00').getTime(), 3.5],
114+
],
115+
},
116+
{
117+
name: 'max_memory',
118+
values: [
119+
[new Date('2025-09-09 05:18:00').getTime(), 277],
120+
[new Date('2025-09-09 05:19:00').getTime(), 156102],
121+
],
122+
},
123+
]);
124+
expect(response.stepMs).toBe(60 * 1000);
125+
expect(response.metadata?.executedQueryString).toBe(
126+
"SELECT toStartOfMinute(timestamp) as time, avg(cpu_usage) as avg_cpu, max(memory_usage) as max_memory FROM system_metrics WHERE timestamp BETWEEN '2025-01-01 00:00:00' AND '2025-01-02 00:00:00' GROUP BY time ORDER BY time"
127+
);
128+
});
129+
130+
it('should infer daily query step from returned timestamps', async () => {
131+
(clickhouseStubClient.query as jest.Mock).mockResolvedValueOnce({
132+
status: 'success',
133+
data: [
134+
{ time: '2026-01-01 22:00:00', flights: 80 },
135+
{ time: '2026-01-02 22:00:00', flights: 56 },
136+
{ time: '2026-01-03 22:00:00', flights: 32 },
137+
],
138+
});
139+
140+
const response = await ClickHouseTimeSeriesQuery.getTimeSeriesData(
141+
{
142+
query:
143+
"SELECT toStartOfDay(timestamp) as time, sum(flights_count) as flights FROM flight WHERE timestamp BETWEEN '{start}' AND '{end}' GROUP BY time ORDER BY time",
144+
},
145+
createStubContext()
146+
);
147+
148+
expect(response.stepMs).toBe(24 * 60 * 60 * 1000);
149+
expect(response.series).toEqual([
150+
{
151+
name: 'flights',
152+
values: [
153+
[new Date('2026-01-01 22:00:00').getTime(), 80],
154+
[new Date('2026-01-02 22:00:00').getTime(), 56],
155+
[new Date('2026-01-03 22:00:00').getTime(), 32],
156+
],
157+
},
158+
]);
159+
});
160+
161+
it('should keep timezone daily buckets daily across daylight saving time changes', async () => {
162+
(clickhouseStubClient.query as jest.Mock).mockResolvedValueOnce({
163+
status: 'success',
164+
data: [
165+
{ time: '2026-03-28 22:00:00', flights: 80 },
166+
{ time: '2026-03-29 21:00:00', flights: 56 },
167+
{ time: '2026-03-30 21:00:00', flights: 32 },
168+
],
169+
});
170+
171+
const response = await ClickHouseTimeSeriesQuery.getTimeSeriesData(
172+
{
173+
query:
174+
"SELECT toStartOfDay(timestamp) as time, sum(flights_count) as flights FROM flight WHERE timestamp BETWEEN '{start}' AND '{end}' GROUP BY time ORDER BY time",
175+
},
176+
createStubContext()
177+
);
178+
179+
expect(response.stepMs).toBe(24 * 60 * 60 * 1000);
99180
});
100181
});

0 commit comments

Comments
 (0)