diff --git a/CHANGELOG.md b/CHANGELOG.md index e5323067..67ea009d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## tip +* FEATURE: add opt-in Grafana dataplane log format support. When enabled in datasource settings, log frames use standard field names (`timestamp`, `body`) and `DataFrameType.LogLines` metadata, enabling automatic `${labelKey}` variables in correlations. + ## v0.26.3 * BUGFIX: fix time range provided in `field_names` and `field_values` requests. Instead of being rounded to a 24-hour time range, the selected time range is now provided. See [pr #581](https://github.com/VictoriaMetrics/victorialogs-datasource/pull/581). diff --git a/src/README.md b/src/README.md index 36e2c8e9..711eefe1 100644 --- a/src/README.md +++ b/src/README.md @@ -46,6 +46,8 @@ datasources: url: http://victorialogs:9428 isDefault: true jsonData: + # Enable Grafana dataplane log format for correlations support + #useDataplaneFormat: true # Multitenancy settings, see https://docs.victoriametrics.com/victorialogs/#multitenancy # to use the multitenancy, uncomment lines below: AccountID and ProjectID multitenancyHeaders: @@ -165,6 +167,19 @@ Where: * `operator` is the comparison operator to use, such as `equals`, `notEquals`, `regex`, `lessThan`, `greaterThan`. * `enabled` is a boolean flag to enable or disable the rule. Defaults to `true` if omitted. +### Dataplane format + +The **Use dataplane format** toggle in the datasource settings enables the [Grafana dataplane log format](https://grafana.com/developers/dataplane/). When enabled, log frames use standard field names (`timestamp`, `body` instead of `Time`, `Line`) and set `DataFrameType.LogLines` metadata. This makes all label keys automatically available as `${labelKey}` variables in [correlations](https://grafana.com/docs/grafana/latest/administration/correlations/), similar to how Loki works with its dataplane support. + +**Note:** Enabling this setting may break existing client-side transformations that reference the `Time` or `Line` field names. + +To enable via provisioning: + +```yaml +jsonData: + useDataplaneFormat: true +``` + ### Variables VictoriaLogs datasource supports [variables](https://grafana.com/docs/grafana/latest/variables/) in queries. diff --git a/src/configuration/LogsSettings.tsx b/src/configuration/LogsSettings.tsx index 4054cba7..31303297 100644 --- a/src/configuration/LogsSettings.tsx +++ b/src/configuration/LogsSettings.tsx @@ -1,7 +1,7 @@ import React, { SyntheticEvent, useMemo } from 'react'; import { SelectableValue } from '@grafana/data'; -import { InlineField, Input, Stack, Text } from '@grafana/ui'; +import { InlineField, InlineSwitch, Input, Stack, Text } from '@grafana/ui'; import { Options } from '../types'; @@ -53,6 +53,26 @@ export const LogsSettings = (props: PropsConfigEditor) => { /> +
+ + { + onOptionsChange({ + ...options, + jsonData: { + ...options.jsonData, + useDataplaneFormat: e.currentTarget.checked, + }, + }); + }} + /> + +
); diff --git a/src/datasource.ts b/src/datasource.ts index b28fceed..bd21c1e2 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -94,6 +94,7 @@ export class VictoriaLogsDatasource queryBuilderLimits?: QueryBuilderLimits; logLevelRules: LogLevelRule[]; multitenancyHeaders?: MultitenancyHeaders; + useDataplaneFormat: boolean; constructor( instanceSettings: DataSourceInstanceSettings, @@ -120,6 +121,7 @@ export class VictoriaLogsDatasource this.queryBuilderLimits = settingsData.queryBuilderLimits; this.logLevelRules = settingsData.logLevelRules || []; this.multitenancyHeaders = this.parseMultitenancyHeaders(settingsData.multitenancyHeaders); + this.useDataplaneFormat = settingsData.useDataplaneFormat ?? false; } query(request: DataQueryRequest): Observable { @@ -158,7 +160,8 @@ export class VictoriaLogsDatasource response, fixedRequest, this.derivedFields ?? [], - this.getActiveLevelRules() + this.getActiveLevelRules(), + this.useDataplaneFormat ) ) ); diff --git a/src/transformers/frameProcessors/streamFrameProcessor.ts b/src/transformers/frameProcessors/streamFrameProcessor.ts index 01374d4d..65f2a705 100644 --- a/src/transformers/frameProcessors/streamFrameProcessor.ts +++ b/src/transformers/frameProcessors/streamFrameProcessor.ts @@ -1,4 +1,4 @@ -import { DataFrame, QueryResultMeta } from '@grafana/data'; +import { DataFrame, DataFrameType, QueryResultMeta } from '@grafana/data'; import { LogLevelRule } from '../../configuration/LogLevelRules/types'; import { getHighlighterExpressionsFromQuery } from '../../queryUtils'; @@ -7,19 +7,20 @@ import { getDerivedFields } from '../fields/derivedField'; import { getStreamFields } from '../fields/labelField'; import { addLevelField } from '../fields/levelField'; import { getStreamIds } from '../fields/streamUtils'; -import { ANNOTATIONS_REF_ID } from '../types'; +import { ANNOTATIONS_REF_ID, FrameField } from '../types'; import { dataFrameHasError, setFrameMeta } from '../utils/frame/frameUtils'; export function processStreamsFrames( frames: DataFrame[], queryMap: Map, derivedFieldConfigs: DerivedFieldConfig[], - logLevelRules: LogLevelRule[] + logLevelRules: LogLevelRule[], + useDataplaneFormat = false, ): DataFrame[] { return frames.map((frame) => { const query = frame.refId !== undefined ? queryMap.get(frame.refId) : undefined; const isAnnotations = query?.refId === ANNOTATIONS_REF_ID; - return processStreamFrame(frame, query, derivedFieldConfigs, logLevelRules, isAnnotations); + return processStreamFrame(frame, query, derivedFieldConfigs, logLevelRules, isAnnotations, useDataplaneFormat); }); } @@ -28,7 +29,8 @@ function processStreamFrame( query: Query | undefined, derivedFieldConfigs: DerivedFieldConfig[], logLevelRules: LogLevelRule[], - transformLabels = false + transformLabels = false, + useDataplaneFormat = false, ): DataFrame { const custom: Record = { ...frame.meta?.custom, // keep the original meta.custom @@ -55,11 +57,26 @@ function processStreamFrame( const derivedFields = getDerivedFields(frameWithLevel, derivedFieldConfigs); const baseFields = getStreamFields(frameWithLevel.fields, transformLabels); + const fields = [...baseFields, ...derivedFields]; + + if (useDataplaneFormat) { + return { + ...frameWithLevel, + fields: fields.map((f) => { + if (f.name === 'Time') { return { ...f, name: 'timestamp' }; } + if (f.name === FrameField.Line) { return { ...f, name: 'body' }; } + return f; + }), + meta: { + ...frameWithLevel.meta, + type: DataFrameType.LogLines, + typeVersion: [0, 0], + }, + }; + } + return { ...frameWithLevel, - fields: [ - ...baseFields, - ...derivedFields - ] + fields, }; } diff --git a/src/transformers/transformBackendResult.test.ts b/src/transformers/transformBackendResult.test.ts index d5257bdc..5a0d50c2 100644 --- a/src/transformers/transformBackendResult.test.ts +++ b/src/transformers/transformBackendResult.test.ts @@ -1,4 +1,4 @@ -import { DataQueryRequest, DataQueryResponse, dateTime, LogLevel } from '@grafana/data'; +import { DataFrameType, DataQueryRequest, DataQueryResponse, dateTime, LogLevel } from '@grafana/data'; import { LogLevelRule, LogLevelRuleType } from '../configuration/LogLevelRules/types'; import { DerivedFieldConfig, Query, QueryType } from '../types'; @@ -187,6 +187,95 @@ describe('transformBackendResult', () => { ]); }); + it('should use dataplane format when useDataplaneFormat is true', () => { + const response = { + 'data': [ + { + 'refId': 'A', + 'meta': { + 'typeVersion': [0, 0] + }, + 'fields': [ + { + 'name': 'Time', + 'type': 'time', + 'typeInfo': { 'frame': 'time.Time' }, + 'config': {}, + 'values': [1760598702731], + 'entities': {}, + 'nanos': [713000] + }, + { + 'name': 'Line', + 'type': 'string', + 'typeInfo': { 'frame': 'string' }, + 'config': {}, + 'values': ['starting application'], + 'entities': {} + }, + { + 'name': 'labels', + 'type': 'other', + 'typeInfo': { 'frame': 'json.RawMessage' }, + 'config': {}, + 'values': [labels[0]], + 'entities': {} + } + ], + 'length': 1 + } + ], + 'state': 'Done' + } as DataQueryResponse; + const request = { + 'app': 'dashboard', + 'requestId': 'SQR100', + 'timezone': 'browser', + 'range': { + 'to': '2025-10-16T07:28:02.475Z', + 'from': '2025-10-16T01:28:02.475Z', + 'raw': { 'from': 'now-6h', 'to': 'now' } + }, + 'interval': '20s', + 'intervalMs': 20000, + 'targets': [ + { + 'datasource': { 'type': 'victoriametrics-logs-datasource', 'uid': 'bexw8wod6s4jke' }, + 'editorMode': 'code', + 'expr': '*', + 'queryType': 'instant', + 'refId': 'A', + 'maxLines': 1000 + } + ], + 'maxDataPoints': 913, + 'scopedVars': { + '__sceneObject': { 'text': '__sceneObject' }, + '__interval': { 'text': '20s', 'value': '20s' }, + '__interval_ms': { 'text': '20000', 'value': 20000 } + }, + 'startTime': 1760599682628, + 'rangeRaw': { 'from': 'now-6h', 'to': 'now' }, + 'dashboardUID': '886b7b9f-97a7-47ee-93b6-9ec7342f6d3e', + 'panelId': 1, + 'panelName': 'New panel', + 'panelPluginId': 'table', + 'dashboardTitle': 'double label info' + } as unknown as DataQueryRequest; + const result = transformBackendResult(response, request, [], [], true); + const frame = result.data[0]; + + // Field names should be renamed to dataplane format + expect(frame.fields[0].name).toBe('timestamp'); + expect(frame.fields[1].name).toBe('body'); + expect(frame.fields[2].name).toBe('labels'); + expect(frame.fields[3].name).toBe('detected_level'); + + // Meta should have dataplane type + expect(frame.meta?.type).toBe(DataFrameType.LogLines); + expect(frame.meta?.typeVersion).toStrictEqual([0, 0]); + }); + it('should parse level according to rules, apply the origin level labels then rule labels', () => { const extendedLabels = labels.map((l, index) => { if (index > 2) { diff --git a/src/transformers/transformBackendResult.ts b/src/transformers/transformBackendResult.ts index 70c6e82b..824d8557 100644 --- a/src/transformers/transformBackendResult.ts +++ b/src/transformers/transformBackendResult.ts @@ -17,6 +17,7 @@ export function transformBackendResult( request: DataQueryRequest, derivedFieldConfigs: DerivedFieldConfig[], logLevelRules: LogLevelRule[], + useDataplaneFormat = false, ): DataQueryResponse { const { data, errors, ...rest } = response; const queries = request.targets; @@ -44,7 +45,7 @@ export function transformBackendResult( data: [ ...processMetricRangeFrames(metricRangeFrames, request.targets, request.range.from.valueOf(), request.range.to.valueOf()), ...processMetricInstantFrames(metricInstantFrames), - ...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs, logLevelRules), + ...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs, logLevelRules, useDataplaneFormat), ...processHistogramFrames(histogramFrames, request.panelPluginId), ], }; diff --git a/src/types.ts b/src/types.ts index 051386a9..ffe7b147 100644 --- a/src/types.ts +++ b/src/types.ts @@ -18,6 +18,7 @@ export interface Options extends DataSourceJsonData { logLevelRules?: LogLevelRule[]; multitenancyHeaders?: Partial>; vmuiUrl?: string; + useDataplaneFormat?: boolean; } export const QUERY_DIRECTION = {