Skip to content

Commit a4ba368

Browse files
zhichlizhichli
authored
Merge pull request #310213 from microsoft/zhichli/otel-genai-message-format
fix: normalize OTel message JSON to GenAI semantic conventions
2 parents 3c1b53d + 2f61834 commit a4ba368

File tree

9 files changed

+239
-29
lines changed

9 files changed

+239
-29
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ test/componentFixtures/.screenshots/*
3131
!test/componentFixtures/.screenshots/baseline/
3232
dist
3333
.playwright-cli
34+
.playwright-mcp
3435
.claude/
3536
.agents/agents/*.local.md
3637
.github/agents/*.local.md

‎extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -489,21 +489,30 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider {
489489
[StdAttr.SERVER_ADDRESS]: 'api.anthropic.com',
490490
},
491491
});
492-
// Opt-in: capture input messages
492+
// Opt-in: capture input messages in OTel GenAI format
493493
if (this._otelService.config.captureContent) {
494494
try {
495495
const roleNames: Record<number, string> = { 1: 'user', 2: 'assistant', 3: 'system' };
496496
const inputMsgs = messages.map(m => {
497497
const msg = m as LanguageModelChatMessage;
498498
const role = roleNames[msg.role] ?? String(msg.role);
499-
const textParts: string[] = [];
499+
const parts: Array<{ type: string; content?: string | unknown; id?: string; name?: string; arguments?: unknown; response?: unknown }> = [];
500500
if (Array.isArray(msg.content)) {
501501
for (const p of msg.content) {
502-
if (p instanceof LanguageModelTextPart) { textParts.push(p.value); }
502+
if (p instanceof LanguageModelTextPart) {
503+
parts.push({ type: 'text', content: p.value });
504+
} else if (p instanceof LanguageModelToolCallPart) {
505+
parts.push({ type: 'tool_call', id: p.callId, name: p.name, arguments: p.input });
506+
} else if (p instanceof LanguageModelToolResultPart) {
507+
const resultText = p.content.map((c: unknown) => c instanceof LanguageModelTextPart ? c.value : '').join('');
508+
parts.push({ type: 'tool_call_response', id: p.callId, response: resultText });
509+
}
503510
}
504511
}
505-
const content = textParts.length > 0 ? textParts.join('') : '[non-text content]';
506-
return { role, parts: [{ type: 'text', content }] };
512+
if (parts.length === 0) {
513+
parts.push({ type: 'text', content: '[non-text content]' });
514+
}
515+
return { role, parts };
507516
});
508517
otelSpan.setAttribute(GenAiAttr.INPUT_MESSAGES, truncateForOTel(JSON.stringify(inputMsgs)));
509518
} catch { /* swallow */ }

‎extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -344,21 +344,27 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide
344344
[StdAttr.SERVER_ADDRESS]: 'generativelanguage.googleapis.com',
345345
},
346346
});
347-
// Opt-in: capture input messages
347+
// Opt-in: capture input messages in OTel GenAI format
348348
if (this._otelService.config.captureContent) {
349349
try {
350350
const roleNames: Record<number, string> = { 1: 'user', 2: 'assistant', 3: 'system' };
351351
const inputMsgs = messages.map(m => {
352352
const msg = m as LanguageModelChatMessage;
353353
const role = roleNames[msg.role] ?? String(msg.role);
354-
const textParts: string[] = [];
354+
const parts: Array<{ type: string; content?: string; id?: string; name?: string; arguments?: unknown }> = [];
355355
if (Array.isArray(msg.content)) {
356356
for (const p of msg.content) {
357-
if (p instanceof LanguageModelTextPart) { textParts.push(p.value); }
357+
if (p instanceof LanguageModelTextPart) {
358+
parts.push({ type: 'text', content: p.value });
359+
} else if (p instanceof LanguageModelToolCallPart) {
360+
parts.push({ type: 'tool_call', id: p.callId, name: p.name, arguments: p.input });
361+
}
358362
}
359363
}
360-
const content = textParts.length > 0 ? textParts.join('') : '[non-text content]';
361-
return { role, parts: [{ type: 'text', content }] };
364+
if (parts.length === 0) {
365+
parts.push({ type: 'text', content: '[non-text content]' });
366+
}
367+
return { role, parts };
362368
});
363369
otelSpan.setAttribute(GenAiAttr.INPUT_MESSAGES, truncateForOTel(JSON.stringify(inputMsgs)));
364370
} catch { /* swallow */ }

‎extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import { sendEngineMessagesTelemetry } from '../../../platform/networking/node/c
2828
import { CAPIWebSocketErrorEvent, IChatWebSocketManager, isCAPIWebSocketError } from '../../../platform/networking/node/chatWebSocketManager';
2929
import { sendCommunicationErrorTelemetry } from '../../../platform/networking/node/stream';
3030
import { ChatFailKind, ChatRequestCanceled, ChatRequestFailed, ChatResults, FetchResponseKind } from '../../../platform/openai/node/fetch';
31-
import { CopilotChatAttr, emitInferenceDetailsEvent, GenAiAttr, GenAiMetrics, GenAiOperationName, GenAiProviderName, StdAttr, toInputMessages, truncateForOTel } from '../../../platform/otel/common/index';
31+
import { CopilotChatAttr, emitInferenceDetailsEvent, GenAiAttr, GenAiMetrics, GenAiOperationName, GenAiProviderName, normalizeProviderMessages, StdAttr, toSystemInstructions, truncateForOTel } from '../../../platform/otel/common/index';
3232
import { IOTelService, ISpanHandle, SpanKind, SpanStatusCode } from '../../../platform/otel/common/otelService';
3333
import { IRequestLogger } from '../../../platform/requestLogger/common/requestLogger';
3434
import { getCurrentCapturingToken } from '../../../platform/requestLogger/node/requestLogger';
@@ -280,20 +280,20 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
280280
} else {
281281
systemText = JSON.stringify(systemContent);
282282
}
283-
otelInferenceSpan.setAttribute(GenAiAttr.SYSTEM_INSTRUCTIONS, systemText);
283+
// Format as OTel GenAI system instruction JSON schema
284+
const systemInstructions = toSystemInstructions(systemText);
285+
if (systemInstructions) {
286+
otelInferenceSpan.setAttribute(GenAiAttr.SYSTEM_INSTRUCTIONS, JSON.stringify(systemInstructions));
287+
}
284288
}
285289
}
286290

287291
// Always capture full request content for the debug panel
288292
if (otelInferenceSpan) {
289-
const capiMessages = (requestBody.messages ?? requestBody.input) as ReadonlyArray<{ role?: string; content?: string | unknown[] }> | undefined;
293+
const capiMessages = (requestBody.messages ?? requestBody.input) as ReadonlyArray<Record<string, unknown>> | undefined;
290294
if (capiMessages) {
291-
// Normalize non-string content (Anthropic arrays, Responses API parts) to strings for OTel schema
292-
const normalized = capiMessages.map(m => ({
293-
...m,
294-
content: typeof m.content === 'string' ? m.content : m.content ? JSON.stringify(m.content) : undefined,
295-
}));
296-
otelInferenceSpan.setAttribute(GenAiAttr.INPUT_MESSAGES, truncateForOTel(JSON.stringify(toInputMessages(normalized))));
295+
// Normalize provider-specific content (Anthropic tool_use/tool_result, OpenAI tool messages) to OTel schema
296+
otelInferenceSpan.setAttribute(GenAiAttr.INPUT_MESSAGES, truncateForOTel(JSON.stringify(normalizeProviderMessages(capiMessages))));
297297
}
298298
}
299299
tokenCount = await countTokens();

‎extensions/copilot/src/platform/otel/common/genAiEvents.ts‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
*--------------------------------------------------------------------------------------------*/
55

66
import { GenAiAttr, GenAiOperationName, StdAttr } from './genAiAttributes';
7-
import { truncateForOTel } from './messageFormatters';
7+
import { normalizeProviderMessages, toSystemInstructions, truncateForOTel } from './messageFormatters';
88
import type { IOTelService } from './otelService';
99
import { type WorkspaceOTelMetadata, workspaceMetadataToOTelAttributes } from './workspaceOTelMetadata';
1010

@@ -52,12 +52,22 @@ export function emitInferenceDetailsEvent(
5252
}
5353

5454
// Full content capture with truncation to prevent OTLP batch failures
55+
// Normalize to OTel GenAI semantic convention format
5556
if (otel.config.captureContent) {
5657
if (request.messages !== undefined) {
57-
attributes[GenAiAttr.INPUT_MESSAGES] = truncateForOTel(JSON.stringify(request.messages));
58+
const msgs = Array.isArray(request.messages) ? request.messages as ReadonlyArray<Record<string, unknown>> : undefined;
59+
attributes[GenAiAttr.INPUT_MESSAGES] = truncateForOTel(JSON.stringify(
60+
msgs ? normalizeProviderMessages(msgs) : request.messages
61+
));
5862
}
5963
if (request.systemMessage !== undefined) {
60-
attributes[GenAiAttr.SYSTEM_INSTRUCTIONS] = truncateForOTel(JSON.stringify(request.systemMessage));
64+
const systemText = typeof request.systemMessage === 'string'
65+
? request.systemMessage
66+
: JSON.stringify(request.systemMessage);
67+
const systemInstructions = toSystemInstructions(systemText);
68+
if (systemInstructions !== undefined) {
69+
attributes[GenAiAttr.SYSTEM_INSTRUCTIONS] = truncateForOTel(JSON.stringify(systemInstructions));
70+
}
6171
}
6272
if (request.tools !== undefined) {
6373
attributes[GenAiAttr.TOOL_DEFINITIONS] = truncateForOTel(JSON.stringify(request.tools));

‎extensions/copilot/src/platform/otel/common/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
export { CopilotChatAttr, GenAiAttr, GenAiOperationName, GenAiProviderName, GenAiTokenType, GenAiToolType, StdAttr } from './genAiAttributes';
77
export { emitAgentTurnEvent, emitInferenceDetailsEvent, emitSessionStartEvent, emitToolCallEvent } from './genAiEvents';
88
export { GenAiMetrics } from './genAiMetrics';
9-
export { toInputMessages, toOutputMessages, toSystemInstructions, toToolDefinitions, truncateForOTel } from './messageFormatters';
9+
export { normalizeProviderMessages, toInputMessages, toOutputMessages, toSystemInstructions, toToolDefinitions, truncateForOTel } from './messageFormatters';
1010
export { NoopOTelService } from './noopOtelService';
1111
export { resolveOTelConfig, DEFAULT_OTLP_ENDPOINT, type OTelConfig, type OTelConfigInput } from './otelConfig';
1212
export { IOTelService, SpanKind, SpanStatusCode, type ICompletedSpanData, type ISpanEventData, type ISpanEventRecord, type ISpanHandle, type OTelModelOptions, type SpanOptions, type TraceContext } from './otelService';

‎extensions/copilot/src/platform/otel/common/messageFormatters.ts‎

Lines changed: 95 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ export interface OTelOutputMessage extends OTelChatMessage {
4040
export type OTelMessagePart =
4141
| { type: 'text'; content: string }
4242
| { type: 'tool_call'; id: string; name: string; arguments: unknown }
43-
| { type: 'tool_call_response'; id: string; content: unknown }
43+
| { type: 'tool_call_response'; id: string; response: unknown }
4444
| { type: 'reasoning'; content: string };
4545

4646
export type OTelSystemInstruction = Array<{ type: 'text'; content: string }>;
@@ -54,11 +54,18 @@ export interface OTelToolDefinition {
5454

5555
/**
5656
* Convert an array of internal messages to OTel input message format.
57+
* Handles OpenAI format (tool_calls, tool_call_id) natively.
5758
*/
58-
export function toInputMessages(messages: ReadonlyArray<{ role?: string; content?: string; tool_calls?: ReadonlyArray<{ id: string; function: { name: string; arguments: string } }> }>): OTelChatMessage[] {
59+
export function toInputMessages(messages: ReadonlyArray<{ role?: string; content?: string; tool_calls?: ReadonlyArray<{ id: string; function: { name: string; arguments: string } }>; tool_call_id?: string }>): OTelChatMessage[] {
5960
return messages.map(msg => {
6061
const parts: OTelMessagePart[] = [];
6162

63+
// OpenAI tool-result message (role=tool): map to tool_call_response
64+
if (msg.role === 'tool' && msg.tool_call_id) {
65+
parts.push({ type: 'tool_call_response', id: msg.tool_call_id, response: msg.content ?? '' });
66+
return { role: msg.role, parts };
67+
}
68+
6269
if (msg.content) {
6370
parts.push({ type: 'text', content: msg.content });
6471
}
@@ -126,6 +133,92 @@ export function toSystemInstructions(systemMessage: string | undefined): OTelSys
126133
return [{ type: 'text', content: systemMessage }];
127134
}
128135

136+
/**
137+
* Normalize provider-specific messages (Anthropic content blocks, OpenAI tool messages)
138+
* to OTel GenAI semantic convention format.
139+
*
140+
* Handles:
141+
* - Anthropic content block arrays: tool_use → tool_call, tool_result → tool_call_response
142+
* - OpenAI format: tool_calls, role=tool with tool_call_id
143+
* - Plain string content
144+
*/
145+
export function normalizeProviderMessages(messages: ReadonlyArray<Record<string, unknown>>): OTelChatMessage[] {
146+
return messages.map(msg => {
147+
const role = msg.role as string | undefined;
148+
const parts: OTelMessagePart[] = [];
149+
const content = msg.content;
150+
151+
// OpenAI tool-result message
152+
if (role === 'tool' && typeof msg.tool_call_id === 'string') {
153+
parts.push({ type: 'tool_call_response', id: msg.tool_call_id, response: content ?? '' });
154+
return { role, parts };
155+
}
156+
157+
if (typeof content === 'string' && content.length > 0) {
158+
parts.push({ type: 'text', content });
159+
} else if (Array.isArray(content)) {
160+
// Anthropic content block array
161+
for (const block of content) {
162+
if (!block || typeof block !== 'object') { continue; }
163+
const b = block as Record<string, unknown>;
164+
switch (b.type) {
165+
case 'text':
166+
if (typeof b.text === 'string') {
167+
parts.push({ type: 'text', content: b.text });
168+
}
169+
break;
170+
case 'tool_use':
171+
parts.push({
172+
type: 'tool_call',
173+
id: String(b.id ?? ''),
174+
name: String(b.name ?? ''),
175+
arguments: b.input,
176+
});
177+
break;
178+
case 'tool_result':
179+
parts.push({
180+
type: 'tool_call_response',
181+
id: String(b.tool_use_id ?? ''),
182+
response: b.content ?? '',
183+
});
184+
break;
185+
case 'thinking':
186+
if (typeof b.thinking === 'string') {
187+
parts.push({ type: 'reasoning', content: b.thinking });
188+
}
189+
break;
190+
default:
191+
// Unknown block type — include as text fallback
192+
parts.push({ type: 'text', content: JSON.stringify(b) });
193+
break;
194+
}
195+
}
196+
}
197+
198+
// OpenAI tool_calls
199+
const toolCalls = msg.tool_calls;
200+
if (Array.isArray(toolCalls)) {
201+
for (const tc of toolCalls) {
202+
if (!tc || typeof tc !== 'object') { continue; }
203+
const call = tc as Record<string, unknown>;
204+
const fn = call.function as Record<string, unknown> | undefined;
205+
if (fn) {
206+
let args: unknown;
207+
try { args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : fn.arguments; } catch { args = fn.arguments; }
208+
parts.push({
209+
type: 'tool_call',
210+
id: String(call.id ?? ''),
211+
name: String(fn.name ?? ''),
212+
arguments: args,
213+
});
214+
}
215+
}
216+
}
217+
218+
return { role, parts };
219+
});
220+
}
221+
129222
/**
130223
* Convert tool definitions to OTel tool definition format.
131224
*/

‎extensions/copilot/src/platform/otel/common/test/genAiEvents.spec.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ describe('emitInferenceDetailsEvent', () => {
7373

7474
it('includes content attributes when captureContent is true', () => {
7575
const otel = createMockOTel(true);
76-
const messages = [{ role: 'user', text: 'hello' }];
76+
const messages = [{ role: 'user', content: 'hello' }];
7777
const systemMsg = 'You are helpful';
7878
const tools = [{ name: 'readFile' }];
7979

@@ -83,8 +83,10 @@ describe('emitInferenceDetailsEvent', () => {
8383
);
8484

8585
const attrs = otel.emitLogRecord.mock.calls[0][1];
86-
expect(attrs[GenAiAttr.INPUT_MESSAGES]).toBe(JSON.stringify(messages));
87-
expect(attrs[GenAiAttr.SYSTEM_INSTRUCTIONS]).toBe(JSON.stringify(systemMsg));
86+
// Messages should be normalized to OTel GenAI format
87+
expect(attrs[GenAiAttr.INPUT_MESSAGES]).toBe(JSON.stringify([{ role: 'user', parts: [{ type: 'text', content: 'hello' }] }]));
88+
// System instructions should be wrapped in OTel format
89+
expect(attrs[GenAiAttr.SYSTEM_INSTRUCTIONS]).toBe(JSON.stringify([{ type: 'text', content: 'You are helpful' }]));
8890
expect(attrs[GenAiAttr.TOOL_DEFINITIONS]).toBe(JSON.stringify(tools));
8991
});
9092

0 commit comments

Comments
 (0)