Skip to content

Commit 5d5de09

Browse files
guillaumefontannirga
guillaumefontan
and
nirga
authored
fix(openai): streaming tool_call + logging multiple tool_call (#463)
Co-authored-by: Nir Gazit <nirga@users.noreply.github.com>
1 parent 3b1f3c8 commit 5d5de09

File tree

5 files changed

+385
-43
lines changed

5 files changed

+385
-43
lines changed

‎.github/workflows/ci.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ jobs:
3333
image: qdrant/qdrant
3434
ports:
3535
- 6333:6333
36+
chroma:
37+
image: chromadb/chroma
38+
ports:
39+
- 8000:8000
3640
permissions:
3741
contents: "read"
3842
id-token: "write"
@@ -47,8 +51,6 @@ jobs:
4751
cache: "npm"
4852
node-version-file: ".nvmrc"
4953

50-
- run: pip3 install chromadb
51-
5254
- uses: nrwl/nx-set-shas@v3
5355
- run: npm ci
5456
- name: Build

‎packages/instrumentation-chromadb/tests/instrumentation.test.ts

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ import {
2626
import * as chromadb from "chromadb";
2727
import * as assert from "assert";
2828

29-
import { exec, ChildProcess } from "child_process";
30-
3129
const memoryExporter = new InMemorySpanExporter();
3230

3331
describe("Test ChromaDB instrumentation", function () {
@@ -36,13 +34,8 @@ describe("Test ChromaDB instrumentation", function () {
3634
let contextManager: AsyncHooksContextManager;
3735
let chromaDbClient: chromadb.ChromaClient;
3836
let collection: chromadb.Collection;
39-
let chromaRun: ChildProcess;
4037

4138
this.beforeAll(async () => {
42-
// Run ChromaDB instance on different terminal instance
43-
chromaRun = exec("/bin/sh");
44-
chromaRun.stdin?.write("chroma run --path .\n");
45-
4639
chromaDbClient = new chromadb.ChromaClient();
4740

4841
// Wait for ChromaDB to spin up
@@ -92,14 +85,6 @@ describe("Test ChromaDB instrumentation", function () {
9285
context.disable();
9386
});
9487

95-
after(() => {
96-
// Terminate the Chroma client process after tests
97-
if (chromaRun.pid) {
98-
// DEBT: chromaRun.kill() is not killing spawned process
99-
process.exit(0);
100-
}
101-
});
102-
10388
it("should set span attributes for Query", async () => {
10489
const input: chromadb.QueryParams = {
10590
nResults: 2,

‎packages/instrumentation-openai/recordings/Test-OpenAI-instrumentation_1770406427/should-set-function_call-attributes-in-span-for-stream-completion-when-multiple-tools-cal_2278551756/recording.har

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
{
2+
"log": {
3+
"_recordingName": "Test OpenAI instrumentation/should set function_call attributes in span for stream completion when multiple tools called",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.6"
8+
},
9+
"entries": [
10+
{
11+
"_id": "4e7f1c094bf406a3975b24c0c8857547",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 1516,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "content-length",
21+
"value": "1516"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "accept",
26+
"value": "application/json"
27+
},
28+
{
29+
"_fromType": "array",
30+
"name": "content-type",
31+
"value": "application/json"
32+
},
33+
{
34+
"_fromType": "array",
35+
"name": "user-agent",
36+
"value": "OpenAI/JS 4.57.0"
37+
},
38+
{
39+
"_fromType": "array",
40+
"name": "x-stainless-lang",
41+
"value": "js"
42+
},
43+
{
44+
"_fromType": "array",
45+
"name": "x-stainless-package-version",
46+
"value": "4.57.0"
47+
},
48+
{
49+
"_fromType": "array",
50+
"name": "x-stainless-os",
51+
"value": "MacOS"
52+
},
53+
{
54+
"_fromType": "array",
55+
"name": "x-stainless-arch",
56+
"value": "arm64"
57+
},
58+
{
59+
"_fromType": "array",
60+
"name": "x-stainless-runtime",
61+
"value": "node"
62+
},
63+
{
64+
"_fromType": "array",
65+
"name": "x-stainless-runtime-version",
66+
"value": "v22.1.0"
67+
},
68+
{
69+
"_fromType": "array",
70+
"name": "accept-encoding",
71+
"value": "gzip,deflate"
72+
},
73+
{
74+
"name": "host",
75+
"value": "api.openai.com"
76+
}
77+
],
78+
"headersSize": 470,
79+
"httpVersion": "HTTP/1.1",
80+
"method": "POST",
81+
"postData": {
82+
"mimeType": "application/json",
83+
"params": [],
84+
"text": "{\n \"model\": \"gpt-4o-mini\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"What's the weather today in Boston and what will the weather be tomorrow in Chicago?\"\n }\n ],\n \"stream\": true,\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather in a given location\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"enum\": [\n \"celsius\",\n \"fahrenheit\"\n ]\n }\n },\n \"required\": [\n \"location\"\n ]\n }\n }\n },\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_tomorrow_weather\",\n \"description\": \"Get tomorrow's weather in a given location\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"enum\": [\n \"celsius\",\n \"fahrenheit\"\n ]\n }\n },\n \"required\": [\n \"location\"\n ]\n }\n }\n }\n ]\n}"
85+
},
86+
"queryString": [],
87+
"url": "https://api.openai.com/v1/chat/completions"
88+
},
89+
"response": {
90+
"bodySize": 5057,
91+
"content": {
92+
"mimeType": "text/event-stream; charset=utf-8",
93+
"size": 5057,
94+
"text": "data: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_5LSV532rbNrhEKj3XddxLXKb\",\"type\":\"function\",\"function\":{\"name\":\"get_current_weather\",\"arguments\":\"\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"lo\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"catio\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"n\\\": \\\"B\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"osto\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"n, MA\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"id\":\"call_8rndUDfjLo174WcwdN7aT7mP\",\"type\":\"function\",\"function\":{\"name\":\"get_tomorrow_weather\",\"arguments\":\"\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"{\\\"lo\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"catio\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"n\\\": \\\"C\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"hica\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"go, I\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"L\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-AFzJUI0700ButGSyUudPGR9IqmGJi\",\"object\":\"chat.completion.chunk\",\"created\":1728373972,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f85bea6784\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n"
95+
},
96+
"cookies": [
97+
{
98+
"domain": ".api.openai.com",
99+
"expires": "2024-10-08T08:22:52.000Z",
100+
"httpOnly": true,
101+
"name": "__cf_bm",
102+
"path": "/",
103+
"sameSite": "None",
104+
"secure": true,
105+
"value": "7VLzjD6bk2vl3dQ1buulluBXUcZWRHRf3FMpMkNfDe4-1728373972-1.0.1.1-M5XcPulIjIntr0YG3dvI4hnwTDHLaXKHuLZd_JUTOH40kvNyQnhrPvm2PB12Tf7rR.E_D7F80cPaEEHKxli0SA"
106+
},
107+
{
108+
"domain": ".api.openai.com",
109+
"httpOnly": true,
110+
"name": "_cfuvid",
111+
"path": "/",
112+
"sameSite": "None",
113+
"secure": true,
114+
"value": "M4JyAwhhEaoDpYpUXniLhAW_YlQUIJCcsDEx1ZsVvTs-1728373972937-0.0.1.1-604800000"
115+
}
116+
],
117+
"headers": [
118+
{
119+
"name": "date",
120+
"value": "Tue, 08 Oct 2024 07:52:52 GMT"
121+
},
122+
{
123+
"name": "content-type",
124+
"value": "text/event-stream; charset=utf-8"
125+
},
126+
{
127+
"name": "transfer-encoding",
128+
"value": "chunked"
129+
},
130+
{
131+
"name": "connection",
132+
"value": "keep-alive"
133+
},
134+
{
135+
"name": "access-control-expose-headers",
136+
"value": "X-Request-ID"
137+
},
138+
{
139+
"name": "openai-organization",
140+
"value": "trubrics"
141+
},
142+
{
143+
"name": "openai-processing-ms",
144+
"value": "625"
145+
},
146+
{
147+
"name": "openai-version",
148+
"value": "2020-10-01"
149+
},
150+
{
151+
"name": "strict-transport-security",
152+
"value": "max-age=31536000; includeSubDomains; preload"
153+
},
154+
{
155+
"name": "x-ratelimit-limit-requests",
156+
"value": "10000"
157+
},
158+
{
159+
"name": "x-ratelimit-limit-tokens",
160+
"value": "10000000"
161+
},
162+
{
163+
"name": "x-ratelimit-remaining-requests",
164+
"value": "9999"
165+
},
166+
{
167+
"name": "x-ratelimit-remaining-tokens",
168+
"value": "9999961"
169+
},
170+
{
171+
"name": "x-ratelimit-reset-requests",
172+
"value": "6ms"
173+
},
174+
{
175+
"name": "x-ratelimit-reset-tokens",
176+
"value": "0s"
177+
},
178+
{
179+
"name": "x-request-id",
180+
"value": "req_a89738719c34e11ea192e73a5497d00a"
181+
},
182+
{
183+
"name": "cf-cache-status",
184+
"value": "DYNAMIC"
185+
},
186+
{
187+
"_fromType": "array",
188+
"name": "set-cookie",
189+
"value": "__cf_bm=7VLzjD6bk2vl3dQ1buulluBXUcZWRHRf3FMpMkNfDe4-1728373972-1.0.1.1-M5XcPulIjIntr0YG3dvI4hnwTDHLaXKHuLZd_JUTOH40kvNyQnhrPvm2PB12Tf7rR.E_D7F80cPaEEHKxli0SA; path=/; expires=Tue, 08-Oct-24 08:22:52 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None"
190+
},
191+
{
192+
"_fromType": "array",
193+
"name": "set-cookie",
194+
"value": "_cfuvid=M4JyAwhhEaoDpYpUXniLhAW_YlQUIJCcsDEx1ZsVvTs-1728373972937-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None"
195+
},
196+
{
197+
"name": "x-content-type-options",
198+
"value": "nosniff"
199+
},
200+
{
201+
"name": "server",
202+
"value": "cloudflare"
203+
},
204+
{
205+
"name": "cf-ray",
206+
"value": "8cf48dcd0a786fdb-CDG"
207+
}
208+
],
209+
"headersSize": 1150,
210+
"httpVersion": "HTTP/1.1",
211+
"redirectURL": "",
212+
"status": 200,
213+
"statusText": "OK"
214+
},
215+
"startedDateTime": "2024-10-08T07:52:51.703Z",
216+
"time": 1328,
217+
"timings": {
218+
"blocked": -1,
219+
"connect": -1,
220+
"dns": -1,
221+
"receive": 0,
222+
"send": 0,
223+
"ssl": -1,
224+
"wait": 1328
225+
}
226+
}
227+
],
228+
"pages": [],
229+
"version": "1.2"
230+
}
231+
}

‎packages/instrumentation-openai/src/instrumentation.ts

Lines changed: 51 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -356,7 +356,12 @@ export class OpenAIInstrumentation extends InstrumentationBase {
356356
index: 0,
357357
logprobs: null,
358358
finish_reason: "stop",
359-
message: { role: "assistant", content: "", refusal: null },
359+
message: {
360+
role: "assistant",
361+
content: "",
362+
refusal: null,
363+
tool_calls: [],
364+
},
360365
},
361366
],
362367
object: "chat.completion",
@@ -388,24 +393,39 @@ export class OpenAIInstrumentation extends InstrumentationBase {
388393
arguments: chunk.choices[0].delta.function_call.arguments,
389394
};
390395
}
391-
if (chunk.choices[0]?.delta.tool_calls) {
392-
// I needed to re-build the object so that Typescript will understand that arguments are not null.
393-
result.choices[0].message.tool_calls = [];
394-
for (const toolCall of chunk.choices[0].delta.tool_calls) {
395-
if (
396-
toolCall.id &&
397-
toolCall.type &&
398-
toolCall.function?.name &&
399-
toolCall.function?.arguments
400-
) {
401-
result.choices[0].message.tool_calls.push({
402-
id: toolCall.id,
403-
type: toolCall.type,
404-
function: {
405-
name: toolCall.function.name,
406-
arguments: toolCall.function.arguments,
407-
},
408-
});
396+
for (const toolCall of chunk.choices[0]?.delta?.tool_calls ?? []) {
397+
if (
398+
(result.choices[0].message.tool_calls?.length ?? 0) <
399+
toolCall.index + 1
400+
) {
401+
result.choices[0].message.tool_calls?.push({
402+
function: {
403+
name: "",
404+
arguments: "",
405+
},
406+
id: "",
407+
type: "function",
408+
});
409+
}
410+
411+
if (result.choices[0].message.tool_calls) {
412+
if (toolCall.id) {
413+
result.choices[0].message.tool_calls[toolCall.index].id +=
414+
toolCall.id;
415+
}
416+
if (toolCall.type) {
417+
result.choices[0].message.tool_calls[toolCall.index].type +=
418+
toolCall.type;
419+
}
420+
if (toolCall.function?.name) {
421+
result.choices[0].message.tool_calls[
422+
toolCall.index
423+
].function.name += toolCall.function.name;
424+
}
425+
if (toolCall.function?.arguments) {
426+
result.choices[0].message.tool_calls[
427+
toolCall.index
428+
].function.arguments += toolCall.function.arguments;
409429
}
410430
}
411431
}
@@ -621,14 +641,17 @@ export class OpenAIInstrumentation extends InstrumentationBase {
621641
choice.message.function_call.arguments,
622642
);
623643
}
624-
if (choice.message.tool_calls) {
644+
for (const [
645+
toolIndex,
646+
toolCall,
647+
] of choice?.message?.tool_calls?.entries() || []) {
625648
span.setAttribute(
626-
`${SpanAttributes.LLM_COMPLETIONS}.${index}.function_call.name`,
627-
choice.message.tool_calls[0].function.name,
649+
`${SpanAttributes.LLM_COMPLETIONS}.${index}.tool_calls.${toolIndex}.name`,
650+
toolCall.function.name,
628651
);
629652
span.setAttribute(
630-
`${SpanAttributes.LLM_COMPLETIONS}.${index}.function_call.arguments`,
631-
choice.message.tool_calls[0].function.arguments,
653+
`${SpanAttributes.LLM_COMPLETIONS}.${index}.tool_calls.${toolIndex}.arguments`,
654+
toolCall.function.arguments,
632655
);
633656
}
634657
});
@@ -722,6 +745,10 @@ export class OpenAIInstrumentation extends InstrumentationBase {
722745
private _encodingCache = new Map<string, Tiktoken>();
723746

724747
private tokenCountFromString(text: string, model: string) {
748+
if (!text) {
749+
return 0;
750+
}
751+
725752
let encoding = this._encodingCache.get(model);
726753

727754
if (!encoding) {

0 commit comments

Comments
 (0)