FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fix (provider/openai): increase transcription model resilience (#5600) · vercel/ai@013faa8 · GitHub

Repository navigation

Commit 013faa8

Browse files
authored
fix (provider/openai): increase transcription model resilience (#5600)
1 parent c96e093 commit 013faa8

10 files changed

Lines changed: 112 additions & 53 deletions

File tree

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@ai-sdk/provider': patch
3+
'@ai-sdk/openai': patch
4+
'ai': patch
5+
---
6+
7+
core (ai): change transcription model mimeType to mediaType

‎.changeset/tender-buses-glow.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@ai-sdk/openai': patch
3+
---
4+
5+
fix (provider/openai): increase transcription model resilience

‎examples/ai-core/src/transcribe/openai-string.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@ import { readFile } from 'fs/promises';
66
async function main() {
77
const result = await transcribe({
88
model: openai.transcription('whisper-1'),
9-
audio: Buffer.from(
10-
await readFile('examples/ai-core/data/galileo.mp3'),
11-
).toString('base64'),
9+
audio: Buffer.from(await readFile('./data/galileo.mp3')).toString('base64'),
1210
});
1311

1412
console.log('Text:', result.text);

‎examples/ai-core/src/transcribe/openai-url.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ async function main() {
66
const result = await transcribe({
77
model: openai.transcription('whisper-1'),
88
audio: new URL(
9-
'/vercel/ai/raw/refs/heads/main/examples/ai-core/data/galileo.mp3',
10-
'https://github.com',
9+
'https://github.com/vercel/ai/raw/refs/heads/main/examples/ai-core/data/galileo.mp3',
1110
),
1211
});
1312

‎examples/ai-core/src/transcribe/openai.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { readFile } from 'fs/promises';
66
async function main() {
77
const result = await transcribe({
88
model: openai.transcription('whisper-1'),
9-
audio: await readFile('examples/ai-core/data/galileo.mp3'),
9+
audio: await readFile('data/galileo.mp3'),
1010
});
1111

1212
console.log('Text:', result.text);

‎packages/ai/core/transcribe/transcribe.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ describe('transcribe', () => {
7878

7979
expect(capturedArgs).toStrictEqual({
8080
audio: audioData,
81-
mimeType: 'audio/wav',
81+
mediaType: 'audio/wav',
8282
headers: { 'custom-request-header': 'request-header-value' },
8383
abortSignal,
8484
providerOptions: {},

‎packages/ai/core/transcribe/transcribe.ts‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
import { TranscriptionModelV1, JSONValue } from '@ai-sdk/provider';
1+
import { JSONValue, TranscriptionModelV1 } from '@ai-sdk/provider';
22
import { NoTranscriptGeneratedError } from '../../errors/no-transcript-generated-error';
3+
import { download } from '../../util/download';
4+
import { DataContent } from '../prompt';
5+
import { convertDataContentToUint8Array } from '../prompt/data-content';
36
import { prepareRetries } from '../prompt/prepare-retries';
7+
import { ProviderOptions } from '../types/provider-metadata';
48
import { TranscriptionWarning } from '../types/transcription-model';
59
import { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
6-
import { TranscriptionResult } from './transcribe-result';
7-
import { DataContent } from '../prompt';
8-
import { convertDataContentToUint8Array } from '../prompt/data-content';
9-
import { download } from '../../util/download';
1010
import {
1111
audioMimeTypeSignatures,
1212
detectMimeType,
1313
} from '../util/detect-mimetype';
14-
import { ProviderOptions } from '../types/provider-metadata';
14+
import { TranscriptionResult } from './transcribe-result';
1515

1616
/**
1717
Generates transcripts using a transcription model.
@@ -81,7 +81,7 @@ Only applicable for HTTP-based providers.
8181
const { retry } = prepareRetries({ maxRetries: maxRetriesArg });
8282
const audioData =
8383
audio instanceof URL
84-
? new Uint8Array((await download({ url: audio })).data)
84+
? (await download({ url: audio })).data
8585
: convertDataContentToUint8Array(audio);
8686

8787
const result = await retry(() =>
@@ -90,7 +90,7 @@ Only applicable for HTTP-based providers.
9090
abortSignal,
9191
headers,
9292
providerOptions,
93-
mimeType:
93+
mediaType:
9494
detectMimeType({
9595
data: audioData,
9696
signatures: audioMimeTypeSignatures,

‎packages/openai/src/openai-transcription-model.test.ts‎

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ describe('doGenerate', () => {
7373

7474
await model.doGenerate({
7575
audio: audioData,
76-
mimeType: 'audio/wav',
76+
mediaType: 'audio/wav',
7777
});
7878

7979
expect(await server.calls[0].requestBodyMultipart).toMatchObject({
@@ -95,7 +95,7 @@ describe('doGenerate', () => {
9595

9696
await provider.transcription('whisper-1').doGenerate({
9797
audio: audioData,
98-
mimeType: 'audio/wav',
98+
mediaType: 'audio/wav',
9999
headers: {
100100
'Custom-Request-Header': 'request-header-value',
101101
},
@@ -118,7 +118,7 @@ describe('doGenerate', () => {
118118

119119
const result = await model.doGenerate({
120120
audio: audioData,
121-
mimeType: 'audio/wav',
121+
mediaType: 'audio/wav',
122122
});
123123

124124
expect(result.text).toBe('Hello from the Vercel AI SDK!');
@@ -144,7 +144,7 @@ describe('doGenerate', () => {
144144

145145
const result = await customModel.doGenerate({
146146
audio: audioData,
147-
mimeType: 'audio/wav',
147+
mediaType: 'audio/wav',
148148
});
149149

150150
expect(result.response).toMatchObject({
@@ -173,10 +173,59 @@ describe('doGenerate', () => {
173173

174174
const result = await customModel.doGenerate({
175175
audio: audioData,
176-
mimeType: 'audio/wav',
176+
mediaType: 'audio/wav',
177177
});
178178

179179
expect(result.response.timestamp.getTime()).toEqual(testDate.getTime());
180180
expect(result.response.modelId).toBe('whisper-1');
181181
});
182+
183+
it('should work when no words, language, or duration are returned', async () => {
184+
server.urls['https://api.openai.com/v1/audio/transcriptions'].response = {
185+
type: 'json-value',
186+
body: {
187+
task: 'transcribe',
188+
text: 'Hello from the Vercel AI SDK!',
189+
_request_id: 'req_1234',
190+
},
191+
};
192+
193+
const testDate = new Date(0);
194+
const customModel = new OpenAITranscriptionModel('whisper-1', {
195+
provider: 'test-provider',
196+
url: () => 'https://api.openai.com/v1/audio/transcriptions',
197+
headers: () => ({}),
198+
_internal: {
199+
currentDate: () => testDate,
200+
},
201+
});
202+
203+
const result = await customModel.doGenerate({
204+
audio: audioData,
205+
mediaType: 'audio/wav',
206+
});
207+
208+
expect(result).toMatchInlineSnapshot(`
209+
{
210+
"durationInSeconds": undefined,
211+
"language": undefined,
212+
"response": {
213+
"body": {
214+
"_request_id": "req_1234",
215+
"task": "transcribe",
216+
"text": "Hello from the Vercel AI SDK!",
217+
},
218+
"headers": {
219+
"content-length": "85",
220+
"content-type": "application/json",
221+
},
222+
"modelId": "whisper-1",
223+
"timestamp": 1970-01-01T00:00:00.000Z,
224+
},
225+
"segments": [],
226+
"text": "Hello from the Vercel AI SDK!",
227+
"warnings": [],
228+
}
229+
`);
230+
});
182231
});

‎packages/openai/src/openai-transcription-model.ts‎

Lines changed: 30 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ export class OpenAITranscriptionModel implements TranscriptionModelV1 {
142142

143143
private getArgs({
144144
audio,
145-
mimeType,
145+
mediaType,
146146
providerOptions,
147147
}: OpenAITranscriptionCallOptions) {
148148
const warnings: TranscriptionModelV1CallWarning[] = [];
@@ -162,7 +162,7 @@ export class OpenAITranscriptionModel implements TranscriptionModelV1 {
162162
: new Blob([convertBase64ToUint8Array(audio)]);
163163

164164
formData.append('model', this.modelId);
165-
formData.append('file', new File([blob], 'audio', { type: mimeType }));
165+
formData.append('file', new File([blob], 'audio', { type: mediaType }));
166166

167167
// Add provider-specific options
168168
if (openAIOptions) {
@@ -197,7 +197,11 @@ export class OpenAITranscriptionModel implements TranscriptionModelV1 {
197197
const currentDate = this.config._internal?.currentDate?.() ?? new Date();
198198
const { formData, warnings } = this.getArgs(options);
199199

200-
const { value: response, responseHeaders } = await postFormDataToApi({
200+
const {
201+
value: response,
202+
responseHeaders,
203+
rawValue: rawResponse,
204+
} = await postFormDataToApi({
201205
url: this.config.url({
202206
path: '/audio/transcriptions',
203207
modelId: this.modelId,
@@ -212,48 +216,43 @@ export class OpenAITranscriptionModel implements TranscriptionModelV1 {
212216
fetch: this.config.fetch,
213217
});
214218

215-
let language: string | undefined;
216-
217-
if (response.language && response.language in languageMap) {
218-
language = languageMap[response.language as keyof typeof languageMap];
219-
}
219+
const language =
220+
response.language != null && response.language in languageMap
221+
? languageMap[response.language as keyof typeof languageMap]
222+
: undefined;
220223

221224
return {
222225
text: response.text,
223-
segments: response.words.map(word => ({
224-
text: word.word,
225-
startSecond: word.start,
226-
endSecond: word.end,
227-
})),
226+
segments:
227+
response.words?.map(word => ({
228+
text: word.word,
229+
startSecond: word.start,
230+
endSecond: word.end,
231+
})) ?? [],
228232
language,
229-
durationInSeconds: response.duration,
233+
durationInSeconds: response.duration ?? undefined,
230234
warnings,
231235
response: {
232236
timestamp: currentDate,
233237
modelId: this.modelId,
234238
headers: responseHeaders,
235-
body: response,
236-
},
237-
238-
// When using format `verbose_json` on `whisper-1`, OpenAI includes the things like `task` and enhanced `segments` information.
239-
providerMetadata: {
240-
openai: {
241-
transcript: response,
242-
},
239+
body: rawResponse,
243240
},
244241
};
245242
}
246243
}
247244

248245
const openaiTranscriptionResponseSchema = z.object({
249246
text: z.string(),
250-
language: z.string().optional(),
251-
duration: z.number().optional(),
252-
words: z.array(
253-
z.object({
254-
word: z.string(),
255-
start: z.number(),
256-
end: z.number(),
257-
}),
258-
),
247+
language: z.string().nullish(),
248+
duration: z.number().nullish(),
249+
words: z
250+
.array(
251+
z.object({
252+
word: z.string(),
253+
start: z.number(),
254+
end: z.number(),
255+
}),
256+
)
257+
.nullish(),
259258
});

‎packages/provider/src/transcription-model/v1/transcription-model-v1-call-options.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@ Accepts a `Uint8Array` or `string`, where `string` is a base64 encoded audio fil
1313
audio: Uint8Array | string;
1414

1515
/**
16-
The MIME type of the audio data.
16+
The IANA media type of the audio data.
17+
18+
@see https://www.iana.org/assignments/media-types/media-types.xhtml
1719
*/
18-
mimeType: string;
20+
mediaType: string;
1921

2022
/**
2123
Additional provider-specific options that are passed through to the provider

0 commit comments

Comments
 (0)

Back | FazBrowse Home | New Git URL