| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,7 @@ | |||
| 1 | + --- | ||
| 2 | + '@ai-sdk/provider': patch | ||
| 3 | + '@ai-sdk/openai': patch | ||
| 4 | + 'ai': patch | ||
| 5 | + --- | ||
| 6 | + | ||
| 7 | + core (ai): change transcription model mimeType to mediaType | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,5 @@ | |||
| 1 | + --- | ||
| 2 | + '@ai-sdk/openai': patch | ||
| 3 | + --- | ||
| 4 | + | ||
| 5 | + fix (provider/openai): increase transcription model resilience | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -6,9 +6,7 @@ import { readFile } from 'fs/promises'; | |||
| 6 | 6 | async function main() { | |
| 7 | 7 | const result = await transcribe({ | |
| 8 | 8 | model: openai.transcription('whisper-1'), | |
| 9 | - audio: Buffer.from( | ||
| 10 | - await readFile('examples/ai-core/data/galileo.mp3'), | ||
| 11 | - ).toString('base64'), | ||
| 9 | + audio: Buffer.from(await readFile('./data/galileo.mp3')).toString('base64'), | ||
| 12 | 10 | }); | |
| 13 | 11 | ||
| 14 | 12 | console.log('Text:', result.text); | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -6,8 +6,7 @@ async function main() { | |||
| 6 | 6 | const result = await transcribe({ | |
| 7 | 7 | model: openai.transcription('whisper-1'), | |
| 8 | 8 | audio: new URL( | |
| 9 | - '/vercel/ai/raw/refs/heads/main/examples/ai-core/data/galileo.mp3', | ||
| 10 | - 'https://github.com', | ||
| 9 | + 'https://github.com/vercel/ai/raw/refs/heads/main/examples/ai-core/data/galileo.mp3', | ||
| 11 | 10 | ), | |
| 12 | 11 | }); | |
| 13 | 12 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -6,7 +6,7 @@ import { readFile } from 'fs/promises'; | |||
| 6 | 6 | async function main() { | |
| 7 | 7 | const result = await transcribe({ | |
| 8 | 8 | model: openai.transcription('whisper-1'), | |
| 9 | - audio: await readFile('examples/ai-core/data/galileo.mp3'), | ||
| 9 | + audio: await readFile('data/galileo.mp3'), | ||
| 10 | 10 | }); | |
| 11 | 11 | ||
| 12 | 12 | console.log('Text:', result.text); | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -78,7 +78,7 @@ describe('transcribe', () => { | |||
| 78 | 78 | ||
| 79 | 79 | expect(capturedArgs).toStrictEqual({ | |
| 80 | 80 | audio: audioData, | |
| 81 | - mimeType: 'audio/wav', | ||
| 81 | + mediaType: 'audio/wav', | ||
| 82 | 82 | headers: { 'custom-request-header': 'request-header-value' }, | |
| 83 | 83 | abortSignal, | |
| 84 | 84 | providerOptions: {}, | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,17 +1,17 @@ | |||
| 1 | - import { TranscriptionModelV1, JSONValue } from '@ai-sdk/provider'; | ||
| 1 | + import { JSONValue, TranscriptionModelV1 } from '@ai-sdk/provider'; | ||
| 2 | 2 | import { NoTranscriptGeneratedError } from '../../errors/no-transcript-generated-error'; | |
| 3 | + import { download } from '../../util/download'; | ||
| 4 | + import { DataContent } from '../prompt'; | ||
| 5 | + import { convertDataContentToUint8Array } from '../prompt/data-content'; | ||
| 3 | 6 | import { prepareRetries } from '../prompt/prepare-retries'; | |
| 7 | + import { ProviderOptions } from '../types/provider-metadata'; | ||
| 4 | 8 | import { TranscriptionWarning } from '../types/transcription-model'; | |
| 5 | 9 | import { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata'; | |
| 6 | - import { TranscriptionResult } from './transcribe-result'; | ||
| 7 | - import { DataContent } from '../prompt'; | ||
| 8 | - import { convertDataContentToUint8Array } from '../prompt/data-content'; | ||
| 9 | - import { download } from '../../util/download'; | ||
| 10 | 10 | import { | |
| 11 | 11 | audioMimeTypeSignatures, | |
| 12 | 12 | detectMimeType, | |
| 13 | 13 | } from '../util/detect-mimetype'; | |
| 14 | - import { ProviderOptions } from '../types/provider-metadata'; | ||
| 14 | + import { TranscriptionResult } from './transcribe-result'; | ||
| 15 | 15 | ||
| 16 | 16 | /** | |
| 17 | 17 | Generates transcripts using a transcription model. | |
@@ -81,7 +81,7 @@ Only applicable for HTTP-based providers. | |||
| 81 | 81 | const { retry } = prepareRetries({ maxRetries: maxRetriesArg }); | |
| 82 | 82 | const audioData = | |
| 83 | 83 | audio instanceof URL | |
| 84 | - ? new Uint8Array((await download({ url: audio })).data) | ||
| 84 | + ? (await download({ url: audio })).data | ||
| 85 | 85 | : convertDataContentToUint8Array(audio); | |
| 86 | 86 | ||
| 87 | 87 | const result = await retry(() => | |
@@ -90,7 +90,7 @@ Only applicable for HTTP-based providers. | |||
| 90 | 90 | abortSignal, | |
| 91 | 91 | headers, | |
| 92 | 92 | providerOptions, | |
| 93 | - mimeType: | ||
| 93 | + mediaType: | ||
| 94 | 94 | detectMimeType({ | |
| 95 | 95 | data: audioData, | |
| 96 | 96 | signatures: audioMimeTypeSignatures, | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -73,7 +73,7 @@ describe('doGenerate', () => { | |||
| 73 | 73 | ||
| 74 | 74 | await model.doGenerate({ | |
| 75 | 75 | audio: audioData, | |
| 76 | - mimeType: 'audio/wav', | ||
| 76 | + mediaType: 'audio/wav', | ||
| 77 | 77 | }); | |
| 78 | 78 | ||
| 79 | 79 | expect(await server.calls[0].requestBodyMultipart).toMatchObject({ | |
@@ -95,7 +95,7 @@ describe('doGenerate', () => { | |||
| 95 | 95 | ||
| 96 | 96 | await provider.transcription('whisper-1').doGenerate({ | |
| 97 | 97 | audio: audioData, | |
| 98 | - mimeType: 'audio/wav', | ||
| 98 | + mediaType: 'audio/wav', | ||
| 99 | 99 | headers: { | |
| 100 | 100 | 'Custom-Request-Header': 'request-header-value', | |
| 101 | 101 | }, | |
@@ -118,7 +118,7 @@ describe('doGenerate', () => { | |||
| 118 | 118 | ||
| 119 | 119 | const result = await model.doGenerate({ | |
| 120 | 120 | audio: audioData, | |
| 121 | - mimeType: 'audio/wav', | ||
| 121 | + mediaType: 'audio/wav', | ||
| 122 | 122 | }); | |
| 123 | 123 | ||
| 124 | 124 | expect(result.text).toBe('Hello from the Vercel AI SDK!'); | |
@@ -144,7 +144,7 @@ describe('doGenerate', () => { | |||
| 144 | 144 | ||
| 145 | 145 | const result = await customModel.doGenerate({ | |
| 146 | 146 | audio: audioData, | |
| 147 | - mimeType: 'audio/wav', | ||
| 147 | + mediaType: 'audio/wav', | ||
| 148 | 148 | }); | |
| 149 | 149 | ||
| 150 | 150 | expect(result.response).toMatchObject({ | |
@@ -173,10 +173,59 @@ describe('doGenerate', () => { | |||
| 173 | 173 | ||
| 174 | 174 | const result = await customModel.doGenerate({ | |
| 175 | 175 | audio: audioData, | |
| 176 | - mimeType: 'audio/wav', | ||
| 176 | + mediaType: 'audio/wav', | ||
| 177 | 177 | }); | |
| 178 | 178 | ||
| 179 | 179 | expect(result.response.timestamp.getTime()).toEqual(testDate.getTime()); | |
| 180 | 180 | expect(result.response.modelId).toBe('whisper-1'); | |
| 181 | 181 | }); | |
| 182 | + | ||
| 183 | + it('should work when no words, language, or duration are returned', async () => { | ||
| 184 | + server.urls['https://api.openai.com/v1/audio/transcriptions'].response = { | ||
| 185 | + type: 'json-value', | ||
| 186 | + body: { | ||
| 187 | + task: 'transcribe', | ||
| 188 | + text: 'Hello from the Vercel AI SDK!', | ||
| 189 | + _request_id: 'req_1234', | ||
| 190 | + }, | ||
| 191 | + }; | ||
| 192 | + | ||
| 193 | + const testDate = new Date(0); | ||
| 194 | + const customModel = new OpenAITranscriptionModel('whisper-1', { | ||
| 195 | + provider: 'test-provider', | ||
| 196 | + url: () => 'https://api.openai.com/v1/audio/transcriptions', | ||
| 197 | + headers: () => ({}), | ||
| 198 | + _internal: { | ||
| 199 | + currentDate: () => testDate, | ||
| 200 | + }, | ||
| 201 | + }); | ||
| 202 | + | ||
| 203 | + const result = await customModel.doGenerate({ | ||
| 204 | + audio: audioData, | ||
| 205 | + mediaType: 'audio/wav', | ||
| 206 | + }); | ||
| 207 | + | ||
| 208 | + expect(result).toMatchInlineSnapshot(` | ||
| 209 | + { | ||
| 210 | + "durationInSeconds": undefined, | ||
| 211 | + "language": undefined, | ||
| 212 | + "response": { | ||
| 213 | + "body": { | ||
| 214 | + "_request_id": "req_1234", | ||
| 215 | + "task": "transcribe", | ||
| 216 | + "text": "Hello from the Vercel AI SDK!", | ||
| 217 | + }, | ||
| 218 | + "headers": { | ||
| 219 | + "content-length": "85", | ||
| 220 | + "content-type": "application/json", | ||
| 221 | + }, | ||
| 222 | + "modelId": "whisper-1", | ||
| 223 | + "timestamp": 1970-01-01T00:00:00.000Z, | ||
| 224 | + }, | ||
| 225 | + "segments": [], | ||
| 226 | + "text": "Hello from the Vercel AI SDK!", | ||
| 227 | + "warnings": [], | ||
| 228 | + } | ||
| 229 | + `); | ||
| 230 | + }); | ||
| 182 | 231 | }); | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -142,7 +142,7 @@ export class OpenAITranscriptionModel implements TranscriptionModelV1 { | |||
| 142 | 142 | ||
| 143 | 143 | private getArgs({ | |
| 144 | 144 | audio, | |
| 145 | - mimeType, | ||
| 145 | + mediaType, | ||
| 146 | 146 | providerOptions, | |
| 147 | 147 | }: OpenAITranscriptionCallOptions) { | |
| 148 | 148 | const warnings: TranscriptionModelV1CallWarning[] = []; | |
@@ -162,7 +162,7 @@ export class OpenAITranscriptionModel implements TranscriptionModelV1 { | |||
| 162 | 162 | : new Blob([convertBase64ToUint8Array(audio)]); | |
| 163 | 163 | ||
| 164 | 164 | formData.append('model', this.modelId); | |
| 165 | - formData.append('file', new File([blob], 'audio', { type: mimeType })); | ||
| 165 | + formData.append('file', new File([blob], 'audio', { type: mediaType })); | ||
| 166 | 166 | ||
| 167 | 167 | // Add provider-specific options | |
| 168 | 168 | if (openAIOptions) { | |
@@ -197,7 +197,11 @@ export class OpenAITranscriptionModel implements TranscriptionModelV1 { | |||
| 197 | 197 | const currentDate = this.config._internal?.currentDate?.() ?? new Date(); | |
| 198 | 198 | const { formData, warnings } = this.getArgs(options); | |
| 199 | 199 | ||
| 200 | - const { value: response, responseHeaders } = await postFormDataToApi({ | ||
| 200 | + const { | ||
| 201 | + value: response, | ||
| 202 | + responseHeaders, | ||
| 203 | + rawValue: rawResponse, | ||
| 204 | + } = await postFormDataToApi({ | ||
| 201 | 205 | url: this.config.url({ | |
| 202 | 206 | path: '/audio/transcriptions', | |
| 203 | 207 | modelId: this.modelId, | |
@@ -212,48 +216,43 @@ export class OpenAITranscriptionModel implements TranscriptionModelV1 { | |||
| 212 | 216 | fetch: this.config.fetch, | |
| 213 | 217 | }); | |
| 214 | 218 | ||
| 215 | - let language: string | undefined; | ||
| 216 | - | ||
| 217 | - if (response.language && response.language in languageMap) { | ||
| 218 | - language = languageMap[response.language as keyof typeof languageMap]; | ||
| 219 | - } | ||
| 219 | + const language = | ||
| 220 | + response.language != null && response.language in languageMap | ||
| 221 | + ? languageMap[response.language as keyof typeof languageMap] | ||
| 222 | + : undefined; | ||
| 220 | 223 | ||
| 221 | 224 | return { | |
| 222 | 225 | text: response.text, | |
| 223 | - segments: response.words.map(word => ({ | ||
| 224 | - text: word.word, | ||
| 225 | - startSecond: word.start, | ||
| 226 | - endSecond: word.end, | ||
| 227 | - })), | ||
| 226 | + segments: | ||
| 227 | + response.words?.map(word => ({ | ||
| 228 | + text: word.word, | ||
| 229 | + startSecond: word.start, | ||
| 230 | + endSecond: word.end, | ||
| 231 | + })) ?? [], | ||
| 228 | 232 | language, | |
| 229 | - durationInSeconds: response.duration, | ||
| 233 | + durationInSeconds: response.duration ?? undefined, | ||
| 230 | 234 | warnings, | |
| 231 | 235 | response: { | |
| 232 | 236 | timestamp: currentDate, | |
| 233 | 237 | modelId: this.modelId, | |
| 234 | 238 | headers: responseHeaders, | |
| 235 | - body: response, | ||
| 236 | - }, | ||
| 237 | - | ||
| 238 | - // When using format `verbose_json` on `whisper-1`, OpenAI includes the things like `task` and enhanced `segments` information. | ||
| 239 | - providerMetadata: { | ||
| 240 | - openai: { | ||
| 241 | - transcript: response, | ||
| 242 | - }, | ||
| 239 | + body: rawResponse, | ||
| 243 | 240 | }, | |
| 244 | 241 | }; | |
| 245 | 242 | } | |
| 246 | 243 | } | |
| 247 | 244 | ||
| 248 | 245 | const openaiTranscriptionResponseSchema = z.object({ | |
| 249 | 246 | text: z.string(), | |
| 250 | - language: z.string().optional(), | ||
| 251 | - duration: z.number().optional(), | ||
| 252 | - words: z.array( | ||
| 253 | - z.object({ | ||
| 254 | - word: z.string(), | ||
| 255 | - start: z.number(), | ||
| 256 | - end: z.number(), | ||
| 257 | - }), | ||
| 258 | - ), | ||
| 247 | + language: z.string().nullish(), | ||
| 248 | + duration: z.number().nullish(), | ||
| 249 | + words: z | ||
| 250 | + .array( | ||
| 251 | + z.object({ | ||
| 252 | + word: z.string(), | ||
| 253 | + start: z.number(), | ||
| 254 | + end: z.number(), | ||
| 255 | + }), | ||
| 256 | + ) | ||
| 257 | + .nullish(), | ||
| 259 | 258 | }); | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -13,9 +13,11 @@ Accepts a `Uint8Array` or `string`, where `string` is a base64 encoded audio fil | |||
| 13 | 13 | audio: Uint8Array | string; | |
| 14 | 14 | ||
| 15 | 15 | /** | |
| 16 | - The MIME type of the audio data. | ||
| 16 | + The IANA media type of the audio data. | ||
| 17 | + | ||
| 18 | + @see https://www.iana.org/assignments/media-types/media-types.xhtml | ||
| 17 | 19 | */ | |
| 18 | - mimeType: string; | ||
| 20 | + mediaType: string; | ||
| 19 | 21 | ||
| 20 | 22 | /** | |
| 21 | 23 | Additional provider-specific options that are passed through to the provider | |
| Back | FazBrowse Home | New Git URL |
0 commit comments