/**
* GitHub Example Fetching Utilities
*
* Functions for fetching example files from GitHub repositories.
*/
import {
cancelUnusedResponseBody,
fetchGitHubRecursiveTree,
getGitHubContentFetchOptions,
GitHubContentError,
isGitHubAuthFailureStatus,
} from './documents.server'
import { getCachedGitHubTextFile } from './github-content-cache.server'
import {
fetchWithTimeout,
readResponseBytesWithLimit,
readResponseTextWithLimit,
} from './outbound-fetch.server'
import { encodeExampleBinaryFile } from './example-workspace'
const RAW_FETCH_CONCURRENCY = 6
const MAX_EXAMPLE_FILES = 500
const MAX_EXAMPLE_FILE_BYTES = 1 * 1024 * 1024
const MAX_EXAMPLE_TOTAL_BYTES = 5 * 1024 * 1024
const EXAMPLE_FETCH_TIMEOUT_MS = 10_000
export interface FetchExampleFilesResult {
success: true
files: Record
binaryFiles?: Record
}
export interface FetchExampleFilesError {
success: false
error: string
reason: 'fetch-failed' | 'not-found' | 'too-large' | 'too-many-files'
}
export function isFetchExampleFilesResponse(
value: unknown,
): value is FetchExampleFilesResult | FetchExampleFilesError {
if (!isRecord(value)) return false
if (value.success === true) {
return (
isStringRecord(value.files) &&
(value.binaryFiles === undefined || isStringRecord(value.binaryFiles))
)
}
return (
value.success === false &&
typeof value.error === 'string' &&
(value.reason === 'fetch-failed' ||
value.reason === 'not-found' ||
value.reason === 'too-large' ||
value.reason === 'too-many-files')
)
}
export function ensureCacheableFetchExampleFilesResponse(
result: FetchExampleFilesResult | FetchExampleFilesError,
) {
if (!result.success && result.reason === 'fetch-failed') {
throw new Error(result.error)
}
return result
}
/**
* Fetch all files from a GitHub example directory.
*/
export async function fetchExampleFiles(
repo: string,
branch: string,
examplePath: string,
options: { preserveBinary?: boolean } = {},
): Promise {
console.log('[fetchExampleFiles] Fetching:', { repo, branch, examplePath })
const tree = await fetchGitHubRecursiveTree(repo, branch)
if (!tree) {
return {
success: false,
error: `Failed to fetch example directory: ${examplePath}`,
reason: 'not-found',
}
}
const normalizedExamplePath = examplePath.replace(/^\/+|\/+$/g, '')
const fileEntries = tree.filter(
(entry) =>
entry.type === 'blob' &&
entry.path.startsWith(`${normalizedExamplePath}/`) &&
!shouldExcludeFile(entry.path.slice(normalizedExamplePath.length + 1)),
)
if (fileEntries.length > MAX_EXAMPLE_FILES) {
return {
success: false,
error: `Example has too many files; maximum is ${MAX_EXAMPLE_FILES}`,
reason: 'too-many-files',
}
}
const knownTotalBytes = fileEntries.reduce(
(total, entry) => total + (entry.size ?? 0),
0,
)
if (knownTotalBytes > MAX_EXAMPLE_TOTAL_BYTES) {
return {
success: false,
error: `Example is too large; maximum is ${MAX_EXAMPLE_TOTAL_BYTES} bytes`,
reason: 'too-large',
}
}
if (fileEntries.length === 0) {
return {
success: false,
error: `No files found in example path: ${examplePath}`,
reason: 'not-found',
}
}
try {
let totalBytes = 0
const fetchedFiles = await mapWithConcurrency(
fileEntries,
RAW_FETCH_CONCURRENCY,
async (entry) => {
const relativePath = entry.path.slice(normalizedExamplePath.length + 1)
const content = options.preserveBinary
? await fetchRawGitHubWorkspaceFile(repo, branch, entry.path)
: await fetchRawGitHubFile(repo, branch, entry.path)
if (content === null) {
throw new Error(`Missing file content for ${entry.path}`)
}
totalBytes +=
typeof content === 'string'
? new TextEncoder().encode(content).byteLength
: content.bytes.byteLength
if (totalBytes > MAX_EXAMPLE_TOTAL_BYTES) {
throw new Error(
`Example is too large; maximum is ${MAX_EXAMPLE_TOTAL_BYTES} bytes`,
)
}
return { content, relativePath }
},
)
const files: Record = {}
const binaryFiles: Record = {}
for (const file of fetchedFiles) {
if (typeof file.content === 'string') {
files[file.relativePath] = file.content
} else {
binaryFiles[file.relativePath] = encodeExampleBinaryFile(
file.content.bytes,
)
}
}
console.log(
'[fetchExampleFiles] Fetched',
Object.keys(files).length + Object.keys(binaryFiles).length,
'files',
)
return {
success: true,
files,
...(Object.keys(binaryFiles).length ? { binaryFiles } : {}),
}
} catch (error) {
console.error('[fetchExampleFiles] Failed:', error)
return {
success: false,
error:
error instanceof Error
? error.message
: 'Failed to fetch example files',
reason: 'fetch-failed',
}
}
}
async function fetchRawGitHubFile(
repo: string,
branch: string,
filePath: string,
) {
return getCachedGitHubTextFile({
repo,
gitRef: branch,
path: filePath,
origin: async () => {
const response = await fetchRawGitHubResponse(repo, branch, filePath)
if (!response) return null
return readResponseTextWithLimit(response, MAX_EXAMPLE_FILE_BYTES)
},
})
}
async function fetchRawGitHubWorkspaceFile(
repo: string,
branch: string,
filePath: string,
) {
const response = await fetchRawGitHubResponse(repo, branch, filePath)
if (!response) return null
const bytes = await readResponseBytesWithLimit(
response,
MAX_EXAMPLE_FILE_BYTES,
)
try {
const text = new TextDecoder('utf-8', {
fatal: true,
ignoreBOM: true,
}).decode(bytes)
const encoded = new TextEncoder().encode(text)
if (
!text.includes('\0') &&
encoded.byteLength === bytes.byteLength &&
encoded.every((byte, index) => byte === bytes[index])
) {
return text
}
} catch {
// Files that are not lossless UTF-8 must remain byte-for-byte intact.
}
return { bytes }
}
async function fetchRawGitHubResponse(
repo: string,
branch: string,
filePath: string,
) {
const href = new URL(
`${repo}/${branch}/${filePath}`,
'https://raw.githubusercontent.com/',
).href
let response: Response
try {
response = await fetchWithTimeout(href, {
...getGitHubContentFetchOptions({
includeApiVersion: false,
userAgent: `examples:${repo}`,
}),
timeoutMs: EXAMPLE_FETCH_TIMEOUT_MS,
})
if (isGitHubAuthFailureStatus(response.status)) {
await cancelUnusedResponseBody(response)
response = await fetchWithTimeout(href, {
...getGitHubContentFetchOptions({
includeApiVersion: false,
includeAuthorization: false,
userAgent: `examples:${repo}`,
}),
timeoutMs: EXAMPLE_FETCH_TIMEOUT_MS,
})
}
} catch (error) {
throw new GitHubContentError(
'network',
`Failed to fetch ${repo}@${branch}:${filePath}`,
{ cause: error },
)
}
if (response.ok) return response
await cancelUnusedResponseBody(response)
if (response.status === 404) return null
throw new GitHubContentError(
response.status === 403 || response.status === 429
? 'rate-limit'
: response.status >= 500
? 'server'
: 'forbidden',
`GitHub failed to serve ${repo}@${branch}:${filePath}`,
{ status: response.status },
)
}
async function mapWithConcurrency(
values: Array,
concurrency: number,
fn: (value: T) => Promise,
) {
const results = new Array(values.length)
let index = 0
const workers = Array.from(
{ length: Math.min(concurrency, values.length) },
async () => {
while (index < values.length) {
const currentIndex = index
index += 1
results[currentIndex] = await fn(values[currentIndex])
}
},
)
await Promise.all(workers)
return results
}
/**
* Files/directories to exclude when copying examples
*/
const EXCLUDED_PATTERNS = [
'node_modules',
'.git',
'dist',
'.output',
'.netlify',
'.vercel',
'.DS_Store',
'.nitro',
'test-results',
'playwright-report',
]
/**
* Check if a file should be excluded
*/
export function shouldExcludeFile(path: string): boolean {
return EXCLUDED_PATTERNS.some(
(pattern) => path === pattern || path.startsWith(`${pattern}/`),
)
}
/**
* Filter out excluded files from a files record
*/
export function filterExcludedFiles(
files: Record,
): Record {
const filtered: Record = {}
for (const [path, content] of Object.entries(files)) {
if (!shouldExcludeFile(path)) {
filtered[path] = content
}
}
return filtered
}
function isRecord(value: unknown): value is Record {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
function isStringRecord(value: unknown): value is Record {
return (
isRecord(value) &&
Object.values(value).every((entry) => typeof entry === 'string')
)
}