[ Web Proxy ]
URL:
Viewing: https://raw.githubusercontent.com/TanStack/tanstack.com/main/src/utils/github-example.server.ts [Back]  [Original]

/**
 * GitHub Example Fetching Utilities
 *
 * Functions for fetching example files from GitHub repositories.
 */

import {
  cancelUnusedResponseBody,
  fetchGitHubRecursiveTree,
  getGitHubContentFetchOptions,
  GitHubContentError,
  isGitHubAuthFailureStatus,
} from './documents.server'
import { getCachedGitHubTextFile } from './github-content-cache.server'
import {
  fetchWithTimeout,
  readResponseBytesWithLimit,
  readResponseTextWithLimit,
} from './outbound-fetch.server'
import { encodeExampleBinaryFile } from './example-workspace'

const RAW_FETCH_CONCURRENCY = 6
const MAX_EXAMPLE_FILES = 500
const MAX_EXAMPLE_FILE_BYTES = 1 * 1024 * 1024
const MAX_EXAMPLE_TOTAL_BYTES = 5 * 1024 * 1024
const EXAMPLE_FETCH_TIMEOUT_MS = 10_000

export interface FetchExampleFilesResult {
  success: true
  files: Record
  binaryFiles?: Record
}

export interface FetchExampleFilesError {
  success: false
  error: string
  reason: 'fetch-failed' | 'not-found' | 'too-large' | 'too-many-files'
}

export function isFetchExampleFilesResponse(
  value: unknown,
): value is FetchExampleFilesResult | FetchExampleFilesError {
  if (!isRecord(value)) return false

  if (value.success === true) {
    return (
      isStringRecord(value.files) &&
      (value.binaryFiles === undefined || isStringRecord(value.binaryFiles))
    )
  }

  return (
    value.success === false &&
    typeof value.error === 'string' &&
    (value.reason === 'fetch-failed' ||
      value.reason === 'not-found' ||
      value.reason === 'too-large' ||
      value.reason === 'too-many-files')
  )
}

export function ensureCacheableFetchExampleFilesResponse(
  result: FetchExampleFilesResult | FetchExampleFilesError,
) {
  if (!result.success && result.reason === 'fetch-failed') {
    throw new Error(result.error)
  }

  return result
}

/**
 * Fetch all files from a GitHub example directory.
 */
export async function fetchExampleFiles(
  repo: string,
  branch: string,
  examplePath: string,
  options: { preserveBinary?: boolean } = {},
): Promise {
  console.log('[fetchExampleFiles] Fetching:', { repo, branch, examplePath })

  const tree = await fetchGitHubRecursiveTree(repo, branch)

  if (!tree) {
    return {
      success: false,
      error: `Failed to fetch example directory: ${examplePath}`,
      reason: 'not-found',
    }
  }

  const normalizedExamplePath = examplePath.replace(/^\/+|\/+$/g, '')
  const fileEntries = tree.filter(
    (entry) =>
      entry.type === 'blob' &&
      entry.path.startsWith(`${normalizedExamplePath}/`) &&
      !shouldExcludeFile(entry.path.slice(normalizedExamplePath.length + 1)),
  )

  if (fileEntries.length > MAX_EXAMPLE_FILES) {
    return {
      success: false,
      error: `Example has too many files; maximum is ${MAX_EXAMPLE_FILES}`,
      reason: 'too-many-files',
    }
  }

  const knownTotalBytes = fileEntries.reduce(
    (total, entry) => total + (entry.size ?? 0),
    0,
  )

  if (knownTotalBytes > MAX_EXAMPLE_TOTAL_BYTES) {
    return {
      success: false,
      error: `Example is too large; maximum is ${MAX_EXAMPLE_TOTAL_BYTES} bytes`,
      reason: 'too-large',
    }
  }

  if (fileEntries.length === 0) {
    return {
      success: false,
      error: `No files found in example path: ${examplePath}`,
      reason: 'not-found',
    }
  }

  try {
    let totalBytes = 0
    const fetchedFiles = await mapWithConcurrency(
      fileEntries,
      RAW_FETCH_CONCURRENCY,
      async (entry) => {
        const relativePath = entry.path.slice(normalizedExamplePath.length + 1)
        const content = options.preserveBinary
          ? await fetchRawGitHubWorkspaceFile(repo, branch, entry.path)
          : await fetchRawGitHubFile(repo, branch, entry.path)

        if (content === null) {
          throw new Error(`Missing file content for ${entry.path}`)
        }

        totalBytes +=
          typeof content === 'string'
            ? new TextEncoder().encode(content).byteLength
            : content.bytes.byteLength
        if (totalBytes > MAX_EXAMPLE_TOTAL_BYTES) {
          throw new Error(
            `Example is too large; maximum is ${MAX_EXAMPLE_TOTAL_BYTES} bytes`,
          )
        }

        return { content, relativePath }
      },
    )
    const files: Record = {}
    const binaryFiles: Record = {}

    for (const file of fetchedFiles) {
      if (typeof file.content === 'string') {
        files[file.relativePath] = file.content
      } else {
        binaryFiles[file.relativePath] = encodeExampleBinaryFile(
          file.content.bytes,
        )
      }
    }

    console.log(
      '[fetchExampleFiles] Fetched',
      Object.keys(files).length + Object.keys(binaryFiles).length,
      'files',
    )

    return {
      success: true,
      files,
      ...(Object.keys(binaryFiles).length ? { binaryFiles } : {}),
    }
  } catch (error) {
    console.error('[fetchExampleFiles] Failed:', error)

    return {
      success: false,
      error:
        error instanceof Error
          ? error.message
          : 'Failed to fetch example files',
      reason: 'fetch-failed',
    }
  }
}

async function fetchRawGitHubFile(
  repo: string,
  branch: string,
  filePath: string,
) {
  return getCachedGitHubTextFile({
    repo,
    gitRef: branch,
    path: filePath,
    origin: async () => {
      const response = await fetchRawGitHubResponse(repo, branch, filePath)
      if (!response) return null
      return readResponseTextWithLimit(response, MAX_EXAMPLE_FILE_BYTES)
    },
  })
}

async function fetchRawGitHubWorkspaceFile(
  repo: string,
  branch: string,
  filePath: string,
) {
  const response = await fetchRawGitHubResponse(repo, branch, filePath)
  if (!response) return null

  const bytes = await readResponseBytesWithLimit(
    response,
    MAX_EXAMPLE_FILE_BYTES,
  )

  try {
    const text = new TextDecoder('utf-8', {
      fatal: true,
      ignoreBOM: true,
    }).decode(bytes)
    const encoded = new TextEncoder().encode(text)
    if (
      !text.includes('\0') &&
      encoded.byteLength === bytes.byteLength &&
      encoded.every((byte, index) => byte === bytes[index])
    ) {
      return text
    }
  } catch {
    // Files that are not lossless UTF-8 must remain byte-for-byte intact.
  }

  return { bytes }
}

async function fetchRawGitHubResponse(
  repo: string,
  branch: string,
  filePath: string,
) {
  const href = new URL(
    `${repo}/${branch}/${filePath}`,
    'https://raw.githubusercontent.com/',
  ).href

  let response: Response

  try {
    response = await fetchWithTimeout(href, {
      ...getGitHubContentFetchOptions({
        includeApiVersion: false,
        userAgent: `examples:${repo}`,
      }),
      timeoutMs: EXAMPLE_FETCH_TIMEOUT_MS,
    })

    if (isGitHubAuthFailureStatus(response.status)) {
      await cancelUnusedResponseBody(response)
      response = await fetchWithTimeout(href, {
        ...getGitHubContentFetchOptions({
          includeApiVersion: false,
          includeAuthorization: false,
          userAgent: `examples:${repo}`,
        }),
        timeoutMs: EXAMPLE_FETCH_TIMEOUT_MS,
      })
    }
  } catch (error) {
    throw new GitHubContentError(
      'network',
      `Failed to fetch ${repo}@${branch}:${filePath}`,
      { cause: error },
    )
  }

  if (response.ok) return response

  await cancelUnusedResponseBody(response)
  if (response.status === 404) return null

  throw new GitHubContentError(
    response.status === 403 || response.status === 429
      ? 'rate-limit'
      : response.status >= 500
        ? 'server'
        : 'forbidden',
    `GitHub failed to serve ${repo}@${branch}:${filePath}`,
    { status: response.status },
  )
}

async function mapWithConcurrency(
  values: Array,
  concurrency: number,
  fn: (value: T) => Promise,
) {
  const results = new Array(values.length)
  let index = 0

  const workers = Array.from(
    { length: Math.min(concurrency, values.length) },
    async () => {
      while (index < values.length) {
        const currentIndex = index
        index += 1
        results[currentIndex] = await fn(values[currentIndex])
      }
    },
  )

  await Promise.all(workers)

  return results
}

/**
 * Files/directories to exclude when copying examples
 */
const EXCLUDED_PATTERNS = [
  'node_modules',
  '.git',
  'dist',
  '.output',
  '.netlify',
  '.vercel',
  '.DS_Store',
  '.nitro',
  'test-results',
  'playwright-report',
]

/**
 * Check if a file should be excluded
 */
export function shouldExcludeFile(path: string): boolean {
  return EXCLUDED_PATTERNS.some(
    (pattern) => path === pattern || path.startsWith(`${pattern}/`),
  )
}

/**
 * Filter out excluded files from a files record
 */
export function filterExcludedFiles(
  files: Record,
): Record {
  const filtered: Record = {}

  for (const [path, content] of Object.entries(files)) {
    if (!shouldExcludeFile(path)) {
      filtered[path] = content
    }
  }

  return filtered
}

function isRecord(value: unknown): value is Record {
  return typeof value === 'object' && value !== null && !Array.isArray(value)
}

function isStringRecord(value: unknown): value is Record {
  return (
    isRecord(value) &&
    Object.values(value).every((entry) => typeof entry === 'string')
  )
}

Web Proxy Viewer  |  New URL  |  Original Page