graph TB
    subgraph "Client Layer"
        Browser["Browser<br/>(React SPA)"]
        API_Client["External API Client<br/>(JWT / API Key)"]
    end

    subgraph "Gateway"
        Nginx["Nginx<br/>(SPA + reverse proxy)"]
    end

    subgraph "Application Layer"
        API["api-server<br/>(Axum HTTP + WS)<br/>N replicas, stateless"]
        Worker["worker<br/>(poll + claim + execute)<br/>N replicas"]
        Scheduler["scheduler-daemon<br/>(advisory-lock singleton)<br/>1 active + 1 standby"]
    end

    subgraph "Data Layer"
        PG[("PostgreSQL 16<br/>jobs · queues · workers<br/>executions · logs · DLQ")]
    end

    subgraph "Observability"
        Prom["Prometheus<br/>/metrics scraping"]
        Grafana["Grafana<br/>dashboards"]
    end

    Browser --> Nginx
    API_Client --> Nginx
    Nginx -->|"REST + WebSocket"| API
    Worker -->|"poll + claim<br/>LISTEN/NOTIFY"| PG
    Scheduler -->|"promote · cron · reap"| PG
    API -->|"queries"| PG
    PG --> Prom
    API --> Prom
    Worker --> Prom
    Scheduler --> Prom
    Prom --> Grafana

graph TB
    subgraph "bin/"
        API_BIN["api-server / main.rs"]
        WORKER_BIN["worker / main.rs"]
        SCHED_BIN["scheduler-daemon / main.rs"]
    end

    subgraph "crates/ — Application"
        API_CRATE["api<br/>Axum routes · middleware · DTOs<br/>error.rs · validation.rs"]
        WORKER_RT["worker-runtime<br/>executor · heartbeat · shutdown"]
    end

    subgraph "crates/ — Domain Logic"
        SCHED_CORE["scheduler-core<br/>claim · retry · cron · reaper"]
    end

    subgraph "crates/ — Domain (zero I/O)"
        DOMAIN["domain<br/>entities · value objects · ports (traits)"]
    end

    subgraph "crates/ — Infrastructure"
        INFRA["infra-postgres<br/>SQLx repository adapters"]
        COMMON["common<br/>config · telemetry · errors"]
    end

    subgraph "External"
        PG[("PostgreSQL")]
    end

    API_BIN --> API_CRATE
    WORKER_BIN --> WORKER_RT
    SCHED_BIN --> SCHED_CORE

    API_CRATE --> DOMAIN
    API_CRATE --> INFRA
    WORKER_RT --> DOMAIN
    WORKER_RT --> INFRA
    SCHED_CORE --> DOMAIN

    DOMAIN -. "defines traits" .-> INFRA
    INFRA --> PG
    COMMON --> DOMAIN

sequenceDiagram
    participant W1 as Worker 1
    participant W2 as Worker 2
    participant PG as PostgreSQL

    W1->>PG: BEGIN
    W2->>PG: BEGIN
    Note over W1,PG: CTE: SELECT id FROM jobs<br/>WHERE status='queued'<br/>ORDER BY priority DESC, run_at ASC<br/>FOR UPDATE SKIP LOCKED<br/>LIMIT free_permits
    W1->>PG: Locks rows 1,2,3 (SKIP LOCKED)
    W2->>PG: Locks rows 4,5 (skips 1,2,3)
    W1->>PG: UPDATE SET status='claimed', locked_by=W1<br/>RETURNING *
    W2->>PG: UPDATE SET status='claimed', locked_by=W2<br/>RETURNING *
    W1->>PG: COMMIT
    W2->>PG: COMMIT
    Note over W1,W2: Zero contention, zero duplicate execution

erDiagram
    users ||--o{ organization_members : "belongs to"
    organizations ||--o{ organization_members : "has"
    organizations ||--o{ projects : "owns"
    projects ||--o{ queues : "contains"
    projects ||--o{ retry_policies : "defines"
    projects ||--o{ job_batches : "groups"
    queues ||--o{ jobs : "holds"
    queues ||--o{ scheduled_jobs : "templates"
    queues ||--o{ dead_letter_queue : "failed"
    retry_policies ||--o| queues : "default for"
    retry_policies ||--o| jobs : "overrides"
    retry_policies ||--o| scheduled_jobs : "applies to"
    workers ||--o{ worker_heartbeats : "emits"
    workers ||--o{ jobs : "locks"
    workers ||--o{ job_executions : "executes"
    jobs ||--o{ job_executions : "has attempts"
    jobs ||--o{ job_logs : "produces"
    jobs ||--o{ job_dependencies : "depends on"
    jobs ||--o| dead_letter_queue : "dead-letters to"
    job_executions ||--o{ job_logs : "contains"
    job_batches ||--o{ jobs : "batches"
    scheduled_jobs ||--o{ jobs : "spawns"

    users {
        uuid id PK
        text email UK
        text password_hash
        text full_name
        boolean is_active
        timestamptz created_at
        timestamptz updated_at
    }

    organizations {
        uuid id PK
        text name
        text slug UK
        timestamptz created_at
    }

    organization_members {
        uuid org_id PK_FK
        uuid user_id PK_FK
        org_role role
    }

    projects {
        uuid id PK
        uuid org_id FK
        text name
        text api_key_hash UK
        uuid created_by FK
        timestamptz created_at
        timestamptz updated_at
    }

    queues {
        uuid id PK
        uuid project_id FK
        text name
        int priority
        int max_concurrency
        boolean is_paused
        uuid default_retry_policy_id FK
        int visibility_timeout_secs
        timestamptz created_at
        timestamptz updated_at
    }

    retry_policies {
        uuid id PK
        uuid project_id FK
        text name
        retry_strategy strategy
        bigint base_delay_ms
        float multiplier
        bigint max_delay_ms
        int max_attempts
        boolean jitter
        timestamptz created_at
    }

    jobs {
        uuid id PK
        uuid queue_id FK
        uuid scheduled_job_id FK
        uuid batch_id FK
        text job_type
        jsonb payload
        int priority
        job_status status
        timestamptz run_at
        text idempotency_key
        uuid retry_policy_id FK
        int max_attempts_override
        int attempt_count
        uuid locked_by FK
        timestamptz locked_at
        timestamptz lease_expires_at
        jsonb result
        text error_message
        text trace_id
        timestamptz created_at
        timestamptz updated_at
        timestamptz completed_at
    }

    job_executions {
        uuid id PK
        uuid job_id FK
        uuid worker_id FK
        int attempt_number
        execution_status status
        timestamptz started_at
        timestamptz finished_at
        bigint duration_ms
        text error_message
        text error_stack
        jsonb result
        timestamptz created_at
    }

    job_logs {
        bigserial id PK
        uuid execution_id FK
        uuid job_id FK
        timestamptz ts
        text level
        text message
    }

    workers {
        uuid id PK
        text hostname
        int pid
        text version
        uuid[] subscribed_queues
        int max_concurrency
        worker_status status
        timestamptz started_at
        timestamptz last_seen_at
    }

    worker_heartbeats {
        bigserial id PK
        uuid worker_id FK
        timestamptz ts
        int active_job_count
        float cpu_percent
        float memory_mb
        jsonb metadata
    }

    dead_letter_queue {
        uuid id PK
        uuid job_id FK
        uuid queue_id FK
        jsonb payload_snapshot
        text failure_reason
        int attempt_count
        text last_error
        timestamptz moved_at
        boolean resolved
        uuid resolved_by FK
        timestamptz resolved_at
        text resolution_action
    }

    scheduled_jobs {
        uuid id PK
        uuid queue_id FK
        text name
        text cron_expression
        text timezone
        text job_type
        jsonb payload_template
        uuid retry_policy_id FK
        boolean is_active
        timestamptz next_run_at
        timestamptz last_run_at
        timestamptz created_at
        timestamptz updated_at
    }

    job_batches {
        uuid id PK
        uuid project_id FK
        text name
        int total_jobs
        int completed_jobs
        int failed_jobs
        batch_status status
        timestamptz created_at
        timestamptz updated_at
    }

    job_dependencies {
        uuid job_id PK_FK
        uuid depends_on_job_id PK_FK
    }

graph LR
    subgraph "job_status"
        scheduled --> queued
        queued --> claimed
        claimed --> running
        running --> completed
        running --> failed
        failed --> scheduled
        failed --> dead_letter
        scheduled --> dead_letter
        claimed --> dead_letter
        running --> dead_letter
        queued --> cancelled
        scheduled --> cancelled
    end

stateDiagram-v2
    [*] --> scheduled : Job created with<br/>run_at in the future
    [*] --> queued : Job created with<br/>run_at ≤ now()
    scheduled --> queued : scheduler-daemon<br/>promotion sweep
    queued --> claimed : worker atomically claims<br/>(FOR UPDATE SKIP LOCKED)
    claimed --> running : worker begins execution
    running --> completed : handler returns Ok
    running --> failed : handler returns Err
    failed --> scheduled : attempt_count < max_attempts<br/>(retry with backoff)
    failed --> dead_letter : attempt_count ≥ max_attempts
    scheduled --> dead_letter : lease expired + attempts exhausted
    claimed --> scheduled : lease expired + attempts remain
    running --> scheduled : lease expired + attempts remain
    queued --> cancelled : user cancels
    scheduled --> cancelled : user cancels
    completed --> [*]
    dead_letter --> [*]
    cancelled --> [*]

    note right of failed
        Retry backoff reuses the
        "scheduled" state.
        attempt_count > 0
        distinguishes retries.
    end note