#! /usr/bin/env ruby

# if :development, submit cases to localhost:3000 and load local mesa_test library
# rather than system version and run out of local lib, not installed
# if :staging, submit to beta version and run out of local lib, not installed
# anything else: trust rubygems to get lib and submit to real deal testhub
MODE = :production

case MODE
when :staging, :development
  require_relative '../lib/mesa_test'
else
  require 'mesa_test'
end

require 'thor'
require 'yaml'
require 'etc'

class MesaTest < Thor
  DEVELOPMENT_URI = 'http://localhost:3000'.freeze
  STAGING_URI = 'https://beta-testhub.herokuapp.com'.freeze

  # Exit status when MESATestHub has nothing for this computer to do, so a
  # script can stop instead of re-testing whatever is already in the work
  # directory:  mesa_test install best || exit
  NOTHING_TO_DO_EXIT = 3

  def self.exit_on_failure?
    true
  end

  # --claim / --no-claim, shared by every command that starts work.
  def self.claim_option
    option :claim, desc: 'Tell MESATestHub this computer is working on the ' \
                         'commit (and its tests) so it shows as pending, not ' \
                         'untested. Use --no-claim if something else already ' \
                         'claimed it.',
                   type: :boolean, default: true
  end

  # Parallel testing (opt-in; with neither set, tests run serially in your
  # shell's environment exactly as before).
  def self.parallel_options
    option :jobs, desc: 'Run this many tests at once (each in its own ' \
                        'process; output in each_test_run.log in the test ' \
                        'directory). Default 1: one after another.',
                  type: :numeric, default: 1, aliases: '-j'
    option :test_threads, desc: 'OMP_NUM_THREADS for each test. Default: ' \
                                "your shell's OMP_NUM_THREADS (else all cores) " \
                                'split evenly across --jobs.',
                          type: :numeric
  end

  # NPROCS (MESA's install runs make -j$NPROCS; all cores if unset) and
  # OMP_NUM_THREADS for the build.
  def self.build_threads_option
    option :build_threads, desc: 'Cores for compiling MESA (sets NPROCS and ' \
                                 'OMP_NUM_THREADS for the install; default: ' \
                                 'all cores).',
                           type: :numeric
  end

  # The run-mode switches. Each is three-way: --x, --no-x, or unset. What
  # unset means depends on the command (see each command's help).
  def self.run_mode_options
    option :skip_optional, desc: 'Skip optional inlists (sets MESA_SKIP_OPTIONAL); ' \
                                 '--no-skip-optional runs them all, as MESA does ' \
                                 'by default.',
                           type: :boolean
    option :fpe, desc: 'Floating-point exception checks (MESA_FPE_CHECKS_ON=1). ' \
                       'Compiled in, so it is chosen when MESA is installed.',
                 type: :boolean
    option :converge, desc: 'Convergence run (sets MESA_TEST_SUITE_RESOLUTION_FACTOR, ' \
                            "keeping your value if set, else #{DEFAULT_CONVERGE_FACTOR}).",
                      type: :boolean
    option :resolution_factor, desc: 'Convergence run at this factor (implies --converge). ' \
                                     'With `best`, a factor the commit names ' \
                                     '([ci converge 0.5]) wins.',
                               type: :numeric
  end

  desc 'test [TEST CASE]', 'run and submit a test case or commit'
  long_desc <<-LONGDESC
    Runs TEST_CASE, or with no TEST_CASE every test case, in the MESA
    installation in your work directory (mesa_work in ~/.mesa_test/config.yml,
    or --mesadir). TEST_CASE can be a name or an integer: its position in
    do1_test_source (or in the modules' lists concatenated, if no --module is
    given). Modules are searched/concatenated in this order:

    #{MesaTestCase.modules.map { |mod| '    ' + mod.to_s }.join(', ')}

    Each test is claimed on MESATestHub as it starts, so it shows as pending,
    and its result is submitted as soon as it finishes. This needs an API key
    (see `mesa_test help setup`). --no-submit runs tests locally only:
    nothing is claimed or sent, and no key is needed.

    With no TEST_CASE, --jobs N runs N tests at once.

    Run modes (--skip-optional, --fpe, --converge): a flag given here wins;
    otherwise the mode recorded when this build was installed (in
    $MESA_DIR/testhub.yml) applies; otherwise your shell's environment
    does. FPE checks are compiled in, so a build installed with or without
    them keeps that setting whatever is passed here.
  LONGDESC

  option :force, desc: 'Use ~/.mesa_test/config.yml as it is. --no-force ' \
                       'shows it and asks you to confirm (or fix) it first.',
                 type: :boolean, default: true, aliases: '-f'
  option 'force-logs', desc: 'Submit logs for test case(s), even if they pass',
                       type: :boolean, default: false
  option :submit, desc: 'Submit results (if any) to MESATestHub.',
                  type: :boolean, default: true
  option :module, desc: 'Which module to test and/or submit, if applicable.',
                  type: :string, default: 'all', aliases: '-m'
  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
                   type: :string
  claim_option
  run_mode_options
  parallel_options

  def test(test_case_name = nil)
    validate_options!
    s = if options[:submit]
          create_and_check_submitter(force: options[:force])
        else
          create_submitter(force: options[:force])
        end
    m = create_and_check_mesa(
      mesa_dir: work_dir(options[:mesadir], s.mesa_work),
      mirror_dir: s.mesa_mirror,
      mod: options[:module].downcase.to_sym,
      github_protocol: s.github_protocol
    )
    modes = test_run_modes(m)
    with_env(run_mode_env(modes).merge(test_thread_env)) do
      m.with_mesa_dir do
        if test_case_name
          # only testing one test case
          t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
                                         mod: options[:module].downcase.to_sym)

          if options[:submit] && options[:claim]
            s.claim(sha: m.sha, scope: 'test', mod: t.mod, test_case: t.test_name,
                    flags: claim_flags(modes), mesa: m)
          end

          say_step(shell, "Testing #{t.mod}/#{t.test_name}")
          started = Time.now
          t.do_one
          finish_test(s, m, t, started, submit: options[:submit])
        else
          # Each test is claimed as it starts and submitted as it finishes.
          # (`install` reports the build.)
          run_tests(s, m, mod: options[:module].downcase.to_sym,
                          submit: options[:submit], flags: claim_flags(modes))
        end
      end
    end
  end

  desc 'submit [TEST CASE]', 'submit a test case or commit'
  long_desc <<-LONGDESC
    Submits results already in your work directory (mesa_work in
    ~/.mesa_test/config.yml, or --mesadir) without running anything: say, from
    a cluster's head node after jobs ran with --no-submit, or to retry after a
    network failure. With TEST_CASE (a name or number, as for `test`), just
    that test's result. Without it, the build status plus every test with
    results from this build (or with --empty, just the build status), in one
    request. Results left over from an earlier build are skipped, and
    MESATestHub ignores results it already has, so submitting again is safe.
    Needs an API key (see `mesa_test help setup`). Modules are
    searched/concatenated in this order:

    #{MesaTestCase.modules.map { |mod| '    ' + mod.to_s }.join(', ')}
  LONGDESC

  option :force, desc: 'Use ~/.mesa_test/config.yml as it is. --no-force ' \
                       'shows it and asks you to confirm (or fix) it first.',
                 type: :boolean, default: true, aliases: '-f'
  option 'force-logs', desc: 'Submit logs for test case(s), even if they pass',
                       type: :boolean, default: false
  option :module, desc: 'Which module to test and/or submit, if applicable.',
                  type: :string, default: 'all', aliases: '-m'
  option :empty, desc: 'Submit only the build status, no test results.',
                 type: :boolean, default: false, aliases: '-e'
  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
                   type: :string

  def submit(test_case_name = nil)
    s = create_and_check_submitter(force: options[:force])
    m = create_and_check_mesa(
      mesa_dir: work_dir(options[:mesadir], s.mesa_work),
      mirror_dir: s.mesa_mirror,
      mod: options[:module].downcase.to_sym,
      github_protocol: s.github_protocol
    )
    empty = options[:empty]

    # default value for now; might change to false if we have a bad
    # installation or if a test case is specified
    m.with_mesa_dir do
      if test_case_name
        # just submitting one test case.
        # this is not an empty submission since there is one test case (the
        # +empty+ option is ignored in this case)
  
        t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
                                       mod: options[:module].downcase.to_sym)
        raise Thor::Error, "#{t.mod}/#{t.test_name} has no results to submit." unless t.ran?

        s.submit_instance(m, t, force_logs: options['force-logs'])
      else
        # The build status plus (unless --empty) every test with results from
        # this build, in one request. MESATestHub stores it as a build row
        # plus one row per test and ignores anything it already has (by
        # fingerprint), so re-running `submit` is safe.
        begin
          m.check_installation
        rescue MesaDirError
          shell.say "This build didn't finish compiling, so only the failed build " \
                    'will be reported.', :yellow
          empty = true
        end
        s.submit_commit(m, empty: empty, force_logs: options['force-logs'])
      end
    end
  end

  desc 'checkout [SHA]', 'checkout a commit'
  long_desc <<-LONGDESC
  Checks out SHA (default: main) into your work directory (mesa_work in
  ~/.mesa_test/config.yml, or --mesadir), using git worktrees off the mirror
  so MESA's history isn't downloaded again. Doesn't build, doesn't contact
  MESATestHub, and doesn't touch your own MESA_DIR or environment.
  LONGDESC

  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
                   type: :string

  def checkout(sha = 'main')
    # assuming config wizard has already been set (perhaps a little reckless)
    # so that we can get at the work directory and mirror directory
    s = create_submitter(force: true)
    Mesa.checkout(sha: sha,
                  work_dir: work_dir(options[:mesadir], s.mesa_work),
                  mirror_dir: s.mesa_mirror,
                  github_protocol: s.github_protocol)
  end

  desc 'install [SHA]', 'checkout and install a commit'
  long_desc <<-LONGDESC
  Checks out SHA (default: main) into your work directory (mesa_work in
  ~/.mesa_test/config.yml, or --mesadir) and builds it: ./clean and
  ./install, with git worktrees off the mirror so MESA's history isn't
  downloaded again. The SDK or compilers must already be set up. Doesn't
  touch your own MESA_DIR or environment. With --no-checkout, builds what's
  already there and ignores SHA.

  The build is claimed and its result (pass or fail, with the build log)
  reported to MESATestHub, which needs an API key (see `mesa_test help
  setup`). A failed build exits with status 1. --no-submit just builds, for
  local work: nothing is claimed or sent, and no key is needed.

  Run modes given here (--skip-optional, --fpe, --converge) are used for
  the build and recorded in $MESA_DIR/testhub.yml, so later `mesa_test test`
  runs on this build (e.g. cluster array jobs) apply them automatically.

  If SHA is `best`, MESATestHub picks the commit this computer should build
  next, and the run modes come from its request instead: the mode flags then
  say what this computer is *willing* to do (overriding `capabilities` in the
  config file). If nothing needs building, mesa_test exits with status
  #{NOTHING_TO_DO_EXIT} without touching the work directory, so a script can
  stop:

      mesa_test install best || exit
  LONGDESC

  option :checkout, desc: 'Whether a checkout needs to happen. Use '\
                          '--no-checkout if you already ran checkout.',
                    type: :boolean, default: true, aliases: '-c'
  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
                   type: :string
  option :submit, desc: 'Report the build (claim it first, then submit its ' \
                        'status) to MESATestHub. --no-submit just builds, ' \
                        'e.g. for local testing.',
                  type: :boolean, default: true
  claim_option
  run_mode_options
  build_threads_option

  def install(sha = 'main')
    validate_options!
    # assuming config wizard has already been set (perhaps a little reckless)
    # so that we can get at the work directory and mirror directory
    s = if options[:submit] || sha == 'best'
          create_and_check_submitter(force: true)
        else
          create_submitter(force: true)
        end
    rec = nil
    if sha == 'best'
      rec = dispatch_build_or_exit(s)
      sha = rec['commit_sha']
    end
    modes = rec ? modes_for(rec) : explicit_run_modes
    m = if options[:checkout] || rec
          Mesa.checkout(
            sha: sha,
            work_dir: work_dir(options[:mesadir], s.mesa_work),
            mirror_dir: s.mesa_mirror,
            github_protocol: s.github_protocol
          )
        else
          create_and_check_mesa(
            mesa_dir: work_dir(options[:mesadir], s.mesa_work),
            mirror_dir: s.mesa_mirror,
            mod: :all,
            github_protocol: s.github_protocol
          )
        end
    if options[:submit] && options[:claim]
      s.claim(sha: m.sha, scope: 'build', flags: claim_flags(modes),
              dispatched_at: rec && rec['dispatched_at'])
    end
    built = begin
      build_with_modes(m, modes, raise_on_failure: true)
      true
    rescue MesaDirError
      false
    end
    # Report the build either way (a failure carries its build log), like
    # `test` reports each test.
    submit_build(s, m) if options[:submit]
    exit 1 unless built
  end

  desc 'install_and_test [SHA]', 'Install, test, and submit an entire commit.'
  long_desc <<-LONGDESC
  Checks out SHA (default: main) into your work directory (mesa_work in
  ~/.mesa_test/config.yml, or --mesadir), builds it (./clean and ./install;
  the SDK or compilers must already be set up), and runs the test suite.
  Doesn't touch your own MESA_DIR or environment. Roughly a shortcut for

    mesa_test install SHA && mesa_test test

  The build status is submitted as soon as the build finishes (a failed
  build is reported and exits 1), then each test is claimed as it starts
  and submitted as it finishes, so results roll in during the run. This
  needs an API key (see `mesa_test help setup`). --no-submit builds and
  tests locally without claiming or sending anything. With --no-checkout,
  the build in the work directory is used and SHA is ignored.

  --jobs N runs N tests at once; see also --test-threads and --build-threads.

  Run modes (--skip-optional, --fpe, --converge) apply to the build and
  every test; unset ones are left to your shell's environment.

  If SHA is `best`, MESATestHub picks the commit (one this computer hasn't
  tested yet) and the run modes (FPE checks, plus full inlists or a
  convergence run if the commit asks for one); everything else is the
  ordinary run above. The mode flags then say what this computer is
  willing to do (overriding `capabilities` in the config file): commits
  asking for [ci optional] or [ci converge] get those runs only if it's
  willing. --no-submit doesn't apply in this mode. If nothing needs
  testing, mesa_test exits with status #{NOTHING_TO_DO_EXIT}.
  LONGDESC

  option :checkout, desc: 'Whether a checkout needs to happen. Use '\
                          '--no-checkout if you already ran checkout.',
                    type: :boolean, default: true, aliases: '-c'
  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
                   type: :string
  option :force, desc: 'Use ~/.mesa_test/config.yml as it is. --no-force ' \
                       'shows it and asks you to confirm (or fix) it first.',
                 type: :boolean, default: true, aliases: '-f'
  option :submit, desc: 'Submit results (if any) to MESATestHub.',
                  type: :boolean, default: true
  option :module, desc: 'Which module to test and/or submit, if applicable.',
                  type: :string, default: 'all', aliases: '-m'
  option :destroy, type: :boolean, desc: 'Delete mesa installation upon ' \
                   'submission of complete results.', default: false
  claim_option
  run_mode_options
  build_threads_option
  parallel_options

  def install_and_test(sha='main')
    validate_options!
    return install_and_test_best if sha == 'best'

    s = if options[:submit]
          create_and_check_submitter(force: options[:force])
        else
          # not submitting? don't check config file (risky!) still need
          # submitter to get work and mirror directories, though
          create_submitter(force: true)
        end

    m = if options[:checkout]
          Mesa.checkout(
            sha: sha,
            work_dir: work_dir(options[:mesadir], s.mesa_work),
            mirror_dir: s.mesa_mirror,
            github_protocol: s.github_protocol
          )
        else
          create_and_check_mesa(
            mesa_dir: work_dir(options[:mesadir], s.mesa_work),
            mirror_dir: s.mesa_mirror,
            mod: options[:module].to_sym,
            github_protocol: s.github_protocol
          )
        end

    modes = explicit_run_modes
    claiming = options[:submit] && options[:claim]
    s.claim(sha: m.sha, scope: 'build', flags: claim_flags(modes)) if claiming

    begin
      build_with_modes(m, modes, raise_on_failure: true)
    rescue MesaDirError
      # Report the failed build (with its log) right away.
      submit_build(s, m) if options[:submit]
      exit 1
    end

    check_mesa(m)
    update_mesa(mesa: m, mod: options[:module].downcase.to_sym)
    # The build's status goes in as soon as it's known, then each test as
    # soon as it finishes -- results roll in rather than arriving in one
    # bulk submission at the end.
    build_submitted = options[:submit] ? submit_build(s, m) : true
    all_submitted = with_env(run_mode_env(m.recorded_run_modes).merge(test_thread_env)) do
      m.with_mesa_dir do
        run_tests(s, m, mod: options[:module].downcase.to_sym,
                        submit: options[:submit], flags: claim_flags(modes))
      end
    end

    # if requested, installation and submission are successful, destroy the
    # directory
    if options[:submit] && build_submitted && all_submitted && options[:destroy] && m.installed?
      m.remove
    end
  end

  desc 'request_work', 'ask MESATestHub what this computer should do next'
  long_desc <<-LONGDESC
  Asks MESATestHub which commit this computer should build next (or, with
  --scope=test --sha=SHA, which test to run next on a commit it built) and
  prints the recommendation as JSON on STDOUT, including the run modes to
  use and the reasons it was picked. The mode flags say what this computer
  is willing to do (overriding `capabilities` in the config file). Prints
  nothing and exits with status #{NOTHING_TO_DO_EXIT} when there is nothing
  useful to do. Read-only: nothing is claimed. Needs an API key (see
  `mesa_test help setup`).
  LONGDESC

  option :scope, desc: 'build or test', type: :string, default: 'build',
                 enum: %w[build test]
  option :sha, desc: 'For --scope=test: the commit you built', type: :string
  run_mode_options

  def request_work
    s = create_submitter(force: true, hub: true)
    rec = dispatch_or_exit(s, scope: options[:scope], sha: options[:sha])
    exit NOTHING_TO_DO_EXIT unless rec

    puts JSON.pretty_generate(rec)
  end

  desc 'setup [CONFIG_FILE]', 'Set up this computer (API key, test modes, directories)'
  long_desc <<-LONGDESC
    Walks through this computer's settings, offering the current ones as
    defaults, and saves them to CONFIG_FILE (default ~/.mesa_test/config.yml,
    which is where the other commands look).

    mesa_test authenticates with a per-computer API key. To get one, log in
    to MESATestHub, open My Computers, open this computer's page (adding the
    computer first if needed; one used only for `search` and `count` is
    fine), and click "Generate key". The key starts with mth_. Paste it when
    the wizard asks, or export MESATESTHUB_API_KEY, which overrides the
    config file (e.g. to keep the key off disk on a cluster).

    The wizard checks the key with MESATestHub, which also supplies this
    computer's name. Then it asks which optional test modes the computer is
    willing to run when a commit asks for them (full_inlists, fpe, converge),
    the logs token, and where the MESA mirror and work directory live.
  LONGDESC
  def setup(config_file = File.join(ENV['HOME'], '.mesa_test', 'config.yml'))
    MesaTestSubmitter.new_from_config(config_file: config_file,
                                      force_setup: true)
  end

  desc 'search "QUERY"', 'retrieve JSON test instances matching QUERY'
  long_desc <<-LONGDESC
  Sends a GET request to the MESATestHub search API. QUERY should be
  quoted. Authenticates with your API key; a key for any of your computers
  works, since search is read-only.

  Query syntax is the same key-value form used by the in-browser search box
  at testhub.mesastar.org/test_instances/search (see the "Query syntax"
  panel on that page). Example:

    mesa_test search "computer: Hercules; passed: false"

  The raw JSON response body is written to STDOUT, suitable for piping to
  jq or redirecting to a file. If the server reports any rejected query
  clauses in a `failures` array, those are also echoed to STDERR so they
  aren't silently dropped from downstream pipelines. On a non-success HTTP
  status, an error is written to STDERR and the command exits non-zero.
  LONGDESC

  def search(query)
    s = create_submitter(force: true, hub: true)
    emit_search_response(s.search(query))
  end

  desc 'count "QUERY"', 'count test instances matching QUERY'
  long_desc <<-LONGDESC
  Like `search`, but hits the count endpoint. The response is a small JSON
  object containing the result-set size (and any `failures`). Use this
  before `search` when you don't know how large a result set will be.
  LONGDESC

  def count(query)
    s = create_submitter(force: true, hub: true)
    emit_search_response(s.search_count(query))
  end

  private

  # `install_and_test best`: MESATestHub picks the commit, then each test.
  def install_and_test_best
    unless options[:submit]
      raise Thor::Error, '`install_and_test best` submits as it goes; ' \
                         '--no-submit is not supported.'
    end
    s = create_and_check_submitter(force: options[:force])
    rec = dispatch_build_or_exit(s)
    modes = modes_for(rec)
    m = Mesa.checkout(sha: rec['commit_sha'],
                      work_dir: work_dir(options[:mesadir], s.mesa_work),
                      mirror_dir: s.mesa_mirror,
                      github_protocol: s.github_protocol)
    if options[:claim]
      s.claim(sha: m.sha, scope: 'build', flags: claim_flags(modes),
              dispatched_at: rec['dispatched_at'])
    end

    begin
      build_with_modes(m, modes, raise_on_failure: true)
    rescue MesaDirError
      submit_build(s, m)
      exit 1
    end
    mod = options[:module].downcase.to_sym
    check_mesa(m)
    update_mesa(mesa: m, mod: mod)

    # Report the build on its own; this also fulfills the build claim.
    build_submitted = submit_build(s, m)
    # MESATestHub chose the commit and its modes; from here it's an ordinary
    # whole-suite run, as `install_and_test SHA` does it: MESA's order,
    # --jobs at a time, each test claimed as it starts and submitted as it
    # finishes.
    all_submitted = with_env(run_mode_env(m.recorded_run_modes).merge(test_thread_env)) do
      m.with_mesa_dir do
        run_tests(s, m, mod: mod, submit: true, flags: claim_flags(modes))
      end
    end
    m.remove if build_submitted && all_submitted && options[:destroy] && m.installed?
  end


  def jobs
    j = options[:jobs].to_i
    raise Thor::Error, '--jobs must be at least 1.' if j < 1

    j
  end

  # OMP_NUM_THREADS for each test. --test-threads if given. Otherwise, with
  # --jobs N, the shell's OMP_NUM_THREADS (else the core count) is treated
  # as the budget for the whole run and split evenly across the N tests,
  # rounded down, at least 1 -- a value set for serial runs shouldn't
  # silently become N times as many threads. Serial runs leave the shell
  # alone. Computed once, before anything changes ENV.
  def test_thread_env
    @test_thread_env ||= begin
      n = options[:test_threads]
      if n
        raise Thor::Error, '--test-threads must be at least 1.' if n.to_i < 1

        { 'OMP_NUM_THREADS' => n.to_i.to_s }
      elsif jobs > 1
        { 'OMP_NUM_THREADS' => split_threads.to_s }
      else
        {}
      end
    end
  end

  def split_threads
    from_shell = ENV['OMP_NUM_THREADS'].to_i
    budget, source = if from_shell.positive?
                       [from_shell, "OMP_NUM_THREADS=#{from_shell}"]
                     else
                       [Etc.nprocessors, "#{Etc.nprocessors} cores"]
                     end
    each = [budget / jobs, 1].max
    if budget < jobs
      shell.say "--jobs #{jobs} is more than your #{source}; each test gets 1 thread " \
                "(#{jobs} in all).", :yellow
    else
      idle = budget - each * jobs
      @thread_note = "#{source} split #{jobs} ways#{"; #{idle} left idle" if idle.positive?}; " \
                     '--test-threads sets it directly'
    end
    each
  end

  def build_thread_env
    n = options[:build_threads]
    return {} if n.nil?
    raise Thor::Error, '--build-threads must be at least 1.' if n.to_i < 1

    { 'NPROCS' => n.to_i.to_s, 'OMP_NUM_THREADS' => n.to_i.to_s }
  end

  # Submit the results +t+ wrote during the run that began at +started+.
  # If it wrote none (each_test_run died early, the run was killed), there
  # is nothing to report: an older testhub.yml must not be sent as this
  # run's result. Its claim is left to lapse, which tells the hub the work
  # was abandoned. Returns whether a result went through.
  def submit_result(s, m, t, started)
    return false unless t.ran_since?(started)

    s.submit_instance(m, t, force_logs: options['force-logs'], quiet: true)
  end

  # Check numeric options before anything is checked out or built, so a
  # typo fails in a second rather than after a long compile.
  def validate_options!
    { jobs: 1, test_threads: 1, build_threads: 1 }.each do |name, min|
      value = options[name]
      next if value.nil?
      next if value.is_a?(Numeric) && value == value.to_i && value >= min

      raise Thor::Error, "--#{name.to_s.tr('_', '-')} must be a whole number of at least #{min}."
    end
    explicit_factor
    nil
  end

  # Report this build's status (and its log, if it failed). Returns whether
  # the report went through; false if there was nothing to report.
  def submit_build(s, m)
    unless m.install_attempted?
      shell.say 'No build to report (the install never got far enough to ' \
                'write testhub.yml).', :red
      return false
    end
    s.submit_commit(m, empty: true)
  end

  # The loaded tests of +mod+ (or every module), in do1_test_source order.
  def tests_for(m, mod)
    mods = mod == :all ? MesaTestCase.modules : [mod]
    mods.flat_map do |md|
      (m.test_case_names[md] || []).filter_map { |name| (m.test_cases[md] || {})[name] }
    end
  end

  # Run every test in +mod+ (or all modules) on +m+'s build, in the current
  # environment and MESA_DIR: one at a time (MESA's output on the terminal,
  # as before), or --jobs at a time (output in each test's
  # each_test_run.log). With +submit+, each test is claimed just as it
  # starts and submitted as soon as it finishes, so a claim always means
  # "running now" and results arrive continuously -- no bulk submission.
  # Returns true if every submission went through.
  def run_tests(s, m, mod:, submit:, flags: {})
    m.check_installation
    tests = tests_for(m, mod)
    sha = m.sha
    claiming = submit && options[:claim]
    if jobs > 1
      threads = test_thread_env['OMP_NUM_THREADS'] || ENV['OMP_NUM_THREADS']
      say_step(shell, "Testing #{tests.size} tests, #{jobs} at a time" \
                      "#{", #{threads} thread#{'s' unless threads == '1'} each" if threads}")
      shell.say "(#{@thread_note})" if @thread_note
      shell.say "Each test's output goes to each_test_run.log in its directory."
    end
    lock = Mutex.new
    done = 0
    all_submitted = true
    run_one = lambda do |t|
      if claiming
        lock.synchronize do
          s.claim(sha: sha, scope: 'test', mod: t.mod, test_case: t.test_name, flags: flags, mesa: m)
        end
      end
      started = Time.now
      if jobs > 1
        t.run_isolated
      else
        say_step(shell, "[#{lock.synchronize { done } + 1}/#{tests.size}] Testing #{t.mod}/#{t.test_name}")
        t.do_one
      end
      lock.synchronize do
        done += 1
        # (not `&&=` on the call: that would skip every later submission)
        submitted = finish_test(s, m, t, started, submit: submit, prefix: "[#{done}/#{tests.size}] ")
        all_submitted &&= submitted
      end
    end
    if jobs > 1
      parallel_each(tests, jobs) { |t| run_one.call(t) }
    else
      tests.each { |t| run_one.call(t) }
    end
    all_submitted
  end

  # Submit a finished test (with +submit+) and print its one progress line:
  #   [12/107] star/1M_pre_ms_to_wd passed in 41.2 min, submitted
  # Only the outcome is colored. Returns whether the result went in (true
  # when not submitting).
  def finish_test(s, m, t, started, submit:, prefix: '')
    elapsed = format_duration(Time.now - started)
    unless t.ran_since?(started)
      line = "#{prefix}#{t.mod}/#{t.test_name} #{shell.set_color('wrote no results', :red)} " \
             "after #{elapsed}"
      shell.say(submit ? "#{line}; nothing submitted" : line)
      return !submit
    end

    passed = begin
      t.passed?
    rescue StandardError
      false
    end
    outcome = passed ? shell.set_color('passed', :green) : shell.set_color('FAILED', :red)
    submitted = submit ? submit_result(s, m, t, started) : true
    suffix = if !submit then ''
             elsif submitted then ', submitted'
             else ", #{shell.set_color('not submitted', :red)}"
             end
    shell.say "#{prefix}#{t.mod}/#{t.test_name} #{outcome} in #{elapsed}#{suffix}"
    submitted
  end

  def format_duration(seconds)
    if seconds < 90 then format('%d s', seconds.round)
    elsif seconds < 2 * 3600 then format('%.1f min', seconds / 60.0)
    else format('%.1f h', seconds / 3600.0)
    end
  end


  # clean + install +m+ with +modes+ in the environment, then record the
  # modes in $MESA_DIR/testhub.yml for later `test`/`submit` runs.
  def build_with_modes(m, modes, raise_on_failure: false)
    say_step(shell, "Building MESA #{short_sha(m.sha)}" \
                    "#{" (#{describe_modes(modes)})" unless modes.empty?}")
    with_env(run_mode_env(modes).merge(build_thread_env)) do
      m.clean
      m.install
    end
  rescue MesaDirError
    shell.say "MESA #{short_sha(m.sha)} failed to build; see #{File.join(m.mesa_dir, 'build.log')}.", :red
    raise if raise_on_failure
  else
    # Recorded so later `mesa_test test` runs on this build use the same modes.
    m.record_run_modes(modes)
    shell.say "Built MESA #{short_sha(m.sha)}.", :green
  end

  # Run modes given explicitly on the command line (unset flags omitted).
  def explicit_run_modes
    modes = {}
    modes[:skip_optional] = options[:skip_optional] unless options[:skip_optional].nil?
    modes[:fpe] = options[:fpe] unless options[:fpe].nil?
    modes[:resolution_factor] = (options[:converge] ? converge_factor : false) unless options[:converge].nil?
    modes[:resolution_factor] = explicit_factor if explicit_factor
    modes
  end

  # --resolution-factor, as a string for the environment; implies
  # --converge, so it can't be combined with --no-converge.
  def explicit_factor
    factor = options[:resolution_factor]
    return nil if factor.nil?
    raise Thor::Error, '--resolution-factor must be positive.' unless factor.positive?
    raise Thor::Error, '--resolution-factor implies --converge; drop --no-converge.' if options[:converge] == false

    factor.to_s
  end

  # The run modes a dispatch asks for, with any --resolution-factor given
  # here as the preferred converge factor (a factor the commit names
  # still wins, with a warning).
  def modes_for(rec)
    if explicit_factor
      dispatch_run_modes(rec, preferred: explicit_factor, preferred_source: '--resolution-factor')
    else
      dispatch_run_modes(rec)
    end
  end

  # Modes for running tests on +m+'s build: what was recorded at install,
  # overridden by flags given now — except FPE, which is compiled in.
  def test_run_modes(m)
    recorded = m.downloaded? ? m.recorded_run_modes : {}
    explicit = explicit_run_modes
    if recorded.key?(:fpe) && explicit.key?(:fpe) && explicit[:fpe] != recorded[:fpe]
      shell.say "Ignoring --#{explicit[:fpe] ? '' : 'no-'}fpe: this build was compiled " \
                "#{recorded[:fpe] ? 'with' : 'without'} FPE checks.", :yellow
      explicit.delete(:fpe)
    end
    # A factor the build recorded (e.g. from `[ci converge 0.5]` via
    # `install best`) beats the shell's; say so when they differ.
    shell_factor = ENV['MESA_TEST_SUITE_RESOLUTION_FACTOR']
    if recorded[:resolution_factor] && !explicit.key?(:resolution_factor) && shell_factor &&
       shell_factor.to_f != recorded[:resolution_factor].to_f
      warn_yellow "This build was set up for resolution factor #{recorded[:resolution_factor]}; " \
                  "using it instead of your MESA_TEST_SUITE_RESOLUTION_FACTOR=#{shell_factor}."
    end
    recorded.merge(explicit)
  end

  # What a claim says the work will run with (MESATestHub's use_* flags).
  def claim_flags(modes)
    {
      fpe: modes[:fpe] == true,
      full_inlists: modes[:skip_optional] == false,
      converge: modes[:resolution_factor] ? true : false
    }
  end

  def describe_modes(modes)
    modes.map do |k, v|
      case k
      when :fpe then v ? 'FPE checks on' : 'FPE checks off'
      when :skip_optional then v ? 'skip optional inlists' : 'run optional inlists'
      when :resolution_factor then v ? "resolution factor #{v}" : 'default resolution'
      end
    end.join(', ')
  end

  # Optional modes this computer is willing to run for MESATestHub: the
  # config file's `capabilities`, overridden by any mode flag given.
  def capabilities_for(s)
    caps = s.capabilities.dup
    caps[:fpe] = options[:fpe] unless options[:fpe].nil?
    caps[:full_inlists] = !options[:skip_optional] unless options[:skip_optional].nil?
    caps[:converge] = options[:converge] unless options[:converge].nil?
    caps[:converge] = true if options[:resolution_factor]
    caps
  end

  # Ask for a commit to build, exiting with NOTHING_TO_DO_EXIT if there is
  # none — before anything touches the work directory.
  def dispatch_build_or_exit(s)
    dispatch_or_exit(s, scope: 'build') || exit(NOTHING_TO_DO_EXIT)
  end

  # Dispatch, or print the server's complaint and exit 1. nil means there's
  # nothing to do (said on STDERR so `request_work` STDOUT stays pure JSON).
  def dispatch_or_exit(s, scope:, sha: nil, capabilities: capabilities_for(s))
    rec = s.dispatch(scope: scope, sha: sha, capabilities: capabilities)
    if rec.nil?
      $stderr.puts 'MESATestHub has nothing for this computer to ' \
                   "#{scope == 'build' ? 'build' : 'test on this commit'} right now."
    elsif scope == 'build'
      # (the hub's first reason is usually "on <branch>", already said)
      reasons = Array(rec['reasons']) - ["on #{rec['branch']}"]
      $stderr.puts "MESATestHub picked #{rec['short_sha']} on #{rec['branch']}: #{reasons.join('; ')}"
    end
    rec
  rescue TestHubError => e
    $stderr.puts e.message
    exit 1
  end

  def emit_search_response(response)
    if response.is_a?(Net::HTTPSuccess)
      puts response.body
      warn_search_failures(response.body)
    else
      $stderr.puts "Search request failed: HTTP #{response.code} "\
                   "#{response.message}"
      $stderr.puts response.body unless response.body.nil? || response.body.empty?
      if response.code == '401'
        $stderr.puts "\nCheck your API key (`mesa_test setup`, or MESATESTHUB_API_KEY).\n" \
                     "#{MesaTestSubmitter::API_KEY_HELP}"
      end
      exit 1
    end
  end

  def warn_search_failures(body)
    parsed = JSON.parse(body)
    failures = parsed['failures']
    return unless failures.is_a?(Array) && !failures.empty?

    $stderr.puts 'Warning: the following query clauses were rejected by '\
                 'the server and ignored:'
    failures.each { |f| $stderr.puts "  - #{f}" }
  rescue JSON::ParserError
    # Body wasn't JSON; nothing to surface.
  end

  def work_dir(given, default)
    if given
      File.expand_path('', given)
    else
      default
    end
  end

  def check_user_and_computer(submitter)
    computer_check = submitter.confirm_computer
    if computer_check['valid']
      puts computer_check['message']
    else
      $stderr.puts computer_check['message']
      unless computer_check['reachable'] == false
        $stderr.puts 'Check your API key (`mesa_test setup`, or MESATESTHUB_API_KEY).'
      end
      exit 1
    end
  end

  # hub: true for commands that talk to MESATestHub, which all need an API
  # key; checked first, before any checkout or build.
  def create_submitter(force: true, hub: false)
    s = case MODE
        when :development
          MesaTestSubmitter.new_from_config(base_uri: DEVELOPMENT_URI)
        when :staging
          MesaTestSubmitter.new_from_config(base_uri: STAGING_URI)
        else
          MesaTestSubmitter.new_from_config
        end
    unless force
      s.setup unless s.confirm_computer_data
    end
    s.require_api_key! if hub
    s
  end

  def create_and_check_submitter(force: true)
    s = create_submitter(force: force, hub: true)
    check_user_and_computer s
    s
  end

  def create_and_check_mesa(mesa_dir: '', mirror_dir: '', mod: :all,
                            github_protocol: nil)
    m = Mesa.new(mesa_dir: mesa_dir, mirror_dir: mirror_dir,
                 github_protocol: github_protocol)
    check_mesa(m)
    update_mesa(mesa: m, mod: mod)
  end

  def check_mesa(mesa)
    return if mesa.downloaded?

    raise MesaDirError, "Invalid MESA_DIR: #{mesa.mesa_dir}. Please "\
      'download and install a valid MESA version or provide the path to one.'
  end

  def update_mesa(mesa:, mod: :all)
    mesa.load_test_source_data(mod: mod)
    mesa
  end

  def create_and_check_test_case(mesa:, test_case_name:, mod: :all)
    t = mesa.find_test_case(test_case_name: test_case_name, mod: mod)
    # return test case if it was found
    return t unless t.nil?

    # test case was not found. Throw an error.
    msg = "No such test case, #{test_case_name} found in any of "
    # this should care about the value in `mod`. Current behavior is only
    # appropriate if `mod` is :all.
    msg << MesaTestCase.modules.map do |this_mod|
      File.join(mesa.test_suite_dir(mod: this_mod), 'do1_test_source')
    end.join(' or ')
    msg << '.'
    raise TestCaseDirError, msg
  end
end

# actually start the CLI
begin
  MesaTest.start(ARGV)
rescue Interrupt
  # run_workers has already stopped any tests running in parallel.
  $stderr.puts "\nInterrupted."
  exit 130
rescue TestHubError => e
  $stderr.puts e.message
  exit 1
end
