Commit c994722
Your Name
Benchmark Harness Updates
1 parent 7106d26 commit c994722
2 files changed
Lines changed: 16 additions & 11 deletions
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -875,6 +875,7 @@ async def run_test_real( | |||
| 875 | 875 | from cecli.coders import Coder | |
| 876 | 876 | from cecli.helpers.conversation import ConversationFiles, ConversationManager | |
| 877 | 877 | from cecli.io import InputOutput | |
| 878 | + from cecli.main import SwitchCoderSignal | ||
| 878 | 879 | ||
| 879 | 880 | if not os.path.isdir(testdir): | |
| 880 | 881 | if dry: | |
@@ -1114,7 +1115,10 @@ async def run_test_real( | |||
| 1114 | 1115 | ||
| 1115 | 1116 | await coder.apply_updates() | |
| 1116 | 1117 | else: | |
| 1117 | - response = await coder.run(with_message=instructions, preproc=False) | ||
| 1118 | + try: | ||
| 1119 | + response = await coder.run(with_message=instructions, preproc=False) | ||
| 1120 | + except SwitchCoderSignal: | ||
| 1121 | + pass | ||
| 1118 | 1122 | ||
| 1119 | 1123 | dur += time.time() - start | |
| 1120 | 1124 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -5,13 +5,13 @@ | |||
| 5 | 5 | set -e # Exit on error | |
| 6 | 6 | ||
| 7 | 7 | # Default values | |
| 8 | - BASE_NAME="hashline-fix-2" | ||
| 8 | + BASE_NAME="cecli-base-hashline-9" | ||
| 9 | 9 | EDIT_FORMAT="hashline" | |
| 10 | 10 | MAP_TOKENS="512" | |
| 11 | 11 | THREADS="1" | |
| 12 | 12 | LANGUAGES="javascript,python,rust,go,java" | |
| 13 | - HASH_RE="^[7f]" | ||
| 14 | - NUM_TESTS="40" | ||
| 13 | + HASH_RE="^[15]" | ||
| 14 | + NUM_TESTS="32" | ||
| 15 | 15 | EXERCISES_DIR="polyglot-benchmark" | |
| 16 | 16 | OUTPUT_DIR="tmp.benchmarks" | |
| 17 | 17 | SLEEP_BETWEEN=30 # Seconds to sleep between runs | |
@@ -21,14 +21,15 @@ SLEEP_BETWEEN=30 # Seconds to sleep between runs | |||
| 21 | 21 | # "openrouter/minimax/minimax-m2.1" | |
| 22 | 22 | # "openrouter/qwen/qwen3-vl-235b-a22b-thinking" | |
| 23 | 23 | MODELS=( | |
| 24 | - "openrouter/deepseek/deepseek-v3.2-exp" | ||
| 25 | - "openrouter/moonshotai/kimi-k2.5" | ||
| 26 | - "openrouter/openai/gpt-oss-120b" | ||
| 27 | - "openrouter/openai/gpt-5.2" | ||
| 28 | 24 | "openrouter/google/gemini-3-flash-preview" | |
| 29 | - "openrouter/google/gemini-3-pro-preview" | ||
| 30 | - "openrouter/anthropic/claude-haiku-4.5" | ||
| 31 | - "openrouter/anthropic/claude-sonnet-4.5" | ||
| 25 | + "openrouter/deepseek/deepseek-v3.2-exp" | ||
| 26 | + # "openrouter/moonshotai/kimi-k2.5" | ||
| 27 | + # "openrouter/openai/gpt-oss-120b" | ||
| 28 | + # "openrouter/openai/gpt-5.2" | ||
| 29 | + # "openrouter/google/gemini-3-flash-preview" | ||
| 30 | + # "openrouter/google/gemini-3-pro-preview" | ||
| 31 | + # "openrouter/anthropic/claude-haiku-4.5" | ||
| 32 | + # "openrouter/anthropic/claude-sonnet-4.5" | ||
| 32 | 33 | ) | |
| 33 | 34 | ||
| 34 | 35 | # Parse command line arguments | |
0 commit comments