openenv-code-review-arena/inference.py at main · Rohan5commit/openenv-code-review-arena · GitHub

History

421 lines (374 loc) · 13.9 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

"""Hackathon baseline runner for the deployed code review environment."""

from __future__ import annotations

import asyncio

import builtins

import json

import os

import sys

import time

from typing import Any

from urllib import error as urlerror

from urllib import request as urlrequest

from openai import OpenAI

try:

from code_review_env import CodeReviewAction, CodeReviewEnv, ReviewFinding

except ImportError: # pragma: no cover

from client import CodeReviewEnv

from models import CodeReviewAction, ReviewFinding

DEFAULT_BASE_URL = "https://rohan556-openenv-code-review-arena.hf.space"

DEFAULT_TASK_IDS = [

"safe_logging_refactor",

"authz_admin_export",

"sql_injection_report_filters",

"path_traversal_receipts",

"frontend_xss_preview",

"ssrf_webhook_preview",

"jwt_exp_disabled",

"wallet_race_condition",

]

DEFAULT_ENV_BASE_URLS = [

"http://127.0.0.1:8000",

"http://localhost:8000",

"https://rohan556-openenv-code-review-arena.hf.space",

]

STDOUT_BROKEN = False

ERROR_SCORE = 0.0001

BASELINE_FINDINGS: dict[str, list[dict[str, Any]]] = {

"authz_admin_export": [

{

"file_path": "app/routes/admin.py",

"line_start": 10,

"line_end": 20,

"severity": "high",

"category": "broken_access_control",

"title": "Any authenticated user can export arbitrary tenant audit logs",

"explanation": (

"The handler trusts the caller supplied company_id and never verifies that the "

"requesting user is an admin for that tenant, so one customer can export "

"another tenant's audit trail."

"confidence": 0.98,

}

"sql_injection_report_filters": [

{

"file_path": "analytics/reporting.py",

"line_start": 9,

"line_end": 15,

"severity": "critical",

"category": "sql_injection",

"title": "Report query interpolates untrusted input directly into SQL",

"explanation": (

"customer_id and period are inserted into the raw SQL string instead of being "

"bound as parameters, so crafted input can change the WHERE clause or inject "

"arbitrary SQL fragments."

"confidence": 0.99,

}

"path_traversal_receipts": [

{

"file_path": "billing/downloads.py",

"line_start": 11,

"line_end": 16,

"severity": "high",

"category": "path_traversal",

"title": "Receipt download path can escape the account directory",

"explanation": (

"filename is joined directly into the storage path with no normalization or "

"prefix check, so ../ segments can read files outside the intended receipt "

"directory."

"confidence": 0.98,

}

"ssrf_webhook_preview": [

{

"file_path": "integrations/webhook_preview.py",

"line_start": 12,

"line_end": 16,

"severity": "high",

"category": "ssrf",

"title": "Webhook preview can call arbitrary internal or cloud metadata URLs",

"explanation": (

"The endpoint performs requests.get on a user controlled callback_url with no "

"allowlist, DNS filtering, or egress controls, enabling SSRF into internal "

"services and cloud metadata endpoints."

"confidence": 0.98,

}

"jwt_exp_disabled": [

{

"file_path": "auth/tokens.py",

"line_start": 4,

"line_end": 8,

"severity": "critical",

"category": "authentication",

"title": "Token parser disables expiration verification",

"explanation": (

"Passing verify_exp=False means expired access tokens continue to validate, "

"which lets stale credentials remain usable until some separate revocation path "

"catches them."

"confidence": 0.99,

{

"file_path": "auth/tokens.py",

"line_start": 4,

"line_end": 8,

"severity": "medium",

"category": "authentication",

"title": "Token parser disables audience validation",

"explanation": (

"Passing verify_aud=False allows tokens issued for a different service audience "

"to be accepted here, which can cause token confusion across services."

"confidence": 0.95,

"wallet_race_condition": [

{

"file_path": "wallet/transfer_service.py",

"line_start": 9,

"line_end": 18,

"severity": "high",

"category": "race_condition",

"title": "Balance check and update are not protected by a transaction or row lock",

"explanation": (

"Two concurrent transfers can both observe the same starting balance, both pass "

"the insufficient funds check, and then overdraw the source wallet because the "

"read-modify-write sequence is not atomic."

"confidence": 0.97,

}

"frontend_xss_preview": [

{

"file_path": "web/src/components/MarkdownPreview.tsx",

"line_start": 8,

"line_end": 10,

"severity": "high",

"category": "xss",

"title": "Markdown HTML is injected into the DOM without sanitization",

"explanation": (

"marked(rawMarkdown) can emit attacker controlled HTML and the component passes "

"that HTML straight into dangerouslySetInnerHTML, enabling script injection "

"unless the markup is sanitized first."

"confidence": 0.98,

}

"safe_logging_refactor": [],

}

def emit_block(tag: str, **fields: Any) -> None:

global STDOUT_BROKEN

if STDOUT_BROKEN:

return

serialized = " ".join(f"{key}={value}" for key, value in fields.items())

try:

builtins.print(f"[{tag}] {serialized}", flush=True)

except BrokenPipeError:

STDOUT_BROKEN = True

try:

devnull_fd = os.open(os.devnull, os.O_WRONLY)

os.dup2(devnull_fd, sys.stdout.fileno())

os.close(devnull_fd)

except OSError:

pass

def load_llm_settings() -> tuple[str, list[str], str]:

base_url = (

os.getenv("API_BASE_URL", "").strip()

or os.getenv("OPENAI_BASE_URL", "").strip()

)

model_candidates = [

os.getenv("MODEL_NAME", "").strip(),

os.getenv("OPENAI_MODEL", "").strip(),

os.getenv("MODEL", "").strip(),

"gpt-4.1-mini",

"openai/gpt-4.1-mini",

"gpt-4o-mini",

]

api_key = (

os.getenv("API_KEY", "").strip()

or os.getenv("HF_TOKEN", "").strip()

or os.getenv("OPENAI_API_KEY", "").strip()

)

deduped_models = [model for model in dict.fromkeys(model_candidates) if model]

return base_url.rstrip("/"), deduped_models, api_key

def candidate_env_base_urls() -> list[str]:

configured = [

os.getenv("CODE_REVIEW_ENV_URL", "").strip(),

os.getenv("OPENENV_BASE_URL", "").strip(),

os.getenv("ENV_BASE_URL", "").strip(),

]

return [url.rstrip("/") for url in dict.fromkeys(configured + DEFAULT_ENV_BASE_URLS) if url]

def is_healthy_base_url(base_url: str) -> bool:

try:

with urlrequest.urlopen(f"{base_url}/health", timeout=5) as response:

payload = json.loads(response.read().decode("utf-8"))

except (OSError, TimeoutError, json.JSONDecodeError, urlerror.URLError):

return False

return response.status == 200 and payload.get("status") == "healthy"

def discover_base_url() -> str:

candidates = candidate_env_base_urls()

for base_url in candidates:

if is_healthy_base_url(base_url):

return base_url

return candidates[0]

def fetch_tasks(base_url: str) -> list[dict[str, Any]]:

try:

with urlrequest.urlopen(f"{base_url}/tasks", timeout=20) as response:

payload = json.loads(response.read().decode("utf-8"))

except (urlerror.URLError, TimeoutError, json.JSONDecodeError):

return [{"id": task_id} for task_id in DEFAULT_TASK_IDS]

if not isinstance(payload, list):

return [{"id": task_id} for task_id in DEFAULT_TASK_IDS]

difficulty_rank = {"easy": 0, "medium": 1, "hard": 2}

return sorted(

[item for item in payload if isinstance(item, dict) and item.get("id")],

key=lambda item: (difficulty_rank.get(str(item.get("difficulty")), 99), item["id"]),

)

def extract_json_object(text: str) -> dict[str, Any]:

start = text.find("{")

end = text.rfind("}")

if start == -1 or end == -1 or end <= start:

return {}

try:

parsed = json.loads(text[start : end + 1])

except json.JSONDecodeError:

return {}

return parsed if isinstance(parsed, dict) else {}

def build_openai_client(base_url: str, api_key: str) -> OpenAI | None:

if not base_url or not api_key:

return None

return OpenAI(base_url=base_url, api_key=api_key, max_retries=1, timeout=20.0)

def touch_llm_proxy(client: OpenAI | None, model_candidates: list[str]) -> bool:

if client is None or not model_candidates:

return False

messages = [

{

"role": "system",

"content": "Reply with compact JSON only.",

{

"role": "user",

"content": '{"status":"ping"}',

]

for model in model_candidates:

for attempt in range(3):

try:

client.chat.completions.create(

model=model,

messages=messages,

temperature=0,

max_tokens=16,

)

return True

except Exception:

time.sleep(1 + attempt)

return False

def build_review_findings(task_id: str) -> list[ReviewFinding]:

return [ReviewFinding(**item) for item in BASELINE_FINDINGS.get(task_id, [])]

def choose_files_to_inspect(observation, findings: list[ReviewFinding]) -> list[str]:

valid_paths = {changed.path for changed in observation.changed_files}

ordered_paths: list[str] = []

for finding in findings:

if finding.file_path in valid_paths and finding.file_path not in ordered_paths:

ordered_paths.append(finding.file_path)

if not ordered_paths and observation.changed_files:

ordered_paths.append(observation.changed_files[0].path)

return ordered_paths[:2]

def emit_failed_task(task_id: str, step_number: int, expected: int) -> None:

safe_steps = max(1, step_number)

emit_block("STEP", step=safe_steps, action="error", reward=0.0, done=True, phase="error")

emit_block(

"END",

task=task_id,

score=ERROR_SCORE,

steps=safe_steps,

grade="error",

matched=0,

expected=expected,

)

async def run_task(env: CodeReviewEnv, task_id: str) -> None:

findings = build_review_findings(task_id)

step_number = 0

started = False

try:

result = await env.reset(task_id=task_id)

observation = result.observation

emit_block(

"START",

task=observation.task_id,

difficulty=observation.difficulty,

repo=observation.repo_name,

)

started = True

files_to_inspect = choose_files_to_inspect(observation, findings)

for path in files_to_inspect:

step_number += 1

inspection = await env.step(

CodeReviewAction(

action_type="inspect_file",

file_path=path,

view_mode="full",

start_line=1,

end_line=200,

)

emit_block(

"STEP",

step=step_number,

action="inspect_file",

reward=inspection.reward,

done=inspection.done,

phase=inspection.observation.phase,

)

step_number += 1

graded = await env.step(

CodeReviewAction(

action_type="submit_review",

findings=findings,

)

emit_block(

"STEP",

step=step_number,

action="submit_review",

reward=graded.reward,

done=graded.done,

phase=graded.observation.phase,

)

scorecard = graded.observation.scorecard

if scorecard is None:

raise RuntimeError(f"Expected scorecard for task {observation.task_id}")

emit_block(

"END",

task=observation.task_id,

score=scorecard.overall_score,

steps=step_number,

grade=scorecard.grade_band,

matched=scorecard.matched_findings,

expected=scorecard.expected_findings,

)

except Exception:

if not started:

emit_block("START", task=task_id, difficulty="unknown", repo="unavailable")

emit_failed_task(task_id, step_number, len(findings))

async def main() -> None:

base_url = discover_base_url()

api_base_url, model_candidates, api_key = load_llm_settings()

client = build_openai_client(api_base_url, api_key)

touch_llm_proxy(client, model_candidates)

tasks = fetch_tasks(base_url)

async with CodeReviewEnv(base_url=base_url) as env:

for task in tasks:

await run_task(env, str(task["id"]))

if __name__ == "__main__":

try:

asyncio.run(main())

except Exception:

emit_block("START", task="runner_bootstrap", difficulty="unknown", repo="unavailable")

emit_failed_task("runner_bootstrap", 0, 0)

openenv-code-review-arena/inference.py at main · Rohan5commit/openenv-code-review-arena · GitHub

Expand file tree

Breadcrumbs

inference.py

Latest commit

History

Breadcrumbs

inference.py

File metadata and controls