binaryen/scripts/fuzz_opt.py at main · WebAssembly/binaryen · GitHub

History

executable file

3126 lines (2671 loc) · 125 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558

559

560

561

562

563

564

565

566

567

568

569

570

571

572

573

574

575

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596

597

598

599

600

601

602

603

604

605

606

607

608

609

610

611

612

613

614

615

616

617

618

619

620

621

622

623

624

625

626

627

628

629

630

631

632

633

634

635

636

637

638

639

640

641

642

643

644

645

646

647

648

649

650

651

652

653

654

655

656

657

658

659

660

661

662

663

664

665

666

667

668

669

670

671

672

673

674

675

676

677

678

679

680

681

682

683

684

685

686

687

688

689

690

691

692

693

694

695

696

697

698

699

700

701

702

703

704

705

706

707

708

709

710

711

712

713

714

715

716

717

718

719

720

721

722

723

724

725

726

727

728

729

730

731

732

733

734

735

736

737

738

739

740

741

742

743

744

745

746

747

748

749

750

751

752

753

754

755

756

757

758

759

760

761

762

763

764

765

766

767

768

769

770

771

772

773

774

775

776

777

778

779

780

781

782

783

784

785

786

787

788

789

790

791

792

793

794

795

796

797

798

799

800

801

802

803

804

805

806

807

808

809

810

811

812

813

814

815

816

817

818

819

820

821

822

823

824

825

826

827

828

829

830

831

832

833

834

835

836

837

838

839

840

841

842

843

844

845

846

847

848

849

850

851

852

853

854

855

856

857

858

859

860

861

862

863

864

865

866

867

868

869

870

871

872

873

874

875

876

877

878

879

880

881

882

883

884

885

886

887

888

889

890

891

892

893

894

895

896

897

898

899

900

901

902

903

904

905

906

907

908

909

910

911

912

913

914

915

916

917

918

919

920

921

922

923

924

925

926

927

928

929

930

931

932

933

934

935

936

937

938

939

940

941

942

943

944

945

946

947

948

949

950

951

952

953

954

955

956

957

958

959

960

961

962

963

964

965

966

967

968

969

970

971

972

973

974

975

976

977

978

979

980

981

982

983

984

985

986

987

988

989

990

991

992

993

994

995

996

997

998

999

1000

#!/usr/bin/python3

"""Run various fuzzing operations on random inputs, using wasm-opt.

See "testcase_handlers" below for the list of fuzzing operations.

Usage:

./scripts/fuzz_opt.py

That will run forever or until it finds a problem.

You can put files in the local directory 'fuzz' (under the top level of the

binaryen repo) and the fuzzer will treat them as important content to fuzz

with high frequency.

Setup: Some tools are optional, like emcc and wasm2c. The v8 shell (d8),

however, is used in various sub-fuzzers and so it is mandatory.

Note: For afl-fuzz integration, you probably don't want this, and can use

something like

BINARYEN_CORES=1 BINARYEN_PASS_DEBUG=1 afl-fuzz -i afl-testcases/ -o afl-findings/ -m 100 -d -- bin/wasm-opt -ttf --fuzz-exec --Os @@

(that is on a fixed set of arguments to wasm-opt, though - this

script covers different options being passed)

"""

# ruff: file-ignore[prohibited-trailing-comma, unused-method-argument]

import contextlib

import difflib

import json

import math

import os

import pathlib

import random

import re

import shutil

import subprocess

import sys

import tarfile

import time

import traceback

from datetime import datetime, timedelta, timezone

from os.path import abspath

from typing import override

from test import fuzzing, shared, support

assert sys.version_info >= (3, 10), 'requires Python 3.10'

# parameters

# feature options that are always passed to the tools.

# XXX fp16 and multibyte are not yet stable, remove from here when they are.

CONSTANT_FEATURE_OPTS = ['--all-features', '--disable-fp16', '--disable-multibyte']

INPUT_SIZE_MIN = 1024

INPUT_SIZE_MEAN = 40 * 1024

INPUT_SIZE_MAX = 5 * INPUT_SIZE_MEAN

PRINT_WATS = False

given_seed = None

CLOSED_WORLD_FLAG = '--closed-world'

# V8 does not support shared memories when running with shared-everything

# enabled, so do not fuzz shared-everything for now. The remaining features are

# not yet implemented in v8.

# Keep this in sync with bundle_clusterfuzz.py.

DISALLOWED_FEATURES_IN_V8 = [

'shared-everything',

'fp16',

'strings',

'stack-switching',

'multibyte',

'relaxed-atomics',

]

# utilities

def in_binaryen(*args):

return os.path.join(shared.options.binaryen_root, *args)

def in_bin(tool):

return os.path.join(shared.options.binaryen_bin, tool)

def random_size():

if random.random() < 0.25:

# sometimes do an exponential distribution, which prefers smaller sizes but may

# also get very high

ret = int(random.expovariate(1.0 / INPUT_SIZE_MEAN))

# if the result is valid, use it, otherwise do the normal thing

# (don't clamp, which would give us a lot of values on the borders)

if ret >= INPUT_SIZE_MIN and ret <= INPUT_SIZE_MAX:

return ret

# most of the time do a simple linear range around the mean

return random.randint(INPUT_SIZE_MIN, 2 * INPUT_SIZE_MEAN - INPUT_SIZE_MIN)

def make_random_input(input_size, raw_input_data):

with open(raw_input_data, 'wb') as f:

f.write(bytes([random.randint(0, 255) for x in range(input_size)]))

def run(cmd, stderr=None, silent=False):

if not silent:

print(' '.join(cmd))

return subprocess.check_output(cmd, stderr=stderr, text=True)

def run_unchecked(cmd):

print(' '.join(cmd))

return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True).stdout

def randomize_pass_debug():

if random.random() < 0.1:

print('[pass-debug]')

os.environ['BINARYEN_PASS_DEBUG'] = '1'

else:

os.environ['BINARYEN_PASS_DEBUG'] = '0'

del os.environ['BINARYEN_PASS_DEBUG']

print('randomized pass debug:', os.environ.get('BINARYEN_PASS_DEBUG', ''))

@contextlib.contextmanager

def no_pass_debug():

old_env = os.environ.copy()

if os.environ.get('BINARYEN_PASS_DEBUG'):

del os.environ['BINARYEN_PASS_DEBUG']

try:

yield

finally:

os.environ.update(old_env)

def randomize_feature_opts():

global FEATURE_OPTS

FEATURE_OPTS = CONSTANT_FEATURE_OPTS[:]

if random.random() < 0.1:

# 10% of the time disable all features, i.e., fuzz the MVP featureset.

# Fuzzing that is less and less important as more features get enabled

# by default, but we don't want to lose all coverage for it entirely

# (and the odds of randomly not selecting any feature, below, is too

# small - at 17 features it is far less than 1%).

FEATURE_OPTS += FEATURE_DISABLE_FLAGS

elif random.random() < 0.333:

# 1/3 of the remaining 90% pick each feature randomly.

for possible in FEATURE_DISABLE_FLAGS:

if random.random() < 0.5:

FEATURE_OPTS.append(possible)

if possible in IMPLIED_FEATURE_OPTS:

FEATURE_OPTS.extend(IMPLIED_FEATURE_OPTS[possible])

# Otherwise, use all the features. This is useful to maximize

# coverage, as enabling more features enables more optimizations and

# code paths, and also allows all initial contents to run. However,

# half the time disable the specific ones that are a problem in V8.

elif random.random() < 0.5:

# Disable features not allowed in V8 to increase V8 fuzzing.

FEATURE_OPTS.extend(f'--disable-{feature}' for feature in DISALLOWED_FEATURES_IN_V8)

# Relaxed SIMD's nondeterminism disables much but not

# all of our V8 fuzzing, so avoid it.

FEATURE_OPTS.append('--disable-relaxed-simd')

print('randomized feature opts:', '\n ' + '\n '.join(FEATURE_OPTS))

# Pick closed or open with equal probability as both matter.

# Closed world is not a feature flag, technically, since it only makes sense

# to pass to wasm-opt (and not other tools). But decide on whether we'll

# be fuzzing in that mode now, as it determinies how we set other things up.

global CLOSED_WORLD

CLOSED_WORLD = random.random() < 0.5

ALL_FEATURE_OPTS = ['--all-features', '-all', '--mvp-features', '-mvp']

def update_feature_opts(wasm):

global FEATURE_OPTS

# we will re-compute the features; leave all other things as they are

EXTRA = [x for x in FEATURE_OPTS if not x.startswith('--enable') and

not x.startswith('--disable') and x not in ALL_FEATURE_OPTS]

FEATURE_OPTS = run([in_bin('wasm-opt'), wasm] + FEATURE_OPTS + ['--print-features']).strip().split('\n')

# filter out '', which can happen if no features are enabled

FEATURE_OPTS = [x for x in FEATURE_OPTS if x]

print(FEATURE_OPTS, EXTRA)

FEATURE_OPTS += EXTRA

def randomize_fuzz_settings():

# a list of the arguments to pass to wasm-opt -ttf when generating the wasm

global GEN_ARGS

GEN_ARGS = []

# a list of the optimizations to run on the wasm

global FUZZ_OPTS

FUZZ_OPTS = []

# a boolean whether NaN values are allowed, or we de-NaN them

global NANS

# a boolean whether out of bounds operations are allowed, or we bounds-enforce them

global OOB

# a boolean whether we legalize the wasm for JS

global LEGALIZE

if random.random() < 0.5:

NANS = True

else:

NANS = False

GEN_ARGS += ['--denan']

if random.random() < 0.5:

OOB = True

else:

OOB = False

GEN_ARGS += ['--no-fuzz-oob']

if random.random() < 0.5:

LEGALIZE = True

GEN_ARGS += ['--legalize-and-prune-js-interface']

else:

LEGALIZE = False

if CLOSED_WORLD:

GEN_ARGS += [CLOSED_WORLD_FLAG]

# Enclose the world much of the time when fuzzing closed-world, so that

# many types are private and hence optimizable.

if random.random() < 0.5:

GEN_ARGS += ['--enclose-world']

# Test JSPI somewhat rarely, as it may be slower, and disables some other

# fuzzing.

global JSPI

JSPI = random.random() < 0.1

if JSPI:

# Fuzzing the start function with JSPI is tricky: if the start function

# calls an import that is marked as suspending, then it traps on

# "SuspendError: trying to suspend without WebAssembly.promising" - the

# start function in fact *cannot* be wrapped as promising, as it is not

# even an export. To make the binaryen interpreter behave the same way,

# we'd need to know JSPI is enabled and act like it, which would add a

# bunch of complexity. Instead, fuzz JSPI without start functions.

GEN_ARGS += ['--remove-start']

print('randomized settings (NaNs, OOB, legalize, JSPI):', NANS, OOB, LEGALIZE, JSPI)

def init_important_initial_contents():

# Fuzz dir contents are always important to us.

fuzz_dir = os.path.join(shared.options.binaryen_root, 'fuzz')

fuzz_cases = shared.get_tests(fuzz_dir, shared.test_suffixes, recursive=True)

FIXED_IMPORTANT_INITIAL_CONTENTS = fuzz_cases

# If auto_initial_contents is set we'll also grab all test files that are

# recent.

RECENT_DAYS = 30

# Returns the list of test wast/wat files added or modified within the

# RECENT_DAYS number of days counting from the commit time of HEAD

def auto_select_recent_initial_contents():

# Print 'git log' with changed file status and without commit messages,

# with commits within RECENT_DAYS number of days, counting from the

# commit time of HEAD. The reason we use the commit time of HEAD instead

# of the current system time is to make the results deterministic given

# the Binaryen HEAD commit.

head_ts_str = run(['git', 'log', '-1', '--format=%cd', '--date=raw'],

silent=True).split()[0]

head_dt = datetime.utcfromtimestamp(int(head_ts_str))

start_dt = head_dt - timedelta(days=RECENT_DAYS)

start_ts = start_dt.replace(tzinfo=timezone.utc).timestamp()

log = run(['git', 'log', '--name-status', '--format=', '--date=raw', '--no-renames', f'--since={start_ts}'], silent=True).splitlines()

# Pick up lines in the form of

# A test/../something.wast

# M test/../something.wast

# (wat extension is also included)

p = re.compile(r'^[AM]\stest' + os.sep + r'(.*\.(wat|wast))$')

matches = [p.match(e) for e in log]

auto_set = {match.group(1) for match in matches if match}

auto_set = auto_set.difference(set(FIXED_IMPORTANT_INITIAL_CONTENTS))

return sorted(auto_set)

def is_git_repo():

try:

ret = run(['git', 'rev-parse', '--is-inside-work-tree'],

silent=True, stderr=subprocess.DEVNULL)

return ret == 'true\n'

except subprocess.CalledProcessError:

return False

if not is_git_repo() and shared.options.auto_initial_contents:

print('Warning: The current directory is not a git repository, ' +

'so not automatically selecting initial contents.')

shared.options.auto_initial_contents = False

print('- Important provided initial contents:')

for test in FIXED_IMPORTANT_INITIAL_CONTENTS:

print(' ' + test)

print()

recent_contents = []

print('- Recently added or modified initial contents ', end='')

if shared.options.auto_initial_contents:

print(f'(automatically selected: within last {RECENT_DAYS} days):')

recent_contents += auto_select_recent_initial_contents()

for test in recent_contents:

print(' ' + test)

print()

initial_contents = FIXED_IMPORTANT_INITIAL_CONTENTS + recent_contents

global IMPORTANT_INITIAL_CONTENTS

IMPORTANT_INITIAL_CONTENTS = [os.path.join(shared.get_test_dir('.'), t) for t in initial_contents]

all_tests = shared.get_all_tests()

def pick_initial_contents():

# if we use an initial wasm file's contents as the basis for the

# fuzzing, then that filename, or None if we start entirely from scratch

global INITIAL_CONTENTS

INITIAL_CONTENTS = None

# half the time don't use any initial contents

if random.random() < 0.5:

return

# some of the time use initial contents that are known to be especially

# important

if IMPORTANT_INITIAL_CONTENTS and random.random() < 0.5:

test_name = random.choice(IMPORTANT_INITIAL_CONTENTS)

else:

test_name = random.choice(all_tests)

print('initial contents:', test_name)

if shared.options.auto_initial_contents:

# when using auto initial contents, we look through the git history to

# find test files. if a test file was renamed or removed then it may

# no longer exist, and we should just skip it.

if not os.path.exists(test_name):

return

if not fuzzing.is_fuzzable(test_name):

return

assert os.path.exists(test_name)

if test_name.endswith('.wast'):

# this can contain multiple modules, pick one

split_parts = support.split_wast(test_name)

if len(split_parts) > 1:

index = random.randint(0, len(split_parts) - 1)

chosen = split_parts[index]

module, _asserts = chosen

if not module:

# there is no module in this choice (just asserts), ignore it

print('initial contents has no module')

return

test_name = abspath('initial.wat')

with open(test_name, 'w') as f:

f.write(module)

print(f' picked submodule {index} from multi-module wast')

global FEATURE_OPTS

FEATURE_OPTS += [

# avoid multivalue for now due to bad interactions with gc non-nullable

# locals in stacky code. for example, this fails to roundtrip as the

# tuple code ends up creating stacky binary code that needs to spill

# non-nullable references to locals, which is not allowed:

# (module

# (type $other (struct))

# (func $foo (result (ref $other))

# (select

# (struct.new $other)

# (tuple.extract 2 1

# (tuple.make 2

# (i32.const 0)

# )

'--disable-multivalue',

]

# the given wasm may not work with the chosen feature opts. for example, if

# we pick atomics.wast but want to run with --disable-atomics, then we'd

# error, so we need to test the wasm. first, make sure it doesn't have a

# features section, as that would enable a feature that we might want to

# be disabled, and our test would not error as we want it to.

if test_name.endswith('.wasm'):

temp_test_name = 'initial.wasm'

try:

run([in_bin('wasm-opt'), test_name, '-all', '--strip-target-features',

'-o', temp_test_name])

except Exception:

# the input can be invalid if e.g. it is raw data that is used with

# -ttf as fuzzer input

print('(initial contents are not valid wasm, ignoring)')

return

test_name = temp_test_name

# Next, test the wasm. Note that we must check for closed world explicitly

# here, as a testcase may only work in an open world, which means we need to

# skip it.

args = FEATURE_OPTS

if CLOSED_WORLD:

args.append(CLOSED_WORLD_FLAG)

try:

run([in_bin('wasm-opt'), test_name] + args,

stderr=subprocess.PIPE,

silent=True)

except Exception:

print('(initial contents not valid for features, ignoring)')

return

INITIAL_CONTENTS = test_name

# Test outputs we want to ignore are marked this way.

IGNORE = '[binaryen-fuzzer-ignore]'

# Traps are reported as [trap REASON]

TRAP_PREFIX = '[trap '

# Host limits are reported as [host limit REASON]

HOST_LIMIT_PREFIX = '[host limit '

# --fuzz-exec reports calls as [fuzz-exec] export foo

FUZZ_EXEC_EXPORT_PREFIX = '[fuzz-exec] export'

# --fuzz-exec reports a stack limit using this notation

STACK_LIMIT = '[trap stack limit]'

# JS exceptions are logged as exception thrown: REASON

EXCEPTION_PREFIX = 'exception thrown: '

# given an export line that includes FUZZ_EXEC_EXPORT_PREFIX, return the export

# that is called

def get_export_from_export_line(export_line):

assert FUZZ_EXEC_EXPORT_PREFIX in export_line

return export_line.split(FUZZ_EXEC_EXPORT_PREFIX)[1].strip()

# compare two strings, strictly

def compare(x, y, context, verbose=True):

if x != y and IGNORE not in {x, y}:

message = ''.join([a + '\n' for a in difflib.unified_diff(x.splitlines(), y.splitlines(), fromfile='expected', tofile='actual')])

if verbose:

raise Exception(f"{context} comparison error, expected to have '{x}' == '{y}', diff:\n\n{message}")

else:

raise Exception(f"{context}\nDiff:\n\n{message}")

# converts a possibly-signed integer to an unsigned integer

def unsign(x, bits):

return x & ((1 << bits) - 1)

# numbers are "close enough" if they just differ in printing, as different

# vms may print at different precision levels and verbosity

def numbers_are_close_enough(x, y):

# handle nan comparisons like -nan:0x7ffff0 vs NaN, ignoring the bits

if 'nan' in x.lower() and 'nan' in y.lower():

return True

# if one input is a pair, then it is in fact a 64-bit integer that is

# reported as two 32-bit chunks. convert such 'low high' pairs into a 64-bit

# integer for comparison to the other value

if ' ' in x or ' ' in y:

def to_64_bit(a):

if ' ' not in a:

return unsign(int(a), bits=64)

low, high = a.split(' ')

return unsign(int(low), 32) + (1 << 32) * unsign(int(high), 32)

return to_64_bit(x) == to_64_bit(y)

# float() on the strings will handle many minor differences, like

# float('1.0') == float('1') , float('inf') == float('Infinity'), etc.

try:

return float(x) == float(y)

except Exception:

pass

# otherwise, try a full eval which can handle i64s too

try:

ex = eval(x)

ey = eval(y)

return ex == ey or float(ex) == float(ey)

except Exception as e:

print('failed to check if numbers are close enough:', e)

return False

FUZZ_EXEC_NOTE_RESULT = '[fuzz-exec] note result'

# compare between vms, which may slightly change how numbers are printed

def compare_between_vms(x, y, context):

x_lines = x.splitlines()

y_lines = y.splitlines()

if len(x_lines) != len(y_lines):

return compare(x, y, context + ' (note: different number of lines between vms)')

num_lines = len(x_lines)

for i in range(num_lines):

x_line = x_lines[i]

y_line = y_lines[i]

if x_line != y_line:

# this is different, but maybe it's a vm difference we can ignore

LOGGING_PREFIX = '[LoggingExternalInterface logging'

if x_line.startswith(LOGGING_PREFIX) and y_line.startswith(LOGGING_PREFIX):

x_val = x_line[len(LOGGING_PREFIX) + 1:-1]

y_val = y_line[len(LOGGING_PREFIX) + 1:-1]

if numbers_are_close_enough(x_val, y_val):

continue

if x_line.startswith(FUZZ_EXEC_NOTE_RESULT) and y_line.startswith(FUZZ_EXEC_NOTE_RESULT):

x_val = x_line.split(' ')[-1]

y_val = y_line.split(' ')[-1]

if numbers_are_close_enough(x_val, y_val):

continue

# this failed to compare. print a custom diff of the relevant lines

MARGIN = 3

start = max(i - MARGIN, 0)

end = min(i + MARGIN, num_lines)

return compare('\n'.join(x_lines[start:end]), '\n'.join(y_lines[start:end]), context)

def fix_output(out):

# large doubles may print slightly different on different VMs

def fix_double(x):

x = x.group(1)

if 'nan' in x or 'NaN' in x:

x = 'nan'

else:

x = x.replace('Infinity', 'inf')

x = str(float(x))

return 'f64.const ' + x

out = re.sub(r'f64\.const (-?[nanN:abcdefxIity\d+-.]+)', fix_double, out)

# mark traps from wasm-opt as exceptions, even though they didn't run in a vm

out = out.replace(TRAP_PREFIX, EXCEPTION_PREFIX + TRAP_PREFIX)

# funcref(0) has the index of the function in it, and optimizations can

# change that index, so ignore it

out = re.sub(r'funcref$[\d\w$+-_:]+$', 'funcref()', out)

# JS prints i31 as just a number, so change "i31ref(N)" (which C++ emits)

# to "N".

out = re.sub(r'i31ref$(-?\d+)$', r'\1', out)

# Tag names may change due to opts, so canonicalize them.

out = re.sub(r' tag\$\d+', ' tag', out)

# If we failed to instantiate, remove everything after it. VMs may print

# additional error info that cannot be compared between VMs, like a JS

# stack trace for where we tried to instantiate.

if INSTANTIATE_ERROR in out:

after = out.find(INSTANTIATE_ERROR)

after = out.find('\n', after)

out = out[:after + 1]

lines = out.splitlines()

for i in range(len(lines)):

line = lines[i]

if 'Warning: unknown flag' in line or 'Try --help for options' in line:

# ignore some VM warnings that don't matter, like if a newer V8 has

# removed a flag that is no longer needed. but print the line so the

# developer can see it.

print(line)

lines[i] = None

elif 'V8 is running with experimental features enabled. Stability and security will suffer.' in line:

# Ignore some boilerplate from VMs

lines[i] = None

elif EXCEPTION_PREFIX in line:

# exceptions may differ when optimizing, but an exception should

# occur, so ignore their types (also js engines print them out

# slightly differently)

lines[i] = ' *exception*'

return '\n'.join([line for line in lines if line is not None])

def fix_spec_output(out):

out = fix_output(out)

# spec shows a pointer when it traps, remove that

out = '\n'.join(ln if 'runtime trap' not in ln else ln[ln.find('runtime trap'):] for ln in out.splitlines())

# https://github.com/WebAssembly/spec/issues/543 , float consts are messed up

out = '\n'.join(ln if 'f32' not in ln and 'f64' not in ln else '' for ln in out.splitlines())

return out

ignored_vm_runs = 0

ignored_vm_run_reasons = {}

# Notes a VM run that we ignore, and the reason for it (for metrics purposes).

# Extra text can also be printed that is not included in the metrics.

def note_ignored_vm_run(reason, extra_text='', amount=1):

global ignored_vm_runs

print(f'(ignore VM run: {reason}{extra_text})')

ignored_vm_runs += amount

ignored_vm_run_reasons.setdefault(reason, 0)

ignored_vm_run_reasons[reason] += amount

# Run a VM command, and filter out known issues.

def run_vm(cmd, checked=True):

def filter_known_issues(output):

known_issues = [

# can be caused by flatten, ssa, etc. passes

'local count too large',

# can be caused by (array.new $type -1) etc.

'requested new array is too large',

# https://github.com/WebAssembly/binaryen/issues/3767

# note that this text is a little too broad, but the problem is rare

# enough that it's unlikely to hide an unrelated issue

'found br_if of type',

# this text is emitted from V8 when it runs out of memory during a

# GC allocation.

'out of memory',

# if the call stack is exceeded we must ignore this, as

# optimizations can change whether this happens or not (e.g. by

# removing locals, which makes stack frames smaller), which is

# noticeable.

'Maximum call stack size exceeded',

# all host limitations are arbitrary and may differ between VMs and

# also be affected by optimizations, so ignore them.

# this is the prefix that the binaryen interpreter emits. For V8,

# there is no single host-limit signal, and we have the earlier

# strings in this list for known issues (to which more need to be

# added as necessary).

HOST_LIMIT_PREFIX,

# V8 does not accept nullable stringviews

# (https://github.com/WebAssembly/binaryen/pull/6574)

'expected (ref stringview_wtf16), got nullref',

'expected type (ref stringview_wtf16), found ref.null of type nullref',

# wasm64 memories have a V8 limit

'larger than implementation limit',

]

for issue in known_issues:

if issue in output:

note_ignored_vm_run(issue)

return IGNORE

return output

try:

# some known issues do not cause the entire process to fail

if checked:

ret = run(cmd)

else:

ret = run_unchecked(cmd)

return filter_known_issues(ret)

except subprocess.CalledProcessError:

# Other known issues do make it fail, so re-run without checking for

# success and see if we should ignore it

raw = run_unchecked(cmd)

filtered = filter_known_issues(raw)

if filtered == IGNORE:

return IGNORE

# Otherwise, raise an error.

raise

MAX_INTERPRETER_ENV_VAR = 'BINARYEN_MAX_INTERPRETER_DEPTH'

MAX_INTERPRETER_DEPTH = 1000

def run_bynterp(wasm, args):

# increase the interpreter stack depth, to test more things

os.environ[MAX_INTERPRETER_ENV_VAR] = str(MAX_INTERPRETER_DEPTH)

try:

return run_vm([in_bin('wasm-opt'), wasm] + FEATURE_OPTS + args)

finally:

del os.environ['BINARYEN_MAX_INTERPRETER_DEPTH']

# Enable even more things than V8_OPTS. V8_OPTS are the flags we want to use

# when testing, on our fixed test suite, but when fuzzing we may want more.

def get_v8_extra_flags():

# It is important to use the --fuzzing flag because it does things like

# enable mixed old and new EH (which is an issue since

# https://github.com/WebAssembly/exception-handling/issues/344 )

flags = ['--fuzzing']

# Sometimes add --future, which may enable new JITs and such, which is good

# to fuzz for V8's sake.

if random.random() < 0.5:

flags += ['--future']

# Sometimes add V8's type assertions.

if random.random() < 0.5:

flags += ['--wasm-assert-types']

# Some other options make sense to use sometimes.

if random.random() < 0.5:

flags += ['--no-wasm-generic-wrapper']

return flags

V8_LIFTOFF_ARGS = ['--liftoff']

V8_NO_LIFTOFF_ARGS = ['--no-liftoff']

# Default to running with liftoff enabled, because we need to pick either

# liftoff or turbo* for consistency (otherwise running the same command twice

# may have different results due to NaN nondeterminism), and liftoff is faster

# for small things.

def run_d8_js(js, args=[], liftoff=True):

cmd = [shared.V8] + shared.V8_OPTS

cmd += get_v8_extra_flags()

if liftoff:

cmd += V8_LIFTOFF_ARGS

cmd += [js]

if args:

cmd += ['--'] + args

return run_vm(cmd)

# For JSPI, we must customize fuzz_shell.js. We do so the first time we need

# it, and save the filename here.

JSPI_JS_FILE = None

def get_fuzz_shell_js():

js = in_binaryen('scripts', 'fuzz_shell.js')

if not JSPI:

# Just use the normal fuzz shell script.

return js

global JSPI_JS_FILE

if JSPI_JS_FILE:

# Use the customized file we've already created.

return JSPI_JS_FILE

JSPI_JS_FILE = os.path.abspath('jspi_fuzz_shell.js')

with open(JSPI_JS_FILE, 'w') as f:

# Enable JSPI.

f.write('var JSPI = 1;\n\n')

# Un-comment the async and await keywords.

with open(js) as g:

code = g.read()

code = code.replace('/* async */', 'async')

code = code.replace('/* await */', 'await')

f.write(code)

return JSPI_JS_FILE

def run_d8_wasm(wasm, liftoff=True, args=[]):

return run_d8_js(get_fuzz_shell_js(), [wasm] + args, liftoff=liftoff)

def all_disallowed(features):

return not any(('--enable-' + x) in FEATURE_OPTS for x in features)

class TestCaseHandler:

# how frequent this handler will be run. 1 means always run it, 0.5 means half the

# time

frequency = 1

def __init__(self):

self.num_runs = 0

# If the core handle_pair() method is not overridden, it calls handle() on

# each of the items. That is useful if you just want the two wasms and don't

# care about their relationship.

@override

def handle_pair(self, input, before_wasm, after_wasm, opts):

self.handle(before_wasm)

# Add some visual space between the independent parts.

print('\n')

self.handle(after_wasm)

@override

def can_run_on_wasm(self, wasm):

return True

def increment_runs(self):

self.num_runs += 1

def count_runs(self):

return self.num_runs

# Fuzz the interpreter with --fuzz-exec.

class FuzzExec(TestCaseHandler):

frequency = 1

@override

def handle_pair(self, input, before_wasm, after_wasm, opts):

run([in_bin('wasm-opt'), before_wasm] + opts + ['--fuzz-exec'])

# VMs

class BinaryenInterpreter:

name = 'binaryen interpreter'

@override

def run(self, wasm):

output = run_bynterp(wasm, ['--fuzz-exec-before'])

if output != IGNORE:

calls = output.count(FUZZ_EXEC_EXPORT_PREFIX)

errors = output.count(TRAP_PREFIX) + output.count(HOST_LIMIT_PREFIX)

if errors > calls / 2:

# A significant amount of execution on this testcase

# simply trapped, and was not very useful, so mark it

# as ignored. Ideally the fuzzer testcases would be

# improved to reduce this number.

# Note that we don't change output=IGNORE as there may

# still be useful testing here (up to 50%), so we only

# note that this is a mostly-ignored run, but we do not

# ignore the parts that are useful.

# Note that we set amount to 0.5 because we are run both

# on the before wasm and the after wasm. Those will be

# in sync (because the optimizer does not remove traps)

# and so by setting 0.5 we only increment by 1 for the

# entire iteration.

note_ignored_vm_run('too many errors vs calls',

extra_text=f' ({calls} calls, {errors} errors)',

amount=0.5)

return output

@override

def can_run(self, wasm):

return True

@override

def can_compare_to_self(self):

return True

@override

def can_compare_to_other(self, other):

return True

class D8:

name = 'd8'

extra_d8_flags = []

def run_js(self, js, wasm, checked=True):

return run_vm([shared.V8, js] + shared.V8_OPTS + get_v8_extra_flags() + self.extra_d8_flags + ['--', wasm], checked=checked)

@override

def run(self, wasm):

return self.run_js(js=get_fuzz_shell_js(), wasm=wasm)

@override

def can_run(self, wasm):

return all_disallowed(DISALLOWED_FEATURES_IN_V8)

@override

def can_compare_to_self(self):

# With nans or relaxed SIMD, VM differences can confuse us, including

# differences between binaryen and V8 (binaryen's behavior can get

# "baked" into the wasm when it precomputes code, so we cannot compare

# V8's output before binaryen opts and after binaryen opts).

return not NANS and all_disallowed(['relaxed-simd'])

@override

def can_compare_to_other(self, other):

# Relaxed SIMD allows different behavior between VMs, so only

# allow comparisons to other d8 variants if it is enabled.

if not all_disallowed(['relaxed-simd']) and not other.name.startswith('d8'):

return False

# If not legalized, the JS will fail immediately, so no point to

# compare to others.

return self.can_compare_to_self() and LEGALIZE

class D8Liftoff(D8):

name = 'd8_liftoff'

extra_d8_flags = V8_LIFTOFF_ARGS

class D8Turboshaft(D8):

name = 'd8_turboshaft'

extra_d8_flags = V8_NO_LIFTOFF_ARGS

class Wasm2C:

name = 'wasm2c'

def __init__(self):

# look for wabt in the path. if it's not here, don't run wasm2c

try:

wabt_bin = shared.which('wasm2c')

wabt_root = os.path.dirname(os.path.dirname(wabt_bin))

self.wasm2c_dir = os.path.join(wabt_root, 'wasm2c')

if not os.path.isdir(self.wasm2c_dir):

print('wabt found, but not wasm2c support dir')

self.wasm2c_dir = None

except Exception as e:

print('warning: no wabt found:', e)

self.wasm2c_dir = None

@override

def can_run(self, wasm):

if self.wasm2c_dir is None:

return False

# if we legalize for JS, the ABI is not what C wants

if LEGALIZE:

return False

# relatively slow, so run it less frequently

if random.random() < 0.5:

return False

# wasm2c doesn't support most features

return all_disallowed(['exception-handling', 'simd', 'threads', 'bulk-memory', 'nontrapping-float-to-int', 'tail-call', 'sign-ext', 'reference-types', 'multivalue', 'gc', 'custom-descriptors', 'acquire-release-atomics', 'relaxed-atomics', 'wide-arithmetic'])

@override

def run(self, wasm):

run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)

run(['wasm2c', wasm, '-o', 'wasm.c'])

compile_cmd = ['clang', 'main.c', 'wasm.c', os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'), '-I' + self.wasm2c_dir, '-lm', '-Werror']

run(compile_cmd)

return run_vm(['./a.out'])

@override

def can_compare_to_self(self):

# The binaryen optimizer changes NaNs in the ways that wasm

# expects, but that's not quite what C has

return not NANS

@override

def can_compare_to_other(self, other):

# C won't trap on OOB, and NaNs can differ from wasm VMs

return not OOB and not NANS

class Wasm2C2Wasm(Wasm2C):

name = 'wasm2c2wasm'

def __init__(self):

super().__init__()

self.has_emcc = shared.which('emcc') is not None

@override

def run(self, wasm):

run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)

run(['wasm2c', wasm, '-o', 'wasm.c'])

compile_cmd = ['emcc', 'main.c', 'wasm.c',

os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'),

'-I' + self.wasm2c_dir,

'-lm',

'-s', 'ENVIRONMENT=shell',

'-s', 'ALLOW_MEMORY_GROWTH']

# disable the signal handler: emcc looks like unix, but wasm has

# no signals

compile_cmd += ['-DWASM_RT_MEMCHECK_SIGNAL_HANDLER=0']

if random.random() < 0.5:

compile_cmd += ['-O' + str(random.randint(1, 3))]

elif random.random() < 0.5:

if random.random() < 0.5:

compile_cmd += ['-Os']

else:

compile_cmd += ['-Oz']

# avoid pass-debug on the emcc invocation itself (which runs

# binaryen to optimize the wasm), as the wasm here can be very

# large and it isn't what we are focused on testing here

with no_pass_debug():

run(compile_cmd)

return run_d8_js(abspath('a.out.js'))

@override

def can_run(self, wasm):

# quite slow (more steps), so run it less frequently

if random.random() < 0.8:

return False

# prefer not to run if the wasm is very large, as it can OOM

# the JS engine.

return super().can_run(wasm) and self.has_emcc and \

os.path.getsize(wasm) <= INPUT_SIZE_MEAN

@override

def can_compare_to_other(self, other):

# NaNs can differ from wasm VMs

return not NANS

class CompareVMs(TestCaseHandler):

frequency = 1

View remainder of file in raw view

binaryen/scripts/fuzz_opt.py at main · WebAssembly/binaryen · GitHub

Uh oh!

Expand file tree

Breadcrumbs

fuzz_opt.py

Latest commit

History

Breadcrumbs

fuzz_opt.py

File metadata and controls