livecode/tests/_compilertestrunnerbehavior.livecodescript at develop · livecode/livecode · GitHub

This repository was archived by the owner on Aug 31, 2021. It is now read-only.

History

472 lines (399 loc) · 18 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

script "CompilerTestRunnerBehavior"

This file is part of LiveCode.

LiveCode is free software; you can redistribute it and/or modify it under

the terms of the GNU General Public License v3 as published by the Free

Software Foundation.

LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY

WARRANTY; without even the implied warranty of MERCHANTABILITY or

FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License

for more details.

You should have received a copy of the GNU General Public License

along with LiveCode. If not see <http://www.gnu.org/licenses/>. */

private function getTestRunnerBehavior

-- Compute the filename of the test runner

local tTestsPath, tStack

set the itemdelimiter to "/"

put item 1 to -2 of the effective filename of this me into tTestsPath

put tTestsPath & "/_testrunnerbehavior.livecodescript" into tStack

return the long id of stack tStack

end getTestRunnerBehavior

on CompilerTestRunnerMain

local tTestRunner

put getTestRunnerBehavior() into tTestRunner

set the behavior of this me to tTestRunner

send "TestRunnerMain" to me in 0

end CompilerTestRunnerMain

command TestRunnerImpl_DoRun pInfo

local tResult

dispatch "CompilerTestRunner_ValidateEnvironment" to me

put the result into tResult

if tResult is not empty then

throw tResult

end if

local tScript, tCommand

put pInfo["args"][2] into tScript

put pInfo["args"][3] into tCommand

if tScript is empty then

runAllScripts pInfo

else if tCommand is empty then

runCompilerTestScript pInfo, tScript

else

runCompilerTest pInfo, tScript, tCommand

end if

return the result for value

end TestRunnerImpl_DoRun

command TestRunnerImpl_DoUsage pStatus

dispatch "CompilerTestRunner_DoUsage" to me with pStatus

end TestRunnerImpl_DoUsage

----------------------------------------------------------------

-- Support for running tests

----------------------------------------------------------------

-- Run all the test scripts that can be found below the current

-- directory

private command runAllScripts pInfo

local tFile, tAnalysis, tExtension

dispatch function "CompilerTestRunner_FileExtension" to me

put the result into tExtension

repeat for each element tFile in TesterGetTestFileNames("", tExtension)

runCompilerTestScript pInfo, tFile

put TesterTapCombine(tAnalysis, the result) into tAnalysis

end repeat

TesterRunPrintSummary tAnalysis

return tAnalysis

end runAllScripts

-- Run the tests found in one specific script file

private command runCompilerTestScript pInfo, pScriptFile

local tTest, tAnalysis

repeat for each element tTest in listCompilerTestsInFile(pScriptFile)

runCompilerTest pInfo, pScriptFile, tTest

put TesterTapCombine(tAnalysis, the result) into tAnalysis

end repeat

return tAnalysis

end runCompilerTestScript

command CompilerRunTestProcessOutput pScriptfile, pCommand, pOutput

-- Create test log

local tTestLog

put "###" && TesterGetPrettyTestName(pScriptFile, "compilertest") && pCommand \

& return & return into tTestLog

put pOutput & return after tTestLog

-- Analyse the results and print a summary line

local tTapResults

put TesterTapAnalyse(tTestLog) into tTapResults

TesterLogSummaryLine tTapResults, (TesterGetPrettyTestName(pScriptFile, "compilertest") & ":" && pCommand)

return tTapResults

end CompilerRunTestProcessOutput

-- Run a specific named test command tCommand in a script file

-- tScriptFile

private command runCompilerTest pInfo, pScriptFile, pTest

local tCommandLine

-- First we need to process the specified test in script file

local tTestInfo

try

processCompilerTest pInfo, pScriptFile, pTest, tTestInfo

catch tError

write tError & return to stderr

quit 1

end try

local tCompilerOutput, tCompilerExitStatus

dispatch "CompilerTestRunner_DoRunTest" to me with tTestInfo, tCompilerOutput, tCompilerExitStatus

reportCompilerTestDiag tCompilerOutput

-- Process the assertions and make sure each one matches up

local tTestOutput

repeat for each line tAssertion in tTestInfo["assertions"]

dispatch function "CompilerTestRunner_DoesOutputSatisfyAssertion" to me \

with tCompilerOutput, tCompilerExitStatus, tAssertion, tTestInfo

if the result is true then

put "ok - " after tTestOutput

else

put "not ok - " after tTestOutput

end if

put compilerPrettyPrintAssertion(tAssertion, tTestInfo["positions"]) after tTestOutput

if tTestInfo["expectation"] is not "PASS" then

if tTestInfo["expectation"] is "FAIL" then

put " # TODO" after tTestOutput

else if tTestInfo["expectation"] is "SKIP" then

put " # SKIP" after tTestOutput

end if

if tTestInfo["reason"] is not empty then

put " " & tTestInfo["reason"] after tTestOutput

end if

put return after tTestOutput

end repeat

CompilerRunTestProcessOutput pScriptFile, pTest, tTestOutput

return the result

end runCompilerTest

----------------------------------------------------------------

-- Support for generating the compiler tests

----------------------------------------------------------------

-- Get a number-indexed array containing the names of all "test"

-- blocks in pFilename.

private function listCompilerTestsInFile pFilename

local tScript

-- Get the contents of the file

open file pFilename for "UTF-8" text read

if the result is not empty then

throw the result

end if

read from file pFilename until end

put it into tScript

close file pFilename

-- Scan the file for "on Test*" definitions

local tTestNames, tCount, tLine, tName

repeat for each line tLine in tScript

if word 1 of tLine is not "%TEST" then

next repeat

end if

put word 2 of tLine into tName

add 1 to tCount

put tName into tTestNames[tCount]

end repeat

return tTestNames

end listCompilerTestsInFile

private function compilerPrettyPrintAssertion pAssertion, pPositions

if item 1 of pAssertion is "success" then

return "compile succeeded"

end if

if item 1 of pAssertion is among the items of "error,warning" then

return format("%s %s at line %d", item 1 of pAssertion, item 2 of pAssertion, pPositions[item 3 of pAssertion])

end if

return "UNKNOWN ASSERTION TYPE"

end compilerPrettyPrintAssertion

private command processCompilerTest pInfo, pScriptFile, pTest, @rCompilerTest

-- Get the contents of the file

open file pScriptFile for "UTF-8" text read

if the result is not empty then

throw format("can't open file '%s'", pScriptFile)

end if

local tScript

read from file pScriptFile until end

put it into tScript

close file pScriptFile

-- First extract the test block we are wanting to run

local tLine, tLineNumber, tTestLineNumber, tState

local tName, tCode, tExpectation, tExpectationReason, tAssertions, tPositions, tSet, tSetCount

put empty into tLine

put 0 into tLineNumber

put 0 into tTestLineNumber

put empty into tState

put empty into tName

put empty into tCode

put empty into tExpectation

put empty into tExpectationReason

put empty into tAssertions

put empty into tPositions

put empty into tSet

put 0 into tSetCount

repeat for each line tLine in tScript

add 1 to tLineNumber

local tIsDirective

put false into tIsDirective

if tState is "code" then

if word 1 of tLine begins with "%" and \

not (word 1 of tLine begins with "%{") then

put true into tIsDirective

end if

else

if word 1 of tLine begins with "%" then

put true into tIsDirective

end if

if tIsDirective then

local tToken

put word 1 of tLine into tToken

if tToken is "%%" then

-- We don't allow comment directives inside code blocks

if tState is "code" then

reportCompilerTestError pScriptFile, tLineNumber, "comment directives not allowed in code block"

end if

-- Ignore comment lines

next repeat

else if tToken is "%TEST" then

-- If we are already in a test block, it is an error

if tState is not empty then

reportCompilerTestError pScriptFile, tLineNumber, "%TEST directive not allowed inside test block"

end if

-- We've started a new test clause, so record the name and move to

-- code state.

reportCompilerTestDiag format("found test '%s' at line %d", word 2 of tLine, tLineNumber)

put word 2 of tLine into tName

put tLineNumber + 1 into tTestLineNumber

put empty into tCode

put empty into tExpectation

put empty into tAssertions

put empty into tPositions

put "set" into tState

put 0 into tSetCount

else if tToken is "%EXPECT" then

-- We only allow %EXPECT directives after code blocks

if tState is not "code" then

reportCompilerTestError pScriptFile, tLineNumber, "%EXPECT directive only allowed after code block"

end if

-- If the expectation isn't one of PASS, FAIL, XFAIL, SKIP then it

-- is an error

if word 2 of tLine is not among the items of "PASS,FAIL,SKIP" then

reportCompilerTestError pScriptFile, tLineNumber, format("invalid expectation '%s'", word 2 of tLine)

end if

if word 3 of tLine contains "#" then

reportCompilerTestError pScriptFile, tLineNumber, format("expectation reason cannot contain '#'")

end if

-- Record the expectation, and move to assertions state.

put word 2 of tLine into tExpectation

put word 3 of tLine into tExpectationReason

if char 1 of tExpectationReason is quote then

delete char 1 of tExpectationReason

delete char -1 of tExpectationReason

end if

put "assertions" into tState

else if tToken is among the items of "%SUCCESS,%WARNING,%ERROR" then

if tState is not "assertions" then

reportCompilerTestError pScriptFile, tLineNumber, "assertion directive outside of assertion block"

end if

if tToken is "%SUCCESS" then

if the number of words in tLine is not 1 then

reportCompilerTestError pScriptFile, tLineNumber, "incorrect syntax, expected: %SUCCESS"

end if

reportCompilerTestDiag format("found assert success at line %d", tLineNumber)

put "success" & return after tAssertions

else if tToken is "%WARNING" then

if (the number of words in tLine is 4 and \

word 3 of tLine is "AT") then

-- Check that the position exists

if tPositions[word 4 of tLine] is empty then

reportCompilerTestError pScriptFile, tLineNumber, format("unknown position '%s'", word 4 of tLine)

end if

reportCompilerTestDiag format("found assert warning %s for position %s (with '%s') at line %d", word 2 of tLine, word 4 of tLine, word 6 of tLine, tLineNumber)

put "warning", word 2 of tLine, word 4 of tLine & return after tAssertions

else

reportCompilerTestError pScriptFile, tLineNumber, "incorrect syntax, expected: %WARNING <id> AT <position>"

end if

else if tToken is "%ERROR" then

if (the number of words in tLine is 4 and \

word 3 of tLine is "AT") then

-- Check that the position exists

if tPositions[word 4 of tLine] is empty then

reportCompilerTestError pScriptFile, tLineNumber, format("unknown position '%s'", word 4 of tLine)

end if

reportCompilerTestDiag format("found assert error %s for position %s at line %d", word 2 of tLine, word 4 of tLine, tLineNumber)

put "error", word 2 of tLine, word 4 of tLine & return after tAssertions

else

reportCompilerTestError pScriptFile, tLineNumber, "incorrect syntax, expected: %ERROR <id> AT <position>"

end if

else if tToken is "%ENDTEST" then

-- If we aren't in assertions state then it isn't a complete test

-- clause

if tState is not "assertions" then

reportCompilerTestError pScriptFile, tLineNumber, "premature %ENDTEST directive"

end if

if tAssertions is empty then

reportCompilerTestError pScriptFile, tLineNumber, "no assertions specified"

end if

if tAssertions contains "%SUCCESS" and \

tAssertions contains "%ERROR" then

reportCompilerTestError pScriptFile, tLineNumber, "both %SUCCESS and %ERROR not allowed in the same set of assertions"

end if

-- If this is the test we are looking for, we are done

if tName is pTest then

exit repeat

end if

-- Move back to initial state

put empty into tName

put empty into tState

else if tToken is "%SET" then

if tState is not "set" then

reportCompilerTestError pScriptFile, tLineNumber, "set directive must precede code"

end if

add 1 to tSetCount

put word 3 of tLine into tSet[word 2 of tLine]

else

-- We've encountered an illegal directive

reportCompilerTestError pScriptFile, tLineNumber, format("unknown directive '%s'", tToken)

end if

else if tState is empty then

-- We only allow empty lines outside of test blocks

if word 1 to -1 of tLine is not empty then

reportCompilerTestError pScriptFile, tLineNumber, "junk outside of test clause"

end if

else if tState is "code" or tState is "set" then

put "code" into tState

-- If we are in a code block, then accumulate the code line after

-- processing for position directives %{...}.

-- We loop through the line, searching for %{...}, and accumulating the

-- code in between as we go.

local tPositionSkip

put 0 into tPositionSkip

repeat forever

local tPositionOffset

put offset("%{", tLine, tPositionSkip) into tPositionOffset

if tPositionOffset is 0 then

put char tPositionSkip to -1 of tLine after tCode

exit repeat

end if

add tPositionSkip to tPositionOffset

put char tPositionSkip to tPositionOffset - 1 of tLine after tCode

-- Look for the end brace, it is an error if it is not there

local tPositionEndOffset

put offset("}", tLine, tPositionOffset) into tPositionEndOffset

if tPositionEndOffset is 0 then

reportCompilerTestError pScriptFile, tLineNumber, "unterminated named position"

end if

add tPositionOffset to tPositionEndOffset

-- Check that the name is not empty

local tPositionName

put char tPositionOffset + 2 to tPositionEndOffset - 1 of tLine into tPositionName

if tPositionName is empty then

reportCompilerTestError pScriptFile, tLineNumber, "empty named position"

end if

-- Check that the position hasn't already been defined

if tPositions[tPositionName] is not empty then

reportCompilerTestError pScriptFile, tLineNumber, "named position already defined"

end if

reportCompilerTestDiag format("found position '%s' at line %d", tPositionName, tLineNumber)

put tLineNumber - tTestLineNumber - tSetCount + 1 into tPositions[tPositionName]

put tPositionEndOffset + 1 into tPositionSkip

end repeat

-- Add the newline to the code

put return after tCode

else if tState is "assertions" then

-- We only allow directives inside assertion blocks

if word 1 to -1 of tLine is not empty then

reportCompilerTestError pScriptFile, tLineNumber, "junk inside assertion clause"

end if

end repeat

if tName is empty then

reportCompilerTestError pScriptFile, tLineNumber, format("test '%s' not found in file", pTest)

end if

-- We now have the code for the test, the expectation and a list of

-- assertions.

put tName into rCompilerTest["name"]

put tCode into rCompilerTest["code"]

put tPositions into rCompilerTest["positions"]

put tExpectation into rCompilerTest["expectation"]

put tExpectation into rCompilerTest["reason"]

put tAssertions into rCompilerTest["assertions"]

put tSet into rCompilerTest["set"]

end processCompilerTest

private command reportCompilerTestError pFile, pLine, pMessage

throw format("'%s', line %d: %s", pFile, pLine, pMessage)

end reportCompilerTestError

command reportCompilerTestDiag pMessage

if $LCC_VERBOSE is not empty then

repeat for each line tLine in pMessage

write "DIAG:" && tLine & return to stderr

end repeat

end if

end reportCompilerTestDiag

command reportCompilerTestDiagWithLineNumbers pMessage

if $LCC_VERBOSE is not empty then

local tLineNumber

put 1 into tLineNumber

repeat for each line tLine in pMessage

write "DIAG:" && format("%4d", tLineNumber) && ":" && tLine & return to stderr

add 1 to tLineNumber

end repeat

end if

end reportCompilerTestDiagWithLineNumbers

livecode/tests/_compilertestrunnerbehavior.livecodescript at develop · livecode/livecode · GitHub

Uh oh!

Expand file tree

Breadcrumbs

_compilertestrunnerbehavior.livecodescript

Latest commit

History

Breadcrumbs

_compilertestrunnerbehavior.livecodescript

File metadata and controls