netdata/python.d/python_modules/pyyaml2/parser.py at master · ed00m/netdata · GitHub

History

589 lines (531 loc) · 24.9 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558

559

560

561

562

563

564

565

566

567

568

569

570

571

572

573

574

575

576

577

578

579

580

581

582

583

584

585

586

587

588

# The following YAML grammar is LL(1) and is parsed by a recursive descent

# parser.

# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END

# implicit_document ::= block_node DOCUMENT-END*

# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*

# block_node_or_indentless_sequence ::=

# ALIAS

# | properties (block_content | indentless_block_sequence)?

# | block_content

# | indentless_block_sequence

# block_node ::= ALIAS

# | properties block_content?

# | block_content

# flow_node ::= ALIAS

# | properties flow_content?

# | flow_content

# properties ::= TAG ANCHOR? | ANCHOR TAG?

# block_content ::= block_collection | flow_collection | SCALAR

# flow_content ::= flow_collection | SCALAR

# block_collection ::= block_sequence | block_mapping

# flow_collection ::= flow_sequence | flow_mapping

# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END

# indentless_sequence ::= (BLOCK-ENTRY block_node?)+

# block_mapping ::= BLOCK-MAPPING_START

# ((KEY block_node_or_indentless_sequence?)?

# (VALUE block_node_or_indentless_sequence?)?)*

# BLOCK-END

# flow_sequence ::= FLOW-SEQUENCE-START

# (flow_sequence_entry FLOW-ENTRY)*

# flow_sequence_entry?

# FLOW-SEQUENCE-END

# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?

# flow_mapping ::= FLOW-MAPPING-START

# (flow_mapping_entry FLOW-ENTRY)*

# flow_mapping_entry?

# FLOW-MAPPING-END

# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?

# FIRST sets:

# stream: { STREAM-START }

# explicit_document: { DIRECTIVE DOCUMENT-START }

# implicit_document: FIRST(block_node)

# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }

# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }

# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }

# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }

# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }

# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }

# block_sequence: { BLOCK-SEQUENCE-START }

# block_mapping: { BLOCK-MAPPING-START }

# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }

# indentless_sequence: { ENTRY }

# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }

# flow_sequence: { FLOW-SEQUENCE-START }

# flow_mapping: { FLOW-MAPPING-START }

# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }

# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }

__all__ = ['Parser', 'ParserError']

from error import MarkedYAMLError

from tokens import *

from events import *

from scanner import *

class ParserError(MarkedYAMLError):

pass

class Parser(object):

# Since writing a recursive-descendant parser is a straightforward task, we

# do not give many comments here.

DEFAULT_TAGS = {

u'!': u'!',

u'!!': u'tag:yaml.org,2002:',

}

def __init__(self):

self.current_event = None

self.yaml_version = None

self.tag_handles = {}

self.states = []

self.marks = []

self.state = self.parse_stream_start

def dispose(self):

# Reset the state attributes (to clear self-references)

self.states = []

self.state = None

def check_event(self, *choices):

# Check the type of the next event.

if self.current_event is None:

if self.state:

self.current_event = self.state()

if self.current_event is not None:

if not choices:

return True

for choice in choices:

if isinstance(self.current_event, choice):

return True

return False

def peek_event(self):

# Get the next event.

if self.current_event is None:

if self.state:

self.current_event = self.state()

return self.current_event

def get_event(self):

# Get the next event and proceed further.

if self.current_event is None:

if self.state:

self.current_event = self.state()

value = self.current_event

self.current_event = None

return value

# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END

# implicit_document ::= block_node DOCUMENT-END*

# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*

def parse_stream_start(self):

# Parse the stream start.

token = self.get_token()

event = StreamStartEvent(token.start_mark, token.end_mark,

encoding=token.encoding)

# Prepare the next state.

self.state = self.parse_implicit_document_start

return event

def parse_implicit_document_start(self):

# Parse an implicit document.

if not self.check_token(DirectiveToken, DocumentStartToken,

StreamEndToken):

self.tag_handles = self.DEFAULT_TAGS

token = self.peek_token()

start_mark = end_mark = token.start_mark

event = DocumentStartEvent(start_mark, end_mark,

explicit=False)

# Prepare the next state.

self.states.append(self.parse_document_end)

self.state = self.parse_block_node

return event

else:

return self.parse_document_start()

def parse_document_start(self):

# Parse any extra document end indicators.

while self.check_token(DocumentEndToken):

self.get_token()

# Parse an explicit document.

if not self.check_token(StreamEndToken):

token = self.peek_token()

start_mark = token.start_mark

version, tags = self.process_directives()

if not self.check_token(DocumentStartToken):

raise ParserError(None, None,

"expected '<document start>', but found %r"

% self.peek_token().id,

self.peek_token().start_mark)

token = self.get_token()

end_mark = token.end_mark

event = DocumentStartEvent(start_mark, end_mark,

explicit=True, version=version, tags=tags)

self.states.append(self.parse_document_end)

self.state = self.parse_document_content

else:

# Parse the end of the stream.

token = self.get_token()

event = StreamEndEvent(token.start_mark, token.end_mark)

assert not self.states

assert not self.marks

self.state = None

return event

def parse_document_end(self):

# Parse the document end.

token = self.peek_token()

start_mark = end_mark = token.start_mark

explicit = False

if self.check_token(DocumentEndToken):

token = self.get_token()

end_mark = token.end_mark

explicit = True

event = DocumentEndEvent(start_mark, end_mark,

explicit=explicit)

# Prepare the next state.

self.state = self.parse_document_start

return event

def parse_document_content(self):

if self.check_token(DirectiveToken,

DocumentStartToken, DocumentEndToken, StreamEndToken):

event = self.process_empty_scalar(self.peek_token().start_mark)

self.state = self.states.pop()

return event

else:

return self.parse_block_node()

def process_directives(self):

self.yaml_version = None

self.tag_handles = {}

while self.check_token(DirectiveToken):

token = self.get_token()

if token.name == u'YAML':

if self.yaml_version is not None:

raise ParserError(None, None,

"found duplicate YAML directive", token.start_mark)

major, minor = token.value

if major != 1:

raise ParserError(None, None,

"found incompatible YAML document (version 1.* is required)",

token.start_mark)

self.yaml_version = token.value

elif token.name == u'TAG':

handle, prefix = token.value

if handle in self.tag_handles:

raise ParserError(None, None,

"duplicate tag handle %r" % handle.encode('utf-8'),

token.start_mark)

self.tag_handles[handle] = prefix

if self.tag_handles:

value = self.yaml_version, self.tag_handles.copy()

else:

value = self.yaml_version, None

for key in self.DEFAULT_TAGS:

if key not in self.tag_handles:

self.tag_handles[key] = self.DEFAULT_TAGS[key]

return value

# block_node_or_indentless_sequence ::= ALIAS

# | properties (block_content | indentless_block_sequence)?

# | block_content

# | indentless_block_sequence

# block_node ::= ALIAS

# | properties block_content?

# | block_content

# flow_node ::= ALIAS

# | properties flow_content?

# | flow_content

# properties ::= TAG ANCHOR? | ANCHOR TAG?

# block_content ::= block_collection | flow_collection | SCALAR

# flow_content ::= flow_collection | SCALAR

# block_collection ::= block_sequence | block_mapping

# flow_collection ::= flow_sequence | flow_mapping

def parse_block_node(self):

return self.parse_node(block=True)

def parse_flow_node(self):

return self.parse_node()

def parse_block_node_or_indentless_sequence(self):

return self.parse_node(block=True, indentless_sequence=True)

def parse_node(self, block=False, indentless_sequence=False):

if self.check_token(AliasToken):

token = self.get_token()

event = AliasEvent(token.value, token.start_mark, token.end_mark)

self.state = self.states.pop()

else:

anchor = None

tag = None

start_mark = end_mark = tag_mark = None

if self.check_token(AnchorToken):

token = self.get_token()

start_mark = token.start_mark

end_mark = token.end_mark

anchor = token.value

if self.check_token(TagToken):

token = self.get_token()

tag_mark = token.start_mark

end_mark = token.end_mark

tag = token.value

elif self.check_token(TagToken):

token = self.get_token()

start_mark = tag_mark = token.start_mark

end_mark = token.end_mark

tag = token.value

if self.check_token(AnchorToken):

token = self.get_token()

end_mark = token.end_mark

anchor = token.value

if tag is not None:

handle, suffix = tag

if handle is not None:

if handle not in self.tag_handles:

raise ParserError("while parsing a node", start_mark,

"found undefined tag handle %r" % handle.encode('utf-8'),

tag_mark)

tag = self.tag_handles[handle]+suffix

else:

tag = suffix

#if tag == u'!':

# raise ParserError("while parsing a node", start_mark,

# "found non-specific tag '!'", tag_mark,

# "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")

if start_mark is None:

start_mark = end_mark = self.peek_token().start_mark

event = None

implicit = (tag is None or tag == u'!')

if indentless_sequence and self.check_token(BlockEntryToken):

end_mark = self.peek_token().end_mark

event = SequenceStartEvent(anchor, tag, implicit,

start_mark, end_mark)

self.state = self.parse_indentless_sequence_entry

else:

if self.check_token(ScalarToken):

token = self.get_token()

end_mark = token.end_mark

if (token.plain and tag is None) or tag == u'!':

implicit = (True, False)

elif tag is None:

implicit = (False, True)

else:

implicit = (False, False)

event = ScalarEvent(anchor, tag, implicit, token.value,

start_mark, end_mark, style=token.style)

self.state = self.states.pop()

elif self.check_token(FlowSequenceStartToken):

end_mark = self.peek_token().end_mark

event = SequenceStartEvent(anchor, tag, implicit,

start_mark, end_mark, flow_style=True)

self.state = self.parse_flow_sequence_first_entry

elif self.check_token(FlowMappingStartToken):

end_mark = self.peek_token().end_mark

event = MappingStartEvent(anchor, tag, implicit,

start_mark, end_mark, flow_style=True)

self.state = self.parse_flow_mapping_first_key

elif block and self.check_token(BlockSequenceStartToken):

end_mark = self.peek_token().start_mark

event = SequenceStartEvent(anchor, tag, implicit,

start_mark, end_mark, flow_style=False)

self.state = self.parse_block_sequence_first_entry

elif block and self.check_token(BlockMappingStartToken):

end_mark = self.peek_token().start_mark

event = MappingStartEvent(anchor, tag, implicit,

start_mark, end_mark, flow_style=False)

self.state = self.parse_block_mapping_first_key

elif anchor is not None or tag is not None:

# Empty scalars are allowed even if a tag or an anchor is

# specified.

event = ScalarEvent(anchor, tag, (implicit, False), u'',

start_mark, end_mark)

self.state = self.states.pop()

else:

if block:

node = 'block'

else:

node = 'flow'

token = self.peek_token()

raise ParserError("while parsing a %s node" % node, start_mark,

"expected the node content, but found %r" % token.id,

token.start_mark)

return event

# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END

def parse_block_sequence_first_entry(self):

token = self.get_token()

self.marks.append(token.start_mark)

return self.parse_block_sequence_entry()

def parse_block_sequence_entry(self):

if self.check_token(BlockEntryToken):

token = self.get_token()

if not self.check_token(BlockEntryToken, BlockEndToken):

self.states.append(self.parse_block_sequence_entry)

return self.parse_block_node()

else:

self.state = self.parse_block_sequence_entry

return self.process_empty_scalar(token.end_mark)

if not self.check_token(BlockEndToken):

token = self.peek_token()

raise ParserError("while parsing a block collection", self.marks[-1],

"expected <block end>, but found %r" % token.id, token.start_mark)

token = self.get_token()

event = SequenceEndEvent(token.start_mark, token.end_mark)

self.state = self.states.pop()

self.marks.pop()

return event

# indentless_sequence ::= (BLOCK-ENTRY block_node?)+

def parse_indentless_sequence_entry(self):

if self.check_token(BlockEntryToken):

token = self.get_token()

if not self.check_token(BlockEntryToken,

KeyToken, ValueToken, BlockEndToken):

self.states.append(self.parse_indentless_sequence_entry)

return self.parse_block_node()

else:

self.state = self.parse_indentless_sequence_entry

return self.process_empty_scalar(token.end_mark)

token = self.peek_token()

event = SequenceEndEvent(token.start_mark, token.start_mark)

self.state = self.states.pop()

return event

# block_mapping ::= BLOCK-MAPPING_START

# ((KEY block_node_or_indentless_sequence?)?

# (VALUE block_node_or_indentless_sequence?)?)*

# BLOCK-END

def parse_block_mapping_first_key(self):

token = self.get_token()

self.marks.append(token.start_mark)

return self.parse_block_mapping_key()

def parse_block_mapping_key(self):

if self.check_token(KeyToken):

token = self.get_token()

if not self.check_token(KeyToken, ValueToken, BlockEndToken):

self.states.append(self.parse_block_mapping_value)

return self.parse_block_node_or_indentless_sequence()

else:

self.state = self.parse_block_mapping_value

return self.process_empty_scalar(token.end_mark)

if not self.check_token(BlockEndToken):

token = self.peek_token()

raise ParserError("while parsing a block mapping", self.marks[-1],

"expected <block end>, but found %r" % token.id, token.start_mark)

token = self.get_token()

event = MappingEndEvent(token.start_mark, token.end_mark)

self.state = self.states.pop()

self.marks.pop()

return event

def parse_block_mapping_value(self):

if self.check_token(ValueToken):

token = self.get_token()

if not self.check_token(KeyToken, ValueToken, BlockEndToken):

self.states.append(self.parse_block_mapping_key)

return self.parse_block_node_or_indentless_sequence()

else:

self.state = self.parse_block_mapping_key

return self.process_empty_scalar(token.end_mark)

else:

self.state = self.parse_block_mapping_key

token = self.peek_token()

return self.process_empty_scalar(token.start_mark)

# flow_sequence ::= FLOW-SEQUENCE-START

# (flow_sequence_entry FLOW-ENTRY)*

# flow_sequence_entry?

# FLOW-SEQUENCE-END

# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?

# Note that while production rules for both flow_sequence_entry and

# flow_mapping_entry are equal, their interpretations are different.

# For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`

# generate an inline mapping (set syntax).

def parse_flow_sequence_first_entry(self):

token = self.get_token()

self.marks.append(token.start_mark)

return self.parse_flow_sequence_entry(first=True)

def parse_flow_sequence_entry(self, first=False):

if not self.check_token(FlowSequenceEndToken):

if not first:

if self.check_token(FlowEntryToken):

self.get_token()

else:

token = self.peek_token()

raise ParserError("while parsing a flow sequence", self.marks[-1],

"expected ',' or ']', but got %r" % token.id, token.start_mark)

if self.check_token(KeyToken):

token = self.peek_token()

event = MappingStartEvent(None, None, True,

token.start_mark, token.end_mark,

flow_style=True)

self.state = self.parse_flow_sequence_entry_mapping_key

return event

elif not self.check_token(FlowSequenceEndToken):

self.states.append(self.parse_flow_sequence_entry)

return self.parse_flow_node()

token = self.get_token()

event = SequenceEndEvent(token.start_mark, token.end_mark)

self.state = self.states.pop()

self.marks.pop()

return event

def parse_flow_sequence_entry_mapping_key(self):

token = self.get_token()

if not self.check_token(ValueToken,

FlowEntryToken, FlowSequenceEndToken):

self.states.append(self.parse_flow_sequence_entry_mapping_value)

return self.parse_flow_node()

else:

self.state = self.parse_flow_sequence_entry_mapping_value

return self.process_empty_scalar(token.end_mark)

def parse_flow_sequence_entry_mapping_value(self):

if self.check_token(ValueToken):

token = self.get_token()

if not self.check_token(FlowEntryToken, FlowSequenceEndToken):

self.states.append(self.parse_flow_sequence_entry_mapping_end)

return self.parse_flow_node()

else:

self.state = self.parse_flow_sequence_entry_mapping_end

return self.process_empty_scalar(token.end_mark)

else:

self.state = self.parse_flow_sequence_entry_mapping_end

token = self.peek_token()

return self.process_empty_scalar(token.start_mark)

def parse_flow_sequence_entry_mapping_end(self):

self.state = self.parse_flow_sequence_entry

token = self.peek_token()

return MappingEndEvent(token.start_mark, token.start_mark)

# flow_mapping ::= FLOW-MAPPING-START

# (flow_mapping_entry FLOW-ENTRY)*

# flow_mapping_entry?

# FLOW-MAPPING-END

# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?

def parse_flow_mapping_first_key(self):

token = self.get_token()

self.marks.append(token.start_mark)

return self.parse_flow_mapping_key(first=True)

def parse_flow_mapping_key(self, first=False):

if not self.check_token(FlowMappingEndToken):

if not first:

if self.check_token(FlowEntryToken):

self.get_token()

else:

token = self.peek_token()

raise ParserError("while parsing a flow mapping", self.marks[-1],

"expected ',' or '}', but got %r" % token.id, token.start_mark)

if self.check_token(KeyToken):

token = self.get_token()

if not self.check_token(ValueToken,

FlowEntryToken, FlowMappingEndToken):

self.states.append(self.parse_flow_mapping_value)

return self.parse_flow_node()

else:

self.state = self.parse_flow_mapping_value

return self.process_empty_scalar(token.end_mark)

elif not self.check_token(FlowMappingEndToken):

self.states.append(self.parse_flow_mapping_empty_value)

return self.parse_flow_node()

token = self.get_token()

event = MappingEndEvent(token.start_mark, token.end_mark)

self.state = self.states.pop()

self.marks.pop()

return event

def parse_flow_mapping_value(self):

if self.check_token(ValueToken):

token = self.get_token()

if not self.check_token(FlowEntryToken, FlowMappingEndToken):

self.states.append(self.parse_flow_mapping_key)

return self.parse_flow_node()

else:

self.state = self.parse_flow_mapping_key

return self.process_empty_scalar(token.end_mark)

else:

self.state = self.parse_flow_mapping_key

token = self.peek_token()

return self.process_empty_scalar(token.start_mark)

def parse_flow_mapping_empty_value(self):

self.state = self.parse_flow_mapping_key

return self.process_empty_scalar(self.peek_token().start_mark)

def process_empty_scalar(self, mark):

return ScalarEvent(None, None, (True, False), u'', mark, mark)

netdata/python.d/python_modules/pyyaml2/parser.py at master · ed00m/netdata · GitHub

Expand file tree

Breadcrumbs

parser.py

Latest commit

History

Breadcrumbs

parser.py

File metadata and controls