| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -2363,6 +2363,26 @@ def test_multiline_readline_chunk(self): | |||
| 2363 | 2363 | )) | |
| 2364 | 2364 | self.assertEqual(tokens, expected) | |
| 2365 | 2365 | ||
| 2366 | + def test_multiline_readline_matches_physical_lines(self): | ||
| 2367 | + source = ( | ||
| 2368 | + 'value = f"""first {\n' | ||
| 2369 | + ' 40 + 2\n' | ||
| 2370 | + '} last"""\n' | ||
| 2371 | + 'result = value\n' | ||
| 2372 | + ) | ||
| 2373 | + physical_lines = iter(source.splitlines(keepends=True) + [""]) | ||
| 2374 | + multiline_chunks = iter([source, ""]) | ||
| 2375 | + | ||
| 2376 | + expected = list(tokenize._generate_tokens_from_c_tokenizer( | ||
| 2377 | + physical_lines.__next__, | ||
| 2378 | + extra_tokens=True, | ||
| 2379 | + )) | ||
| 2380 | + actual = list(tokenize._generate_tokens_from_c_tokenizer( | ||
| 2381 | + multiline_chunks.__next__, | ||
| 2382 | + extra_tokens=True, | ||
| 2383 | + )) | ||
| 2384 | + self.assertEqual(actual, expected) | ||
| 2385 | + | ||
| 2366 | 2386 | def test_multiline_readline_chunk_with_unterminated_tail(self): | |
| 2367 | 2387 | readline = mock.Mock(side_effect=["x\nz", ""]) | |
| 2368 | 2388 | iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) | |
@@ -2427,6 +2447,34 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self): | |||
| 2427 | 2447 | (token.ENDMARKER, "", (1, 0), (1, 0), ""), | |
| 2428 | 2448 | ) | |
| 2429 | 2449 | ||
| 2450 | + def test_fstring_offsets_survive_source_reallocation(self): | ||
| 2451 | + padding = " " * 9000 | ||
| 2452 | + expression_line = ")=:>{2}}\n" | ||
| 2453 | + physical_lines = [ | ||
| 2454 | + 'f"""\n', | ||
| 2455 | + "{(\n", | ||
| 2456 | + padding + "1\n", | ||
| 2457 | + expression_line, | ||
| 2458 | + '"""\n', | ||
| 2459 | + ] | ||
| 2460 | + source = "".join(physical_lines) | ||
| 2461 | + chunks = iter([ | ||
| 2462 | + "".join(physical_lines[:2]), | ||
| 2463 | + "".join(physical_lines[2:4]), | ||
| 2464 | + physical_lines[4], | ||
| 2465 | + "", | ||
| 2466 | + ]) | ||
| 2467 | + | ||
| 2468 | + expected = self._get_tokens(source, extra_tokens=True) | ||
| 2469 | + tokens = list(tokenize._generate_tokens_from_c_tokenizer( | ||
| 2470 | + chunks.__next__, | ||
| 2471 | + extra_tokens=True, | ||
| 2472 | + )) | ||
| 2473 | + self.assertEqual(tokens, expected) | ||
| 2474 | + number = next(tok for tok in tokens if tok.type == token.NUMBER) | ||
| 2475 | + self.assertEqual(number.string, "1") | ||
| 2476 | + self.assertEqual((number.start, number.end), ((3, 9000), (3, 9001))) | ||
| 2477 | + | ||
| 2430 | 2478 | def test_extra_tokens_relaxes_lexer_errors(self): | |
| 2431 | 2479 | cases = [ | |
| 2432 | 2480 | ( | |
| Back | FazBrowse Home | New Git URL |
0 commit comments