FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
cpython/Python/specialize.c at 3.15 · python/cpython · GitHub
python
cpython
Repository navigation
Code
Issues
5k+
(5k+)
Pull requests
2.3k
(2.3k)
Actions
Projects
Security and quality
Insights
Expand file tree
Breadcrumbs
cpython
/
Python
/
specialize.c
Copy path
More file actions
More file actions
Latest commit
History
History
History
3109 lines (2936 loc) · 109 KB
Breadcrumbs
cpython
/
Python
/
specialize.c
Copy path
File metadata and controls
3109 lines (2936 loc) · 109 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include
"Python.h"
#include
"opcode.h"
#include
"pycore_bytesobject.h"
// _PyBytes_Concat
#include
"pycore_code.h"
#include
"pycore_critical_section.h"
#include
"pycore_descrobject.h"
// _PyMethodWrapper_Type
#include
"pycore_dict.h"
// DICT_KEYS_UNICODE
#include
"pycore_function.h"
// _PyFunction_GetVersionForCurrentState()
#include
"pycore_interpframe.h"
// FRAME_SPECIALS_SIZE
#include
"pycore_lazyimportobject.h"
// PyLazyImport_CheckExact
#include
"pycore_list.h"
// _PyListIterObject, _PyList_Concat
#include
"pycore_tuple.h"
// _PyTuple_Concat
#include
"pycore_long.h"
// _PyLong_IsNonNegativeCompact()
#include
"pycore_moduleobject.h"
#include
"pycore_object.h"
#include
"pycore_opcode_metadata.h"
// _PyOpcode_Caches
#include
"pycore_uop_metadata.h"
// _PyOpcode_uop_name
#include
"pycore_uop_ids.h"
// MAX_UOP_ID
#include
"pycore_opcode_utils.h"
// RESUME_AT_FUNC_START
#include
"pycore_pylifecycle.h"
// _PyOS_URandomNonblock()
#include
"pycore_runtime.h"
// _Py_ID()
#include
"pycore_unicodeobject.h"
// _PyUnicodeASCIIIter_Type
#include
<stdlib.h>
// rand()
/* For guidance on adding or extending families of instructions see
* InternalDocs/interpreter.md `Specialization` section.
*/
#if
Py_STATS
#define
SPECIALIZATION_FAIL
(
opcode
,
kind
) \
do { \
PyStats *s = _PyStats_GET(); \
if (s) { \
int _kind = (kind); \
assert(_kind < SPECIALIZATION_FAILURE_KINDS); \
s->opcode_stats[opcode].specialization.failure_kinds[_kind]++; \
} \
} while (0)
#else
# define
SPECIALIZATION_FAIL
(
opcode
,
kind
) ((void)0)
#endif
// Py_STATS
static
void
fixup_getiter
(
_Py_CODEUNIT
*
instruction
,
int
flags
)
{
// Compiler can't know if types.coroutine() will be called,
// so fix up here
if
(
instruction
->
op
.
arg
) {
if
(
flags
&
(
CO_COROUTINE
|
CO_ITERABLE_COROUTINE
)) {
instruction
->
op
.
arg
=
GET_ITER_YIELD_FROM_NO_CHECK
;
}
else
{
instruction
->
op
.
arg
=
GET_ITER_YIELD_FROM_CORO_CHECK
;
}
}
}
// Initialize warmup counters and optimize instructions. This cannot fail.
void
_PyCode_Quicken
(
_Py_CODEUNIT
*
instructions
,
Py_ssize_t
size
,
int
enable_counters
,
int
flags
)
{
#if
ENABLE_SPECIALIZATION
_Py_BackoffCounter
jump_counter
,
adaptive_counter
,
resume_counter
;
if
(
enable_counters
) {
PyThreadState
*
tstate
=
_PyThreadState_GET
();
PyInterpreterState
*
interp
=
tstate
->
interp
;
jump_counter
=
initial_jump_backoff_counter
(
&
interp
->
opt_config
);
adaptive_counter
=
adaptive_counter_warmup
();
resume_counter
=
initial_resume_backoff_counter
(
&
interp
->
opt_config
);
}
else
{
jump_counter
=
initial_unreachable_backoff_counter
();
adaptive_counter
=
initial_unreachable_backoff_counter
();
resume_counter
=
initial_unreachable_backoff_counter
();
}
int
opcode
=
0
;
int
oparg
=
0
;
/* The last code unit cannot have a cache, so we don't need to check it */
for
(
Py_ssize_t
i
=
0
;
i
<
size
-
1
;
i
++
) {
opcode
=
instructions
[
i
].
op
.
code
;
int
caches
=
_PyOpcode_Caches
[
opcode
];
oparg
=
(
oparg
<<
8
) |
instructions
[
i
].
op
.
arg
;
if
(
opcode
==
GET_ITER
) {
fixup_getiter
(
&
instructions
[
i
],
flags
);
}
if
(
caches
) {
// The initial value depends on the opcode
switch
(
opcode
) {
case
JUMP_BACKWARD
:
instructions
[
i
+
1
].
counter
=
jump_counter
;
break
;
case
RESUME
:
instructions
[
i
+
1
].
counter
=
resume_counter
;
break
;
case
POP_JUMP_IF_FALSE
:
case
POP_JUMP_IF_TRUE
:
case
POP_JUMP_IF_NONE
:
case
POP_JUMP_IF_NOT_NONE
:
instructions
[
i
+
1
].
cache
=
0x5555
;
// Alternating 0, 1 bits
break
;
default
:
instructions
[
i
+
1
].
counter
=
adaptive_counter
;
break
;
}
i
+=
caches
;
}
if
(
opcode
!=
EXTENDED_ARG
) {
oparg
=
0
;
}
}
#else
for
(
Py_ssize_t
i
=
0
;
i
<
size
-
1
;
i
++
) {
int
opcode
=
instructions
[
i
].
op
.
code
;
if
(
opcode
==
GET_ITER
) {
fixup_getiter
(
&
instructions
[
i
],
flags
);
}
i
+=
_PyOpcode_Caches
[
opcode
];
}
#endif
/* ENABLE_SPECIALIZATION */
}
#define
SIMPLE_FUNCTION
0
/* Common */
#define
SPEC_FAIL_OTHER
0
#define
SPEC_FAIL_NO_DICT
1
#define
SPEC_FAIL_OVERRIDDEN
2
#define
SPEC_FAIL_OUT_OF_VERSIONS
3
#define
SPEC_FAIL_OUT_OF_RANGE
4
#define
SPEC_FAIL_EXPECTED_ERROR
5
#define
SPEC_FAIL_WRONG_NUMBER_ARGUMENTS
6
#define
SPEC_FAIL_CODE_COMPLEX_PARAMETERS
7
#define
SPEC_FAIL_CODE_NOT_OPTIMIZED
8
#define
SPEC_FAIL_LOAD_GLOBAL_NON_DICT
17
#define
SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT
18
/* Super */
#define
SPEC_FAIL_SUPER_BAD_CLASS
9
#define
SPEC_FAIL_SUPER_SHADOWED
10
/* Attributes */
#define
SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR
9
#define
SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR
10
#define
SPEC_FAIL_ATTR_NOT_DESCRIPTOR
11
#define
SPEC_FAIL_ATTR_METHOD
12
#define
SPEC_FAIL_ATTR_MUTABLE_CLASS
13
#define
SPEC_FAIL_ATTR_PROPERTY
14
#define
SPEC_FAIL_ATTR_NON_OBJECT_SLOT
15
#define
SPEC_FAIL_ATTR_READ_ONLY
16
#define
SPEC_FAIL_ATTR_AUDITED_SLOT
17
#define
SPEC_FAIL_ATTR_NOT_MANAGED_DICT
18
#define
SPEC_FAIL_ATTR_NON_STRING
19
#define
SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND
20
#define
SPEC_FAIL_ATTR_SHADOWED
21
#define
SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD
22
#define
SPEC_FAIL_ATTR_CLASS_METHOD_OBJ
23
#define
SPEC_FAIL_ATTR_OBJECT_SLOT
24
#define
SPEC_FAIL_ATTR_MODULE_LAZY_VALUE
25
#define
SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE
26
#define
SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE
27
#define
SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION
28
#define
SPEC_FAIL_ATTR_NOT_IN_KEYS
29
#define
SPEC_FAIL_ATTR_NOT_IN_DICT
30
#define
SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE
31
#define
SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR
32
#define
SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ
33
#define
SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN
34
#define
SPEC_FAIL_ATTR_SPLIT_DICT
35
#define
SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED
36
#define
SPEC_FAIL_ATTR_SLOT_AFTER_ITEMS
37
/* Binary subscr and store subscr */
#define
SPEC_FAIL_SUBSCR_ARRAY_INT
9
#define
SPEC_FAIL_SUBSCR_ARRAY_SLICE
10
#define
SPEC_FAIL_SUBSCR_LIST_SLICE
11
#define
SPEC_FAIL_SUBSCR_BUFFER_INT
12
#define
SPEC_FAIL_SUBSCR_BUFFER_SLICE
13
/* Store subscr */
#define
SPEC_FAIL_SUBSCR_BYTEARRAY_INT
18
#define
SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE
19
#define
SPEC_FAIL_SUBSCR_PY_SIMPLE
20
#define
SPEC_FAIL_SUBSCR_PY_OTHER
21
#define
SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE
22
#define
SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE
23
/* Binary op */
#define
SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES
9
#define
SPEC_FAIL_BINARY_OP_ADD_OTHER
10
#define
SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES
11
#define
SPEC_FAIL_BINARY_OP_AND_INT
12
#define
SPEC_FAIL_BINARY_OP_AND_OTHER
13
#define
SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE
14
#define
SPEC_FAIL_BINARY_OP_LSHIFT
15
#define
SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY
16
#define
SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES
17
#define
SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER
18
#define
SPEC_FAIL_BINARY_OP_OR
19
#define
SPEC_FAIL_BINARY_OP_POWER
20
#define
SPEC_FAIL_BINARY_OP_REMAINDER
21
#define
SPEC_FAIL_BINARY_OP_RSHIFT
22
#define
SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES
23
#define
SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER
24
#define
SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES
25
#define
SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT
26
#define
SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER
27
#define
SPEC_FAIL_BINARY_OP_XOR
28
#define
SPEC_FAIL_BINARY_OP_OR_INT
29
#define
SPEC_FAIL_BINARY_OP_OR_DIFFERENT_TYPES
30
#define
SPEC_FAIL_BINARY_OP_XOR_INT
31
#define
SPEC_FAIL_BINARY_OP_XOR_DIFFERENT_TYPES
32
#define
SPEC_FAIL_BINARY_OP_SUBSCR
33
#define
SPEC_FAIL_BINARY_OP_SUBSCR_LIST_SLICE
34
#define
SPEC_FAIL_BINARY_OP_SUBSCR_TUPLE_SLICE
35
#define
SPEC_FAIL_BINARY_OP_SUBSCR_STRING_SLICE
36
#define
SPEC_FAIL_BINARY_OP_SUBSCR_NOT_HEAP_TYPE
37
#define
SPEC_FAIL_BINARY_OP_SUBSCR_OTHER_SLICE
38
#define
SPEC_FAIL_BINARY_OP_SUBSCR_MAPPINGPROXY
39
#define
SPEC_FAIL_BINARY_OP_SUBSCR_RE_MATCH
40
#define
SPEC_FAIL_BINARY_OP_SUBSCR_ARRAY
41
#define
SPEC_FAIL_BINARY_OP_SUBSCR_DEQUE
42
#define
SPEC_FAIL_BINARY_OP_SUBSCR_ENUMDICT
43
#define
SPEC_FAIL_BINARY_OP_SUBSCR_STACKSUMMARY
44
#define
SPEC_FAIL_BINARY_OP_SUBSCR_DEFAULTDICT
45
#define
SPEC_FAIL_BINARY_OP_SUBSCR_COUNTER
46
#define
SPEC_FAIL_BINARY_OP_SUBSCR_ORDEREDDICT
47
#define
SPEC_FAIL_BINARY_OP_SUBSCR_BYTES
48
#define
SPEC_FAIL_BINARY_OP_SUBSCR_STRUCTTIME
49
#define
SPEC_FAIL_BINARY_OP_SUBSCR_RANGE
50
/* Calls */
#define
SPEC_FAIL_CALL_INSTANCE_METHOD
11
#define
SPEC_FAIL_CALL_CMETHOD
12
#define
SPEC_FAIL_CALL_CFUNC_VARARGS
13
#define
SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS
14
#define
SPEC_FAIL_CALL_CFUNC_NOARGS
15
#define
SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS
16
#define
SPEC_FAIL_CALL_METH_DESCR_VARARGS
17
#define
SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS
18
#define
SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS
19
#define
SPEC_FAIL_CALL_BAD_CALL_FLAGS
20
#define
SPEC_FAIL_CALL_INIT_NOT_PYTHON
21
#define
SPEC_FAIL_CALL_PEP_523
22
#define
SPEC_FAIL_CALL_BOUND_METHOD
23
#define
SPEC_FAIL_CALL_VECTORCALL
24
#define
SPEC_FAIL_CALL_CLASS_MUTABLE
26
#define
SPEC_FAIL_CALL_METHOD_WRAPPER
28
#define
SPEC_FAIL_CALL_OPERATOR_WRAPPER
29
#define
SPEC_FAIL_CALL_INIT_NOT_SIMPLE
30
#define
SPEC_FAIL_CALL_METACLASS
31
#define
SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES
32
/* COMPARE_OP */
#define
SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES
12
#define
SPEC_FAIL_COMPARE_OP_STRING
13
#define
SPEC_FAIL_COMPARE_OP_BIG_INT
14
#define
SPEC_FAIL_COMPARE_OP_BYTES
15
#define
SPEC_FAIL_COMPARE_OP_TUPLE
16
#define
SPEC_FAIL_COMPARE_OP_LIST
17
#define
SPEC_FAIL_COMPARE_OP_SET
18
#define
SPEC_FAIL_COMPARE_OP_BOOL
19
#define
SPEC_FAIL_COMPARE_OP_BASEOBJECT
20
#define
SPEC_FAIL_COMPARE_OP_FLOAT_LONG
21
#define
SPEC_FAIL_COMPARE_OP_LONG_FLOAT
22
/* FOR_ITER and SEND */
#define
SPEC_FAIL_ITER_GENERATOR
10
#define
SPEC_FAIL_ITER_COROUTINE
11
#define
SPEC_FAIL_ITER_ASYNC_GENERATOR
12
#define
SPEC_FAIL_ITER_LIST
13
#define
SPEC_FAIL_ITER_TUPLE
14
#define
SPEC_FAIL_ITER_SET
15
#define
SPEC_FAIL_ITER_STRING
16
#define
SPEC_FAIL_ITER_BYTES
17
#define
SPEC_FAIL_ITER_RANGE
18
#define
SPEC_FAIL_ITER_ITERTOOLS
19
#define
SPEC_FAIL_ITER_DICT_KEYS
20
#define
SPEC_FAIL_ITER_DICT_ITEMS
21
#define
SPEC_FAIL_ITER_DICT_VALUES
22
#define
SPEC_FAIL_ITER_ENUMERATE
23
#define
SPEC_FAIL_ITER_MAP
24
#define
SPEC_FAIL_ITER_ZIP
25
#define
SPEC_FAIL_ITER_SEQ_ITER
26
#define
SPEC_FAIL_ITER_REVERSED_LIST
27
#define
SPEC_FAIL_ITER_CALLABLE
28
#define
SPEC_FAIL_ITER_ASCII_STRING
29
#define
SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND
30
#define
SPEC_FAIL_ITER_SELF
31
// UNPACK_SEQUENCE
#define
SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR
9
#define
SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE
10
// TO_BOOL
#define
SPEC_FAIL_TO_BOOL_BYTEARRAY
9
#define
SPEC_FAIL_TO_BOOL_BYTES
10
#define
SPEC_FAIL_TO_BOOL_DICT
11
#define
SPEC_FAIL_TO_BOOL_FLOAT
12
#define
SPEC_FAIL_TO_BOOL_MAPPING
13
#define
SPEC_FAIL_TO_BOOL_MEMORY_VIEW
14
#define
SPEC_FAIL_TO_BOOL_NUMBER
15
#define
SPEC_FAIL_TO_BOOL_SEQUENCE
16
#define
SPEC_FAIL_TO_BOOL_SET
17
#define
SPEC_FAIL_TO_BOOL_TUPLE
18
// CONTAINS_OP
#define
SPEC_FAIL_CONTAINS_OP_STR
9
#define
SPEC_FAIL_CONTAINS_OP_TUPLE
10
#define
SPEC_FAIL_CONTAINS_OP_LIST
11
#define
SPEC_FAIL_CONTAINS_OP_USER_CLASS
12
static
inline
int
set_opcode
(
_Py_CODEUNIT
*
instr
,
uint8_t
opcode
)
{
#ifdef
Py_GIL_DISABLED
uint8_t
old_op
=
_Py_atomic_load_uint8_relaxed
(
&
instr
->
op
.
code
);
if
(
old_op
>=
MIN_INSTRUMENTED_OPCODE
) {
/* Lost race with instrumentation */
return
0
;
}
if
(!
_Py_atomic_compare_exchange_uint8
(
&
instr
->
op
.
code
,
&
old_op
,
opcode
)) {
/* Lost race with instrumentation */
assert
(
old_op
>=
MIN_INSTRUMENTED_OPCODE
);
return
0
;
}
return
1
;
#else
instr
->
op
.
code
=
opcode
;
return
1
;
#endif
}
static
inline
void
set_counter
(
_Py_BackoffCounter
*
counter
,
_Py_BackoffCounter
value
)
{
FT_ATOMIC_STORE_UINT16_RELAXED
(
counter
->
value_and_backoff
,
value
.
value_and_backoff
);
}
static
inline
_Py_BackoffCounter
load_counter
(
_Py_BackoffCounter
*
counter
)
{
_Py_BackoffCounter
result
=
{
.
value_and_backoff
=
FT_ATOMIC_LOAD_UINT16_RELAXED
(
counter
->
value_and_backoff
)};
return
result
;
}
static
inline
void
specialize
(
_Py_CODEUNIT
*
instr
,
uint8_t
specialized_opcode
)
{
assert
(!
PyErr_Occurred
());
if
(!
set_opcode
(
instr
,
specialized_opcode
)) {
STAT_INC
(
_PyOpcode_Deopt
[
specialized_opcode
],
failure
);
SPECIALIZATION_FAIL
(
_PyOpcode_Deopt
[
specialized_opcode
],
SPEC_FAIL_OTHER
);
return
;
}
STAT_INC
(
_PyOpcode_Deopt
[
specialized_opcode
],
success
);
set_counter
((
_Py_BackoffCounter
*
)
instr
+
1
,
adaptive_counter_cooldown
());
}
static
inline
void
unspecialize
(
_Py_CODEUNIT
*
instr
)
{
assert
(!
PyErr_Occurred
());
uint8_t
opcode
=
FT_ATOMIC_LOAD_UINT8_RELAXED
(
instr
->
op
.
code
);
uint8_t
generic_opcode
=
_PyOpcode_Deopt
[
opcode
];
STAT_INC
(
generic_opcode
,
failure
);
if
(!
set_opcode
(
instr
,
generic_opcode
)) {
SPECIALIZATION_FAIL
(
generic_opcode
,
SPEC_FAIL_OTHER
);
return
;
}
_Py_BackoffCounter
*
counter
=
(
_Py_BackoffCounter
*
)
instr
+
1
;
_Py_BackoffCounter
cur
=
load_counter
(
counter
);
set_counter
(
counter
,
adaptive_counter_backoff
(
cur
));
}
static
int
function_kind
(
PyCodeObject
*
code
);
static
bool
function_check_args
(
PyObject
*
o
,
int
expected_argcount
,
int
opcode
);
static
uint32_t
function_get_version
(
PyObject
*
o
,
int
opcode
);
#ifdef
Py_GIL_DISABLED
static
void
maybe_enable_deferred_ref_count
(
PyObject
*
op
)
{
if
(!
_Py_IsOwnedByCurrentThread
(
op
)
&&
_PyObject_GC_IS_TRACKED
(
op
)) {
// For module level variables that are heavily used from multiple
// threads, deferred reference counting provides good scaling
// benefits. The downside is that the object will only be deallocated
// by a GC run.
PyUnstable_Object_EnableDeferredRefcount
(
op
);
}
}
#endif
static
int
specialize_module_load_attr_lock_held
(
PyDictObject
*
dict
,
_Py_CODEUNIT
*
instr
,
PyObject
*
name
)
{
_PyAttrCache
*
cache
=
(
_PyAttrCache
*
)(
instr
+
1
);
if
(
dict
->
ma_keys
->
dk_kind
!=
DICT_KEYS_UNICODE
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_NON_STRING
);
return
-1
;
}
PyObject
*
value
;
Py_ssize_t
index
=
_PyDict_LookupIndexAndValue
(
dict
,
name
,
&
value
);
if
(
value
!=
NULL
&&
PyLazyImport_CheckExact
(
value
)) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_MODULE_LAZY_VALUE
);
return
-1
;
}
assert
(
index
!=
DKIX_ERROR
);
if
(
index
!=
(
uint16_t
)
index
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
index
==
DKIX_EMPTY
?
SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND
:
SPEC_FAIL_OUT_OF_RANGE
);
return
-1
;
}
uint32_t
keys_version
=
_PyDict_GetKeysVersionForCurrentState
(
_PyInterpreterState_GET
(),
dict
);
if
(
keys_version
==
0
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_OUT_OF_VERSIONS
);
return
-1
;
}
#ifdef
Py_GIL_DISABLED
maybe_enable_deferred_ref_count
(
value
);
#endif
write_u32
(
cache
->
version
,
keys_version
);
cache
->
index
=
(
uint16_t
)
index
;
specialize
(
instr
,
LOAD_ATTR_MODULE
);
return
0
;
}
static
int
specialize_module_load_attr
(
PyObject
*
owner
,
_Py_CODEUNIT
*
instr
,
PyObject
*
name
)
{
PyModuleObject
*
m
=
(
PyModuleObject
*
)
owner
;
assert
((
Py_TYPE
(
owner
)
->
tp_flags
&
Py_TPFLAGS_MANAGED_DICT
)
==
0
);
PyDictObject
*
dict
=
(
PyDictObject
*
)
m
->
md_dict
;
if
(
dict
==
NULL
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_NO_DICT
);
return
-1
;
}
int
result
;
Py_BEGIN_CRITICAL_SECTION
(
dict
);
result
=
specialize_module_load_attr_lock_held
(
dict
,
instr
,
name
);
Py_END_CRITICAL_SECTION
();
return
result
;
}
/* Attribute specialization */
Py_NO_INLINE
void
_Py_Specialize_LoadSuperAttr
(
_PyStackRef
global_super_st
,
_PyStackRef
cls_st
,
_Py_CODEUNIT
*
instr
,
int
load_method
) {
PyObject
*
global_super
=
PyStackRef_AsPyObjectBorrow
(
global_super_st
);
PyObject
*
cls
=
PyStackRef_AsPyObjectBorrow
(
cls_st
);
assert
(
ENABLE_SPECIALIZATION
);
assert
(
_PyOpcode_Caches
[
LOAD_SUPER_ATTR
]
==
INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR
);
if
(
global_super
!=
(
PyObject
*
)
&
PySuper_Type
) {
SPECIALIZATION_FAIL
(
LOAD_SUPER_ATTR
,
SPEC_FAIL_SUPER_SHADOWED
);
goto
fail
;
}
if
(!
PyType_Check
(
cls
)) {
SPECIALIZATION_FAIL
(
LOAD_SUPER_ATTR
,
SPEC_FAIL_SUPER_BAD_CLASS
);
goto
fail
;
}
uint8_t
load_code
=
load_method
?
LOAD_SUPER_ATTR_METHOD
:
LOAD_SUPER_ATTR_ATTR
;
specialize
(
instr
,
load_code
);
return
;
fail
:
unspecialize
(
instr
);
}
typedef
enum
{
OVERRIDING
,
/* Is an overriding descriptor, and will remain so. */
METHOD
,
/* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */
PROPERTY
,
/* Is a property */
OBJECT_SLOT
,
/* Is an object slot descriptor */
OTHER_SLOT
,
/* Is a slot descriptor of another type */
NON_OVERRIDING
,
/* Is another non-overriding descriptor, and is an instance of an immutable class*/
BUILTIN_CLASSMETHOD
,
/* Builtin methods with METH_CLASS */
PYTHON_CLASSMETHOD
,
/* Python classmethod(func) object */
NON_DESCRIPTOR
,
/* Is not a descriptor, and is an instance of an immutable class */
MUTABLE
,
/* Instance of a mutable class; might, or might not, be a descriptor */
ABSENT
,
/* Attribute is not present on the class */
DUNDER_CLASS
,
/* __class__ attribute */
GETSET_OVERRIDDEN
,
/* __getattribute__ or __setattr__ has been overridden */
GETATTRIBUTE_IS_PYTHON_FUNCTION
/* Descriptor requires calling a Python __getattribute__ */
}
DescriptorClassification
;
static
DescriptorClassification
classify_descriptor
(
PyObject
*
descriptor
,
bool
has_getattr
)
{
if
(
descriptor
==
NULL
) {
return
ABSENT
;
}
PyTypeObject
*
desc_cls
=
Py_TYPE
(
descriptor
);
if
(!(
desc_cls
->
tp_flags
&
Py_TPFLAGS_IMMUTABLETYPE
)) {
return
MUTABLE
;
}
if
(
desc_cls
->
tp_descr_set
) {
if
(
desc_cls
==
&
PyMemberDescr_Type
) {
PyMemberDescrObject
*
member
=
(
PyMemberDescrObject
*
)
descriptor
;
struct
PyMemberDef
*
dmem
=
member
->
d_member
;
if
(
dmem
->
type
==
Py_T_OBJECT_EX
||
dmem
->
type
==
_Py_T_OBJECT
) {
return
OBJECT_SLOT
;
}
return
OTHER_SLOT
;
}
if
(
desc_cls
==
&
PyProperty_Type
) {
/* We can't detect at runtime whether an attribute exists
with property. So that means we may have to call
__getattr__. */
return
has_getattr
?
GETSET_OVERRIDDEN
:
PROPERTY
;
}
return
OVERRIDING
;
}
if
(
desc_cls
->
tp_descr_get
) {
if
(
desc_cls
->
tp_flags
&
Py_TPFLAGS_METHOD_DESCRIPTOR
) {
return
METHOD
;
}
if
(
Py_IS_TYPE
(
descriptor
,
&
PyClassMethodDescr_Type
)) {
return
BUILTIN_CLASSMETHOD
;
}
if
(
Py_IS_TYPE
(
descriptor
,
&
PyClassMethod_Type
)) {
return
PYTHON_CLASSMETHOD
;
}
return
NON_OVERRIDING
;
}
return
NON_DESCRIPTOR
;
}
static
bool
descriptor_is_class
(
PyObject
*
descriptor
,
PyObject
*
name
)
{
return
((
PyUnicode_CompareWithASCIIString
(
name
,
"__class__"
)
==
0
)
&&
(
descriptor
==
_PyType_Lookup
(
&
PyBaseObject_Type
,
name
)));
}
static
DescriptorClassification
analyze_descriptor_load
(
PyTypeObject
*
type
,
PyObject
*
name
,
PyObject
*
*
descr
,
unsigned
int
*
tp_version
) {
bool
has_getattr
=
false;
bool
have_ga_version
=
false;
unsigned
int
ga_version
;
getattrofunc
getattro_slot
=
type
->
tp_getattro
;
if
(
getattro_slot
==
PyObject_GenericGetAttr
) {
/* Normal attribute lookup; */
has_getattr
=
false;
}
else
if
(
getattro_slot
==
_Py_slot_tp_getattr_hook
||
getattro_slot
==
_Py_slot_tp_getattro
) {
/* One or both of __getattribute__ or __getattr__ may have been
overridden See typeobject.c for why these functions are special. */
PyObject
*
getattribute
=
_PyType_LookupRefAndVersion
(
type
,
&
_Py_ID
(
__getattribute__
),
&
ga_version
);
have_ga_version
=
true;
PyInterpreterState
*
interp
=
_PyInterpreterState_GET
();
bool
has_custom_getattribute
=
getattribute
!=
NULL
&&
getattribute
!=
interp
->
callable_cache
.
object__getattribute__
;
PyObject
*
getattr
=
_PyType_Lookup
(
type
,
&
_Py_ID
(
__getattr__
));
has_getattr
=
getattr
!=
NULL
;
if
(
has_custom_getattribute
) {
if
(!
has_getattr
&&
Py_IS_TYPE
(
getattribute
,
&
PyFunction_Type
)) {
*
descr
=
getattribute
;
*
tp_version
=
ga_version
;
return
GETATTRIBUTE_IS_PYTHON_FUNCTION
;
}
/* Potentially both __getattr__ and __getattribute__ are set.
Too complicated */
Py_DECREF
(
getattribute
);
*
descr
=
NULL
;
*
tp_version
=
ga_version
;
return
GETSET_OVERRIDDEN
;
}
/* Potentially has __getattr__ but no custom __getattribute__.
Fall through to usual descriptor analysis.
Usual attribute lookup should only be allowed at runtime
if we can guarantee that there is no way an exception can be
raised. This means some specializations, e.g. specializing
for property() isn't safe.
*/
Py_XDECREF
(
getattribute
);
}
else
{
*
descr
=
NULL
;
*
tp_version
=
FT_ATOMIC_LOAD_UINT_RELAXED
(
type
->
tp_version_tag
);
return
GETSET_OVERRIDDEN
;
}
unsigned
int
descr_version
;
PyObject
*
descriptor
=
_PyType_LookupRefAndVersion
(
type
,
name
,
&
descr_version
);
*
descr
=
descriptor
;
*
tp_version
=
have_ga_version
?
ga_version
:
descr_version
;
if
(
descriptor_is_class
(
descriptor
,
name
)) {
return
DUNDER_CLASS
;
}
return
classify_descriptor
(
descriptor
,
has_getattr
);
}
static
DescriptorClassification
analyze_descriptor_store
(
PyTypeObject
*
type
,
PyObject
*
name
,
PyObject
*
*
descr
,
unsigned
int
*
tp_version
)
{
if
(
type
->
tp_setattro
!=
PyObject_GenericSetAttr
) {
*
descr
=
NULL
;
return
GETSET_OVERRIDDEN
;
}
PyObject
*
descriptor
=
_PyType_LookupRefAndVersion
(
type
,
name
,
tp_version
);
*
descr
=
descriptor
;
if
(
descriptor_is_class
(
descriptor
,
name
)) {
return
DUNDER_CLASS
;
}
return
classify_descriptor
(
descriptor
, false);
}
static
int
specialize_dict_access_inline
(
PyObject
*
owner
,
_Py_CODEUNIT
*
instr
,
PyTypeObject
*
type
,
PyObject
*
name
,
unsigned
int
tp_version
,
int
base_op
,
int
values_op
)
{
_PyAttrCache
*
cache
=
(
_PyAttrCache
*
)(
instr
+
1
);
PyDictKeysObject
*
keys
=
((
PyHeapTypeObject
*
)
type
)
->
ht_cached_keys
;
assert
(
PyUnicode_CheckExact
(
name
));
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED
(
owner
);
Py_ssize_t
index
=
_PyDictKeys_StringLookupSplit
(
keys
,
name
);
assert
(
index
!=
DKIX_ERROR
);
if
(
index
==
DKIX_EMPTY
) {
SPECIALIZATION_FAIL
(
base_op
,
SPEC_FAIL_ATTR_NOT_IN_KEYS
);
return
0
;
}
assert
(
index
>=
0
);
assert
(
_PyObject_InlineValues
(
owner
)
->
valid
);
char
*
value_addr
=
(
char
*
)
&
_PyObject_InlineValues
(
owner
)
->
values
[
index
];
Py_ssize_t
offset
=
value_addr
-
(
char
*
)
owner
;
if
(
offset
!=
(
uint16_t
)
offset
) {
SPECIALIZATION_FAIL
(
base_op
,
SPEC_FAIL_OUT_OF_RANGE
);
return
0
;
}
cache
->
index
=
(
uint16_t
)
offset
;
write_u32
(
cache
->
version
,
tp_version
);
specialize
(
instr
,
values_op
);
return
1
;
}
static
int
specialize_dict_access_hint
(
PyDictObject
*
dict
,
_Py_CODEUNIT
*
instr
,
PyTypeObject
*
type
,
PyObject
*
name
,
unsigned
int
tp_version
,
int
base_op
,
int
hint_op
)
{
_PyAttrCache
*
cache
=
(
_PyAttrCache
*
)(
instr
+
1
);
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED
(
dict
);
#ifdef
Py_GIL_DISABLED
_PyDict_EnsureSharedOnRead
(
dict
);
#endif
// We found an instance with a __dict__.
if
(
_PyDict_HasSplitTable
(
dict
)) {
SPECIALIZATION_FAIL
(
base_op
,
SPEC_FAIL_ATTR_SPLIT_DICT
);
return
0
;
}
Py_ssize_t
index
=
_PyDict_LookupIndex
(
dict
,
name
);
if
(
index
!=
(
uint16_t
)
index
) {
SPECIALIZATION_FAIL
(
base_op
,
index
==
DKIX_EMPTY
?
SPEC_FAIL_ATTR_NOT_IN_DICT
:
SPEC_FAIL_OUT_OF_RANGE
);
return
0
;
}
cache
->
index
=
(
uint16_t
)
index
;
write_u32
(
cache
->
version
,
tp_version
);
specialize
(
instr
,
hint_op
);
return
1
;
}
static
int
specialize_dict_access
(
PyObject
*
owner
,
_Py_CODEUNIT
*
instr
,
PyTypeObject
*
type
,
DescriptorClassification
kind
,
PyObject
*
name
,
unsigned
int
tp_version
,
int
base_op
,
int
values_op
,
int
hint_op
)
{
assert
(
kind
==
NON_OVERRIDING
||
kind
==
NON_DESCRIPTOR
||
kind
==
ABSENT
||
kind
==
BUILTIN_CLASSMETHOD
||
kind
==
PYTHON_CLASSMETHOD
||
kind
==
METHOD
);
// No descriptor, or non overriding.
if
((
type
->
tp_flags
&
Py_TPFLAGS_MANAGED_DICT
)
==
0
) {
SPECIALIZATION_FAIL
(
base_op
,
SPEC_FAIL_ATTR_NOT_MANAGED_DICT
);
return
0
;
}
if
(
type
->
tp_flags
&
Py_TPFLAGS_INLINE_VALUES
&&
FT_ATOMIC_LOAD_UINT8
(
_PyObject_InlineValues
(
owner
)
->
valid
)
&&
!(
base_op
==
STORE_ATTR
&&
_PyObject_GetManagedDict
(
owner
)
!=
NULL
))
{
int
res
;
Py_BEGIN_CRITICAL_SECTION
(
owner
);
PyDictObject
*
dict
=
_PyObject_GetManagedDict
(
owner
);
if
(
dict
==
NULL
) {
// managed dict, not materialized, inline values valid
res
=
specialize_dict_access_inline
(
owner
,
instr
,
type
,
name
,
tp_version
,
base_op
,
values_op
);
}
else
{
// lost race and dict was created, fail specialization
SPECIALIZATION_FAIL
(
STORE_ATTR
,
SPEC_FAIL_OTHER
);
res
=
0
;
}
Py_END_CRITICAL_SECTION
();
return
res
;
}
else
{
PyDictObject
*
dict
=
_PyObject_GetManagedDict
(
owner
);
if
(
dict
==
NULL
||
!
PyDict_CheckExact
(
dict
)) {
SPECIALIZATION_FAIL
(
base_op
,
SPEC_FAIL_NO_DICT
);
return
0
;
}
int
res
;
Py_BEGIN_CRITICAL_SECTION
(
dict
);
// materialized managed dict
res
=
specialize_dict_access_hint
(
dict
,
instr
,
type
,
name
,
tp_version
,
base_op
,
hint_op
);
Py_END_CRITICAL_SECTION
();
return
res
;
}
}
static
int
specialize_attr_loadclassattr
(
PyObject
*
owner
,
_Py_CODEUNIT
*
instr
,
PyObject
*
name
,
PyObject
*
descr
,
unsigned
int
tp_version
,
DescriptorClassification
kind
,
bool
is_method
,
uint32_t
shared_keys_version
);
static
int
specialize_class_load_attr
(
PyObject
*
owner
,
_Py_CODEUNIT
*
instr
,
PyObject
*
name
);
/* Returns true if instances of obj's class are
* likely to have `name` in their __dict__.
* For objects with inline values, we check in the shared keys.
* For other objects, we check their actual dictionary.
*/
static
bool
instance_has_key
(
PyObject
*
obj
,
PyObject
*
name
,
uint32_t
*
shared_keys_version
)
{
PyTypeObject
*
cls
=
Py_TYPE
(
obj
);
if
((
cls
->
tp_flags
&
Py_TPFLAGS_MANAGED_DICT
)
==
0
) {
return
false;
}
if
(
cls
->
tp_flags
&
Py_TPFLAGS_INLINE_VALUES
) {
PyDictKeysObject
*
keys
=
((
PyHeapTypeObject
*
)
cls
)
->
ht_cached_keys
;
Py_ssize_t
index
=
_PyDictKeys_StringLookupAndVersion
(
keys
,
name
,
shared_keys_version
);
return
index
>=
0
;
}
PyDictObject
*
dict
=
_PyObject_GetManagedDict
(
obj
);
if
(
dict
==
NULL
||
!
PyDict_CheckExact
(
dict
)) {
return
false;
}
bool
result
;
Py_BEGIN_CRITICAL_SECTION
(
dict
);
if
(
dict
->
ma_values
) {
result
=
false;
}
else
{
result
=
(
_PyDict_LookupIndex
(
dict
,
name
) >=
0
);
}
Py_END_CRITICAL_SECTION
();
return
result
;
}
static
int
do_specialize_instance_load_attr
(
PyObject
*
owner
,
_Py_CODEUNIT
*
instr
,
PyObject
*
name
,
bool
shadow
,
uint32_t
shared_keys_version
,
DescriptorClassification
kind
,
PyObject
*
descr
,
unsigned
int
tp_version
)
{
_PyAttrCache
*
cache
=
(
_PyAttrCache
*
)(
instr
+
1
);
PyTypeObject
*
type
=
Py_TYPE
(
owner
);
if
(
tp_version
==
0
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_OUT_OF_VERSIONS
);
return
-1
;
}
uint8_t
oparg
=
FT_ATOMIC_LOAD_UINT8_RELAXED
(
instr
->
op
.
arg
);
switch
(
kind
) {
case
OVERRIDING
:
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR
);
return
-1
;
case
METHOD
:
{
if
(
shadow
) {
goto
try_instance
;
}
if
(
oparg
&
1
) {
if
(
specialize_attr_loadclassattr
(
owner
,
instr
,
name
,
descr
,
tp_version
,
kind
, true,
shared_keys_version
)) {
return
0
;
}
else
{
return
-1
;
}
}
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_METHOD
);
return
-1
;
}
case
PROPERTY
:
{
_PyLoadMethodCache
*
lm_cache
=
(
_PyLoadMethodCache
*
)(
instr
+
1
);
assert
(
Py_TYPE
(
descr
)
==
&
PyProperty_Type
);
PyObject
*
fget
=
((
_PyPropertyObject
*
)
descr
)
->
prop_get
;
if
(
fget
==
NULL
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_EXPECTED_ERROR
);
return
-1
;
}
if
(!
Py_IS_TYPE
(
fget
,
&
PyFunction_Type
)) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION
);
return
-1
;
}
if
(!
function_check_args
(
fget
,
1
,
LOAD_ATTR
)) {
return
-1
;
}
if
(
oparg
&
1
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_METHOD
);
return
-1
;
}
/* Don't specialize if PEP 523 is active */
if
(!
_PyInterpreterState_IsSpecializationEnabled
(
_PyInterpreterState_GET
())) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_OTHER
);
return
-1
;
}
#ifdef
Py_GIL_DISABLED
if
(!
_PyObject_HasDeferredRefcount
(
fget
)) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED
);
return
-1
;
}
#endif
uint32_t
func_version
=
function_get_version
(
fget
,
LOAD_ATTR
);
if
(
func_version
==
0
) {
return
-1
;
}
assert
(
tp_version
!=
0
);
write_u32
(
lm_cache
->
type_version
,
tp_version
);
write_u32
(
lm_cache
->
keys_version
,
func_version
);
/* borrowed */
write_ptr
(
lm_cache
->
descr
,
fget
);
specialize
(
instr
,
LOAD_ATTR_PROPERTY
);
return
0
;
}
case
OBJECT_SLOT
:
{
PyMemberDescrObject
*
member
=
(
PyMemberDescrObject
*
)
descr
;
struct
PyMemberDef
*
dmem
=
member
->
d_member
;
Py_ssize_t
offset
=
dmem
->
offset
;
if
(!
PyObject_TypeCheck
(
owner
,
member
->
d_common
.
d_type
)) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_EXPECTED_ERROR
);
return
-1
;
}
if
(
dmem
->
flags
&
_Py_AFTER_ITEMS
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_SLOT_AFTER_ITEMS
);
return
-1
;
}
if
(
dmem
->
flags
&
Py_AUDIT_READ
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_AUDITED_SLOT
);
return
-1
;
}
if
(
offset
!=
(
uint16_t
)
offset
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_OUT_OF_RANGE
);
return
-1
;
}
assert
(
dmem
->
type
==
Py_T_OBJECT_EX
||
dmem
->
type
==
_Py_T_OBJECT
);
assert
(
offset
>
0
);
cache
->
index
=
(
uint16_t
)
offset
;
write_u32
(
cache
->
version
,
tp_version
);
specialize
(
instr
,
LOAD_ATTR_SLOT
);
return
0
;
}
case
DUNDER_CLASS
:
{
Py_ssize_t
offset
=
offsetof(
PyObject
,
ob_type
);
assert
(
offset
==
(
uint16_t
)
offset
);
cache
->
index
=
(
uint16_t
)
offset
;
write_u32
(
cache
->
version
,
tp_version
);
specialize
(
instr
,
LOAD_ATTR_SLOT
);
return
0
;
}
case
OTHER_SLOT
:
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_NON_OBJECT_SLOT
);
return
-1
;
case
MUTABLE
:
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_MUTABLE_CLASS
);
return
-1
;
case
GETSET_OVERRIDDEN
:
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_OVERRIDDEN
);
return
-1
;
case
GETATTRIBUTE_IS_PYTHON_FUNCTION
:
{
assert
(
Py_IS_TYPE
(
descr
,
&
PyFunction_Type
));
_PyLoadMethodCache
*
lm_cache
=
(
_PyLoadMethodCache
*
)(
instr
+
1
);
if
(!
function_check_args
(
descr
,
2
,
LOAD_ATTR
)) {
return
-1
;
}
if
(
oparg
&
1
) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_METHOD
);
return
-1
;
}
uint32_t
version
=
function_get_version
(
descr
,
LOAD_ATTR
);
if
(
version
==
0
) {
return
-1
;
}
/* Don't specialize if PEP 523 is active */
if
(!
_PyInterpreterState_IsSpecializationEnabled
(
_PyInterpreterState_GET
())) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_OTHER
);
return
-1
;
}
#ifdef
Py_GIL_DISABLED
if
(!
_PyObject_HasDeferredRefcount
(
descr
)) {
SPECIALIZATION_FAIL
(
LOAD_ATTR
,
SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED
);
return
-1
;
}
#endif
write_u32
(
lm_cache
->
keys_version
,
version
);
/* borrowed */
write_ptr
(
lm_cache
->
descr
,
descr
);
write_u32
(
lm_cache
->
type_version
,
tp_version
);
specialize
(
instr
,
LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
);
return
0
;
}
case
BUILTIN_CLASSMETHOD
:
case
PYTHON_CLASSMETHOD
:
case
NON_OVERRIDING
:
if
(
shadow
) {
goto
try_instance
;
}
return
-1
;
case
NON_DESCRIPTOR
:
if
(
shadow
) {
goto
try_instance
;
}
if
((
oparg
&
1
)
==
0
) {
if
(
specialize_attr_loadclassattr
(
owner
,
instr
,
name
,
descr
,
tp_version
,
kind
, false,
shared_keys_version
)) {
return
0
;
}
}
return
-1
;
case
ABSENT
:
if
(
shadow
) {
goto
try_instance
;
}
set_counter
((
_Py_BackoffCounter
*
)
instr
+
1
,
adaptive_counter_cooldown
());
return
0
;
}
Py_UNREACHABLE
();
try_instance
:
if
(
specialize_dict_access
(
owner
,
instr
,
type
,
kind
,
name
,
tp_version
,
LOAD_ATTR
,
LOAD_ATTR_INSTANCE_VALUE
,
LOAD_ATTR_WITH_HINT
))
{
return
0
;
}
return
-1
;
}
static
int
specialize_instance_load_attr
(
PyObject
*
owner
,
_Py_CODEUNIT
*
instr
,
PyObject
*
name
)
{
// 0 is not a valid version
uint32_t
shared_keys_version
=
0
;
bool
shadow
=
instance_has_key
(
owner
,
name
,
&
shared_keys_version
);
PyObject
*
descr
=
NULL
;
unsigned
int
tp_version
=
0
;
PyTypeObject
*
type
=
Py_TYPE
(
owner
);
DescriptorClassification
kind
=
analyze_descriptor_load
(
type
,
name
,
&
descr
,
&
tp_version
);
int
result
=
do_specialize_instance_load_attr
(
owner
,
instr
,
name
,
shadow
,
shared_keys_version
,
kind
,
descr
,
tp_version
);
Py_XDECREF
(
descr
);
return
result
;
}
Py_NO_INLINE
void
_Py_Specialize_LoadAttr
(
_PyStackRef
owner_st
,
_Py_CODEUNIT
*
instr
,
PyObject
*
name
)
{
PyObject
*
owner
=
PyStackRef_AsPyObjectBorrow
(
owner_st
);
assert
(
ENABLE_SPECIALIZATION
);
assert
(
_PyOpcode_Caches
[
LOAD_ATTR
]
==
INLINE_CACHE_ENTRIES_LOAD_ATTR
);
View remainder of file in raw view
Back
|
FazBrowse Home
|
New Git URL