diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 188da775eb1cc7..6565910d848206 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -615,6 +615,25 @@ PyStackRef_FromPyObjectBorrow(PyObject *obj) return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; } +/* Create a stackref from a pre-tagged operand (tag bits already set). + Used by _LOAD_CONST_INLINE_BORROW variants where the operand is + tagged at trace creation time to avoid tagging on every execution. */ +static inline _PyStackRef +_PyStackRef_FromPreTagged(uintptr_t tagged) +{ + assert(tagged & Py_TAG_REFCNT); + return (_PyStackRef){ .bits = tagged }; +} +#define PyStackRef_FromPreTagged(ptr) _PyStackRef_FromPreTagged((uintptr_t)(ptr)) + +/* Tag a PyObject pointer as a borrowed operand for BORROW variants. */ +#define PyStackRef_TagBorrow(ptr) \ + ((uintptr_t)(ptr) | Py_TAG_REFCNT) + +/* Strip tag bits from a pre-tagged operand to recover the PyObject pointer. */ +#define PyStackRef_UntagBorrow(tagged) \ + ((PyObject *)((uintptr_t)(tagged) & ~Py_TAG_BITS)) + /* WARNING: This macro evaluates its argument more than once */ #ifdef _WIN32 #define PyStackRef_DUP(REF) \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index dd319778b1f2e8..5f5d9d23bb0d5c 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -945,449 +945,435 @@ extern "C" { #define _INSERT_1_LOAD_CONST_INLINE_r02 1153 #define _INSERT_1_LOAD_CONST_INLINE_r12 1154 #define _INSERT_1_LOAD_CONST_INLINE_r23 1155 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r02 1156 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 1157 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r23 1158 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r03 1159 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r13 1160 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 1161 -#define _INSERT_NULL_r10 1162 -#define _INSTRUMENTED_FOR_ITER_r23 1163 -#define _INSTRUMENTED_INSTRUCTION_r00 1164 -#define _INSTRUMENTED_JUMP_FORWARD_r00 1165 -#define _INSTRUMENTED_JUMP_FORWARD_r11 1166 -#define _INSTRUMENTED_JUMP_FORWARD_r22 1167 -#define _INSTRUMENTED_JUMP_FORWARD_r33 1168 -#define _INSTRUMENTED_LINE_r00 1169 -#define _INSTRUMENTED_NOT_TAKEN_r00 1170 -#define _INSTRUMENTED_NOT_TAKEN_r11 1171 -#define _INSTRUMENTED_NOT_TAKEN_r22 1172 -#define _INSTRUMENTED_NOT_TAKEN_r33 1173 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1174 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1175 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1176 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1177 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1178 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1179 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1180 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1181 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1182 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1183 -#define _IS_NONE_r11 1184 -#define _IS_OP_r03 1185 -#define _IS_OP_r13 1186 -#define _IS_OP_r23 1187 -#define _ITER_CHECK_LIST_r02 1188 -#define _ITER_CHECK_LIST_r12 1189 -#define _ITER_CHECK_LIST_r22 1190 -#define _ITER_CHECK_LIST_r33 1191 -#define _ITER_CHECK_RANGE_r02 1192 -#define _ITER_CHECK_RANGE_r12 1193 -#define _ITER_CHECK_RANGE_r22 1194 -#define _ITER_CHECK_RANGE_r33 1195 -#define _ITER_CHECK_TUPLE_r02 1196 -#define _ITER_CHECK_TUPLE_r12 1197 -#define _ITER_CHECK_TUPLE_r22 1198 -#define _ITER_CHECK_TUPLE_r33 1199 -#define _ITER_JUMP_LIST_r02 1200 -#define _ITER_JUMP_LIST_r12 1201 -#define _ITER_JUMP_LIST_r22 1202 -#define _ITER_JUMP_LIST_r33 1203 -#define _ITER_JUMP_RANGE_r02 1204 -#define _ITER_JUMP_RANGE_r12 1205 -#define _ITER_JUMP_RANGE_r22 1206 -#define _ITER_JUMP_RANGE_r33 1207 -#define _ITER_JUMP_TUPLE_r02 1208 -#define _ITER_JUMP_TUPLE_r12 1209 -#define _ITER_JUMP_TUPLE_r22 1210 -#define _ITER_JUMP_TUPLE_r33 1211 -#define _ITER_NEXT_LIST_r23 1212 -#define _ITER_NEXT_LIST_TIER_TWO_r23 1213 -#define _ITER_NEXT_RANGE_r03 1214 -#define _ITER_NEXT_RANGE_r13 1215 -#define _ITER_NEXT_RANGE_r23 1216 -#define _ITER_NEXT_TUPLE_r03 1217 -#define _ITER_NEXT_TUPLE_r13 1218 -#define _ITER_NEXT_TUPLE_r23 1219 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1220 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1221 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1222 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1223 -#define _JUMP_TO_TOP_r00 1224 -#define _LIST_APPEND_r10 1225 -#define _LIST_EXTEND_r11 1226 -#define _LOAD_ATTR_r10 1227 -#define _LOAD_ATTR_CLASS_r11 1228 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 1229 -#define _LOAD_ATTR_INSTANCE_VALUE_r02 1230 -#define _LOAD_ATTR_INSTANCE_VALUE_r12 1231 -#define _LOAD_ATTR_INSTANCE_VALUE_r23 1232 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1233 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1234 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1235 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 1236 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 1237 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 1238 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1239 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1240 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1241 -#define _LOAD_ATTR_MODULE_r12 1242 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1243 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1244 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1245 -#define _LOAD_ATTR_SLOT_r02 1246 -#define _LOAD_ATTR_SLOT_r12 1247 -#define _LOAD_ATTR_SLOT_r23 1248 -#define _LOAD_ATTR_WITH_HINT_r12 1249 -#define _LOAD_BUILD_CLASS_r01 1250 -#define _LOAD_BYTECODE_r00 1251 -#define _LOAD_COMMON_CONSTANT_r01 1252 -#define _LOAD_COMMON_CONSTANT_r12 1253 -#define _LOAD_COMMON_CONSTANT_r23 1254 -#define _LOAD_CONST_r01 1255 -#define _LOAD_CONST_r12 1256 -#define _LOAD_CONST_r23 1257 -#define _LOAD_CONST_INLINE_r01 1258 -#define _LOAD_CONST_INLINE_r12 1259 -#define _LOAD_CONST_INLINE_r23 1260 -#define _LOAD_CONST_INLINE_BORROW_r01 1261 -#define _LOAD_CONST_INLINE_BORROW_r12 1262 -#define _LOAD_CONST_INLINE_BORROW_r23 1263 -#define _LOAD_CONST_UNDER_INLINE_r02 1264 -#define _LOAD_CONST_UNDER_INLINE_r12 1265 -#define _LOAD_CONST_UNDER_INLINE_r23 1266 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1267 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1268 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1269 -#define _LOAD_DEREF_r01 1270 -#define _LOAD_FAST_r01 1271 -#define _LOAD_FAST_r12 1272 -#define _LOAD_FAST_r23 1273 -#define _LOAD_FAST_0_r01 1274 -#define _LOAD_FAST_0_r12 1275 -#define _LOAD_FAST_0_r23 1276 -#define _LOAD_FAST_1_r01 1277 -#define _LOAD_FAST_1_r12 1278 -#define _LOAD_FAST_1_r23 1279 -#define _LOAD_FAST_2_r01 1280 -#define _LOAD_FAST_2_r12 1281 -#define _LOAD_FAST_2_r23 1282 -#define _LOAD_FAST_3_r01 1283 -#define _LOAD_FAST_3_r12 1284 -#define _LOAD_FAST_3_r23 1285 -#define _LOAD_FAST_4_r01 1286 -#define _LOAD_FAST_4_r12 1287 -#define _LOAD_FAST_4_r23 1288 -#define _LOAD_FAST_5_r01 1289 -#define _LOAD_FAST_5_r12 1290 -#define _LOAD_FAST_5_r23 1291 -#define _LOAD_FAST_6_r01 1292 -#define _LOAD_FAST_6_r12 1293 -#define _LOAD_FAST_6_r23 1294 -#define _LOAD_FAST_7_r01 1295 -#define _LOAD_FAST_7_r12 1296 -#define _LOAD_FAST_7_r23 1297 -#define _LOAD_FAST_AND_CLEAR_r01 1298 -#define _LOAD_FAST_AND_CLEAR_r12 1299 -#define _LOAD_FAST_AND_CLEAR_r23 1300 -#define _LOAD_FAST_BORROW_r01 1301 -#define _LOAD_FAST_BORROW_r12 1302 -#define _LOAD_FAST_BORROW_r23 1303 -#define _LOAD_FAST_BORROW_0_r01 1304 -#define _LOAD_FAST_BORROW_0_r12 1305 -#define _LOAD_FAST_BORROW_0_r23 1306 -#define _LOAD_FAST_BORROW_1_r01 1307 -#define _LOAD_FAST_BORROW_1_r12 1308 -#define _LOAD_FAST_BORROW_1_r23 1309 -#define _LOAD_FAST_BORROW_2_r01 1310 -#define _LOAD_FAST_BORROW_2_r12 1311 -#define _LOAD_FAST_BORROW_2_r23 1312 -#define _LOAD_FAST_BORROW_3_r01 1313 -#define _LOAD_FAST_BORROW_3_r12 1314 -#define _LOAD_FAST_BORROW_3_r23 1315 -#define _LOAD_FAST_BORROW_4_r01 1316 -#define _LOAD_FAST_BORROW_4_r12 1317 -#define _LOAD_FAST_BORROW_4_r23 1318 -#define _LOAD_FAST_BORROW_5_r01 1319 -#define _LOAD_FAST_BORROW_5_r12 1320 -#define _LOAD_FAST_BORROW_5_r23 1321 -#define _LOAD_FAST_BORROW_6_r01 1322 -#define _LOAD_FAST_BORROW_6_r12 1323 -#define _LOAD_FAST_BORROW_6_r23 1324 -#define _LOAD_FAST_BORROW_7_r01 1325 -#define _LOAD_FAST_BORROW_7_r12 1326 -#define _LOAD_FAST_BORROW_7_r23 1327 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1328 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1329 -#define _LOAD_FAST_CHECK_r01 1330 -#define _LOAD_FAST_CHECK_r12 1331 -#define _LOAD_FAST_CHECK_r23 1332 -#define _LOAD_FAST_LOAD_FAST_r02 1333 -#define _LOAD_FAST_LOAD_FAST_r13 1334 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1335 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1336 -#define _LOAD_GLOBAL_r00 1337 -#define _LOAD_GLOBAL_BUILTINS_r01 1338 -#define _LOAD_GLOBAL_MODULE_r01 1339 -#define _LOAD_LOCALS_r01 1340 -#define _LOAD_LOCALS_r12 1341 -#define _LOAD_LOCALS_r23 1342 -#define _LOAD_NAME_r01 1343 -#define _LOAD_SMALL_INT_r01 1344 -#define _LOAD_SMALL_INT_r12 1345 -#define _LOAD_SMALL_INT_r23 1346 -#define _LOAD_SMALL_INT_0_r01 1347 -#define _LOAD_SMALL_INT_0_r12 1348 -#define _LOAD_SMALL_INT_0_r23 1349 -#define _LOAD_SMALL_INT_1_r01 1350 -#define _LOAD_SMALL_INT_1_r12 1351 -#define _LOAD_SMALL_INT_1_r23 1352 -#define _LOAD_SMALL_INT_2_r01 1353 -#define _LOAD_SMALL_INT_2_r12 1354 -#define _LOAD_SMALL_INT_2_r23 1355 -#define _LOAD_SMALL_INT_3_r01 1356 -#define _LOAD_SMALL_INT_3_r12 1357 -#define _LOAD_SMALL_INT_3_r23 1358 -#define _LOAD_SPECIAL_r00 1359 -#define _LOAD_SUPER_ATTR_ATTR_r31 1360 -#define _LOAD_SUPER_ATTR_METHOD_r32 1361 -#define _LOCK_OBJECT_r01 1362 -#define _LOCK_OBJECT_r11 1363 -#define _LOCK_OBJECT_r22 1364 -#define _LOCK_OBJECT_r33 1365 -#define _MAKE_CALLARGS_A_TUPLE_r33 1366 -#define _MAKE_CELL_r00 1367 -#define _MAKE_FUNCTION_r11 1368 -#define _MAKE_HEAP_SAFE_r01 1369 -#define _MAKE_HEAP_SAFE_r11 1370 -#define _MAKE_HEAP_SAFE_r22 1371 -#define _MAKE_HEAP_SAFE_r33 1372 -#define _MAKE_WARM_r00 1373 -#define _MAKE_WARM_r11 1374 -#define _MAKE_WARM_r22 1375 -#define _MAKE_WARM_r33 1376 -#define _MAP_ADD_r20 1377 -#define _MATCH_CLASS_r33 1378 -#define _MATCH_KEYS_r23 1379 -#define _MATCH_MAPPING_r02 1380 -#define _MATCH_MAPPING_r12 1381 -#define _MATCH_MAPPING_r23 1382 -#define _MATCH_SEQUENCE_r02 1383 -#define _MATCH_SEQUENCE_r12 1384 -#define _MATCH_SEQUENCE_r23 1385 -#define _MAYBE_EXPAND_METHOD_r00 1386 -#define _MAYBE_EXPAND_METHOD_KW_r11 1387 -#define _MONITOR_CALL_r00 1388 -#define _MONITOR_CALL_KW_r11 1389 -#define _MONITOR_JUMP_BACKWARD_r00 1390 -#define _MONITOR_JUMP_BACKWARD_r11 1391 -#define _MONITOR_JUMP_BACKWARD_r22 1392 -#define _MONITOR_JUMP_BACKWARD_r33 1393 -#define _MONITOR_RESUME_r00 1394 -#define _NOP_r00 1395 -#define _NOP_r11 1396 -#define _NOP_r22 1397 -#define _NOP_r33 1398 -#define _POP_CALL_r20 1399 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1400 -#define _POP_CALL_ONE_r30 1401 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1402 -#define _POP_CALL_TWO_r30 1403 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1404 -#define _POP_EXCEPT_r10 1405 -#define _POP_ITER_r20 1406 -#define _POP_JUMP_IF_FALSE_r00 1407 -#define _POP_JUMP_IF_FALSE_r10 1408 -#define _POP_JUMP_IF_FALSE_r21 1409 -#define _POP_JUMP_IF_FALSE_r32 1410 -#define _POP_JUMP_IF_TRUE_r00 1411 -#define _POP_JUMP_IF_TRUE_r10 1412 -#define _POP_JUMP_IF_TRUE_r21 1413 -#define _POP_JUMP_IF_TRUE_r32 1414 -#define _POP_TOP_r10 1415 -#define _POP_TOP_FLOAT_r00 1416 -#define _POP_TOP_FLOAT_r10 1417 -#define _POP_TOP_FLOAT_r21 1418 -#define _POP_TOP_FLOAT_r32 1419 -#define _POP_TOP_INT_r00 1420 -#define _POP_TOP_INT_r10 1421 -#define _POP_TOP_INT_r21 1422 -#define _POP_TOP_INT_r32 1423 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1424 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1425 -#define _POP_TOP_NOP_r00 1426 -#define _POP_TOP_NOP_r10 1427 -#define _POP_TOP_NOP_r21 1428 -#define _POP_TOP_NOP_r32 1429 -#define _POP_TOP_UNICODE_r00 1430 -#define _POP_TOP_UNICODE_r10 1431 -#define _POP_TOP_UNICODE_r21 1432 -#define _POP_TOP_UNICODE_r32 1433 -#define _POP_TWO_r20 1434 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1435 -#define _PUSH_EXC_INFO_r02 1436 -#define _PUSH_EXC_INFO_r12 1437 -#define _PUSH_EXC_INFO_r23 1438 -#define _PUSH_FRAME_r10 1439 -#define _PUSH_NULL_r01 1440 -#define _PUSH_NULL_r12 1441 -#define _PUSH_NULL_r23 1442 -#define _PUSH_NULL_CONDITIONAL_r00 1443 -#define _PY_FRAME_EX_r31 1444 -#define _PY_FRAME_GENERAL_r01 1445 -#define _PY_FRAME_KW_r11 1446 -#define _REPLACE_WITH_TRUE_r02 1447 -#define _REPLACE_WITH_TRUE_r12 1448 -#define _REPLACE_WITH_TRUE_r23 1449 -#define _RESUME_CHECK_r00 1450 -#define _RESUME_CHECK_r11 1451 -#define _RESUME_CHECK_r22 1452 -#define _RESUME_CHECK_r33 1453 -#define _RETURN_GENERATOR_r01 1454 -#define _RETURN_VALUE_r11 1455 -#define _SAVE_RETURN_OFFSET_r00 1456 -#define _SAVE_RETURN_OFFSET_r11 1457 -#define _SAVE_RETURN_OFFSET_r22 1458 -#define _SAVE_RETURN_OFFSET_r33 1459 -#define _SEND_r33 1460 -#define _SEND_GEN_FRAME_r33 1461 -#define _SETUP_ANNOTATIONS_r00 1462 -#define _SET_ADD_r10 1463 -#define _SET_FUNCTION_ATTRIBUTE_r01 1464 -#define _SET_FUNCTION_ATTRIBUTE_r11 1465 -#define _SET_FUNCTION_ATTRIBUTE_r21 1466 -#define _SET_FUNCTION_ATTRIBUTE_r32 1467 -#define _SET_IP_r00 1468 -#define _SET_IP_r11 1469 -#define _SET_IP_r22 1470 -#define _SET_IP_r33 1471 -#define _SET_UPDATE_r11 1472 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1473 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1474 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1475 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1476 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1477 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1478 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1479 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1480 -#define _SPILL_OR_RELOAD_r01 1481 -#define _SPILL_OR_RELOAD_r02 1482 -#define _SPILL_OR_RELOAD_r03 1483 -#define _SPILL_OR_RELOAD_r10 1484 -#define _SPILL_OR_RELOAD_r12 1485 -#define _SPILL_OR_RELOAD_r13 1486 -#define _SPILL_OR_RELOAD_r20 1487 -#define _SPILL_OR_RELOAD_r21 1488 -#define _SPILL_OR_RELOAD_r23 1489 -#define _SPILL_OR_RELOAD_r30 1490 -#define _SPILL_OR_RELOAD_r31 1491 -#define _SPILL_OR_RELOAD_r32 1492 -#define _START_EXECUTOR_r00 1493 -#define _STORE_ATTR_r20 1494 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1495 -#define _STORE_ATTR_SLOT_r21 1496 -#define _STORE_ATTR_WITH_HINT_r21 1497 -#define _STORE_DEREF_r10 1498 -#define _STORE_FAST_LOAD_FAST_r11 1499 -#define _STORE_FAST_STORE_FAST_r20 1500 -#define _STORE_GLOBAL_r10 1501 -#define _STORE_NAME_r10 1502 -#define _STORE_SLICE_r30 1503 -#define _STORE_SUBSCR_r30 1504 -#define _STORE_SUBSCR_DICT_r31 1505 -#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1506 -#define _STORE_SUBSCR_LIST_INT_r32 1507 -#define _SWAP_r11 1508 -#define _SWAP_2_r02 1509 -#define _SWAP_2_r12 1510 -#define _SWAP_2_r22 1511 -#define _SWAP_2_r33 1512 -#define _SWAP_3_r03 1513 -#define _SWAP_3_r13 1514 -#define _SWAP_3_r23 1515 -#define _SWAP_3_r33 1516 -#define _SWAP_FAST_r01 1517 -#define _SWAP_FAST_r11 1518 -#define _SWAP_FAST_r22 1519 -#define _SWAP_FAST_r33 1520 -#define _SWAP_FAST_0_r01 1521 -#define _SWAP_FAST_0_r11 1522 -#define _SWAP_FAST_0_r22 1523 -#define _SWAP_FAST_0_r33 1524 -#define _SWAP_FAST_1_r01 1525 -#define _SWAP_FAST_1_r11 1526 -#define _SWAP_FAST_1_r22 1527 -#define _SWAP_FAST_1_r33 1528 -#define _SWAP_FAST_2_r01 1529 -#define _SWAP_FAST_2_r11 1530 -#define _SWAP_FAST_2_r22 1531 -#define _SWAP_FAST_2_r33 1532 -#define _SWAP_FAST_3_r01 1533 -#define _SWAP_FAST_3_r11 1534 -#define _SWAP_FAST_3_r22 1535 -#define _SWAP_FAST_3_r33 1536 -#define _SWAP_FAST_4_r01 1537 -#define _SWAP_FAST_4_r11 1538 -#define _SWAP_FAST_4_r22 1539 -#define _SWAP_FAST_4_r33 1540 -#define _SWAP_FAST_5_r01 1541 -#define _SWAP_FAST_5_r11 1542 -#define _SWAP_FAST_5_r22 1543 -#define _SWAP_FAST_5_r33 1544 -#define _SWAP_FAST_6_r01 1545 -#define _SWAP_FAST_6_r11 1546 -#define _SWAP_FAST_6_r22 1547 -#define _SWAP_FAST_6_r33 1548 -#define _SWAP_FAST_7_r01 1549 -#define _SWAP_FAST_7_r11 1550 -#define _SWAP_FAST_7_r22 1551 -#define _SWAP_FAST_7_r33 1552 -#define _TIER2_RESUME_CHECK_r00 1553 -#define _TIER2_RESUME_CHECK_r11 1554 -#define _TIER2_RESUME_CHECK_r22 1555 -#define _TIER2_RESUME_CHECK_r33 1556 -#define _TO_BOOL_r11 1557 -#define _TO_BOOL_BOOL_r01 1558 -#define _TO_BOOL_BOOL_r11 1559 -#define _TO_BOOL_BOOL_r22 1560 -#define _TO_BOOL_BOOL_r33 1561 -#define _TO_BOOL_INT_r02 1562 -#define _TO_BOOL_INT_r12 1563 -#define _TO_BOOL_INT_r23 1564 -#define _TO_BOOL_LIST_r02 1565 -#define _TO_BOOL_LIST_r12 1566 -#define _TO_BOOL_LIST_r23 1567 -#define _TO_BOOL_NONE_r01 1568 -#define _TO_BOOL_NONE_r11 1569 -#define _TO_BOOL_NONE_r22 1570 -#define _TO_BOOL_NONE_r33 1571 -#define _TO_BOOL_STR_r02 1572 -#define _TO_BOOL_STR_r12 1573 -#define _TO_BOOL_STR_r23 1574 -#define _TRACE_RECORD_r00 1575 -#define _UNARY_INVERT_r12 1576 -#define _UNARY_NEGATIVE_r12 1577 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1578 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1579 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1580 -#define _UNARY_NOT_r01 1581 -#define _UNARY_NOT_r11 1582 -#define _UNARY_NOT_r22 1583 -#define _UNARY_NOT_r33 1584 -#define _UNPACK_EX_r10 1585 -#define _UNPACK_SEQUENCE_r10 1586 -#define _UNPACK_SEQUENCE_LIST_r10 1587 -#define _UNPACK_SEQUENCE_TUPLE_r10 1588 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1589 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1590 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1591 -#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1592 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1593 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1594 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1595 -#define _WITH_EXCEPT_START_r33 1596 -#define _YIELD_VALUE_r11 1597 -#define MAX_UOP_REGS_ID 1597 +#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 1156 +#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 1157 +#define _INSERT_NULL_r10 1158 +#define _INSTRUMENTED_FOR_ITER_r23 1159 +#define _INSTRUMENTED_INSTRUCTION_r00 1160 +#define _INSTRUMENTED_JUMP_FORWARD_r00 1161 +#define _INSTRUMENTED_JUMP_FORWARD_r11 1162 +#define _INSTRUMENTED_JUMP_FORWARD_r22 1163 +#define _INSTRUMENTED_JUMP_FORWARD_r33 1164 +#define _INSTRUMENTED_LINE_r00 1165 +#define _INSTRUMENTED_NOT_TAKEN_r00 1166 +#define _INSTRUMENTED_NOT_TAKEN_r11 1167 +#define _INSTRUMENTED_NOT_TAKEN_r22 1168 +#define _INSTRUMENTED_NOT_TAKEN_r33 1169 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1170 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1171 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1172 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1173 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1174 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1175 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1176 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1177 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1178 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1179 +#define _IS_NONE_r11 1180 +#define _IS_OP_r03 1181 +#define _IS_OP_r13 1182 +#define _IS_OP_r23 1183 +#define _ITER_CHECK_LIST_r02 1184 +#define _ITER_CHECK_LIST_r12 1185 +#define _ITER_CHECK_LIST_r22 1186 +#define _ITER_CHECK_LIST_r33 1187 +#define _ITER_CHECK_RANGE_r02 1188 +#define _ITER_CHECK_RANGE_r12 1189 +#define _ITER_CHECK_RANGE_r22 1190 +#define _ITER_CHECK_RANGE_r33 1191 +#define _ITER_CHECK_TUPLE_r02 1192 +#define _ITER_CHECK_TUPLE_r12 1193 +#define _ITER_CHECK_TUPLE_r22 1194 +#define _ITER_CHECK_TUPLE_r33 1195 +#define _ITER_JUMP_LIST_r02 1196 +#define _ITER_JUMP_LIST_r12 1197 +#define _ITER_JUMP_LIST_r22 1198 +#define _ITER_JUMP_LIST_r33 1199 +#define _ITER_JUMP_RANGE_r02 1200 +#define _ITER_JUMP_RANGE_r12 1201 +#define _ITER_JUMP_RANGE_r22 1202 +#define _ITER_JUMP_RANGE_r33 1203 +#define _ITER_JUMP_TUPLE_r02 1204 +#define _ITER_JUMP_TUPLE_r12 1205 +#define _ITER_JUMP_TUPLE_r22 1206 +#define _ITER_JUMP_TUPLE_r33 1207 +#define _ITER_NEXT_LIST_r23 1208 +#define _ITER_NEXT_LIST_TIER_TWO_r23 1209 +#define _ITER_NEXT_RANGE_r03 1210 +#define _ITER_NEXT_RANGE_r13 1211 +#define _ITER_NEXT_RANGE_r23 1212 +#define _ITER_NEXT_TUPLE_r03 1213 +#define _ITER_NEXT_TUPLE_r13 1214 +#define _ITER_NEXT_TUPLE_r23 1215 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1216 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1217 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1218 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1219 +#define _JUMP_TO_TOP_r00 1220 +#define _LIST_APPEND_r10 1221 +#define _LIST_EXTEND_r11 1222 +#define _LOAD_ATTR_r10 1223 +#define _LOAD_ATTR_CLASS_r11 1224 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 1225 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 1226 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 1227 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 1228 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1229 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1230 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1231 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1232 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1233 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1234 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1235 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1236 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1237 +#define _LOAD_ATTR_MODULE_r12 1238 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1239 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1240 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1241 +#define _LOAD_ATTR_SLOT_r02 1242 +#define _LOAD_ATTR_SLOT_r12 1243 +#define _LOAD_ATTR_SLOT_r23 1244 +#define _LOAD_ATTR_WITH_HINT_r12 1245 +#define _LOAD_BUILD_CLASS_r01 1246 +#define _LOAD_BYTECODE_r00 1247 +#define _LOAD_COMMON_CONSTANT_r01 1248 +#define _LOAD_COMMON_CONSTANT_r12 1249 +#define _LOAD_COMMON_CONSTANT_r23 1250 +#define _LOAD_CONST_r01 1251 +#define _LOAD_CONST_r12 1252 +#define _LOAD_CONST_r23 1253 +#define _LOAD_CONST_INLINE_r01 1254 +#define _LOAD_CONST_INLINE_r12 1255 +#define _LOAD_CONST_INLINE_r23 1256 +#define _LOAD_CONST_INLINE_BORROW_r01 1257 +#define _LOAD_CONST_UNDER_INLINE_r02 1258 +#define _LOAD_CONST_UNDER_INLINE_r12 1259 +#define _LOAD_CONST_UNDER_INLINE_r23 1260 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1261 +#define _LOAD_DEREF_r01 1262 +#define _LOAD_FAST_r01 1263 +#define _LOAD_FAST_r12 1264 +#define _LOAD_FAST_r23 1265 +#define _LOAD_FAST_0_r01 1266 +#define _LOAD_FAST_0_r12 1267 +#define _LOAD_FAST_0_r23 1268 +#define _LOAD_FAST_1_r01 1269 +#define _LOAD_FAST_1_r12 1270 +#define _LOAD_FAST_1_r23 1271 +#define _LOAD_FAST_2_r01 1272 +#define _LOAD_FAST_2_r12 1273 +#define _LOAD_FAST_2_r23 1274 +#define _LOAD_FAST_3_r01 1275 +#define _LOAD_FAST_3_r12 1276 +#define _LOAD_FAST_3_r23 1277 +#define _LOAD_FAST_4_r01 1278 +#define _LOAD_FAST_4_r12 1279 +#define _LOAD_FAST_4_r23 1280 +#define _LOAD_FAST_5_r01 1281 +#define _LOAD_FAST_5_r12 1282 +#define _LOAD_FAST_5_r23 1283 +#define _LOAD_FAST_6_r01 1284 +#define _LOAD_FAST_6_r12 1285 +#define _LOAD_FAST_6_r23 1286 +#define _LOAD_FAST_7_r01 1287 +#define _LOAD_FAST_7_r12 1288 +#define _LOAD_FAST_7_r23 1289 +#define _LOAD_FAST_AND_CLEAR_r01 1290 +#define _LOAD_FAST_AND_CLEAR_r12 1291 +#define _LOAD_FAST_AND_CLEAR_r23 1292 +#define _LOAD_FAST_BORROW_r01 1293 +#define _LOAD_FAST_BORROW_r12 1294 +#define _LOAD_FAST_BORROW_r23 1295 +#define _LOAD_FAST_BORROW_0_r01 1296 +#define _LOAD_FAST_BORROW_0_r12 1297 +#define _LOAD_FAST_BORROW_0_r23 1298 +#define _LOAD_FAST_BORROW_1_r01 1299 +#define _LOAD_FAST_BORROW_1_r12 1300 +#define _LOAD_FAST_BORROW_1_r23 1301 +#define _LOAD_FAST_BORROW_2_r01 1302 +#define _LOAD_FAST_BORROW_2_r12 1303 +#define _LOAD_FAST_BORROW_2_r23 1304 +#define _LOAD_FAST_BORROW_3_r01 1305 +#define _LOAD_FAST_BORROW_3_r12 1306 +#define _LOAD_FAST_BORROW_3_r23 1307 +#define _LOAD_FAST_BORROW_4_r01 1308 +#define _LOAD_FAST_BORROW_4_r12 1309 +#define _LOAD_FAST_BORROW_4_r23 1310 +#define _LOAD_FAST_BORROW_5_r01 1311 +#define _LOAD_FAST_BORROW_5_r12 1312 +#define _LOAD_FAST_BORROW_5_r23 1313 +#define _LOAD_FAST_BORROW_6_r01 1314 +#define _LOAD_FAST_BORROW_6_r12 1315 +#define _LOAD_FAST_BORROW_6_r23 1316 +#define _LOAD_FAST_BORROW_7_r01 1317 +#define _LOAD_FAST_BORROW_7_r12 1318 +#define _LOAD_FAST_BORROW_7_r23 1319 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1320 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1321 +#define _LOAD_FAST_CHECK_r01 1322 +#define _LOAD_FAST_CHECK_r12 1323 +#define _LOAD_FAST_CHECK_r23 1324 +#define _LOAD_FAST_LOAD_FAST_r02 1325 +#define _LOAD_FAST_LOAD_FAST_r13 1326 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1327 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1328 +#define _LOAD_GLOBAL_r00 1329 +#define _LOAD_GLOBAL_BUILTINS_r01 1330 +#define _LOAD_GLOBAL_MODULE_r01 1331 +#define _LOAD_LOCALS_r01 1332 +#define _LOAD_LOCALS_r12 1333 +#define _LOAD_LOCALS_r23 1334 +#define _LOAD_NAME_r01 1335 +#define _LOAD_SMALL_INT_r01 1336 +#define _LOAD_SMALL_INT_r12 1337 +#define _LOAD_SMALL_INT_r23 1338 +#define _LOAD_SMALL_INT_0_r01 1339 +#define _LOAD_SMALL_INT_0_r12 1340 +#define _LOAD_SMALL_INT_0_r23 1341 +#define _LOAD_SMALL_INT_1_r01 1342 +#define _LOAD_SMALL_INT_1_r12 1343 +#define _LOAD_SMALL_INT_1_r23 1344 +#define _LOAD_SMALL_INT_2_r01 1345 +#define _LOAD_SMALL_INT_2_r12 1346 +#define _LOAD_SMALL_INT_2_r23 1347 +#define _LOAD_SMALL_INT_3_r01 1348 +#define _LOAD_SMALL_INT_3_r12 1349 +#define _LOAD_SMALL_INT_3_r23 1350 +#define _LOAD_SPECIAL_r00 1351 +#define _LOAD_SUPER_ATTR_ATTR_r31 1352 +#define _LOAD_SUPER_ATTR_METHOD_r32 1353 +#define _LOCK_OBJECT_r01 1354 +#define _LOCK_OBJECT_r11 1355 +#define _LOCK_OBJECT_r22 1356 +#define _LOCK_OBJECT_r33 1357 +#define _MAKE_CALLARGS_A_TUPLE_r33 1358 +#define _MAKE_CELL_r00 1359 +#define _MAKE_FUNCTION_r11 1360 +#define _MAKE_HEAP_SAFE_r01 1361 +#define _MAKE_HEAP_SAFE_r11 1362 +#define _MAKE_HEAP_SAFE_r22 1363 +#define _MAKE_HEAP_SAFE_r33 1364 +#define _MAKE_WARM_r00 1365 +#define _MAKE_WARM_r11 1366 +#define _MAKE_WARM_r22 1367 +#define _MAKE_WARM_r33 1368 +#define _MAP_ADD_r20 1369 +#define _MATCH_CLASS_r33 1370 +#define _MATCH_KEYS_r23 1371 +#define _MATCH_MAPPING_r02 1372 +#define _MATCH_MAPPING_r12 1373 +#define _MATCH_MAPPING_r23 1374 +#define _MATCH_SEQUENCE_r02 1375 +#define _MATCH_SEQUENCE_r12 1376 +#define _MATCH_SEQUENCE_r23 1377 +#define _MAYBE_EXPAND_METHOD_r00 1378 +#define _MAYBE_EXPAND_METHOD_KW_r11 1379 +#define _MONITOR_CALL_r00 1380 +#define _MONITOR_CALL_KW_r11 1381 +#define _MONITOR_JUMP_BACKWARD_r00 1382 +#define _MONITOR_JUMP_BACKWARD_r11 1383 +#define _MONITOR_JUMP_BACKWARD_r22 1384 +#define _MONITOR_JUMP_BACKWARD_r33 1385 +#define _MONITOR_RESUME_r00 1386 +#define _NOP_r00 1387 +#define _NOP_r11 1388 +#define _NOP_r22 1389 +#define _NOP_r33 1390 +#define _POP_CALL_r20 1391 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1392 +#define _POP_CALL_ONE_r30 1393 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1394 +#define _POP_CALL_TWO_r30 1395 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1396 +#define _POP_EXCEPT_r10 1397 +#define _POP_ITER_r20 1398 +#define _POP_JUMP_IF_FALSE_r00 1399 +#define _POP_JUMP_IF_FALSE_r10 1400 +#define _POP_JUMP_IF_FALSE_r21 1401 +#define _POP_JUMP_IF_FALSE_r32 1402 +#define _POP_JUMP_IF_TRUE_r00 1403 +#define _POP_JUMP_IF_TRUE_r10 1404 +#define _POP_JUMP_IF_TRUE_r21 1405 +#define _POP_JUMP_IF_TRUE_r32 1406 +#define _POP_TOP_r10 1407 +#define _POP_TOP_FLOAT_r00 1408 +#define _POP_TOP_FLOAT_r10 1409 +#define _POP_TOP_FLOAT_r21 1410 +#define _POP_TOP_FLOAT_r32 1411 +#define _POP_TOP_INT_r00 1412 +#define _POP_TOP_INT_r10 1413 +#define _POP_TOP_INT_r21 1414 +#define _POP_TOP_INT_r32 1415 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1416 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1417 +#define _POP_TOP_NOP_r00 1418 +#define _POP_TOP_NOP_r10 1419 +#define _POP_TOP_NOP_r21 1420 +#define _POP_TOP_NOP_r32 1421 +#define _POP_TOP_UNICODE_r00 1422 +#define _POP_TOP_UNICODE_r10 1423 +#define _POP_TOP_UNICODE_r21 1424 +#define _POP_TOP_UNICODE_r32 1425 +#define _POP_TWO_r20 1426 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1427 +#define _PUSH_EXC_INFO_r02 1428 +#define _PUSH_EXC_INFO_r12 1429 +#define _PUSH_EXC_INFO_r23 1430 +#define _PUSH_FRAME_r10 1431 +#define _PUSH_NULL_r01 1432 +#define _PUSH_NULL_r12 1433 +#define _PUSH_NULL_r23 1434 +#define _PUSH_NULL_CONDITIONAL_r00 1435 +#define _PY_FRAME_EX_r31 1436 +#define _PY_FRAME_GENERAL_r01 1437 +#define _PY_FRAME_KW_r11 1438 +#define _REPLACE_WITH_TRUE_r02 1439 +#define _REPLACE_WITH_TRUE_r12 1440 +#define _REPLACE_WITH_TRUE_r23 1441 +#define _RESUME_CHECK_r00 1442 +#define _RESUME_CHECK_r11 1443 +#define _RESUME_CHECK_r22 1444 +#define _RESUME_CHECK_r33 1445 +#define _RETURN_GENERATOR_r01 1446 +#define _RETURN_VALUE_r11 1447 +#define _SAVE_RETURN_OFFSET_r00 1448 +#define _SAVE_RETURN_OFFSET_r11 1449 +#define _SAVE_RETURN_OFFSET_r22 1450 +#define _SAVE_RETURN_OFFSET_r33 1451 +#define _SEND_r33 1452 +#define _SEND_GEN_FRAME_r33 1453 +#define _SETUP_ANNOTATIONS_r00 1454 +#define _SET_ADD_r10 1455 +#define _SET_FUNCTION_ATTRIBUTE_r01 1456 +#define _SET_FUNCTION_ATTRIBUTE_r11 1457 +#define _SET_FUNCTION_ATTRIBUTE_r21 1458 +#define _SET_FUNCTION_ATTRIBUTE_r32 1459 +#define _SET_IP_r00 1460 +#define _SET_IP_r11 1461 +#define _SET_IP_r22 1462 +#define _SET_IP_r33 1463 +#define _SET_UPDATE_r11 1464 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1465 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1466 +#define _SPILL_OR_RELOAD_r01 1467 +#define _SPILL_OR_RELOAD_r02 1468 +#define _SPILL_OR_RELOAD_r03 1469 +#define _SPILL_OR_RELOAD_r10 1470 +#define _SPILL_OR_RELOAD_r12 1471 +#define _SPILL_OR_RELOAD_r13 1472 +#define _SPILL_OR_RELOAD_r20 1473 +#define _SPILL_OR_RELOAD_r21 1474 +#define _SPILL_OR_RELOAD_r23 1475 +#define _SPILL_OR_RELOAD_r30 1476 +#define _SPILL_OR_RELOAD_r31 1477 +#define _SPILL_OR_RELOAD_r32 1478 +#define _START_EXECUTOR_r00 1479 +#define _STORE_ATTR_r20 1480 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1481 +#define _STORE_ATTR_SLOT_r21 1482 +#define _STORE_ATTR_WITH_HINT_r21 1483 +#define _STORE_DEREF_r10 1484 +#define _STORE_FAST_LOAD_FAST_r11 1485 +#define _STORE_FAST_STORE_FAST_r20 1486 +#define _STORE_GLOBAL_r10 1487 +#define _STORE_NAME_r10 1488 +#define _STORE_SLICE_r30 1489 +#define _STORE_SUBSCR_r30 1490 +#define _STORE_SUBSCR_DICT_r31 1491 +#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1492 +#define _STORE_SUBSCR_LIST_INT_r32 1493 +#define _SWAP_r11 1494 +#define _SWAP_2_r02 1495 +#define _SWAP_2_r12 1496 +#define _SWAP_2_r22 1497 +#define _SWAP_2_r33 1498 +#define _SWAP_3_r03 1499 +#define _SWAP_3_r13 1500 +#define _SWAP_3_r23 1501 +#define _SWAP_3_r33 1502 +#define _SWAP_FAST_r01 1503 +#define _SWAP_FAST_r11 1504 +#define _SWAP_FAST_r22 1505 +#define _SWAP_FAST_r33 1506 +#define _SWAP_FAST_0_r01 1507 +#define _SWAP_FAST_0_r11 1508 +#define _SWAP_FAST_0_r22 1509 +#define _SWAP_FAST_0_r33 1510 +#define _SWAP_FAST_1_r01 1511 +#define _SWAP_FAST_1_r11 1512 +#define _SWAP_FAST_1_r22 1513 +#define _SWAP_FAST_1_r33 1514 +#define _SWAP_FAST_2_r01 1515 +#define _SWAP_FAST_2_r11 1516 +#define _SWAP_FAST_2_r22 1517 +#define _SWAP_FAST_2_r33 1518 +#define _SWAP_FAST_3_r01 1519 +#define _SWAP_FAST_3_r11 1520 +#define _SWAP_FAST_3_r22 1521 +#define _SWAP_FAST_3_r33 1522 +#define _SWAP_FAST_4_r01 1523 +#define _SWAP_FAST_4_r11 1524 +#define _SWAP_FAST_4_r22 1525 +#define _SWAP_FAST_4_r33 1526 +#define _SWAP_FAST_5_r01 1527 +#define _SWAP_FAST_5_r11 1528 +#define _SWAP_FAST_5_r22 1529 +#define _SWAP_FAST_5_r33 1530 +#define _SWAP_FAST_6_r01 1531 +#define _SWAP_FAST_6_r11 1532 +#define _SWAP_FAST_6_r22 1533 +#define _SWAP_FAST_6_r33 1534 +#define _SWAP_FAST_7_r01 1535 +#define _SWAP_FAST_7_r11 1536 +#define _SWAP_FAST_7_r22 1537 +#define _SWAP_FAST_7_r33 1538 +#define _TIER2_RESUME_CHECK_r00 1539 +#define _TIER2_RESUME_CHECK_r11 1540 +#define _TIER2_RESUME_CHECK_r22 1541 +#define _TIER2_RESUME_CHECK_r33 1542 +#define _TO_BOOL_r11 1543 +#define _TO_BOOL_BOOL_r01 1544 +#define _TO_BOOL_BOOL_r11 1545 +#define _TO_BOOL_BOOL_r22 1546 +#define _TO_BOOL_BOOL_r33 1547 +#define _TO_BOOL_INT_r02 1548 +#define _TO_BOOL_INT_r12 1549 +#define _TO_BOOL_INT_r23 1550 +#define _TO_BOOL_LIST_r02 1551 +#define _TO_BOOL_LIST_r12 1552 +#define _TO_BOOL_LIST_r23 1553 +#define _TO_BOOL_NONE_r01 1554 +#define _TO_BOOL_NONE_r11 1555 +#define _TO_BOOL_NONE_r22 1556 +#define _TO_BOOL_NONE_r33 1557 +#define _TO_BOOL_STR_r02 1558 +#define _TO_BOOL_STR_r12 1559 +#define _TO_BOOL_STR_r23 1560 +#define _TRACE_RECORD_r00 1561 +#define _UNARY_INVERT_r12 1562 +#define _UNARY_NEGATIVE_r12 1563 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1564 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1565 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1566 +#define _UNARY_NOT_r01 1567 +#define _UNARY_NOT_r11 1568 +#define _UNARY_NOT_r22 1569 +#define _UNARY_NOT_r33 1570 +#define _UNPACK_EX_r10 1571 +#define _UNPACK_SEQUENCE_r10 1572 +#define _UNPACK_SEQUENCE_LIST_r10 1573 +#define _UNPACK_SEQUENCE_TUPLE_r10 1574 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1575 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1576 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1577 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1578 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1579 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1580 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1581 +#define _WITH_EXCEPT_START_r33 1582 +#define _YIELD_VALUE_r11 1583 +#define MAX_UOP_REGS_ID 1583 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 016bed17bf15aa..4bbcf3cf67ea1e 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -380,7 +380,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_POP_CALL] = HAS_ESCAPES_FLAG, [_POP_CALL_ONE] = HAS_ESCAPES_FLAG, [_POP_CALL_TWO] = HAS_ESCAPES_FLAG, @@ -389,13 +389,13 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_POP_CALL_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_INSERT_1_LOAD_CONST_INLINE] = 0, - [_INSERT_1_LOAD_CONST_INLINE_BORROW] = 0, - [_INSERT_2_LOAD_CONST_INLINE_BORROW] = 0, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = 0, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = 0, + [_INSERT_1_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_INSERT_2_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_LOAD_CONST_UNDER_INLINE] = 0, - [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, + [_LOAD_CONST_UNDER_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, @@ -3544,11 +3544,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, + .best = { 0, 0, 0, 0 }, .entries = { { 1, 0, _LOAD_CONST_INLINE_BORROW_r01 }, - { 2, 1, _LOAD_CONST_INLINE_BORROW_r12 }, - { 3, 2, _LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { -1, -1, -1 }, }, }, @@ -3625,38 +3625,38 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_INSERT_1_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, + .best = { 1, 1, 1, 1 }, .entries = { - { 2, 0, _INSERT_1_LOAD_CONST_INLINE_BORROW_r02 }, + { -1, -1, -1 }, { 2, 1, _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 }, - { 3, 2, _INSERT_1_LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, { -1, -1, -1 }, }, }, [_INSERT_2_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, + .best = { 2, 2, 2, 2 }, .entries = { - { 3, 0, _INSERT_2_LOAD_CONST_INLINE_BORROW_r03 }, - { 3, 1, _INSERT_2_LOAD_CONST_INLINE_BORROW_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 3, 2, _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 }, { -1, -1, -1 }, }, }, [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 3 }, + .best = { 3, 3, 3, 3 }, .entries = { - { 2, 0, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 }, - { 2, 1, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 }, - { 2, 2, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 2, 3, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 }, }, }, [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 3 }, + .best = { 3, 3, 3, 3 }, .entries = { - { 3, 0, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 }, - { 3, 1, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 }, - { 3, 2, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 3, 3, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 }, }, }, @@ -3679,11 +3679,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_LOAD_CONST_UNDER_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, + .best = { 1, 1, 1, 1 }, .entries = { - { 2, 0, _LOAD_CONST_UNDER_INLINE_BORROW_r02 }, + { -1, -1, -1 }, { 2, 1, _LOAD_CONST_UNDER_INLINE_BORROW_r12 }, - { 3, 2, _LOAD_CONST_UNDER_INLINE_BORROW_r23 }, + { -1, -1, -1 }, { -1, -1, -1 }, }, }, @@ -4632,8 +4632,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_CONST_INLINE_r23] = _LOAD_CONST_INLINE, [_POP_TOP_LOAD_CONST_INLINE_r11] = _POP_TOP_LOAD_CONST_INLINE, [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, - [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, - [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, [_POP_CALL_r20] = _POP_CALL, [_POP_CALL_ONE_r30] = _POP_CALL_ONE, [_POP_CALL_TWO_r30] = _POP_CALL_TWO, @@ -4644,27 +4642,15 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_INSERT_1_LOAD_CONST_INLINE_r02] = _INSERT_1_LOAD_CONST_INLINE, [_INSERT_1_LOAD_CONST_INLINE_r12] = _INSERT_1_LOAD_CONST_INLINE, [_INSERT_1_LOAD_CONST_INLINE_r23] = _INSERT_1_LOAD_CONST_INLINE, - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r02] = _INSERT_1_LOAD_CONST_INLINE_BORROW, [_INSERT_1_LOAD_CONST_INLINE_BORROW_r12] = _INSERT_1_LOAD_CONST_INLINE_BORROW, - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r23] = _INSERT_1_LOAD_CONST_INLINE_BORROW, - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r03] = _INSERT_2_LOAD_CONST_INLINE_BORROW, - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r13] = _INSERT_2_LOAD_CONST_INLINE_BORROW, [_INSERT_2_LOAD_CONST_INLINE_BORROW_r23] = _INSERT_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31] = _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, [_LOAD_CONST_UNDER_INLINE_r02] = _LOAD_CONST_UNDER_INLINE, [_LOAD_CONST_UNDER_INLINE_r12] = _LOAD_CONST_UNDER_INLINE, [_LOAD_CONST_UNDER_INLINE_r23] = _LOAD_CONST_UNDER_INLINE, - [_LOAD_CONST_UNDER_INLINE_BORROW_r02] = _LOAD_CONST_UNDER_INLINE_BORROW, [_LOAD_CONST_UNDER_INLINE_BORROW_r12] = _LOAD_CONST_UNDER_INLINE_BORROW, - [_LOAD_CONST_UNDER_INLINE_BORROW_r23] = _LOAD_CONST_UNDER_INLINE_BORROW, [_START_EXECUTOR_r00] = _START_EXECUTOR, [_MAKE_WARM_r00] = _MAKE_WARM, [_MAKE_WARM_r11] = _MAKE_WARM, @@ -5477,12 +5463,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_INSERT_1_LOAD_CONST_INLINE_r12] = "_INSERT_1_LOAD_CONST_INLINE_r12", [_INSERT_1_LOAD_CONST_INLINE_r23] = "_INSERT_1_LOAD_CONST_INLINE_r23", [_INSERT_1_LOAD_CONST_INLINE_BORROW] = "_INSERT_1_LOAD_CONST_INLINE_BORROW", - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r02] = "_INSERT_1_LOAD_CONST_INLINE_BORROW_r02", [_INSERT_1_LOAD_CONST_INLINE_BORROW_r12] = "_INSERT_1_LOAD_CONST_INLINE_BORROW_r12", - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r23] = "_INSERT_1_LOAD_CONST_INLINE_BORROW_r23", [_INSERT_2_LOAD_CONST_INLINE_BORROW] = "_INSERT_2_LOAD_CONST_INLINE_BORROW", - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r03] = "_INSERT_2_LOAD_CONST_INLINE_BORROW_r03", - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r13] = "_INSERT_2_LOAD_CONST_INLINE_BORROW_r13", [_INSERT_2_LOAD_CONST_INLINE_BORROW_r23] = "_INSERT_2_LOAD_CONST_INLINE_BORROW_r23", [_INSERT_NULL] = "_INSERT_NULL", [_INSERT_NULL_r10] = "_INSERT_NULL_r10", @@ -5573,16 +5555,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_CONST_INLINE_r23] = "_LOAD_CONST_INLINE_r23", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", [_LOAD_CONST_INLINE_BORROW_r01] = "_LOAD_CONST_INLINE_BORROW_r01", - [_LOAD_CONST_INLINE_BORROW_r12] = "_LOAD_CONST_INLINE_BORROW_r12", - [_LOAD_CONST_INLINE_BORROW_r23] = "_LOAD_CONST_INLINE_BORROW_r23", [_LOAD_CONST_UNDER_INLINE] = "_LOAD_CONST_UNDER_INLINE", [_LOAD_CONST_UNDER_INLINE_r02] = "_LOAD_CONST_UNDER_INLINE_r02", [_LOAD_CONST_UNDER_INLINE_r12] = "_LOAD_CONST_UNDER_INLINE_r12", [_LOAD_CONST_UNDER_INLINE_r23] = "_LOAD_CONST_UNDER_INLINE_r23", [_LOAD_CONST_UNDER_INLINE_BORROW] = "_LOAD_CONST_UNDER_INLINE_BORROW", - [_LOAD_CONST_UNDER_INLINE_BORROW_r02] = "_LOAD_CONST_UNDER_INLINE_BORROW_r02", [_LOAD_CONST_UNDER_INLINE_BORROW_r12] = "_LOAD_CONST_UNDER_INLINE_BORROW_r12", - [_LOAD_CONST_UNDER_INLINE_BORROW_r23] = "_LOAD_CONST_UNDER_INLINE_BORROW_r23", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_DEREF_r01] = "_LOAD_DEREF_r01", [_LOAD_FAST] = "_LOAD_FAST", @@ -5859,14 +5837,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SET_UPDATE] = "_SET_UPDATE", [_SET_UPDATE_r11] = "_SET_UPDATE_r11", [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22", [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32", [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23", [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33", [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-19-35-32.gh-issue-145742.SU9RYL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-19-35-32.gh-issue-145742.SU9RYL.rst new file mode 100644 index 00000000000000..0909ffe44bd331 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-19-35-32.gh-issue-145742.SU9RYL.rst @@ -0,0 +1,2 @@ +Optimize _LOAD_CONST_INLINE_BORROW by pre-tagging operands at trace +creation. Patch by Donghee Na. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index edba5a89cc0f29..9a5e8b61a88587 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5874,7 +5874,7 @@ dummy_func( } tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_POP_CALL, (callable, null --)) { @@ -5900,20 +5900,20 @@ dummy_func( tier2 op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { PyStackRef_CLOSE(pop); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_POP_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, pop1, pop2 -- value)) { PyStackRef_CLOSE(pop2); PyStackRef_CLOSE(pop1); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null -- value)) { (void)null; // Silence compiler warnings about unused variables DEAD(null); PyStackRef_CLOSE(callable); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, pop -- value)) { @@ -5921,7 +5921,7 @@ dummy_func( (void)null; // Silence compiler warnings about unused variables DEAD(null); PyStackRef_CLOSE(callable); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_INSERT_1_LOAD_CONST_INLINE, (ptr/4, left -- res, l)) { @@ -5931,26 +5931,26 @@ dummy_func( } tier2 op(_INSERT_1_LOAD_CONST_INLINE_BORROW, (ptr/4, left -- res, l)) { - res = PyStackRef_FromPyObjectBorrow(ptr); + res = PyStackRef_FromPreTagged(ptr); l = left; INPUTS_DEAD(); } tier2 op(_INSERT_2_LOAD_CONST_INLINE_BORROW, (ptr/4, left, right -- res, l, r)) { - res = PyStackRef_FromPyObjectBorrow(ptr); + res = PyStackRef_FromPreTagged(ptr); l = left; r = right; INPUTS_DEAD(); } tier2 op(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, arg -- res, a)) { - res = PyStackRef_FromPyObjectBorrow(ptr); + res = PyStackRef_FromPreTagged(ptr); a = arg; INPUTS_DEAD(); } tier2 op(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, arg -- res, a, c)) { - res = PyStackRef_FromPyObjectBorrow(ptr); + res = PyStackRef_FromPreTagged(ptr); a = arg; c = callable; INPUTS_DEAD(); @@ -5962,7 +5962,7 @@ dummy_func( (void)null; // Silence compiler warnings about unused variables DEAD(null); PyStackRef_CLOSE(callable); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_LOAD_CONST_UNDER_INLINE, (ptr/4, old -- value, new)) { @@ -5974,7 +5974,7 @@ dummy_func( tier2 op(_LOAD_CONST_UNDER_INLINE_BORROW, (ptr/4, old -- value, new)) { new = old; DEAD(old); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_START_EXECUTOR, (executor/4 --)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6600accb37e3f2..fa100874e036ad 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -20991,43 +20991,17 @@ assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _LOAD_CONST_INLINE_BORROW_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _POP_CALL_r20: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21132,8 +21106,8 @@ PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -21163,8 +21137,8 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop1); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -21187,8 +21161,8 @@ (void)null; _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -21223,8 +21197,8 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -21290,25 +21264,6 @@ break; } - case _INSERT_1_LOAD_CONST_INLINE_BORROW_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - left = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - l = left; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _INSERT_1_LOAD_CONST_INLINE_BORROW_r12: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21318,78 +21273,17 @@ _PyStackRef _stack_item_0 = _tos_cache0; left = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - l = left; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INSERT_1_LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - left = _stack_item_1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - l = left; - _tos_cache2 = l; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INSERT_2_LOAD_CONST_INLINE_BORROW_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; + stack_pointer[0] = left; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INSERT_2_LOAD_CONST_INLINE_BORROW_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); + _PyFrame_SetStackPointer(frame, stack_pointer); + res = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); l = left; - r = right; - _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21409,77 +21303,25 @@ right = _stack_item_1; left = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); l = left; r = right; _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - arg = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - _PyStackRef _stack_item_0 = _tos_cache0; - arg = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - arg = _stack_item_1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32: { CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21491,90 +21333,25 @@ _PyStackRef _stack_item_2 = _tos_cache2; arg = _stack_item_2; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = arg; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); a = arg; _tos_cache1 = a; _tos_cache0 = res; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - arg = stack_pointer[-1]; - callable = stack_pointer[-3]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - arg = _stack_item_0; - callable = stack_pointer[-2]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - arg = _stack_item_1; - callable = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33: { CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21589,13 +21366,22 @@ arg = _stack_item_2; callable = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = callable; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = arg; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); a = arg; c = callable; _tos_cache2 = c; _tos_cache1 = a; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -21633,8 +21419,8 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -21700,25 +21486,6 @@ break; } - case _LOAD_CONST_UNDER_INLINE_BORROW_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef old; - _PyStackRef value; - _PyStackRef new; - old = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - new = old; - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache1 = new; - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _LOAD_CONST_UNDER_INLINE_BORROW_r12: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21729,34 +21496,17 @@ old = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; - value = PyStackRef_FromPyObjectBorrow(ptr); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); _tos_cache1 = new; _tos_cache0 = value; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _LOAD_CONST_UNDER_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef old; - _PyStackRef value; - _PyStackRef new; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - old = _stack_item_1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - new = old; - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache2 = new; - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _START_EXECUTOR_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 92e1c081d524db..fbe96bf2c77f3f 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -175,14 +175,15 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop, bool i if (res == NULL) { return NULL; } + bool borrow = _Py_IsImmortal(res); if (insert) { - if (_Py_IsImmortal(res)) { + if (borrow) { inst->opcode = _INSERT_1_LOAD_CONST_INLINE_BORROW; } else { inst->opcode = _INSERT_1_LOAD_CONST_INLINE; } } else { - if (_Py_IsImmortal(res)) { + if (borrow) { inst->opcode = pop ? _POP_TOP_LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE_BORROW; } else { inst->opcode = pop ? _POP_TOP_LOAD_CONST_INLINE : _LOAD_CONST_INLINE; @@ -192,7 +193,7 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop, bool i assert(inst[1].oparg & 1); } } - inst->operand0 = (uint64_t)res; + inst->operand0 = borrow ? PyStackRef_TagBorrow(res) : (uint64_t)res; return res; } @@ -343,7 +344,7 @@ optimize_to_bool( int opcode = insert_mode ? _INSERT_1_LOAD_CONST_INLINE_BORROW : _POP_TOP_LOAD_CONST_INLINE_BORROW; - ADD_OP(opcode, 0, (uintptr_t)load); + ADD_OP(opcode, 0, PyStackRef_TagBorrow(load)); *result_ptr = sym_new_const(ctx, load); return 1; } @@ -397,11 +398,13 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction PyObject *lookup = _PyType_Lookup(type, name); if (lookup) { int opcode = mortal; + uintptr_t operand = (uintptr_t)lookup; // if the object is immortal or the type is immutable, borrowing is safe if (_Py_IsImmortal(lookup) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) { opcode = immortal; + operand = PyStackRef_TagBorrow(lookup); } - ADD_OP(opcode, 0, (uintptr_t)lookup); + ADD_OP(opcode, 0, operand); PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); return sym_new_const(ctx, lookup); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index f2645553513f3d..2ec139c471f197 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -722,14 +722,14 @@ dummy_func(void) { op(_LOAD_CONST, (-- value)) { PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_COMMON_CONSTANT, (-- value)) { assert(oparg < NUM_COMMON_CONSTANTS); PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -737,7 +737,7 @@ dummy_func(void) { PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -746,7 +746,7 @@ dummy_func(void) { } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); } op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { @@ -754,19 +754,19 @@ dummy_func(void) { } op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); } op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); } op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); } op(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, arg -- res, a)) { - res = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + res = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); a = arg; } @@ -1221,7 +1221,7 @@ dummy_func(void) { if (type) { res = sym_new_const(ctx, type); ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, - (uintptr_t)type); + PyStackRef_TagBorrow(type)); } else { res = sym_new_not_null(ctx); @@ -1258,7 +1258,7 @@ dummy_func(void) { out = Py_True; } sym_set_const(res, out); - ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(out)); } } @@ -1602,7 +1602,7 @@ dummy_func(void) { } op(_REPLACE_WITH_TRUE, (value -- res, v)) { - ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(Py_True)); res = sym_new_const(ctx, Py_True); v = value; } @@ -1887,7 +1887,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(temp)); } res = sym_new_const(ctx, temp); Py_DECREF(temp); @@ -1908,7 +1908,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(temp)); } len = sym_new_const(ctx, temp); Py_DECREF(temp); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index fb3ec39a42eabc..05fc77cdf60140 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -78,7 +78,7 @@ JitOptRef value; PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -92,7 +92,7 @@ PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -247,7 +247,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -315,7 +315,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } stack_pointer[-1] = res; @@ -486,7 +486,7 @@ JitOptRef res; JitOptRef v; value = stack_pointer[-1]; - ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(Py_True)); res = sym_new_const(ctx, Py_True); v = value; CHECK_STACK_BOUNDS(1); @@ -525,7 +525,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -638,7 +638,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -708,7 +708,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -778,7 +778,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -1499,7 +1499,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -1802,7 +1802,7 @@ JitOptRef value; assert(oparg < NUM_COMMON_CONSTANTS); PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -2634,7 +2634,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(-1); @@ -2705,7 +2705,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -2778,7 +2778,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -2840,7 +2840,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -2920,7 +2920,7 @@ PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -3027,7 +3027,7 @@ PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -3133,7 +3133,7 @@ goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(temp)); } len = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(1); @@ -3736,7 +3736,7 @@ if (type) { res = sym_new_const(ctx, type); ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, - (uintptr_t)type); + PyStackRef_TagBorrow(type)); } else { res = sym_new_not_null(ctx); @@ -4023,7 +4023,7 @@ goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(temp)); } res = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(-2); @@ -4069,7 +4069,7 @@ out = Py_True; } sym_set_const(res, out); - ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(out)); } CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = res; @@ -4616,7 +4616,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -4858,7 +4858,7 @@ case _LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; @@ -4890,7 +4890,7 @@ case _POP_TOP_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); stack_pointer[-1] = value; break; } @@ -4908,7 +4908,7 @@ case _POP_CALL_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = value; stack_pointer += -1; @@ -4974,7 +4974,7 @@ JitOptRef a; arg = stack_pointer[-1]; PyObject *ptr = (PyObject *)this_instr->operand0; - res = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + res = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); a = arg; CHECK_STACK_BOUNDS(-1); stack_pointer[-3] = res; @@ -5000,7 +5000,7 @@ case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = value; stack_pointer += -3; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 6ba9c43ef1f0c3..030bf7c0bf16b8 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -766,7 +766,7 @@ def escaping_call_in_simple_stmt(stmt: SimpleStmt, result: dict[SimpleStmt, Esca continue #if not tkn.text.startswith(("Py", "_Py", "monitor")): # continue - if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef", "PyStackRef_Tag", "PyStackRef_Untag")): # Optimize functions continue if tkn.text.endswith("Check"): diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 65896221ba7a05..cfd62510afc8e1 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -270,7 +270,7 @@ def replace_opcode_if_evaluates_pure( emitter.emit(f"PyObject *result = sym_get_const(ctx, {output_identifier.text});\n") emitter.emit(f"if (_Py_IsImmortal(result)) {{\n") emitter.emit(f"// Replace with {replacement_uop} since we have {input_desc} and an immortal result\n") - emitter.emit(f"ADD_OP({replacement_uop}, 0, (uintptr_t)result);\n") + emitter.emit(f"ADD_OP({replacement_uop}, 0, PyStackRef_TagBorrow(result));\n") emitter.emit("}\n") emitter.emit("}\n")