pythonGH-113710: Add a "globals to constants" pass (pythonGH-114592)

Converts specializations of `LOAD_GLOBAL` into constants during tier 2 optimization.
akulakov · Feb 2, 2024 · 0e71a29 · 0e71a29
1 parent 2091fb2
commit 0e71a29
Show file tree

Hide file tree

Showing 16 changed files with 375 additions and 55 deletions.
diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h
@@ -17,6 +17,9 @@ typedef struct {
     /* Dictionary version: globally unique, value change each time
        the dictionary is modified */
 #ifdef Py_BUILD_CORE
+    /* Bits 0-7 are for dict watchers.
+     * Bits 8-11 are for the watched mutation counter (used by tier2 optimization)
+     * The remaining bits (12-63) are the actual version tag. */
     uint64_t ma_version_tag;
 #else
     Py_DEPRECATED(3.12) uint64_t ma_version_tag;

diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h
@@ -47,7 +47,10 @@ typedef struct _PyExecutorObject {
 typedef struct _PyOptimizerObject _PyOptimizerObject;
 
 /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */
-typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **, int curr_stackentries);
+typedef int (*optimize_func)(
+    _PyOptimizerObject* self, struct _PyInterpreterFrame *frame,
+    _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr,
+    int curr_stackentries);
 
 typedef struct _PyOptimizerObject {
     PyObject_HEAD
@@ -94,6 +97,9 @@ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void);
 /* Minimum of 16 additional executions before retry */
 #define MINIMUM_TIER2_BACKOFF 4
 
+#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3
+#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h
@@ -207,8 +207,8 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
 
 #define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL)
 
-#define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS)
-#define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1)
+#define DICT_VERSION_INCREMENT (1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS))
+#define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1)
 
 #ifdef Py_GIL_DISABLED
 #define DICT_NEXT_VERSION(INTERP) \
@@ -234,7 +234,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
                     PyObject *value)
 {
     assert(Py_REFCNT((PyObject*)mp) > 0);
-    int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK;
+    int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK;
     if (watcher_bits) {
         _PyDict_SendEvent(watcher_bits, event, mp, key, value);
         return DICT_NEXT_VERSION(interp) | watcher_bits;

diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h
@@ -9,6 +9,7 @@ extern "C" {
 #endif
 
 #define DICT_MAX_WATCHERS 8
+#define DICT_WATCHED_MUTATION_BITS 4
 
 struct _Py_dict_state {
     /*Global counter used to set ma_version_tag field of dictionary.

diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
@@ -72,7 +72,6 @@ typedef struct _rare_events {
     uint8_t set_eval_frame_func;
     /* Modifying the builtins,  __builtins__.__dict__[var] = ... */
     uint8_t builtin_dict;
-    int builtins_dict_watcher_id;
     /* Modifying a function, e.g. func.__defaults__ = ..., etc. */
     uint8_t func_modification;
 } _rare_events;
@@ -243,6 +242,7 @@ struct _is {
     uint16_t optimizer_backedge_threshold;
     uint32_t next_func_version;
     _rare_events rare_events;
+    PyDict_WatchCallback builtins_dict_watcher;
 
     _Py_GlobalMonitors monitors;
     bool sys_profile_initialized;

diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
@@ -8,8 +8,9 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-int _Py_uop_analyze_and_optimize(PyCodeObject *code,
-    _PyUOpInstruction *trace, int trace_len, int curr_stackentries);
+int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame,
+    _PyUOpInstruction *trace, int trace_len, int curr_stackentries,
+    _PyBloomFilter *dependencies);
 
 extern PyTypeObject _PyCounterExecutor_Type;
 extern PyTypeObject _PyCounterOptimizer_Type;

diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h
diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h
@@ -204,6 +204,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_CHECK_VALIDITY] = HAS_DEOPT_FLAG,
     [_LOAD_CONST_INLINE] = 0,
     [_LOAD_CONST_INLINE_BORROW] = 0,
+    [_LOAD_CONST_INLINE_WITH_NULL] = 0,
+    [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = 0,
+    [_CHECK_GLOBALS] = HAS_DEOPT_FLAG,
+    [_CHECK_BUILTINS] = HAS_DEOPT_FLAG,
     [_INTERNAL_INCREMENT_OPT_COUNTER] = 0,
 };
 
@@ -250,10 +254,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT",
     [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE",
     [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT",
+    [_CHECK_BUILTINS] = "_CHECK_BUILTINS",
     [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS",
     [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH",
     [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH",
     [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS",
+    [_CHECK_GLOBALS] = "_CHECK_GLOBALS",
     [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES",
     [_CHECK_PEP_523] = "_CHECK_PEP_523",
     [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE",
@@ -332,6 +338,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_LOAD_CONST] = "_LOAD_CONST",
     [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE",
     [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW",
+    [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = "_LOAD_CONST_INLINE_BORROW_WITH_NULL",
+    [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL",
     [_LOAD_DEREF] = "_LOAD_DEREF",
     [_LOAD_FAST] = "_LOAD_FAST",
     [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR",

diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py
@@ -151,8 +151,8 @@ def test_watch_out_of_range_watcher_id(self):
 
     def test_watch_unassigned_watcher_id(self):
         d = {}
-        with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"):
-            self.watch(1, d)
+        with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"):
+            self.watch(3, d)
 
     def test_unwatch_non_dict(self):
         with self.watcher() as wid:
@@ -168,8 +168,8 @@ def test_unwatch_out_of_range_watcher_id(self):
 
     def test_unwatch_unassigned_watcher_id(self):
         d = {}
-        with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"):
-            self.unwatch(1, d)
+        with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"):
+            self.unwatch(3, d)
 
     def test_clear_out_of_range_watcher_id(self):
         with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID -1"):
@@ -178,8 +178,8 @@ def test_clear_out_of_range_watcher_id(self):
             self.clear_watcher(8)  # DICT_MAX_WATCHERS = 8
 
     def test_clear_unassigned_watcher_id(self):
-        with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"):
-            self.clear_watcher(1)
+        with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 3"):
+            self.clear_watcher(3)
 
 
 class TestTypeWatchers(unittest.TestCase):

diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c
@@ -15,8 +15,8 @@ module _testcapi
 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/
 
 // Test dict watching
-static PyObject *g_dict_watch_events;
-static int g_dict_watchers_installed;
+static PyObject *g_dict_watch_events = NULL;
+static int g_dict_watchers_installed = 0;
 
 static int
 dict_watch_callback(PyDict_WatchEvent event,

diff --git a/Objects/dictobject.c b/Objects/dictobject.c
@@ -5943,7 +5943,8 @@ PyDict_AddWatcher(PyDict_WatchCallback callback)
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
 
-    for (int i = 0; i < DICT_MAX_WATCHERS; i++) {
+    /* Start at 2, as 0 and 1 are reserved for CPython */
+    for (int i = 2; i < DICT_MAX_WATCHERS; i++) {
         if (!interp->dict_state.watchers[i]) {
             interp->dict_state.watchers[i] = callback;
             return i;

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
@@ -4071,11 +4071,35 @@ dummy_func(
         }
 
         op(_LOAD_CONST_INLINE, (ptr/4 -- value)) {
+            TIER_TWO_ONLY
             value = Py_NewRef(ptr);
         }
 
         op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) {
+            TIER_TWO_ONLY
+            value = ptr;
+        }
+
+        op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) {
+            TIER_TWO_ONLY
+            value = Py_NewRef(ptr);
+            null = NULL;
+        }
+
+        op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) {
+            TIER_TWO_ONLY
             value = ptr;
+            null = NULL;
+        }
+
+        op(_CHECK_GLOBALS, (dict/4 -- )) {
+            TIER_TWO_ONLY
+            DEOPT_IF(GLOBALS() != dict);
+        }
+
+        op(_CHECK_BUILTINS, (dict/4 -- )) {
+            TIER_TWO_ONLY
+            DEOPT_IF(BUILTINS() != dict);
         }
 
         /* Internal -- for testing executors */

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h