Merge specialization code more (stats are still broken)

Lib/opcode.py

@@ -284,12 +284,12 @@ def jabs_op(name, op):
     "LOAD_ATTR": [
         "LOAD_ATTR_ADAPTIVE",
         # LOAD_ATTR
+        "LOAD_ATTR_CLASS",
         "LOAD_ATTR_INSTANCE_VALUE",
         "LOAD_ATTR_MODULE",
         "LOAD_ATTR_SLOT",
         "LOAD_ATTR_WITH_HINT",
         # LOAD_METHOD
-        "LOAD_ATTR_METHOD_CLASS",
         "LOAD_ATTR_METHOD_LAZY_DICT",
         "LOAD_ATTR_METHOD_NO_DICT",
         "LOAD_ATTR_METHOD_WITH_DICT",

Python/ceval.c

@@ -3626,6 +3626,30 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             NOTRACE_DISPATCH();
         }
 
+        TARGET(LOAD_ATTR_CLASS) {
+            /* LOAD_METHOD, for class methods */
+            assert(cframe.use_tracing == 0);
+            _PyLoadMethodCache* cache = (_PyLoadMethodCache*)next_instr;
+
+            PyObject* cls = TOP();
+            DEOPT_IF(!PyType_Check(cls), LOAD_ATTR);
+            uint32_t type_version = read_u32(cache->type_version);
+            DEOPT_IF(((PyTypeObject*)cls)->tp_version_tag != type_version,
+                LOAD_ATTR);
+            assert(type_version != 0);
+
+            STAT_INC(LOAD_ATTR, hit);
+            PyObject* res = read_obj(cache->descr);
+            assert(res != NULL);
+            Py_INCREF(res);
+            SET_TOP(NULL);
+            STACK_GROW((oparg & 1));
+            SET_TOP(res);
+            Py_DECREF(cls);
+            JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
+            NOTRACE_DISPATCH();
+        }
+
         TARGET(STORE_ATTR_ADAPTIVE) {
             assert(cframe.use_tracing == 0);
             _PyAttrCache *cache = (_PyAttrCache *)next_instr;
@@ -4621,29 +4645,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             NOTRACE_DISPATCH();
         }
 
-        TARGET(LOAD_ATTR_METHOD_CLASS) {
-            /* LOAD_METHOD, for class methods */
-            assert(cframe.use_tracing == 0);
-            _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr;
-
-            PyObject *cls = TOP();
-            DEOPT_IF(!PyType_Check(cls), LOAD_ATTR);
-            uint32_t type_version = read_u32(cache->type_version);
-            DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version,
-                     LOAD_ATTR);
-            assert(type_version != 0);
-
-            STAT_INC(LOAD_ATTR, hit);
-            PyObject *res = read_obj(cache->descr);
-            assert(res != NULL);
-            Py_INCREF(res);
-            SET_TOP(NULL);
-            Py_DECREF(cls);
-            PUSH(res);
-            JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
-            NOTRACE_DISPATCH();
-        }
-
         TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
             DEOPT_IF(is_method(stack_pointer, oparg), CALL);
             PyObject *function = PEEK(oparg + 1);

Python/specialize.c

@@ -653,8 +653,9 @@ specialize_dict_access(
     return 1;
 }
 
-static int
-specialize_attr_loadmethod(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);
+static int specialize_attr_loadmethod(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
+    PyObject* descr, DescriptorClassification kind);
+static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);
 
 int
 _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
@@ -669,25 +670,29 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
         }
         goto success;
     }
-    int oparg = _Py_OPARG(*instr);
-    if (oparg & 1) {
-        int res = specialize_attr_loadmethod(owner, instr, name);
-        if (res < 0) {
-            return -1;
-        }
-        else if (res) {
-            goto success;
+    if (PyType_Check(owner)) {
+        int err = specialize_class_load_attr(owner, instr, name);
+        if (err) {
+            goto fail;
         }
-        /* res == 0 (fail), fall through and let LOAD_ATTR specialize for it */
+        goto success;
     }
-    PyTypeObject *type = Py_TYPE(owner);
+    int oparg = _Py_OPARG(*instr);
+    PyTypeObject* type = Py_TYPE(owner);
     if (type->tp_dict == NULL) {
         if (PyType_Ready(type) < 0) {
             return -1;
         }
     }
-    PyObject *descr;
+    PyObject* descr = NULL;
     DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);
+    assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
+    if ((oparg & 1) && kind == METHOD) {
+        if (specialize_attr_loadmethod(owner, instr, name, descr, kind)) {
+            goto success;
+        }
+        /* res == 0 (fail), fall through and let LOAD_ATTR specialize for it */
+    }
     switch(kind) {
         case OVERRIDING:
             SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
@@ -885,7 +890,7 @@ load_method_fail_kind(DescriptorClassification kind)
 #endif
 
 static int
-specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr,
+specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
                              PyObject *name)
 {
     _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
@@ -897,7 +902,7 @@ specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr,
         case NON_DESCRIPTOR:
             write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag);
             write_obj(cache->descr, descr);
-            _Py_SET_OPCODE(*instr, LOAD_ATTR_METHOD_CLASS);
+            _Py_SET_OPCODE(*instr, LOAD_ATTR_CLASS);
             return 0;
 #ifdef Py_STATS
         case ABSENT:
@@ -927,32 +932,13 @@ typedef enum {
 // can cause a significant drop in cache hits. A possible test is
 // python.exe -m test_typing test_re test_dis test_zlib.
 static int
-specialize_attr_loadmethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
+specialize_attr_loadmethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
+PyObject *descr, DescriptorClassification kind)
 {
     _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
     PyTypeObject *owner_cls = Py_TYPE(owner);
 
-    if (owner_cls->tp_dict == NULL) {
-        if (PyType_Ready(owner_cls) < 0) {
-            return -1;
-        }
-    }
-    if (PyType_Check(owner)) {
-        int err = specialize_class_load_method(owner, instr, name);
-        if (err) {
-            goto fail;
-        }
-        goto success;
-    }
-
-    PyObject *descr = NULL;
-    DescriptorClassification kind = 0;
-    kind = analyze_descriptor(owner_cls, name, &descr, 0);
-    assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
-    if (kind != METHOD) {
-        SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind));
-        goto fail;
-    }
+    assert(kind == METHOD && descr != NULL);
     ObjectDictKind dictkind;
     PyDictKeysObject *keys;
     if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
@@ -968,7 +954,7 @@ specialize_attr_loadmethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
     else {
         Py_ssize_t dictoffset = owner_cls->tp_dictoffset;
         if (dictoffset < 0 || dictoffset > INT16_MAX) {
-            SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_RANGE);
+            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
             goto fail;
         }
         if (dictoffset == 0) {
@@ -996,7 +982,7 @@ specialize_attr_loadmethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
         }
         uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys);
         if (keys_version == 0) {
-            SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_VERSIONS);
+            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
             goto fail;
         }
         write_u32(cache->keys_version, keys_version);
@@ -1037,7 +1023,6 @@ specialize_attr_loadmethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
     write_u32(cache->type_version, owner_cls->tp_version_tag);
     write_obj(cache->descr, descr);
     // Fall through.
-success:
     return 1;
 fail:
     return 0;