bpo-39702: Relax grammar restrictions on decorators (PEP 614) (python…

…GH-18570)

Grammar/Grammar

@@ -14,7 +14,7 @@ single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
 file_input: (NEWLINE | stmt)* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
-decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+decorator: '@' namedexpr_test NEWLINE
 decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)
 

Include/node.h

@@ -31,7 +31,6 @@ PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
 #define NCH(n)          ((n)->n_nchildren)
 
 #define CHILD(n, i)     (&(n)->n_child[i])
-#define RCHILD(n, i)    (CHILD(n, NCH(n) + i))
 #define TYPE(n)         ((n)->n_type)
 #define STR(n)          ((n)->n_str)
 #define LINENO(n)       ((n)->n_lineno)

Lib/test/test_decorators.py

@@ -151,21 +151,18 @@ def double(x):
         self.assertEqual(counts['double'], 4)
 
     def test_errors(self):
-        # Test syntax restrictions - these are all compile-time errors:
-        #
-        for expr in [ "1+2", "x[3]", "(1, 2)" ]:
-            # Sanity check: is expr is a valid expression by itself?
-            compile(expr, "testexpr", "exec")
-
-            codestr = "@%s\ndef f(): pass" % expr
-            self.assertRaises(SyntaxError, compile, codestr, "test", "exec")
 
-        # You can't put multiple decorators on a single line:
-        #
-        self.assertRaises(SyntaxError, compile,
-                          "@f1 @f2\ndef f(): pass", "test", "exec")
+        # Test SyntaxErrors:
+        for stmt in ("x,", "x, y", "x = y", "pass", "import sys"):
+            compile(stmt, "test", "exec")  # Sanity check.
+            with self.assertRaises(SyntaxError):
+                compile(f"@{stmt}\ndef f(): pass", "test", "exec")
 
-        # Test runtime errors
+        # Test TypeErrors that used to be SyntaxErrors:
+        for expr in ("1.+2j", "[1, 2][-1]", "(1, 2)", "True", "...", "None"):
+            compile(expr, "test", "eval")  # Sanity check.
+            with self.assertRaises(TypeError):
+                exec(f"@{expr}\ndef f(): pass")
 
         def unimp(func):
             raise NotImplementedError
@@ -179,6 +176,13 @@ def unimp(func):
             code = compile(codestr, "test", "exec")
             self.assertRaises(exc, eval, code, context)
 
+    def test_expressions(self):
+        for expr in (
+            "(x,)", "(x, y)", "x := y", "(x := y)", "x @y", "(x @ y)", "x[0]",
+            "w[x].y.z", "w + x - (y + z)", "x(y)()(z)", "[w, x, y][z]", "x.y",
+        ):
+            compile(f"@{expr}\ndef f(): pass", "test", "exec")
+
     def test_double(self):
         class C(object):
             @funcattrs(abc=1, xyz="haha")

Lib/test/test_grammar.py

@@ -460,7 +460,7 @@ def test_var_annot_rhs(self):
 
     def test_funcdef(self):
         ### [decorators] 'def' NAME parameters ['->' test] ':' suite
-        ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        ### decorator: '@' namedexpr_test NEWLINE
         ### decorators: decorator+
         ### parameters: '(' [typedargslist] ')'
         ### typedargslist: ((tfpdef ['=' test] ',')*
@@ -666,6 +666,20 @@ def null(x): return x
         def f(x) -> list: pass
         self.assertEqual(f.__annotations__, {'return': list})
 
+        # Test expressions as decorators (PEP 614):
+        @False or null
+        def f(x): pass
+        @d := null
+        def f(x): pass
+        @lambda f: null(f)
+        def f(x): pass
+        @[..., null, ...][1]
+        def f(x): pass
+        @null(null)(null)
+        def f(x): pass
+        @[null][0].__call__.__call__
+        def f(x): pass
+
         # test closures with a variety of opargs
         closure = 1
         def f(): return closure
@@ -1515,13 +1529,27 @@ def meth1(self): pass
             def meth2(self, arg): pass
             def meth3(self, a1, a2): pass
 
-        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorator: '@' namedexpr_test NEWLINE
         # decorators: decorator+
         # decorated: decorators (classdef | funcdef)
         def class_decorator(x): return x
         @class_decorator
         class G: pass
 
+        # Test expressions as decorators (PEP 614):
+        @False or class_decorator
+        class H: pass
+        @d := class_decorator
+        class I: pass
+        @lambda c: class_decorator(c)
+        class J: pass
+        @[..., class_decorator, ...][1]
+        class K: pass
+        @class_decorator(class_decorator)(class_decorator)
+        class L: pass
+        @[class_decorator][0].__call__.__call__
+        class M: pass
+
     def test_dictcomps(self):
         # dictorsetmaker: ( (test ':' test (comp_for |
         #                                   (',' test ':' test)* [','])) |

Lib/test/test_parser.py

@@ -226,6 +226,27 @@ def test_function_defs(self):
         self.check_suite("@funcattrs()\n"
                          "def f(): pass")
 
+        self.check_suite("@False or x\n"
+                         "def f(): pass")
+        self.check_suite("@d := x\n"
+                         "def f(): pass")
+        self.check_suite("@lambda f: x(f)\n"
+                         "def f(): pass")
+        self.check_suite("@[..., x, ...][1]\n"
+                         "def f(): pass")
+        self.check_suite("@x(x)(x)\n"
+                         "def f(): pass")
+        self.check_suite("@(x, x)\n"
+                         "def f(): pass")
+        self.check_suite("@...\n"
+                         "def f(): pass")
+        self.check_suite("@None\n"
+                         "def f(): pass")
+        self.check_suite("@w @(x @y) @(z)\n"
+                         "def f(): pass")
+        self.check_suite("@w[x].y.z\n"
+                         "def f(): pass")
+
         # keyword-only arguments
         self.check_suite("def f(*, a): pass")
         self.check_suite("def f(*, a = 5): pass")
@@ -270,6 +291,27 @@ def test_class_defs(self):
                          "@decorator2\n"
                          "class foo():pass")
 
+        self.check_suite("@False or x\n"
+                         "class C: pass")
+        self.check_suite("@d := x\n"
+                         "class C: pass")
+        self.check_suite("@lambda f: x(f)\n"
+                         "class C: pass")
+        self.check_suite("@[..., x, ...][1]\n"
+                         "class C: pass")
+        self.check_suite("@x(x)(x)\n"
+                         "class C: pass")
+        self.check_suite("@(x, x)\n"
+                         "class C: pass")
+        self.check_suite("@...\n"
+                         "class C: pass")
+        self.check_suite("@None\n"
+                         "class C: pass")
+        self.check_suite("@w @(x @y) @(z)\n"
+                         "class C: pass")
+        self.check_suite("@w[x].y.z\n"
+                         "class C: pass")
+
     def test_import_from_statement(self):
         self.check_suite("from sys.path import *")
         self.check_suite("from sys.path import dirname")

Misc/NEWS.d/next/Core and Builtins/2020-02-20-08-12-52.bpo-39702.4_AmyF.rst

@@ -0,0 +1,2 @@
+Relax :term:`decorator` grammar restrictions to allow any valid expression
+(:pep:`614`).

Python/ast.c

@@ -1691,80 +1691,16 @@ ast_for_arguments(struct compiling *c, const node *n)
     return arguments(posonlyargs, posargs, vararg, kwonlyargs, kwdefaults, kwarg, posdefaults, c->c_arena);
 }
 
-static expr_ty
-ast_for_dotted_name(struct compiling *c, const node *n)
-{
-    expr_ty e;
-    identifier id;
-    int lineno, col_offset;
-    int i;
-    node *ch;
-
-    REQ(n, dotted_name);
-
-    lineno = LINENO(n);
-    col_offset = n->n_col_offset;
-
-    ch = CHILD(n, 0);
-    id = NEW_IDENTIFIER(ch);
-    if (!id)
-        return NULL;
-    e = Name(id, Load, lineno, col_offset,
-             ch->n_end_lineno, ch->n_end_col_offset, c->c_arena);
-    if (!e)
-        return NULL;
-
-    for (i = 2; i < NCH(n); i+=2) {
-        const node *child = CHILD(n, i);
-        id = NEW_IDENTIFIER(child);
-        if (!id)
-            return NULL;
-        e = Attribute(e, id, Load, lineno, col_offset,
-                      child->n_end_lineno, child->n_end_col_offset, c->c_arena);
-        if (!e)
-            return NULL;
-    }
-
-    return e;
-}
-
 static expr_ty
 ast_for_decorator(struct compiling *c, const node *n)
 {
-    /* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */
-    expr_ty d = NULL;
-    expr_ty name_expr;
+    /* decorator: '@' namedexpr_test NEWLINE */
 
     REQ(n, decorator);
     REQ(CHILD(n, 0), AT);
-    REQ(RCHILD(n, -1), NEWLINE);
-
-    name_expr = ast_for_dotted_name(c, CHILD(n, 1));
-    if (!name_expr)
-        return NULL;
-
-    if (NCH(n) == 3) { /* No arguments */
-        d = name_expr;
-        name_expr = NULL;
-    }
-    else if (NCH(n) == 5) { /* Call with no arguments */
-        d = Call(name_expr, NULL, NULL,
-                 name_expr->lineno, name_expr->col_offset,
-                 CHILD(n, 3)->n_end_lineno, CHILD(n, 3)->n_end_col_offset,
-                 c->c_arena);
-        if (!d)
-            return NULL;
-        name_expr = NULL;
-    }
-    else {
-        d = ast_for_call(c, CHILD(n, 3), name_expr,
-                         CHILD(n, 1), CHILD(n, 2), CHILD(n, 4));
-        if (!d)
-            return NULL;
-        name_expr = NULL;
-    }
-
-    return d;
+    REQ(CHILD(n, 2), NEWLINE);
+
+    return ast_for_expr(c, CHILD(n, 1));
 }
 
 static asdl_seq*