bpo-29869: Allow underscores in numeric literals in lib2to3. (pythonG…

…H-1119)

* Allow underscores in numeric literals in lib2to3.
* Stricter literal parsing for Python 3.6 in lib2to3.pgen2.tokenize.
* Add test case for underscores in literals in Python 3.

Lib/lib2to3/pgen2/tokenize.py

@@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Name = r'[a-zA-Z_]\w*'
 
-Binnumber = r'0[bB][01]*'
-Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
-Octnumber = r'0[oO]?[0-7]*[lL]?'
-Decnumber = r'[1-9]\d*[lL]?'
+Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
+Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
+Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?'
+Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?')
 Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber)
-Exponent = r'[eE][-+]?\d+'
-Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
-Expfloat = r'\d+' + Exponent
+Exponent = r'[eE][-+]?\d+(?:_\d+)*'
+Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent)
+Expfloat = r'\d+(?:_\d+)*' + Exponent
 Floatnumber = group(Pointfloat, Expfloat)
-Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
+Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]')
 Number = group(Imagnumber, Floatnumber, Intnumber)
 
 # Tail end of ' string.

Lib/lib2to3/tests/data/py3_test_grammar.py

@@ -72,6 +72,28 @@ def testLongIntegers(self):
         x = 0b100000000000000000000000000000000000000000000000000000000000000000000
         x = 0B111111111111111111111111111111111111111111111111111111111111111111111
 
+    def testUnderscoresInNumbers(self):
+        # Integers
+        x = 1_0
+        x = 123_456_7_89
+        x = 0xabc_123_4_5
+        x = 0X_abc_123
+        x = 0B11_01
+        x = 0b_11_01
+        x = 0o45_67
+        x = 0O_45_67
+
+        # Floats
+        x = 3_1.4
+        x = 03_1.4
+        x = 3_1.
+        x = .3_1
+        x = 3.1_4
+        x = 0_3.1_4
+        x = 3e1_4
+        x = 3_1e+4_1
+        x = 3_1E-4_1
+
     def testFloats(self):
         x = 3.14
         x = 314.