Feature/pep 701 syntactic formalization of f strings

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.11"]  # no particular need for 3.9 or 3.10
+        python-version: ["3.7", "3.8", "3.11", "3.12.0-rc.1"]  # no particular need for 3.9 or 3.10
         os: [macos-latest, ubuntu-latest, windows-latest]
     steps:
     - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744  # v3.6.0

pep-701.py

@@ -0,0 +1,14 @@
+bag = {'wand': 'unicorn'}
+
+print(f'Magic wand: { bag['wand'] }')
+
+things = ['wand', 'cloak']
+
+source = 'a_file.py'
+
+print(f'These are the things: {', '.join(things)}')
+print(f'{source.removesuffix('.py')}.c: $(srcdir)/{source}')
+print(f'{f'{f'infinite'}'}' + ' ' + f'{f'nesting!!!'}')
+
+a = ['hello', 'world']
+print(f"{'\n'.join(a)}")

third_party/yapf_third_party/_ylib2to3/Grammar.txt

@@ -1,150 +1,109 @@
-# Grammar for 2to3. This grammar supports Python 2.x and 3.x.
+# Grammar for Python
 
 # NOTE WELL: You should also follow all the steps listed at
 # https://devguide.python.org/grammar/
 
 # Start symbols for the grammar:
-#	file_input is a module or sequence of commands read from an input file;
-#	single_input is a single interactive statement;
-#	eval_input is the input for the eval() and input() functions.
+#       single_input is a single interactive statement;
+#       file_input is a module or sequence of commands read from an input file;
+#       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
-file_input: (NEWLINE | stmt)* ENDMARKER
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' namedexpr_test NEWLINE
 decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)
-async_funcdef: ASYNC funcdef
-funcdef: 'def' NAME parameters ['->' test] ':' suite
-parameters: '(' [typedargslist] ')'
-
-# The following definition for typedarglist is equivalent to this set of rules:
-#
-#     arguments = argument (',' argument)*
-#     argument = tfpdef ['=' test]
-#     kwargs = '**' tname [',']
-#     args = '*' [tname_star]
-#     kwonly_kwargs = (',' argument)* [',' [kwargs]]
-#     args_kwonly_kwargs = args kwonly_kwargs | kwargs
-#     poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]]
-#     typedargslist_no_posonly  = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
-#     typedarglist = arguments ',' '/' [',' [typedargslist_no_posonly]])|(typedargslist_no_posonly)"
-#
-# It needs to be fully expanded to allow our LL(1) parser to work on it.
-
-typedargslist: tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [
-                     ',' [((tfpdef ['=' test] ',')* ('*' [tname_star] (',' tname ['=' test])*
-                            [',' ['**' tname [',']]] | '**' tname [','])
-                     | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])]
-                ] | ((tfpdef ['=' test] ',')* ('*' [tname_star] (',' tname ['=' test])*
-                     [',' ['**' tname [',']]] | '**' tname [','])
-                     | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
-
-tname: NAME [':' test]
-tname_star: NAME [':' (test|star_expr)]
-tfpdef: tname | '(' tfplist ')'
-tfplist: tfpdef (',' tfpdef)* [',']
-
-# The following definition for varargslist is equivalent to this set of rules:
-#
-#     arguments = argument (',' argument )*
-#     argument = vfpdef ['=' test]
-#     kwargs = '**' vname [',']
-#     args = '*' [vname]
-#     kwonly_kwargs = (',' argument )* [',' [kwargs]]
-#     args_kwonly_kwargs = args kwonly_kwargs | kwargs
-#     poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]]
-#     vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
-#     varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly)
-#
-# It needs to be fully expanded to allow our LL(1) parser to work on it.
 
-varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [
-                     ((vfpdef ['=' test] ',')* ('*' [vname] (',' vname ['=' test])*
-                            [',' ['**' vname [',']]] | '**' vname [','])
-                            | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
-                     ]] | ((vfpdef ['=' test] ',')*
-                     ('*' [vname] (',' vname ['=' test])*  [',' ['**' vname [',']]]| '**' vname [','])
-                     | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
-
-vname: NAME
-vfpdef: vname | '(' vfplist ')'
-vfplist: vfpdef (',' vfpdef)* [',']
+async_funcdef: 'async' funcdef
+funcdef: 'def' NAME parameters ['->' test] ':' suite
 
-stmt: simple_stmt | compound_stmt
+parameters: '(' [typedargslist] ')'
+typedargslist: (
+  (tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [',' [ tfpdef ['=' test] (
+        ',' tfpdef ['=' test])* ([',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]])
+  | '*' [tfpdef] (',' tfpdef ['=' test])* ([',' ['**' tfpdef [',']]])
+  | '**' tfpdef [',']]] )
+|  (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]]
+  | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+  | '**' tfpdef [','])
+)
+tfpdef: NAME [':' test]
+varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']
+)
+vfpdef: NAME
+
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
-small_stmt: (expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
-             import_stmt | global_stmt | exec_stmt | assert_stmt)
+small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
 expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist_star_expr))*)
 annassign: ':' test ['=' (yield_expr|testlist_star_expr)]
 testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
 augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
             '<<=' | '>>=' | '**=' | '//=')
 # For normal and annotated assignments, additional restrictions enforced by the interpreter
-print_stmt: 'print' ( [ test (',' test)* [','] ] |
-                      '>>' test [ (',' test)+ [','] ] )
 del_stmt: 'del' exprlist
 pass_stmt: 'pass'
 flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
 break_stmt: 'break'
 continue_stmt: 'continue'
 return_stmt: 'return' [testlist_star_expr]
 yield_stmt: yield_expr
-raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]]
+raise_stmt: 'raise' [test ['from' test]]
 import_stmt: import_name | import_from
 import_name: 'import' dotted_as_names
-import_from: ('from' ('.'* dotted_name | '.'+)
+# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
               'import' ('*' | '(' import_as_names ')' | import_as_names))
 import_as_name: NAME ['as' NAME]
 dotted_as_name: dotted_name ['as' NAME]
 import_as_names: import_as_name (',' import_as_name)* [',']
 dotted_as_names: dotted_as_name (',' dotted_as_name)*
 dotted_name: NAME ('.' NAME)*
-global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
-exec_stmt: 'exec' expr ['in' test [',' test]]
+global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]
 
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt
-async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
-for_stmt: 'for' exprlist 'in' testlist_star_expr ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
 try_stmt: ('try' ':' suite
            ((except_clause ':' suite)+
-	    ['else' ':' suite]
-	    ['finally' ':' suite] |
-	   'finally' ':' suite))
-with_stmt: 'with' asexpr_test (',' asexpr_test)*  ':' suite
-
+            ['else' ':' suite]
+            ['finally' ':' suite] |
+           'finally' ':' suite))
+with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_item: test ['as' expr]
 # NB compile.c makes sure that the default except clause is last
-except_clause: 'except' ['*'] [test [(',' | 'as') test]]
+except_clause: 'except' [test ['as' NAME]]
 suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
 
-# Backward compatibility cruft to support:
-# [ x for x in lambda: True, lambda: False if x() ]
-# even while also allowing:
-# lambda x: 5 if x else 2
-# (But not a mix of the two)
-testlist_safe: old_test [(',' old_test)+ [',']]
-old_test: or_test | old_lambdef
-old_lambdef: 'lambda' [varargslist] ':' old_test
-
-namedexpr_test: asexpr_test [':=' asexpr_test]
-
-# This is actually not a real rule, though since the parser is very
-# limited in terms of the strategy about match/case rules, we are inserting
-# a virtual case (<expr> as <expr>) as a valid expression. Unless a better
-# approach is thought, the only side effect of this seem to be just allowing
-# more stuff to be parser (which would fail on the ast).
-asexpr_test: test ['as' test]
-
+namedexpr_test: test [':=' test]
 test: or_test ['if' or_test 'else' test] | lambdef
+lambdef: 'lambda' [varargslist] ':' test
 or_test: and_test ('or' and_test)*
 and_test: not_test ('and' not_test)*
 not_test: 'not' not_test | comparison
 comparison: expr (comp_op expr)*
+# <> isn't actually a valid comparison operator in Python. It's here for the
+# sake of a __future__ import described in PEP 401 (which really works :-)
 comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
 star_expr: '*' expr
 expr: xor_expr ('|' xor_expr)*
@@ -154,30 +113,30 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
-power: [AWAIT] atom trailer* ['**' factor]
-atom: ('(' [yield_expr|testlist_gexp] ')' |
-       '[' [listmaker] ']' |
-       '{' [dictsetmaker] '}' |
-       '`' testlist1 '`' |
-       NAME | NUMBER | STRING+ | '.' '.' '.')
-listmaker: (namedexpr_test|star_expr) ( old_comp_for | (',' (namedexpr_test|star_expr))* [','] )
-testlist_gexp: (namedexpr_test|star_expr) ( old_comp_for | (',' (namedexpr_test|star_expr))* [','] )
-lambdef: 'lambda' [varargslist] ':' test
+power: atom_expr ['**' factor]
+atom_expr: ['await'] atom trailer*
+atom: ('(' [yield_expr|testlist_comp] ')' |
+       '[' [testlist_comp] ']' |
+       '{' [dictorsetmaker] '}' |
+       NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
+testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
-subscriptlist: (subscript|star_expr) (',' (subscript|star_expr))* [',']
+subscriptlist: subscript (',' subscript)* [',']
 subscript: test [':=' test] | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
-dictsetmaker: ( ((test ':' asexpr_test | '**' expr)
-                 (comp_for | (',' (test ':' asexpr_test | '**' expr))* [','])) |
-                ((test [':=' test] | star_expr)
-		 (comp_for | (',' (test [':=' test] | star_expr))* [','])) )
+dictorsetmaker: ( ((test ':' test | '**' expr)
+                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                  ((test [':=' test] | star_expr)
+                   (comp_for | (',' (test [':=' test] | star_expr))* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
-arglist: argument (',' argument)* [',']
+arglist: argument (',' argument)*  [',']
 
+# The reason that keywords are test nodes instead of NAME is that using NAME
+# results in an ambiguity. ast.c makes sure it's a NAME.
 # "test '=' test" is really "keyword '=' test", but we have no such token.
 # These need to be in a single rule to avoid grammar that is ambiguous
 # to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
@@ -186,67 +145,25 @@ arglist: argument (',' argument)* [',']
 # multiple (test comp_for) arguments are blocked; keyword unpackings
 # that precede iterable unpackings are blocked; etc.
 argument: ( test [comp_for] |
-            test ':=' test [comp_for] |
-            test 'as' test |
-            test '=' asexpr_test |
-	    '**' test |
+            test ':=' test |
+            test '=' test |
+            '**' test |
             '*' test )
 
 comp_iter: comp_for | comp_if
-comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
-comp_if: 'if' old_test [comp_iter]
-
-# As noted above, testlist_safe extends the syntax allowed in list
-# comprehensions and generators. We can't use it indiscriminately in all
-# derivations using a comp_for-like pattern because the testlist_safe derivation
-# contains comma which clashes with trailing comma in arglist.
-#
-# This was an issue because the parser would not follow the correct derivation
-# when parsing syntactically valid Python code. Since testlist_safe was created
-# specifically to handle list comprehensions and generator expressions enclosed
-# with parentheses, it's safe to only use it in those. That avoids the issue; we
-# can parse code like set(x for x in [],).
-#
-# The syntax supported by this set of rules is not a valid Python 3 syntax,
-# hence the prefix "old".
-#
-# See https://bugs.python.org/issue27494
-old_comp_iter: old_comp_for | old_comp_if
-old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter]
-old_comp_if: 'if' old_test [old_comp_iter]
-
-testlist1: test (',' test)*
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
+comp_if: 'if' or_test [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler
 encoding_decl: NAME
 
 yield_expr: 'yield' [yield_arg]
 yield_arg: 'from' test | testlist_star_expr
 
-
-# 3.10 match statement definition
-
-# PS: normally the grammar is much much more restricted, but
-# at this moment for not trying to bother much with encoding the
-# exact same DSL in a LL(1) parser, we will just accept an expression
-# and let the ast.parse() step of the safe mode to reject invalid
-# grammar.
-
-# The reason why it is more restricted is that, patterns are some
-# sort of a DSL (more advanced than our LHS on assignments, but
-# still in a very limited python subset). They are not really
-# expressions, but who cares. If we can parse them, that is enough
-# to reformat them.
-
-match_stmt: "match" subject_expr ':' NEWLINE INDENT case_block+ DEDENT
-
-# This is more permissive than the actual version. For example it
-# accepts `match *something:`, even though single-item starred expressions
-# are forbidden.
-subject_expr: (namedexpr_test|star_expr) (',' (namedexpr_test|star_expr))* [',']
-
-# cases
-case_block: "case" patterns [guard] ':' suite
-guard: 'if' namedexpr_test
-patterns: pattern (',' pattern)* [',']
-pattern: (expr|star_expr) ['as' expr]
+strings: (STRING | fstring)+
+fstring: FSTRING_START fstring_content* FSTRING_END
+fstring_content: FSTRING_STRING | fstring_expr
+fstring_conversion: '!' NAME
+fstring_expr: '{' (testlist_comp | yield_expr) ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_format_spec: ':' fstring_content*

third_party/yapf_third_party/_ylib2to3/pgen2/grammar.py

@@ -179,6 +179,8 @@ def report(self):
 //= DOUBLESLASHEQUAL
 -> RARROW
 := COLONEQUAL
+! FSTRINGCONVERSION
+... ELLIPSIS
 """
 
 opmap = {}

third_party/yapf_third_party/_ylib2to3/pgen2/token.py

@@ -66,6 +66,11 @@
 ERRORTOKEN = 58
 COLONEQUAL = 59
 N_TOKENS = 60
+FSTRING_START = 61
+FSTRING_END = 62
+FSTRING_STRING = 63
+FSTRINGCONVERSION = 64
+ELLIPSIS = 65
 NT_OFFSET = 256
 # --end constants--
 

third_party/yapf_third_party/_ylib2to3/pygram.py

@@ -31,10 +31,10 @@ def __init__(self, grammar):
 python_symbols = Symbols(python_grammar)
 
 python_grammar_no_print_statement = python_grammar.copy()
-del python_grammar_no_print_statement.keywords['print']
+# del python_grammar_no_print_statement.keywords['print']
 
 python_grammar_no_print_and_exec_statement = python_grammar_no_print_statement.copy()  # yapf: disable # noqa: E501
-del python_grammar_no_print_and_exec_statement.keywords['exec']
+# del python_grammar_no_print_and_exec_statement.keywords['exec']
 
 pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE)
 pattern_symbols = Symbols(pattern_grammar)

yapf/pytree/pytree_utils.py

@@ -87,7 +87,7 @@ def LastLeafNode(node):
 # Note that pygram.python_grammar_no_print_and_exec_statement with "_and_exec"
 # will require Python >=3.8.
 _PYTHON_GRAMMAR = pygram.python_grammar_no_print_statement.copy()
-del _PYTHON_GRAMMAR.keywords['exec']
+# del _PYTHON_GRAMMAR.keywords['exec']
 
 
 def ParseCodeToTree(code):