# Copyright (c) 2017-2020 Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ Python 3.7 grammar for the spark Earley-algorithm parser. """ from __future__ import print_function from uncompyle6.parser import PythonParserSingle, nop_func from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from uncompyle6.parsers.parse37base import Python37BaseParser class Python37Parser(Python37BaseParser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): super(Python37Parser, self).__init__(debug_parser) self.customized = {} ############################################### # Python 3.7 grammar rules ############################################### def p_start(self, args): """ # The start or goal symbol stmts ::= sstmt+ """ def p_call_stmt(self, args): """ # eval-mode compilation. Single-mode interactive compilation # adds another rule. call_stmt ::= expr POP_TOP """ def p_stmt(self, args): """ pass ::= _stmts ::= stmt+ # statements with continue and break c_stmts ::= _stmts c_stmts ::= _stmts lastc_stmt c_stmts ::= lastc_stmt c_stmts ::= continues lastc_stmt ::= iflaststmt lastc_stmt ::= forelselaststmt lastc_stmt ::= ifelsestmtc # Statements in a loop lstmt ::= stmt l_stmts ::= lstmt+ c_stmts_opt ::= c_stmts c_stmts_opt ::= pass # statements inside a loop l_stmts ::= _stmts l_stmts ::= returns l_stmts ::= continues l_stmts ::= _stmts lastl_stmt l_stmts ::= lastl_stmt lastl_stmt ::= iflaststmtl lastl_stmt ::= ifelsestmtl lastl_stmt ::= forelselaststmtl lastl_stmt ::= tryelsestmtl l_stmts_opt ::= l_stmts l_stmts_opt ::= pass suite_stmts ::= _stmts suite_stmts ::= returns suite_stmts ::= continues suite_stmts_opt ::= suite_stmts # passtmt is needed for semantic actions to add "pass" suite_stmts_opt ::= pass else_suite ::= suite_stmts else_suitel ::= l_stmts else_suitec ::= c_stmts else_suitec ::= returns else_suite_opt ::= else_suite else_suite_opt ::= pass stmt ::= classdef stmt ::= call_stmt stmt ::= ifstmt stmt ::= ifelsestmt stmt ::= whilestmt stmt ::= while1stmt stmt ::= whileelsestmt stmt ::= while1elsestmt stmt ::= for stmt ::= forelsestmt stmt ::= try_except stmt ::= tryelsestmt stmt ::= tryfinallystmt stmt ::= del_stmt del_stmt ::= DELETE_FAST del_stmt ::= DELETE_NAME del_stmt ::= DELETE_GLOBAL stmt ::= return return ::= ret_expr RETURN_VALUE # "returns" nonterminal is a sequence of statements that ends in a RETURN statement. # In later Python versions with jump optimization, this can cause JUMPs # that would normally appear to be omitted. returns ::= return returns ::= _stmts return stmt ::= genexpr_func genexpr_func ::= LOAD_FAST _come_froms FOR_ITER store comp_iter JUMP_BACK """ pass def p_expr(self, args): """ expr ::= LOAD_CODE expr ::= LOAD_CONST expr ::= LOAD_DEREF expr ::= LOAD_FAST expr ::= LOAD_GLOBAL expr ::= LOAD_NAME expr ::= LOAD_STR expr ::= _mklambda expr ::= and expr ::= bin_op expr ::= call expr ::= compare expr ::= dict expr ::= generator_exp expr ::= list expr ::= or expr ::= subscript expr ::= subscript2 expr ::= unary_not expr ::= unary_op expr ::= yield # bin_op (formerly "binary_expr") is the Python AST BinOp bin_op ::= expr expr binary_operator binary_operator ::= BINARY_ADD binary_operator ::= BINARY_MULTIPLY binary_operator ::= BINARY_AND binary_operator ::= BINARY_OR binary_operator ::= BINARY_XOR binary_operator ::= BINARY_SUBTRACT binary_operator ::= BINARY_TRUE_DIVIDE binary_operator ::= BINARY_FLOOR_DIVIDE binary_operator ::= BINARY_MODULO binary_operator ::= BINARY_LSHIFT binary_operator ::= BINARY_RSHIFT binary_operator ::= BINARY_POWER # unary_op (formerly "unary_expr") is the Python AST UnaryOp unary_op ::= expr unary_operator unary_operator ::= UNARY_POSITIVE unary_operator ::= UNARY_NEGATIVE unary_operator ::= UNARY_INVERT unary_not ::= expr UNARY_NOT subscript ::= expr expr BINARY_SUBSCR get_iter ::= expr GET_ITER yield ::= expr YIELD_VALUE _mklambda ::= mklambda expr ::= if_exp ret_expr ::= expr ret_expr ::= ret_and ret_expr ::= ret_or ret_expr_or_cond ::= ret_expr ret_expr_or_cond ::= if_exp_ret stmt ::= return_lambda return_lambda ::= ret_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER return_lambda ::= ret_expr RETURN_VALUE_LAMBDA compare ::= compare_chained compare ::= compare_single compare_single ::= expr expr COMPARE_OP # A compare_chained is two comparisions like x <= y <= z compare_chained ::= expr compare_chained1 ROT_TWO POP_TOP _come_froms compare_chained2 ::= expr COMPARE_OP JUMP_FORWARD # Non-null kvlist items are broken out in the indiviual grammars kvlist ::= # Positional arguments in make_function pos_arg ::= expr """ def p_function_def(self, args): """ stmt ::= function_def function_def ::= mkfunc store stmt ::= function_def_deco function_def_deco ::= mkfuncdeco store mkfuncdeco ::= expr mkfuncdeco CALL_FUNCTION_1 mkfuncdeco ::= expr mkfuncdeco0 CALL_FUNCTION_1 mkfuncdeco0 ::= mkfunc load_closure ::= load_closure LOAD_CLOSURE load_closure ::= LOAD_CLOSURE """ def p_generator_exp(self, args): """ """ def p_jump(self, args): """ _jump ::= JUMP_ABSOLUTE _jump ::= JUMP_FORWARD _jump ::= JUMP_BACK # Zero or more COME_FROMs - loops can have this _come_froms ::= COME_FROM* _come_froms ::= _come_froms COME_FROM_LOOP # One or more COME_FROMs - joins of tryelse's have this come_froms ::= COME_FROM+ # Zero or one COME_FROM # And/or expressions have this come_from_opt ::= COME_FROM? """ def p_augmented_assign(self, args): """ stmt ::= aug_assign1 stmt ::= aug_assign2 # This is odd in that other aug_assign1's have only 3 slots # The store isn't used as that's supposed to be also # indicated in the first expr aug_assign1 ::= expr expr inplace_op store aug_assign1 ::= expr expr inplace_op ROT_THREE STORE_SUBSCR aug_assign2 ::= expr DUP_TOP LOAD_ATTR expr inplace_op ROT_TWO STORE_ATTR inplace_op ::= INPLACE_ADD inplace_op ::= INPLACE_SUBTRACT inplace_op ::= INPLACE_MULTIPLY inplace_op ::= INPLACE_TRUE_DIVIDE inplace_op ::= INPLACE_FLOOR_DIVIDE inplace_op ::= INPLACE_MODULO inplace_op ::= INPLACE_POWER inplace_op ::= INPLACE_LSHIFT inplace_op ::= INPLACE_RSHIFT inplace_op ::= INPLACE_AND inplace_op ::= INPLACE_XOR inplace_op ::= INPLACE_OR """ def p_assign(self, args): """ stmt ::= assign assign ::= expr DUP_TOP designList assign ::= expr store stmt ::= assign2 stmt ::= assign3 assign2 ::= expr expr ROT_TWO store store assign3 ::= expr expr expr ROT_THREE ROT_TWO store store store """ def p_forstmt(self, args): """ get_for_iter ::= GET_ITER _come_froms FOR_ITER for_block ::= l_stmts_opt _come_froms JUMP_BACK forelsestmt ::= SETUP_LOOP expr get_for_iter store for_block POP_BLOCK else_suite _come_froms forelselaststmt ::= SETUP_LOOP expr get_for_iter store for_block POP_BLOCK else_suitec _come_froms forelselaststmtl ::= SETUP_LOOP expr get_for_iter store for_block POP_BLOCK else_suitel _come_froms """ def p_import20(self, args): """ stmt ::= import stmt ::= import_from stmt ::= import_from_star stmt ::= importmultiple importlist ::= importlist alias importlist ::= alias alias ::= IMPORT_NAME store alias ::= IMPORT_FROM store alias ::= IMPORT_NAME attributes store import ::= LOAD_CONST LOAD_CONST alias import_from_star ::= LOAD_CONST LOAD_CONST IMPORT_NAME IMPORT_STAR import_from_star ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR IMPORT_STAR import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist POP_TOP importmultiple ::= LOAD_CONST LOAD_CONST alias imports_cont imports_cont ::= import_cont+ import_cont ::= LOAD_CONST LOAD_CONST alias attributes ::= LOAD_ATTR+ """ def p_import37(self, args): """ stmt ::= import_as37 import_as37 ::= LOAD_CONST LOAD_CONST importlist37 store POP_TOP importlist37 ::= importlist37 ROT_TWO IMPORT_FROM importlist37 ::= importlist37 ROT_TWO POP_TOP IMPORT_FROM importlist37 ::= importattr37 importattr37 ::= IMPORT_NAME_ATTR IMPORT_FROM # The 3.7base scanner adds IMPORT_NAME_ATTR alias ::= IMPORT_NAME_ATTR attributes store alias ::= IMPORT_NAME_ATTR store import_from ::= LOAD_CONST LOAD_CONST importlist POP_TOP expr ::= attribute37 attribute37 ::= expr LOAD_METHOD stmt ::= import_from37 importlist37 ::= importlist37 alias37 importlist37 ::= alias37 alias37 ::= IMPORT_NAME store alias37 ::= IMPORT_FROM store import_from37 ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist37 POP_TOP """ def p_list_comprehension(self, args): """ expr ::= list_comp list_iter ::= list_for list_iter ::= list_if list_iter ::= list_if_not list_iter ::= lc_body list_if ::= expr jmp_false list_iter list_if_not ::= expr jmp_true list_iter """ def p_set_comp(self, args): """ comp_iter ::= comp_for comp_body ::= gen_comp_body gen_comp_body ::= expr YIELD_VALUE POP_TOP comp_if ::= expr jmp_false comp_iter """ def p_store(self, args): """ # Note. The below is right-recursive: designList ::= store store designList ::= store DUP_TOP designList ## Can we replace with left-recursive, and redo with: ## ## designList ::= designLists store store ## designLists ::= designLists store DUP_TOP ## designLists ::= ## Will need to redo semantic actiion store ::= STORE_FAST store ::= STORE_NAME store ::= STORE_GLOBAL store ::= STORE_DEREF store ::= expr STORE_ATTR store ::= store_subscript store_subscript ::= expr expr STORE_SUBSCR store ::= unpack """ def p_32on(self, args): """ if_exp::= expr jmp_false expr jump_forward_else expr COME_FROM # compare_chained2 is used in a "chained_compare": x <= y <= z # used exclusively in compare_chained compare_chained2 ::= expr COMPARE_OP RETURN_VALUE compare_chained2 ::= expr COMPARE_OP RETURN_VALUE_LAMBDA # Python < 3.5 no POP BLOCK whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM_LOOP # Python 3.5+ has jump optimization to remove the redundant # jump_excepts. But in 3.3 we need them added except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK except_handler else_suite jump_excepts come_from_except_clauses jump_excepts ::= jump_except+ subscript2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR # FIXME: The below rule was in uncompyle6. # In decompyle6 though "_ifstmts_jump" is part of an "ifstmt" # where as the below rule is appropriate for an "ifelsesmt" # Investigate and reconcile # _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_froms kv3 ::= expr expr STORE_MAP """ return def p_33on(self, args): """ # Python 3.3+ adds yield from. expr ::= yield_from yield_from ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM # We do the grammar hackery below for semantics # actions that want c_stmts_opt at index 1 # Python 3.5+ has jump optimization to remove the redundant # jump_excepts. But in 3.3 we need them added try_except ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK except_handler jump_excepts come_from_except_clauses """ def p_34on(self, args): """ whilestmt ::= setup_loop testexpr returns come_froms POP_BLOCK COME_FROM_LOOP # Seems to be needed starting 3.4.4 or so while1stmt ::= setup_loop l_stmts COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP while1stmt ::= setup_loop l_stmts POP_BLOCK COME_FROM_LOOP # FIXME the below masks a bug in not detecting COME_FROM_LOOP # grammar rules with COME_FROM -> COME_FROM_LOOP already exist whileelsestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK POP_BLOCK else_suitel COME_FROM while1elsestmt ::= setup_loop l_stmts JUMP_BACK _come_froms POP_BLOCK else_suitel COME_FROM_LOOP # Python 3.4+ optimizes the trailing two JUMPS away _ifstmts_jump ::= c_stmts_opt JUMP_ABSOLUTE JUMP_FORWARD _come_froms """ def p_35on(self, args): """ while1elsestmt ::= setup_loop l_stmts JUMP_BACK POP_BLOCK else_suite COME_FROM_LOOP # The following rule is for Python 3.5+ where we can have stuff like # while .. # if # ... # the end of the if will jump back to the loop and there will be a COME_FROM # after the jump l_stmts ::= lastl_stmt come_froms l_stmts # Python 3.5+ Await statement expr ::= await_expr await_expr ::= expr GET_AWAITABLE LOAD_CONST YIELD_FROM stmt ::= await_stmt await_stmt ::= await_expr POP_TOP # Python 3.5+ async additions inplace_op ::= INPLACE_MATRIX_MULTIPLY binary_operator ::= BINARY_MATRIX_MULTIPLY # Python 3.5+ does jump optimization # In <.3.5 the below is a JUMP_FORWARD to a JUMP_ABSOLUTE. return_if_stmt ::= ret_expr RETURN_END_IF POP_BLOCK return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM jb_else ::= JUMP_BACK ELSE jb_else ::= JUMP_BACK COME_FROM ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec ifelsestmtl ::= testexpr c_stmts_opt jb_else else_suitel # We want to keep the positions of the "then" and # "else" statements in "ifelstmtl" similar to others of this ilk. testexpr_cf ::= testexpr come_froms ifelsestmtl ::= testexpr_cf c_stmts_opt jb_else else_suitel # 3.5 Has jump optimization which can route the end of an # "if/then" back to to a loop just before an else. jump_absolute_else ::= jb_else jump_absolute_else ::= CONTINUE ELSE # Our hacky "ELSE" determination doesn't do a good job and really # determine the start of an "else". It could also be the end of an # "if-then" which ends in a "continue". Perhaps with real control-flow # analysis we'll sort this out. Or call "ELSE" something more appropriate. _ifstmts_jump ::= c_stmts_opt ELSE # ifstmt ::= testexpr c_stmts_opt iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD """ def p_37async(self, args): """ stmt ::= async_for_stmt37 stmt ::= async_for_stmt stmt ::= async_forelse_stmt async_for_stmt ::= setup_loop expr GET_AITER SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM store POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE END_FINALLY COME_FROM for_block COME_FROM POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK COME_FROM_LOOP # Order of LOAD_CONST YIELD_FROM is switched from 3.6 to save a LOAD_CONST async_for_stmt37 ::= setup_loop expr GET_AITER SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM store POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE END_FINALLY for_block COME_FROM POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK COME_FROM_LOOP async_forelse_stmt ::= setup_loop expr GET_AITER SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM store POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE END_FINALLY COME_FROM for_block COME_FROM POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK else_suite COME_FROM_LOOP """ def p_37chained(self, args): """ testtrue ::= compare_chained37 testfalse ::= compare_chained37_false compare_chained ::= compare_chained37 compare_chained ::= compare_chained37_false compare_chained37 ::= expr compare_chained1a_37 compare_chained37 ::= expr compare_chained1c_37 compare_chained37_false ::= expr compare_chained1_false_37 compare_chained37_false ::= expr compare_chained1b_false_37 compare_chained37_false ::= expr compare_chained2_false_37 compare_chained1a_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE compare_chained1a_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE compare_chained2a_37 COME_FROM POP_TOP COME_FROM compare_chained1b_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE compare_chained2b_false_37 POP_TOP _jump COME_FROM compare_chained1c_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE compare_chained2a_37 POP_TOP compare_chained1_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE compare_chained2c_37 POP_TOP JUMP_FORWARD COME_FROM compare_chained1_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE compare_chained2b_false_37 POP_TOP _jump COME_FROM compare_chained2_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE compare_chained2a_false_37 POP_TOP JUMP_BACK COME_FROM compare_chained2a_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_TRUE JUMP_FORWARD compare_chained2a_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_TRUE JUMP_BACK compare_chained2a_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE jf_cfs compare_chained2b_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD COME_FROM compare_chained2b_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD compare_chained2c_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE compare_chained2a_false_37 ELSE compare_chained2c_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE compare_chained2a_false_37 """ def p_37conditionals(self, args): """ expr ::= if_exp37 if_exp37 ::= expr expr jf_cfs expr COME_FROM jf_cfs ::= JUMP_FORWARD _come_froms ifelsestmt ::= testexpr c_stmts_opt jf_cfs else_suite opt_come_from_except # This is probably more realistically an "ifstmt" (with a null else) # see _cmp() of python3.8/distutils/__pycache__/version.cpython-38.opt-1.pyc ifelsestmt ::= testexpr stmts jf_cfs else_suite_opt opt_come_from_except expr_pjit ::= expr POP_JUMP_IF_TRUE expr_jit ::= expr JUMP_IF_TRUE expr_jt ::= expr jmp_true jmp_false37 ::= POP_JUMP_IF_FALSE COME_FROM list_if ::= expr jmp_false37 list_iter list_iter ::= list_if37 list_iter ::= list_if37_not list_if37 ::= compare_chained37_false list_iter list_if37_not ::= compare_chained37 list_iter _ifstmts_jump ::= c_stmts_opt come_froms _ifstmts_jump ::= COME_FROM c_stmts come_froms and_not ::= expr jmp_false expr POP_JUMP_IF_TRUE testfalse ::= and_not expr ::= if_exp_37a expr ::= if_exp_37b if_exp_37a ::= and_not expr JUMP_FORWARD come_froms expr COME_FROM if_exp_37b ::= expr jmp_false expr POP_JUMP_IF_FALSE jump_forward_else expr jmp_false_cf ::= POP_JUMP_IF_FALSE COME_FROM comp_if ::= or jmp_false_cf comp_iter """ def p_comprehension3(self, args): """ # Python3 scanner adds LOAD_LISTCOMP. Python3 does list comprehension like # other comprehensions (set, dictionary). # Our "continue" heuristic - in two successive JUMP_BACKS, the first # one may be a continue - sometimes classifies a JUMP_BACK # as a CONTINUE. The two are kind of the same in a comprehension. comp_for ::= expr get_for_iter store comp_iter CONTINUE comp_for ::= expr get_for_iter store comp_iter JUMP_BACK for_iter ::= _come_froms FOR_ITER list_comp ::= BUILD_LIST_0 list_iter lc_body ::= expr LIST_APPEND list_for ::= expr for_iter store list_iter jb_or_c # This is seen in PyPy, but possibly it appears on other Python 3? list_if ::= expr jmp_false list_iter COME_FROM list_if_not ::= expr jmp_true list_iter COME_FROM jb_or_c ::= JUMP_BACK jb_or_c ::= CONTINUE stmt ::= set_comp_func set_comp_func ::= BUILD_SET_0 LOAD_FAST for_iter store comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST set_comp_func ::= BUILD_SET_0 LOAD_FAST for_iter store comp_iter COME_FROM JUMP_BACK RETURN_VALUE RETURN_LAST comp_body ::= dict_comp_body comp_body ::= set_comp_body dict_comp_body ::= expr expr MAP_ADD set_comp_body ::= expr SET_ADD # See also common Python p_list_comprehension """ def p_dict_comp3(self, args): """" expr ::= dict_comp stmt ::= dict_comp_func dict_comp_func ::= BUILD_MAP_0 LOAD_FAST for_iter store comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST comp_iter ::= comp_if comp_iter ::= comp_if_not comp_if_not ::= expr jmp_true comp_iter comp_iter ::= comp_body """ def p_expr3(self, args): """ expr ::= if_exp_not if_exp_not ::= expr jmp_true expr jump_forward_else expr COME_FROM # a JUMP_FORWARD to another JUMP_FORWARD can get turned into # a JUMP_ABSOLUTE with no COME_FROM if_exp ::= expr jmp_false expr jump_absolute_else expr # if_exp_true are for conditions which always evaluate true # There is dead or non-optional remnants of the condition code though, # and we use that to match on to reconstruct the source more accurately expr ::= if_exp_true if_exp_true ::= expr JUMP_FORWARD expr COME_FROM """ def p_generator_exp3(self, args): """ load_genexpr ::= LOAD_GENEXPR load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR """ def p_grammar(self, args): """ sstmt ::= stmt sstmt ::= ifelsestmtr sstmt ::= return RETURN_LAST return_if_stmts ::= return_if_stmt come_from_opt return_if_stmts ::= _stmts return_if_stmt _come_froms return_if_stmt ::= ret_expr RETURN_END_IF returns ::= _stmts return_if_stmt stmt ::= break break ::= BREAK_LOOP stmt ::= continue continue ::= CONTINUE continues ::= _stmts lastl_stmt continue continues ::= lastl_stmt continue continues ::= continue kwarg ::= LOAD_STR expr kwargs ::= kwarg+ classdef ::= build_class store # FIXME: we need to add these because don't detect this properly # in custom rules. Specifically if one of the exprs is CALL_FUNCTION # then we'll mistake that for the final CALL_FUNCTION. # We can fix by triggering on the CALL_FUNCTION op # Python3 introduced LOAD_BUILD_CLASS # Other definitions are in a custom rule build_class ::= LOAD_BUILD_CLASS mkfunc expr call CALL_FUNCTION_3 build_class ::= LOAD_BUILD_CLASS mkfunc expr call expr CALL_FUNCTION_4 stmt ::= classdefdeco classdefdeco ::= classdefdeco1 store # In 3.7 there are some LOAD_GLOBALs we don't convert to LOAD_ASSERT stmt ::= assert2 assert2 ::= expr jmp_true LOAD_GLOBAL expr CALL_FUNCTION_1 RAISE_VARARGS_1 # "assert_invert" tests on the negative of the condition given stmt ::= assert_invert assert_invert ::= testtrue LOAD_GLOBAL RAISE_VARARGS_1 expr ::= LOAD_ASSERT # FIXME: add this: # expr ::= assert_expr_or ifstmt ::= testexpr _ifstmts_jump testexpr ::= testfalse testexpr ::= testtrue testfalse ::= expr jmp_false testtrue ::= expr jmp_true _ifstmts_jump ::= return_if_stmts _ifstmts_jump ::= c_stmts_opt COME_FROM iflaststmt ::= testexpr c_stmts iflaststmt ::= testexpr c_stmts JUMP_ABSOLUTE iflaststmtl ::= testexpr c_stmts JUMP_BACK iflaststmtl ::= testexpr c_stmts JUMP_BACK COME_FROM_LOOP iflaststmtl ::= testexpr c_stmts JUMP_BACK POP_BLOCK # These are used to keep parse tree indices the same jump_forward_else ::= JUMP_FORWARD jump_forward_else ::= JUMP_FORWARD ELSE jump_forward_else ::= JUMP_FORWARD COME_FROM jump_absolute_else ::= JUMP_ABSOLUTE ELSE jump_absolute_else ::= JUMP_ABSOLUTE _come_froms jump_absolute_else ::= come_froms _jump COME_FROM # Note: in if/else kinds of statements, we err on the side # of missing "else" clauses. Therefore we include grammar # rules with and without ELSE. ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite opt_come_from_except ifelsestmt ::= testexpr c_stmts_opt jump_forward_else else_suite _come_froms # This handles the case where a "JUMP_ABSOLUTE" is part # of an inner if in c_stmts_opt ifelsestmt ::= testexpr c_stmts come_froms else_suite come_froms # ifelsestmt ::= testexpr c_stmts_opt jump_forward_else # pass _come_froms ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec ifelsestmtr ::= testexpr return_if_stmts returns ifelsestmtl ::= testexpr c_stmts_opt cf_jump_back else_suitel cf_jump_back ::= COME_FROM JUMP_BACK # FIXME: this feels like a hack. Is it just 1 or two # COME_FROMs? the parsed tree for this and even with just the # one COME_FROM for Python 2.7 seems to associate the # COME_FROM targets from the wrong places # this is nested inside a try_except tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_FINALLY suite_stmts_opt END_FINALLY except_handler ::= jmp_abs COME_FROM except_stmts _come_froms END_FINALLY except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts _come_froms END_FINALLY # FIXME: remove this except_handler ::= JUMP_FORWARD COME_FROM except_stmts come_froms END_FINALLY come_from_opt except_stmts ::= except_stmt+ except_stmt ::= except_cond1 except_suite come_from_opt except_stmt ::= except_cond2 except_suite come_from_opt except_stmt ::= except_cond2 except_suite_finalize except_stmt ::= except ## FIXME: what's except_pop_except? except_stmt ::= except_pop_except # Python3 introduced POP_EXCEPT except_suite ::= c_stmts_opt POP_EXCEPT jump_except jump_except ::= JUMP_ABSOLUTE jump_except ::= JUMP_BACK jump_except ::= JUMP_FORWARD jump_except ::= CONTINUE # This is used in Python 3 in # "except ... as e" to remove 'e' after the c_stmts_opt finishes except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize END_FINALLY _jump except_var_finalize ::= POP_BLOCK POP_EXCEPT LOAD_CONST COME_FROM_FINALLY LOAD_CONST store del_stmt except_suite ::= returns except_cond1 ::= DUP_TOP expr COMPARE_OP jmp_false POP_TOP POP_TOP POP_TOP except_cond2 ::= DUP_TOP expr COMPARE_OP jmp_false POP_TOP store POP_TOP come_from_opt except ::= POP_TOP POP_TOP POP_TOP c_stmts_opt POP_EXCEPT _jump except ::= POP_TOP POP_TOP POP_TOP returns jmp_abs ::= JUMP_ABSOLUTE jmp_abs ::= JUMP_BACK """ def p_misc3(self, args): """ except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts come_froms END_FINALLY for_block ::= l_stmts_opt COME_FROM_LOOP JUMP_BACK for_block ::= l_stmts for_block ::= l_stmts JUMP_BACK iflaststmtl ::= testexpr c_stmts """ def p_come_from3(self, args): """ opt_come_from_except ::= COME_FROM_EXCEPT opt_come_from_except ::= _come_froms opt_come_from_except ::= come_from_except_clauses come_from_except_clauses ::= COME_FROM_EXCEPT_CLAUSE+ """ def p_jump3(self, args): """ jmp_false ::= POP_JUMP_IF_FALSE jmp_true ::= POP_JUMP_IF_TRUE # FIXME: Common with 2.7 ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond jitop_come_from_expr ::= JUMP_IF_TRUE_OR_POP come_froms expr jifop_come_from ::= JUMP_IF_FALSE_OR_POP come_froms expr_jitop ::= expr JUMP_IF_TRUE_OR_POP or ::= and jitop_come_from_expr COME_FROM or ::= expr_jitop expr COME_FROM or ::= expr_jit expr COME_FROM or ::= expr_pjit expr POP_JUMP_IF_FALSE COME_FROM testfalse_not_or ::= expr jmp_false expr jmp_false COME_FROM testfalse_not_and ::= and jmp_true come_froms testfalse_not_and ::= expr jmp_false expr jmp_true COME_FROM testfalse ::= testfalse_not_or testfalse ::= testfalse_not_and testfalse ::= or jmp_false COME_FROM iflaststmtl ::= testexprl c_stmts JUMP_BACK iflaststmtl ::= testexprl c_stmts JUMP_BACK COME_FROM_LOOP iflaststmtl ::= testexprl c_stmts JUMP_BACK POP_BLOCK testexprl ::= testfalsel testfalsel ::= expr jmp_true or ::= expr_jt expr and ::= expr JUMP_IF_FALSE_OR_POP expr come_from_opt and ::= expr jifop_come_from expr expr_pjit_come_from ::= expr POP_JUMP_IF_TRUE COME_FROM or ::= expr_pjit_come_from expr ## Note that "jmp_false" is what we check on in the "and" reduce rule. and ::= expr jmp_false expr COME_FROM or ::= expr_jt expr COME_FROM # compare_chained1 is used exclusively in chained_compare compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP compare_chained1 COME_FROM compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP compare_chained2 COME_FROM """ def p_stmt3(self, args): """ stmt ::= if_exp_lambda stmt ::= if_exp_not_lambda # If statement inside a loop: stmt ::= ifstmtl if_exp_lambda ::= expr jmp_false expr return_if_lambda return_stmt_lambda LAMBDA_MARKER if_exp_not_lambda ::= expr jmp_true expr return_if_lambda return_stmt_lambda LAMBDA_MARKER return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA return_if_lambda ::= RETURN_END_IF_LAMBDA stmt ::= return_closure return_closure ::= LOAD_CLOSURE RETURN_VALUE RETURN_LAST stmt ::= whileTruestmt ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_froms ifelsestmtl ::= testexpr c_stmts_opt jump_forward_else else_suitec ifstmtl ::= testexpr _ifstmts_jumpl _ifstmts_jumpl ::= c_stmts JUMP_BACK _ifstmts_jumpl ::= _ifstmts_jump # The following can happen when the jump offset is large and # Python is looking to do a small jump to a larger jump to get # around the problem that the offset can't be represented in # the size allowed for the jump offset. This is more likely to # happen in wordcode Python since the offset range has been # reduced. FIXME: We should add a reduction check that the # final jump goes to another jump. _ifstmts_jumpl ::= COME_FROM c_stmts JUMP_BACK _ifstmts_jumpl ::= COME_FROM c_stmts JUMP_FORWARD """ def p_loop_stmt3(self, args): """ setup_loop ::= SETUP_LOOP _come_froms for ::= setup_loop expr get_for_iter store for_block POP_BLOCK for ::= setup_loop expr get_for_iter store for_block POP_BLOCK COME_FROM_LOOP forelsestmt ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suite COME_FROM_LOOP forelselaststmt ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec COME_FROM_LOOP forelselaststmtl ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitel COME_FROM_LOOP whilestmt ::= setup_loop testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP whilestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP whilestmt ::= setup_loop testexpr returns POP_BLOCK COME_FROM_LOOP # We can be missing a COME_FROM_LOOP if the "while" statement is nested inside an if/else # so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if" # This is undoubtedly some sort of JUMP optimization going on. whilestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK come_froms POP_BLOCK while1elsestmt ::= setup_loop l_stmts JUMP_BACK else_suitel whileelsestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK POP_BLOCK else_suitel COME_FROM_LOOP whileTruestmt ::= setup_loop l_stmts_opt JUMP_BACK POP_BLOCK _come_froms # FIXME: Python 3.? starts adding branch optimization? Put this starting there. while1stmt ::= setup_loop l_stmts COME_FROM_LOOP while1stmt ::= setup_loop l_stmts COME_FROM_LOOP JUMP_BACK POP_BLOCK COME_FROM_LOOP while1stmt ::= setup_loop l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP while1elsestmt ::= setup_loop l_stmts JUMP_BACK else_suite COME_FROM_LOOP # FIXME: investigate - can code really produce a NOP? for ::= setup_loop expr get_for_iter store for_block POP_BLOCK NOP COME_FROM_LOOP """ def p_36misc(self, args): """ sstmt ::= sstmt RETURN_LAST # 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE return_closure ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST for_block ::= l_stmts_opt come_from_loops JUMP_BACK come_from_loops ::= COME_FROM_LOOP* whilestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP whilestmt ::= setup_loop testexpr l_stmts_opt come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where # RETURN_VALUE is meant. Specifcally this can happen in # ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt return ::= ret_expr RETURN_END_IF return ::= ret_expr RETURN_VALUE COME_FROM return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA COME_FROM # A COME_FROM is dropped off because of JUMP-to-JUMP optimization and ::= expr jmp_false expr and ::= expr jmp_false expr jmp_false jf_cf ::= JUMP_FORWARD COME_FROM cf_jf_else ::= come_froms JUMP_FORWARD ELSE if_exp ::= expr jmp_false expr jf_cf expr COME_FROM async_for_stmt ::= setup_loop expr GET_AITER LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM store POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK JUMP_ABSOLUTE END_FINALLY COME_FROM for_block POP_BLOCK COME_FROM_LOOP # Adds a COME_FROM_ASYNC_WITH over 3.5 # FIXME: remove corresponding rule for 3.5? except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM jb_cfs ::= come_from_opt JUMP_BACK come_froms ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel # In 3.6+, A sequence of statements ending in a RETURN can cause # JUMP_FORWARD END_FINALLY to be omitted from try middle except_return ::= POP_TOP POP_TOP POP_TOP returns except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_return # Try middle following a returns except_handler36 ::= COME_FROM_EXCEPT except_stmts END_FINALLY stmt ::= try_except36 try_except36 ::= SETUP_EXCEPT returns except_handler36 opt_come_from_except try_except36 ::= SETUP_EXCEPT suite_stmts try_except36 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK except_handler36 come_from_opt # 3.6 omits END_FINALLY sometimes except_handler36 ::= COME_FROM_EXCEPT except_stmts except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts stmt ::= tryfinally36 tryfinally36 ::= SETUP_FINALLY returns COME_FROM_FINALLY suite_stmts tryfinally36 ::= SETUP_FINALLY returns COME_FROM_FINALLY suite_stmts_opt END_FINALLY except_suite_finalize ::= SETUP_FINALLY returns COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump stmt ::= tryfinally_return_stmt tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_FINALLY compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD """ def p_37misc(self, args): """ # long except clauses in a loop can sometimes cause a JUMP_BACK to turn into a # JUMP_FORWARD to a JUMP_BACK. And when this happens there is an additional # ELSE added to the except_suite. With better flow control perhaps we can # sort this out better. except_suite ::= c_stmts_opt POP_EXCEPT jump_except ELSE # FIXME: the below is to work around test_grammar expecting a "call" to be # on the LHS because it is also somewhere on in a rule. call ::= expr CALL_METHOD_0 """ def customize_grammar_rules(self, tokens, customize): super(Python37Parser, self).customize_grammar_rules(tokens, customize) self.check_reduce["call_kw"] = "AST" for i, token in enumerate(tokens): opname = token.kind if opname == "LOAD_ASSERT": if "PyPy" in customize: rules_str = """ stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM """ self.add_unique_doc_rules(rules_str, customize) elif opname == "FORMAT_VALUE": rules_str = """ expr ::= formatted_value1 formatted_value1 ::= expr FORMAT_VALUE """ self.add_unique_doc_rules(rules_str, customize) elif opname == "FORMAT_VALUE_ATTR": rules_str = """ expr ::= formatted_value2 formatted_value2 ::= expr expr FORMAT_VALUE_ATTR """ self.add_unique_doc_rules(rules_str, customize) elif opname == "MAKE_FUNCTION_8": if "LOAD_DICTCOMP" in self.seen_ops: # Is there something general going on here? rule = """ dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_8 expr GET_ITER CALL_FUNCTION_1 """ self.addRule(rule, nop_func) elif "LOAD_SETCOMP" in self.seen_ops: rule = """ set_comp ::= load_closure LOAD_SETCOMP LOAD_STR MAKE_FUNCTION_8 expr GET_ITER CALL_FUNCTION_1 """ self.addRule(rule, nop_func) elif opname == "BEFORE_ASYNC_WITH": rules_str = """ stmt ::= async_with_stmt SETUP_ASYNC_WITH async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH async_with_post ::= COME_FROM_ASYNC_WITH WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM WITH_CLEANUP_FINISH END_FINALLY stmt ::= async_with_as_stmt async_with_as_stmt ::= expr async_with_pre store suite_stmts_opt POP_BLOCK LOAD_CONST async_with_post async_with_stmt ::= expr async_with_pre POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST async_with_post async_with_stmt ::= expr async_with_pre POP_TOP suite_stmts_opt async_with_post """ self.addRule(rules_str, nop_func) elif opname.startswith("BUILD_STRING"): v = token.attr rules_str = """ expr ::= joined_str joined_str ::= %sBUILD_STRING_%d """ % ( "expr " * v, v, ) self.add_unique_doc_rules(rules_str, customize) if "FORMAT_VALUE_ATTR" in self.seen_ops: rules_str = """ formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING expr ::= formatted_value_attr """ self.add_unique_doc_rules(rules_str, customize) elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"): v = token.attr rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname) self.addRule(rule, nop_func) elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"): v = token.attr rule = ( "build_tuple_unpack_with_call ::= " + "expr1024 " * int(v // 1024) + "expr32 " * int((v // 32) % 32) + "expr " * (v % 32) + opname ) self.addRule(rule, nop_func) rule = "starred ::= %s %s" % ("expr " * v, opname) self.addRule(rule, nop_func) elif opname == "SETUP_WITH": rules_str = """ with ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY # Removes POP_BLOCK LOAD_CONST from 3.6- withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY """ if self.version < 3.8: rules_str += """ with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY """ else: rules_str += """ with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK BEGIN_FINALLY COME_FROM_WITH WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY """ self.addRule(rules_str, nop_func) pass pass def custom_classfunc_rule(self, opname, token, customize, next_token): args_pos, args_kw = self.get_pos_kw(token) # Additional exprs for * and ** args: # 0 if neither # 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW # 2 for * and ** args (CALL_FUNCTION_VAR_KW). # Yes, this computation based on instruction name is a little bit hoaky. nak = (len(opname) - len("CALL_FUNCTION")) // 3 uniq_param = args_kw + args_pos if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops): rule = ( "async_call ::= expr " + ("pos_arg " * args_pos) + ("kwarg " * args_kw) + "expr " * nak + token.kind + " GET_AWAITABLE LOAD_CONST YIELD_FROM" ) self.add_unique_rule(rule, token.kind, uniq_param, customize) self.add_unique_rule( "expr ::= async_call", token.kind, uniq_param, customize ) if opname.startswith("CALL_FUNCTION_KW"): self.addRule("expr ::= call_kw36", nop_func) values = "expr " * token.attr rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals()) self.add_unique_rule(rule, token.kind, token.attr, customize) elif opname == "CALL_FUNCTION_EX_KW": # Note: this doesn't exist in 3.7 and later self.addRule( """expr ::= call_ex_kw4 call_ex_kw4 ::= expr expr expr CALL_FUNCTION_EX_KW """, nop_func, ) if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames: self.addRule( """expr ::= call_ex_kw call_ex_kw ::= expr expr build_map_unpack_with_call CALL_FUNCTION_EX_KW """, nop_func, ) if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames: # FIXME: should this be parameterized by EX value? self.addRule( """expr ::= call_ex_kw3 call_ex_kw3 ::= expr build_tuple_unpack_with_call expr CALL_FUNCTION_EX_KW """, nop_func, ) if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames: # FIXME: should this be parameterized by EX value? self.addRule( """expr ::= call_ex_kw2 call_ex_kw2 ::= expr build_tuple_unpack_with_call build_map_unpack_with_call CALL_FUNCTION_EX_KW """, nop_func, ) elif opname == "CALL_FUNCTION_EX": self.addRule( """ expr ::= call_ex starred ::= expr call_ex ::= expr starred CALL_FUNCTION_EX """, nop_func, ) if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops: self.addRule( """ expr ::= call_ex_kw call_ex_kw ::= expr expr build_map_unpack_with_call CALL_FUNCTION_EX """, nop_func, ) if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops: self.addRule( """ expr ::= call_ex_kw3 call_ex_kw3 ::= expr build_tuple_unpack_with_call %s CALL_FUNCTION_EX """ % "expr " * token.attr, nop_func, ) pass # FIXME: Is this right? self.addRule( """ expr ::= call_ex_kw4 call_ex_kw4 ::= expr expr expr CALL_FUNCTION_EX """, nop_func, ) pass else: super(Python37Parser, self).custom_classfunc_rule( opname, token, customize, next_token ) def reduce_is_invalid(self, rule, ast, tokens, first, last): invalid = super(Python37Parser, self).reduce_is_invalid( rule, ast, tokens, first, last ) if invalid: return invalid if rule[0] == "call_kw": # Make sure we don't derive call_kw nt = ast[0] while not isinstance(nt, Token): if nt[0] == "call_kw": return True nt = nt[0] pass pass return False def info(args): # Check grammar p = Python37Parser() if len(args) > 0: arg = args[0] if arg == "3.7": from uncompyle6.parser.parse37 import Python37Parser p = Python37Parser() elif arg == "3.8": from uncompyle6.parser.parse38 import Python38Parser p = Python38Parser() else: raise RuntimeError("Only 3.7 and 3.8 supported") p.check_grammar() if len(sys.argv) > 1 and sys.argv[1] == "dump": print("-" * 50) p.dump_grammar() class Python37ParserSingle(Python37Parser, PythonParserSingle): pass if __name__ == "__main__": # Check grammar # FIXME: DRY this with other parseXX.py routines p = Python37Parser() p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 3.7: lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union( set( """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME LAMBDA_MARKER RETURN_LAST """.split() ) ) remain_tokens = set(tokens) - opcode_set import re remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens]) remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens]) remain_tokens = set(remain_tokens) - opcode_set print(remain_tokens) import sys if len(sys.argv) > 1: from spark_parser.spark import rule2str for rule in sorted(p.rule2name.items()): print(rule2str(rule[0]))