Merge pull request #199 from structuredllm/fix_iter

Fix IterGen issue

Author: shubhamugare

syncode/parsers/__init__.py

@@ -22,7 +22,7 @@ def create_parser(
         cache_filename = parser_cache_dir + f'{grammar}_{parser}_{grammar.hash()}_parser.pkl'
         os.makedirs(os.path.dirname(parser_cache_dir), exist_ok=True)
 
-        if grammar.name == 'python':
+        if grammar.name == 'python' and not use_symbol_pos_map:
             indenter = PythonIndenter()
 
         base_parser = create_base_parser(grammar, parser, indenter, cache_filename)

syncode/parsers/incremental_parser.py

@@ -136,12 +136,12 @@ def get_acceptable_next_terminals(self, partial_code) -> ParseResult:
         Returns the set of acceptable terminals at the current partial code position.
         """
         # Stores the sequence of tokens that the parser has seen in the order  
-        interactive = self.interactive
         lexer_tokens, lexing_incomplete = self._lex_code(partial_code)
-        self.next_ac_terminals = self._accepts(interactive)
+        self.next_ac_terminals = self._accepts(self.interactive)
 
         # Restore the previous state of the parser
         self._restore_recent_parser_state(lexer_tokens)
+        interactive = self.interactive
 
         # Parse the tokens
         self.time_accepts = 0

syncode/parsers/itergen_parser.py

@@ -232,7 +232,7 @@ def _store_parser_state(
         # Store parsed tokens, parser state, terminal sets, indent levels, dedent queue, and symbol pos map
         self.cur_pos_to_parser_state[key] = (
             copy.deepcopy(self.parsed_lexer_tokens), 
-            parser_state, 
+            parser_state.copy(),
             cur_ac_terminals, 
             next_ac_terminals, 
             indent_levels, 
@@ -338,12 +338,12 @@ def get_acceptable_next_terminals(
         Now handles updating the symbol position map during parsing.
         """
         # Get lexer tokens and initialize state
-        interactive = self.interactive
         lexer_tokens, lexing_incomplete = self._lex_code(partial_code)
-        self.next_ac_terminals = self._accepts(interactive)
+        self.next_ac_terminals = self._accepts(self.interactive)
 
         # Restore the previous state of the parser
         self._restore_recent_parser_state(lexer_tokens, symbol_pos_map=symbol_pos_map)
+        interactive = self.interactive
 
         # Update symbol position map for terminals if provided
         if symbol_pos_map is not None: