X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=scripts%2Fjinja2%2Fparser.py;h=e036db08b66cc25a09c7db04f539621d5029d127;hb=3d61544bd177bdb8c880ee87f5cdad2b7d1606e5;hp=d6f1b3647d96e909033eb358294f1832c07a400e;hpb=279822e9e2a4f26deaf67b5fe8423b27837a75d7;p=libfirm

diff --git a/scripts/jinja2/parser.py b/scripts/jinja2/parser.py
index d6f1b3647..e036db08b 100644
--- a/scripts/jinja2/parser.py
+++ b/scripts/jinja2/parser.py
@@ -5,13 +5,16 @@
 
     Implements the template parser.
 
-    :copyright: 2008 by Armin Ronacher.
+    :copyright: (c) 2010 by the Jinja Team.
     :license: BSD, see LICENSE for more details.
 """
 from jinja2 import nodes
 from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
+from jinja2.utils import next
+from jinja2.lexer import describe_token, describe_token_expr
 
 
+#: statements that callinto
 _statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
                                  'macro', 'include', 'from', 'import',
                                  'set'])
@@ -31,10 +34,12 @@ class Parser(object):
         self.filename = filename
         self.closed = False
         self.extensions = {}
-        for extension in environment.extensions.itervalues():
+        for extension in environment.iter_extensions():
             for tag in extension.tags:
                 self.extensions[tag] = extension.parse
         self._last_identifier = 0
+        self._tag_stack = []
+        self._end_token_stack = []
 
     def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
         """Convenience method that raises `exc` with the message, passed
@@ -45,6 +50,51 @@ class Parser(object):
             lineno = self.stream.current.lineno
         raise exc(msg, lineno, self.name, self.filename)
 
+    def _fail_ut_eof(self, name, end_token_stack, lineno):
+        expected = []
+        for exprs in end_token_stack:
+            expected.extend(map(describe_token_expr, exprs))
+        if end_token_stack:
+            currently_looking = ' or '.join(
+                "'%s'" % describe_token_expr(expr)
+                for expr in end_token_stack[-1])
+        else:
+            currently_looking = None
+
+        if name is None:
+            message = ['Unexpected end of template.']
+        else:
+            message = ['Encountered unknown tag \'%s\'.' % name]
+
+        if currently_looking:
+            if name is not None and name in expected:
+                message.append('You probably made a nesting mistake. Jinja '
+                               'is expecting this tag, but currently looking '
+                               'for %s.' % currently_looking)
+            else:
+                message.append('Jinja was looking for the following tags: '
+                               '%s.' % currently_looking)
+
+        if self._tag_stack:
+            message.append('The innermost block that needs to be '
+                           'closed is \'%s\'.' % self._tag_stack[-1])
+
+        self.fail(' '.join(message), lineno)
+
+    def fail_unknown_tag(self, name, lineno=None):
+        """Called if the parser encounters an unknown tag.  Tries to fail
+        with a human readable error message that could help to identify
+        the problem.
+        """
+        return self._fail_ut_eof(name, self._end_token_stack, lineno)
+
+    def fail_eof(self, end_tokens=None, lineno=None):
+        """Like fail_unknown_tag but for end of template situations."""
+        stack = list(self._end_token_stack)
+        if end_tokens is not None:
+            stack.append(end_tokens)
+        return self._fail_ut_eof(None, stack, lineno)
+
     def is_tuple_end(self, extra_end_rules=None):
         """Are we at the end of a tuple?"""
         if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
@@ -63,18 +113,30 @@ class Parser(object):
     def parse_statement(self):
         """Parse a single statement."""
         token = self.stream.current
-        if token.type is not 'name':
+        if token.type != 'name':
             self.fail('tag name expected', token.lineno)
-        if token.value in _statement_keywords:
-            return getattr(self, 'parse_' + self.stream.current.value)()
-        if token.value == 'call':
-            return self.parse_call_block()
-        if token.value == 'filter':
-            return self.parse_filter_block()
-        ext = self.extensions.get(token.value)
-        if ext is not None:
-            return ext(self)
-        self.fail('unknown tag %r' % token.value, token.lineno)
+        self._tag_stack.append(token.value)
+        pop_tag = True
+        try:
+            if token.value in _statement_keywords:
+                return getattr(self, 'parse_' + self.stream.current.value)()
+            if token.value == 'call':
+                return self.parse_call_block()
+            if token.value == 'filter':
+                return self.parse_filter_block()
+            ext = self.extensions.get(token.value)
+            if ext is not None:
+                return ext(self)
+
+            # did not work out, remove the token we pushed by accident
+            # from the stack so that the unknown tag fail function can
+            # produce a proper error message.
+            self._tag_stack.pop()
+            pop_tag = False
+            self.fail_unknown_tag(token.value, token.lineno)
+        finally:
+            if pop_tag:
+                self._tag_stack.pop()
 
     def parse_statements(self, end_tokens, drop_needle=False):
         """Parse multiple statements into a list until one of the end tokens
@@ -94,13 +156,18 @@ class Parser(object):
         self.stream.expect('block_end')
         result = self.subparse(end_tokens)
 
+        # we reached the end of the template too early, the subparser
+        # does not check for this, so we do that now
+        if self.stream.current.type == 'eof':
+            self.fail_eof(end_tokens)
+
         if drop_needle:
-            self.stream.next()
+            next(self.stream)
         return result
 
     def parse_set(self):
         """Parse an assign statement."""
-        lineno = self.stream.next().lineno
+        lineno = next(self.stream).lineno
         target = self.parse_assign_target()
         self.stream.expect('assign')
         expr = self.parse_tuple()
@@ -118,7 +185,7 @@ class Parser(object):
             test = self.parse_expression()
         recursive = self.stream.skip_if('name:recursive')
         body = self.parse_statements(('name:endfor', 'name:else'))
-        if self.stream.next().value == 'endfor':
+        if next(self.stream).value == 'endfor':
             else_ = []
         else:
             else_ = self.parse_statements(('name:endfor',), drop_needle=True)
@@ -132,7 +199,7 @@ class Parser(object):
             node.test = self.parse_tuple(with_condexpr=False)
             node.body = self.parse_statements(('name:elif', 'name:else',
                                                'name:endif'))
-            token = self.stream.next()
+            token = next(self.stream)
             if token.test('name:elif'):
                 new_node = nodes.If(lineno=self.stream.current.lineno)
                 node.else_ = [new_node]
@@ -147,40 +214,56 @@ class Parser(object):
         return result
 
     def parse_block(self):
-        node = nodes.Block(lineno=self.stream.next().lineno)
+        node = nodes.Block(lineno=next(self.stream).lineno)
         node.name = self.stream.expect('name').value
+        node.scoped = self.stream.skip_if('name:scoped')
+
+        # common problem people encounter when switching from django
+        # to jinja.  we do not support hyphens in block names, so let's
+        # raise a nicer error message in that case.
+        if self.stream.current.type == 'sub':
+            self.fail('Block names in Jinja have to be valid Python '
+                      'identifiers and may not contain hypens, use an '
+                      'underscore instead.')
+
         node.body = self.parse_statements(('name:endblock',), drop_needle=True)
         self.stream.skip_if('name:' + node.name)
         return node
 
     def parse_extends(self):
-        node = nodes.Extends(lineno=self.stream.next().lineno)
+        node = nodes.Extends(lineno=next(self.stream).lineno)
         node.template = self.parse_expression()
         return node
 
     def parse_import_context(self, node, default):
         if self.stream.current.test_any('name:with', 'name:without') and \
            self.stream.look().test('name:context'):
-            node.with_context = self.stream.next().value == 'with'
+            node.with_context = next(self.stream).value == 'with'
             self.stream.skip()
         else:
             node.with_context = default
         return node
 
     def parse_include(self):
-        node = nodes.Include(lineno=self.stream.next().lineno)
+        node = nodes.Include(lineno=next(self.stream).lineno)
         node.template = self.parse_expression()
+        if self.stream.current.test('name:ignore') and \
+           self.stream.look().test('name:missing'):
+            node.ignore_missing = True
+            self.stream.skip(2)
+        else:
+            node.ignore_missing = False
         return self.parse_import_context(node, True)
 
     def parse_import(self):
-        node = nodes.Import(lineno=self.stream.next().lineno)
+        node = nodes.Import(lineno=next(self.stream).lineno)
         node.template = self.parse_expression()
         self.stream.expect('name:as')
         node.target = self.parse_assign_target(name_only=True).name
         return self.parse_import_context(node, False)
 
     def parse_from(self):
-        node = nodes.FromImport(lineno=self.stream.next().lineno)
+        node = nodes.FromImport(lineno=next(self.stream).lineno)
         node.template = self.parse_expression()
         self.stream.expect('name:import')
         node.names = []
@@ -188,7 +271,7 @@ class Parser(object):
         def parse_context():
             if self.stream.current.value in ('with', 'without') and \
                self.stream.look().test('name:context'):
-                node.with_context = self.stream.next().value == 'with'
+                node.with_context = next(self.stream).value == 'with'
                 self.stream.skip()
                 return True
             return False
@@ -196,7 +279,7 @@ class Parser(object):
         while 1:
             if node.names:
                 self.stream.expect('comma')
-            if self.stream.current.type is 'name':
+            if self.stream.current.type == 'name':
                 if parse_context():
                     break
                 target = self.parse_assign_target(name_only=True)
@@ -209,7 +292,7 @@ class Parser(object):
                     node.names.append((target.name, alias.name))
                 else:
                     node.names.append(target.name)
-                if parse_context() or self.stream.current.type is not 'comma':
+                if parse_context() or self.stream.current.type != 'comma':
                     break
             else:
                 break
@@ -222,7 +305,7 @@ class Parser(object):
         node.args = args = []
         node.defaults = defaults = []
         self.stream.expect('lparen')
-        while self.stream.current.type is not 'rparen':
+        while self.stream.current.type != 'rparen':
             if args:
                 self.stream.expect('comma')
             arg = self.parse_assign_target(name_only=True)
@@ -233,8 +316,8 @@ class Parser(object):
         self.stream.expect('rparen')
 
     def parse_call_block(self):
-        node = nodes.CallBlock(lineno=self.stream.next().lineno)
-        if self.stream.current.type is 'lparen':
+        node = nodes.CallBlock(lineno=next(self.stream).lineno)
+        if self.stream.current.type == 'lparen':
             self.parse_signature(node)
         else:
             node.args = []
@@ -247,14 +330,14 @@ class Parser(object):
         return node
 
     def parse_filter_block(self):
-        node = nodes.FilterBlock(lineno=self.stream.next().lineno)
+        node = nodes.FilterBlock(lineno=next(self.stream).lineno)
         node.filter = self.parse_filter(None, start_inline=True)
         node.body = self.parse_statements(('name:endfilter',),
                                           drop_needle=True)
         return node
 
     def parse_macro(self):
-        node = nodes.Macro(lineno=self.stream.next().lineno)
+        node = nodes.Macro(lineno=next(self.stream).lineno)
         node.name = self.parse_assign_target(name_only=True).name
         self.parse_signature(node)
         node.body = self.parse_statements(('name:endmacro',),
@@ -262,9 +345,9 @@ class Parser(object):
         return node
 
     def parse_print(self):
-        node = nodes.Output(lineno=self.stream.next().lineno)
+        node = nodes.Output(lineno=next(self.stream).lineno)
         node.nodes = []
-        while self.stream.current.type is not 'block_end':
+        while self.stream.current.type != 'block_end':
             if node.nodes:
                 self.stream.expect('comma')
             node.nodes.append(self.parse_expression())
@@ -287,7 +370,7 @@ class Parser(object):
                 target = self.parse_tuple(simplified=True,
                                           extra_end_rules=extra_end_rules)
             else:
-                target = self.parse_primary(with_postfix=False)
+                target = self.parse_primary()
             target.set_ctx('store')
         if not target.can_assign():
             self.fail('can\'t assign to %r' % target.__class__.
@@ -327,13 +410,19 @@ class Parser(object):
 
     def parse_and(self):
         lineno = self.stream.current.lineno
-        left = self.parse_compare()
+        left = self.parse_not()
         while self.stream.skip_if('name:and'):
-            right = self.parse_compare()
+            right = self.parse_not()
             left = nodes.And(left, right, lineno=lineno)
             lineno = self.stream.current.lineno
         return left
 
+    def parse_not(self):
+        if self.stream.current.test('name:not'):
+            lineno = next(self.stream).lineno
+            return nodes.Not(self.parse_not(), lineno=lineno)
+        return self.parse_compare()
+
     def parse_compare(self):
         lineno = self.stream.current.lineno
         expr = self.parse_add()
@@ -341,7 +430,7 @@ class Parser(object):
         while 1:
             token_type = self.stream.current.type
             if token_type in _compare_operators:
-                self.stream.next()
+                next(self.stream)
                 ops.append(nodes.Operand(token_type, self.parse_add()))
             elif self.stream.skip_if('name:in'):
                 ops.append(nodes.Operand('in', self.parse_add()))
@@ -359,8 +448,8 @@ class Parser(object):
     def parse_add(self):
         lineno = self.stream.current.lineno
         left = self.parse_sub()
-        while self.stream.current.type is 'add':
-            self.stream.next()
+        while self.stream.current.type == 'add':
+            next(self.stream)
             right = self.parse_sub()
             left = nodes.Add(left, right, lineno=lineno)
             lineno = self.stream.current.lineno
@@ -369,8 +458,8 @@ class Parser(object):
     def parse_sub(self):
         lineno = self.stream.current.lineno
         left = self.parse_concat()
-        while self.stream.current.type is 'sub':
-            self.stream.next()
+        while self.stream.current.type == 'sub':
+            next(self.stream)
             right = self.parse_concat()
             left = nodes.Sub(left, right, lineno=lineno)
             lineno = self.stream.current.lineno
@@ -379,8 +468,8 @@ class Parser(object):
     def parse_concat(self):
         lineno = self.stream.current.lineno
         args = [self.parse_mul()]
-        while self.stream.current.type is 'tilde':
-            self.stream.next()
+        while self.stream.current.type == 'tilde':
+            next(self.stream)
             args.append(self.parse_mul())
         if len(args) == 1:
             return args[0]
@@ -389,8 +478,8 @@ class Parser(object):
     def parse_mul(self):
         lineno = self.stream.current.lineno
         left = self.parse_div()
-        while self.stream.current.type is 'mul':
-            self.stream.next()
+        while self.stream.current.type == 'mul':
+            next(self.stream)
             right = self.parse_div()
             left = nodes.Mul(left, right, lineno=lineno)
             lineno = self.stream.current.lineno
@@ -399,8 +488,8 @@ class Parser(object):
     def parse_div(self):
         lineno = self.stream.current.lineno
         left = self.parse_floordiv()
-        while self.stream.current.type is 'div':
-            self.stream.next()
+        while self.stream.current.type == 'div':
+            next(self.stream)
             right = self.parse_floordiv()
             left = nodes.Div(left, right, lineno=lineno)
             lineno = self.stream.current.lineno
@@ -409,8 +498,8 @@ class Parser(object):
     def parse_floordiv(self):
         lineno = self.stream.current.lineno
         left = self.parse_mod()
-        while self.stream.current.type is 'floordiv':
-            self.stream.next()
+        while self.stream.current.type == 'floordiv':
+            next(self.stream)
             right = self.parse_mod()
             left = nodes.FloorDiv(left, right, lineno=lineno)
             lineno = self.stream.current.lineno
@@ -419,8 +508,8 @@ class Parser(object):
     def parse_mod(self):
         lineno = self.stream.current.lineno
         left = self.parse_pow()
-        while self.stream.current.type is 'mod':
-            self.stream.next()
+        while self.stream.current.type == 'mod':
+            next(self.stream)
             right = self.parse_pow()
             left = nodes.Mod(left, right, lineno=lineno)
             lineno = self.stream.current.lineno
@@ -429,33 +518,32 @@ class Parser(object):
     def parse_pow(self):
         lineno = self.stream.current.lineno
         left = self.parse_unary()
-        while self.stream.current.type is 'pow':
-            self.stream.next()
+        while self.stream.current.type == 'pow':
+            next(self.stream)
             right = self.parse_unary()
             left = nodes.Pow(left, right, lineno=lineno)
             lineno = self.stream.current.lineno
         return left
 
-    def parse_unary(self):
+    def parse_unary(self, with_filter=True):
         token_type = self.stream.current.type
         lineno = self.stream.current.lineno
-        if token_type is 'name' and self.stream.current.value == 'not':
-            self.stream.next()
-            node = self.parse_unary()
-            return nodes.Not(node, lineno=lineno)
-        if token_type is 'sub':
-            self.stream.next()
-            node = self.parse_unary()
-            return nodes.Neg(node, lineno=lineno)
-        if token_type is 'add':
-            self.stream.next()
-            node = self.parse_unary()
-            return nodes.Pos(node, lineno=lineno)
-        return self.parse_primary()
-
-    def parse_primary(self, with_postfix=True):
+        if token_type == 'sub':
+            next(self.stream)
+            node = nodes.Neg(self.parse_unary(False), lineno=lineno)
+        elif token_type == 'add':
+            next(self.stream)
+            node = nodes.Pos(self.parse_unary(False), lineno=lineno)
+        else:
+            node = self.parse_primary()
+        node = self.parse_postfix(node)
+        if with_filter:
+            node = self.parse_filter_expr(node)
+        return node
+
+    def parse_primary(self):
         token = self.stream.current
-        if token.type is 'name':
+        if token.type == 'name':
             if token.value in ('true', 'false', 'True', 'False'):
                 node = nodes.Const(token.value in ('true', 'True'),
                                    lineno=token.lineno)
@@ -463,34 +551,32 @@ class Parser(object):
                 node = nodes.Const(None, lineno=token.lineno)
             else:
                 node = nodes.Name(token.value, 'load', lineno=token.lineno)
-            self.stream.next()
-        elif token.type is 'string':
-            self.stream.next()
+            next(self.stream)
+        elif token.type == 'string':
+            next(self.stream)
             buf = [token.value]
             lineno = token.lineno
-            while self.stream.current.type is 'string':
+            while self.stream.current.type == 'string':
                 buf.append(self.stream.current.value)
-                self.stream.next()
+                next(self.stream)
             node = nodes.Const(''.join(buf), lineno=lineno)
         elif token.type in ('integer', 'float'):
-            self.stream.next()
+            next(self.stream)
             node = nodes.Const(token.value, lineno=token.lineno)
-        elif token.type is 'lparen':
-            self.stream.next()
-            node = self.parse_tuple()
+        elif token.type == 'lparen':
+            next(self.stream)
+            node = self.parse_tuple(explicit_parentheses=True)
             self.stream.expect('rparen')
-        elif token.type is 'lbracket':
+        elif token.type == 'lbracket':
             node = self.parse_list()
-        elif token.type is 'lbrace':
+        elif token.type == 'lbrace':
             node = self.parse_dict()
         else:
-            self.fail("unexpected token '%s'" % (token,), token.lineno)
-        if with_postfix:
-            node = self.parse_postfix(node)
+            self.fail("unexpected '%s'" % describe_token(token), token.lineno)
         return node
 
     def parse_tuple(self, simplified=False, with_condexpr=True,
-                    extra_end_rules=None):
+                    extra_end_rules=None, explicit_parentheses=False):
         """Works like `parse_expression` but if multiple expressions are
         delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
         This method could also return a regular expression instead of a tuple
@@ -504,10 +590,14 @@ class Parser(object):
         an extra hint is needed that marks the end of a tuple.  For example
         for loops support tuples between `for` and `in`.  In that case the
         `extra_end_rules` is set to ``['name:in']``.
+
+        `explicit_parentheses` is true if the parsing was triggered by an
+        expression in parentheses.  This is used to figure out if an empty
+        tuple is a valid expression or not.
         """
         lineno = self.stream.current.lineno
         if simplified:
-            parse = lambda: self.parse_primary(with_postfix=False)
+            parse = self.parse_primary
         elif with_condexpr:
             parse = self.parse_expression
         else:
@@ -520,19 +610,30 @@ class Parser(object):
             if self.is_tuple_end(extra_end_rules):
                 break
             args.append(parse())
-            if self.stream.current.type is 'comma':
+            if self.stream.current.type == 'comma':
                 is_tuple = True
             else:
                 break
             lineno = self.stream.current.lineno
-        if not is_tuple and args:
-            return args[0]
+
+        if not is_tuple:
+            if args:
+                return args[0]
+
+            # if we don't have explicit parentheses, an empty tuple is
+            # not a valid expression.  This would mean nothing (literally
+            # nothing) in the spot of an expression would be an empty
+            # tuple.
+            if not explicit_parentheses:
+                self.fail('Expected an expression, got \'%s\'' %
+                          describe_token(self.stream.current))
+
         return nodes.Tuple(args, 'load', lineno=lineno)
 
     def parse_list(self):
         token = self.stream.expect('lbracket')
         items = []
-        while self.stream.current.type is not 'rbracket':
+        while self.stream.current.type != 'rbracket':
             if items:
                 self.stream.expect('comma')
             if self.stream.current.type == 'rbracket':
@@ -544,7 +645,7 @@ class Parser(object):
     def parse_dict(self):
         token = self.stream.expect('lbrace')
         items = []
-        while self.stream.current.type is not 'rbrace':
+        while self.stream.current.type != 'rbrace':
             if items:
                 self.stream.expect('comma')
             if self.stream.current.type == 'rbrace':
@@ -559,34 +660,47 @@ class Parser(object):
     def parse_postfix(self, node):
         while 1:
             token_type = self.stream.current.type
-            if token_type is 'dot' or token_type is 'lbracket':
+            if token_type == 'dot' or token_type == 'lbracket':
                 node = self.parse_subscript(node)
-            elif token_type is 'lparen':
+            # calls are valid both after postfix expressions (getattr
+            # and getitem) as well as filters and tests
+            elif token_type == 'lparen':
                 node = self.parse_call(node)
-            elif token_type is 'pipe':
+            else:
+                break
+        return node
+
+    def parse_filter_expr(self, node):
+        while 1:
+            token_type = self.stream.current.type
+            if token_type == 'pipe':
                 node = self.parse_filter(node)
-            elif token_type is 'name' and self.stream.current.value == 'is':
+            elif token_type == 'name' and self.stream.current.value == 'is':
                 node = self.parse_test(node)
+            # calls are valid both after postfix expressions (getattr
+            # and getitem) as well as filters and tests
+            elif token_type == 'lparen':
+                node = self.parse_call(node)
             else:
                 break
         return node
 
     def parse_subscript(self, node):
-        token = self.stream.next()
-        if token.type is 'dot':
+        token = next(self.stream)
+        if token.type == 'dot':
             attr_token = self.stream.current
-            self.stream.next()
-            if attr_token.type is 'name':
+            next(self.stream)
+            if attr_token.type == 'name':
                 return nodes.Getattr(node, attr_token.value, 'load',
                                      lineno=token.lineno)
-            elif attr_token.type is not 'integer':
+            elif attr_token.type != 'integer':
                 self.fail('expected name or number', attr_token.lineno)
             arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
-        if token.type is 'lbracket':
+        if token.type == 'lbracket':
             priority_on_attribute = False
             args = []
-            while self.stream.current.type is not 'rbracket':
+            while self.stream.current.type != 'rbracket':
                 if args:
                     self.stream.expect('comma')
                 args.append(self.parse_subscribed())
@@ -594,32 +708,32 @@ class Parser(object):
             if len(args) == 1:
                 arg = args[0]
             else:
-                arg = nodes.Tuple(args, self.lineno, self.filename)
+                arg = nodes.Tuple(args, 'load', lineno=token.lineno)
             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
         self.fail('expected subscript expression', self.lineno)
 
     def parse_subscribed(self):
         lineno = self.stream.current.lineno
 
-        if self.stream.current.type is 'colon':
-            self.stream.next()
+        if self.stream.current.type == 'colon':
+            next(self.stream)
             args = [None]
         else:
             node = self.parse_expression()
-            if self.stream.current.type is not 'colon':
+            if self.stream.current.type != 'colon':
                 return node
-            self.stream.next()
+            next(self.stream)
             args = [node]
 
-        if self.stream.current.type is 'colon':
+        if self.stream.current.type == 'colon':
             args.append(None)
         elif self.stream.current.type not in ('rbracket', 'comma'):
             args.append(self.parse_expression())
         else:
             args.append(None)
 
-        if self.stream.current.type is 'colon':
-            self.stream.next()
+        if self.stream.current.type == 'colon':
+            next(self.stream)
             if self.stream.current.type not in ('rbracket', 'comma'):
                 args.append(self.parse_expression())
             else:
@@ -641,24 +755,24 @@ class Parser(object):
                 self.fail('invalid syntax for function call expression',
                           token.lineno)
 
-        while self.stream.current.type is not 'rparen':
+        while self.stream.current.type != 'rparen':
             if require_comma:
                 self.stream.expect('comma')
                 # support for trailing comma
-                if self.stream.current.type is 'rparen':
+                if self.stream.current.type == 'rparen':
                     break
-            if self.stream.current.type is 'mul':
+            if self.stream.current.type == 'mul':
                 ensure(dyn_args is None and dyn_kwargs is None)
-                self.stream.next()
+                next(self.stream)
                 dyn_args = self.parse_expression()
-            elif self.stream.current.type is 'pow':
+            elif self.stream.current.type == 'pow':
                 ensure(dyn_kwargs is None)
-                self.stream.next()
+                next(self.stream)
                 dyn_kwargs = self.parse_expression()
             else:
                 ensure(dyn_args is None and dyn_kwargs is None)
-                if self.stream.current.type is 'name' and \
-                    self.stream.look().type is 'assign':
+                if self.stream.current.type == 'name' and \
+                    self.stream.look().type == 'assign':
                     key = self.stream.current.value
                     self.stream.skip(2)
                     value = self.parse_expression()
@@ -679,13 +793,13 @@ class Parser(object):
     def parse_filter(self, node, start_inline=False):
         while self.stream.current.type == 'pipe' or start_inline:
             if not start_inline:
-                self.stream.next()
+                next(self.stream)
             token = self.stream.expect('name')
             name = token.value
-            while self.stream.current.type is 'dot':
-                self.stream.next()
+            while self.stream.current.type == 'dot':
+                next(self.stream)
                 name += '.' + self.stream.expect('name').value
-            if self.stream.current.type is 'lparen':
+            if self.stream.current.type == 'lparen':
                 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
             else:
                 args = []
@@ -697,19 +811,19 @@ class Parser(object):
         return node
 
     def parse_test(self, node):
-        token = self.stream.next()
+        token = next(self.stream)
         if self.stream.current.test('name:not'):
-            self.stream.next()
+            next(self.stream)
             negated = True
         else:
             negated = False
         name = self.stream.expect('name').value
-        while self.stream.current.type is 'dot':
-            self.stream.next()
+        while self.stream.current.type == 'dot':
+            next(self.stream)
             name += '.' + self.stream.expect('name').value
         dyn_args = dyn_kwargs = None
         kwargs = []
-        if self.stream.current.type is 'lparen':
+        if self.stream.current.type == 'lparen':
             args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
         elif self.stream.current.type in ('name', 'string', 'integer',
                                           'float', 'lparen', 'lbracket',
@@ -732,39 +846,47 @@ class Parser(object):
         data_buffer = []
         add_data = data_buffer.append
 
+        if end_tokens is not None:
+            self._end_token_stack.append(end_tokens)
+
         def flush_data():
             if data_buffer:
                 lineno = data_buffer[0].lineno
                 body.append(nodes.Output(data_buffer[:], lineno=lineno))
                 del data_buffer[:]
 
-        while self.stream:
-            token = self.stream.current
-            if token.type is 'data':
-                if token.value:
-                    add_data(nodes.TemplateData(token.value,
-                                                lineno=token.lineno))
-                self.stream.next()
-            elif token.type is 'variable_begin':
-                self.stream.next()
-                add_data(self.parse_tuple(with_condexpr=True))
-                self.stream.expect('variable_end')
-            elif token.type is 'block_begin':
-                flush_data()
-                self.stream.next()
-                if end_tokens is not None and \
-                   self.stream.current.test_any(*end_tokens):
-                    return body
-                rv = self.parse_statement()
-                if isinstance(rv, list):
-                    body.extend(rv)
+        try:
+            while self.stream:
+                token = self.stream.current
+                if token.type == 'data':
+                    if token.value:
+                        add_data(nodes.TemplateData(token.value,
+                                                    lineno=token.lineno))
+                    next(self.stream)
+                elif token.type == 'variable_begin':
+                    next(self.stream)
+                    add_data(self.parse_tuple(with_condexpr=True))
+                    self.stream.expect('variable_end')
+                elif token.type == 'block_begin':
+                    flush_data()
+                    next(self.stream)
+                    if end_tokens is not None and \
+                       self.stream.current.test_any(*end_tokens):
+                        return body
+                    rv = self.parse_statement()
+                    if isinstance(rv, list):
+                        body.extend(rv)
+                    else:
+                        body.append(rv)
+                    self.stream.expect('block_end')
                 else:
-                    body.append(rv)
-                self.stream.expect('block_end')
-            else:
-                raise AssertionError('internal parsing error')
+                    raise AssertionError('internal parsing error')
+
+            flush_data()
+        finally:
+            if end_tokens is not None:
+                self._end_token_stack.pop()
 
-        flush_data()
         return body
 
     def parse(self):