nsz Git - libfirm/blob - scripts/jinja2/parser.py

   1 # -*- coding: utf-8 -*-
   2 """
   3     jinja2.parser
   4     ~~~~~~~~~~~~~
   5
   6     Implements the template parser.
   7
   8     :copyright: 2008 by Armin Ronacher.
   9     :license: BSD, see LICENSE for more details.
  10 """
  11 from jinja2 import nodes
  12 from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
  13
  14
  15 _statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
  16                                  'macro', 'include', 'from', 'import',
  17                                  'set'])
  18 _compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
  19
  20
  21 class Parser(object):
  22     """This is the central parsing class Jinja2 uses.  It's passed to
  23     extensions and can be used to parse expressions or statements.
  24     """
  25
  26     def __init__(self, environment, source, name=None, filename=None,
  27                  state=None):
  28         self.environment = environment
  29         self.stream = environment._tokenize(source, name, filename, state)
  30         self.name = name
  31         self.filename = filename
  32         self.closed = False
  33         self.extensions = {}
  34         for extension in environment.extensions.itervalues():
  35             for tag in extension.tags:
  36                 self.extensions[tag] = extension.parse
  37         self._last_identifier = 0
  38
  39     def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
  40         """Convenience method that raises `exc` with the message, passed
  41         line number or last line number as well as the current name and
  42         filename.
  43         """
  44         if lineno is None:
  45             lineno = self.stream.current.lineno
  46         raise exc(msg, lineno, self.name, self.filename)
  47
  48     def is_tuple_end(self, extra_end_rules=None):
  49         """Are we at the end of a tuple?"""
  50         if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
  51             return True
  52         elif extra_end_rules is not None:
  53             return self.stream.current.test_any(extra_end_rules)
  54         return False
  55
  56     def free_identifier(self, lineno=None):
  57         """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
  58         self._last_identifier += 1
  59         rv = object.__new__(nodes.InternalName)
  60         nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
  61         return rv
  62
  63     def parse_statement(self):
  64         """Parse a single statement."""
  65         token = self.stream.current
  66         if token.type is not 'name':
  67             self.fail('tag name expected', token.lineno)
  68         if token.value in _statement_keywords:
  69             return getattr(self, 'parse_' + self.stream.current.value)()
  70         if token.value == 'call':
  71             return self.parse_call_block()
  72         if token.value == 'filter':
  73             return self.parse_filter_block()
  74         ext = self.extensions.get(token.value)
  75         if ext is not None:
  76             return ext(self)
  77         self.fail('unknown tag %r' % token.value, token.lineno)
  78
  79     def parse_statements(self, end_tokens, drop_needle=False):
  80         """Parse multiple statements into a list until one of the end tokens
  81         is reached.  This is used to parse the body of statements as it also
  82         parses template data if appropriate.  The parser checks first if the
  83         current token is a colon and skips it if there is one.  Then it checks
  84         for the block end and parses until if one of the `end_tokens` is
  85         reached.  Per default the active token in the stream at the end of
  86         the call is the matched end token.  If this is not wanted `drop_needle`
  87         can be set to `True` and the end token is removed.
  88         """
  89         # the first token may be a colon for python compatibility
  90         self.stream.skip_if('colon')
  91
  92         # in the future it would be possible to add whole code sections
  93         # by adding some sort of end of statement token and parsing those here.
  94         self.stream.expect('block_end')
  95         result = self.subparse(end_tokens)
  96
  97         if drop_needle:
  98             self.stream.next()
  99         return result
 100
 101     def parse_set(self):
 102         """Parse an assign statement."""
 103         lineno = self.stream.next().lineno
 104         target = self.parse_assign_target()
 105         self.stream.expect('assign')
 106         expr = self.parse_tuple()
 107         return nodes.Assign(target, expr, lineno=lineno)
 108
 109     def parse_for(self):
 110         """Parse a for loop."""
 111         lineno = self.stream.expect('name:for').lineno
 112         target = self.parse_assign_target(extra_end_rules=('name:in',))
 113         self.stream.expect('name:in')
 114         iter = self.parse_tuple(with_condexpr=False,
 115                                 extra_end_rules=('name:recursive',))
 116         test = None
 117         if self.stream.skip_if('name:if'):
 118             test = self.parse_expression()
 119         recursive = self.stream.skip_if('name:recursive')
 120         body = self.parse_statements(('name:endfor', 'name:else'))
 121         if self.stream.next().value == 'endfor':
 122             else_ = []
 123         else:
 124             else_ = self.parse_statements(('name:endfor',), drop_needle=True)
 125         return nodes.For(target, iter, body, else_, test,
 126                          recursive, lineno=lineno)
 127
 128     def parse_if(self):
 129         """Parse an if construct."""
 130         node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
 131         while 1:
 132             node.test = self.parse_tuple(with_condexpr=False)
 133             node.body = self.parse_statements(('name:elif', 'name:else',
 134                                                'name:endif'))
 135             token = self.stream.next()
 136             if token.test('name:elif'):
 137                 new_node = nodes.If(lineno=self.stream.current.lineno)
 138                 node.else_ = [new_node]
 139                 node = new_node
 140                 continue
 141             elif token.test('name:else'):
 142                 node.else_ = self.parse_statements(('name:endif',),
 143                                                    drop_needle=True)
 144             else:
 145                 node.else_ = []
 146             break
 147         return result
 148
 149     def parse_block(self):
 150         node = nodes.Block(lineno=self.stream.next().lineno)
 151         node.name = self.stream.expect('name').value
 152         node.body = self.parse_statements(('name:endblock',), drop_needle=True)
 153         self.stream.skip_if('name:' + node.name)
 154         return node
 155
 156     def parse_extends(self):
 157         node = nodes.Extends(lineno=self.stream.next().lineno)
 158         node.template = self.parse_expression()
 159         return node
 160
 161     def parse_import_context(self, node, default):
 162         if self.stream.current.test_any('name:with', 'name:without') and \
 163            self.stream.look().test('name:context'):
 164             node.with_context = self.stream.next().value == 'with'
 165             self.stream.skip()
 166         else:
 167             node.with_context = default
 168         return node
 169
 170     def parse_include(self):
 171         node = nodes.Include(lineno=self.stream.next().lineno)
 172         node.template = self.parse_expression()
 173         return self.parse_import_context(node, True)
 174
 175     def parse_import(self):
 176         node = nodes.Import(lineno=self.stream.next().lineno)
 177         node.template = self.parse_expression()
 178         self.stream.expect('name:as')
 179         node.target = self.parse_assign_target(name_only=True).name
 180         return self.parse_import_context(node, False)
 181
 182     def parse_from(self):
 183         node = nodes.FromImport(lineno=self.stream.next().lineno)
 184         node.template = self.parse_expression()
 185         self.stream.expect('name:import')
 186         node.names = []
 187
 188         def parse_context():
 189             if self.stream.current.value in ('with', 'without') and \
 190                self.stream.look().test('name:context'):
 191                 node.with_context = self.stream.next().value == 'with'
 192                 self.stream.skip()
 193                 return True
 194             return False
 195
 196         while 1:
 197             if node.names:
 198                 self.stream.expect('comma')
 199             if self.stream.current.type is 'name':
 200                 if parse_context():
 201                     break
 202                 target = self.parse_assign_target(name_only=True)
 203                 if target.name.startswith('_'):
 204                     self.fail('names starting with an underline can not '
 205                               'be imported', target.lineno,
 206                               exc=TemplateAssertionError)
 207                 if self.stream.skip_if('name:as'):
 208                     alias = self.parse_assign_target(name_only=True)
 209                     node.names.append((target.name, alias.name))
 210                 else:
 211                     node.names.append(target.name)
 212                 if parse_context() or self.stream.current.type is not 'comma':
 213                     break
 214             else:
 215                 break
 216         if not hasattr(node, 'with_context'):
 217             node.with_context = False
 218             self.stream.skip_if('comma')
 219         return node
 220
 221     def parse_signature(self, node):
 222         node.args = args = []
 223         node.defaults = defaults = []
 224         self.stream.expect('lparen')
 225         while self.stream.current.type is not 'rparen':
 226             if args:
 227                 self.stream.expect('comma')
 228             arg = self.parse_assign_target(name_only=True)
 229             arg.set_ctx('param')
 230             if self.stream.skip_if('assign'):
 231                 defaults.append(self.parse_expression())
 232             args.append(arg)
 233         self.stream.expect('rparen')
 234
 235     def parse_call_block(self):
 236         node = nodes.CallBlock(lineno=self.stream.next().lineno)
 237         if self.stream.current.type is 'lparen':
 238             self.parse_signature(node)
 239         else:
 240             node.args = []
 241             node.defaults = []
 242
 243         node.call = self.parse_expression()
 244         if not isinstance(node.call, nodes.Call):
 245             self.fail('expected call', node.lineno)
 246         node.body = self.parse_statements(('name:endcall',), drop_needle=True)
 247         return node
 248
 249     def parse_filter_block(self):
 250         node = nodes.FilterBlock(lineno=self.stream.next().lineno)
 251         node.filter = self.parse_filter(None, start_inline=True)
 252         node.body = self.parse_statements(('name:endfilter',),
 253                                           drop_needle=True)
 254         return node
 255
 256     def parse_macro(self):
 257         node = nodes.Macro(lineno=self.stream.next().lineno)
 258         node.name = self.parse_assign_target(name_only=True).name
 259         self.parse_signature(node)
 260         node.body = self.parse_statements(('name:endmacro',),
 261                                           drop_needle=True)
 262         return node
 263
 264     def parse_print(self):
 265         node = nodes.Output(lineno=self.stream.next().lineno)
 266         node.nodes = []
 267         while self.stream.current.type is not 'block_end':
 268             if node.nodes:
 269                 self.stream.expect('comma')
 270             node.nodes.append(self.parse_expression())
 271         return node
 272
 273     def parse_assign_target(self, with_tuple=True, name_only=False,
 274                             extra_end_rules=None):
 275         """Parse an assignment target.  As Jinja2 allows assignments to
 276         tuples, this function can parse all allowed assignment targets.  Per
 277         default assignments to tuples are parsed, that can be disable however
 278         by setting `with_tuple` to `False`.  If only assignments to names are
 279         wanted `name_only` can be set to `True`.  The `extra_end_rules`
 280         parameter is forwarded to the tuple parsing function.
 281         """
 282         if name_only:
 283             token = self.stream.expect('name')
 284             target = nodes.Name(token.value, 'store', lineno=token.lineno)
 285         else:
 286             if with_tuple:
 287                 target = self.parse_tuple(simplified=True,
 288                                           extra_end_rules=extra_end_rules)
 289             else:
 290                 target = self.parse_primary(with_postfix=False)
 291             target.set_ctx('store')
 292         if not target.can_assign():
 293             self.fail('can\'t assign to %r' % target.__class__.
 294                       __name__.lower(), target.lineno)
 295         return target
 296
 297     def parse_expression(self, with_condexpr=True):
 298         """Parse an expression.  Per default all expressions are parsed, if
 299         the optional `with_condexpr` parameter is set to `False` conditional
 300         expressions are not parsed.
 301         """
 302         if with_condexpr:
 303             return self.parse_condexpr()
 304         return self.parse_or()
 305
 306     def parse_condexpr(self):
 307         lineno = self.stream.current.lineno
 308         expr1 = self.parse_or()
 309         while self.stream.skip_if('name:if'):
 310             expr2 = self.parse_or()
 311             if self.stream.skip_if('name:else'):
 312                 expr3 = self.parse_condexpr()
 313             else:
 314                 expr3 = None
 315             expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
 316             lineno = self.stream.current.lineno
 317         return expr1
 318
 319     def parse_or(self):
 320         lineno = self.stream.current.lineno
 321         left = self.parse_and()
 322         while self.stream.skip_if('name:or'):
 323             right = self.parse_and()
 324             left = nodes.Or(left, right, lineno=lineno)
 325             lineno = self.stream.current.lineno
 326         return left
 327
 328     def parse_and(self):
 329         lineno = self.stream.current.lineno
 330         left = self.parse_compare()
 331         while self.stream.skip_if('name:and'):
 332             right = self.parse_compare()
 333             left = nodes.And(left, right, lineno=lineno)
 334             lineno = self.stream.current.lineno
 335         return left
 336
 337     def parse_compare(self):
 338         lineno = self.stream.current.lineno
 339         expr = self.parse_add()
 340         ops = []
 341         while 1:
 342             token_type = self.stream.current.type
 343             if token_type in _compare_operators:
 344                 self.stream.next()
 345                 ops.append(nodes.Operand(token_type, self.parse_add()))
 346             elif self.stream.skip_if('name:in'):
 347                 ops.append(nodes.Operand('in', self.parse_add()))
 348             elif self.stream.current.test('name:not') and \
 349                  self.stream.look().test('name:in'):
 350                 self.stream.skip(2)
 351                 ops.append(nodes.Operand('notin', self.parse_add()))
 352             else:
 353                 break
 354             lineno = self.stream.current.lineno
 355         if not ops:
 356             return expr
 357         return nodes.Compare(expr, ops, lineno=lineno)
 358
 359     def parse_add(self):
 360         lineno = self.stream.current.lineno
 361         left = self.parse_sub()
 362         while self.stream.current.type is 'add':
 363             self.stream.next()
 364             right = self.parse_sub()
 365             left = nodes.Add(left, right, lineno=lineno)
 366             lineno = self.stream.current.lineno
 367         return left
 368
 369     def parse_sub(self):
 370         lineno = self.stream.current.lineno
 371         left = self.parse_concat()
 372         while self.stream.current.type is 'sub':
 373             self.stream.next()
 374             right = self.parse_concat()
 375             left = nodes.Sub(left, right, lineno=lineno)
 376             lineno = self.stream.current.lineno
 377         return left
 378
 379     def parse_concat(self):
 380         lineno = self.stream.current.lineno
 381         args = [self.parse_mul()]
 382         while self.stream.current.type is 'tilde':
 383             self.stream.next()
 384             args.append(self.parse_mul())
 385         if len(args) == 1:
 386             return args[0]
 387         return nodes.Concat(args, lineno=lineno)
 388
 389     def parse_mul(self):
 390         lineno = self.stream.current.lineno
 391         left = self.parse_div()
 392         while self.stream.current.type is 'mul':
 393             self.stream.next()
 394             right = self.parse_div()
 395             left = nodes.Mul(left, right, lineno=lineno)
 396             lineno = self.stream.current.lineno
 397         return left
 398
 399     def parse_div(self):
 400         lineno = self.stream.current.lineno
 401         left = self.parse_floordiv()
 402         while self.stream.current.type is 'div':
 403             self.stream.next()
 404             right = self.parse_floordiv()
 405             left = nodes.Div(left, right, lineno=lineno)
 406             lineno = self.stream.current.lineno
 407         return left
 408
 409     def parse_floordiv(self):
 410         lineno = self.stream.current.lineno
 411         left = self.parse_mod()
 412         while self.stream.current.type is 'floordiv':
 413             self.stream.next()
 414             right = self.parse_mod()
 415             left = nodes.FloorDiv(left, right, lineno=lineno)
 416             lineno = self.stream.current.lineno
 417         return left
 418
 419     def parse_mod(self):
 420         lineno = self.stream.current.lineno
 421         left = self.parse_pow()
 422         while self.stream.current.type is 'mod':
 423             self.stream.next()
 424             right = self.parse_pow()
 425             left = nodes.Mod(left, right, lineno=lineno)
 426             lineno = self.stream.current.lineno
 427         return left
 428
 429     def parse_pow(self):
 430         lineno = self.stream.current.lineno
 431         left = self.parse_unary()
 432         while self.stream.current.type is 'pow':
 433             self.stream.next()
 434             right = self.parse_unary()
 435             left = nodes.Pow(left, right, lineno=lineno)
 436             lineno = self.stream.current.lineno
 437         return left
 438
 439     def parse_unary(self):
 440         token_type = self.stream.current.type
 441         lineno = self.stream.current.lineno
 442         if token_type is 'name' and self.stream.current.value == 'not':
 443             self.stream.next()
 444             node = self.parse_unary()
 445             return nodes.Not(node, lineno=lineno)
 446         if token_type is 'sub':
 447             self.stream.next()
 448             node = self.parse_unary()
 449             return nodes.Neg(node, lineno=lineno)
 450         if token_type is 'add':
 451             self.stream.next()
 452             node = self.parse_unary()
 453             return nodes.Pos(node, lineno=lineno)
 454         return self.parse_primary()
 455
 456     def parse_primary(self, with_postfix=True):
 457         token = self.stream.current
 458         if token.type is 'name':
 459             if token.value in ('true', 'false', 'True', 'False'):
 460                 node = nodes.Const(token.value in ('true', 'True'),
 461                                    lineno=token.lineno)
 462             elif token.value in ('none', 'None'):
 463                 node = nodes.Const(None, lineno=token.lineno)
 464             else:
 465                 node = nodes.Name(token.value, 'load', lineno=token.lineno)
 466             self.stream.next()
 467         elif token.type is 'string':
 468             self.stream.next()
 469             buf = [token.value]
 470             lineno = token.lineno
 471             while self.stream.current.type is 'string':
 472                 buf.append(self.stream.current.value)
 473                 self.stream.next()
 474             node = nodes.Const(''.join(buf), lineno=lineno)
 475         elif token.type in ('integer', 'float'):
 476             self.stream.next()
 477             node = nodes.Const(token.value, lineno=token.lineno)
 478         elif token.type is 'lparen':
 479             self.stream.next()
 480             node = self.parse_tuple()
 481             self.stream.expect('rparen')
 482         elif token.type is 'lbracket':
 483             node = self.parse_list()
 484         elif token.type is 'lbrace':
 485             node = self.parse_dict()
 486         else:
 487             self.fail("unexpected token '%s'" % (token,), token.lineno)
 488         if with_postfix:
 489             node = self.parse_postfix(node)
 490         return node
 491
 492     def parse_tuple(self, simplified=False, with_condexpr=True,
 493                     extra_end_rules=None):
 494         """Works like `parse_expression` but if multiple expressions are
 495         delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
 496         This method could also return a regular expression instead of a tuple
 497         if no commas where found.
 498
 499         The default parsing mode is a full tuple.  If `simplified` is `True`
 500         only names and literals are parsed.  The `no_condexpr` parameter is
 501         forwarded to :meth:`parse_expression`.
 502
 503         Because tuples do not require delimiters and may end in a bogus comma
 504         an extra hint is needed that marks the end of a tuple.  For example
 505         for loops support tuples between `for` and `in`.  In that case the
 506         `extra_end_rules` is set to ``['name:in']``.
 507         """
 508         lineno = self.stream.current.lineno
 509         if simplified:
 510             parse = lambda: self.parse_primary(with_postfix=False)
 511         elif with_condexpr:
 512             parse = self.parse_expression
 513         else:
 514             parse = lambda: self.parse_expression(with_condexpr=False)
 515         args = []
 516         is_tuple = False
 517         while 1:
 518             if args:
 519                 self.stream.expect('comma')
 520             if self.is_tuple_end(extra_end_rules):
 521                 break
 522             args.append(parse())
 523             if self.stream.current.type is 'comma':
 524                 is_tuple = True
 525             else:
 526                 break
 527             lineno = self.stream.current.lineno
 528         if not is_tuple and args:
 529             return args[0]
 530         return nodes.Tuple(args, 'load', lineno=lineno)
 531
 532     def parse_list(self):
 533         token = self.stream.expect('lbracket')
 534         items = []
 535         while self.stream.current.type is not 'rbracket':
 536             if items:
 537                 self.stream.expect('comma')
 538             if self.stream.current.type == 'rbracket':
 539                 break
 540             items.append(self.parse_expression())
 541         self.stream.expect('rbracket')
 542         return nodes.List(items, lineno=token.lineno)
 543
 544     def parse_dict(self):
 545         token = self.stream.expect('lbrace')
 546         items = []
 547         while self.stream.current.type is not 'rbrace':
 548             if items:
 549                 self.stream.expect('comma')
 550             if self.stream.current.type == 'rbrace':
 551                 break
 552             key = self.parse_expression()
 553             self.stream.expect('colon')
 554             value = self.parse_expression()
 555             items.append(nodes.Pair(key, value, lineno=key.lineno))
 556         self.stream.expect('rbrace')
 557         return nodes.Dict(items, lineno=token.lineno)
 558
 559     def parse_postfix(self, node):
 560         while 1:
 561             token_type = self.stream.current.type
 562             if token_type is 'dot' or token_type is 'lbracket':
 563                 node = self.parse_subscript(node)
 564             elif token_type is 'lparen':
 565                 node = self.parse_call(node)
 566             elif token_type is 'pipe':
 567                 node = self.parse_filter(node)
 568             elif token_type is 'name' and self.stream.current.value == 'is':
 569                 node = self.parse_test(node)
 570             else:
 571                 break
 572         return node
 573
 574     def parse_subscript(self, node):
 575         token = self.stream.next()
 576         if token.type is 'dot':
 577             attr_token = self.stream.current
 578             self.stream.next()
 579             if attr_token.type is 'name':
 580                 return nodes.Getattr(node, attr_token.value, 'load',
 581                                      lineno=token.lineno)
 582             elif attr_token.type is not 'integer':
 583                 self.fail('expected name or number', attr_token.lineno)
 584             arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
 585             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
 586         if token.type is 'lbracket':
 587             priority_on_attribute = False
 588             args = []
 589             while self.stream.current.type is not 'rbracket':
 590                 if args:
 591                     self.stream.expect('comma')
 592                 args.append(self.parse_subscribed())
 593             self.stream.expect('rbracket')
 594             if len(args) == 1:
 595                 arg = args[0]
 596             else:
 597                 arg = nodes.Tuple(args, self.lineno, self.filename)
 598             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
 599         self.fail('expected subscript expression', self.lineno)
 600
 601     def parse_subscribed(self):
 602         lineno = self.stream.current.lineno
 603
 604         if self.stream.current.type is 'colon':
 605             self.stream.next()
 606             args = [None]
 607         else:
 608             node = self.parse_expression()
 609             if self.stream.current.type is not 'colon':
 610                 return node
 611             self.stream.next()
 612             args = [node]
 613
 614         if self.stream.current.type is 'colon':
 615             args.append(None)
 616         elif self.stream.current.type not in ('rbracket', 'comma'):
 617             args.append(self.parse_expression())
 618         else:
 619             args.append(None)
 620
 621         if self.stream.current.type is 'colon':
 622             self.stream.next()
 623             if self.stream.current.type not in ('rbracket', 'comma'):
 624                 args.append(self.parse_expression())
 625             else:
 626                 args.append(None)
 627         else:
 628             args.append(None)
 629
 630         return nodes.Slice(lineno=lineno, *args)
 631
 632     def parse_call(self, node):
 633         token = self.stream.expect('lparen')
 634         args = []
 635         kwargs = []
 636         dyn_args = dyn_kwargs = None
 637         require_comma = False
 638
 639         def ensure(expr):
 640             if not expr:
 641                 self.fail('invalid syntax for function call expression',
 642                           token.lineno)
 643
 644         while self.stream.current.type is not 'rparen':
 645             if require_comma:
 646                 self.stream.expect('comma')
 647                 # support for trailing comma
 648                 if self.stream.current.type is 'rparen':
 649                     break
 650             if self.stream.current.type is 'mul':
 651                 ensure(dyn_args is None and dyn_kwargs is None)
 652                 self.stream.next()
 653                 dyn_args = self.parse_expression()
 654             elif self.stream.current.type is 'pow':
 655                 ensure(dyn_kwargs is None)
 656                 self.stream.next()
 657                 dyn_kwargs = self.parse_expression()
 658             else:
 659                 ensure(dyn_args is None and dyn_kwargs is None)
 660                 if self.stream.current.type is 'name' and \
 661                     self.stream.look().type is 'assign':
 662                     key = self.stream.current.value
 663                     self.stream.skip(2)
 664                     value = self.parse_expression()
 665                     kwargs.append(nodes.Keyword(key, value,
 666                                                 lineno=value.lineno))
 667                 else:
 668                     ensure(not kwargs)
 669                     args.append(self.parse_expression())
 670
 671             require_comma = True
 672         self.stream.expect('rparen')
 673
 674         if node is None:
 675             return args, kwargs, dyn_args, dyn_kwargs
 676         return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
 677                           lineno=token.lineno)
 678
 679     def parse_filter(self, node, start_inline=False):
 680         while self.stream.current.type == 'pipe' or start_inline:
 681             if not start_inline:
 682                 self.stream.next()
 683             token = self.stream.expect('name')
 684             name = token.value
 685             while self.stream.current.type is 'dot':
 686                 self.stream.next()
 687                 name += '.' + self.stream.expect('name').value
 688             if self.stream.current.type is 'lparen':
 689                 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 690             else:
 691                 args = []
 692                 kwargs = []
 693                 dyn_args = dyn_kwargs = None
 694             node = nodes.Filter(node, name, args, kwargs, dyn_args,
 695                                 dyn_kwargs, lineno=token.lineno)
 696             start_inline = False
 697         return node
 698
 699     def parse_test(self, node):
 700         token = self.stream.next()
 701         if self.stream.current.test('name:not'):
 702             self.stream.next()
 703             negated = True
 704         else:
 705             negated = False
 706         name = self.stream.expect('name').value
 707         while self.stream.current.type is 'dot':
 708             self.stream.next()
 709             name += '.' + self.stream.expect('name').value
 710         dyn_args = dyn_kwargs = None
 711         kwargs = []
 712         if self.stream.current.type is 'lparen':
 713             args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 714         elif self.stream.current.type in ('name', 'string', 'integer',
 715                                           'float', 'lparen', 'lbracket',
 716                                           'lbrace') and not \
 717              self.stream.current.test_any('name:else', 'name:or',
 718                                           'name:and'):
 719             if self.stream.current.test('name:is'):
 720                 self.fail('You cannot chain multiple tests with is')
 721             args = [self.parse_expression()]
 722         else:
 723             args = []
 724         node = nodes.Test(node, name, args, kwargs, dyn_args,
 725                           dyn_kwargs, lineno=token.lineno)
 726         if negated:
 727             node = nodes.Not(node, lineno=token.lineno)
 728         return node
 729
 730     def subparse(self, end_tokens=None):
 731         body = []
 732         data_buffer = []
 733         add_data = data_buffer.append
 734
 735         def flush_data():
 736             if data_buffer:
 737                 lineno = data_buffer[0].lineno
 738                 body.append(nodes.Output(data_buffer[:], lineno=lineno))
 739                 del data_buffer[:]
 740
 741         while self.stream:
 742             token = self.stream.current
 743             if token.type is 'data':
 744                 if token.value:
 745                     add_data(nodes.TemplateData(token.value,
 746                                                 lineno=token.lineno))
 747                 self.stream.next()
 748             elif token.type is 'variable_begin':
 749                 self.stream.next()
 750                 add_data(self.parse_tuple(with_condexpr=True))
 751                 self.stream.expect('variable_end')
 752             elif token.type is 'block_begin':
 753                 flush_data()
 754                 self.stream.next()
 755                 if end_tokens is not None and \
 756                    self.stream.current.test_any(*end_tokens):
 757                     return body
 758                 rv = self.parse_statement()
 759                 if isinstance(rv, list):
 760                     body.extend(rv)
 761                 else:
 762                     body.append(rv)
 763                 self.stream.expect('block_end')
 764             else:
 765                 raise AssertionError('internal parsing error')
 766
 767         flush_data()
 768         return body
 769
 770     def parse(self):
 771         """Parse the whole template into a `Template` node."""
 772         result = nodes.Template(self.subparse(), lineno=1)
 773         result.set_environment(self.environment)
 774         return result