nsz Git - libfirm/blob - scripts/jinja2/parser.py

   1 # -*- coding: utf-8 -*-
   2 """
   3     jinja2.parser
   4     ~~~~~~~~~~~~~
   5
   6     Implements the template parser.
   7
   8     :copyright: (c) 2010 by the Jinja Team.
   9     :license: BSD, see LICENSE for more details.
  10 """
  11 from jinja2 import nodes
  12 from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
  13 from jinja2.utils import next
  14 from jinja2.lexer import describe_token, describe_token_expr
  15
  16
  17 #: statements that callinto
  18 _statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
  19                                  'macro', 'include', 'from', 'import',
  20                                  'set'])
  21 _compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
  22
  23
  24 class Parser(object):
  25     """This is the central parsing class Jinja2 uses.  It's passed to
  26     extensions and can be used to parse expressions or statements.
  27     """
  28
  29     def __init__(self, environment, source, name=None, filename=None,
  30                  state=None):
  31         self.environment = environment
  32         self.stream = environment._tokenize(source, name, filename, state)
  33         self.name = name
  34         self.filename = filename
  35         self.closed = False
  36         self.extensions = {}
  37         for extension in environment.extensions.itervalues():
  38             for tag in extension.tags:
  39                 self.extensions[tag] = extension.parse
  40         self._last_identifier = 0
  41         self._tag_stack = []
  42         self._end_token_stack = []
  43
  44     def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
  45         """Convenience method that raises `exc` with the message, passed
  46         line number or last line number as well as the current name and
  47         filename.
  48         """
  49         if lineno is None:
  50             lineno = self.stream.current.lineno
  51         raise exc(msg, lineno, self.name, self.filename)
  52
  53     def _fail_ut_eof(self, name, end_token_stack, lineno):
  54         expected = []
  55         for exprs in end_token_stack:
  56             expected.extend(map(describe_token_expr, exprs))
  57         if end_token_stack:
  58             currently_looking = ' or '.join(
  59                 "'%s'" % describe_token_expr(expr)
  60                 for expr in end_token_stack[-1])
  61         else:
  62             currently_looking = None
  63
  64         if name is None:
  65             message = ['Unexpected end of template.']
  66         else:
  67             message = ['Encountered unknown tag \'%s\'.' % name]
  68
  69         if currently_looking:
  70             if name is not None and name in expected:
  71                 message.append('You probably made a nesting mistake. Jinja '
  72                                'is expecting this tag, but currently looking '
  73                                'for %s.' % currently_looking)
  74             else:
  75                 message.append('Jinja was looking for the following tags: '
  76                                '%s.' % currently_looking)
  77
  78         if self._tag_stack:
  79             message.append('The innermost block that needs to be '
  80                            'closed is \'%s\'.' % self._tag_stack[-1])
  81
  82         self.fail(' '.join(message), lineno)
  83
  84     def fail_unknown_tag(self, name, lineno=None):
  85         """Called if the parser encounters an unknown tag.  Tries to fail
  86         with a human readable error message that could help to identify
  87         the problem.
  88         """
  89         return self._fail_ut_eof(name, self._end_token_stack, lineno)
  90
  91     def fail_eof(self, end_tokens=None, lineno=None):
  92         """Like fail_unknown_tag but for end of template situations."""
  93         stack = list(self._end_token_stack)
  94         if end_tokens is not None:
  95             stack.append(end_tokens)
  96         return self._fail_ut_eof(None, stack, lineno)
  97
  98     def is_tuple_end(self, extra_end_rules=None):
  99         """Are we at the end of a tuple?"""
 100         if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
 101             return True
 102         elif extra_end_rules is not None:
 103             return self.stream.current.test_any(extra_end_rules)
 104         return False
 105
 106     def free_identifier(self, lineno=None):
 107         """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
 108         self._last_identifier += 1
 109         rv = object.__new__(nodes.InternalName)
 110         nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
 111         return rv
 112
 113     def parse_statement(self):
 114         """Parse a single statement."""
 115         token = self.stream.current
 116         if token.type != 'name':
 117             self.fail('tag name expected', token.lineno)
 118         self._tag_stack.append(token.value)
 119         pop_tag = True
 120         try:
 121             if token.value in _statement_keywords:
 122                 return getattr(self, 'parse_' + self.stream.current.value)()
 123             if token.value == 'call':
 124                 return self.parse_call_block()
 125             if token.value == 'filter':
 126                 return self.parse_filter_block()
 127             ext = self.extensions.get(token.value)
 128             if ext is not None:
 129                 return ext(self)
 130
 131             # did not work out, remove the token we pushed by accident
 132             # from the stack so that the unknown tag fail function can
 133             # produce a proper error message.
 134             self._tag_stack.pop()
 135             pop_tag = False
 136             self.fail_unknown_tag(token.value, token.lineno)
 137         finally:
 138             if pop_tag:
 139                 self._tag_stack.pop()
 140
 141     def parse_statements(self, end_tokens, drop_needle=False):
 142         """Parse multiple statements into a list until one of the end tokens
 143         is reached.  This is used to parse the body of statements as it also
 144         parses template data if appropriate.  The parser checks first if the
 145         current token is a colon and skips it if there is one.  Then it checks
 146         for the block end and parses until if one of the `end_tokens` is
 147         reached.  Per default the active token in the stream at the end of
 148         the call is the matched end token.  If this is not wanted `drop_needle`
 149         can be set to `True` and the end token is removed.
 150         """
 151         # the first token may be a colon for python compatibility
 152         self.stream.skip_if('colon')
 153
 154         # in the future it would be possible to add whole code sections
 155         # by adding some sort of end of statement token and parsing those here.
 156         self.stream.expect('block_end')
 157         result = self.subparse(end_tokens)
 158
 159         # we reached the end of the template too early, the subparser
 160         # does not check for this, so we do that now
 161         if self.stream.current.type == 'eof':
 162             self.fail_eof(end_tokens)
 163
 164         if drop_needle:
 165             next(self.stream)
 166         return result
 167
 168     def parse_set(self):
 169         """Parse an assign statement."""
 170         lineno = next(self.stream).lineno
 171         target = self.parse_assign_target()
 172         self.stream.expect('assign')
 173         expr = self.parse_tuple()
 174         return nodes.Assign(target, expr, lineno=lineno)
 175
 176     def parse_for(self):
 177         """Parse a for loop."""
 178         lineno = self.stream.expect('name:for').lineno
 179         target = self.parse_assign_target(extra_end_rules=('name:in',))
 180         self.stream.expect('name:in')
 181         iter = self.parse_tuple(with_condexpr=False,
 182                                 extra_end_rules=('name:recursive',))
 183         test = None
 184         if self.stream.skip_if('name:if'):
 185             test = self.parse_expression()
 186         recursive = self.stream.skip_if('name:recursive')
 187         body = self.parse_statements(('name:endfor', 'name:else'))
 188         if next(self.stream).value == 'endfor':
 189             else_ = []
 190         else:
 191             else_ = self.parse_statements(('name:endfor',), drop_needle=True)
 192         return nodes.For(target, iter, body, else_, test,
 193                          recursive, lineno=lineno)
 194
 195     def parse_if(self):
 196         """Parse an if construct."""
 197         node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
 198         while 1:
 199             node.test = self.parse_tuple(with_condexpr=False)
 200             node.body = self.parse_statements(('name:elif', 'name:else',
 201                                                'name:endif'))
 202             token = next(self.stream)
 203             if token.test('name:elif'):
 204                 new_node = nodes.If(lineno=self.stream.current.lineno)
 205                 node.else_ = [new_node]
 206                 node = new_node
 207                 continue
 208             elif token.test('name:else'):
 209                 node.else_ = self.parse_statements(('name:endif',),
 210                                                    drop_needle=True)
 211             else:
 212                 node.else_ = []
 213             break
 214         return result
 215
 216     def parse_block(self):
 217         node = nodes.Block(lineno=next(self.stream).lineno)
 218         node.name = self.stream.expect('name').value
 219         node.scoped = self.stream.skip_if('name:scoped')
 220
 221         # common problem people encounter when switching from django
 222         # to jinja.  we do not support hyphens in block names, so let's
 223         # raise a nicer error message in that case.
 224         if self.stream.current.type == 'sub':
 225             self.fail('Block names in Jinja have to be valid Python '
 226                       'identifiers and may not contain hypens, use an '
 227                       'underscore instead.')
 228
 229         node.body = self.parse_statements(('name:endblock',), drop_needle=True)
 230         self.stream.skip_if('name:' + node.name)
 231         return node
 232
 233     def parse_extends(self):
 234         node = nodes.Extends(lineno=next(self.stream).lineno)
 235         node.template = self.parse_expression()
 236         return node
 237
 238     def parse_import_context(self, node, default):
 239         if self.stream.current.test_any('name:with', 'name:without') and \
 240            self.stream.look().test('name:context'):
 241             node.with_context = next(self.stream).value == 'with'
 242             self.stream.skip()
 243         else:
 244             node.with_context = default
 245         return node
 246
 247     def parse_include(self):
 248         node = nodes.Include(lineno=next(self.stream).lineno)
 249         node.template = self.parse_expression()
 250         if self.stream.current.test('name:ignore') and \
 251            self.stream.look().test('name:missing'):
 252             node.ignore_missing = True
 253             self.stream.skip(2)
 254         else:
 255             node.ignore_missing = False
 256         return self.parse_import_context(node, True)
 257
 258     def parse_import(self):
 259         node = nodes.Import(lineno=next(self.stream).lineno)
 260         node.template = self.parse_expression()
 261         self.stream.expect('name:as')
 262         node.target = self.parse_assign_target(name_only=True).name
 263         return self.parse_import_context(node, False)
 264
 265     def parse_from(self):
 266         node = nodes.FromImport(lineno=next(self.stream).lineno)
 267         node.template = self.parse_expression()
 268         self.stream.expect('name:import')
 269         node.names = []
 270
 271         def parse_context():
 272             if self.stream.current.value in ('with', 'without') and \
 273                self.stream.look().test('name:context'):
 274                 node.with_context = next(self.stream).value == 'with'
 275                 self.stream.skip()
 276                 return True
 277             return False
 278
 279         while 1:
 280             if node.names:
 281                 self.stream.expect('comma')
 282             if self.stream.current.type == 'name':
 283                 if parse_context():
 284                     break
 285                 target = self.parse_assign_target(name_only=True)
 286                 if target.name.startswith('_'):
 287                     self.fail('names starting with an underline can not '
 288                               'be imported', target.lineno,
 289                               exc=TemplateAssertionError)
 290                 if self.stream.skip_if('name:as'):
 291                     alias = self.parse_assign_target(name_only=True)
 292                     node.names.append((target.name, alias.name))
 293                 else:
 294                     node.names.append(target.name)
 295                 if parse_context() or self.stream.current.type != 'comma':
 296                     break
 297             else:
 298                 break
 299         if not hasattr(node, 'with_context'):
 300             node.with_context = False
 301             self.stream.skip_if('comma')
 302         return node
 303
 304     def parse_signature(self, node):
 305         node.args = args = []
 306         node.defaults = defaults = []
 307         self.stream.expect('lparen')
 308         while self.stream.current.type != 'rparen':
 309             if args:
 310                 self.stream.expect('comma')
 311             arg = self.parse_assign_target(name_only=True)
 312             arg.set_ctx('param')
 313             if self.stream.skip_if('assign'):
 314                 defaults.append(self.parse_expression())
 315             args.append(arg)
 316         self.stream.expect('rparen')
 317
 318     def parse_call_block(self):
 319         node = nodes.CallBlock(lineno=next(self.stream).lineno)
 320         if self.stream.current.type == 'lparen':
 321             self.parse_signature(node)
 322         else:
 323             node.args = []
 324             node.defaults = []
 325
 326         node.call = self.parse_expression()
 327         if not isinstance(node.call, nodes.Call):
 328             self.fail('expected call', node.lineno)
 329         node.body = self.parse_statements(('name:endcall',), drop_needle=True)
 330         return node
 331
 332     def parse_filter_block(self):
 333         node = nodes.FilterBlock(lineno=next(self.stream).lineno)
 334         node.filter = self.parse_filter(None, start_inline=True)
 335         node.body = self.parse_statements(('name:endfilter',),
 336                                           drop_needle=True)
 337         return node
 338
 339     def parse_macro(self):
 340         node = nodes.Macro(lineno=next(self.stream).lineno)
 341         node.name = self.parse_assign_target(name_only=True).name
 342         self.parse_signature(node)
 343         node.body = self.parse_statements(('name:endmacro',),
 344                                           drop_needle=True)
 345         return node
 346
 347     def parse_print(self):
 348         node = nodes.Output(lineno=next(self.stream).lineno)
 349         node.nodes = []
 350         while self.stream.current.type != 'block_end':
 351             if node.nodes:
 352                 self.stream.expect('comma')
 353             node.nodes.append(self.parse_expression())
 354         return node
 355
 356     def parse_assign_target(self, with_tuple=True, name_only=False,
 357                             extra_end_rules=None):
 358         """Parse an assignment target.  As Jinja2 allows assignments to
 359         tuples, this function can parse all allowed assignment targets.  Per
 360         default assignments to tuples are parsed, that can be disable however
 361         by setting `with_tuple` to `False`.  If only assignments to names are
 362         wanted `name_only` can be set to `True`.  The `extra_end_rules`
 363         parameter is forwarded to the tuple parsing function.
 364         """
 365         if name_only:
 366             token = self.stream.expect('name')
 367             target = nodes.Name(token.value, 'store', lineno=token.lineno)
 368         else:
 369             if with_tuple:
 370                 target = self.parse_tuple(simplified=True,
 371                                           extra_end_rules=extra_end_rules)
 372             else:
 373                 target = self.parse_primary(with_postfix=False)
 374             target.set_ctx('store')
 375         if not target.can_assign():
 376             self.fail('can\'t assign to %r' % target.__class__.
 377                       __name__.lower(), target.lineno)
 378         return target
 379
 380     def parse_expression(self, with_condexpr=True):
 381         """Parse an expression.  Per default all expressions are parsed, if
 382         the optional `with_condexpr` parameter is set to `False` conditional
 383         expressions are not parsed.
 384         """
 385         if with_condexpr:
 386             return self.parse_condexpr()
 387         return self.parse_or()
 388
 389     def parse_condexpr(self):
 390         lineno = self.stream.current.lineno
 391         expr1 = self.parse_or()
 392         while self.stream.skip_if('name:if'):
 393             expr2 = self.parse_or()
 394             if self.stream.skip_if('name:else'):
 395                 expr3 = self.parse_condexpr()
 396             else:
 397                 expr3 = None
 398             expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
 399             lineno = self.stream.current.lineno
 400         return expr1
 401
 402     def parse_or(self):
 403         lineno = self.stream.current.lineno
 404         left = self.parse_and()
 405         while self.stream.skip_if('name:or'):
 406             right = self.parse_and()
 407             left = nodes.Or(left, right, lineno=lineno)
 408             lineno = self.stream.current.lineno
 409         return left
 410
 411     def parse_and(self):
 412         lineno = self.stream.current.lineno
 413         left = self.parse_not()
 414         while self.stream.skip_if('name:and'):
 415             right = self.parse_not()
 416             left = nodes.And(left, right, lineno=lineno)
 417             lineno = self.stream.current.lineno
 418         return left
 419
 420     def parse_not(self):
 421         if self.stream.current.test('name:not'):
 422             lineno = next(self.stream).lineno
 423             return nodes.Not(self.parse_not(), lineno=lineno)
 424         return self.parse_compare()
 425
 426     def parse_compare(self):
 427         lineno = self.stream.current.lineno
 428         expr = self.parse_add()
 429         ops = []
 430         while 1:
 431             token_type = self.stream.current.type
 432             if token_type in _compare_operators:
 433                 next(self.stream)
 434                 ops.append(nodes.Operand(token_type, self.parse_add()))
 435             elif self.stream.skip_if('name:in'):
 436                 ops.append(nodes.Operand('in', self.parse_add()))
 437             elif self.stream.current.test('name:not') and \
 438                  self.stream.look().test('name:in'):
 439                 self.stream.skip(2)
 440                 ops.append(nodes.Operand('notin', self.parse_add()))
 441             else:
 442                 break
 443             lineno = self.stream.current.lineno
 444         if not ops:
 445             return expr
 446         return nodes.Compare(expr, ops, lineno=lineno)
 447
 448     def parse_add(self):
 449         lineno = self.stream.current.lineno
 450         left = self.parse_sub()
 451         while self.stream.current.type == 'add':
 452             next(self.stream)
 453             right = self.parse_sub()
 454             left = nodes.Add(left, right, lineno=lineno)
 455             lineno = self.stream.current.lineno
 456         return left
 457
 458     def parse_sub(self):
 459         lineno = self.stream.current.lineno
 460         left = self.parse_concat()
 461         while self.stream.current.type == 'sub':
 462             next(self.stream)
 463             right = self.parse_concat()
 464             left = nodes.Sub(left, right, lineno=lineno)
 465             lineno = self.stream.current.lineno
 466         return left
 467
 468     def parse_concat(self):
 469         lineno = self.stream.current.lineno
 470         args = [self.parse_mul()]
 471         while self.stream.current.type == 'tilde':
 472             next(self.stream)
 473             args.append(self.parse_mul())
 474         if len(args) == 1:
 475             return args[0]
 476         return nodes.Concat(args, lineno=lineno)
 477
 478     def parse_mul(self):
 479         lineno = self.stream.current.lineno
 480         left = self.parse_div()
 481         while self.stream.current.type == 'mul':
 482             next(self.stream)
 483             right = self.parse_div()
 484             left = nodes.Mul(left, right, lineno=lineno)
 485             lineno = self.stream.current.lineno
 486         return left
 487
 488     def parse_div(self):
 489         lineno = self.stream.current.lineno
 490         left = self.parse_floordiv()
 491         while self.stream.current.type == 'div':
 492             next(self.stream)
 493             right = self.parse_floordiv()
 494             left = nodes.Div(left, right, lineno=lineno)
 495             lineno = self.stream.current.lineno
 496         return left
 497
 498     def parse_floordiv(self):
 499         lineno = self.stream.current.lineno
 500         left = self.parse_mod()
 501         while self.stream.current.type == 'floordiv':
 502             next(self.stream)
 503             right = self.parse_mod()
 504             left = nodes.FloorDiv(left, right, lineno=lineno)
 505             lineno = self.stream.current.lineno
 506         return left
 507
 508     def parse_mod(self):
 509         lineno = self.stream.current.lineno
 510         left = self.parse_pow()
 511         while self.stream.current.type == 'mod':
 512             next(self.stream)
 513             right = self.parse_pow()
 514             left = nodes.Mod(left, right, lineno=lineno)
 515             lineno = self.stream.current.lineno
 516         return left
 517
 518     def parse_pow(self):
 519         lineno = self.stream.current.lineno
 520         left = self.parse_unary()
 521         while self.stream.current.type == 'pow':
 522             next(self.stream)
 523             right = self.parse_unary()
 524             left = nodes.Pow(left, right, lineno=lineno)
 525             lineno = self.stream.current.lineno
 526         return left
 527
 528     def parse_unary(self):
 529         token_type = self.stream.current.type
 530         lineno = self.stream.current.lineno
 531         if token_type == 'sub':
 532             next(self.stream)
 533             node = self.parse_unary()
 534             return nodes.Neg(node, lineno=lineno)
 535         if token_type == 'add':
 536             next(self.stream)
 537             node = self.parse_unary()
 538             return nodes.Pos(node, lineno=lineno)
 539         return self.parse_primary()
 540
 541     def parse_primary(self, with_postfix=True):
 542         token = self.stream.current
 543         if token.type == 'name':
 544             if token.value in ('true', 'false', 'True', 'False'):
 545                 node = nodes.Const(token.value in ('true', 'True'),
 546                                    lineno=token.lineno)
 547             elif token.value in ('none', 'None'):
 548                 node = nodes.Const(None, lineno=token.lineno)
 549             else:
 550                 node = nodes.Name(token.value, 'load', lineno=token.lineno)
 551             next(self.stream)
 552         elif token.type == 'string':
 553             next(self.stream)
 554             buf = [token.value]
 555             lineno = token.lineno
 556             while self.stream.current.type == 'string':
 557                 buf.append(self.stream.current.value)
 558                 next(self.stream)
 559             node = nodes.Const(''.join(buf), lineno=lineno)
 560         elif token.type in ('integer', 'float'):
 561             next(self.stream)
 562             node = nodes.Const(token.value, lineno=token.lineno)
 563         elif token.type == 'lparen':
 564             next(self.stream)
 565             node = self.parse_tuple(explicit_parentheses=True)
 566             self.stream.expect('rparen')
 567         elif token.type == 'lbracket':
 568             node = self.parse_list()
 569         elif token.type == 'lbrace':
 570             node = self.parse_dict()
 571         else:
 572             self.fail("unexpected '%s'" % describe_token(token), token.lineno)
 573         if with_postfix:
 574             node = self.parse_postfix(node)
 575         return node
 576
 577     def parse_tuple(self, simplified=False, with_condexpr=True,
 578                     extra_end_rules=None, explicit_parentheses=False):
 579         """Works like `parse_expression` but if multiple expressions are
 580         delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
 581         This method could also return a regular expression instead of a tuple
 582         if no commas where found.
 583
 584         The default parsing mode is a full tuple.  If `simplified` is `True`
 585         only names and literals are parsed.  The `no_condexpr` parameter is
 586         forwarded to :meth:`parse_expression`.
 587
 588         Because tuples do not require delimiters and may end in a bogus comma
 589         an extra hint is needed that marks the end of a tuple.  For example
 590         for loops support tuples between `for` and `in`.  In that case the
 591         `extra_end_rules` is set to ``['name:in']``.
 592
 593         `explicit_parentheses` is true if the parsing was triggered by an
 594         expression in parentheses.  This is used to figure out if an empty
 595         tuple is a valid expression or not.
 596         """
 597         lineno = self.stream.current.lineno
 598         if simplified:
 599             parse = lambda: self.parse_primary(with_postfix=False)
 600         elif with_condexpr:
 601             parse = self.parse_expression
 602         else:
 603             parse = lambda: self.parse_expression(with_condexpr=False)
 604         args = []
 605         is_tuple = False
 606         while 1:
 607             if args:
 608                 self.stream.expect('comma')
 609             if self.is_tuple_end(extra_end_rules):
 610                 break
 611             args.append(parse())
 612             if self.stream.current.type == 'comma':
 613                 is_tuple = True
 614             else:
 615                 break
 616             lineno = self.stream.current.lineno
 617
 618         if not is_tuple:
 619             if args:
 620                 return args[0]
 621
 622             # if we don't have explicit parentheses, an empty tuple is
 623             # not a valid expression.  This would mean nothing (literally
 624             # nothing) in the spot of an expression would be an empty
 625             # tuple.
 626             if not explicit_parentheses:
 627                 self.fail('Expected an expression, got \'%s\'' %
 628                           describe_token(self.stream.current))
 629
 630         return nodes.Tuple(args, 'load', lineno=lineno)
 631
 632     def parse_list(self):
 633         token = self.stream.expect('lbracket')
 634         items = []
 635         while self.stream.current.type != 'rbracket':
 636             if items:
 637                 self.stream.expect('comma')
 638             if self.stream.current.type == 'rbracket':
 639                 break
 640             items.append(self.parse_expression())
 641         self.stream.expect('rbracket')
 642         return nodes.List(items, lineno=token.lineno)
 643
 644     def parse_dict(self):
 645         token = self.stream.expect('lbrace')
 646         items = []
 647         while self.stream.current.type != 'rbrace':
 648             if items:
 649                 self.stream.expect('comma')
 650             if self.stream.current.type == 'rbrace':
 651                 break
 652             key = self.parse_expression()
 653             self.stream.expect('colon')
 654             value = self.parse_expression()
 655             items.append(nodes.Pair(key, value, lineno=key.lineno))
 656         self.stream.expect('rbrace')
 657         return nodes.Dict(items, lineno=token.lineno)
 658
 659     def parse_postfix(self, node):
 660         while 1:
 661             token_type = self.stream.current.type
 662             if token_type == 'dot' or token_type == 'lbracket':
 663                 node = self.parse_subscript(node)
 664             elif token_type == 'lparen':
 665                 node = self.parse_call(node)
 666             elif token_type == 'pipe':
 667                 node = self.parse_filter(node)
 668             elif token_type == 'name' and self.stream.current.value == 'is':
 669                 node = self.parse_test(node)
 670             else:
 671                 break
 672         return node
 673
 674     def parse_subscript(self, node):
 675         token = next(self.stream)
 676         if token.type == 'dot':
 677             attr_token = self.stream.current
 678             next(self.stream)
 679             if attr_token.type == 'name':
 680                 return nodes.Getattr(node, attr_token.value, 'load',
 681                                      lineno=token.lineno)
 682             elif attr_token.type != 'integer':
 683                 self.fail('expected name or number', attr_token.lineno)
 684             arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
 685             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
 686         if token.type == 'lbracket':
 687             priority_on_attribute = False
 688             args = []
 689             while self.stream.current.type != 'rbracket':
 690                 if args:
 691                     self.stream.expect('comma')
 692                 args.append(self.parse_subscribed())
 693             self.stream.expect('rbracket')
 694             if len(args) == 1:
 695                 arg = args[0]
 696             else:
 697                 arg = nodes.Tuple(args, 'load', lineno=token.lineno)
 698             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
 699         self.fail('expected subscript expression', self.lineno)
 700
 701     def parse_subscribed(self):
 702         lineno = self.stream.current.lineno
 703
 704         if self.stream.current.type == 'colon':
 705             next(self.stream)
 706             args = [None]
 707         else:
 708             node = self.parse_expression()
 709             if self.stream.current.type != 'colon':
 710                 return node
 711             next(self.stream)
 712             args = [node]
 713
 714         if self.stream.current.type == 'colon':
 715             args.append(None)
 716         elif self.stream.current.type not in ('rbracket', 'comma'):
 717             args.append(self.parse_expression())
 718         else:
 719             args.append(None)
 720
 721         if self.stream.current.type == 'colon':
 722             next(self.stream)
 723             if self.stream.current.type not in ('rbracket', 'comma'):
 724                 args.append(self.parse_expression())
 725             else:
 726                 args.append(None)
 727         else:
 728             args.append(None)
 729
 730         return nodes.Slice(lineno=lineno, *args)
 731
 732     def parse_call(self, node):
 733         token = self.stream.expect('lparen')
 734         args = []
 735         kwargs = []
 736         dyn_args = dyn_kwargs = None
 737         require_comma = False
 738
 739         def ensure(expr):
 740             if not expr:
 741                 self.fail('invalid syntax for function call expression',
 742                           token.lineno)
 743
 744         while self.stream.current.type != 'rparen':
 745             if require_comma:
 746                 self.stream.expect('comma')
 747                 # support for trailing comma
 748                 if self.stream.current.type == 'rparen':
 749                     break
 750             if self.stream.current.type == 'mul':
 751                 ensure(dyn_args is None and dyn_kwargs is None)
 752                 next(self.stream)
 753                 dyn_args = self.parse_expression()
 754             elif self.stream.current.type == 'pow':
 755                 ensure(dyn_kwargs is None)
 756                 next(self.stream)
 757                 dyn_kwargs = self.parse_expression()
 758             else:
 759                 ensure(dyn_args is None and dyn_kwargs is None)
 760                 if self.stream.current.type == 'name' and \
 761                     self.stream.look().type == 'assign':
 762                     key = self.stream.current.value
 763                     self.stream.skip(2)
 764                     value = self.parse_expression()
 765                     kwargs.append(nodes.Keyword(key, value,
 766                                                 lineno=value.lineno))
 767                 else:
 768                     ensure(not kwargs)
 769                     args.append(self.parse_expression())
 770
 771             require_comma = True
 772         self.stream.expect('rparen')
 773
 774         if node is None:
 775             return args, kwargs, dyn_args, dyn_kwargs
 776         return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
 777                           lineno=token.lineno)
 778
 779     def parse_filter(self, node, start_inline=False):
 780         while self.stream.current.type == 'pipe' or start_inline:
 781             if not start_inline:
 782                 next(self.stream)
 783             token = self.stream.expect('name')
 784             name = token.value
 785             while self.stream.current.type == 'dot':
 786                 next(self.stream)
 787                 name += '.' + self.stream.expect('name').value
 788             if self.stream.current.type == 'lparen':
 789                 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 790             else:
 791                 args = []
 792                 kwargs = []
 793                 dyn_args = dyn_kwargs = None
 794             node = nodes.Filter(node, name, args, kwargs, dyn_args,
 795                                 dyn_kwargs, lineno=token.lineno)
 796             start_inline = False
 797         return node
 798
 799     def parse_test(self, node):
 800         token = next(self.stream)
 801         if self.stream.current.test('name:not'):
 802             next(self.stream)
 803             negated = True
 804         else:
 805             negated = False
 806         name = self.stream.expect('name').value
 807         while self.stream.current.type == 'dot':
 808             next(self.stream)
 809             name += '.' + self.stream.expect('name').value
 810         dyn_args = dyn_kwargs = None
 811         kwargs = []
 812         if self.stream.current.type == 'lparen':
 813             args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 814         elif self.stream.current.type in ('name', 'string', 'integer',
 815                                           'float', 'lparen', 'lbracket',
 816                                           'lbrace') and not \
 817              self.stream.current.test_any('name:else', 'name:or',
 818                                           'name:and'):
 819             if self.stream.current.test('name:is'):
 820                 self.fail('You cannot chain multiple tests with is')
 821             args = [self.parse_expression()]
 822         else:
 823             args = []
 824         node = nodes.Test(node, name, args, kwargs, dyn_args,
 825                           dyn_kwargs, lineno=token.lineno)
 826         if negated:
 827             node = nodes.Not(node, lineno=token.lineno)
 828         return node
 829
 830     def subparse(self, end_tokens=None):
 831         body = []
 832         data_buffer = []
 833         add_data = data_buffer.append
 834
 835         if end_tokens is not None:
 836             self._end_token_stack.append(end_tokens)
 837
 838         def flush_data():
 839             if data_buffer:
 840                 lineno = data_buffer[0].lineno
 841                 body.append(nodes.Output(data_buffer[:], lineno=lineno))
 842                 del data_buffer[:]
 843
 844         try:
 845             while self.stream:
 846                 token = self.stream.current
 847                 if token.type == 'data':
 848                     if token.value:
 849                         add_data(nodes.TemplateData(token.value,
 850                                                     lineno=token.lineno))
 851                     next(self.stream)
 852                 elif token.type == 'variable_begin':
 853                     next(self.stream)
 854                     add_data(self.parse_tuple(with_condexpr=True))
 855                     self.stream.expect('variable_end')
 856                 elif token.type == 'block_begin':
 857                     flush_data()
 858                     next(self.stream)
 859                     if end_tokens is not None and \
 860                        self.stream.current.test_any(*end_tokens):
 861                         return body
 862                     rv = self.parse_statement()
 863                     if isinstance(rv, list):
 864                         body.extend(rv)
 865                     else:
 866                         body.append(rv)
 867                     self.stream.expect('block_end')
 868                 else:
 869                     raise AssertionError('internal parsing error')
 870
 871             flush_data()
 872         finally:
 873             if end_tokens is not None:
 874                 self._end_token_stack.pop()
 875
 876         return body
 877
 878     def parse(self):
 879         """Parse the whole template into a `Template` node."""
 880         result = nodes.Template(self.subparse(), lineno=1)
 881         result.set_environment(self.environment)
 882         return result