1 # -*- coding: utf-8 -*-
6 Implements the template parser.
8 :copyright: (c) 2010 by the Jinja Team.
9 :license: BSD, see LICENSE for more details.
11 from jinja2 import nodes
12 from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
13 from jinja2.utils import next
14 from jinja2.lexer import describe_token, describe_token_expr
17 #: statements that callinto
18 _statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
19 'macro', 'include', 'from', 'import',
21 _compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
25 """This is the central parsing class Jinja2 uses. It's passed to
26 extensions and can be used to parse expressions or statements.
29 def __init__(self, environment, source, name=None, filename=None,
31 self.environment = environment
32 self.stream = environment._tokenize(source, name, filename, state)
34 self.filename = filename
37 for extension in environment.extensions.itervalues():
38 for tag in extension.tags:
39 self.extensions[tag] = extension.parse
40 self._last_identifier = 0
42 self._end_token_stack = []
44 def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
45 """Convenience method that raises `exc` with the message, passed
46 line number or last line number as well as the current name and
50 lineno = self.stream.current.lineno
51 raise exc(msg, lineno, self.name, self.filename)
53 def _fail_ut_eof(self, name, end_token_stack, lineno):
55 for exprs in end_token_stack:
56 expected.extend(map(describe_token_expr, exprs))
58 currently_looking = ' or '.join(
59 "'%s'" % describe_token_expr(expr)
60 for expr in end_token_stack[-1])
62 currently_looking = None
65 message = ['Unexpected end of template.']
67 message = ['Encountered unknown tag \'%s\'.' % name]
70 if name is not None and name in expected:
71 message.append('You probably made a nesting mistake. Jinja '
72 'is expecting this tag, but currently looking '
73 'for %s.' % currently_looking)
75 message.append('Jinja was looking for the following tags: '
76 '%s.' % currently_looking)
79 message.append('The innermost block that needs to be '
80 'closed is \'%s\'.' % self._tag_stack[-1])
82 self.fail(' '.join(message), lineno)
84 def fail_unknown_tag(self, name, lineno=None):
85 """Called if the parser encounters an unknown tag. Tries to fail
86 with a human readable error message that could help to identify
89 return self._fail_ut_eof(name, self._end_token_stack, lineno)
91 def fail_eof(self, end_tokens=None, lineno=None):
92 """Like fail_unknown_tag but for end of template situations."""
93 stack = list(self._end_token_stack)
94 if end_tokens is not None:
95 stack.append(end_tokens)
96 return self._fail_ut_eof(None, stack, lineno)
98 def is_tuple_end(self, extra_end_rules=None):
99 """Are we at the end of a tuple?"""
100 if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
102 elif extra_end_rules is not None:
103 return self.stream.current.test_any(extra_end_rules)
106 def free_identifier(self, lineno=None):
107 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
108 self._last_identifier += 1
109 rv = object.__new__(nodes.InternalName)
110 nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
113 def parse_statement(self):
114 """Parse a single statement."""
115 token = self.stream.current
116 if token.type != 'name':
117 self.fail('tag name expected', token.lineno)
118 self._tag_stack.append(token.value)
121 if token.value in _statement_keywords:
122 return getattr(self, 'parse_' + self.stream.current.value)()
123 if token.value == 'call':
124 return self.parse_call_block()
125 if token.value == 'filter':
126 return self.parse_filter_block()
127 ext = self.extensions.get(token.value)
131 # did not work out, remove the token we pushed by accident
132 # from the stack so that the unknown tag fail function can
133 # produce a proper error message.
134 self._tag_stack.pop()
136 self.fail_unknown_tag(token.value, token.lineno)
139 self._tag_stack.pop()
141 def parse_statements(self, end_tokens, drop_needle=False):
142 """Parse multiple statements into a list until one of the end tokens
143 is reached. This is used to parse the body of statements as it also
144 parses template data if appropriate. The parser checks first if the
145 current token is a colon and skips it if there is one. Then it checks
146 for the block end and parses until if one of the `end_tokens` is
147 reached. Per default the active token in the stream at the end of
148 the call is the matched end token. If this is not wanted `drop_needle`
149 can be set to `True` and the end token is removed.
151 # the first token may be a colon for python compatibility
152 self.stream.skip_if('colon')
154 # in the future it would be possible to add whole code sections
155 # by adding some sort of end of statement token and parsing those here.
156 self.stream.expect('block_end')
157 result = self.subparse(end_tokens)
159 # we reached the end of the template too early, the subparser
160 # does not check for this, so we do that now
161 if self.stream.current.type == 'eof':
162 self.fail_eof(end_tokens)
169 """Parse an assign statement."""
170 lineno = next(self.stream).lineno
171 target = self.parse_assign_target()
172 self.stream.expect('assign')
173 expr = self.parse_tuple()
174 return nodes.Assign(target, expr, lineno=lineno)
177 """Parse a for loop."""
178 lineno = self.stream.expect('name:for').lineno
179 target = self.parse_assign_target(extra_end_rules=('name:in',))
180 self.stream.expect('name:in')
181 iter = self.parse_tuple(with_condexpr=False,
182 extra_end_rules=('name:recursive',))
184 if self.stream.skip_if('name:if'):
185 test = self.parse_expression()
186 recursive = self.stream.skip_if('name:recursive')
187 body = self.parse_statements(('name:endfor', 'name:else'))
188 if next(self.stream).value == 'endfor':
191 else_ = self.parse_statements(('name:endfor',), drop_needle=True)
192 return nodes.For(target, iter, body, else_, test,
193 recursive, lineno=lineno)
196 """Parse an if construct."""
197 node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
199 node.test = self.parse_tuple(with_condexpr=False)
200 node.body = self.parse_statements(('name:elif', 'name:else',
202 token = next(self.stream)
203 if token.test('name:elif'):
204 new_node = nodes.If(lineno=self.stream.current.lineno)
205 node.else_ = [new_node]
208 elif token.test('name:else'):
209 node.else_ = self.parse_statements(('name:endif',),
216 def parse_block(self):
217 node = nodes.Block(lineno=next(self.stream).lineno)
218 node.name = self.stream.expect('name').value
219 node.scoped = self.stream.skip_if('name:scoped')
221 # common problem people encounter when switching from django
222 # to jinja. we do not support hyphens in block names, so let's
223 # raise a nicer error message in that case.
224 if self.stream.current.type == 'sub':
225 self.fail('Block names in Jinja have to be valid Python '
226 'identifiers and may not contain hypens, use an '
227 'underscore instead.')
229 node.body = self.parse_statements(('name:endblock',), drop_needle=True)
230 self.stream.skip_if('name:' + node.name)
233 def parse_extends(self):
234 node = nodes.Extends(lineno=next(self.stream).lineno)
235 node.template = self.parse_expression()
238 def parse_import_context(self, node, default):
239 if self.stream.current.test_any('name:with', 'name:without') and \
240 self.stream.look().test('name:context'):
241 node.with_context = next(self.stream).value == 'with'
244 node.with_context = default
247 def parse_include(self):
248 node = nodes.Include(lineno=next(self.stream).lineno)
249 node.template = self.parse_expression()
250 if self.stream.current.test('name:ignore') and \
251 self.stream.look().test('name:missing'):
252 node.ignore_missing = True
255 node.ignore_missing = False
256 return self.parse_import_context(node, True)
258 def parse_import(self):
259 node = nodes.Import(lineno=next(self.stream).lineno)
260 node.template = self.parse_expression()
261 self.stream.expect('name:as')
262 node.target = self.parse_assign_target(name_only=True).name
263 return self.parse_import_context(node, False)
265 def parse_from(self):
266 node = nodes.FromImport(lineno=next(self.stream).lineno)
267 node.template = self.parse_expression()
268 self.stream.expect('name:import')
272 if self.stream.current.value in ('with', 'without') and \
273 self.stream.look().test('name:context'):
274 node.with_context = next(self.stream).value == 'with'
281 self.stream.expect('comma')
282 if self.stream.current.type == 'name':
285 target = self.parse_assign_target(name_only=True)
286 if target.name.startswith('_'):
287 self.fail('names starting with an underline can not '
288 'be imported', target.lineno,
289 exc=TemplateAssertionError)
290 if self.stream.skip_if('name:as'):
291 alias = self.parse_assign_target(name_only=True)
292 node.names.append((target.name, alias.name))
294 node.names.append(target.name)
295 if parse_context() or self.stream.current.type != 'comma':
299 if not hasattr(node, 'with_context'):
300 node.with_context = False
301 self.stream.skip_if('comma')
304 def parse_signature(self, node):
305 node.args = args = []
306 node.defaults = defaults = []
307 self.stream.expect('lparen')
308 while self.stream.current.type != 'rparen':
310 self.stream.expect('comma')
311 arg = self.parse_assign_target(name_only=True)
313 if self.stream.skip_if('assign'):
314 defaults.append(self.parse_expression())
316 self.stream.expect('rparen')
318 def parse_call_block(self):
319 node = nodes.CallBlock(lineno=next(self.stream).lineno)
320 if self.stream.current.type == 'lparen':
321 self.parse_signature(node)
326 node.call = self.parse_expression()
327 if not isinstance(node.call, nodes.Call):
328 self.fail('expected call', node.lineno)
329 node.body = self.parse_statements(('name:endcall',), drop_needle=True)
332 def parse_filter_block(self):
333 node = nodes.FilterBlock(lineno=next(self.stream).lineno)
334 node.filter = self.parse_filter(None, start_inline=True)
335 node.body = self.parse_statements(('name:endfilter',),
339 def parse_macro(self):
340 node = nodes.Macro(lineno=next(self.stream).lineno)
341 node.name = self.parse_assign_target(name_only=True).name
342 self.parse_signature(node)
343 node.body = self.parse_statements(('name:endmacro',),
347 def parse_print(self):
348 node = nodes.Output(lineno=next(self.stream).lineno)
350 while self.stream.current.type != 'block_end':
352 self.stream.expect('comma')
353 node.nodes.append(self.parse_expression())
356 def parse_assign_target(self, with_tuple=True, name_only=False,
357 extra_end_rules=None):
358 """Parse an assignment target. As Jinja2 allows assignments to
359 tuples, this function can parse all allowed assignment targets. Per
360 default assignments to tuples are parsed, that can be disable however
361 by setting `with_tuple` to `False`. If only assignments to names are
362 wanted `name_only` can be set to `True`. The `extra_end_rules`
363 parameter is forwarded to the tuple parsing function.
366 token = self.stream.expect('name')
367 target = nodes.Name(token.value, 'store', lineno=token.lineno)
370 target = self.parse_tuple(simplified=True,
371 extra_end_rules=extra_end_rules)
373 target = self.parse_primary(with_postfix=False)
374 target.set_ctx('store')
375 if not target.can_assign():
376 self.fail('can\'t assign to %r' % target.__class__.
377 __name__.lower(), target.lineno)
380 def parse_expression(self, with_condexpr=True):
381 """Parse an expression. Per default all expressions are parsed, if
382 the optional `with_condexpr` parameter is set to `False` conditional
383 expressions are not parsed.
386 return self.parse_condexpr()
387 return self.parse_or()
389 def parse_condexpr(self):
390 lineno = self.stream.current.lineno
391 expr1 = self.parse_or()
392 while self.stream.skip_if('name:if'):
393 expr2 = self.parse_or()
394 if self.stream.skip_if('name:else'):
395 expr3 = self.parse_condexpr()
398 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
399 lineno = self.stream.current.lineno
403 lineno = self.stream.current.lineno
404 left = self.parse_and()
405 while self.stream.skip_if('name:or'):
406 right = self.parse_and()
407 left = nodes.Or(left, right, lineno=lineno)
408 lineno = self.stream.current.lineno
412 lineno = self.stream.current.lineno
413 left = self.parse_not()
414 while self.stream.skip_if('name:and'):
415 right = self.parse_not()
416 left = nodes.And(left, right, lineno=lineno)
417 lineno = self.stream.current.lineno
421 if self.stream.current.test('name:not'):
422 lineno = next(self.stream).lineno
423 return nodes.Not(self.parse_not(), lineno=lineno)
424 return self.parse_compare()
426 def parse_compare(self):
427 lineno = self.stream.current.lineno
428 expr = self.parse_add()
431 token_type = self.stream.current.type
432 if token_type in _compare_operators:
434 ops.append(nodes.Operand(token_type, self.parse_add()))
435 elif self.stream.skip_if('name:in'):
436 ops.append(nodes.Operand('in', self.parse_add()))
437 elif self.stream.current.test('name:not') and \
438 self.stream.look().test('name:in'):
440 ops.append(nodes.Operand('notin', self.parse_add()))
443 lineno = self.stream.current.lineno
446 return nodes.Compare(expr, ops, lineno=lineno)
449 lineno = self.stream.current.lineno
450 left = self.parse_sub()
451 while self.stream.current.type == 'add':
453 right = self.parse_sub()
454 left = nodes.Add(left, right, lineno=lineno)
455 lineno = self.stream.current.lineno
459 lineno = self.stream.current.lineno
460 left = self.parse_concat()
461 while self.stream.current.type == 'sub':
463 right = self.parse_concat()
464 left = nodes.Sub(left, right, lineno=lineno)
465 lineno = self.stream.current.lineno
468 def parse_concat(self):
469 lineno = self.stream.current.lineno
470 args = [self.parse_mul()]
471 while self.stream.current.type == 'tilde':
473 args.append(self.parse_mul())
476 return nodes.Concat(args, lineno=lineno)
479 lineno = self.stream.current.lineno
480 left = self.parse_div()
481 while self.stream.current.type == 'mul':
483 right = self.parse_div()
484 left = nodes.Mul(left, right, lineno=lineno)
485 lineno = self.stream.current.lineno
489 lineno = self.stream.current.lineno
490 left = self.parse_floordiv()
491 while self.stream.current.type == 'div':
493 right = self.parse_floordiv()
494 left = nodes.Div(left, right, lineno=lineno)
495 lineno = self.stream.current.lineno
498 def parse_floordiv(self):
499 lineno = self.stream.current.lineno
500 left = self.parse_mod()
501 while self.stream.current.type == 'floordiv':
503 right = self.parse_mod()
504 left = nodes.FloorDiv(left, right, lineno=lineno)
505 lineno = self.stream.current.lineno
509 lineno = self.stream.current.lineno
510 left = self.parse_pow()
511 while self.stream.current.type == 'mod':
513 right = self.parse_pow()
514 left = nodes.Mod(left, right, lineno=lineno)
515 lineno = self.stream.current.lineno
519 lineno = self.stream.current.lineno
520 left = self.parse_unary()
521 while self.stream.current.type == 'pow':
523 right = self.parse_unary()
524 left = nodes.Pow(left, right, lineno=lineno)
525 lineno = self.stream.current.lineno
528 def parse_unary(self):
529 token_type = self.stream.current.type
530 lineno = self.stream.current.lineno
531 if token_type == 'sub':
533 node = self.parse_unary()
534 return nodes.Neg(node, lineno=lineno)
535 if token_type == 'add':
537 node = self.parse_unary()
538 return nodes.Pos(node, lineno=lineno)
539 return self.parse_primary()
541 def parse_primary(self, with_postfix=True):
542 token = self.stream.current
543 if token.type == 'name':
544 if token.value in ('true', 'false', 'True', 'False'):
545 node = nodes.Const(token.value in ('true', 'True'),
547 elif token.value in ('none', 'None'):
548 node = nodes.Const(None, lineno=token.lineno)
550 node = nodes.Name(token.value, 'load', lineno=token.lineno)
552 elif token.type == 'string':
555 lineno = token.lineno
556 while self.stream.current.type == 'string':
557 buf.append(self.stream.current.value)
559 node = nodes.Const(''.join(buf), lineno=lineno)
560 elif token.type in ('integer', 'float'):
562 node = nodes.Const(token.value, lineno=token.lineno)
563 elif token.type == 'lparen':
565 node = self.parse_tuple(explicit_parentheses=True)
566 self.stream.expect('rparen')
567 elif token.type == 'lbracket':
568 node = self.parse_list()
569 elif token.type == 'lbrace':
570 node = self.parse_dict()
572 self.fail("unexpected '%s'" % describe_token(token), token.lineno)
574 node = self.parse_postfix(node)
577 def parse_tuple(self, simplified=False, with_condexpr=True,
578 extra_end_rules=None, explicit_parentheses=False):
579 """Works like `parse_expression` but if multiple expressions are
580 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
581 This method could also return a regular expression instead of a tuple
582 if no commas where found.
584 The default parsing mode is a full tuple. If `simplified` is `True`
585 only names and literals are parsed. The `no_condexpr` parameter is
586 forwarded to :meth:`parse_expression`.
588 Because tuples do not require delimiters and may end in a bogus comma
589 an extra hint is needed that marks the end of a tuple. For example
590 for loops support tuples between `for` and `in`. In that case the
591 `extra_end_rules` is set to ``['name:in']``.
593 `explicit_parentheses` is true if the parsing was triggered by an
594 expression in parentheses. This is used to figure out if an empty
595 tuple is a valid expression or not.
597 lineno = self.stream.current.lineno
599 parse = lambda: self.parse_primary(with_postfix=False)
601 parse = self.parse_expression
603 parse = lambda: self.parse_expression(with_condexpr=False)
608 self.stream.expect('comma')
609 if self.is_tuple_end(extra_end_rules):
612 if self.stream.current.type == 'comma':
616 lineno = self.stream.current.lineno
622 # if we don't have explicit parentheses, an empty tuple is
623 # not a valid expression. This would mean nothing (literally
624 # nothing) in the spot of an expression would be an empty
626 if not explicit_parentheses:
627 self.fail('Expected an expression, got \'%s\'' %
628 describe_token(self.stream.current))
630 return nodes.Tuple(args, 'load', lineno=lineno)
632 def parse_list(self):
633 token = self.stream.expect('lbracket')
635 while self.stream.current.type != 'rbracket':
637 self.stream.expect('comma')
638 if self.stream.current.type == 'rbracket':
640 items.append(self.parse_expression())
641 self.stream.expect('rbracket')
642 return nodes.List(items, lineno=token.lineno)
644 def parse_dict(self):
645 token = self.stream.expect('lbrace')
647 while self.stream.current.type != 'rbrace':
649 self.stream.expect('comma')
650 if self.stream.current.type == 'rbrace':
652 key = self.parse_expression()
653 self.stream.expect('colon')
654 value = self.parse_expression()
655 items.append(nodes.Pair(key, value, lineno=key.lineno))
656 self.stream.expect('rbrace')
657 return nodes.Dict(items, lineno=token.lineno)
659 def parse_postfix(self, node):
661 token_type = self.stream.current.type
662 if token_type == 'dot' or token_type == 'lbracket':
663 node = self.parse_subscript(node)
664 elif token_type == 'lparen':
665 node = self.parse_call(node)
666 elif token_type == 'pipe':
667 node = self.parse_filter(node)
668 elif token_type == 'name' and self.stream.current.value == 'is':
669 node = self.parse_test(node)
674 def parse_subscript(self, node):
675 token = next(self.stream)
676 if token.type == 'dot':
677 attr_token = self.stream.current
679 if attr_token.type == 'name':
680 return nodes.Getattr(node, attr_token.value, 'load',
682 elif attr_token.type != 'integer':
683 self.fail('expected name or number', attr_token.lineno)
684 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
685 return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
686 if token.type == 'lbracket':
687 priority_on_attribute = False
689 while self.stream.current.type != 'rbracket':
691 self.stream.expect('comma')
692 args.append(self.parse_subscribed())
693 self.stream.expect('rbracket')
697 arg = nodes.Tuple(args, 'load', lineno=token.lineno)
698 return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
699 self.fail('expected subscript expression', self.lineno)
701 def parse_subscribed(self):
702 lineno = self.stream.current.lineno
704 if self.stream.current.type == 'colon':
708 node = self.parse_expression()
709 if self.stream.current.type != 'colon':
714 if self.stream.current.type == 'colon':
716 elif self.stream.current.type not in ('rbracket', 'comma'):
717 args.append(self.parse_expression())
721 if self.stream.current.type == 'colon':
723 if self.stream.current.type not in ('rbracket', 'comma'):
724 args.append(self.parse_expression())
730 return nodes.Slice(lineno=lineno, *args)
732 def parse_call(self, node):
733 token = self.stream.expect('lparen')
736 dyn_args = dyn_kwargs = None
737 require_comma = False
741 self.fail('invalid syntax for function call expression',
744 while self.stream.current.type != 'rparen':
746 self.stream.expect('comma')
747 # support for trailing comma
748 if self.stream.current.type == 'rparen':
750 if self.stream.current.type == 'mul':
751 ensure(dyn_args is None and dyn_kwargs is None)
753 dyn_args = self.parse_expression()
754 elif self.stream.current.type == 'pow':
755 ensure(dyn_kwargs is None)
757 dyn_kwargs = self.parse_expression()
759 ensure(dyn_args is None and dyn_kwargs is None)
760 if self.stream.current.type == 'name' and \
761 self.stream.look().type == 'assign':
762 key = self.stream.current.value
764 value = self.parse_expression()
765 kwargs.append(nodes.Keyword(key, value,
766 lineno=value.lineno))
769 args.append(self.parse_expression())
772 self.stream.expect('rparen')
775 return args, kwargs, dyn_args, dyn_kwargs
776 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
779 def parse_filter(self, node, start_inline=False):
780 while self.stream.current.type == 'pipe' or start_inline:
783 token = self.stream.expect('name')
785 while self.stream.current.type == 'dot':
787 name += '.' + self.stream.expect('name').value
788 if self.stream.current.type == 'lparen':
789 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
793 dyn_args = dyn_kwargs = None
794 node = nodes.Filter(node, name, args, kwargs, dyn_args,
795 dyn_kwargs, lineno=token.lineno)
799 def parse_test(self, node):
800 token = next(self.stream)
801 if self.stream.current.test('name:not'):
806 name = self.stream.expect('name').value
807 while self.stream.current.type == 'dot':
809 name += '.' + self.stream.expect('name').value
810 dyn_args = dyn_kwargs = None
812 if self.stream.current.type == 'lparen':
813 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
814 elif self.stream.current.type in ('name', 'string', 'integer',
815 'float', 'lparen', 'lbracket',
817 self.stream.current.test_any('name:else', 'name:or',
819 if self.stream.current.test('name:is'):
820 self.fail('You cannot chain multiple tests with is')
821 args = [self.parse_expression()]
824 node = nodes.Test(node, name, args, kwargs, dyn_args,
825 dyn_kwargs, lineno=token.lineno)
827 node = nodes.Not(node, lineno=token.lineno)
830 def subparse(self, end_tokens=None):
833 add_data = data_buffer.append
835 if end_tokens is not None:
836 self._end_token_stack.append(end_tokens)
840 lineno = data_buffer[0].lineno
841 body.append(nodes.Output(data_buffer[:], lineno=lineno))
846 token = self.stream.current
847 if token.type == 'data':
849 add_data(nodes.TemplateData(token.value,
850 lineno=token.lineno))
852 elif token.type == 'variable_begin':
854 add_data(self.parse_tuple(with_condexpr=True))
855 self.stream.expect('variable_end')
856 elif token.type == 'block_begin':
859 if end_tokens is not None and \
860 self.stream.current.test_any(*end_tokens):
862 rv = self.parse_statement()
863 if isinstance(rv, list):
867 self.stream.expect('block_end')
869 raise AssertionError('internal parsing error')
873 if end_tokens is not None:
874 self._end_token_stack.pop()
879 """Parse the whole template into a `Template` node."""
880 result = nodes.Template(self.subparse(), lineno=1)
881 result.set_environment(self.environment)