1 # -*- coding: utf-8 -*-
6 Implements the template parser.
8 :copyright: 2008 by Armin Ronacher.
9 :license: BSD, see LICENSE for more details.
11 from jinja2 import nodes
12 from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
15 _statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
16 'macro', 'include', 'from', 'import',
18 _compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
22 """This is the central parsing class Jinja2 uses. It's passed to
23 extensions and can be used to parse expressions or statements.
26 def __init__(self, environment, source, name=None, filename=None,
28 self.environment = environment
29 self.stream = environment._tokenize(source, name, filename, state)
31 self.filename = filename
34 for extension in environment.extensions.itervalues():
35 for tag in extension.tags:
36 self.extensions[tag] = extension.parse
37 self._last_identifier = 0
39 def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
40 """Convenience method that raises `exc` with the message, passed
41 line number or last line number as well as the current name and
45 lineno = self.stream.current.lineno
46 raise exc(msg, lineno, self.name, self.filename)
48 def is_tuple_end(self, extra_end_rules=None):
49 """Are we at the end of a tuple?"""
50 if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
52 elif extra_end_rules is not None:
53 return self.stream.current.test_any(extra_end_rules)
56 def free_identifier(self, lineno=None):
57 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
58 self._last_identifier += 1
59 rv = object.__new__(nodes.InternalName)
60 nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
63 def parse_statement(self):
64 """Parse a single statement."""
65 token = self.stream.current
66 if token.type is not 'name':
67 self.fail('tag name expected', token.lineno)
68 if token.value in _statement_keywords:
69 return getattr(self, 'parse_' + self.stream.current.value)()
70 if token.value == 'call':
71 return self.parse_call_block()
72 if token.value == 'filter':
73 return self.parse_filter_block()
74 ext = self.extensions.get(token.value)
77 self.fail('unknown tag %r' % token.value, token.lineno)
79 def parse_statements(self, end_tokens, drop_needle=False):
80 """Parse multiple statements into a list until one of the end tokens
81 is reached. This is used to parse the body of statements as it also
82 parses template data if appropriate. The parser checks first if the
83 current token is a colon and skips it if there is one. Then it checks
84 for the block end and parses until if one of the `end_tokens` is
85 reached. Per default the active token in the stream at the end of
86 the call is the matched end token. If this is not wanted `drop_needle`
87 can be set to `True` and the end token is removed.
89 # the first token may be a colon for python compatibility
90 self.stream.skip_if('colon')
92 # in the future it would be possible to add whole code sections
93 # by adding some sort of end of statement token and parsing those here.
94 self.stream.expect('block_end')
95 result = self.subparse(end_tokens)
102 """Parse an assign statement."""
103 lineno = self.stream.next().lineno
104 target = self.parse_assign_target()
105 self.stream.expect('assign')
106 expr = self.parse_tuple()
107 return nodes.Assign(target, expr, lineno=lineno)
110 """Parse a for loop."""
111 lineno = self.stream.expect('name:for').lineno
112 target = self.parse_assign_target(extra_end_rules=('name:in',))
113 self.stream.expect('name:in')
114 iter = self.parse_tuple(with_condexpr=False,
115 extra_end_rules=('name:recursive',))
117 if self.stream.skip_if('name:if'):
118 test = self.parse_expression()
119 recursive = self.stream.skip_if('name:recursive')
120 body = self.parse_statements(('name:endfor', 'name:else'))
121 if self.stream.next().value == 'endfor':
124 else_ = self.parse_statements(('name:endfor',), drop_needle=True)
125 return nodes.For(target, iter, body, else_, test,
126 recursive, lineno=lineno)
129 """Parse an if construct."""
130 node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
132 node.test = self.parse_tuple(with_condexpr=False)
133 node.body = self.parse_statements(('name:elif', 'name:else',
135 token = self.stream.next()
136 if token.test('name:elif'):
137 new_node = nodes.If(lineno=self.stream.current.lineno)
138 node.else_ = [new_node]
141 elif token.test('name:else'):
142 node.else_ = self.parse_statements(('name:endif',),
149 def parse_block(self):
150 node = nodes.Block(lineno=self.stream.next().lineno)
151 node.name = self.stream.expect('name').value
152 node.body = self.parse_statements(('name:endblock',), drop_needle=True)
153 self.stream.skip_if('name:' + node.name)
156 def parse_extends(self):
157 node = nodes.Extends(lineno=self.stream.next().lineno)
158 node.template = self.parse_expression()
161 def parse_import_context(self, node, default):
162 if self.stream.current.test_any('name:with', 'name:without') and \
163 self.stream.look().test('name:context'):
164 node.with_context = self.stream.next().value == 'with'
167 node.with_context = default
170 def parse_include(self):
171 node = nodes.Include(lineno=self.stream.next().lineno)
172 node.template = self.parse_expression()
173 return self.parse_import_context(node, True)
175 def parse_import(self):
176 node = nodes.Import(lineno=self.stream.next().lineno)
177 node.template = self.parse_expression()
178 self.stream.expect('name:as')
179 node.target = self.parse_assign_target(name_only=True).name
180 return self.parse_import_context(node, False)
182 def parse_from(self):
183 node = nodes.FromImport(lineno=self.stream.next().lineno)
184 node.template = self.parse_expression()
185 self.stream.expect('name:import')
189 if self.stream.current.value in ('with', 'without') and \
190 self.stream.look().test('name:context'):
191 node.with_context = self.stream.next().value == 'with'
198 self.stream.expect('comma')
199 if self.stream.current.type is 'name':
202 target = self.parse_assign_target(name_only=True)
203 if target.name.startswith('_'):
204 self.fail('names starting with an underline can not '
205 'be imported', target.lineno,
206 exc=TemplateAssertionError)
207 if self.stream.skip_if('name:as'):
208 alias = self.parse_assign_target(name_only=True)
209 node.names.append((target.name, alias.name))
211 node.names.append(target.name)
212 if parse_context() or self.stream.current.type is not 'comma':
216 if not hasattr(node, 'with_context'):
217 node.with_context = False
218 self.stream.skip_if('comma')
221 def parse_signature(self, node):
222 node.args = args = []
223 node.defaults = defaults = []
224 self.stream.expect('lparen')
225 while self.stream.current.type is not 'rparen':
227 self.stream.expect('comma')
228 arg = self.parse_assign_target(name_only=True)
230 if self.stream.skip_if('assign'):
231 defaults.append(self.parse_expression())
233 self.stream.expect('rparen')
235 def parse_call_block(self):
236 node = nodes.CallBlock(lineno=self.stream.next().lineno)
237 if self.stream.current.type is 'lparen':
238 self.parse_signature(node)
243 node.call = self.parse_expression()
244 if not isinstance(node.call, nodes.Call):
245 self.fail('expected call', node.lineno)
246 node.body = self.parse_statements(('name:endcall',), drop_needle=True)
249 def parse_filter_block(self):
250 node = nodes.FilterBlock(lineno=self.stream.next().lineno)
251 node.filter = self.parse_filter(None, start_inline=True)
252 node.body = self.parse_statements(('name:endfilter',),
256 def parse_macro(self):
257 node = nodes.Macro(lineno=self.stream.next().lineno)
258 node.name = self.parse_assign_target(name_only=True).name
259 self.parse_signature(node)
260 node.body = self.parse_statements(('name:endmacro',),
264 def parse_print(self):
265 node = nodes.Output(lineno=self.stream.next().lineno)
267 while self.stream.current.type is not 'block_end':
269 self.stream.expect('comma')
270 node.nodes.append(self.parse_expression())
273 def parse_assign_target(self, with_tuple=True, name_only=False,
274 extra_end_rules=None):
275 """Parse an assignment target. As Jinja2 allows assignments to
276 tuples, this function can parse all allowed assignment targets. Per
277 default assignments to tuples are parsed, that can be disable however
278 by setting `with_tuple` to `False`. If only assignments to names are
279 wanted `name_only` can be set to `True`. The `extra_end_rules`
280 parameter is forwarded to the tuple parsing function.
283 token = self.stream.expect('name')
284 target = nodes.Name(token.value, 'store', lineno=token.lineno)
287 target = self.parse_tuple(simplified=True,
288 extra_end_rules=extra_end_rules)
290 target = self.parse_primary(with_postfix=False)
291 target.set_ctx('store')
292 if not target.can_assign():
293 self.fail('can\'t assign to %r' % target.__class__.
294 __name__.lower(), target.lineno)
297 def parse_expression(self, with_condexpr=True):
298 """Parse an expression. Per default all expressions are parsed, if
299 the optional `with_condexpr` parameter is set to `False` conditional
300 expressions are not parsed.
303 return self.parse_condexpr()
304 return self.parse_or()
306 def parse_condexpr(self):
307 lineno = self.stream.current.lineno
308 expr1 = self.parse_or()
309 while self.stream.skip_if('name:if'):
310 expr2 = self.parse_or()
311 if self.stream.skip_if('name:else'):
312 expr3 = self.parse_condexpr()
315 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
316 lineno = self.stream.current.lineno
320 lineno = self.stream.current.lineno
321 left = self.parse_and()
322 while self.stream.skip_if('name:or'):
323 right = self.parse_and()
324 left = nodes.Or(left, right, lineno=lineno)
325 lineno = self.stream.current.lineno
329 lineno = self.stream.current.lineno
330 left = self.parse_compare()
331 while self.stream.skip_if('name:and'):
332 right = self.parse_compare()
333 left = nodes.And(left, right, lineno=lineno)
334 lineno = self.stream.current.lineno
337 def parse_compare(self):
338 lineno = self.stream.current.lineno
339 expr = self.parse_add()
342 token_type = self.stream.current.type
343 if token_type in _compare_operators:
345 ops.append(nodes.Operand(token_type, self.parse_add()))
346 elif self.stream.skip_if('name:in'):
347 ops.append(nodes.Operand('in', self.parse_add()))
348 elif self.stream.current.test('name:not') and \
349 self.stream.look().test('name:in'):
351 ops.append(nodes.Operand('notin', self.parse_add()))
354 lineno = self.stream.current.lineno
357 return nodes.Compare(expr, ops, lineno=lineno)
360 lineno = self.stream.current.lineno
361 left = self.parse_sub()
362 while self.stream.current.type is 'add':
364 right = self.parse_sub()
365 left = nodes.Add(left, right, lineno=lineno)
366 lineno = self.stream.current.lineno
370 lineno = self.stream.current.lineno
371 left = self.parse_concat()
372 while self.stream.current.type is 'sub':
374 right = self.parse_concat()
375 left = nodes.Sub(left, right, lineno=lineno)
376 lineno = self.stream.current.lineno
379 def parse_concat(self):
380 lineno = self.stream.current.lineno
381 args = [self.parse_mul()]
382 while self.stream.current.type is 'tilde':
384 args.append(self.parse_mul())
387 return nodes.Concat(args, lineno=lineno)
390 lineno = self.stream.current.lineno
391 left = self.parse_div()
392 while self.stream.current.type is 'mul':
394 right = self.parse_div()
395 left = nodes.Mul(left, right, lineno=lineno)
396 lineno = self.stream.current.lineno
400 lineno = self.stream.current.lineno
401 left = self.parse_floordiv()
402 while self.stream.current.type is 'div':
404 right = self.parse_floordiv()
405 left = nodes.Div(left, right, lineno=lineno)
406 lineno = self.stream.current.lineno
409 def parse_floordiv(self):
410 lineno = self.stream.current.lineno
411 left = self.parse_mod()
412 while self.stream.current.type is 'floordiv':
414 right = self.parse_mod()
415 left = nodes.FloorDiv(left, right, lineno=lineno)
416 lineno = self.stream.current.lineno
420 lineno = self.stream.current.lineno
421 left = self.parse_pow()
422 while self.stream.current.type is 'mod':
424 right = self.parse_pow()
425 left = nodes.Mod(left, right, lineno=lineno)
426 lineno = self.stream.current.lineno
430 lineno = self.stream.current.lineno
431 left = self.parse_unary()
432 while self.stream.current.type is 'pow':
434 right = self.parse_unary()
435 left = nodes.Pow(left, right, lineno=lineno)
436 lineno = self.stream.current.lineno
439 def parse_unary(self):
440 token_type = self.stream.current.type
441 lineno = self.stream.current.lineno
442 if token_type is 'name' and self.stream.current.value == 'not':
444 node = self.parse_unary()
445 return nodes.Not(node, lineno=lineno)
446 if token_type is 'sub':
448 node = self.parse_unary()
449 return nodes.Neg(node, lineno=lineno)
450 if token_type is 'add':
452 node = self.parse_unary()
453 return nodes.Pos(node, lineno=lineno)
454 return self.parse_primary()
456 def parse_primary(self, with_postfix=True):
457 token = self.stream.current
458 if token.type is 'name':
459 if token.value in ('true', 'false', 'True', 'False'):
460 node = nodes.Const(token.value in ('true', 'True'),
462 elif token.value in ('none', 'None'):
463 node = nodes.Const(None, lineno=token.lineno)
465 node = nodes.Name(token.value, 'load', lineno=token.lineno)
467 elif token.type is 'string':
470 lineno = token.lineno
471 while self.stream.current.type is 'string':
472 buf.append(self.stream.current.value)
474 node = nodes.Const(''.join(buf), lineno=lineno)
475 elif token.type in ('integer', 'float'):
477 node = nodes.Const(token.value, lineno=token.lineno)
478 elif token.type is 'lparen':
480 node = self.parse_tuple()
481 self.stream.expect('rparen')
482 elif token.type is 'lbracket':
483 node = self.parse_list()
484 elif token.type is 'lbrace':
485 node = self.parse_dict()
487 self.fail("unexpected token '%s'" % (token,), token.lineno)
489 node = self.parse_postfix(node)
492 def parse_tuple(self, simplified=False, with_condexpr=True,
493 extra_end_rules=None):
494 """Works like `parse_expression` but if multiple expressions are
495 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
496 This method could also return a regular expression instead of a tuple
497 if no commas where found.
499 The default parsing mode is a full tuple. If `simplified` is `True`
500 only names and literals are parsed. The `no_condexpr` parameter is
501 forwarded to :meth:`parse_expression`.
503 Because tuples do not require delimiters and may end in a bogus comma
504 an extra hint is needed that marks the end of a tuple. For example
505 for loops support tuples between `for` and `in`. In that case the
506 `extra_end_rules` is set to ``['name:in']``.
508 lineno = self.stream.current.lineno
510 parse = lambda: self.parse_primary(with_postfix=False)
512 parse = self.parse_expression
514 parse = lambda: self.parse_expression(with_condexpr=False)
519 self.stream.expect('comma')
520 if self.is_tuple_end(extra_end_rules):
523 if self.stream.current.type is 'comma':
527 lineno = self.stream.current.lineno
528 if not is_tuple and args:
530 return nodes.Tuple(args, 'load', lineno=lineno)
532 def parse_list(self):
533 token = self.stream.expect('lbracket')
535 while self.stream.current.type is not 'rbracket':
537 self.stream.expect('comma')
538 if self.stream.current.type == 'rbracket':
540 items.append(self.parse_expression())
541 self.stream.expect('rbracket')
542 return nodes.List(items, lineno=token.lineno)
544 def parse_dict(self):
545 token = self.stream.expect('lbrace')
547 while self.stream.current.type is not 'rbrace':
549 self.stream.expect('comma')
550 if self.stream.current.type == 'rbrace':
552 key = self.parse_expression()
553 self.stream.expect('colon')
554 value = self.parse_expression()
555 items.append(nodes.Pair(key, value, lineno=key.lineno))
556 self.stream.expect('rbrace')
557 return nodes.Dict(items, lineno=token.lineno)
559 def parse_postfix(self, node):
561 token_type = self.stream.current.type
562 if token_type is 'dot' or token_type is 'lbracket':
563 node = self.parse_subscript(node)
564 elif token_type is 'lparen':
565 node = self.parse_call(node)
566 elif token_type is 'pipe':
567 node = self.parse_filter(node)
568 elif token_type is 'name' and self.stream.current.value == 'is':
569 node = self.parse_test(node)
574 def parse_subscript(self, node):
575 token = self.stream.next()
576 if token.type is 'dot':
577 attr_token = self.stream.current
579 if attr_token.type is 'name':
580 return nodes.Getattr(node, attr_token.value, 'load',
582 elif attr_token.type is not 'integer':
583 self.fail('expected name or number', attr_token.lineno)
584 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
585 return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
586 if token.type is 'lbracket':
587 priority_on_attribute = False
589 while self.stream.current.type is not 'rbracket':
591 self.stream.expect('comma')
592 args.append(self.parse_subscribed())
593 self.stream.expect('rbracket')
597 arg = nodes.Tuple(args, self.lineno, self.filename)
598 return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
599 self.fail('expected subscript expression', self.lineno)
601 def parse_subscribed(self):
602 lineno = self.stream.current.lineno
604 if self.stream.current.type is 'colon':
608 node = self.parse_expression()
609 if self.stream.current.type is not 'colon':
614 if self.stream.current.type is 'colon':
616 elif self.stream.current.type not in ('rbracket', 'comma'):
617 args.append(self.parse_expression())
621 if self.stream.current.type is 'colon':
623 if self.stream.current.type not in ('rbracket', 'comma'):
624 args.append(self.parse_expression())
630 return nodes.Slice(lineno=lineno, *args)
632 def parse_call(self, node):
633 token = self.stream.expect('lparen')
636 dyn_args = dyn_kwargs = None
637 require_comma = False
641 self.fail('invalid syntax for function call expression',
644 while self.stream.current.type is not 'rparen':
646 self.stream.expect('comma')
647 # support for trailing comma
648 if self.stream.current.type is 'rparen':
650 if self.stream.current.type is 'mul':
651 ensure(dyn_args is None and dyn_kwargs is None)
653 dyn_args = self.parse_expression()
654 elif self.stream.current.type is 'pow':
655 ensure(dyn_kwargs is None)
657 dyn_kwargs = self.parse_expression()
659 ensure(dyn_args is None and dyn_kwargs is None)
660 if self.stream.current.type is 'name' and \
661 self.stream.look().type is 'assign':
662 key = self.stream.current.value
664 value = self.parse_expression()
665 kwargs.append(nodes.Keyword(key, value,
666 lineno=value.lineno))
669 args.append(self.parse_expression())
672 self.stream.expect('rparen')
675 return args, kwargs, dyn_args, dyn_kwargs
676 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
679 def parse_filter(self, node, start_inline=False):
680 while self.stream.current.type == 'pipe' or start_inline:
683 token = self.stream.expect('name')
685 while self.stream.current.type is 'dot':
687 name += '.' + self.stream.expect('name').value
688 if self.stream.current.type is 'lparen':
689 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
693 dyn_args = dyn_kwargs = None
694 node = nodes.Filter(node, name, args, kwargs, dyn_args,
695 dyn_kwargs, lineno=token.lineno)
699 def parse_test(self, node):
700 token = self.stream.next()
701 if self.stream.current.test('name:not'):
706 name = self.stream.expect('name').value
707 while self.stream.current.type is 'dot':
709 name += '.' + self.stream.expect('name').value
710 dyn_args = dyn_kwargs = None
712 if self.stream.current.type is 'lparen':
713 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
714 elif self.stream.current.type in ('name', 'string', 'integer',
715 'float', 'lparen', 'lbracket',
717 self.stream.current.test_any('name:else', 'name:or',
719 if self.stream.current.test('name:is'):
720 self.fail('You cannot chain multiple tests with is')
721 args = [self.parse_expression()]
724 node = nodes.Test(node, name, args, kwargs, dyn_args,
725 dyn_kwargs, lineno=token.lineno)
727 node = nodes.Not(node, lineno=token.lineno)
730 def subparse(self, end_tokens=None):
733 add_data = data_buffer.append
737 lineno = data_buffer[0].lineno
738 body.append(nodes.Output(data_buffer[:], lineno=lineno))
742 token = self.stream.current
743 if token.type is 'data':
745 add_data(nodes.TemplateData(token.value,
746 lineno=token.lineno))
748 elif token.type is 'variable_begin':
750 add_data(self.parse_tuple(with_condexpr=True))
751 self.stream.expect('variable_end')
752 elif token.type is 'block_begin':
755 if end_tokens is not None and \
756 self.stream.current.test_any(*end_tokens):
758 rv = self.parse_statement()
759 if isinstance(rv, list):
763 self.stream.expect('block_end')
765 raise AssertionError('internal parsing error')
771 """Parse the whole template into a `Template` node."""
772 result = nodes.Template(self.subparse(), lineno=1)
773 result.set_environment(self.environment)