diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index acaaffb59..9d40c904d 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -252,6 +252,10 @@ class Environment: will reload the template. For higher performance it's possible to disable that. + `parser_tolerate_faults` + Instruct the parser to tolerate some invalid constructs that don't cause much semantic uncertainty, useful for linters and LSP to provide output on incomplete templates. + Defaults to False. + `bytecode_cache` If set to a bytecode cache object, this object will provide a cache for the internal Jinja bytecode so that templates don't @@ -316,6 +320,7 @@ def __init__( auto_reload: bool = True, bytecode_cache: t.Optional["BytecodeCache"] = None, enable_async: bool = False, + parser_tolerate_faults: bool = False, ): # !!Important notice!! # The constructor accepts quite a few arguments that should be @@ -360,6 +365,7 @@ def __init__( self.auto_reload = auto_reload # configurable policies + self.parser_tolerate_faults = parser_tolerate_faults self.policies = DEFAULT_POLICIES.copy() # load extensions diff --git a/src/jinja2/lexer.py b/src/jinja2/lexer.py index e35cd471e..7cd5c9cf0 100644 --- a/src/jinja2/lexer.py +++ b/src/jinja2/lexer.py @@ -8,6 +8,7 @@ import typing as t from ast import literal_eval from collections import deque +from dataclasses import dataclass from sys import intern from ._identifier import pattern as name_re @@ -266,10 +267,12 @@ def __call__(self, lineno: int, filename: str | None) -> "te.NoReturn": raise self.error_class(self.message, lineno, filename) -class Token(t.NamedTuple): +@dataclass +class Token: lineno: int type: str value: str + linepos: int def __str__(self) -> str: return describe_token(self) @@ -333,7 +336,7 @@ def __init__( self.name = name self.filename = filename self.closed = False - self.current = Token(1, TOKEN_INITIAL, "") + self.current = Token(1, TOKEN_INITIAL, "", 0) next(self) def __iter__(self) -> TokenStreamIterator: @@ -396,7 +399,11 @@ def __next__(self) -> Token: def close(self) -> None: """Close the stream.""" - self.current = Token(self.current.lineno, TOKEN_EOF, "") + lineno, linepos = self.current.lineno, self.current.linepos + value = self.current.value + lineno += value.count("\n") + linepos += len(value.rsplit("\n", 1)[-1]) + self.current = Token(lineno, TOKEN_EOF, "", linepos=linepos) self._iter = iter(()) self.closed = True @@ -609,19 +616,23 @@ def tokenize( state: str | None = None, ) -> TokenStream: """Calls tokeniter + tokenize and wraps it in a token stream.""" - stream = self.tokeniter(source, name, filename, state) + stream = self.tokeniter_linepos(source, name, filename, state) return TokenStream(self.wrap(stream, name, filename), name, filename) def wrap( self, - stream: t.Iterable[tuple[int, str, str]], + stream: t.Iterable[tuple[int, str, str] | tuple[int, str, str, int]], name: str | None = None, filename: str | None = None, ) -> t.Iterator[Token]: """This is called with the stream as returned by `tokenize` and wraps every token in a :class:`Token` and converts the value. """ - for lineno, token, value_str in stream: + for tup in stream: + if len(tup) == 3: + tup = (*tup, -1) + assert len(tup) == 4 + lineno, token, value_str, linepos = tup if token in ignored_tokens: continue @@ -664,15 +675,9 @@ def wrap( elif token == TOKEN_OPERATOR: token = operators[value_str] - yield Token(lineno, token, value) + yield Token(lineno, token, value, linepos) - def tokeniter( - self, - source: str, - name: str | None, - filename: str | None = None, - state: str | None = None, - ) -> t.Iterator[tuple[int, str, str]]: + def tokeniter(self, *kargs, **kwargs) -> t.Iterator[tuple[int, str, str]]: """This method tokenizes the text and returns the tokens in a generator. Use this method if you just want to tokenize a template. @@ -680,6 +685,15 @@ def tokeniter( Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line breaks. """ + yield from (tup[0:3] for tup in self.tokeniter_linepos(*kargs, **kwargs)) + + def tokeniter_linepos( + self, + source: str, + name: str | None, + filename: str | None = None, + state: str | None = None, + ) -> t.Iterator[tuple[int, str, str, int]]: lines = newline_re.split(source)[::2] if not self.keep_trailing_newline and lines[-1] == "": @@ -688,6 +702,7 @@ def tokeniter( source = "\n".join(lines) pos = 0 lineno = 1 + linepos = 0 stack = ["root"] if state is not None and state != "root": @@ -700,11 +715,39 @@ def tokeniter( newlines_stripped = 0 line_starting = True + def linepos_from_str(line_or_more: str) -> int: + line = line_or_more.rsplit("\n", 1)[-1] + return len(line) + + old_pos = pos + while True: # tokenizer loop for regex, tokens, new_state in statetokens: - m = regex.match(source, pos) + if old_pos != pos: + lineno = source[:pos].count("\n") + 1 + inbetween = source[old_pos:pos] + if "\n" in inbetween: + linepos = len(inbetween.rsplit("\n", 1)[-1]) + else: + for backwards_buffer_expo in range(5): + backwards_offset = 10**backwards_buffer_expo + backwards_location = max(0, pos - backwards_offset) + lookbehind = source[backwards_location:pos] + last_line = lookbehind.rsplit("\n", 1)[-1] + if ( + len(last_line) != len(lookbehind) + or len(lookbehind) >= pos + ): + # we found a line break + linepos = len(last_line) + break + if backwards_location <= 0: + break + + old_pos = pos + m = regex.match(source, pos) # if no match we try again with the next rule if m is None: continue @@ -737,6 +780,7 @@ def tokeniter( # Strip all whitespace between the text and the tag. stripped = text.rstrip() newlines_stripped = text[len(stripped) :].count("\n") + linepos = len(text.rsplit("\n", 1)[-1]) groups = [stripped, *groups[1:]] elif ( # Not marked for preserving whitespace. @@ -765,7 +809,10 @@ def tokeniter( elif token == "#bygroup": for key, value in m.groupdict().items(): if value is not None: - yield lineno, key, value + yield lineno, key, value, linepos + linepos = len(value.splitlines(keepends=False)[-1]) + # linepos = pos + # pos = linepos lineno += value.count("\n") break else: @@ -778,14 +825,17 @@ def tokeniter( data = groups[idx] if data or token not in ignore_if_empty: - yield lineno, token, data # type: ignore[misc] + yield lineno, token, data, linepos # type: ignore[misc] lineno += data.count("\n") + newlines_stripped + if "\n" in data: + linepos = 0 + linepos += len(data.rsplit("\n")[-1]) newlines_stripped = 0 # strings as token just are yielded as it. else: - data = m.group() + data: str = m.group() # update brace/parentheses balance if tokens == TOKEN_OPERATOR: @@ -813,11 +863,14 @@ def tokeniter( # yield items if data or tokens not in ignore_if_empty: - yield lineno, tokens, data + yield lineno, tokens, data, linepos lineno += data.count("\n") + if "\n" in data: + linepos = len(data.rsplit("\n", 1)[-1]) line_starting = m.group()[-1:] == "\n" + # fetch new position into new variable so that we can check # if there is a internal parsing error which would result # in an infinite loop diff --git a/src/jinja2/nodes.py b/src/jinja2/nodes.py index 1e08d3c59..e7631c7e3 100644 --- a/src/jinja2/nodes.py +++ b/src/jinja2/nodes.py @@ -120,12 +120,25 @@ class Node(metaclass=NodeType): """ fields: tuple[str, ...] = () - attributes: tuple[str, ...] = ("lineno", "environment") + attributes: tuple[str, ...] = ( + "lineno", + "linepos", + "environment", + "issues", + "lineno_end", + "linepos_end", + ) abstract = True lineno: int + linepos: int environment: t.Optional["Environment"] + # only filled in diagnostic mode + issues: list["Node"] + lineno_end: int | None + linepos_end: int | None + def __init__(self, *fields: t.Any, **attributes: t.Any) -> None: if self.abstract: raise TypeError("abstract nodes are not instantiable") @@ -170,35 +183,47 @@ def iter_child_nodes( self, exclude: t.Container[str] | None = None, only: t.Container[str] | None = None, + reverse: bool = False, ) -> t.Iterator["Node"]: """Iterates over all direct child nodes of the node. This iterates over all fields and yields the values of they are nodes. If the value of a field is a list all the nodes in that list are returned. """ - for _, item in self.iter_fields(exclude, only): + items: t.Iterable[tuple[str, t.Any]] = self.iter_fields(exclude, only) + if reverse: + items = reversed(list(items)) + for _, item in items: if isinstance(item, list): + if reverse: + item = reversed(item) for n in item: if isinstance(n, Node): yield n elif isinstance(item, Node): yield item - def find(self, node_type: type[_NodeBound]) -> _NodeBound | None: + def find( + self, node_type: type[_NodeBound], *, reverse: bool = False + ) -> _NodeBound | None: """Find the first node of a given type. If no such node exists the return value is `None`. + With reverse=True, the last node is returned instead """ - for result in self.find_all(node_type): + for result in self.find_all(node_type, reverse=reverse): return result return None def find_all( - self, node_type: type[_NodeBound] | tuple[type[_NodeBound], ...] + self, + node_type: type[_NodeBound] | tuple[type[_NodeBound], ...], + *, + reverse: bool = False, ) -> t.Iterator[_NodeBound]: """Find all the nodes of a given type. If the type is a tuple, the check is performed for any of the tuple items. """ - for child in self.iter_child_nodes(): + for child in self.iter_child_nodes(reverse=reverse): if isinstance(child, node_type): yield child # type: ignore yield from child.find_all(node_type) @@ -279,12 +304,28 @@ def _dump(node: Node | t.Any) -> None: return "".join(buf) +class ParserIssue(Node): + attributes: tuple[str, ...] = ("message", "issue_context") + message: str + issue_context: str | None + + class Stmt(Node): """Base node for all statements.""" abstract = True +class EmptyStatement(Stmt): + """Node where a statement should be but an empty statement was given. + Returned in Fault-tolerant Mode only + """ + + attributes: tuple[str, ...] = ("message", "issue_context") + message: str | None + issue_context: str | None + + class Helper(Node): """Nodes that exist in a specific context only.""" @@ -398,10 +439,12 @@ class Block(Stmt): """ fields = ("name", "body", "scoped", "required") + attributes = ("endblock_with_name",) name: str body: list[Node] scoped: bool required: bool + endblock_with_name: bool | None class Include(Stmt): @@ -487,6 +530,28 @@ def can_assign(self) -> bool: return False +class ExprIssue(Expr): + attributes: tuple[str, ...] = ("message", "issue_context") + message: str + issue_context: str | None + + +class EmptyExpression(ExprIssue): + """Node where an expression should be but an empty expression was given. + Returned in Fault-tolerant Mode only + """ + + +class InvalidExpression(ExprIssue): + """Node where an expression should be but an unparsable expression was given. + Returned in Fault-tolerant Mode only + """ + + attributes: tuple[str, ...] = ("original_str",) + + original_str: str + + class BinExpr(Expr): """Baseclass for all binary expressions.""" diff --git a/src/jinja2/parser.py b/src/jinja2/parser.py index 3ae857ebe..018aac08d 100644 --- a/src/jinja2/parser.py +++ b/src/jinja2/parser.py @@ -153,18 +153,37 @@ def is_tuple_end(self, extra_end_rules: tuple[str, ...] | None = None) -> bool: return self.stream.current.test_any(extra_end_rules) # type: ignore return False - def free_identifier(self, lineno: int | None = None) -> nodes.InternalName: + def free_identifier( + self, lineno: int | None = None, linepos: int | None = None + ) -> nodes.InternalName: """Return a new free identifier as :class:`~jinja2.nodes.InternalName`.""" self._last_identifier += 1 rv = object.__new__(nodes.InternalName) - nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno) + nodes.Node.__init__( + rv, + f"fi{self._last_identifier}", + lineno=lineno, + linepos=linepos, + lineno_end=lineno, + linepos_end=linepos, + ) return rv def parse_statement(self) -> nodes.Node | list[nodes.Node]: """Parse a single statement.""" token = self.stream.current if token.type != "name": - self.fail("tag name expected", token.lineno) + if not self.environment.parser_tolerate_faults: + self.fail("tag name expected", token.lineno) + nxt = self.stream.look() if not self.stream.closed else self.stream.current + return nodes.EmptyStatement( + message="Tag name expected", + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + issue_context="tag", + ) self._tag_stack.append(token.value) pop_tag = True try: @@ -177,7 +196,10 @@ def parse_statement(self) -> nodes.Node | list[nodes.Node]: return self.parse_filter_block() ext = self.extensions.get(token.value) if ext is not None: - return ext(self) + res = ext(self) + if hasattr(res, "linepos") and res.linepos is None: + res.linepos = token.linepos + return res # did not work out, remove the token we pushed by accident # from the stack so that the unknown tag fail function can @@ -220,23 +242,62 @@ def parse_statements( def parse_set(self) -> nodes.Assign | nodes.AssignBlock: """Parse an assign statement.""" - lineno = next(self.stream).lineno + _next = next(self.stream) + lineno = _next.lineno + linepos = _next.linepos target = self.parse_assign_target(with_namespace=True) - if self.stream.skip_if("assign"): - expr = self.parse_tuple() - return nodes.Assign(target, expr, lineno=lineno) + expr_start = self.stream.next_if("assign") + if expr_start: + expr = self.parse_tuple(allow_empty=self.environment.parser_tolerate_faults) + end_token = self.stream.current + result = nodes.Assign( + target, + expr, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) + if isinstance(expr, nodes.EmptyExpression): + expr.message = "Assignment to empty expression" + expr.issue_context = "assignment" + expr.lineno, expr.linepos = expr_start.lineno, expr_start.linepos + expr.linepos_end += 1 + return result filter_node = self.parse_filter(None) body = self.parse_statements(("name:endset",), drop_needle=True) - return nodes.AssignBlock(target, filter_node, body, lineno=lineno) + end_token = self.stream.current + return nodes.AssignBlock( + target, + filter_node, + body, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) def parse_for(self) -> nodes.For: """Parse a for loop.""" - lineno = self.stream.expect("name:for").lineno + _next = self.stream.expect("name:for") + lineno = _next.lineno + linepos = _next.linepos target = self.parse_assign_target(extra_end_rules=("name:in",)) - self.stream.expect("name:in") + iter_start = self.stream.expect("name:in") iter = self.parse_tuple( - with_condexpr=False, extra_end_rules=("name:recursive",) + with_condexpr=False, + extra_end_rules=("name:recursive",), + allow_empty=self.environment.parser_tolerate_faults, ) + if self.environment.parser_tolerate_faults and isinstance( + iter, nodes.EmptyExpression + ): + iter.message = "Empty For-loop iterator" + iter.lineno, iter.linepos = iter_start.lineno, iter_start.linepos + assert iter.linepos_end is not None + iter.linepos_end += 1 + iter.issue_context = "for_iterator" + test = None if self.stream.skip_if("name:if"): test = self.parse_expression() @@ -246,28 +307,60 @@ def parse_for(self) -> nodes.For: else_ = [] else: else_ = self.parse_statements(("name:endfor",), drop_needle=True) - return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno) + end_token = self.stream.current + return nodes.For( + target, + iter, + body, + else_, + test, + recursive, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) def parse_if(self) -> nodes.If: """Parse an if construct.""" - node = result = nodes.If(lineno=self.stream.expect("name:if").lineno) + current = self.stream.current + _next = self.stream.expect("name:if") + node = result = nodes.If( + lineno=current.lineno, + linepos=current.linepos, + lineno_end=_next.lineno, + linepos_end=_next.linepos, + ) while True: - node.test = self.parse_tuple(with_condexpr=False) + node.test = self.parse_tuple( + with_condexpr=False, + allow_empty=self.environment.parser_tolerate_faults, + ) node.body = self.parse_statements(("name:elif", "name:else", "name:endif")) node.elif_ = [] node.else_ = [] token = next(self.stream) + nxt = self.stream.look() if not self.stream.closed else self.stream.current if token.test("name:elif"): - node = nodes.If(lineno=self.stream.current.lineno) + node = nodes.If( + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) result.elif_.append(node) continue elif token.test("name:else"): result.else_ = self.parse_statements(("name:endif",), drop_needle=True) break + end_token = self.stream.current + result.lineno_end = end_token.lineno + result.linepos_end = end_token.linepos return result def parse_with(self) -> nodes.With: - node = nodes.With(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.With(lineno=_next.lineno, linepos=_next.linepos) targets: list[nodes.Expr] = [] values: list[nodes.Expr] = [] while self.stream.current.type != "block_end": @@ -281,16 +374,32 @@ def parse_with(self) -> nodes.With: node.targets = targets node.values = values node.body = self.parse_statements(("name:endwith",), drop_needle=True) + end_token = self.stream.current + node.lineno_end = end_token.lineno + node.linepos_end = end_token.linepos return node def parse_autoescape(self) -> nodes.Scope: - node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.ScopedEvalContextModifier( + lineno=_next.lineno, linepos=_next.linepos + ) node.options = [nodes.Keyword("autoescape", self.parse_expression())] node.body = self.parse_statements(("name:endautoescape",), drop_needle=True) - return nodes.Scope([node]) + end_token = self.stream.current + node.lineno_end = end_token.lineno + node.linepos_end = end_token.linepos + return nodes.Scope( + [node], + lineno=node.lineno, + linepos=node.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) def parse_block(self) -> nodes.Block: - node = nodes.Block(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.Block(lineno=_next.lineno, linepos=_next.linepos) node.name = self.stream.expect("name").value node.scoped = self.stream.skip_if("name:scoped") node.required = self.stream.skip_if("name:required") @@ -318,12 +427,35 @@ def parse_block(self) -> nodes.Block: ): self.fail("Required blocks can only contain comments or whitespace") - self.stream.skip_if("name:" + node.name) + if not self.environment.parser_tolerate_faults: + self.stream.skip_if("name:" + node.name) + elif self.stream.current.test("name"): + node.endblock_with_name = True + wrong = self.stream.expect("name") + if wrong.value != node.name: + node.issues = node.issues or [] + node.issues.append( + nodes.ParserIssue( + message=f"endblock used with incorrect name {wrong.value!r} for block {node.name!r}", + lineno=wrong.lineno, + linepos=wrong.linepos, + lineno_end=wrong.lineno, + linepos_end=wrong.linepos + len(wrong.value), + issue_context="endblock", + ) + ) + end_token = self.stream.current + node.lineno_end = end_token.lineno + node.linepos_end = end_token.linepos return node def parse_extends(self) -> nodes.Extends: - node = nodes.Extends(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.Extends(lineno=_next.lineno, linepos=_next.linepos) node.template = self.parse_expression() + end_token = self.stream.current + node.lineno_end = end_token.lineno + node.linepos_end = end_token.linepos return node def parse_import_context( @@ -339,7 +471,8 @@ def parse_import_context( return node def parse_include(self) -> nodes.Include: - node = nodes.Include(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.Include(lineno=_next.lineno, linepos=_next.linepos) node.template = self.parse_expression() if self.stream.current.test("name:ignore") and self.stream.look().test( "name:missing" @@ -348,17 +481,27 @@ def parse_include(self) -> nodes.Include: self.stream.skip(2) else: node.ignore_missing = False - return self.parse_import_context(node, True) + result = self.parse_import_context(node, True) + end_token = self.stream.current + result.lineno_end = end_token.lineno + result.linepos_end = end_token.linepos + return result def parse_import(self) -> nodes.Import: - node = nodes.Import(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.Import(lineno=_next.lineno, linepos=_next.linepos) node.template = self.parse_expression() self.stream.expect("name:as") node.target = self.parse_assign_target(name_only=True).name - return self.parse_import_context(node, False) + result = self.parse_import_context(node, False) + end_token = self.stream.current + result.lineno_end = end_token.lineno + result.linepos_end = end_token.linepos + return result def parse_from(self) -> nodes.FromImport: - node = nodes.FromImport(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.FromImport(lineno=_next.lineno, linepos=_next.linepos) node.template = self.parse_expression() self.stream.expect("name:import") node.names = [] @@ -397,11 +540,33 @@ def parse_context() -> bool: self.stream.expect("name") if not hasattr(node, "with_context"): node.with_context = False + end_token = self.stream.current + node.lineno_end = end_token.lineno + node.linepos_end = end_token.linepos return node - def parse_signature(self, node: _MacroCall) -> None: + def parse_signature( + self, node: _MacroCall + ) -> None | nodes.EmptyExpression | nodes.InvalidExpression: args = node.args = [] defaults = node.defaults = [] + if ( + self.environment.parser_tolerate_faults + and self.stream.current.type != "lparen" + ): + if self.stream.current.type == "block_end": + node.issues = node.issues or [] + issue = nodes.EmptyExpression( # type: ignore[assignment] + lineno=node.lineno, + linepos=node.linepos, + lineno_end=self.stream.current.lineno, + linepos_end=self.stream.current.linepos, + message=f"Missing {type(node).__name__} signature", + issue_context="signature", + ) + node.issues.append(issue) + return issue + self.stream.expect("lparen") while self.stream.current.type != "rparen": if args: @@ -411,45 +576,90 @@ def parse_signature(self, node: _MacroCall) -> None: if self.stream.skip_if("assign"): defaults.append(self.parse_expression()) elif defaults: - self.fail("non-default argument follows default argument") + msg = "non-default argument follows default argument" + if not self.environment.parser_tolerate_faults: + self.fail(msg) + err = nodes.InvalidExpression( + lineno=arg.lineno, + linepos=arg.linepos, + lineno_end=self.stream.current.lineno, + linepos_end=self.stream.current.linepos, + message=msg, + original_str=arg.name, + ) + arg.issues = arg.issues or [] + arg.issues.append(err) args.append(arg) self.stream.expect("rparen") + return None def parse_call_block(self) -> nodes.CallBlock: - node = nodes.CallBlock(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.CallBlock(lineno=_next.lineno, linepos=_next.linepos) if self.stream.current.type == "lparen": - self.parse_signature(node) + signature_issue = self.parse_signature(node) + if signature_issue: + assert self.environment.parser_tolerate_faults + node.args = signature_issue # type: ignore[assignment] else: node.args = [] node.defaults = [] call_node = self.parse_expression() if not isinstance(call_node, nodes.Call): - self.fail("expected call", node.lineno) + if not ( + self.environment.parser_tolerate_faults or isinstance(call_node, Name) + ): + self.fail("expected call", node.lineno) + call_node.issues = call_node.issues or [] + issue = nodes.EmptyExpression( + message="Expected function call; missing parentheses", + lineno=call_node.lineno, + linepos=call_node.linepos, + lineno_end=call_node.lineno_end, + linepos_end=call_node.linepos_end, + issue_context="function_call", + ) + call_node.issues.append(issue) node.call = call_node node.body = self.parse_statements(("name:endcall",), drop_needle=True) + end_token = self.stream.current + node.lineno_end = end_token.lineno + node.linepos_end = end_token.linepos return node def parse_filter_block(self) -> nodes.FilterBlock: - node = nodes.FilterBlock(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.FilterBlock(lineno=_next.lineno, linepos=_next.linepos) node.filter = self.parse_filter(None, start_inline=True) # type: ignore node.body = self.parse_statements(("name:endfilter",), drop_needle=True) + end_token = self.stream.current + node.lineno_end = end_token.lineno + node.linepos_end = end_token.linepos return node def parse_macro(self) -> nodes.Macro: - node = nodes.Macro(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.Macro(lineno=_next.lineno, linepos=_next.linepos, issues=None) node.name = self.parse_assign_target(name_only=True).name - self.parse_signature(node) + signature_issue = self.parse_signature(node) node.body = self.parse_statements(("name:endmacro",), drop_needle=True) + end_token = self.stream.current + node.lineno_end = end_token.lineno + node.linepos_end = end_token.linepos return node def parse_print(self) -> nodes.Output: - node = nodes.Output(lineno=next(self.stream).lineno) + _next = next(self.stream) + node = nodes.Output(lineno=_next.lineno, linepos=_next.linepos) node.nodes = [] while self.stream.current.type != "block_end": if node.nodes: self.stream.expect("comma") node.nodes.append(self.parse_expression()) + end_token = self.stream.current + node.lineno_end = end_token.lineno + node.linepos_end = end_token.linepos return node @typing.overload @@ -485,7 +695,15 @@ def parse_assign_target( if name_only: token = self.stream.expect("name") - target = nodes.Name(token.value, "store", lineno=token.lineno) + nxt = self.stream.look() if not self.stream.closed else self.stream.current + target = nodes.Name( + token.value, + "store", + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) else: if with_tuple: target = self.parse_tuple( @@ -516,6 +734,7 @@ def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr: def parse_condexpr(self) -> nodes.Expr: lineno = self.stream.current.lineno + linepos = self.stream.current.linepos expr1 = self.parse_or() expr3: nodes.Expr | None @@ -525,112 +744,260 @@ def parse_condexpr(self) -> nodes.Expr: expr3 = self.parse_condexpr() else: expr3 = None - expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno) + end_token = self.stream.current + expr1 = nodes.CondExpr( + expr2, + expr1, + expr3, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) lineno = self.stream.current.lineno + linepos = self.stream.current.linepos return expr1 def parse_or(self) -> nodes.Expr: lineno = self.stream.current.lineno + linepos = self.stream.current.linepos left = self.parse_and() - while self.stream.skip_if("name:or"): + while self.stream.current.test("name:or"): + token = next(self.stream) right = self.parse_and() - left = nodes.Or(left, right, lineno=lineno) + end_token = self.stream.current + if isinstance(right, nodes.EmptyExpression): + right.lineno, right.linepos = token.lineno, token.linepos + left = nodes.Or( + left, + right, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) lineno = self.stream.current.lineno + linepos = self.stream.current.linepos return left def parse_and(self) -> nodes.Expr: lineno = self.stream.current.lineno + linepos = self.stream.current.linepos left = self.parse_not() - while self.stream.skip_if("name:and"): + while self.stream.current.test("name:and"): + token = next(self.stream) right = self.parse_not() - left = nodes.And(left, right, lineno=lineno) + if isinstance(right, nodes.EmptyExpression): + right.lineno, right.linepos = token.lineno, token.linepos + end_token = self.stream.current + left = nodes.And( + left, + right, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) lineno = self.stream.current.lineno + linepos = self.stream.current.linepos return left def parse_not(self) -> nodes.Expr: if self.stream.current.test("name:not"): - lineno = next(self.stream).lineno - return nodes.Not(self.parse_not(), lineno=lineno) + _next = next(self.stream) + lineno = _next.lineno + linepos = _next.linepos + result = self.parse_not() + end_token = self.stream.current + return nodes.Not( + result, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) return self.parse_compare() def parse_compare(self) -> nodes.Expr: lineno = self.stream.current.lineno + linepos = self.stream.current.linepos expr = self.parse_math1() ops = [] while True: - token_type = self.stream.current.type + token = self.stream.current + token_type = token.type if token_type in _compare_operators: next(self.stream) - ops.append(nodes.Operand(token_type, self.parse_math1())) + nxt = self.stream.current + ops.append( + nodes.Operand( + token_type, + self.parse_math1(), + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) + ) elif self.stream.skip_if("name:in"): - ops.append(nodes.Operand("in", self.parse_math1())) + nxt = self.stream.look() if not self.stream.closed else token + ops.append( + nodes.Operand( + "in", + self.parse_math1(), + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) + ) elif self.stream.current.test("name:not") and self.stream.look().test( "name:in" ): + token = self.stream.current self.stream.skip(2) - ops.append(nodes.Operand("notin", self.parse_math1())) + nxt = ( + self.stream.look() + if not self.stream.closed + else self.stream.current + ) + ops.append( + nodes.Operand( + "notin", + self.parse_math1(), + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) + ) else: break - lineno = self.stream.current.lineno if not ops: + if isinstance(expr, nodes.EmptyExpression): + expr.lineno, expr.linepos = lineno, linepos return expr - return nodes.Compare(expr, ops, lineno=lineno) + end_token = self.stream.current + return nodes.Compare( + expr, + ops, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) def parse_math1(self) -> nodes.Expr: lineno = self.stream.current.lineno + linepos = self.stream.current.linepos left = self.parse_concat() while self.stream.current.type in ("add", "sub"): cls = _math_nodes[self.stream.current.type] next(self.stream) right = self.parse_concat() - left = cls(left, right, lineno=lineno) + end_token = self.stream.current + left = cls( + left, + right, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) lineno = self.stream.current.lineno + linepos = self.stream.current.linepos return left def parse_concat(self) -> nodes.Expr: lineno = self.stream.current.lineno + linepos = self.stream.current.linepos args = [self.parse_math2()] while self.stream.current.type == "tilde": next(self.stream) args.append(self.parse_math2()) if len(args) == 1: return args[0] - return nodes.Concat(args, lineno=lineno) + end_token = self.stream.current + return nodes.Concat( + args, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) def parse_math2(self) -> nodes.Expr: lineno = self.stream.current.lineno + linepos = self.stream.current.linepos left = self.parse_pow() while self.stream.current.type in ("mul", "div", "floordiv", "mod"): cls = _math_nodes[self.stream.current.type] next(self.stream) right = self.parse_pow() - left = cls(left, right, lineno=lineno) + end_token = self.stream.current + left = cls( + left, + right, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) lineno = self.stream.current.lineno + linepos = self.stream.current.linepos return left def parse_pow(self) -> nodes.Expr: lineno = self.stream.current.lineno + linepos = self.stream.current.linepos left = self.parse_unary() while self.stream.current.type == "pow": next(self.stream) right = self.parse_unary() - left = nodes.Pow(left, right, lineno=lineno) + end_token = self.stream.current + left = nodes.Pow( + left, + right, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) lineno = self.stream.current.lineno + linepos = self.stream.current.linepos return left def parse_unary(self, with_filter: bool = True) -> nodes.Expr: token_type = self.stream.current.type lineno = self.stream.current.lineno + linepos = self.stream.current.linepos node: nodes.Expr if token_type == "sub": next(self.stream) - node = nodes.Neg(self.parse_unary(False), lineno=lineno) + inner = self.parse_unary(False) + end_token = self.stream.current + node = nodes.Neg( + inner, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) elif token_type == "add": next(self.stream) - node = nodes.Pos(self.parse_unary(False), lineno=lineno) + inner = self.parse_unary(False) + end_token = self.stream.current + node = nodes.Pos( + inner, + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) else: node = self.parse_primary() + node.lineno, node.linepos = lineno, linepos node = self.parse_postfix(node) if with_filter: node = self.parse_filter_expr(node) @@ -644,28 +1011,75 @@ def parse_primary(self, with_namespace: bool = False) -> nodes.Expr: if token.type == "name": next(self.stream) if token.value in ("true", "false", "True", "False"): - node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) + node = nodes.Const( + token.value in ("true", "True"), + lineno=token.lineno, + linepos=token.linepos, + lineno_end=token.lineno, + linepos_end=token.linepos + len(token.value), + ) elif token.value in ("none", "None"): - node = nodes.Const(None, lineno=token.lineno) + node = nodes.Const( + None, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=token.lineno, + linepos_end=token.linepos + len(token.value), + ) elif with_namespace and self.stream.current.type == "dot": # If namespace attributes are allowed at this point, and the next # token is a dot, produce a namespace reference. next(self.stream) attr = self.stream.expect("name") - node = nodes.NSRef(token.value, attr.value, lineno=token.lineno) + nxt = self.stream.current + node = nodes.NSRef( + token.value, + attr.value, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) else: - node = nodes.Name(token.value, "load", lineno=token.lineno) + nxt = ( + self.stream.look() + if not self.stream.closed + else self.stream.current + ) + node = nodes.Name( + token.value, + "load", + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) elif token.type == "string": next(self.stream) buf = [token.value] lineno = token.lineno + linepos_end = token.linepos while self.stream.current.type == "string": buf.append(self.stream.current.value) + linepos_end = self.stream.current.linepos next(self.stream) - node = nodes.Const("".join(buf), lineno=lineno) + node = nodes.Const( + "".join(buf), + lineno=lineno, + linepos=token.linepos, + lineno_end=self.stream.current.lineno, + linepos_end=linepos_end, + ) elif token.type in ("integer", "float"): next(self.stream) - node = nodes.Const(token.value, lineno=token.lineno) + nxt = self.stream.look() if not self.stream.closed else self.stream.current + node = nodes.Const( + token.value, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) elif token.type == "lparen": next(self.stream) node = self.parse_tuple(explicit_parentheses=True) @@ -675,7 +1089,25 @@ def parse_primary(self, with_namespace: bool = False) -> nodes.Expr: elif token.type == "lbrace": node = self.parse_dict() else: - self.fail(f"unexpected {describe_token(token)!r}", token.lineno) + msg = f"unexpected {describe_token(token)!r}" + if not self.environment.parser_tolerate_faults: + self.fail(msg, token.lineno) + if token.type == "variable_end": + nxt = ( + self.stream.look() + if not self.stream.closed + else self.stream.current + ) + node = nodes.EmptyExpression( + message="Unexpected end of primary statement", + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + issue_context="primary", + ) + else: + self.fail(msg, token.lineno) return node def parse_tuple( @@ -685,6 +1117,7 @@ def parse_tuple( extra_end_rules: tuple[str, ...] | None = None, explicit_parentheses: bool = False, with_namespace: bool = False, + allow_empty: bool = False, ) -> nodes.Tuple | nodes.Expr: """Works like `parse_expression` but if multiple expressions are delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. @@ -706,6 +1139,7 @@ def parse_tuple( tuple is a valid expression or not. """ lineno = self.stream.current.lineno + lineno_start = lineno if simplified: def parse() -> nodes.Expr: @@ -718,6 +1152,7 @@ def parse() -> nodes.Expr: args: list[nodes.Expr] = [] is_tuple = False + linepos_start = self.stream.current.linepos while True: if args: @@ -740,12 +1175,28 @@ def parse() -> nodes.Expr: # nothing) in the spot of an expression would be an empty # tuple. if not explicit_parentheses: + if allow_empty: + empty = nodes.EmptyExpression( + lineno=lineno_start, + linepos=linepos_start, + lineno_end=self.stream.current.lineno, + linepos_end=self.stream.current.linepos, + message="Expected an expression", + ) + return empty self.fail( "Expected an expression," f" got {describe_token(self.stream.current)!r}" ) - - return nodes.Tuple(args, "load", lineno=lineno) + end_token = self.stream.current + return nodes.Tuple( + args, + "load", + lineno=lineno, + linepos=linepos_start, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) def parse_list(self) -> nodes.List: token = self.stream.expect("lbracket") @@ -756,8 +1207,14 @@ def parse_list(self) -> nodes.List: if self.stream.current.type == "rbracket": break items.append(self.parse_expression()) - self.stream.expect("rbracket") - return nodes.List(items, lineno=token.lineno) + end_token = self.stream.expect("rbracket") + return nodes.List( + items, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) def parse_dict(self) -> nodes.Dict: token = self.stream.expect("lbrace") @@ -770,9 +1227,15 @@ def parse_dict(self) -> nodes.Dict: key = self.parse_expression() self.stream.expect("colon") value = self.parse_expression() - items.append(nodes.Pair(key, value, lineno=key.lineno)) - self.stream.expect("rbrace") - return nodes.Dict(items, lineno=token.lineno) + items.append(nodes.Pair(key, value, lineno=key.lineno, linepos=key.linepos)) + end_token = self.stream.expect("rbrace") + return nodes.Dict( + items, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) def parse_postfix(self, node: nodes.Expr) -> nodes.Expr: while True: @@ -808,31 +1271,81 @@ def parse_subscript(self, node: nodes.Expr) -> nodes.Getattr | nodes.Getitem: if token.type == "dot": attr_token = self.stream.current - next(self.stream) if attr_token.type == "name": + next(self.stream) + nxt = self.stream.current return nodes.Getattr( - node, attr_token.value, "load", lineno=token.lineno + node, + attr_token.value, + "load", + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) + if attr_token.type != "integer": + if not self.environment.parser_tolerate_faults: + self.fail("expected name or number", attr_token.lineno) + arg = nodes.EmptyExpression( + message=f"Missing name for dot access! Got {attr_token.type}", + lineno=token.lineno, + linepos=token.linepos, + lineno_end=attr_token.lineno, + linepos_end=attr_token.linepos, + issue_context="attribute", ) - elif attr_token.type != "integer": - self.fail("expected name or number", attr_token.lineno) - arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) - return nodes.Getitem(node, arg, "load", lineno=token.lineno) + else: + next(self.stream) + nxt = self.stream.current + arg = nodes.Const( + attr_token.value, + lineno=attr_token.lineno, + linepos=attr_token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) + end_token = self.stream.current + return nodes.Getitem( + node, + arg, + "load", + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) if token.type == "lbracket": args: list[nodes.Expr] = [] while self.stream.current.type != "rbracket": if args: self.stream.expect("comma") args.append(self.parse_subscribed()) - self.stream.expect("rbracket") + end_token = self.stream.expect("rbracket") if len(args) == 1: arg = args[0] else: - arg = nodes.Tuple(args, "load", lineno=token.lineno) - return nodes.Getitem(node, arg, "load", lineno=token.lineno) + arg = nodes.Tuple( + args, + "load", + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) + return nodes.Getitem( + node, + arg, + "load", + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) self.fail("expected subscript expression", token.lineno) def parse_subscribed(self) -> nodes.Expr: lineno = self.stream.current.lineno + linepos = self.stream.current.linepos args: list[nodes.Expr | None] if self.stream.current.type == "colon": @@ -861,7 +1374,14 @@ def parse_subscribed(self) -> nodes.Expr: else: args.append(None) - return nodes.Slice(lineno=lineno, *args) # noqa: B026 + end_token = self.stream.current + return nodes.Slice( + lineno=lineno, + linepos=linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + *args, + ) # noqa: B026 def parse_call_args( self, @@ -908,7 +1428,11 @@ def ensure(expr: bool) -> None: key = self.stream.current.value self.stream.skip(2) value = self.parse_expression() - kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) + kwargs.append( + nodes.Keyword( + key, value, lineno=value.lineno, linepos=value.linepos + ) + ) else: # Parsing an arg ensure(dyn_args is None and dyn_kwargs is None and not kwargs) @@ -924,7 +1448,18 @@ def parse_call(self, node: nodes.Expr) -> nodes.Call: # needs to be recorded before the stream is advanced. token = self.stream.current args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() - return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) + end_token = self.stream.current + return nodes.Call( + node, + args, + kwargs, + dyn_args, + dyn_kwargs, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) def parse_filter( self, node: nodes.Expr | None, start_inline: bool = False @@ -932,8 +1467,29 @@ def parse_filter( while self.stream.current.type == "pipe" or start_inline: if not start_inline: next(self.stream) - token = self.stream.expect("name") - name = token.value + issues: list[nodes.ExprIssue] = [] + + def _get_name() -> str: + nonlocal issues + if ( + self.environment.parser_tolerate_faults + and not self.stream.current.test("name") + ): + issues.append( + nodes.EmptyExpression( + message="Missing name: Filter expected", + lineno=self.stream.current.lineno, + linepos=self.stream.current.linepos, + lineno_end=self.stream.current.lineno, + linepos_end=self.stream.current.linepos, + issue_context="filter", + ) + ) + return "" + return self.stream.expect("name").value + + name = _get_name() + token = self.stream.current while self.stream.current.type == "dot": next(self.stream) name += "." + self.stream.expect("name").value @@ -943,8 +1499,19 @@ def parse_filter( args = [] kwargs = [] dyn_args = dyn_kwargs = None + end_token = self.stream.current node = nodes.Filter( - node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + node, + name, + args, + kwargs, + dyn_args, + dyn_kwargs, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + issues=issues, ) start_inline = False return node @@ -956,10 +1523,32 @@ def parse_test(self, node: nodes.Expr) -> nodes.Expr: negated = True else: negated = False - name = self.stream.expect("name").value + issues: list[nodes.ExprIssue] = [] + + def _get_name() -> str: + nonlocal issues + if self.environment.parser_tolerate_faults and not self.stream.current.test( + "name" + ): + issues.append( + nodes.EmptyExpression( + message="Missing name: Test expected", + lineno=self.stream.current.lineno, + linepos=self.stream.current.linepos, + lineno_end=self.stream.current.lineno, + linepos_end=self.stream.current.linepos, + issue_context="test", + ) + ) + return "" + + return self.stream.expect("name").value + + name = _get_name() + while self.stream.current.type == "dot": next(self.stream) - name += "." + self.stream.expect("name").value + name += "." + _get_name() dyn_args = dyn_kwargs = None kwargs: list[nodes.Keyword] = [] if self.stream.current.type == "lparen": @@ -980,11 +1569,28 @@ def parse_test(self, node: nodes.Expr) -> nodes.Expr: args = [arg_node] else: args = [] + end_token = self.stream.current node = nodes.Test( - node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + node, + name, + args, + kwargs, + dyn_args, + dyn_kwargs, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + issues=issues, ) if negated: - node = nodes.Not(node, lineno=token.lineno) + node = nodes.Not( + node, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end_token.lineno, + linepos_end=end_token.linepos, + ) return node def subparse(self, end_tokens: tuple[str, ...] | None = None) -> list[nodes.Node]: @@ -998,19 +1604,57 @@ def subparse(self, end_tokens: tuple[str, ...] | None = None) -> list[nodes.Node def flush_data() -> None: if data_buffer: lineno = data_buffer[0].lineno - body.append(nodes.Output(data_buffer[:], lineno=lineno)) + linepos = data_buffer[0].linepos + lineno_end = data_buffer[-1].lineno_end + linepos_end = data_buffer[-1].linepos_end + body.append( + nodes.Output( + data_buffer[:], + lineno=lineno, + linepos=linepos, + lineno_end=lineno_end, + linepos_end=linepos_end, + ) + ) del data_buffer[:] try: while self.stream: token = self.stream.current if token.type == "data": + if "\n" not in token.value: + end = token.lineno, token.linepos + len(token.value) + else: + end = ( + token.lineno + token.value.count("\n"), + len(token.value.rsplit("\n", 1)[-1]), + ) if token.value: - add_data(nodes.TemplateData(token.value, lineno=token.lineno)) + add_data( + nodes.TemplateData( + token.value, + lineno=token.lineno, + linepos=token.linepos, + lineno_end=end[0], + linepos_end=end[1], + ) + ) next(self.stream) elif token.type == "variable_begin": next(self.stream) - add_data(self.parse_tuple(with_condexpr=True)) + data = self.parse_tuple( + with_condexpr=True, + allow_empty=self.environment.parser_tolerate_faults, + ) + if isinstance(data, nodes.EmptyExpression): + data.lineno, data.linepos = token.lineno, token.linepos + nxt = self.stream.current + data.lineno_end, data.linepos_end = nxt.lineno, nxt.linepos + if nxt.type == "variable_end": + data.linepos_end += len(nxt.value) + data.message = "Empty expression inside print statement" + data.issue_context = "print" + add_data(data) self.stream.expect("variable_end") elif token.type == "block_begin": flush_data() @@ -1023,6 +1667,18 @@ def flush_data() -> None: if isinstance(rv, list): body.extend(rv) else: + nxt = self.stream.current + if self.environment.parser_tolerate_faults and isinstance( + rv, (nodes.ParserIssue, nodes.EmptyStatement) + ): + rv.lineno, rv.linepos = token.lineno, token.linepos + rv = nodes.Output( + [rv], + lineno=token.lineno, + linepos=token.linepos, + lineno_end=nxt.lineno, + linepos_end=nxt.linepos, + ) body.append(rv) self.stream.expect("block_end") else: @@ -1036,6 +1692,17 @@ def flush_data() -> None: def parse(self) -> nodes.Template: """Parse the whole template into a `Template` node.""" - result = nodes.Template(self.subparse(), lineno=1) + result = nodes.Template(self.subparse(), lineno=1, linepos=0) result.set_environment(self.environment) + # Set end position to the last token + end_token = self.stream.current + end_element = result.body[-1] if result.body else None + end = max( + (end_token.lineno, end_token.linepos), + (-1, -1) + if not end_element + else (end_element.lineno_end, end_element.linepos_end), + ) + result.lineno_end = end[0] + result.linepos_end = end[1] return result