JS: Support TemplatePlaceholderTag.getEnclosingExpr

fixup! makeLocation

Author: asgerf

javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java

@@ -6,6 +6,7 @@
 import java.util.List;
 import java.util.Set;
 import java.util.Stack;
+import java.util.regex.Matcher;
 
 import com.semmle.js.ast.AClass;
 import com.semmle.js.ast.AFunction;
@@ -570,6 +571,16 @@ public Label visit(Literal nd, Context c) {
         regexpExtractor.extract(source.substring(1, source.lastIndexOf('/')), offsets, nd, false);
       } else if (nd.isStringLiteral() && !c.isInsideType() && nd.getRaw().length() < 1000) {
         regexpExtractor.extract(valueString, makeStringLiteralOffsets(nd.getRaw()), nd, true);
+
+        // Scan the string for template tags, if we're in a context where such tags are relevant.
+        if (scopeManager.isInTemplateFile()) {
+          Matcher m = TemplateEngines.TEMPLATE_TAGS.matcher(nd.getRaw());
+          int offset = nd.getLoc().getStart().getOffset();
+          while (m.find()) {
+            Label locationLbl = TemplateEngines.makeLocation(lexicalExtractor.getTextualExtractor(), offset + m.start(), offset + m.end());
+            trapwriter.addTuple("expr_contains_template_tag_location", key, locationLbl);
+          }
+        }
       }
       return key;
     }
@@ -2225,7 +2236,8 @@ public Label visit(AngularPipeRef nd, Context c) {
     @Override
     public Label visit(GeneratedCodeExpr nd, Context c) {
         Label key = super.visit(nd, c);
-        trapwriter.addTuple("generated_code_expr_info", key, nd.getOpeningDelimiter(), nd.getClosingDelimiter(), nd.getBody());
+        Label templateLbl = TemplateEngines.makeLocation(lexicalExtractor.getTextualExtractor(), nd.getLoc().getStart().getOffset(), nd.getLoc().getEnd().getOffset());
+        trapwriter.addTuple("expr_contains_template_tag_location", key, templateLbl);
         return key;
     }
   }

javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java

@@ -39,7 +39,7 @@ public JavaScriptHTMLElementHandler(TextualExtractor textualExtractor) {
       this.textualExtractor = textualExtractor;
 
       this.scopeManager =
-          new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion());
+          new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion(), true);
     }
 
     /*
@@ -382,21 +382,6 @@ private void emitTopLevelXmlNodeBinding(
     writer.addTuple("toplevel_parent_xml_node", topLevelLabel, htmlNodeLabel);
   }
 
-  private static final String MUSTACHE_TAG_DOUBLE = "\\{\\{(?!\\{)(.*?)\\}\\}"; // {{ x }}
-  private static final String MUSTACHE_TAG_TRIPLE = "\\{\\{\\{(.*?)\\}\\}\\}"; // {{{ x }}}
-  private static final String MUSTACHE_TAG_PERCENT = "\\{%(?!>)(.*?)%\\}"; // {% x %}
-  private static final String EJS_TAG = "<%(?![%<>}])[-=]?(.*?)[_-]?%>"; // <% x %>
-
-  /** Pattern for a template tag whose contents should be parsed as an expression */
-  private static final Pattern TEMPLATE_EXPR_OPENING_TAG =
-      Pattern.compile("^(?:\\{\\{\\{?|<%[-=])"); // {{, {{{, <%=, <%-
-
-  private static final Pattern TEMPLATE_TAGS =
-      Pattern.compile(
-          StringUtil.glue(
-              "|", MUSTACHE_TAG_DOUBLE, MUSTACHE_TAG_TRIPLE, MUSTACHE_TAG_PERCENT, EJS_TAG),
-          Pattern.DOTALL);
-
   private void extractTemplateTags(
       TextualExtractor textualExtractor,
       ScopeManager scopeManager,
@@ -407,9 +392,8 @@ private void extractTemplateTags(
     if (start >= end) return;
     if (isEmbedded) return; // Do not extract template tags for HTML snippets embedded in a JS file
 
-    LocationManager locationManager = textualExtractor.getLocationManager();
     TrapWriter trapwriter = textualExtractor.getTrapwriter();
-    Matcher m = TEMPLATE_TAGS.matcher(textualExtractor.getSource()).region(start, end);
+    Matcher m = TemplateEngines.TEMPLATE_TAGS.matcher(textualExtractor.getSource()).region(start, end);
     while (m.find()) {
       int startOffset = m.start();
       int endOffset = m.end();
@@ -424,15 +408,12 @@ private void extractTemplateTags(
       String rawText = m.group();
       trapwriter.addTuple("template_placeholder_tag_info", lbl, parentLabel.get(), rawText);
 
-      // Emit location
-      Position startPos = textualExtractor.getSourceMap().getStart(startOffset);
-      Position endPos = textualExtractor.getSourceMap().getEnd(endOffset - 1);
-      int endColumn = endPos.getColumn() - 1; // Convert to inclusive end position (still 1-based)
-      locationManager.emitFileLocation(
-          lbl, startPos.getLine(), startPos.getColumn(), endPos.getLine(), endColumn);
+      // Emit location entity
+      Label locationLbl = TemplateEngines.makeLocation(textualExtractor, startOffset, endOffset);
+      trapwriter.addTuple("hasLocation", lbl, locationLbl);
 
       // Parse the contents as a template expression, if the delimiter expects an expression.
-      Matcher delimMatcher = TEMPLATE_EXPR_OPENING_TAG.matcher(rawText);
+      Matcher delimMatcher = TemplateEngines.TEMPLATE_EXPR_OPENING_TAG.matcher(rawText);
       if (delimMatcher.find()) {
         // The body of the template tag is stored in the first capture group of each pattern
         int bodyGroup = getNonNullCaptureGroup(m);

javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java

@@ -103,12 +103,22 @@ public Label lookupNamespace(String name) {
   private final ECMAVersion ecmaVersion;
   private final Set<String> implicitGlobals = new LinkedHashSet<String>();
   private Scope implicitVariableScope;
+  private final boolean isInTemplateScope;
 
-  public ScopeManager(TrapWriter trapWriter, ECMAVersion ecmaVersion) {
+  public ScopeManager(TrapWriter trapWriter, ECMAVersion ecmaVersion, boolean isInTemplateScope) {
     this.trapWriter = trapWriter;
     this.toplevelScope = enterScope(ScopeKind.GLOBAL, trapWriter.globalID("global_scope"), null);
     this.ecmaVersion = ecmaVersion;
     this.implicitVariableScope = toplevelScope; 
+    this.isInTemplateScope = isInTemplateScope;
+  }
+
+  /**
+   * Returns true the current scope is potentially in a template file, and may contain
+   * relevant template tags.
+   */
+  public boolean isInTemplateFile() {
+    return isInTemplateScope;
   }
 
   /**

javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java

@@ -77,7 +77,7 @@ public LoCInfo extract(TextualExtractor textualExtractor) {
     }
 
     ScopeManager scopeManager =
-        new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion());
+        new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion(), false);
     Label toplevelLabel = null;
     LoCInfo loc;
     try {

javascript/extractor/src/com/semmle/js/extractor/TemplateEngines.java

@@ -0,0 +1,37 @@
+package com.semmle.js.extractor;
+
+import java.util.regex.Pattern;
+
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.locations.Position;
+import com.semmle.util.trap.TrapWriter.Label;
+
+public class TemplateEngines {
+  private static final String MUSTACHE_TAG_TRIPLE = "\\{\\{\\{(.*?)\\}\\}\\}"; // {{{ x }}}
+  private static final String MUSTACHE_TAG_DOUBLE = "\\{\\{(?!\\{)(.*?)\\}\\}"; // {{ x }}}
+  private static final String MUSTACHE_TAG_PERCENT = "\\{%(?!>)(.*?)%\\}"; // {% x %}
+  private static final String EJS_TAG = "<%(?![%<>}])[-=]?(.*?)[_-]?%>"; // <% x %>
+
+  /** Pattern for a template tag whose contents should be parsed as an expression */
+  public static final Pattern TEMPLATE_EXPR_OPENING_TAG =
+      Pattern.compile("^(?:\\{\\{\\{?|<%[-=])"); // {{, {{{, <%=, <%-
+
+  /**
+   * Pattern matching a template tag from a supported template engine.
+   */
+  public static final Pattern TEMPLATE_TAGS =
+      Pattern.compile(
+          StringUtil.glue(
+              "|", TemplateEngines.MUSTACHE_TAG_DOUBLE, MUSTACHE_TAG_TRIPLE, MUSTACHE_TAG_PERCENT, EJS_TAG),
+          Pattern.DOTALL);
+
+  /**
+   * Returns the location label for a template tag at the given offsets.
+   */
+  public static Label makeLocation(TextualExtractor extractor, int startOffset, int endOffset) {
+    Position startPos = extractor.getSourceMap().getStart(startOffset);
+    Position endPos = extractor.getSourceMap().getEnd(endOffset - 1);
+    int endColumn = endPos.getColumn() - 1; // Convert to inclusive end position (still 1-based)
+    return extractor.getLocationManager().emitLocationsDefault(startPos.getLine(), startPos.getColumn(), endPos.getLine(), endColumn);
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java

@@ -23,7 +23,7 @@ public LoCInfo extract(TextualExtractor textualExtractor) {
     File sourceFile = textualExtractor.getExtractedFile();
     Result res = state.getTypeScriptParser().parse(sourceFile, source, textualExtractor.getMetrics());
     ScopeManager scopeManager =
-        new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2017);
+        new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2017, false);
     try {
       FileSnippet snippet = state.getSnippets().get(sourceFile.toPath());
       SourceType sourceType = snippet != null ? snippet.getSourceType() : jsExtractor.establishSourceType(source, false);

javascript/ql/src/semmle/javascript/Expr.qll

@@ -2893,18 +2893,9 @@ class ImportMetaExpr extends @import_meta_expr, Expr {
  * ```
  */
 class GeneratedCodeExpr extends @generated_code_expr, Expr {
-  /** Gets the opening delimiter, such as `{{` or `{{{`. */
-  string getOpeningDelimiter() { generated_code_expr_info(this, result, _, _) }
-
-  /** Gets the closing delimiter, such as `}}` or `}}}`. */
-  string getClosingDelimiter() { generated_code_expr_info(this, _, result, _) }
-
-  /** Gets the text between the delimiters, including any surrounding whitespace, such as the `x` in `{{x}}`. */
-  string getBody() { generated_code_expr_info(this, _, _, result) }
-
   /** Gets the placeholder tag that was parsed as an expression. */
   Templating::TemplatePlaceholderTag getPlaceholderTag() {
-    result.getLocation() = this.getLocation()
+    this = result.getEnclosingExpr()
   }
 
   override string getAPrimaryQlClass() { result = "GeneratedCodeExpr" }

javascript/ql/src/semmle/javascript/frameworks/Templating.qll

@@ -114,6 +114,13 @@ module Templating {
      * that is, without being enclosed in a string literal or similar.
      */
     predicate isInPlainCodeContext() { this = any(GeneratedCodeExpr e).getPlaceholderTag() }
+
+    /**
+     * Gets the innermost JavaScript expression containing this template tag, if any.
+     */
+    Expr getEnclosingExpr() {
+      expr_contains_template_tag_location(result, getLocation())
+    }
   }
 
   /**

javascript/ql/src/semmlecode.javascript.dbscheme

@@ -417,11 +417,9 @@ case @expr.kind of
 @e4x_xml_attribute_selector = @e4x_xml_static_attribute_selector | @e4x_xml_dynamic_attribute_selector;
 @e4x_xml_qualident = @e4x_xml_static_qualident | @e4x_xml_dynamic_qualident;
 
-generated_code_expr_info(
-  unique int expr: @generated_code_expr ref,
-  varchar(900) openingDelimiter: string ref,
-  varchar(900) closingDelimiter: string ref,
-  varchar(900) body: string ref
+expr_contains_template_tag_location(
+  int expr: @expr ref,
+  int location: @location ref
 );
 
 @template_placeholder_tag_parent = @xmlelement | @xmlattribute | @file;