Fix 14 bugs found during code audit

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@microsoft/powerquery-parser",
-    "version": "0.19.1",
+    "version": "1.0.0",
     "description": "A parser for the Power Query/M formula language.",
     "author": "Microsoft",
     "license": "MIT",

src/powerquery-parser/common/patterns.ts

@@ -2,15 +2,15 @@
 // Licensed under the MIT license.
 
 export const IdentifierStartCharacter: RegExp =
-    /(?:[\p{Uppercase_Letter}|\p{Lowercase_Letter}|\p{Titlecase_Letter}|\p{Modifier_Letter}|\p{Other_Letter}|\p{Letter_Number}|\u{5F}]+)/gu;
+    /(?:[\p{Uppercase_Letter}\p{Lowercase_Letter}\p{Titlecase_Letter}\p{Modifier_Letter}\p{Other_Letter}\p{Letter_Number}\u{5F}]+)/gu;
 
 export const IdentifierPartCharacters: RegExp =
-    /(?:[\p{Uppercase_Letter}|\p{Lowercase_Letter}|\p{Titlecase_Letter}|\p{Modifier_Letter}|\p{Other_Letter}|\p{Letter_Number}|\p{Decimal_Number}|\p{Connector_Punctuation}|\p{Spacing_Mark}|\p{Nonspacing_Mark}|\p{Format}]+)/gu;
+    /(?:[\p{Uppercase_Letter}\p{Lowercase_Letter}\p{Titlecase_Letter}\p{Modifier_Letter}\p{Other_Letter}\p{Letter_Number}\p{Decimal_Number}\p{Connector_Punctuation}\p{Spacing_Mark}\p{Nonspacing_Mark}\p{Format}]+)/gu;
 
 export const Whitespace: RegExp =
     // eslint-disable-next-line no-control-regex
-    /(:?[\u000b-\u000c\u2000-\u200a])|(?:\u0009)|(?:\u0020)|(?:\u00a0)|(?:\u1680)|(?:\u202f)|(?:\u205f)|(?:\u3000)/g;
+    /(?:[\u000b-\u000c\u2000-\u200a])|(?:\u0009)|(?:\u0020)|(?:\u00a0)|(?:\u1680)|(?:\u202f)|(?:\u205f)|(?:\u3000)/g;
 
 export const Hex: RegExp = /0[xX][a-fA-F0-9]+/g;
 
-export const Numeric: RegExp = /(([0-9]*\.[0-9]+)|([0-9]+))([eE][\\+\\-]?[0-9]+)?/g;
+export const Numeric: RegExp = /(([0-9]*\.[0-9]+)|([0-9]+))([eE][+-]?[0-9]+)?/g;

src/powerquery-parser/common/stringUtils.ts

@@ -147,6 +147,8 @@ export function findQuotes(text: string, indexStart: number): FoundQuotes | unde
                 continue;
             } else {
                 index += 2;
+
+                continue;
             }
         }
 
@@ -184,6 +186,7 @@ export function newlineKindAt(text: string, index: number): NewlineKind | undefi
         case `\u2028`:
             return NewlineKind.SingleCharacter;
 
+        case undefined:
         default:
             return undefined;
     }

src/powerquery-parser/language/token.ts

@@ -7,7 +7,7 @@ export enum LineTokenKindAdditions {
     MultilineCommentContent = "MultilineCommentContent",
     MultilineCommentEnd = "MultilineCommentEnd",
     MultilineCommentStart = "MultilineCommentStart",
-    TextLiteralContent = "TextContent",
+    TextLiteralContent = "TextLiteralContent",
     TextLiteralEnd = "TextLiteralEnd",
     TextLiteralStart = "TextLiteralStart",
     QuotedIdentifierContent = "QuotedIdentifierContent",

src/powerquery-parser/language/type/typeUtils/isCompatible.ts

@@ -777,7 +777,14 @@ function isCompatibleDefinedListOrDefinedListType<T extends Type.DefinedList | T
     const numElements: number = leftElements.length;
 
     for (let index: number = 0; index < numElements; index += 1) {
-        if (!isCompatible(ArrayUtils.assertGet(leftElements, index), ArrayUtils.assertGet(rightElements, index), traceManager, trace.id)) {
+        if (
+            !isCompatible(
+                ArrayUtils.assertGet(leftElements, index),
+                ArrayUtils.assertGet(rightElements, index),
+                traceManager,
+                trace.id,
+            )
+        ) {
             trace.exit();
 
             return false;

src/powerquery-parser/language/type/typeUtils/typeUtils.ts

@@ -1,10 +1,10 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import { ArrayUtils, Assert } from "../../../common";
 import { Ast, AstUtils } from "../..";
 import { NodeIdMap, NodeIdMapUtils, ParseContext, XorNode, XorNodeKind } from "../../../parser";
 import { Trace, TraceManager } from "../../../common/trace";
-import { ArrayUtils, Assert } from "../../../common";
 import { isCompatible } from "./isCompatible";
 import { isEqualType } from "./isEqualType";
 import { primitiveType } from "./factories";

src/powerquery-parser/lexer/lexer.ts

@@ -168,7 +168,12 @@ export function equalLines(leftLines: ReadonlyArray<TLine>, rightLines: Readonly
         const numTokens: number = leftTokens.length;
 
         for (let tokenIndex: number = 0; tokenIndex < numTokens; tokenIndex += 1) {
-            if (!equalTokens(ArrayUtils.assertGet(leftTokens, tokenIndex), ArrayUtils.assertGet(rightTokens, tokenIndex))) {
+            if (
+                !equalTokens(
+                    ArrayUtils.assertGet(leftTokens, tokenIndex),
+                    ArrayUtils.assertGet(rightTokens, tokenIndex),
+                )
+            ) {
                 return false;
             }
         }
@@ -400,7 +405,11 @@ function updateRange(state: State, range: Range, text: string): State {
     const lines: ReadonlyArray<TLine> = [
         ...state.lines.slice(0, rangeStart.lineNumber),
         ...newLines,
-        ...retokenizeLines(state, rangeEnd.lineNumber + 1, ArrayUtils.assertGet(newLines, newLines.length - 1).lineModeEnd),
+        ...retokenizeLines(
+            state,
+            rangeEnd.lineNumber + 1,
+            ArrayUtils.assertGet(newLines, newLines.length - 1).lineModeEnd,
+        ),
     ];
 
     return {
@@ -506,7 +515,7 @@ function retokenizeLines(state: State, lineNumber: number, previousLineModeEnd:
                 lineNumber += 1;
                 currentLine = lines[lineNumber];
             } else {
-                return [...retokenizedLines, ...lines.slice(lineNumber + 1)];
+                return [...retokenizedLines, ...lines.slice(lineNumber)];
             }
         }
 

src/powerquery-parser/lexer/lexerSnapshot.ts

@@ -57,7 +57,7 @@ export class LexerSnapshot {
             text.substring(lineBounds.substringPositionStart, lineBounds.substringPositionEnd),
             flatLineToken.positionStart.lineCodeUnit,
             flatLineToken.positionStart.lineNumber,
-            flatLineToken.positionEnd.codeUnit,
+            flatLineToken.positionStart.codeUnit,
         );
     }
 
@@ -84,7 +84,7 @@ export class LexerSnapshot {
                 cached.lineText,
                 token.positionStart.lineCodeUnit,
             ),
-            codeUnit: token.positionEnd.codeUnit,
+            codeUnit: token.positionStart.codeUnit,
         };
     }
 

src/powerquery-parser/parser/context/contextUtils.ts

@@ -58,7 +58,7 @@ export function isNodeKind<T extends Ast.TNode>(
     node: ParseContext.TNode,
     expectedNodeKinds: ReadonlyArray<T["kind"]> | T["kind"],
 ): node is ParseContext.Node<T> {
-    return node.kind === expectedNodeKinds || expectedNodeKinds.includes(node.kind);
+    return Array.isArray(expectedNodeKinds) ? expectedNodeKinds.includes(node.kind) : node.kind === expectedNodeKinds;
 }
 
 export function nextId(state: ParseContext.State): number {

src/powerquery-parser/parser/disambiguation/disambiguationUtils.ts

@@ -234,7 +234,10 @@ export async function disambiguateParenthesis(
                 try {
                     // eslint-disable-next-line no-await-in-loop
                     await parser.readNullablePrimitiveType(state, parser, trace.id);
-                } catch {
+                } catch (error: unknown) {
+                    Assert.isInstanceofError(error);
+                    CommonError.throwIfCancellationError(error);
+
                     // eslint-disable-next-line no-await-in-loop
                     await parser.restoreCheckpoint(state, checkpoint);
 

src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts

@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import { ArrayUtils, Assert } from "../../common";
 import { Ast, Constant, IdentifierUtils } from "../../language";
 import { NodeIdMap, NodeIdMapUtils, TXorNode, XorNodeKind, XorNodeUtils } from ".";
-import { ArrayUtils, Assert } from "../../common";
 import { parameterIdentifier } from "./nodeIdMapUtils";
 import { XorNode } from "./xorNode";
 
@@ -124,7 +124,7 @@ export function nthSiblingXor(
         parentXorNode.node.id,
     );
 
-    if (childIds.length >= attributeIndex) {
+    if (attributeIndex >= childIds.length) {
         return undefined;
     }
 

src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/leafSelectors.ts

@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import { ArrayUtils, Assert } from "../../../common";
 import { AstNodeById, Collection } from "../nodeIdMap";
 import { NodeIdMap, XorNodeUtils } from "..";
-import { ArrayUtils, Assert } from "../../../common";
 import { Ast } from "../../../language";
 import { TXorNode } from "../xorNode";
 import { xor } from "./commonSelectors";

src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/nodeIdMapUtils.ts

@@ -1,10 +1,10 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import { ArrayUtils, Assert } from "../../../common";
 import { Ast, Token } from "../../../language";
 import { Collection, CollectionValidation, IdsByNodeKind, NodeSummary } from "../nodeIdMap";
 import { TXorNode, XorNodeKind, XorNodeTokenRange } from "../xorNode";
-import { ArrayUtils, Assert } from "../../../common";
 import { ParseContext } from "../../context";
 import { rightMostLeaf } from "./leafSelectors";
 

src/powerquery-parser/parser/parseState/parseStateUtils.ts

@@ -37,7 +37,7 @@ export function newState(lexerSnapshot: LexerSnapshot, overrides?: Partial<Parse
         locale: overrides?.locale ?? DefaultLocale,
         cancellationToken: overrides?.cancellationToken,
         traceManager: overrides?.traceManager ?? NoOpTraceManagerInstance,
-        contextState: overrides?.contextState ?? ParseContextUtils.newState(),
+        contextState,
         currentToken,
         currentContextNode,
         currentTokenKind,
@@ -58,9 +58,15 @@ export async function applyState(state: ParseState, update: ParseState): Promise
 // If you have a custom parser + parser state, then you'll have to create your own copyState/applyState functions.
 // eslint-disable-next-line require-await
 export async function copyState(state: ParseState): Promise<ParseState> {
+    const contextState: ParseContext.State = ParseContextUtils.copyState(state.contextState);
+
     return {
         ...state,
-        contextState: ParseContextUtils.copyState(state.contextState),
+        contextState,
+        currentContextNode:
+            state.currentContextNode !== undefined
+                ? MapUtils.assertGet(contextState.nodeIdMapCollection.contextNodeById, state.currentContextNode.id)
+                : undefined,
     };
 }
 

src/powerquery-parser/parser/parser/parserUtils.ts

@@ -188,7 +188,7 @@ async function tryParseDocument(parseSettings: ParseSettings, lexerSnapshot: Lex
         }
 
         default:
-            Assert.isNever(parseSettings.parseBehavior);
+            throw Assert.isNever(parseSettings.parseBehavior);
     }
 }
 

src/powerquery-parser/parser/parsers/combinatorialParserV2/combineOperatorsAndOperands.ts

@@ -188,7 +188,7 @@ interface Validator<Node extends Ast.TBinOpExpression> {
 
 const ValidatorForAsExpression: Validator<Ast.AsExpression> = {
     tag: "AsExpression",
-    validateLeftOperand: (node: Ast.TNode): node is Ast.TEqualityExpression => AstUtils.isTAsExpression(node),
+    validateLeftOperand: (node: Ast.TNode): node is Ast.TAsExpression => AstUtils.isTAsExpression(node),
     validateRightOperand: (node: Ast.TNode): node is Ast.TNullablePrimitiveType =>
         AstUtils.isTNullablePrimitiveType(node),
     fallbackLeftOperand: (state: ParseState, parser: Parser, correlationId: number) =>
@@ -199,8 +199,8 @@ const ValidatorForAsExpression: Validator<Ast.AsExpression> = {
 
 const ValidatorForEqualityExpressionAndBelow: Validator<TEqualityExpressionAndBelow> = {
     tag: "EqualityExpression",
-    validateLeftOperand: (node: Ast.TNode): node is Ast.TMetadataExpression => AstUtils.isTEqualityExpression(node),
-    validateRightOperand: (node: Ast.TNode): node is Ast.TMetadataExpression => AstUtils.isTEqualityExpression(node),
+    validateLeftOperand: (node: Ast.TNode): node is Ast.TEqualityExpression => AstUtils.isTEqualityExpression(node),
+    validateRightOperand: (node: Ast.TNode): node is Ast.TEqualityExpression => AstUtils.isTEqualityExpression(node),
     fallbackLeftOperand: (state: ParseState, parser: Parser, correlationId: number) =>
         NaiveParseSteps.readMetadataExpression(state, parser, correlationId),
     fallbackRightOperand: (state: ParseState, parser: Parser, correlationId: number) =>

src/powerquery-parser/parser/parsers/naiveParseSteps.ts

@@ -122,8 +122,13 @@ export async function readGeneralizedIdentifier(
 
     const lexerSnapshot: LexerSnapshot = state.lexerSnapshot;
     const tokens: ReadonlyArray<Token.Token> = lexerSnapshot.tokens;
-    const contiguousIdentifierStartIndex: number = ArrayUtils.assertGet(tokens, tokenRangeStartIndex).positionStart.codeUnit;
-    const contiguousIdentifierEndIndex: number = ArrayUtils.assertGet(tokens, tokenRangeEndIndex - 1).positionEnd.codeUnit;
+
+    const contiguousIdentifierStartIndex: number = ArrayUtils.assertGet(tokens, tokenRangeStartIndex).positionStart
+        .codeUnit;
+
+    const contiguousIdentifierEndIndex: number = ArrayUtils.assertGet(tokens, tokenRangeEndIndex - 1).positionEnd
+        .codeUnit;
+
     const literal: string = lexerSnapshot.text.slice(contiguousIdentifierStartIndex, contiguousIdentifierEndIndex);
 
     const literalKind: IdentifierUtils.IdentifierKind = IdentifierUtils.getIdentifierKind(literal, {

src/test/libraryTest/common/cancellationToken.test.ts

@@ -3,8 +3,11 @@
 
 import "mocha";
 
+import { expect } from "chai";
+
 import {
     CommonError,
+    CounterCancellationToken,
     DefaultSettings,
     Lexer,
     Result,
@@ -47,6 +50,36 @@ function defaultSettingsWithExpiredCancellationToken(): Settings {
 }
 
 describe("CancellationToken", () => {
+    describe(`CounterCancellationToken`, () => {
+        it(`throwIfCancelled should consume exactly 1 count`, () => {
+            // With threshold of 3, we should be able to call throwIfCancelled 2 times
+            // without throwing (counts 1 and 2), then the 3rd should throw (count 3).
+            const token: CounterCancellationToken = new CounterCancellationToken(3);
+
+            // First call: counter goes to 1, threshold is 3, no throw
+            token.throwIfCancelled();
+
+            // Second call: counter goes to 2, threshold is 3, no throw
+            token.throwIfCancelled();
+
+            // Third call: counter goes to 3, threshold is 3, should throw
+            expect(() => token.throwIfCancelled()).to.throw(CommonError.CancellationError);
+        });
+
+        it(`isCancelled should consume exactly 1 count`, () => {
+            const token: CounterCancellationToken = new CounterCancellationToken(3);
+
+            // First call: counter goes to 1, not cancelled
+            expect(token.isCancelled()).to.be.false;
+
+            // Second call: counter goes to 2, not cancelled
+            expect(token.isCancelled()).to.be.false;
+
+            // Third call: counter goes to 3, cancelled
+            expect(token.isCancelled()).to.be.true;
+        });
+    });
+
     describe(`lexer`, () => {
         it(`Lexer.tryLex`, () => {
             const triedLex: Lexer.TriedLex = Lexer.tryLex(defaultSettingsWithExpiredCancellationToken(), "foo");

src/test/libraryTest/common/stringUtils.test.ts

@@ -91,6 +91,33 @@ describe("StringUtils", () => {
 
             expect(actual).to.deep.equal(expected);
         });
+
+        it(`"a""""b" - consecutive escaped quotes`, () => {
+            // Content represents: a""b (two escaped quotes then 'b')
+            // The "" escape at index 2-3 must not cause index 4 to be skipped
+            const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`"a""""b"`, 0);
+
+            const expected: StringUtils.FoundQuotes = {
+                indexStart: 0,
+                indexEnd: 8,
+                quoteLength: 8,
+            };
+
+            expect(actual).to.deep.equal(expected);
+        });
+
+        it(`"x""y""z" - multiple escaped quotes`, () => {
+            // Content represents: x"y"z
+            const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`"x""y""z"`, 0);
+
+            const expected: StringUtils.FoundQuotes = {
+                indexStart: 0,
+                indexEnd: 9,
+                quoteLength: 9,
+            };
+
+            expect(actual).to.deep.equal(expected);
+        });
     });
 
     describe(`normalizeNumber`, () => {