diff --git a/package-lock.json b/package-lock.json index 1a8eeb18..f6010dbc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@microsoft/powerquery-parser", - "version": "0.19.1", + "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@microsoft/powerquery-parser", - "version": "0.19.1", + "version": "1.0.0", "license": "MIT", "dependencies": { "grapheme-splitter": "^1.0.4", diff --git a/package.json b/package.json index 791a1dc9..08b9df1e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@microsoft/powerquery-parser", - "version": "0.19.1", + "version": "1.0.0", "description": "A parser for the Power Query/M formula language.", "author": "Microsoft", "license": "MIT", diff --git a/src/powerquery-parser/common/patterns.ts b/src/powerquery-parser/common/patterns.ts index d472fe51..1b73f51f 100644 --- a/src/powerquery-parser/common/patterns.ts +++ b/src/powerquery-parser/common/patterns.ts @@ -2,15 +2,15 @@ // Licensed under the MIT license. export const IdentifierStartCharacter: RegExp = - /(?:[\p{Uppercase_Letter}|\p{Lowercase_Letter}|\p{Titlecase_Letter}|\p{Modifier_Letter}|\p{Other_Letter}|\p{Letter_Number}|\u{5F}]+)/gu; + /(?:[\p{Uppercase_Letter}\p{Lowercase_Letter}\p{Titlecase_Letter}\p{Modifier_Letter}\p{Other_Letter}\p{Letter_Number}\u{5F}]+)/gu; export const IdentifierPartCharacters: RegExp = - /(?:[\p{Uppercase_Letter}|\p{Lowercase_Letter}|\p{Titlecase_Letter}|\p{Modifier_Letter}|\p{Other_Letter}|\p{Letter_Number}|\p{Decimal_Number}|\p{Connector_Punctuation}|\p{Spacing_Mark}|\p{Nonspacing_Mark}|\p{Format}]+)/gu; + /(?:[\p{Uppercase_Letter}\p{Lowercase_Letter}\p{Titlecase_Letter}\p{Modifier_Letter}\p{Other_Letter}\p{Letter_Number}\p{Decimal_Number}\p{Connector_Punctuation}\p{Spacing_Mark}\p{Nonspacing_Mark}\p{Format}]+)/gu; export const Whitespace: RegExp = // eslint-disable-next-line no-control-regex - /(:?[\u000b-\u000c\u2000-\u200a])|(?:\u0009)|(?:\u0020)|(?:\u00a0)|(?:\u1680)|(?:\u202f)|(?:\u205f)|(?:\u3000)/g; + /(?:[\u000b-\u000c\u2000-\u200a])|(?:\u0009)|(?:\u0020)|(?:\u00a0)|(?:\u1680)|(?:\u202f)|(?:\u205f)|(?:\u3000)/g; export const Hex: RegExp = /0[xX][a-fA-F0-9]+/g; -export const Numeric: RegExp = /(([0-9]*\.[0-9]+)|([0-9]+))([eE][\\+\\-]?[0-9]+)?/g; +export const Numeric: RegExp = /(([0-9]*\.[0-9]+)|([0-9]+))([eE][+-]?[0-9]+)?/g; diff --git a/src/powerquery-parser/common/stringUtils.ts b/src/powerquery-parser/common/stringUtils.ts index 4bf5dcea..1b3a6c3a 100644 --- a/src/powerquery-parser/common/stringUtils.ts +++ b/src/powerquery-parser/common/stringUtils.ts @@ -147,6 +147,8 @@ export function findQuotes(text: string, indexStart: number): FoundQuotes | unde continue; } else { index += 2; + + continue; } } @@ -184,6 +186,7 @@ export function newlineKindAt(text: string, index: number): NewlineKind | undefi case `\u2028`: return NewlineKind.SingleCharacter; + case undefined: default: return undefined; } diff --git a/src/powerquery-parser/language/token.ts b/src/powerquery-parser/language/token.ts index 44454f5a..a8b72caa 100644 --- a/src/powerquery-parser/language/token.ts +++ b/src/powerquery-parser/language/token.ts @@ -7,7 +7,7 @@ export enum LineTokenKindAdditions { MultilineCommentContent = "MultilineCommentContent", MultilineCommentEnd = "MultilineCommentEnd", MultilineCommentStart = "MultilineCommentStart", - TextLiteralContent = "TextContent", + TextLiteralContent = "TextLiteralContent", TextLiteralEnd = "TextLiteralEnd", TextLiteralStart = "TextLiteralStart", QuotedIdentifierContent = "QuotedIdentifierContent", diff --git a/src/powerquery-parser/language/type/typeUtils/isCompatible.ts b/src/powerquery-parser/language/type/typeUtils/isCompatible.ts index 09821c55..b4978c91 100644 --- a/src/powerquery-parser/language/type/typeUtils/isCompatible.ts +++ b/src/powerquery-parser/language/type/typeUtils/isCompatible.ts @@ -777,7 +777,14 @@ function isCompatibleDefinedListOrDefinedListType, rightLines: Readonly const numTokens: number = leftTokens.length; for (let tokenIndex: number = 0; tokenIndex < numTokens; tokenIndex += 1) { - if (!equalTokens(ArrayUtils.assertGet(leftTokens, tokenIndex), ArrayUtils.assertGet(rightTokens, tokenIndex))) { + if ( + !equalTokens( + ArrayUtils.assertGet(leftTokens, tokenIndex), + ArrayUtils.assertGet(rightTokens, tokenIndex), + ) + ) { return false; } } @@ -400,7 +405,11 @@ function updateRange(state: State, range: Range, text: string): State { const lines: ReadonlyArray = [ ...state.lines.slice(0, rangeStart.lineNumber), ...newLines, - ...retokenizeLines(state, rangeEnd.lineNumber + 1, ArrayUtils.assertGet(newLines, newLines.length - 1).lineModeEnd), + ...retokenizeLines( + state, + rangeEnd.lineNumber + 1, + ArrayUtils.assertGet(newLines, newLines.length - 1).lineModeEnd, + ), ]; return { @@ -506,7 +515,7 @@ function retokenizeLines(state: State, lineNumber: number, previousLineModeEnd: lineNumber += 1; currentLine = lines[lineNumber]; } else { - return [...retokenizedLines, ...lines.slice(lineNumber + 1)]; + return [...retokenizedLines, ...lines.slice(lineNumber)]; } } diff --git a/src/powerquery-parser/lexer/lexerSnapshot.ts b/src/powerquery-parser/lexer/lexerSnapshot.ts index 81c95298..674ebafd 100644 --- a/src/powerquery-parser/lexer/lexerSnapshot.ts +++ b/src/powerquery-parser/lexer/lexerSnapshot.ts @@ -57,7 +57,7 @@ export class LexerSnapshot { text.substring(lineBounds.substringPositionStart, lineBounds.substringPositionEnd), flatLineToken.positionStart.lineCodeUnit, flatLineToken.positionStart.lineNumber, - flatLineToken.positionEnd.codeUnit, + flatLineToken.positionStart.codeUnit, ); } @@ -84,7 +84,7 @@ export class LexerSnapshot { cached.lineText, token.positionStart.lineCodeUnit, ), - codeUnit: token.positionEnd.codeUnit, + codeUnit: token.positionStart.codeUnit, }; } diff --git a/src/powerquery-parser/parser/context/contextUtils.ts b/src/powerquery-parser/parser/context/contextUtils.ts index ada19796..8effbb25 100644 --- a/src/powerquery-parser/parser/context/contextUtils.ts +++ b/src/powerquery-parser/parser/context/contextUtils.ts @@ -58,7 +58,7 @@ export function isNodeKind( node: ParseContext.TNode, expectedNodeKinds: ReadonlyArray | T["kind"], ): node is ParseContext.Node { - return node.kind === expectedNodeKinds || expectedNodeKinds.includes(node.kind); + return Array.isArray(expectedNodeKinds) ? expectedNodeKinds.includes(node.kind) : node.kind === expectedNodeKinds; } export function nextId(state: ParseContext.State): number { diff --git a/src/powerquery-parser/parser/disambiguation/disambiguationUtils.ts b/src/powerquery-parser/parser/disambiguation/disambiguationUtils.ts index 51acc1d2..5f4b0634 100644 --- a/src/powerquery-parser/parser/disambiguation/disambiguationUtils.ts +++ b/src/powerquery-parser/parser/disambiguation/disambiguationUtils.ts @@ -234,7 +234,10 @@ export async function disambiguateParenthesis( try { // eslint-disable-next-line no-await-in-loop await parser.readNullablePrimitiveType(state, parser, trace.id); - } catch { + } catch (error: unknown) { + Assert.isInstanceofError(error); + CommonError.throwIfCancellationError(error); + // eslint-disable-next-line no-await-in-loop await parser.restoreCheckpoint(state, checkpoint); diff --git a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts index 8ef64687..a54e4797 100644 --- a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts +++ b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { ArrayUtils, Assert } from "../../common"; import { Ast, Constant, IdentifierUtils } from "../../language"; import { NodeIdMap, NodeIdMapUtils, TXorNode, XorNodeKind, XorNodeUtils } from "."; -import { ArrayUtils, Assert } from "../../common"; import { parameterIdentifier } from "./nodeIdMapUtils"; import { XorNode } from "./xorNode"; @@ -124,7 +124,7 @@ export function nthSiblingXor( parentXorNode.node.id, ); - if (childIds.length >= attributeIndex) { + if (attributeIndex >= childIds.length) { return undefined; } diff --git a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/leafSelectors.ts b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/leafSelectors.ts index 8fe88b63..ae851ce3 100644 --- a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/leafSelectors.ts +++ b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/leafSelectors.ts @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { ArrayUtils, Assert } from "../../../common"; import { AstNodeById, Collection } from "../nodeIdMap"; import { NodeIdMap, XorNodeUtils } from ".."; -import { ArrayUtils, Assert } from "../../../common"; import { Ast } from "../../../language"; import { TXorNode } from "../xorNode"; import { xor } from "./commonSelectors"; diff --git a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/nodeIdMapUtils.ts b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/nodeIdMapUtils.ts index 8c1fe021..c7cfdd1c 100644 --- a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/nodeIdMapUtils.ts +++ b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/nodeIdMapUtils.ts @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { ArrayUtils, Assert } from "../../../common"; import { Ast, Token } from "../../../language"; import { Collection, CollectionValidation, IdsByNodeKind, NodeSummary } from "../nodeIdMap"; import { TXorNode, XorNodeKind, XorNodeTokenRange } from "../xorNode"; -import { ArrayUtils, Assert } from "../../../common"; import { ParseContext } from "../../context"; import { rightMostLeaf } from "./leafSelectors"; diff --git a/src/powerquery-parser/parser/parseState/parseStateUtils.ts b/src/powerquery-parser/parser/parseState/parseStateUtils.ts index 435fbf73..196edecc 100644 --- a/src/powerquery-parser/parser/parseState/parseStateUtils.ts +++ b/src/powerquery-parser/parser/parseState/parseStateUtils.ts @@ -37,7 +37,7 @@ export function newState(lexerSnapshot: LexerSnapshot, overrides?: Partial { + const contextState: ParseContext.State = ParseContextUtils.copyState(state.contextState); + return { ...state, - contextState: ParseContextUtils.copyState(state.contextState), + contextState, + currentContextNode: + state.currentContextNode !== undefined + ? MapUtils.assertGet(contextState.nodeIdMapCollection.contextNodeById, state.currentContextNode.id) + : undefined, }; } diff --git a/src/powerquery-parser/parser/parser/parserUtils.ts b/src/powerquery-parser/parser/parser/parserUtils.ts index d581043d..5b5b9485 100644 --- a/src/powerquery-parser/parser/parser/parserUtils.ts +++ b/src/powerquery-parser/parser/parser/parserUtils.ts @@ -188,7 +188,7 @@ async function tryParseDocument(parseSettings: ParseSettings, lexerSnapshot: Lex } default: - Assert.isNever(parseSettings.parseBehavior); + throw Assert.isNever(parseSettings.parseBehavior); } } diff --git a/src/powerquery-parser/parser/parsers/combinatorialParserV2/combineOperatorsAndOperands.ts b/src/powerquery-parser/parser/parsers/combinatorialParserV2/combineOperatorsAndOperands.ts index 6fbd286c..3c494359 100644 --- a/src/powerquery-parser/parser/parsers/combinatorialParserV2/combineOperatorsAndOperands.ts +++ b/src/powerquery-parser/parser/parsers/combinatorialParserV2/combineOperatorsAndOperands.ts @@ -188,7 +188,7 @@ interface Validator { const ValidatorForAsExpression: Validator = { tag: "AsExpression", - validateLeftOperand: (node: Ast.TNode): node is Ast.TEqualityExpression => AstUtils.isTAsExpression(node), + validateLeftOperand: (node: Ast.TNode): node is Ast.TAsExpression => AstUtils.isTAsExpression(node), validateRightOperand: (node: Ast.TNode): node is Ast.TNullablePrimitiveType => AstUtils.isTNullablePrimitiveType(node), fallbackLeftOperand: (state: ParseState, parser: Parser, correlationId: number) => @@ -199,8 +199,8 @@ const ValidatorForAsExpression: Validator = { const ValidatorForEqualityExpressionAndBelow: Validator = { tag: "EqualityExpression", - validateLeftOperand: (node: Ast.TNode): node is Ast.TMetadataExpression => AstUtils.isTEqualityExpression(node), - validateRightOperand: (node: Ast.TNode): node is Ast.TMetadataExpression => AstUtils.isTEqualityExpression(node), + validateLeftOperand: (node: Ast.TNode): node is Ast.TEqualityExpression => AstUtils.isTEqualityExpression(node), + validateRightOperand: (node: Ast.TNode): node is Ast.TEqualityExpression => AstUtils.isTEqualityExpression(node), fallbackLeftOperand: (state: ParseState, parser: Parser, correlationId: number) => NaiveParseSteps.readMetadataExpression(state, parser, correlationId), fallbackRightOperand: (state: ParseState, parser: Parser, correlationId: number) => diff --git a/src/powerquery-parser/parser/parsers/naiveParseSteps.ts b/src/powerquery-parser/parser/parsers/naiveParseSteps.ts index a5c543c3..571aab2d 100644 --- a/src/powerquery-parser/parser/parsers/naiveParseSteps.ts +++ b/src/powerquery-parser/parser/parsers/naiveParseSteps.ts @@ -122,8 +122,13 @@ export async function readGeneralizedIdentifier( const lexerSnapshot: LexerSnapshot = state.lexerSnapshot; const tokens: ReadonlyArray = lexerSnapshot.tokens; - const contiguousIdentifierStartIndex: number = ArrayUtils.assertGet(tokens, tokenRangeStartIndex).positionStart.codeUnit; - const contiguousIdentifierEndIndex: number = ArrayUtils.assertGet(tokens, tokenRangeEndIndex - 1).positionEnd.codeUnit; + + const contiguousIdentifierStartIndex: number = ArrayUtils.assertGet(tokens, tokenRangeStartIndex).positionStart + .codeUnit; + + const contiguousIdentifierEndIndex: number = ArrayUtils.assertGet(tokens, tokenRangeEndIndex - 1).positionEnd + .codeUnit; + const literal: string = lexerSnapshot.text.slice(contiguousIdentifierStartIndex, contiguousIdentifierEndIndex); const literalKind: IdentifierUtils.IdentifierKind = IdentifierUtils.getIdentifierKind(literal, { diff --git a/src/test/libraryTest/common/cancellationToken.test.ts b/src/test/libraryTest/common/cancellationToken.test.ts index 7d508c1b..375fd530 100644 --- a/src/test/libraryTest/common/cancellationToken.test.ts +++ b/src/test/libraryTest/common/cancellationToken.test.ts @@ -3,8 +3,11 @@ import "mocha"; +import { expect } from "chai"; + import { CommonError, + CounterCancellationToken, DefaultSettings, Lexer, Result, @@ -47,6 +50,36 @@ function defaultSettingsWithExpiredCancellationToken(): Settings { } describe("CancellationToken", () => { + describe(`CounterCancellationToken`, () => { + it(`throwIfCancelled should consume exactly 1 count`, () => { + // With threshold of 3, we should be able to call throwIfCancelled 2 times + // without throwing (counts 1 and 2), then the 3rd should throw (count 3). + const token: CounterCancellationToken = new CounterCancellationToken(3); + + // First call: counter goes to 1, threshold is 3, no throw + token.throwIfCancelled(); + + // Second call: counter goes to 2, threshold is 3, no throw + token.throwIfCancelled(); + + // Third call: counter goes to 3, threshold is 3, should throw + expect(() => token.throwIfCancelled()).to.throw(CommonError.CancellationError); + }); + + it(`isCancelled should consume exactly 1 count`, () => { + const token: CounterCancellationToken = new CounterCancellationToken(3); + + // First call: counter goes to 1, not cancelled + expect(token.isCancelled()).to.be.false; + + // Second call: counter goes to 2, not cancelled + expect(token.isCancelled()).to.be.false; + + // Third call: counter goes to 3, cancelled + expect(token.isCancelled()).to.be.true; + }); + }); + describe(`lexer`, () => { it(`Lexer.tryLex`, () => { const triedLex: Lexer.TriedLex = Lexer.tryLex(defaultSettingsWithExpiredCancellationToken(), "foo"); diff --git a/src/test/libraryTest/common/stringUtils.test.ts b/src/test/libraryTest/common/stringUtils.test.ts index e09161cf..aa1f7944 100644 --- a/src/test/libraryTest/common/stringUtils.test.ts +++ b/src/test/libraryTest/common/stringUtils.test.ts @@ -91,6 +91,33 @@ describe("StringUtils", () => { expect(actual).to.deep.equal(expected); }); + + it(`"a""""b" - consecutive escaped quotes`, () => { + // Content represents: a""b (two escaped quotes then 'b') + // The "" escape at index 2-3 must not cause index 4 to be skipped + const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`"a""""b"`, 0); + + const expected: StringUtils.FoundQuotes = { + indexStart: 0, + indexEnd: 8, + quoteLength: 8, + }; + + expect(actual).to.deep.equal(expected); + }); + + it(`"x""y""z" - multiple escaped quotes`, () => { + // Content represents: x"y"z + const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`"x""y""z"`, 0); + + const expected: StringUtils.FoundQuotes = { + indexStart: 0, + indexEnd: 9, + quoteLength: 9, + }; + + expect(actual).to.deep.equal(expected); + }); }); describe(`normalizeNumber`, () => { diff --git a/src/test/libraryTest/language/astUtils.test.ts b/src/test/libraryTest/language/astUtils.test.ts index 3d75e8ce..c8c83cfb 100644 --- a/src/test/libraryTest/language/astUtils.test.ts +++ b/src/test/libraryTest/language/astUtils.test.ts @@ -5,8 +5,8 @@ import "mocha"; import { expect } from "chai"; import { Ast, AstUtils } from "../../../powerquery-parser/language"; +import { DefaultSettings, Task } from "../../../powerquery-parser"; import { AssertTestUtils } from "../../testUtils"; -import { DefaultSettings } from "../../../powerquery-parser"; import { ParseOk } from "../../../powerquery-parser/parser"; describe(`AstUtils`, () => { @@ -57,4 +57,46 @@ describe(`AstUtils`, () => { }); }); }); + + describe("Type predicates for validator operands", () => { + it("isTAsExpression should accept AsExpression nodes", async () => { + // Parse "1 as number as text" — the outer AsExpression's left child is itself an AsExpression. + // The validator's type predicate for left must accept TAsExpression (which includes AsExpression). + const parseTaskOk: Task.ParseTaskOk = await AssertTestUtils.assertGetLexParseOk( + DefaultSettings, + `1 as number as text`, + ); + + const root: Ast.TNode = parseTaskOk.ast; + expect(root.kind).to.equal(Ast.NodeKind.AsExpression); + + const asExpr: Ast.AsExpression = root as Ast.AsExpression; + expect(asExpr.left.kind).to.equal(Ast.NodeKind.AsExpression); + + expect(AstUtils.isTAsExpression(asExpr.left)).to.equal( + true, + "isTAsExpression should accept AsExpression nodes", + ); + + // Verify the left operand is NOT one of the narrower TEqualityExpression kinds + expect(asExpr.left.kind).to.not.be.oneOf( + [Ast.NodeKind.EqualityExpression, Ast.NodeKind.RelationalExpression, Ast.NodeKind.ArithmeticExpression], + "The left operand is an AsExpression, in TAsExpression but not TEqualityExpression", + ); + }); + + it("isTEqualityExpression should accept left operand of EqualityExpression", async () => { + const parseTaskOk: Task.ParseTaskOk = await AssertTestUtils.assertGetLexParseOk(DefaultSettings, `1 = 2`); + + const root: Ast.TNode = parseTaskOk.ast; + expect(root.kind).to.equal(Ast.NodeKind.EqualityExpression); + + const eqExpr: Ast.EqualityExpression = root as Ast.EqualityExpression; + + expect(AstUtils.isTEqualityExpression(eqExpr.left)).to.equal( + true, + "isTEqualityExpression should accept left operand of EqualityExpression", + ); + }); + }); }); diff --git a/src/test/libraryTest/language/typeUtils/typeCheck.test.ts b/src/test/libraryTest/language/typeUtils/typeCheck.test.ts index a0d0f04e..41418772 100644 --- a/src/test/libraryTest/language/typeUtils/typeCheck.test.ts +++ b/src/test/libraryTest/language/typeUtils/typeCheck.test.ts @@ -6,8 +6,8 @@ import { expect } from "chai"; import { CheckedDefinedList, CheckedInvocation } from "../../../../powerquery-parser/language/type/typeUtils"; import { Type, TypeUtils } from "../../../../powerquery-parser/language"; -import { Language } from "../../../.."; import { ArrayUtils } from "../../../../powerquery-parser/common"; +import { Language } from "../../../.."; import { NoOpTraceManagerInstance } from "../../../../powerquery-parser/common/trace"; import { OrderedMap } from "../../../../powerquery-parser"; @@ -360,7 +360,9 @@ describe(`TypeUtils.typeCheck`, () => { const expected: TypeUtils.CheckedInvocation = { valid: [], - invalid: new Map([[0, { actual: args[0], expected: ArrayUtils.assertGet(definedFunction.parameters, 0) }]]), + invalid: new Map([ + [0, { actual: args[0], expected: ArrayUtils.assertGet(definedFunction.parameters, 0) }], + ]), extraneous: [], missing: [], }; @@ -388,7 +390,9 @@ describe(`TypeUtils.typeCheck`, () => { const expected: TypeUtils.CheckedInvocation = { valid: [], - invalid: new Map([[0, { actual: args[0], expected: ArrayUtils.assertGet(definedFunction.parameters, 0) }]]), + invalid: new Map([ + [0, { actual: args[0], expected: ArrayUtils.assertGet(definedFunction.parameters, 0) }], + ]), extraneous: [], missing: [], }; diff --git a/src/test/libraryTest/language/typeUtils/typeUtils.test.ts b/src/test/libraryTest/language/typeUtils/typeUtils.test.ts index 2e21c37d..4c09d698 100644 --- a/src/test/libraryTest/language/typeUtils/typeUtils.test.ts +++ b/src/test/libraryTest/language/typeUtils/typeUtils.test.ts @@ -5,9 +5,9 @@ import "mocha"; import { expect } from "chai"; import { Type, TypeUtils } from "../../../../powerquery-parser/language"; +import { ArrayUtils } from "../../../../powerquery-parser/common"; import { NoOpTraceManagerInstance } from "../../../../powerquery-parser/common/trace"; import { OrderedMap } from "../../../../powerquery-parser"; -import { ArrayUtils } from "../../../../powerquery-parser/common"; interface AbridgedType { readonly kind: Type.TypeKind; diff --git a/src/test/libraryTest/lexer/lexIncremental.test.ts b/src/test/libraryTest/lexer/lexIncremental.test.ts index b0fd6b2e..13286806 100644 --- a/src/test/libraryTest/lexer/lexIncremental.test.ts +++ b/src/test/libraryTest/lexer/lexIncremental.test.ts @@ -399,4 +399,35 @@ describe(`Lexer.Incremental`, () => { }); }); }); + + describe(`retokenizeLines does not drop lines`, () => { + it(`changing block comment start preserves subsequent lines`, () => { + // Original: 4 lines — block comment on lines 0-1, then "beta" and "charlie" + // Line 0: /* → Default→Comment + // Line 1: */ → Comment→Default + // Line 2: beta → Default→Default + // Line 3: charlie → Default→Default + const originalText: string = `/*\n*/\nbeta\ncharlie`; + + const originalExpected: AbridgedTLexerLine = [ + [Lexer.LineKind.Touched, Lexer.LineMode.Default, Lexer.LineMode.Comment, `/*`], + [Lexer.LineKind.Touched, Lexer.LineMode.Comment, Lexer.LineMode.Default, `*/`], + [Lexer.LineKind.Touched, Lexer.LineMode.Default, Lexer.LineMode.Default, `beta`], + [Lexer.LineKind.Touched, Lexer.LineMode.Default, Lexer.LineMode.Default, `charlie`], + ]; + + // Change line 0 from "/*" to "alpha" — removes the comment context + // retokenizeLines retokenizes line 1 ("*/" in Default mode → Default→Default) + // then hits line 2 which already starts in Default → early exit + // lines.slice(lineNumber + 1) drops line 2 ("beta") + const state: Lexer.State = assertGetLexerUpdateLine(originalText, originalExpected, 0, `alpha`, [ + [Lexer.LineKind.Touched, Lexer.LineMode.Default, Lexer.LineMode.Default, `alpha`], + [Lexer.LineKind.Touched, Lexer.LineMode.Default, Lexer.LineMode.Default, `*/`], + [Lexer.LineKind.Touched, Lexer.LineMode.Default, Lexer.LineMode.Default, `beta`], + [Lexer.LineKind.Touched, Lexer.LineMode.Default, Lexer.LineMode.Default, `charlie`], + ]); + + expect(state.lines.length).to.equal(4, "retokenizeLines dropped a line"); + }); + }); }); diff --git a/src/test/libraryTest/lexer/lexMultilineTokens.test.ts b/src/test/libraryTest/lexer/lexMultilineTokens.test.ts index e3a76ed7..e6305ebc 100644 --- a/src/test/libraryTest/lexer/lexMultilineTokens.test.ts +++ b/src/test/libraryTest/lexer/lexMultilineTokens.test.ts @@ -129,6 +129,21 @@ describe(`Lexer`, () => { assertGetLineTokenMatch(text, expected, true); }); }); + + describe(`LineTokenKindAdditions enum values match member names`, () => { + it(`TextLiteralContent value should equal "TextLiteralContent"`, () => { + // TextLiteralContent = "TextContent" (missing "Literal" in value) + // All other members have values matching their names exactly + const expected: string = "TextLiteralContent"; + const actual: string = Language.Token.LineTokenKind.TextLiteralContent; + + if (actual !== expected) { + throw new Error( + `LineTokenKindAdditions.TextLiteralContent has value "${actual}" but should be "${expected}"`, + ); + } + }); + }); }); describe(`MultilineTokens Abridged LexerSnapshot`, () => { diff --git a/src/test/libraryTest/lexer/lexSimple.test.ts b/src/test/libraryTest/lexer/lexSimple.test.ts index 7f0697eb..9a357a74 100644 --- a/src/test/libraryTest/lexer/lexSimple.test.ts +++ b/src/test/libraryTest/lexer/lexSimple.test.ts @@ -4,8 +4,8 @@ import "mocha"; import { expect } from "chai"; +import { Language, Pattern, StringUtils } from "../../.."; import { assertGetSnapshotAbridgedTokens } from "../../testUtils/lexTestUtils"; -import { Language } from "../../.."; describe(`Lexer.Simple.TokenKinds`, () => { it(`HexLiteral`, () => { @@ -136,6 +136,26 @@ type assertGetSnapshotAbridgedTokens(text, expected, true); }); + it(`NumericLiteral - backslash in exponent is not valid`, () => { + // The old Pattern.Numeric regex used [\\+\\-] which included literal backslash. + // This meant "1e\\3" would incorrectly match as a valid numeric literal. + // After the fix, only + and - are valid exponent signs. + expect(StringUtils.isNumeric(`1e\\3`)).to.equal(false, `"1e\\3" should not be a valid numeric literal`); + expect(StringUtils.isNumeric(`1e+3`)).to.equal(true, `"1e+3" should be valid`); + expect(StringUtils.isNumeric(`1e-3`)).to.equal(true, `"1e-3" should be valid`); + expect(StringUtils.isNumeric(`1e3`)).to.equal(true, `"1e3" should be valid`); + }); + + it(`DotDot at end of input`, () => { + // chr3 was typed as `string` but text[positionStart + 2] can be undefined + // when ".." appears at end-of-input. Ensure it correctly lexes as DotDot. + const text: string = `..`; + + const expected: ReadonlyArray<[Language.Token.TokenKind, string]> = [[Language.Token.TokenKind.DotDot, `..`]]; + + assertGetSnapshotAbridgedTokens(text, expected, false); + }); + it(`operator-or-punctuation`, () => { const text: string = ` , @@ -299,3 +319,33 @@ describe(`Lexer.Simple.Whitespace`, () => { assertGetSnapshotAbridgedTokens(text, expected, true); }); }); + +describe(`Lexer.Simple.Whitespace Pattern`, () => { + it(`should not match colon followed by whitespace char as a single whitespace token`, () => { + // (:?[\u000b-\u000c\u2000-\u200a]) is a CAPTURING group with optional ':' + // This means ":" + VT (vertical tab) matches as "whitespace", consuming the colon. + // The correct syntax (?:[...]) is a non-capturing group that only matches the whitespace chars. + const regex: RegExp = Pattern.Whitespace; + regex.lastIndex = 0; + + const testStr: string = ":\u000B"; // colon + vertical tab + const match: RegExpExecArray | null = regex.exec(testStr); + + // With bug: matches ":\u000B" (length 2) — colon consumed as whitespace + // With fix: matches "\u000B" (length 1) — only the vertical tab + expect(match).to.not.equal(null, "should match the vertical tab"); + expect(match![0]).to.equal("\u000B", "should match only the whitespace char, not the preceding colon"); + }); + + it(`should match vertical tab (U+000B) as whitespace`, () => { + // U+000B is in the first alternative's character class + const regex: RegExp = Pattern.Whitespace; + regex.lastIndex = 0; + + const testStr: string = "\u000B"; + const match: RegExpExecArray | null = regex.exec(testStr); + + expect(match).to.not.equal(null, "Whitespace pattern should match vertical tab U+000B"); + expect(match![0]).to.equal("\u000B"); + }); +}); diff --git a/src/test/libraryTest/lexer/lexerSnapshotCache.test.ts b/src/test/libraryTest/lexer/lexerSnapshotCache.test.ts index 0fe9ef33..b0f1c8f6 100644 --- a/src/test/libraryTest/lexer/lexerSnapshotCache.test.ts +++ b/src/test/libraryTest/lexer/lexerSnapshotCache.test.ts @@ -105,7 +105,9 @@ describe("LexerSnapshot.graphemePositionStartFrom cache", () => { // Column numbers should be increasing for (let i: number = 1; i < positions.length; i += 1) { - expect(ArrayUtils.assertGet(positions, i).columnNumber).to.be.greaterThan(ArrayUtils.assertGet(positions, i - 1).columnNumber); + expect(ArrayUtils.assertGet(positions, i).columnNumber).to.be.greaterThan( + ArrayUtils.assertGet(positions, i - 1).columnNumber, + ); } }); @@ -123,4 +125,38 @@ describe("LexerSnapshot.graphemePositionStartFrom cache", () => { expect(ArrayUtils.assertGet(positions, 3).lineNumber).to.equal(2); }); }); + + describe("codeUnit correctness", () => { + it("codeUnit should reflect the token start position, not end", () => { + const snapshot: Lexer.LexerSnapshot = assertGetLexerSnapshot("let x = 1"); + + for (const token of snapshot.tokens) { + const position: StringUtils.GraphemePosition = snapshot.graphemePositionStartFrom(token); + + expect(position.codeUnit).to.equal( + token.positionStart.codeUnit, + `graphemePositionStartFrom("${token.data}") returned codeUnit ${position.codeUnit} ` + + `but token.positionStart.codeUnit is ${token.positionStart.codeUnit}`, + ); + } + }); + + it("static graphemePositionStartFrom returns start codeUnit", () => { + const snapshot: Lexer.LexerSnapshot = assertGetLexerSnapshot("let x = 1"); + + for (const token of snapshot.tokens) { + const position: StringUtils.GraphemePosition = Lexer.LexerSnapshot.graphemePositionStartFrom( + snapshot.text, + snapshot.lineTerminators, + token, + ); + + expect(position.codeUnit).to.equal( + token.positionStart.codeUnit, + `static graphemePositionStartFrom("${token.data}") returned codeUnit ${position.codeUnit} ` + + `but token.positionStart.codeUnit is ${token.positionStart.codeUnit}`, + ); + } + }); + }); }); diff --git a/src/test/libraryTest/lexer/retokenizeLineNumbers.test.ts b/src/test/libraryTest/lexer/retokenizeLineNumbers.test.ts index 2b8432f3..3536583d 100644 --- a/src/test/libraryTest/lexer/retokenizeLineNumbers.test.ts +++ b/src/test/libraryTest/lexer/retokenizeLineNumbers.test.ts @@ -31,9 +31,20 @@ describe("Lexer.retokenizeLines line numbers", () => { expect(updated.lines.length).to.equal(3, "expected 3 lines after update"); // All lines should now be in Default mode since line 0 is a complete string - expect(ArrayUtils.assertGet(updated.lines, 0).lineModeEnd).to.equal(Lexer.LineMode.Default, "line 0 should end in Default mode"); - expect(ArrayUtils.assertGet(updated.lines, 1).lineModeStart).to.equal(Lexer.LineMode.Default, "line 1 should start in Default mode"); - expect(ArrayUtils.assertGet(updated.lines, 2).lineModeStart).to.equal(Lexer.LineMode.Default, "line 2 should start in Default mode"); + expect(ArrayUtils.assertGet(updated.lines, 0).lineModeEnd).to.equal( + Lexer.LineMode.Default, + "line 0 should end in Default mode", + ); + + expect(ArrayUtils.assertGet(updated.lines, 1).lineModeStart).to.equal( + Lexer.LineMode.Default, + "line 1 should start in Default mode", + ); + + expect(ArrayUtils.assertGet(updated.lines, 2).lineModeStart).to.equal( + Lexer.LineMode.Default, + "line 2 should start in Default mode", + ); // Now snapshot to get token positions and verify line numbers are correct. const triedSnapshot: Lexer.TriedLexerSnapshot = Lexer.trySnapshot(updated); diff --git a/src/test/libraryTest/parser/identifierContextKind.test.ts b/src/test/libraryTest/parser/identifierContextKind.test.ts index 597380d5..e6066a57 100644 --- a/src/test/libraryTest/parser/identifierContextKind.test.ts +++ b/src/test/libraryTest/parser/identifierContextKind.test.ts @@ -4,8 +4,8 @@ import "mocha"; import { expect } from "chai"; -import { ArrayUtils } from "../../../powerquery-parser/common"; import { NodeIdMap, NodeIdMapUtils, ParseOk } from "../../../powerquery-parser/parser"; +import { ArrayUtils } from "../../../powerquery-parser/common"; import { AssertTestUtils } from "../../testUtils"; import { Ast } from "../../../powerquery-parser/language"; import { DefaultSettings } from "../../.."; diff --git a/src/test/libraryTest/parser/parseBehavior.test.ts b/src/test/libraryTest/parser/parseBehavior.test.ts index d56ce742..fc22bac1 100644 --- a/src/test/libraryTest/parser/parseBehavior.test.ts +++ b/src/test/libraryTest/parser/parseBehavior.test.ts @@ -5,7 +5,8 @@ import "mocha"; import { expect } from "chai"; import * as ParserTestUtils from "./parserTestUtils"; -import { Assert, DefaultSettings, Task, TaskUtils } from "../../../powerquery-parser"; +import { Assert, DefaultSettings, Language, Task, TaskUtils } from "../../../powerquery-parser"; +import { AssertTestUtils } from "../../testUtils"; import { NodeKind } from "../../../powerquery-parser/language/ast/ast"; import { ParseBehavior } from "../../../powerquery-parser/parser/parseBehavior"; import { ParseError } from "../../../powerquery-parser/parser"; @@ -45,7 +46,7 @@ describe("ParseBehavior", () => { break; default: - Assert.isNever(params.expectedStatus); + throw Assert.isNever(params.expectedStatus); } return result; @@ -130,4 +131,77 @@ describe("ParseBehavior", () => { expectedStatus: "ParseStageOk", }); }); + + it(`invalid parseBehavior throws InvariantError`, async () => { + // Missing `throw` before Assert.isNever means TypeScript + // doesn't guarantee the default branch is recognized as unreachable. + // Verify the default branch actually throws for invalid enum values. + const invalidBehavior: ParseBehavior = "InvalidBehavior" as unknown as ParseBehavior; + + let threw: boolean = false; + + try { + await TaskUtils.tryLexParse( + { + ...DefaultSettings, + parseBehavior: invalidBehavior, + }, + "1", + ); + } catch (error: unknown) { + threw = true; + expect((error as Error).message).to.contain("Should never be reached"); + } + + expect(threw).to.equal(true, "An invalid parseBehavior should throw an InvariantError"); + }); +}); + +type ParseOk = Awaited>; + +describe("Type directives - regression", () => { + // BUG: type directive scoping — directive on x's line incorrectly attaches to y + xit("should not attach directive from previous variable to next variable", async () => { + const parseOk: ParseOk = await AssertTestUtils.assertGetLexParseOk( + { + ...DefaultSettings, + isTypeDirectiveAllowed: true, + }, + `let + x = 1, /// @type number + y = 2 +in + y`, + ); + + const letExpression: Language.Ast.LetExpression = parseOk.ast as Language.Ast.LetExpression; + const yVariable: Language.Ast.IdentifierPairedExpression = letExpression.variableList.elements[1]!.node; + + expect(yVariable.precedingDirectives).to.equal(undefined, "y should not have directives from x's line"); + }); + + it("should handle multiple consecutive directives in correct order", async () => { + const parseOk: ParseOk = await AssertTestUtils.assertGetLexParseOk( + { + ...DefaultSettings, + isTypeDirectiveAllowed: true, + }, + `let + /// @type text + /// @type number + value = [] +in + value`, + ); + + const letExpression: Language.Ast.LetExpression = parseOk.ast as Language.Ast.LetExpression; + const variable: Language.Ast.IdentifierPairedExpression = letExpression.variableList.elements[0]!.node; + + expect(variable.precedingDirectives).to.not.equal(undefined); + expect(variable.precedingDirectives?.length).to.equal(2); + + expect( + variable.precedingDirectives?.map((directive: Language.Comment.TDirective) => directive.value), + ).to.deep.equal(["text", "number"]); + }); }); diff --git a/src/test/libraryTest/parser/parseNodeIdMapUtils.test.ts b/src/test/libraryTest/parser/parseNodeIdMapUtils.test.ts index 7254234e..499a1b76 100644 --- a/src/test/libraryTest/parser/parseNodeIdMapUtils.test.ts +++ b/src/test/libraryTest/parser/parseNodeIdMapUtils.test.ts @@ -4,7 +4,7 @@ import "mocha"; import { expect } from "chai"; -import { ArrayUtils, Assert, Language, MapUtils, Parser } from "../../../powerquery-parser"; +import { ArrayUtils, Assert, Language, MapUtils, Parser, TaskUtils } from "../../../powerquery-parser"; import { DefaultSettings, Task } from "../../.."; import { FieldSpecificationKeyValuePair, @@ -46,11 +46,19 @@ describe("nodeIdMapIterator", () => { expect(fieldSpecificationKeyValuePairs.length).to.equal(2); - const firstKeyValuePair: FieldSpecificationKeyValuePair = ArrayUtils.assertGet(fieldSpecificationKeyValuePairs, 0); + const firstKeyValuePair: FieldSpecificationKeyValuePair = ArrayUtils.assertGet( + fieldSpecificationKeyValuePairs, + 0, + ); + expect(firstKeyValuePair.optional).to.equal(undefined); expect(firstKeyValuePair.normalizedKeyLiteral).to.equal("foo"); - const secondKeyValuePair: FieldSpecificationKeyValuePair = ArrayUtils.assertGet(fieldSpecificationKeyValuePairs, 1); + const secondKeyValuePair: FieldSpecificationKeyValuePair = ArrayUtils.assertGet( + fieldSpecificationKeyValuePairs, + 1, + ); + expect(Boolean(secondKeyValuePair.optional)).to.equal(true); expect(secondKeyValuePair.normalizedKeyLiteral).to.equal("bar"); }); @@ -235,11 +243,19 @@ describe("nodeIdMapIterator", () => { expect(fieldSpecificationKeyValuePairs.length).to.equal(2); - const firstKeyValuePair: FieldSpecificationKeyValuePair = ArrayUtils.assertGet(fieldSpecificationKeyValuePairs, 0); + const firstKeyValuePair: FieldSpecificationKeyValuePair = ArrayUtils.assertGet( + fieldSpecificationKeyValuePairs, + 0, + ); + expect(firstKeyValuePair.optional).to.equal(undefined); expect(firstKeyValuePair.normalizedKeyLiteral).to.equal("foo"); - const secondKeyValuePair: FieldSpecificationKeyValuePair = ArrayUtils.assertGet(fieldSpecificationKeyValuePairs, 1); + const secondKeyValuePair: FieldSpecificationKeyValuePair = ArrayUtils.assertGet( + fieldSpecificationKeyValuePairs, + 1, + ); + expect(Boolean(secondKeyValuePair.optional)).to.equal(true); expect(secondKeyValuePair.normalizedKeyLiteral).to.equal("bar"); }); @@ -335,4 +351,48 @@ describe(`nodeIdMapUtils`, () => { ); }); }); + + describe("copyState - currentContextNode isolation", () => { + it("mutating currentContextNode on copy should not affect original", async () => { + const triedLexParseTask: Task.TriedLexParseTask = await TaskUtils.tryLexParse(DefaultSettings, `let x =`); + + TaskUtils.assertIsParseStageParseError(triedLexParseTask); + const originalState: Parser.ParseState = triedLexParseTask.parseState; + + expect(originalState.currentContextNode).to.not.be.undefined; + const originalAttributeCounter: number = originalState.currentContextNode!.attributeCounter; + + const copiedState: Parser.ParseState = await Parser.ParseStateUtils.copyState(originalState); + + expect(copiedState.currentContextNode).to.not.be.undefined; + copiedState.currentContextNode!.attributeCounter += 1; + + expect(originalState.currentContextNode!.attributeCounter).to.equal( + originalAttributeCounter, + "Mutating currentContextNode on copied state should not affect the original state", + ); + }); + + it("copied currentContextNode should have same values but different reference", async () => { + const triedLexParseTask: Task.TriedLexParseTask = await TaskUtils.tryLexParse(DefaultSettings, `let x =`); + + TaskUtils.assertIsParseStageParseError(triedLexParseTask); + const originalState: Parser.ParseState = triedLexParseTask.parseState; + expect(originalState.currentContextNode).to.not.be.undefined; + + const copiedState: Parser.ParseState = await Parser.ParseStateUtils.copyState(originalState); + expect(copiedState.currentContextNode).to.not.be.undefined; + + expect(copiedState.currentContextNode!.id).to.equal(originalState.currentContextNode!.id); + expect(copiedState.currentContextNode!.kind).to.equal(originalState.currentContextNode!.kind); + + expect(copiedState.currentContextNode!.attributeCounter).to.equal( + originalState.currentContextNode!.attributeCounter, + ); + + expect(copiedState.currentContextNode!.isClosed).to.equal(originalState.currentContextNode!.isClosed); + + expect(copiedState.currentContextNode).to.not.equal(originalState.currentContextNode); + }); + }); }); diff --git a/src/test/libraryTest/parser/typeDirective.test.ts b/src/test/libraryTest/parser/typeDirective.test.ts index fef3c3a3..46064d3b 100644 --- a/src/test/libraryTest/parser/typeDirective.test.ts +++ b/src/test/libraryTest/parser/typeDirective.test.ts @@ -22,7 +22,11 @@ in ); const letExpression: Language.Ast.LetExpression = parseOk.ast as Language.Ast.LetExpression; - const variable: Language.Ast.IdentifierPairedExpression = ArrayUtils.assertGet(letExpression.variableList.elements, 0).node; + + const variable: Language.Ast.IdentifierPairedExpression = ArrayUtils.assertGet( + letExpression.variableList.elements, + 0, + ).node; expect(variable.precedingDirectives).to.equal(undefined); }); @@ -41,7 +45,11 @@ in ); const letExpression: Language.Ast.LetExpression = parseOk.ast as Language.Ast.LetExpression; - const variable: Language.Ast.IdentifierPairedExpression = ArrayUtils.assertGet(letExpression.variableList.elements, 0).node; + + const variable: Language.Ast.IdentifierPairedExpression = ArrayUtils.assertGet( + letExpression.variableList.elements, + 0, + ).node; expect(variable.precedingDirectives).to.not.equal(undefined); @@ -83,7 +91,11 @@ in ); const letExpression: Language.Ast.LetExpression = parseOk.ast as Language.Ast.LetExpression; - const variable: Language.Ast.IdentifierPairedExpression = ArrayUtils.assertGet(letExpression.variableList.elements, 0).node; + + const variable: Language.Ast.IdentifierPairedExpression = ArrayUtils.assertGet( + letExpression.variableList.elements, + 0, + ).node; expect( variable.precedingDirectives?.map((directive: Language.Comment.TDirective) => directive.value), @@ -105,7 +117,11 @@ in ); const letExpression: Language.Ast.LetExpression = parseOk.ast as Language.Ast.LetExpression; - const variable: Language.Ast.IdentifierPairedExpression = ArrayUtils.assertGet(letExpression.variableList.elements, 0).node; + + const variable: Language.Ast.IdentifierPairedExpression = ArrayUtils.assertGet( + letExpression.variableList.elements, + 0, + ).node; expect(variable.precedingDirectives).to.equal(undefined); }); diff --git a/src/test/libraryTest/tokenizer/tokenizerIncremental.test.ts b/src/test/libraryTest/tokenizer/tokenizerIncremental.test.ts index d7e6d946..527d7f57 100644 --- a/src/test/libraryTest/tokenizer/tokenizerIncremental.test.ts +++ b/src/test/libraryTest/tokenizer/tokenizerIncremental.test.ts @@ -5,8 +5,8 @@ import "mocha"; import { expect } from "chai"; import { DefaultSettings, Lexer, ResultUtils } from "../../.."; -import { ArrayUtils } from "../../../powerquery-parser/common"; import { ILineTokens, IState, IToken, Tokenizer } from "../../testUtils/tokenizerTestUtils"; +import { ArrayUtils } from "../../../powerquery-parser/common"; const tokenizer: Tokenizer = new Tokenizer("\n"); @@ -184,7 +184,7 @@ describe("Incremental updates", () => { const lineTokens: ReadonlyArray = ArrayUtils.assertGet(document.lineTokens, index); lineTokens.forEach((token: IToken) => { - expect(token.scopes).equals("TextContent", "expecting remaining tokens to be strings"); + expect(token.scopes).equals("TextLiteralContent", "expecting remaining tokens to be strings"); }); } }); @@ -192,7 +192,12 @@ describe("Incremental updates", () => { it("Re-parse with unterminated block comment", () => { const lineNumber: number = 3; const document: MockDocument = new MockDocument(ORIGINAL_QUERY); - const modified: string = ArrayUtils.assertGet(document.lines, lineNumber).replace(`rce),`, `rce), /* my open comment`); + + const modified: string = ArrayUtils.assertGet(document.lines, lineNumber).replace( + `rce),`, + `rce), /* my open comment`, + ); + const count: number = document.applyChangeAndTokenize(modified, lineNumber); expect(count).equals(document.lines.length - lineNumber, "remaining lines should have been tokenized"); diff --git a/src/test/testUtils/tokenizerTestUtils.ts b/src/test/testUtils/tokenizerTestUtils.ts index 4d133520..2e410715 100644 --- a/src/test/testUtils/tokenizerTestUtils.ts +++ b/src/test/testUtils/tokenizerTestUtils.ts @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { ArrayUtils } from "../../powerquery-parser/common"; import { DefaultLocale, Language, ResultUtils } from "../../powerquery-parser"; +import { ArrayUtils } from "../../powerquery-parser/common"; import { Lexer } from "../.."; export class Tokenizer implements TokensProvider { @@ -50,7 +50,9 @@ export class Tokenizer implements TokensProvider { const newLexerState: Lexer.State = triedLex.value; return { - tokens: ArrayUtils.assertGet(newLexerState.lines, newLexerState.lines.length - 1).tokens.map(Tokenizer.ITokenFrom), + tokens: ArrayUtils.assertGet(newLexerState.lines, newLexerState.lines.length - 1).tokens.map( + Tokenizer.ITokenFrom, + ), endState: new TokenizerState(newLexerState), }; } @@ -84,7 +86,11 @@ export class TokenizerState implements IState { // Compare last line state. const leftLastLine: Lexer.TLine = ArrayUtils.assertGet(this.lexerState.lines, this.lexerState.lines.length - 1); - const rightLastLine: Lexer.TLine = ArrayUtils.assertGet(rightLexerState.lines, rightLexerState.lines.length - 1); + + const rightLastLine: Lexer.TLine = ArrayUtils.assertGet( + rightLexerState.lines, + rightLexerState.lines.length - 1, + ); return leftLastLine.lineModeEnd === rightLastLine.lineModeEnd; }