From 23cdb66a9e6c3bf9376276dbaabc142dd7e7e2e9 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Sun, 16 Jan 2022 19:53:11 -0800 Subject: [PATCH 01/21] install moo --- package.json | 4 +++- yarn.lock | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 2b36ecb6f2..4a775e4b95 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,9 @@ "url": "https://github.com/inferrinizzard/prettier-sql/issues" }, "dependencies": { - "argparse": "^2.0.1" + "@types/moo": "^0.5.5", + "argparse": "^2.0.1", + "moo": "^0.5.1" }, "devDependencies": { "@babel/cli": "^7.10.4", diff --git a/yarn.lock b/yarn.lock index 35d3401d6e..859a698c5f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1458,6 +1458,11 @@ dependencies: "@types/node" "*" +"@types/moo@^0.5.5": + version "0.5.5" + resolved "https://registry.yarnpkg.com/@types/moo/-/moo-0.5.5.tgz#83220b7349c59fd4bb1bc14b1d4ea0041899dc15" + integrity sha512-eXQpwnkI4Ntw5uJg6i2PINdRFWLr55dqjuYQaLHNjvqTzF14QdNWbCbml9sza0byyXNA0hZlHtcdN+VNDcgVHA== + "@types/node@*": version "16.6.0" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.6.0.tgz#0d5685f85066f94e97f19e8a67fe003c5fadacc4" @@ -5039,6 +5044,11 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" +moo@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/moo/-/moo-0.5.1.tgz#7aae7f384b9b09f620b6abf6f74ebbcd1b65dbc4" + integrity sha512-I1mnb5xn4fO80BH9BLcF0yLypy2UKl+Cb01Fu0hJRkJjlCRtxZMWkTdAtDd5ZqCOxtCkhmRwyI57vWT+1iZ67w== + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" From db38b66fbd9d04e1aeb5f1fee4169d6cefffd801 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Wed, 19 Jan 2022 07:38:00 -0800 Subject: [PATCH 02/21] basic word, operator lexer --- src/lexer/tokenizer.ts | 133 +++++++++++++++++++++++++++++++++++++++++ test/moo.test.js | 52 ++++++++++++++++ test/test.sql | 18 ++++++ 3 files changed, 203 insertions(+) create mode 100644 src/lexer/tokenizer.ts create mode 100644 test/moo.test.js create mode 100644 test/test.sql diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts new file mode 100644 index 0000000000..6700f2e5c0 --- /dev/null +++ b/src/lexer/tokenizer.ts @@ -0,0 +1,133 @@ +import * as moo from 'moo'; + +import * as regexFactory from '../core/regexFactory'; +import { escapeRegExp } from '../utils'; +import { Token, TokenType } from '../core/token'; // convert to partial type import in TS 4.5 + +const NULL_REGEX = /(?!)/; // zero-width negative lookahead, matches nothing + +interface TokenizerOptions { + reservedKeywords: string[]; + reservedCommands: string[]; + reservedLogicalOperators: string[]; + reservedDependentClauses: string[]; + reservedBinaryCommands: string[]; + stringTypes: regexFactory.StringPatternType[]; + blockStart: string[]; + blockEnd: string[]; + indexedPlaceholderTypes?: string[]; + namedPlaceholderTypes: string[]; + lineCommentTypes: string[]; + specialWordChars?: { prefix?: string; any?: string; suffix?: string }; + operators?: string[]; +} + +export default class Tokenizer { + WHITESPACE_REGEX: RegExp; + REGEX_MAP: { [tokenType in TokenType]: RegExp }; + + INDEXED_PLACEHOLDER_REGEX?: RegExp; + IDENT_NAMED_PLACEHOLDER_REGEX?: RegExp; + STRING_NAMED_PLACEHOLDER_REGEX?: RegExp; + + /** + * @param {TokenizerOptions} cfg + * @param {String[]} cfg.reservedKeywords: Reserved words in SQL + * @param {String[]} cfg.reservedDependentClauses: Words that following a specific Statement and must have data attached + * @param {String[]} cfg.reservedLogicalOperators: Words that are set to newline + * @param {String[]} cfg.reservedCommands: Words that are set to new line separately + * @param {String[]} cfg.reservedBinaryCommands: Words that are top level but have no indentation + * @param {String[]} cfg.stringTypes: String types to enable: "", '', ``, [], N'' + * @param {String[]} cfg.blockStart: Opening parentheses to enable, like (, [ + * @param {String[]} cfg.blockEnd: Closing parentheses to enable, like ), ] + * @param {String[]} cfg.indexedPlaceholderTypes: Prefixes for indexed placeholders, like ? + * @param {String[]} cfg.namedPlaceholderTypes: Prefixes for named placeholders, like @ and : + * @param {String[]} cfg.lineCommentTypes: Line comments to enable, like # and -- + * @param {String[]} cfg.specialWordChars: Special chars that can be found inside of words, like @ and # + * @param {String[]} cfg.operators: Additional operators to recognize + */ + constructor(cfg: TokenizerOptions) { + this.WHITESPACE_REGEX = /^(\s+)/u; + + const specialWordCharsAll = Object.values(cfg.specialWordChars ?? {}).join(''); + this.REGEX_MAP = { + [TokenType.WORD]: regexFactory.createWordRegex(cfg.specialWordChars), + [TokenType.STRING]: regexFactory.createStringRegex(cfg.stringTypes), + [TokenType.RESERVED_KEYWORD]: regexFactory.createReservedWordRegex( + cfg.reservedKeywords, + specialWordCharsAll + ), + [TokenType.RESERVED_DEPENDENT_CLAUSE]: regexFactory.createReservedWordRegex( + cfg.reservedDependentClauses ?? [], + specialWordCharsAll + ), + [TokenType.RESERVED_LOGICAL_OPERATOR]: regexFactory.createReservedWordRegex( + cfg.reservedLogicalOperators, + specialWordCharsAll + ), + [TokenType.RESERVED_COMMAND]: regexFactory.createReservedWordRegex( + cfg.reservedCommands, + specialWordCharsAll + ), + [TokenType.RESERVED_BINARY_COMMAND]: regexFactory.createReservedWordRegex( + cfg.reservedBinaryCommands, + specialWordCharsAll + ), + [TokenType.OPERATOR]: regexFactory.createOperatorRegex('+-/*%&|^><=.,;[]{}`:$', [ + '<>', + '<=', + '>=', + '!=', + ...(cfg.operators ?? []), + ]), + [TokenType.BLOCK_START]: regexFactory.createParenRegex(cfg.blockStart), + [TokenType.BLOCK_END]: regexFactory.createParenRegex(cfg.blockEnd), + [TokenType.LINE_COMMENT]: regexFactory.createLineCommentRegex(cfg.lineCommentTypes), + [TokenType.BLOCK_COMMENT]: /^(\/\*[^]*?(?:\*\/|$))/u, + [TokenType.NUMBER]: + /^((-\s*)?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b/u, + [TokenType.PLACEHOLDER]: NULL_REGEX, // matches nothing + }; + + this.INDEXED_PLACEHOLDER_REGEX = regexFactory.createPlaceholderRegex( + cfg.indexedPlaceholderTypes ?? [], + '[0-9]*' + ); + this.IDENT_NAMED_PLACEHOLDER_REGEX = regexFactory.createPlaceholderRegex( + cfg.namedPlaceholderTypes, + '[a-zA-Z0-9._$]+' + ); + this.STRING_NAMED_PLACEHOLDER_REGEX = regexFactory.createPlaceholderRegex( + cfg.namedPlaceholderTypes, + regexFactory.createStringPattern(cfg.stringTypes) + ); + } + + tokenize(input: string) { + let lexerOptions: { [key: string]: moo.Rule | RegExp } = { + WS: /[ \t]+/, + NL: { match: /\n/, lineBreaks: true }, + [TokenType.WORD]: new RegExp( + /(?:\p{Alphabetic}|\p{Mark}|\p{Decimal_Number}|\p{Connector_Punctuation}|\p{Join_Control})+/u, + 'u' + ), + [TokenType.OPERATOR]: new RegExp('[+-/*%&|^><=.,;\\[\\]\\}\\{`:$]', 'u'), + [TokenType.BLOCK_START]: /[(]/, + [TokenType.BLOCK_END]: /[)]/, + }; + lexerOptions = Object.entries(lexerOptions).reduce( + (acc, [name, regex]) => ({ + ...acc, + [name]: + regex instanceof RegExp + ? new RegExp(regex, 'u') + : { ...regex, match: new RegExp(regex.match as string | RegExp, 'u') }, + }), + {} as { [key: string]: moo.Rule | RegExp } + ); + + const lexer = moo.compile(lexerOptions); + lexer.reset(input); + return Array.from(lexer); + } +} diff --git a/test/moo.test.js b/test/moo.test.js new file mode 100644 index 0000000000..40175e2ffc --- /dev/null +++ b/test/moo.test.js @@ -0,0 +1,52 @@ +import StandardSqlFormatter from '../src/languages/standardsql.formatter'; +import Tokenizer from '../src/core/Tokenizer'; +import MooTokenizer from '../src/lexer/tokenizer'; + +const testSql = ` +select + alpha + 1, + beta b, + gamma AS g, + CASE WHEN iota THEN i END +FROM ( + SELECT + MAX(epsilon) + FROM zeta + GROUP BY eta, iota HAVING chi, psi, phi + UNION + SELECT MIN(rho) + FROM omega +) +JOIN sigma ON tau = theta +WHERE kappa AND lambda OR mu +; +SELECT upsilon AS y, omicron AS o FROM xi; +`; + +const options = { + reservedCommands: StandardSqlFormatter.reservedCommands, + reservedBinaryCommands: StandardSqlFormatter.reservedBinaryCommands, + reservedDependentClauses: StandardSqlFormatter.reservedDependentClauses, + reservedLogicalOperators: StandardSqlFormatter.reservedLogicalOperators, + reservedKeywords: StandardSqlFormatter.reservedKeywords, + stringTypes: StandardSqlFormatter.stringTypes, + blockStart: StandardSqlFormatter.blockStart, + blockEnd: StandardSqlFormatter.blockEnd, + indexedPlaceholderTypes: StandardSqlFormatter.indexedPlaceholderTypes, + namedPlaceholderTypes: StandardSqlFormatter.namedPlaceholderTypes, + lineCommentTypes: StandardSqlFormatter.lineCommentTypes, +}; + +const tokenizer = new Tokenizer(options); +const stream = tokenizer.tokenize(testSql); + +const mooTokenizer = new MooTokenizer(options); +const mooStream = mooTokenizer.tokenize(testSql); + +const filtered = mooStream.filter(token => token.type !== 'WS' && token.type !== 'NL'); +console.log(stream.length); +console.log(filtered.length); + +stream.forEach((t, i) => { + console.log(t.value, filtered[i].value); +}); diff --git a/test/test.sql b/test/test.sql new file mode 100644 index 0000000000..c00bc117b2 --- /dev/null +++ b/test/test.sql @@ -0,0 +1,18 @@ +select + alpha + 1, + beta b, + gamma AS g, + CASE WHEN iota THEN i END +FROM ( + SELECT + MAX(epsilon) + FROM zeta + GROUP BY eta, iota HAVING chi, psi, phi + UNION + SELECT MIN(rho) + FROM omega +) +JOIN sigma ON tau = theta +WHERE kappa AND lambda OR mu +; +SELECT upsilon AS y, omicron AS o FROM xi; From bc88e9964feec3f150797b4ba6513801cd51cef9 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Wed, 19 Jan 2022 08:46:03 -0800 Subject: [PATCH 03/21] add babel plugin to inline import sql --- .babelrc | 6 +++++- package.json | 1 + test/moo.test.js | 21 +-------------------- test/test.sql | 12 ++++++------ yarn.lock | 26 ++++++++++++++++++++++++++ 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/.babelrc b/.babelrc index d0cf44740d..a49d1e2e9e 100644 --- a/.babelrc +++ b/.babelrc @@ -1,4 +1,8 @@ { "presets": ["@babel/preset-env", "@babel/preset-typescript"], - "plugins": ["@babel/plugin-proposal-class-properties", "add-module-exports"] + "plugins": [ + "@babel/plugin-proposal-class-properties", + "add-module-exports", + ["babel-plugin-inline-import", { "extensions": [".sql"] }] + ] } diff --git a/package.json b/package.json index 4a775e4b95..34764d6a45 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "babel-jest": "^26.1.0", "babel-loader": "^8.1.0", "babel-plugin-add-module-exports": "^1.0.2", + "babel-plugin-inline-import": "^3.0.0", "dedent-js": "^1.0.1", "eslint": "^7.4.0", "eslint-config-airbnb-base": "^14.2.0", diff --git a/test/moo.test.js b/test/moo.test.js index 40175e2ffc..860f37b66a 100644 --- a/test/moo.test.js +++ b/test/moo.test.js @@ -2,26 +2,7 @@ import StandardSqlFormatter from '../src/languages/standardsql.formatter'; import Tokenizer from '../src/core/Tokenizer'; import MooTokenizer from '../src/lexer/tokenizer'; -const testSql = ` -select - alpha + 1, - beta b, - gamma AS g, - CASE WHEN iota THEN i END -FROM ( - SELECT - MAX(epsilon) - FROM zeta - GROUP BY eta, iota HAVING chi, psi, phi - UNION - SELECT MIN(rho) - FROM omega -) -JOIN sigma ON tau = theta -WHERE kappa AND lambda OR mu -; -SELECT upsilon AS y, omicron AS o FROM xi; -`; +import testSql from './test.sql'; const options = { reservedCommands: StandardSqlFormatter.reservedCommands, diff --git a/test/test.sql b/test/test.sql index c00bc117b2..26bc949767 100644 --- a/test/test.sql +++ b/test/test.sql @@ -1,18 +1,18 @@ select alpha + 1, - beta b, - gamma AS g, + beta[0] b, + gamma['radiation'] AS g, CASE WHEN iota THEN i END FROM ( SELECT MAX(epsilon) - FROM zeta - GROUP BY eta, iota HAVING chi, psi, phi + FROM europe.greek.zeta + GROUP BY $eta, iota HAVING chi, psi, phi UNION SELECT MIN(rho) - FROM omega + FROM greek.omega ) -JOIN sigma ON tau = theta +JOIN `greek_dev.sigma` ON tau = theta WHERE kappa AND lambda OR mu ; SELECT upsilon AS y, omicron AS o FROM xi; diff --git a/yarn.lock b/yarn.lock index 859a698c5f..585cfe81a6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2029,6 +2029,13 @@ babel-plugin-dynamic-import-node@^2.3.3: dependencies: object.assign "^4.1.0" +babel-plugin-inline-import@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/babel-plugin-inline-import/-/babel-plugin-inline-import-3.0.0.tgz#220eb2a52f8e779d8fb89447f950275e1e3f5981" + integrity sha512-thnykl4FMb8QjMjVCuZoUmAM7r2mnTn5qJwrryCvDv6rugbJlTHZMctdjDtEgD0WBAXJOLJSGXN3loooEwx7UQ== + dependencies: + require-resolve "0.0.2" + babel-plugin-istanbul@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/babel-plugin-istanbul/-/babel-plugin-istanbul-6.0.0.tgz#e159ccdc9af95e0b570c75b4573b7c34d671d765" @@ -5471,6 +5478,11 @@ path-exists@^4.0.0: resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== +path-extra@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/path-extra/-/path-extra-1.0.3.tgz#7c112189a6e50d595790e7ad2037e44e410c1166" + integrity sha1-fBEhiablDVlXkOetIDfkTkEMEWY= + path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" @@ -5932,6 +5944,13 @@ require-main-filename@^2.0.0: resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== +require-resolve@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/require-resolve/-/require-resolve-0.0.2.tgz#bab410ab1aee2f3f55b79317451dd3428764e6f3" + integrity sha1-urQQqxruLz9Vt5MXRR3TQodk5vM= + dependencies: + x-path "^0.0.2" + resolve-alpn@^1.0.0: version "1.2.1" resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9" @@ -7153,6 +7172,13 @@ ws@^7.4.6: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74" integrity sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg== +x-path@^0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/x-path/-/x-path-0.0.2.tgz#294d076bb97a7706cc070bbb2a6fd8c54df67b12" + integrity sha1-KU0Ha7l6dwbMBwu7Km/YxU32exI= + dependencies: + path-extra "^1.0.2" + xdg-basedir@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" From a9dd035e5013f66e7e8416734f5eb497cb42fd88 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Wed, 19 Jan 2022 20:16:40 -0800 Subject: [PATCH 04/21] update regex functions for word, string to moo --- src/core/Tokenizer.ts | 5 ++-- src/core/mooRegexFactory.ts | 59 +++++++++++++++++++++++++++++++++++++ src/core/token.ts | 5 ++++ src/lexer/tokenizer.ts | 46 +++++++++++++++-------------- 4 files changed, 91 insertions(+), 24 deletions(-) create mode 100644 src/core/mooRegexFactory.ts diff --git a/src/core/Tokenizer.ts b/src/core/Tokenizer.ts index 729899ff41..62f10b6a20 100644 --- a/src/core/Tokenizer.ts +++ b/src/core/Tokenizer.ts @@ -22,7 +22,7 @@ interface TokenizerOptions { export default class Tokenizer { WHITESPACE_REGEX: RegExp; - REGEX_MAP: { [tokenType in TokenType]: RegExp }; + REGEX_MAP: Partial<{ [tokenType in TokenType]: RegExp }>; INDEXED_PLACEHOLDER_REGEX?: RegExp; IDENT_NAMED_PLACEHOLDER_REGEX?: RegExp; @@ -139,10 +139,11 @@ export default class Tokenizer { } matchToken = (tokenType: TokenType) => (input: string) => + this.REGEX_MAP[tokenType] && this.getTokenOnFirstMatch({ input, type: tokenType, - regex: this.REGEX_MAP[tokenType], + regex: this.REGEX_MAP[tokenType]!, }); getNextToken(input: string, previousToken?: Token) { diff --git a/src/core/mooRegexFactory.ts b/src/core/mooRegexFactory.ts new file mode 100644 index 0000000000..a815f9397e --- /dev/null +++ b/src/core/mooRegexFactory.ts @@ -0,0 +1,59 @@ +import { escapeRegExp } from '../utils'; + +// This enables the following string patterns: +// 1. backtick quoted string using `` to escape +// 2. square bracket quoted string (SQL Server) using ]] to escape +// 3. double quoted string using "" or \" to escape, with optional prefix for format-specific strings +// 4. single quoted string using '' or \' to escape, with optional prefix for format-specific strings +// 8. PostgreSQL dollar-quoted strings (does not check for matching tags due to moo not allowing capturing groups) + +const stringPrefixList = ['[Nn]', '_utf8', 'U&', 'x']; +const createStringPattern = (stringPrefixes: string) => ({ + '``': '(?:`[^`]*(?:$|`))+', + '{}': '(?:\\{[^\\}]*(?:$|\\}))+', + '[]': '(?:\\[[^\\]]*(?:$|\\]))(\\][^\\]]*(?:$|\\]))*', + '""': `(?:${stringPrefixes}"[^"\\\\]*(?:\\\\.[^"\\\\]*)*(?:"|$))+`, + "''": `(?:${stringPrefixes}'[^'\\\\]*(?:\\\\.[^'\\\\]*)*(?:'|$))+`, + // '$$': '(?\\$\\w*\\$)[\\s\\S]*?(?:\\k|$)', // does not work with moo + '$$': '(?:\\$\\w*\\$)[\\s\\S]*?(?:\\$\\w*\\$)', +}); +export type StringPatternType = keyof ReturnType; +export type StringPatternPrefix = typeof stringPrefixList[number]; + +export const stringRegex = ({ + stringTypes, + stringPrefixes, +}: { + stringTypes: StringPatternType[]; + stringPrefixes?: StringPatternPrefix[]; +}) => { + const stringPrefix = stringPrefixes?.length ? `(?:${stringPrefixes.join('|')})?` : ''; + const stringPatternMap = createStringPattern(stringPrefix); + const stringPattern = stringTypes.map(stringType => stringPatternMap[stringType]).join('|'); + return new RegExp(`^${stringPattern}`, 'u'); +}; + +export const wordRegex = ( + specialChars: { any?: string; suffix?: string; prefix?: string } = {} +) => { + // lookbehind for specialChars that only appear at start + const prefixLookBehind = specialChars.prefix?.length + ? `[${escapeRegExp(specialChars.prefix)}]*` + : ''; + // lookahead for specialChars that only appear at end + const suffixLookAhead = specialChars.suffix?.length + ? `[${escapeRegExp(specialChars.suffix)}]*` + : ''; + + // unicode character categories + specialChars + const wordChar = [ + '\\p{Alphabetic}', + '\\p{Mark}', + '\\p{Decimal_Number}', + '\\p{Connector_Punctuation}', + '\\p{Join_Control}', + ...(specialChars.any?.length ? [`[${escapeRegExp(specialChars.any)}]`] : []), + ].join('|'); + + return new RegExp(`${prefixLookBehind}(?:${wordChar})+${suffixLookAhead}`, 'iu'); +}; diff --git a/src/core/token.ts b/src/core/token.ts index 722cc256f6..776c5e2780 100644 --- a/src/core/token.ts +++ b/src/core/token.ts @@ -7,6 +7,11 @@ export enum TokenType { RESERVED_BINARY_COMMAND = 'RESERVED_BINARY_COMMAND', RESERVED_COMMAND = 'RESERVED_COMMAND', OPERATOR = 'OPERATOR', + COMMA = 'COMMA', + OPEN_PAREN = 'OPEN_PAREN', + OPEN_BRACKET = 'OPEN_BRACKET', + CLOSE_PAREN = 'CLOSE_PAREN', + CLOSE_BRACKET = 'CLOSE_BRACKET', BLOCK_START = 'BLOCK_START', BLOCK_END = 'BLOCK_END', LINE_COMMENT = 'LINE_COMMENT', diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts index 6700f2e5c0..2ee6ca629a 100644 --- a/src/lexer/tokenizer.ts +++ b/src/lexer/tokenizer.ts @@ -1,8 +1,8 @@ import * as moo from 'moo'; import * as regexFactory from '../core/regexFactory'; -import { escapeRegExp } from '../utils'; -import { Token, TokenType } from '../core/token'; // convert to partial type import in TS 4.5 +import { TokenType } from '../core/token'; // convert to partial type import in TS 4.5 +import { StringPatternType, stringRegex, wordRegex } from '../core/mooRegexFactory'; const NULL_REGEX = /(?!)/; // zero-width negative lookahead, matches nothing @@ -12,7 +12,7 @@ interface TokenizerOptions { reservedLogicalOperators: string[]; reservedDependentClauses: string[]; reservedBinaryCommands: string[]; - stringTypes: regexFactory.StringPatternType[]; + stringTypes: StringPatternType[]; blockStart: string[]; blockEnd: string[]; indexedPlaceholderTypes?: string[]; @@ -24,12 +24,14 @@ interface TokenizerOptions { export default class Tokenizer { WHITESPACE_REGEX: RegExp; - REGEX_MAP: { [tokenType in TokenType]: RegExp }; + REGEX_MAP: Partial<{ [tokenType in TokenType]: RegExp }>; INDEXED_PLACEHOLDER_REGEX?: RegExp; IDENT_NAMED_PLACEHOLDER_REGEX?: RegExp; STRING_NAMED_PLACEHOLDER_REGEX?: RegExp; + LEXER_OPTIONS: { [key: string]: moo.Rule }; + /** * @param {TokenizerOptions} cfg * @param {String[]} cfg.reservedKeywords: Reserved words in SQL @@ -101,32 +103,32 @@ export default class Tokenizer { cfg.namedPlaceholderTypes, regexFactory.createStringPattern(cfg.stringTypes) ); - } - tokenize(input: string) { - let lexerOptions: { [key: string]: moo.Rule | RegExp } = { - WS: /[ \t]+/, + this.LEXER_OPTIONS = { + WS: { match: /[ \t]+/ }, NL: { match: /\n/, lineBreaks: true }, - [TokenType.WORD]: new RegExp( - /(?:\p{Alphabetic}|\p{Mark}|\p{Decimal_Number}|\p{Connector_Punctuation}|\p{Join_Control})+/u, - 'u' - ), - [TokenType.OPERATOR]: new RegExp('[+-/*%&|^><=.,;\\[\\]\\}\\{`:$]', 'u'), - [TokenType.BLOCK_START]: /[(]/, - [TokenType.BLOCK_END]: /[)]/, + [TokenType.COMMA]: { match: /[,]/ }, + [TokenType.OPEN_PAREN]: { match: /[(]/ }, + [TokenType.CLOSE_PAREN]: { match: /[)]/ }, + [TokenType.OPEN_BRACKET]: { match: /[[]/ }, + [TokenType.CLOSE_BRACKET]: { match: /[\]]/ }, + [TokenType.OPERATOR]: { match: new RegExp('[+-/*%&|^><=.;{}`:$]', 'u') }, + [TokenType.STRING]: { match: stringRegex({ stringTypes: cfg.stringTypes }) }, + [TokenType.WORD]: { match: wordRegex(cfg.specialWordChars) }, + WIP: { match: '.' }, }; - lexerOptions = Object.entries(lexerOptions).reduce( + this.LEXER_OPTIONS = Object.entries(this.LEXER_OPTIONS).reduce( (acc, [name, regex]) => ({ ...acc, - [name]: - regex instanceof RegExp - ? new RegExp(regex, 'u') - : { ...regex, match: new RegExp(regex.match as string | RegExp, 'u') }, + [name]: { ...regex, match: new RegExp(regex.match as string | RegExp, 'u') }, }), - {} as { [key: string]: moo.Rule | RegExp } + {} as { [key: string]: moo.Rule } ); + } + + tokenize(input: string) { + const lexer = moo.compile(this.LEXER_OPTIONS); - const lexer = moo.compile(lexerOptions); lexer.reset(input); return Array.from(lexer); } From a80cfc8f13127bc7cf59e655a3ffa85386d813af Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Wed, 19 Jan 2022 22:14:04 -0800 Subject: [PATCH 05/21] update comments, case/end, operators, numbers for moo --- src/core/mooRegexFactory.ts | 12 +++++++++- src/lexer/tokenizer.ts | 47 ++++++++++++++++++++++--------------- test/moo.test.js | 27 ++++++++++++++------- test/test.sql | 2 +- 4 files changed, 58 insertions(+), 30 deletions(-) diff --git a/src/core/mooRegexFactory.ts b/src/core/mooRegexFactory.ts index a815f9397e..4e18265f4d 100644 --- a/src/core/mooRegexFactory.ts +++ b/src/core/mooRegexFactory.ts @@ -1,4 +1,14 @@ -import { escapeRegExp } from '../utils'; +import { escapeRegExp, sortByLengthDesc } from '../utils'; + +export const lineCommentRegex = (lineCommentTypes: string[]) => + new RegExp(`^(?:${lineCommentTypes.map(escapeRegExp).join('|')}.*?)(?:\r\n|\r|\n|$)`, 'u'); + +export const operatorRegex = (monadOperators: string, polyadOperators: string[]) => + new RegExp( + `^${sortByLengthDesc(polyadOperators).map(escapeRegExp).join('|')}|` + + `[${monadOperators.split('').map(escapeRegExp).join('')}]`, + 'u' + ); // This enables the following string patterns: // 1. backtick quoted string using `` to escape diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts index 2ee6ca629a..f582f455b9 100644 --- a/src/lexer/tokenizer.ts +++ b/src/lexer/tokenizer.ts @@ -2,7 +2,13 @@ import * as moo from 'moo'; import * as regexFactory from '../core/regexFactory'; import { TokenType } from '../core/token'; // convert to partial type import in TS 4.5 -import { StringPatternType, stringRegex, wordRegex } from '../core/mooRegexFactory'; +import { + lineCommentRegex, + operatorRegex, + StringPatternType, + stringRegex, + wordRegex, +} from '../core/mooRegexFactory'; const NULL_REGEX = /(?!)/; // zero-width negative lookahead, matches nothing @@ -53,8 +59,6 @@ export default class Tokenizer { const specialWordCharsAll = Object.values(cfg.specialWordChars ?? {}).join(''); this.REGEX_MAP = { - [TokenType.WORD]: regexFactory.createWordRegex(cfg.specialWordChars), - [TokenType.STRING]: regexFactory.createStringRegex(cfg.stringTypes), [TokenType.RESERVED_KEYWORD]: regexFactory.createReservedWordRegex( cfg.reservedKeywords, specialWordCharsAll @@ -75,20 +79,6 @@ export default class Tokenizer { cfg.reservedBinaryCommands, specialWordCharsAll ), - [TokenType.OPERATOR]: regexFactory.createOperatorRegex('+-/*%&|^><=.,;[]{}`:$', [ - '<>', - '<=', - '>=', - '!=', - ...(cfg.operators ?? []), - ]), - [TokenType.BLOCK_START]: regexFactory.createParenRegex(cfg.blockStart), - [TokenType.BLOCK_END]: regexFactory.createParenRegex(cfg.blockEnd), - [TokenType.LINE_COMMENT]: regexFactory.createLineCommentRegex(cfg.lineCommentTypes), - [TokenType.BLOCK_COMMENT]: /^(\/\*[^]*?(?:\*\/|$))/u, - [TokenType.NUMBER]: - /^((-\s*)?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b/u, - [TokenType.PLACEHOLDER]: NULL_REGEX, // matches nothing }; this.INDEXED_PLACEHOLDER_REGEX = regexFactory.createPlaceholderRegex( @@ -107,14 +97,33 @@ export default class Tokenizer { this.LEXER_OPTIONS = { WS: { match: /[ \t]+/ }, NL: { match: /\n/, lineBreaks: true }, + [TokenType.BLOCK_COMMENT]: { match: /^(?:\/\*[^]*?(?:\*\/|$))/u, lineBreaks: true }, + [TokenType.LINE_COMMENT]: { + match: lineCommentRegex(cfg.lineCommentTypes), + }, [TokenType.COMMA]: { match: /[,]/ }, [TokenType.OPEN_PAREN]: { match: /[(]/ }, [TokenType.CLOSE_PAREN]: { match: /[)]/ }, [TokenType.OPEN_BRACKET]: { match: /[[]/ }, [TokenType.CLOSE_BRACKET]: { match: /[\]]/ }, - [TokenType.OPERATOR]: { match: new RegExp('[+-/*%&|^><=.;{}`:$]', 'u') }, + [TokenType.OPERATOR]: { + match: operatorRegex('+-/*%&|^><=.;{}`:$', [ + '<>', + '<=', + '>=', + '!=', + ...(cfg.operators ?? []), + ]), + }, + [TokenType.NUMBER]: { + match: + /^(?:(?:-\s*)?[0-9]+(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b/u, + }, [TokenType.STRING]: { match: stringRegex({ stringTypes: cfg.stringTypes }) }, - [TokenType.WORD]: { match: wordRegex(cfg.specialWordChars) }, + [TokenType.WORD]: { + match: wordRegex(cfg.specialWordChars), + type: moo.keywords({ BLOCK_CASE: 'CASE', BLOCK_END: 'END' }), + }, WIP: { match: '.' }, }; this.LEXER_OPTIONS = Object.entries(this.LEXER_OPTIONS).reduce( diff --git a/test/moo.test.js b/test/moo.test.js index 860f37b66a..b605406d72 100644 --- a/test/moo.test.js +++ b/test/moo.test.js @@ -18,16 +18,25 @@ const options = { lineCommentTypes: StandardSqlFormatter.lineCommentTypes, }; -const tokenizer = new Tokenizer(options); -const stream = tokenizer.tokenize(testSql); +describe('MySqlFormatter', () => { + const tokenizer = new Tokenizer(options); + const stream = tokenizer.tokenize(testSql); -const mooTokenizer = new MooTokenizer(options); -const mooStream = mooTokenizer.tokenize(testSql); + const mooTokenizer = new MooTokenizer(options); + const mooStream = mooTokenizer.tokenize(testSql); -const filtered = mooStream.filter(token => token.type !== 'WS' && token.type !== 'NL'); -console.log(stream.length); -console.log(filtered.length); + const filtered = mooStream.filter(token => token.type !== 'WS' && token.type !== 'NL'); + console.log('old:', stream.length, 'new:', filtered.length); -stream.forEach((t, i) => { - console.log(t.value, filtered[i].value); + // console.log(filtered); + + console.log(filtered.filter(token => token.type === 'WIP').length); + + it('does not have any WIP tokens', () => { + expect(filtered).not.toContainEqual( + expect.objectContaining({ + type: 'WIP', + }) + ); + }); }); diff --git a/test/test.sql b/test/test.sql index 26bc949767..d789f7c4c8 100644 --- a/test/test.sql +++ b/test/test.sql @@ -2,7 +2,7 @@ select alpha + 1, beta[0] b, gamma['radiation'] AS g, - CASE WHEN iota THEN i END + case WHEN iota THEN i END FROM ( SELECT MAX(epsilon) From 9473e19baa6e4e8082b0f929b3352ea479cc4de0 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Thu, 20 Jan 2022 07:12:19 -0800 Subject: [PATCH 06/21] add moo support for keywords --- src/core/mooRegexFactory.ts | 20 ++++++++++ src/core/token.ts | 4 +- src/lexer/tokenizer.ts | 73 ++++++++++++++++++------------------- test/moo.test.js | 2 +- 4 files changed, 60 insertions(+), 39 deletions(-) diff --git a/src/core/mooRegexFactory.ts b/src/core/mooRegexFactory.ts index 4e18265f4d..36e0aca601 100644 --- a/src/core/mooRegexFactory.ts +++ b/src/core/mooRegexFactory.ts @@ -67,3 +67,23 @@ export const wordRegex = ( return new RegExp(`${prefixLookBehind}(?:${wordChar})+${suffixLookAhead}`, 'iu'); }; + +export const reservedWordRegex = (reservedKeywords: string[], specialWordChars: string) => { + if (reservedKeywords.length === 0) { + return new RegExp(`^\b$`, 'u'); + } + const reservedKeywordsPattern = sortByLengthDesc(reservedKeywords) + .map(keyword => + keyword + .split('') + .map(char => (/ /gu.test(char) ? '\\s+' : `[${char.toUpperCase()}${char.toLowerCase()}]`)) + .join('') + ) + .join('|') + .replace(/ /gu, '\\s+'); + + const specialCharPattern = specialWordChars.length + ? `(?![${escapeRegExp(specialWordChars)}]+)` + : ''; + return new RegExp(`^${reservedKeywordsPattern}${specialCharPattern}\\b`, 'iu'); +}; diff --git a/src/core/token.ts b/src/core/token.ts index 776c5e2780..12db6a9098 100644 --- a/src/core/token.ts +++ b/src/core/token.ts @@ -9,9 +9,11 @@ export enum TokenType { OPERATOR = 'OPERATOR', COMMA = 'COMMA', OPEN_PAREN = 'OPEN_PAREN', - OPEN_BRACKET = 'OPEN_BRACKET', CLOSE_PAREN = 'CLOSE_PAREN', + OPEN_BRACKET = 'OPEN_BRACKET', CLOSE_BRACKET = 'CLOSE_BRACKET', + CASE_START = 'CASE_START', + CASE_END = 'CASE_END', BLOCK_START = 'BLOCK_START', BLOCK_END = 'BLOCK_END', LINE_COMMENT = 'LINE_COMMENT', diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts index f582f455b9..fd2db89ec7 100644 --- a/src/lexer/tokenizer.ts +++ b/src/lexer/tokenizer.ts @@ -5,13 +5,12 @@ import { TokenType } from '../core/token'; // convert to partial type import in import { lineCommentRegex, operatorRegex, + reservedWordRegex, StringPatternType, stringRegex, wordRegex, } from '../core/mooRegexFactory'; -const NULL_REGEX = /(?!)/; // zero-width negative lookahead, matches nothing - interface TokenizerOptions { reservedKeywords: string[]; reservedCommands: string[]; @@ -29,14 +28,12 @@ interface TokenizerOptions { } export default class Tokenizer { - WHITESPACE_REGEX: RegExp; - REGEX_MAP: Partial<{ [tokenType in TokenType]: RegExp }>; - INDEXED_PLACEHOLDER_REGEX?: RegExp; IDENT_NAMED_PLACEHOLDER_REGEX?: RegExp; STRING_NAMED_PLACEHOLDER_REGEX?: RegExp; LEXER_OPTIONS: { [key: string]: moo.Rule }; + LEXER: moo.Lexer; /** * @param {TokenizerOptions} cfg @@ -55,32 +52,6 @@ export default class Tokenizer { * @param {String[]} cfg.operators: Additional operators to recognize */ constructor(cfg: TokenizerOptions) { - this.WHITESPACE_REGEX = /^(\s+)/u; - - const specialWordCharsAll = Object.values(cfg.specialWordChars ?? {}).join(''); - this.REGEX_MAP = { - [TokenType.RESERVED_KEYWORD]: regexFactory.createReservedWordRegex( - cfg.reservedKeywords, - specialWordCharsAll - ), - [TokenType.RESERVED_DEPENDENT_CLAUSE]: regexFactory.createReservedWordRegex( - cfg.reservedDependentClauses ?? [], - specialWordCharsAll - ), - [TokenType.RESERVED_LOGICAL_OPERATOR]: regexFactory.createReservedWordRegex( - cfg.reservedLogicalOperators, - specialWordCharsAll - ), - [TokenType.RESERVED_COMMAND]: regexFactory.createReservedWordRegex( - cfg.reservedCommands, - specialWordCharsAll - ), - [TokenType.RESERVED_BINARY_COMMAND]: regexFactory.createReservedWordRegex( - cfg.reservedBinaryCommands, - specialWordCharsAll - ), - }; - this.INDEXED_PLACEHOLDER_REGEX = regexFactory.createPlaceholderRegex( cfg.indexedPlaceholderTypes ?? [], '[0-9]*' @@ -94,6 +65,8 @@ export default class Tokenizer { regexFactory.createStringPattern(cfg.stringTypes) ); + const specialWordCharsAll = Object.values(cfg.specialWordChars ?? {}).join(''); + this.LEXER_OPTIONS = { WS: { match: /[ \t]+/ }, NL: { match: /\n/, lineBreaks: true }, @@ -119,26 +92,52 @@ export default class Tokenizer { match: /^(?:(?:-\s*)?[0-9]+(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b/u, }, + [TokenType.CASE_START]: { match: /[Cc][Aa][Ss][Ee]/u }, + [TokenType.CASE_END]: { match: /[Ee][Nn][Dd]/u }, + [TokenType.RESERVED_COMMAND]: { + match: reservedWordRegex(cfg.reservedCommands, specialWordCharsAll), + }, + [TokenType.RESERVED_BINARY_COMMAND]: { + match: reservedWordRegex(cfg.reservedBinaryCommands, specialWordCharsAll), + }, + [TokenType.RESERVED_DEPENDENT_CLAUSE]: { + match: reservedWordRegex(cfg.reservedDependentClauses, specialWordCharsAll), + }, + [TokenType.RESERVED_LOGICAL_OPERATOR]: { + match: reservedWordRegex(cfg.reservedLogicalOperators, specialWordCharsAll), + }, + [TokenType.RESERVED_KEYWORD]: { + match: reservedWordRegex(cfg.reservedKeywords, specialWordCharsAll), + }, [TokenType.STRING]: { match: stringRegex({ stringTypes: cfg.stringTypes }) }, [TokenType.WORD]: { match: wordRegex(cfg.specialWordChars), - type: moo.keywords({ BLOCK_CASE: 'CASE', BLOCK_END: 'END' }), + // type: moo.keywords({ [TokenType.RESERVED_COMMAND]: cfg.reservedCommands }), // case sensitivity currently broken, see moo#122 }, WIP: { match: '.' }, }; + this.LEXER_OPTIONS = Object.entries(this.LEXER_OPTIONS).reduce( (acc, [name, regex]) => ({ ...acc, - [name]: { ...regex, match: new RegExp(regex.match as string | RegExp, 'u') }, + [name]: { + ...regex, + match: new RegExp( + regex.match as string | RegExp, + [...(regex.match instanceof RegExp ? regex.match.flags.split('') : [])] + .filter(flag => !'ium'.includes(flag)) // disallowed flags + .join('') + 'u' + ), + }, }), {} as { [key: string]: moo.Rule } ); + + this.LEXER = moo.compile(this.LEXER_OPTIONS); } tokenize(input: string) { - const lexer = moo.compile(this.LEXER_OPTIONS); - - lexer.reset(input); - return Array.from(lexer); + this.LEXER.reset(input); + return Array.from(this.LEXER); } } diff --git a/test/moo.test.js b/test/moo.test.js index b605406d72..3a71d10964 100644 --- a/test/moo.test.js +++ b/test/moo.test.js @@ -28,7 +28,7 @@ describe('MySqlFormatter', () => { const filtered = mooStream.filter(token => token.type !== 'WS' && token.type !== 'NL'); console.log('old:', stream.length, 'new:', filtered.length); - // console.log(filtered); + console.log(filtered.map(({ type, text }) => ({ type, text }))); console.log(filtered.filter(token => token.type === 'WIP').length); From 39e4989ae050169829ab2409de5bc4c875fbb82f Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Thu, 20 Jan 2022 08:21:43 -0800 Subject: [PATCH 07/21] add support for placeholders --- src/core/mooRegexFactory.ts | 12 ++++++++ src/lexer/tokenizer.ts | 55 ++++++++++++++++--------------------- test/moo.test.js | 3 +- test/test.sql | 1 + 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/src/core/mooRegexFactory.ts b/src/core/mooRegexFactory.ts index 36e0aca601..b73ac6a695 100644 --- a/src/core/mooRegexFactory.ts +++ b/src/core/mooRegexFactory.ts @@ -1,5 +1,7 @@ import { escapeRegExp, sortByLengthDesc } from '../utils'; +export const NULL_REGEX = /(?!)/; // zero-width negative lookahead, matches nothing + export const lineCommentRegex = (lineCommentTypes: string[]) => new RegExp(`^(?:${lineCommentTypes.map(escapeRegExp).join('|')}.*?)(?:\r\n|\r|\n|$)`, 'u'); @@ -87,3 +89,13 @@ export const reservedWordRegex = (reservedKeywords: string[], specialWordChars: : ''; return new RegExp(`^${reservedKeywordsPattern}${specialCharPattern}\\b`, 'iu'); }; + +export const placeholderRegex = (types: string[], pattern: string) => { + if (!types.length) { + // return NULL_REGEX; + return undefined; + } + const typesRegex = types.map(escapeRegExp).join('|'); + + return new RegExp(`${typesRegex}(?:${pattern})`, 'u'); +}; diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts index fd2db89ec7..17d735e4a8 100644 --- a/src/lexer/tokenizer.ts +++ b/src/lexer/tokenizer.ts @@ -1,10 +1,10 @@ import * as moo from 'moo'; -import * as regexFactory from '../core/regexFactory'; import { TokenType } from '../core/token'; // convert to partial type import in TS 4.5 import { lineCommentRegex, operatorRegex, + placeholderRegex, reservedWordRegex, StringPatternType, stringRegex, @@ -28,10 +28,6 @@ interface TokenizerOptions { } export default class Tokenizer { - INDEXED_PLACEHOLDER_REGEX?: RegExp; - IDENT_NAMED_PLACEHOLDER_REGEX?: RegExp; - STRING_NAMED_PLACEHOLDER_REGEX?: RegExp; - LEXER_OPTIONS: { [key: string]: moo.Rule }; LEXER: moo.Lexer; @@ -52,19 +48,6 @@ export default class Tokenizer { * @param {String[]} cfg.operators: Additional operators to recognize */ constructor(cfg: TokenizerOptions) { - this.INDEXED_PLACEHOLDER_REGEX = regexFactory.createPlaceholderRegex( - cfg.indexedPlaceholderTypes ?? [], - '[0-9]*' - ); - this.IDENT_NAMED_PLACEHOLDER_REGEX = regexFactory.createPlaceholderRegex( - cfg.namedPlaceholderTypes, - '[a-zA-Z0-9._$]+' - ); - this.STRING_NAMED_PLACEHOLDER_REGEX = regexFactory.createPlaceholderRegex( - cfg.namedPlaceholderTypes, - regexFactory.createStringPattern(cfg.stringTypes) - ); - const specialWordCharsAll = Object.values(cfg.specialWordChars ?? {}).join(''); this.LEXER_OPTIONS = { @@ -109,27 +92,37 @@ export default class Tokenizer { [TokenType.RESERVED_KEYWORD]: { match: reservedWordRegex(cfg.reservedKeywords, specialWordCharsAll), }, + INDEXED_PLACEHOLDER: { match: placeholderRegex(cfg.indexedPlaceholderTypes ?? [], '[0-9]*') }, + NAMED_PLACEHOLDER: { match: placeholderRegex(cfg.namedPlaceholderTypes, '[a-zA-Z0-9._$]+') }, + STRING_PLACEHOLDER: { + match: placeholderRegex( + cfg.namedPlaceholderTypes, + stringRegex({ stringTypes: cfg.stringTypes }).source + ), + }, [TokenType.STRING]: { match: stringRegex({ stringTypes: cfg.stringTypes }) }, [TokenType.WORD]: { match: wordRegex(cfg.specialWordChars), // type: moo.keywords({ [TokenType.RESERVED_COMMAND]: cfg.reservedCommands }), // case sensitivity currently broken, see moo#122 }, - WIP: { match: '.' }, }; this.LEXER_OPTIONS = Object.entries(this.LEXER_OPTIONS).reduce( - (acc, [name, regex]) => ({ - ...acc, - [name]: { - ...regex, - match: new RegExp( - regex.match as string | RegExp, - [...(regex.match instanceof RegExp ? regex.match.flags.split('') : [])] - .filter(flag => !'ium'.includes(flag)) // disallowed flags - .join('') + 'u' - ), - }, - }), + (rules, [name, regex]) => + regex.match + ? { + ...rules, + [name]: { + ...regex, + match: new RegExp( + regex.match as string | RegExp, + [...(regex.match instanceof RegExp ? regex.match.flags.split('') : [])] + .filter(flag => !'iumgy'.includes(flag)) // disallowed flags + .join('') + 'u' + ), + }, + } + : rules, {} as { [key: string]: moo.Rule } ); diff --git a/test/moo.test.js b/test/moo.test.js index 3a71d10964..e97f35ad36 100644 --- a/test/moo.test.js +++ b/test/moo.test.js @@ -18,7 +18,8 @@ const options = { lineCommentTypes: StandardSqlFormatter.lineCommentTypes, }; -describe('MySqlFormatter', () => { +describe('Moo', () => { + console.log(testSql); const tokenizer = new Tokenizer(options); const stream = tokenizer.tokenize(testSql); diff --git a/test/test.sql b/test/test.sql index d789f7c4c8..b18a68608b 100644 --- a/test/test.sql +++ b/test/test.sql @@ -1,4 +1,5 @@ select + ?, alpha + 1, beta[0] b, gamma['radiation'] AS g, From 755ffb0eee33dbda49eaf1ec720e7728a78232ca Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Thu, 20 Jan 2022 08:24:13 -0800 Subject: [PATCH 08/21] rename mooRegex functions --- src/core/mooRegexFactory.ts | 20 ++++++--------- src/lexer/tokenizer.ts | 50 ++++++++++++++++--------------------- 2 files changed, 30 insertions(+), 40 deletions(-) diff --git a/src/core/mooRegexFactory.ts b/src/core/mooRegexFactory.ts index b73ac6a695..86a0a6b2fa 100644 --- a/src/core/mooRegexFactory.ts +++ b/src/core/mooRegexFactory.ts @@ -2,10 +2,10 @@ import { escapeRegExp, sortByLengthDesc } from '../utils'; export const NULL_REGEX = /(?!)/; // zero-width negative lookahead, matches nothing -export const lineCommentRegex = (lineCommentTypes: string[]) => +export const lineComment = (lineCommentTypes: string[]) => new RegExp(`^(?:${lineCommentTypes.map(escapeRegExp).join('|')}.*?)(?:\r\n|\r|\n|$)`, 'u'); -export const operatorRegex = (monadOperators: string, polyadOperators: string[]) => +export const operator = (monadOperators: string, polyadOperators: string[]) => new RegExp( `^${sortByLengthDesc(polyadOperators).map(escapeRegExp).join('|')}|` + `[${monadOperators.split('').map(escapeRegExp).join('')}]`, @@ -32,7 +32,7 @@ const createStringPattern = (stringPrefixes: string) => ({ export type StringPatternType = keyof ReturnType; export type StringPatternPrefix = typeof stringPrefixList[number]; -export const stringRegex = ({ +export const string = ({ stringTypes, stringPrefixes, }: { @@ -45,9 +45,7 @@ export const stringRegex = ({ return new RegExp(`^${stringPattern}`, 'u'); }; -export const wordRegex = ( - specialChars: { any?: string; suffix?: string; prefix?: string } = {} -) => { +export const word = (specialChars: { any?: string; suffix?: string; prefix?: string } = {}) => { // lookbehind for specialChars that only appear at start const prefixLookBehind = specialChars.prefix?.length ? `[${escapeRegExp(specialChars.prefix)}]*` @@ -70,7 +68,7 @@ export const wordRegex = ( return new RegExp(`${prefixLookBehind}(?:${wordChar})+${suffixLookAhead}`, 'iu'); }; -export const reservedWordRegex = (reservedKeywords: string[], specialWordChars: string) => { +export const reservedWord = (reservedKeywords: string[], specialWordChars: string) => { if (reservedKeywords.length === 0) { return new RegExp(`^\b$`, 'u'); } @@ -90,12 +88,10 @@ export const reservedWordRegex = (reservedKeywords: string[], specialWordChars: return new RegExp(`^${reservedKeywordsPattern}${specialCharPattern}\\b`, 'iu'); }; -export const placeholderRegex = (types: string[], pattern: string) => { +export const placeholder = (types: string[], pattern: string) => { if (!types.length) { - // return NULL_REGEX; return undefined; } - const typesRegex = types.map(escapeRegExp).join('|'); - - return new RegExp(`${typesRegex}(?:${pattern})`, 'u'); + const typesPattern = types.map(escapeRegExp).join('|'); + return new RegExp(`${typesPattern}(?:${pattern})`, 'u'); }; diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts index 17d735e4a8..1ce7ac5bd0 100644 --- a/src/lexer/tokenizer.ts +++ b/src/lexer/tokenizer.ts @@ -1,15 +1,7 @@ import * as moo from 'moo'; import { TokenType } from '../core/token'; // convert to partial type import in TS 4.5 -import { - lineCommentRegex, - operatorRegex, - placeholderRegex, - reservedWordRegex, - StringPatternType, - stringRegex, - wordRegex, -} from '../core/mooRegexFactory'; +import * as regex from '../core/mooRegexFactory'; interface TokenizerOptions { reservedKeywords: string[]; @@ -17,7 +9,7 @@ interface TokenizerOptions { reservedLogicalOperators: string[]; reservedDependentClauses: string[]; reservedBinaryCommands: string[]; - stringTypes: StringPatternType[]; + stringTypes: regex.StringPatternType[]; blockStart: string[]; blockEnd: string[]; indexedPlaceholderTypes?: string[]; @@ -55,7 +47,7 @@ export default class Tokenizer { NL: { match: /\n/, lineBreaks: true }, [TokenType.BLOCK_COMMENT]: { match: /^(?:\/\*[^]*?(?:\*\/|$))/u, lineBreaks: true }, [TokenType.LINE_COMMENT]: { - match: lineCommentRegex(cfg.lineCommentTypes), + match: regex.lineComment(cfg.lineCommentTypes), }, [TokenType.COMMA]: { match: /[,]/ }, [TokenType.OPEN_PAREN]: { match: /[(]/ }, @@ -63,7 +55,7 @@ export default class Tokenizer { [TokenType.OPEN_BRACKET]: { match: /[[]/ }, [TokenType.CLOSE_BRACKET]: { match: /[\]]/ }, [TokenType.OPERATOR]: { - match: operatorRegex('+-/*%&|^><=.;{}`:$', [ + match: regex.operator('+-/*%&|^><=.;{}`:$', [ '<>', '<=', '>=', @@ -78,45 +70,47 @@ export default class Tokenizer { [TokenType.CASE_START]: { match: /[Cc][Aa][Ss][Ee]/u }, [TokenType.CASE_END]: { match: /[Ee][Nn][Dd]/u }, [TokenType.RESERVED_COMMAND]: { - match: reservedWordRegex(cfg.reservedCommands, specialWordCharsAll), + match: regex.reservedWord(cfg.reservedCommands, specialWordCharsAll), }, [TokenType.RESERVED_BINARY_COMMAND]: { - match: reservedWordRegex(cfg.reservedBinaryCommands, specialWordCharsAll), + match: regex.reservedWord(cfg.reservedBinaryCommands, specialWordCharsAll), }, [TokenType.RESERVED_DEPENDENT_CLAUSE]: { - match: reservedWordRegex(cfg.reservedDependentClauses, specialWordCharsAll), + match: regex.reservedWord(cfg.reservedDependentClauses, specialWordCharsAll), }, [TokenType.RESERVED_LOGICAL_OPERATOR]: { - match: reservedWordRegex(cfg.reservedLogicalOperators, specialWordCharsAll), + match: regex.reservedWord(cfg.reservedLogicalOperators, specialWordCharsAll), }, [TokenType.RESERVED_KEYWORD]: { - match: reservedWordRegex(cfg.reservedKeywords, specialWordCharsAll), + match: regex.reservedWord(cfg.reservedKeywords, specialWordCharsAll), }, - INDEXED_PLACEHOLDER: { match: placeholderRegex(cfg.indexedPlaceholderTypes ?? [], '[0-9]*') }, - NAMED_PLACEHOLDER: { match: placeholderRegex(cfg.namedPlaceholderTypes, '[a-zA-Z0-9._$]+') }, + INDEXED_PLACEHOLDER: { + match: regex.placeholder(cfg.indexedPlaceholderTypes ?? [], '[0-9]*'), + }, + NAMED_PLACEHOLDER: { match: regex.placeholder(cfg.namedPlaceholderTypes, '[a-zA-Z0-9._$]+') }, STRING_PLACEHOLDER: { - match: placeholderRegex( + match: regex.placeholder( cfg.namedPlaceholderTypes, - stringRegex({ stringTypes: cfg.stringTypes }).source + regex.string({ stringTypes: cfg.stringTypes }).source ), }, - [TokenType.STRING]: { match: stringRegex({ stringTypes: cfg.stringTypes }) }, + [TokenType.STRING]: { match: regex.string({ stringTypes: cfg.stringTypes }) }, [TokenType.WORD]: { - match: wordRegex(cfg.specialWordChars), + match: regex.word(cfg.specialWordChars), // type: moo.keywords({ [TokenType.RESERVED_COMMAND]: cfg.reservedCommands }), // case sensitivity currently broken, see moo#122 }, }; this.LEXER_OPTIONS = Object.entries(this.LEXER_OPTIONS).reduce( - (rules, [name, regex]) => - regex.match + (rules, [name, rule]) => + rule.match ? { ...rules, [name]: { - ...regex, + ...rule, match: new RegExp( - regex.match as string | RegExp, - [...(regex.match instanceof RegExp ? regex.match.flags.split('') : [])] + rule.match as string | RegExp, + [...(rule.match instanceof RegExp ? rule.match.flags.split('') : [])] .filter(flag => !'iumgy'.includes(flag)) // disallowed flags .join('') + 'u' ), From 7a8cc4d7372030c0762780a504afa941289e9991 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Mon, 30 May 2022 16:14:47 -0700 Subject: [PATCH 09/21] update tsconfig to allow absolute imports --- tsconfig.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tsconfig.json b/tsconfig.json index ab54a99bac..d19e158b7f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,6 +5,10 @@ "lib": ["es6", "dom"], "rootDirs": ["src"], "outDir": "lib", + "baseUrl": "./", + "paths": { + "src/*": ["./src/*"] + }, "sourceMap": true, "declaration": true, "esModuleInterop": true, From e863898c2d07cb00babc5c53a2ec8291e956eb1a Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Mon, 30 May 2022 16:15:19 -0700 Subject: [PATCH 10/21] update import paths for src --- src/core/AliasAs.ts | 3 ++- src/core/AsTokenFactory.ts | 3 ++- src/core/Indentation.ts | 2 +- src/core/StatementFormatter.ts | 5 +++-- src/core/Tokenizer.ts | 3 ++- src/core/WhitespaceBuilder.ts | 3 ++- src/core/config.ts | 2 +- src/core/formatAliasPositions.ts | 2 +- src/core/formatCommaPositions.ts | 5 +++-- src/core/regexFactory.ts | 2 +- src/core/tabularStyle.ts | 2 +- src/languages/bigquery.formatter.ts | 10 +++++----- src/languages/db2.formatter.ts | 8 ++++---- src/languages/hive.formatter.ts | 8 ++++---- src/languages/mariadb.formatter.ts | 10 +++++----- src/languages/mysql.formatter.ts | 10 +++++----- src/languages/n1ql.formatter.ts | 8 ++++---- src/languages/plsql.formatter.ts | 10 +++++----- src/languages/postgresql.formatter.ts | 8 ++++---- src/languages/redshift.formatter.ts | 8 ++++---- src/languages/spark.formatter.ts | 10 +++++----- src/languages/sql.formatter.ts | 8 ++++---- src/languages/sqlite.formatter.ts | 6 +++--- src/languages/tsql.formatter.ts | 8 ++++---- src/sqlFormatter.ts | 26 +++++++++++++------------- 25 files changed, 88 insertions(+), 82 deletions(-) diff --git a/src/core/AliasAs.ts b/src/core/AliasAs.ts index 592d21786c..982c6149f8 100644 --- a/src/core/AliasAs.ts +++ b/src/core/AliasAs.ts @@ -1,4 +1,5 @@ -import { AliasMode } from '../types'; +import { AliasMode } from 'src/types'; + import { isCommand, isToken, Token, TokenType } from './token'; export interface TokenStream { diff --git a/src/core/AsTokenFactory.ts b/src/core/AsTokenFactory.ts index e8810e4386..2babae17d9 100644 --- a/src/core/AsTokenFactory.ts +++ b/src/core/AsTokenFactory.ts @@ -1,4 +1,5 @@ -import { KeywordCase } from '../types'; +import { KeywordCase } from 'src/types'; + import { isToken, Token, TokenType } from './token'; export default class AsTokenFactory { diff --git a/src/core/Indentation.ts b/src/core/Indentation.ts index cf27972ec2..f64e1ef972 100644 --- a/src/core/Indentation.ts +++ b/src/core/Indentation.ts @@ -1,4 +1,4 @@ -import { last } from '../utils'; +import { last } from 'src/utils'; const INDENT_TYPE_TOP_LEVEL = 'top-level'; const INDENT_TYPE_BLOCK_LEVEL = 'block-level'; diff --git a/src/core/StatementFormatter.ts b/src/core/StatementFormatter.ts index 91f3f8ef6b..610c69c632 100644 --- a/src/core/StatementFormatter.ts +++ b/src/core/StatementFormatter.ts @@ -1,9 +1,10 @@ +import { FormatOptions } from 'src/types'; +import { equalizeWhitespace } from 'src/utils'; + import Indentation from './Indentation'; import InlineBlock from './InlineBlock'; import Params from './Params'; -import { equalizeWhitespace } from '../utils'; import { isReserved, isCommand, isToken, Token, TokenType, EOF_TOKEN } from './token'; -import { FormatOptions } from '../types'; import toTabularFormat from './tabularStyle'; import AliasAs from './AliasAs'; import AsTokenFactory from './AsTokenFactory'; diff --git a/src/core/Tokenizer.ts b/src/core/Tokenizer.ts index 66caad1fc9..9f2c92b42d 100644 --- a/src/core/Tokenizer.ts +++ b/src/core/Tokenizer.ts @@ -1,5 +1,6 @@ +import { equalizeWhitespace, escapeRegExp, id } from 'src/utils'; + import * as regexFactory from './regexFactory'; -import { equalizeWhitespace, escapeRegExp, id } from '../utils'; import { Token, TokenType } from './token'; // convert to partial type import in TS 4.5 export const WHITESPACE_REGEX = /^(\s+)/u; diff --git a/src/core/WhitespaceBuilder.ts b/src/core/WhitespaceBuilder.ts index 2f4cae4800..c19bf1207b 100644 --- a/src/core/WhitespaceBuilder.ts +++ b/src/core/WhitespaceBuilder.ts @@ -1,4 +1,5 @@ -import { last } from '../utils'; +import { last } from 'src/utils'; + import Indentation from './Indentation'; /** Whitespace modifiers to be used with add() method */ diff --git a/src/core/config.ts b/src/core/config.ts index f39cab4136..2ad1d11f99 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,4 +1,4 @@ -import { FormatOptions } from '../types'; +import { FormatOptions } from 'src/types'; // Utility functions for config options diff --git a/src/core/formatAliasPositions.ts b/src/core/formatAliasPositions.ts index fcd3035f97..205c3681fa 100644 --- a/src/core/formatAliasPositions.ts +++ b/src/core/formatAliasPositions.ts @@ -1,4 +1,4 @@ -import { maxLength } from '../utils'; +import { maxLength } from 'src/utils'; /** * Handles select alias placement - tabulates if enabled diff --git a/src/core/formatCommaPositions.ts b/src/core/formatCommaPositions.ts index 7d80a877ed..bc17e18960 100644 --- a/src/core/formatCommaPositions.ts +++ b/src/core/formatCommaPositions.ts @@ -1,5 +1,6 @@ -import { CommaPosition } from '../types'; -import { maxLength } from '../utils'; +import { CommaPosition } from 'src/types'; +import { maxLength } from 'src/utils'; + import { WHITESPACE_REGEX } from './Tokenizer'; /** diff --git a/src/core/regexFactory.ts b/src/core/regexFactory.ts index b5f790d3aa..a9fda754bc 100644 --- a/src/core/regexFactory.ts +++ b/src/core/regexFactory.ts @@ -1,4 +1,4 @@ -import { escapeRegExp, isEmpty, sortByLengthDesc } from '../utils'; +import { escapeRegExp, isEmpty, sortByLengthDesc } from 'src/utils'; /** * Builds a RegExp containing all operators for a SQL dialect diff --git a/src/core/tabularStyle.ts b/src/core/tabularStyle.ts index 0ddfa71f83..ec90614770 100644 --- a/src/core/tabularStyle.ts +++ b/src/core/tabularStyle.ts @@ -1,4 +1,4 @@ -import { IndentStyle } from '../types'; +import { IndentStyle } from 'src/types'; /** * When tabular style enabled, diff --git a/src/languages/bigquery.formatter.ts b/src/languages/bigquery.formatter.ts index 673bd88969..3fae04f6c2 100644 --- a/src/languages/bigquery.formatter.ts +++ b/src/languages/bigquery.formatter.ts @@ -1,8 +1,8 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import type { StringPatternType } from '../core/regexFactory'; -import { EOF_TOKEN, Token } from '../core/token'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { EOF_TOKEN, Token } from 'src/core/token'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/languages/db2.formatter.ts b/src/languages/db2.formatter.ts index f8ec66d284..aaf58fb672 100644 --- a/src/languages/db2.formatter.ts +++ b/src/languages/db2.formatter.ts @@ -1,7 +1,7 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/languages/hive.formatter.ts b/src/languages/hive.formatter.ts index ed25cb431d..25a6952525 100644 --- a/src/languages/hive.formatter.ts +++ b/src/languages/hive.formatter.ts @@ -1,7 +1,7 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/languages/mariadb.formatter.ts b/src/languages/mariadb.formatter.ts index f945eaf6fd..0cba8c09da 100644 --- a/src/languages/mariadb.formatter.ts +++ b/src/languages/mariadb.formatter.ts @@ -1,8 +1,8 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import { EOF_TOKEN, isToken, Token, TokenType } from '../core/token'; -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import { EOF_TOKEN, isToken, Token, TokenType } from 'src/core/token'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/languages/mysql.formatter.ts b/src/languages/mysql.formatter.ts index 3613f156d7..d4f04092b8 100644 --- a/src/languages/mysql.formatter.ts +++ b/src/languages/mysql.formatter.ts @@ -1,8 +1,8 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import { EOF_TOKEN, isToken, Token, TokenType } from '../core/token'; -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import { EOF_TOKEN, isToken, Token, TokenType } from 'src/core/token'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; // TODO: split this into object with function categories /** diff --git a/src/languages/n1ql.formatter.ts b/src/languages/n1ql.formatter.ts index 32e54b56a0..a34bd0a408 100644 --- a/src/languages/n1ql.formatter.ts +++ b/src/languages/n1ql.formatter.ts @@ -1,7 +1,7 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; // TODO: split this into object with function categories /** diff --git a/src/languages/plsql.formatter.ts b/src/languages/plsql.formatter.ts index 4558255803..00755c40cb 100644 --- a/src/languages/plsql.formatter.ts +++ b/src/languages/plsql.formatter.ts @@ -1,8 +1,8 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import { EOF_TOKEN, isReserved, isToken, Token, TokenType } from '../core/token'; // convert to partial type import in TS 4.5 -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import { EOF_TOKEN, isReserved, isToken, Token, TokenType } from 'src/core/token'; // convert to partial type import in TS 4.5 +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/languages/postgresql.formatter.ts b/src/languages/postgresql.formatter.ts index 05df3a908f..1cf177f12a 100644 --- a/src/languages/postgresql.formatter.ts +++ b/src/languages/postgresql.formatter.ts @@ -1,7 +1,7 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/languages/redshift.formatter.ts b/src/languages/redshift.formatter.ts index 7391f0db6d..b4bc3f8bc3 100644 --- a/src/languages/redshift.formatter.ts +++ b/src/languages/redshift.formatter.ts @@ -1,7 +1,7 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/languages/spark.formatter.ts b/src/languages/spark.formatter.ts index 129e7a8f89..3df212ee6c 100644 --- a/src/languages/spark.formatter.ts +++ b/src/languages/spark.formatter.ts @@ -1,8 +1,8 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import { EOF_TOKEN, isToken, Token, TokenType } from '../core/token'; // convert to partial type import in TS 4.5 -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import { EOF_TOKEN, isToken, Token, TokenType } from 'src/core/token'; // convert to partial type import in TS 4.5 +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/languages/sql.formatter.ts b/src/languages/sql.formatter.ts index 9301b64d42..43b4c2ce03 100644 --- a/src/languages/sql.formatter.ts +++ b/src/languages/sql.formatter.ts @@ -1,7 +1,7 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/languages/sqlite.formatter.ts b/src/languages/sqlite.formatter.ts index ee2bd95b07..8a4100800c 100644 --- a/src/languages/sqlite.formatter.ts +++ b/src/languages/sqlite.formatter.ts @@ -1,6 +1,6 @@ -import Formatter from '../core/Formatter'; -import { StringPatternType } from '../core/regexFactory'; -import Tokenizer from '../core/Tokenizer'; +import Formatter from 'src/core/Formatter'; +import { StringPatternType } from 'src/core/regexFactory'; +import Tokenizer from 'src/core/Tokenizer'; // https://jakewheat.github.io/sql-overview/sql-2008-foundation-grammar.html#reserved-word const standardReservedWords = [ diff --git a/src/languages/tsql.formatter.ts b/src/languages/tsql.formatter.ts index 0db9b68c28..8932def52c 100644 --- a/src/languages/tsql.formatter.ts +++ b/src/languages/tsql.formatter.ts @@ -1,7 +1,7 @@ -import Formatter from '../core/Formatter'; -import Tokenizer from '../core/Tokenizer'; -import type { StringPatternType } from '../core/regexFactory'; -import { dedupe } from '../utils'; +import Formatter from 'src/core/Formatter'; +import Tokenizer from 'src/core/Tokenizer'; +import type { StringPatternType } from 'src/core/regexFactory'; +import { dedupe } from 'src/utils'; /** * Priority 5 (last) diff --git a/src/sqlFormatter.ts b/src/sqlFormatter.ts index e6febe62f9..7bfa5b0bee 100644 --- a/src/sqlFormatter.ts +++ b/src/sqlFormatter.ts @@ -1,16 +1,16 @@ -import BigQueryFormatter from './languages/bigquery.formatter'; -import Db2Formatter from './languages/db2.formatter'; -import HiveFormatter from './languages/hive.formatter'; -import MariaDbFormatter from './languages/mariadb.formatter'; -import MySqlFormatter from './languages/mysql.formatter'; -import N1qlFormatter from './languages/n1ql.formatter'; -import PlSqlFormatter from './languages/plsql.formatter'; -import PostgreSqlFormatter from './languages/postgresql.formatter'; -import RedshiftFormatter from './languages/redshift.formatter'; -import SparkFormatter from './languages/spark.formatter'; -import SqliteFormatter from './languages/sqlite.formatter'; -import SqlFormatter from './languages/sql.formatter'; -import TSqlFormatter from './languages/tsql.formatter'; +import BigQueryFormatter from 'src/languages/bigquery.formatter'; +import Db2Formatter from 'src/languages/db2.formatter'; +import HiveFormatter from 'src/languages/hive.formatter'; +import MariaDbFormatter from 'src/languages/mariadb.formatter'; +import MySqlFormatter from 'src/languages/mysql.formatter'; +import N1qlFormatter from 'src/languages/n1ql.formatter'; +import PlSqlFormatter from 'src/languages/plsql.formatter'; +import PostgreSqlFormatter from 'src/languages/postgresql.formatter'; +import RedshiftFormatter from 'src/languages/redshift.formatter'; +import SparkFormatter from 'src/languages/spark.formatter'; +import SqliteFormatter from 'src/languages/sqlite.formatter'; +import SqlFormatter from 'src/languages/sql.formatter'; +import TSqlFormatter from 'src/languages/tsql.formatter'; import { FormatOptions } from './types'; import { isNumber } from './utils'; From 79ed1f7b1acf1247030009819b5b9e65640c362d Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Mon, 30 May 2022 16:21:16 -0700 Subject: [PATCH 11/21] update import paths for test --- test/behavesLikeMariaDbFormatter.ts | 5 +++-- test/behavesLikeSqlFormatter.ts | 5 +++-- test/bigquery.test.ts | 7 ++++--- test/db2.test.ts | 7 ++++--- test/features/alterTable.ts | 3 ++- test/features/alterTableModify.ts | 3 ++- test/features/array.ts | 3 ++- test/features/between.ts | 2 +- test/features/case.ts | 3 ++- test/features/comments.ts | 3 ++- test/features/constraints.ts | 3 ++- test/features/createTable.ts | 3 ++- test/features/deleteFrom.ts | 3 ++- test/features/join.ts | 3 ++- test/features/numbers.ts | 3 ++- test/features/operators.ts | 3 ++- test/features/returning.ts | 3 ++- test/features/schema.ts | 3 ++- test/features/strings.ts | 3 ++- test/hive.test.ts | 7 ++++--- test/mariadb.test.ts | 4 ++-- test/mysql.test.ts | 5 +++-- test/n1ql.test.ts | 7 ++++--- test/options/aliasAs.ts | 3 ++- test/options/commaPosition.ts | 3 ++- test/options/expressionWidth.ts | 3 ++- test/options/indentStyle.ts | 3 ++- test/options/keywordCase.ts | 3 ++- test/options/linesBetweenQueries.ts | 3 ++- test/options/logicalOperatorNewline.ts | 3 ++- test/options/multilineLists.ts | 3 ++- test/options/newlineBeforeParen.ts | 3 ++- test/options/newlineBeforeSemicolon.ts | 3 ++- test/options/param.ts | 3 ++- test/options/tabWidth.ts | 3 ++- test/options/tabulateAlias.ts | 3 ++- test/options/useTabs.ts | 2 +- test/plsql.test.ts | 7 ++++--- test/postgresql.test.ts | 6 +++--- test/redshift.test.ts | 7 ++++--- test/spark.test.ts | 7 ++++--- test/sql.test.ts | 7 ++++--- test/sqlFormatter.test.ts | 2 +- test/sqlite.test.ts | 7 ++++--- test/tsql.test.ts | 7 ++++--- test/unit/tabularStyle.test.ts | 2 +- 46 files changed, 112 insertions(+), 72 deletions(-) diff --git a/test/behavesLikeMariaDbFormatter.ts b/test/behavesLikeMariaDbFormatter.ts index 77b98374ad..6c0ae532bf 100644 --- a/test/behavesLikeMariaDbFormatter.ts +++ b/test/behavesLikeMariaDbFormatter.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; + +import { FormatFn } from 'src/sqlFormatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; -import { FormatFn } from '../src/sqlFormatter'; import supportsCreateTable from './features/createTable'; import supportsAlterTable from './features/alterTable'; @@ -8,8 +9,8 @@ import supportsBetween from './features/between'; import supportsJoin from './features/join'; import supportsConstraints from './features/constraints'; import supportsDeleteFrom from './features/deleteFrom'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; /** * Shared tests for MySQL and MariaDB diff --git a/test/behavesLikeSqlFormatter.ts b/test/behavesLikeSqlFormatter.ts index 1d096c9239..c823a601fa 100644 --- a/test/behavesLikeSqlFormatter.ts +++ b/test/behavesLikeSqlFormatter.ts @@ -1,7 +1,9 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../src/sqlFormatter'; +import { FormatFn } from 'src/sqlFormatter'; + import supportsCase from './features/case'; +import supportsNumbers from './features/numbers'; import supportsTabWidth from './options/tabWidth'; import supportsUseTabs from './options/useTabs'; import supportsAliasAs from './options/aliasAs'; @@ -15,7 +17,6 @@ import supportsLinesBetweenQueries from './options/linesBetweenQueries'; import supportsNewlineBeforeSemicolon from './options/newlineBeforeSemicolon'; import supportsLogicalOperatorNewline from './options/logicalOperatorNewline'; import supportsTabulateAlias from './options/tabulateAlias'; -import supportsNumbers from './features/numbers'; /** * Core tests for all SQL formatters diff --git a/test/bigquery.test.ts b/test/bigquery.test.ts index dfb0e65ca1..203cf195ae 100644 --- a/test/bigquery.test.ts +++ b/test/bigquery.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import BigQueryFormatter from '../src/languages/bigquery.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import BigQueryFormatter from 'src/languages/bigquery.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsCreateTable from './features/createTable'; @@ -10,8 +11,8 @@ import supportsBetween from './features/between'; import supportsJoin from './features/join'; import supportsOperators from './features/operators'; import supportsDeleteFrom from './features/deleteFrom'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('BigQueryFormatter', () => { const language = 'bigquery'; diff --git a/test/db2.test.ts b/test/db2.test.ts index d562d59e0a..bf5992f411 100644 --- a/test/db2.test.ts +++ b/test/db2.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import Db2Formatter from '../src/languages/db2.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import Db2Formatter from 'src/languages/db2.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsAlterTable from './features/alterTable'; @@ -12,8 +13,8 @@ import supportsSchema from './features/schema'; import supportsStrings from './features/strings'; import supportsConstraints from './features/constraints'; import supportsDeleteFrom from './features/deleteFrom'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('Db2Formatter', () => { const language = 'db2'; diff --git a/test/features/alterTable.ts b/test/features/alterTable.ts index 6f894139eb..b5abbae7a5 100644 --- a/test/features/alterTable.ts +++ b/test/features/alterTable.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsAlterTable(format: FormatFn) { it('formats ALTER TABLE ... ALTER COLUMN query', () => { diff --git a/test/features/alterTableModify.ts b/test/features/alterTableModify.ts index c6f215a772..e648dd5065 100644 --- a/test/features/alterTableModify.ts +++ b/test/features/alterTableModify.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsAlterTableModify(format: FormatFn) { it('formats ALTER TABLE ... MODIFY statement', () => { diff --git a/test/features/array.ts b/test/features/array.ts index 3ab15d2f87..38fda580cc 100644 --- a/test/features/array.ts +++ b/test/features/array.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsArray(format: FormatFn) { it('supports square brackets for array indexing', () => { diff --git a/test/features/between.ts b/test/features/between.ts index 76ab10c2e5..1f4eafd4e4 100644 --- a/test/features/between.ts +++ b/test/features/between.ts @@ -1,4 +1,4 @@ -import { FormatFn } from '../../src/sqlFormatter'; +import { FormatFn } from 'src/sqlFormatter'; export default function supportsBetween(format: FormatFn) { it('formats BETWEEN _ AND _ on single line', () => { diff --git a/test/features/case.ts b/test/features/case.ts index 41e77719d4..4522b9969e 100644 --- a/test/features/case.ts +++ b/test/features/case.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsCase(format: FormatFn) { it('formats CASE ... WHEN with a blank expression', () => { diff --git a/test/features/comments.ts b/test/features/comments.ts index 2d0aeb92c8..d67ca9e957 100644 --- a/test/features/comments.ts +++ b/test/features/comments.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; import { itIf } from '../utils'; interface CommentsConfig { diff --git a/test/features/constraints.ts b/test/features/constraints.ts index deb38c2748..4eba80f0dc 100644 --- a/test/features/constraints.ts +++ b/test/features/constraints.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsConstraints(format: FormatFn) { it('treats ON UPDATE & ON DELETE as distinct keywords from ON', () => { diff --git a/test/features/createTable.ts b/test/features/createTable.ts index bc23d0dfc5..98a247db57 100644 --- a/test/features/createTable.ts +++ b/test/features/createTable.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsCreateTable(format: FormatFn) { it('formats short CREATE TABLE', () => { diff --git a/test/features/deleteFrom.ts b/test/features/deleteFrom.ts index 78ab4224f8..e81a2e3643 100644 --- a/test/features/deleteFrom.ts +++ b/test/features/deleteFrom.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsDeleteFrom(format: FormatFn) { it('formats simple DELETE FROM statement', () => { diff --git a/test/features/join.ts b/test/features/join.ts index 115e5b6a69..8f43f3020d 100644 --- a/test/features/join.ts +++ b/test/features/join.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; type Options = { without?: string[]; additionally?: string[] }; diff --git a/test/features/numbers.ts b/test/features/numbers.ts index 8c03bc17d8..f342b671f4 100644 --- a/test/features/numbers.ts +++ b/test/features/numbers.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsNumbers(format: FormatFn) { it('supports decimal numbers', () => { diff --git a/test/features/operators.ts b/test/features/operators.ts index 8b802f727d..ac255bd9f7 100644 --- a/test/features/operators.ts +++ b/test/features/operators.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsOperators( format: FormatFn, diff --git a/test/features/returning.ts b/test/features/returning.ts index 3f3a2183aa..f1991da5fa 100644 --- a/test/features/returning.ts +++ b/test/features/returning.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsReturning(format: FormatFn) { it('places RETURNING to new line', () => { diff --git a/test/features/schema.ts b/test/features/schema.ts index 2e53db8fee..2b10a2155d 100644 --- a/test/features/schema.ts +++ b/test/features/schema.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsSchema(format: FormatFn) { it('formats simple SET SCHEMA statements', () => { diff --git a/test/features/strings.ts b/test/features/strings.ts index 00d8113cc1..47faa1110f 100644 --- a/test/features/strings.ts +++ b/test/features/strings.ts @@ -1,6 +1,7 @@ import { expect } from '@jest/globals'; import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsStrings(format: FormatFn, stringTypes: string[]) { if (stringTypes.includes('""')) { diff --git a/test/hive.test.ts b/test/hive.test.ts index 8f0a2cd414..013df350b8 100644 --- a/test/hive.test.ts +++ b/test/hive.test.ts @@ -1,5 +1,6 @@ -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import HiveFormatter from '../src/languages/hive.formatter'; +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; + +import HiveFormatter from 'src/languages/hive.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsCreateTable from './features/createTable'; @@ -10,8 +11,8 @@ import supportsBetween from './features/between'; import supportsJoin from './features/join'; import supportsOperators from './features/operators'; import supportsArray from './features/array'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('HiveFormatter', () => { const language = 'hive'; diff --git a/test/mariadb.test.ts b/test/mariadb.test.ts index 455db797c7..0adb2f1a82 100644 --- a/test/mariadb.test.ts +++ b/test/mariadb.test.ts @@ -1,5 +1,5 @@ -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import MariaDbFormatter from '../src/languages/mariadb.formatter'; +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import MariaDbFormatter from 'src/languages/mariadb.formatter'; import behavesLikeMariaDbFormatter from './behavesLikeMariaDbFormatter'; import supportsStrings from './features/strings'; diff --git a/test/mysql.test.ts b/test/mysql.test.ts index 5ee17ec8e3..1fe93ecfce 100644 --- a/test/mysql.test.ts +++ b/test/mysql.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import MySqlFormatter from '../src/languages/mysql.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import MySqlFormatter from 'src/languages/mysql.formatter'; import behavesLikeMariaDbFormatter from './behavesLikeMariaDbFormatter'; import supportsStrings from './features/strings'; diff --git a/test/n1ql.test.ts b/test/n1ql.test.ts index ff078c181f..1cee0fb391 100644 --- a/test/n1ql.test.ts +++ b/test/n1ql.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import N1qlFormatter from '../src/languages/n1ql.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import N1qlFormatter from 'src/languages/n1ql.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsBetween from './features/between'; @@ -11,8 +12,8 @@ import supportsStrings from './features/strings'; import supportsReturning from './features/returning'; import supportsDeleteFrom from './features/deleteFrom'; import supportsArray from './features/array'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('N1qlFormatter', () => { const language = 'n1ql'; diff --git a/test/options/aliasAs.ts b/test/options/aliasAs.ts index f1d06939ce..4943b58ed2 100644 --- a/test/options/aliasAs.ts +++ b/test/options/aliasAs.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsAliasAs(format: FormatFn) { describe('by default', () => { diff --git a/test/options/commaPosition.ts b/test/options/commaPosition.ts index d7ce002438..11c50f5b69 100644 --- a/test/options/commaPosition.ts +++ b/test/options/commaPosition.ts @@ -1,6 +1,7 @@ import { expect } from '@jest/globals'; import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsCommaPosition(format: FormatFn) { it('defaults to comma after column', () => { diff --git a/test/options/expressionWidth.ts b/test/options/expressionWidth.ts index aceb7beb3f..f58a081243 100644 --- a/test/options/expressionWidth.ts +++ b/test/options/expressionWidth.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsExpressionWidth(format: FormatFn) { it('throws error when expressionWidth negative number', () => { diff --git a/test/options/indentStyle.ts b/test/options/indentStyle.ts index beb375072d..4eaba09f67 100644 --- a/test/options/indentStyle.ts +++ b/test/options/indentStyle.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsIndentStyle(format: FormatFn) { const baseQuery = ` diff --git a/test/options/keywordCase.ts b/test/options/keywordCase.ts index 84b398208a..03d03a0240 100644 --- a/test/options/keywordCase.ts +++ b/test/options/keywordCase.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsKeywordCase(format: FormatFn) { it('preserves keyword case by default', () => { diff --git a/test/options/linesBetweenQueries.ts b/test/options/linesBetweenQueries.ts index adc18683e2..976988ef66 100644 --- a/test/options/linesBetweenQueries.ts +++ b/test/options/linesBetweenQueries.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsLinesBetweenQueries(format: FormatFn) { it('defaults to single empty line between queries', () => { diff --git a/test/options/logicalOperatorNewline.ts b/test/options/logicalOperatorNewline.ts index da820259c3..8348becada 100644 --- a/test/options/logicalOperatorNewline.ts +++ b/test/options/logicalOperatorNewline.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsLogicalOperatorNewline(format: FormatFn) { it('by default adds newline before logical operator', () => { diff --git a/test/options/multilineLists.ts b/test/options/multilineLists.ts index ece8c99f3e..1c8abd7463 100644 --- a/test/options/multilineLists.ts +++ b/test/options/multilineLists.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsMultilineLists(format: FormatFn) { it('throws error when multilineLists is negative number', () => { diff --git a/test/options/newlineBeforeParen.ts b/test/options/newlineBeforeParen.ts index 6d68d1b8ef..a96addd519 100644 --- a/test/options/newlineBeforeParen.ts +++ b/test/options/newlineBeforeParen.ts @@ -1,6 +1,7 @@ import { expect } from '@jest/globals'; import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsNewlineBeforeParen(format: FormatFn) { it('defaults to newline before opening and closing parenthesis', () => { diff --git a/test/options/newlineBeforeSemicolon.ts b/test/options/newlineBeforeSemicolon.ts index ed9f3d6235..3705861452 100644 --- a/test/options/newlineBeforeSemicolon.ts +++ b/test/options/newlineBeforeSemicolon.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsNewlineBeforeSemicolon(format: FormatFn) { it('formats lonely semicolon', () => { diff --git a/test/options/param.ts b/test/options/param.ts index 881dfaa240..ae89cd84a3 100644 --- a/test/options/param.ts +++ b/test/options/param.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; interface ParamsTypes { indexed?: ('?' | '$')[]; diff --git a/test/options/tabWidth.ts b/test/options/tabWidth.ts index e5d33d79d7..66ee57dd0c 100644 --- a/test/options/tabWidth.ts +++ b/test/options/tabWidth.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsTabWidth(format: FormatFn) { it('indents with 2 spaces by default', () => { diff --git a/test/options/tabulateAlias.ts b/test/options/tabulateAlias.ts index f4d20ddaa3..c3dd07ea35 100644 --- a/test/options/tabulateAlias.ts +++ b/test/options/tabulateAlias.ts @@ -1,5 +1,6 @@ import dedent from 'dedent-js'; -import { FormatFn } from '../../src/sqlFormatter'; + +import { FormatFn } from 'src/sqlFormatter'; export default function supportsTabulateAlias(format: FormatFn) { it('tabulates aliases which use AS keyword', () => { diff --git a/test/options/useTabs.ts b/test/options/useTabs.ts index 61c8c45c5a..beaf63bd84 100644 --- a/test/options/useTabs.ts +++ b/test/options/useTabs.ts @@ -1,4 +1,4 @@ -import { FormatFn } from '../../src/sqlFormatter'; +import { FormatFn } from 'src/sqlFormatter'; export default function supportsUseTabs(format: FormatFn) { it('supports indenting with tabs', () => { diff --git a/test/plsql.test.ts b/test/plsql.test.ts index d2f80db172..96ee26c70c 100644 --- a/test/plsql.test.ts +++ b/test/plsql.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import PlSqlFormatter from '../src/languages/plsql.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import PlSqlFormatter from 'src/languages/plsql.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsAlterTable from './features/alterTable'; @@ -14,8 +15,8 @@ import supportsStrings from './features/strings'; import supportsReturning from './features/returning'; import supportsConstraints from './features/constraints'; import supportsDeleteFrom from './features/deleteFrom'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('PlSqlFormatter', () => { const language = 'plsql'; diff --git a/test/postgresql.test.ts b/test/postgresql.test.ts index 4b9be128a0..4c4d7a2196 100644 --- a/test/postgresql.test.ts +++ b/test/postgresql.test.ts @@ -1,5 +1,5 @@ -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import PostgreSqlFormatter from '../src/languages/postgresql.formatter'; +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import PostgreSqlFormatter from 'src/languages/postgresql.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsAlterTable from './features/alterTable'; @@ -12,8 +12,8 @@ import supportsStrings from './features/strings'; import supportsReturning from './features/returning'; import supportsConstraints from './features/constraints'; import supportsDeleteFrom from './features/deleteFrom'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('PostgreSqlFormatter', () => { const language = 'postgresql'; diff --git a/test/redshift.test.ts b/test/redshift.test.ts index 340e436038..f2d2dda5ca 100644 --- a/test/redshift.test.ts +++ b/test/redshift.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import RedshiftFormatter from '../src/languages/redshift.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import RedshiftFormatter from 'src/languages/redshift.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsAlterTable from './features/alterTable'; @@ -11,8 +12,8 @@ import supportsOperators from './features/operators'; import supportsSchema from './features/schema'; import supportsStrings from './features/strings'; import supportsDeleteFrom from './features/deleteFrom'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('RedshiftFormatter', () => { const language = 'redshift'; diff --git a/test/spark.test.ts b/test/spark.test.ts index fb20ee1228..6e3e248815 100644 --- a/test/spark.test.ts +++ b/test/spark.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import SparkFormatter from '../src/languages/spark.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import SparkFormatter from 'src/languages/spark.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsAlterTable from './features/alterTable'; @@ -11,8 +12,8 @@ import supportsOperators from './features/operators'; import supportsSchema from './features/schema'; import supportsStrings from './features/strings'; import supportsArray from './features/array'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('SparkFormatter', () => { const language = 'spark'; diff --git a/test/sql.test.ts b/test/sql.test.ts index efb4557567..05171f6090 100644 --- a/test/sql.test.ts +++ b/test/sql.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import SqlFormatter from '../src/languages/sql.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import SqlFormatter from 'src/languages/sql.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsCreateTable from './features/createTable'; @@ -12,8 +13,8 @@ import supportsJoin from './features/join'; import supportsOperators from './features/operators'; import supportsConstraints from './features/constraints'; import supportsDeleteFrom from './features/deleteFrom'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('SqlFormatter', () => { const language = 'sql'; diff --git a/test/sqlFormatter.test.ts b/test/sqlFormatter.test.ts index ec17e0fb91..3e16d00686 100644 --- a/test/sqlFormatter.test.ts +++ b/test/sqlFormatter.test.ts @@ -1,4 +1,4 @@ -import { format, SqlLanguage } from '../src/sqlFormatter'; +import { format, SqlLanguage } from 'src/sqlFormatter'; describe('sqlFormatter', () => { it('throws error when unsupported language parameter specified', () => { diff --git a/test/sqlite.test.ts b/test/sqlite.test.ts index 1938d15c61..e54dfdf526 100644 --- a/test/sqlite.test.ts +++ b/test/sqlite.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import SqliteFormatter from '../src/languages/sqlite.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import SqliteFormatter from 'src/languages/sqlite.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsCreateTable from './features/createTable'; @@ -12,8 +13,8 @@ import supportsJoin from './features/join'; import supportsOperators from './features/operators'; import supportsConstraints from './features/constraints'; import supportsDeleteFrom from './features/deleteFrom'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('SqliteFormatter', () => { const language = 'sqlite'; diff --git a/test/tsql.test.ts b/test/tsql.test.ts index 953f08bce3..6f04440ce4 100644 --- a/test/tsql.test.ts +++ b/test/tsql.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent-js'; -import { format as originalFormat, FormatFn } from '../src/sqlFormatter'; -import TSqlFormatter from '../src/languages/tsql.formatter'; + +import { format as originalFormat, FormatFn } from 'src/sqlFormatter'; +import TSqlFormatter from 'src/languages/tsql.formatter'; import behavesLikeSqlFormatter from './behavesLikeSqlFormatter'; import supportsCreateTable from './features/createTable'; @@ -12,8 +13,8 @@ import supportsOperators from './features/operators'; import supportsJoin from './features/join'; import supportsConstraints from './features/constraints'; import supportsDeleteFrom from './features/deleteFrom'; -import supportsParams from './options/param'; import supportsComments from './features/comments'; +import supportsParams from './options/param'; describe('TSqlFormatter', () => { const language = 'tsql'; diff --git a/test/unit/tabularStyle.test.ts b/test/unit/tabularStyle.test.ts index 429e6b974e..7aca39ea1f 100644 --- a/test/unit/tabularStyle.test.ts +++ b/test/unit/tabularStyle.test.ts @@ -1,4 +1,4 @@ -import toTabularFormat from '../../src/core/tabularStyle'; +import toTabularFormat from 'src/core/tabularStyle'; describe('toTabularFormat()', () => { it('does nothing in standard style', () => { From c1d4150d4d04c827cfff0ae4b996763a4f9ec108 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Mon, 30 May 2022 16:22:26 -0700 Subject: [PATCH 12/21] update typescript version --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 7976910655..acbdb0499f 100644 --- a/package.json +++ b/package.json @@ -128,7 +128,7 @@ "rimraf": "^3.0.2", "ts-jest": "^28.0.0", "ts-loader": "^9.2.6", - "typescript": "^4.3.5", + "typescript": "^4.7.2", "webpack": "^5.64.1", "webpack-cli": "^4.9.1", "webpack-merge": "^5.8.0" diff --git a/yarn.lock b/yarn.lock index 35558dec8c..fcc4edab5e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7070,10 +7070,10 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript@^4.3.5: - version "4.6.3" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.3.tgz#eefeafa6afdd31d725584c67a0eaba80f6fc6c6c" - integrity sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw== +typescript@^4.7.2: + version "4.7.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.2.tgz#1f9aa2ceb9af87cca227813b4310fff0b51593c4" + integrity sha512-Mamb1iX2FDUpcTRzltPxgWMKy3fhg0TN378ylbktPGPK/99KbDtMQ4W1hwgsbPAsG3a0xKa1vmw4VKZQbkvz5A== uid-number@0.0.6: version "0.0.6" From dca3a8fb7cabb45848c947214196acedfde603f5 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Mon, 30 May 2022 16:28:33 -0700 Subject: [PATCH 13/21] update partial type imports --- src/core/AliasAs.ts | 4 ++-- src/core/AsTokenFactory.ts | 4 ++-- src/core/Formatter.ts | 6 +++--- src/core/InlineBlock.ts | 2 +- src/core/Params.ts | 2 +- src/core/Parser.ts | 2 +- src/core/StatementFormatter.ts | 6 +++--- src/core/Tokenizer.ts | 2 +- src/core/config.ts | 2 +- src/core/formatCommaPositions.ts | 2 +- src/core/tabularStyle.ts | 2 +- src/languages/bigquery.formatter.ts | 4 ++-- src/languages/db2.formatter.ts | 2 +- src/languages/hive.formatter.ts | 2 +- src/languages/mariadb.formatter.ts | 4 ++-- src/languages/mysql.formatter.ts | 4 ++-- src/languages/n1ql.formatter.ts | 2 +- src/languages/plsql.formatter.ts | 4 ++-- src/languages/postgresql.formatter.ts | 2 +- src/languages/redshift.formatter.ts | 2 +- src/languages/spark.formatter.ts | 4 ++-- src/languages/sql.formatter.ts | 2 +- src/languages/sqlite.formatter.ts | 2 +- src/languages/tsql.formatter.ts | 2 +- src/sqlFormatter.ts | 2 +- src/types.ts | 2 +- 26 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/core/AliasAs.ts b/src/core/AliasAs.ts index 982c6149f8..0f77ae19c6 100644 --- a/src/core/AliasAs.ts +++ b/src/core/AliasAs.ts @@ -1,6 +1,6 @@ -import { AliasMode } from 'src/types'; +import type { AliasMode } from 'src/types'; -import { isCommand, isToken, Token, TokenType } from './token'; +import { isCommand, isToken, type Token, TokenType } from './token'; export interface TokenStream { isWithinSelect(): boolean; diff --git a/src/core/AsTokenFactory.ts b/src/core/AsTokenFactory.ts index 2babae17d9..b4162b093b 100644 --- a/src/core/AsTokenFactory.ts +++ b/src/core/AsTokenFactory.ts @@ -1,6 +1,6 @@ -import { KeywordCase } from 'src/types'; +import type { KeywordCase } from 'src/types'; -import { isToken, Token, TokenType } from './token'; +import { isToken, type Token, TokenType } from './token'; export default class AsTokenFactory { private detectedCase: KeywordCase; diff --git a/src/core/Formatter.ts b/src/core/Formatter.ts index bcfdcdffd0..add598f945 100644 --- a/src/core/Formatter.ts +++ b/src/core/Formatter.ts @@ -1,12 +1,12 @@ import Params from './Params'; import Tokenizer from './Tokenizer'; -import { FormatOptions } from '../types'; +import type { FormatOptions } from '../types'; import formatCommaPositions from './formatCommaPositions'; import formatAliasPositions from './formatAliasPositions'; import AsTokenFactory from './AsTokenFactory'; -import Parser, { Statement } from './Parser'; +import Parser, { type Statement } from './Parser'; import StatementFormatter from './StatementFormatter'; -import { Token } from './token'; +import { type Token } from './token'; import { indentString } from './config'; /** Main formatter class that produces a final output string from list of tokens */ diff --git a/src/core/InlineBlock.ts b/src/core/InlineBlock.ts index b48e17b4ac..9b00c43745 100644 --- a/src/core/InlineBlock.ts +++ b/src/core/InlineBlock.ts @@ -1,4 +1,4 @@ -import { isToken, Token, TokenType } from './token'; +import { isToken, type Token, TokenType } from './token'; /** * Bookkeeper for inline blocks. diff --git a/src/core/Params.ts b/src/core/Params.ts index a0fdce0b7a..33d78d3267 100644 --- a/src/core/Params.ts +++ b/src/core/Params.ts @@ -1,4 +1,4 @@ -import type { Token } from './token'; +import { type Token } from './token'; export type ParamItems = { [k: string]: string }; diff --git a/src/core/Parser.ts b/src/core/Parser.ts index cdf2bd72d4..e50dc4383e 100644 --- a/src/core/Parser.ts +++ b/src/core/Parser.ts @@ -1,4 +1,4 @@ -import { EOF_TOKEN, Token, TokenType } from './token'; +import { EOF_TOKEN, type Token, TokenType } from './token'; /* eslint-disable no-cond-assign */ export type Statement = { diff --git a/src/core/StatementFormatter.ts b/src/core/StatementFormatter.ts index 610c69c632..d8cb2ff112 100644 --- a/src/core/StatementFormatter.ts +++ b/src/core/StatementFormatter.ts @@ -1,14 +1,14 @@ -import { FormatOptions } from 'src/types'; +import type { FormatOptions } from 'src/types'; import { equalizeWhitespace } from 'src/utils'; import Indentation from './Indentation'; import InlineBlock from './InlineBlock'; import Params from './Params'; -import { isReserved, isCommand, isToken, Token, TokenType, EOF_TOKEN } from './token'; +import { isReserved, isCommand, isToken, type Token, TokenType, EOF_TOKEN } from './token'; import toTabularFormat from './tabularStyle'; import AliasAs from './AliasAs'; import AsTokenFactory from './AsTokenFactory'; -import { Statement } from './Parser'; +import { type Statement } from './Parser'; import { indentString, isTabularStyle } from './config'; import WhitespaceBuilder, { WS } from './WhitespaceBuilder'; diff --git a/src/core/Tokenizer.ts b/src/core/Tokenizer.ts index 9f2c92b42d..d3150dc18b 100644 --- a/src/core/Tokenizer.ts +++ b/src/core/Tokenizer.ts @@ -1,7 +1,7 @@ import { equalizeWhitespace, escapeRegExp, id } from 'src/utils'; import * as regexFactory from './regexFactory'; -import { Token, TokenType } from './token'; // convert to partial type import in TS 4.5 +import { type Token, TokenType } from './token'; export const WHITESPACE_REGEX = /^(\s+)/u; const NULL_REGEX = /(?!)/; // zero-width negative lookahead, matches nothing diff --git a/src/core/config.ts b/src/core/config.ts index 2ad1d11f99..186c4c47aa 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,4 +1,4 @@ -import { FormatOptions } from 'src/types'; +import type { FormatOptions } from 'src/types'; // Utility functions for config options diff --git a/src/core/formatCommaPositions.ts b/src/core/formatCommaPositions.ts index bc17e18960..cfda17899f 100644 --- a/src/core/formatCommaPositions.ts +++ b/src/core/formatCommaPositions.ts @@ -1,4 +1,4 @@ -import { CommaPosition } from 'src/types'; +import type { CommaPosition } from 'src/types'; import { maxLength } from 'src/utils'; import { WHITESPACE_REGEX } from './Tokenizer'; diff --git a/src/core/tabularStyle.ts b/src/core/tabularStyle.ts index ec90614770..b0057382b9 100644 --- a/src/core/tabularStyle.ts +++ b/src/core/tabularStyle.ts @@ -1,4 +1,4 @@ -import { IndentStyle } from 'src/types'; +import type { IndentStyle } from 'src/types'; /** * When tabular style enabled, diff --git a/src/languages/bigquery.formatter.ts b/src/languages/bigquery.formatter.ts index 3fae04f6c2..329b95be29 100644 --- a/src/languages/bigquery.formatter.ts +++ b/src/languages/bigquery.formatter.ts @@ -1,7 +1,7 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import type { StringPatternType } from 'src/core/regexFactory'; -import { EOF_TOKEN, Token } from 'src/core/token'; +import { type StringPatternType } from 'src/core/regexFactory'; +import { EOF_TOKEN, type Token } from 'src/core/token'; import { dedupe } from 'src/utils'; /** diff --git a/src/languages/db2.formatter.ts b/src/languages/db2.formatter.ts index aaf58fb672..c3b54ad006 100644 --- a/src/languages/db2.formatter.ts +++ b/src/languages/db2.formatter.ts @@ -1,6 +1,6 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import type { StringPatternType } from 'src/core/regexFactory'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; /** diff --git a/src/languages/hive.formatter.ts b/src/languages/hive.formatter.ts index 25a6952525..eb0ff7d741 100644 --- a/src/languages/hive.formatter.ts +++ b/src/languages/hive.formatter.ts @@ -1,6 +1,6 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import type { StringPatternType } from 'src/core/regexFactory'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; /** diff --git a/src/languages/mariadb.formatter.ts b/src/languages/mariadb.formatter.ts index 0cba8c09da..4751fac06c 100644 --- a/src/languages/mariadb.formatter.ts +++ b/src/languages/mariadb.formatter.ts @@ -1,7 +1,7 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import { EOF_TOKEN, isToken, Token, TokenType } from 'src/core/token'; -import type { StringPatternType } from 'src/core/regexFactory'; +import { EOF_TOKEN, isToken, type Token, TokenType } from 'src/core/token'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; /** diff --git a/src/languages/mysql.formatter.ts b/src/languages/mysql.formatter.ts index d4f04092b8..4a5ec7b530 100644 --- a/src/languages/mysql.formatter.ts +++ b/src/languages/mysql.formatter.ts @@ -1,7 +1,7 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import { EOF_TOKEN, isToken, Token, TokenType } from 'src/core/token'; -import type { StringPatternType } from 'src/core/regexFactory'; +import { EOF_TOKEN, isToken, type Token, TokenType } from 'src/core/token'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; // TODO: split this into object with function categories diff --git a/src/languages/n1ql.formatter.ts b/src/languages/n1ql.formatter.ts index a34bd0a408..5693050482 100644 --- a/src/languages/n1ql.formatter.ts +++ b/src/languages/n1ql.formatter.ts @@ -1,6 +1,6 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import type { StringPatternType } from 'src/core/regexFactory'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; // TODO: split this into object with function categories diff --git a/src/languages/plsql.formatter.ts b/src/languages/plsql.formatter.ts index 00755c40cb..f13618bac7 100644 --- a/src/languages/plsql.formatter.ts +++ b/src/languages/plsql.formatter.ts @@ -1,7 +1,7 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import { EOF_TOKEN, isReserved, isToken, Token, TokenType } from 'src/core/token'; // convert to partial type import in TS 4.5 -import type { StringPatternType } from 'src/core/regexFactory'; +import { EOF_TOKEN, isReserved, isToken, type Token, TokenType } from 'src/core/token'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; /** diff --git a/src/languages/postgresql.formatter.ts b/src/languages/postgresql.formatter.ts index 1cf177f12a..e459bb792a 100644 --- a/src/languages/postgresql.formatter.ts +++ b/src/languages/postgresql.formatter.ts @@ -1,6 +1,6 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import type { StringPatternType } from 'src/core/regexFactory'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; /** diff --git a/src/languages/redshift.formatter.ts b/src/languages/redshift.formatter.ts index b4bc3f8bc3..d80a82ac10 100644 --- a/src/languages/redshift.formatter.ts +++ b/src/languages/redshift.formatter.ts @@ -1,6 +1,6 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import type { StringPatternType } from 'src/core/regexFactory'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; /** diff --git a/src/languages/spark.formatter.ts b/src/languages/spark.formatter.ts index 3df212ee6c..98590b4f3c 100644 --- a/src/languages/spark.formatter.ts +++ b/src/languages/spark.formatter.ts @@ -1,7 +1,7 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import { EOF_TOKEN, isToken, Token, TokenType } from 'src/core/token'; // convert to partial type import in TS 4.5 -import type { StringPatternType } from 'src/core/regexFactory'; +import { EOF_TOKEN, isToken, type Token, TokenType } from 'src/core/token'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; /** diff --git a/src/languages/sql.formatter.ts b/src/languages/sql.formatter.ts index 43b4c2ce03..a22a4077a2 100644 --- a/src/languages/sql.formatter.ts +++ b/src/languages/sql.formatter.ts @@ -1,6 +1,6 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import type { StringPatternType } from 'src/core/regexFactory'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; /** diff --git a/src/languages/sqlite.formatter.ts b/src/languages/sqlite.formatter.ts index 8a4100800c..f90c9eb366 100644 --- a/src/languages/sqlite.formatter.ts +++ b/src/languages/sqlite.formatter.ts @@ -1,5 +1,5 @@ import Formatter from 'src/core/Formatter'; -import { StringPatternType } from 'src/core/regexFactory'; +import { type StringPatternType } from 'src/core/regexFactory'; import Tokenizer from 'src/core/Tokenizer'; // https://jakewheat.github.io/sql-overview/sql-2008-foundation-grammar.html#reserved-word diff --git a/src/languages/tsql.formatter.ts b/src/languages/tsql.formatter.ts index 8932def52c..09e0358361 100644 --- a/src/languages/tsql.formatter.ts +++ b/src/languages/tsql.formatter.ts @@ -1,6 +1,6 @@ import Formatter from 'src/core/Formatter'; import Tokenizer from 'src/core/Tokenizer'; -import type { StringPatternType } from 'src/core/regexFactory'; +import { type StringPatternType } from 'src/core/regexFactory'; import { dedupe } from 'src/utils'; /** diff --git a/src/sqlFormatter.ts b/src/sqlFormatter.ts index 7bfa5b0bee..b210538b08 100644 --- a/src/sqlFormatter.ts +++ b/src/sqlFormatter.ts @@ -12,7 +12,7 @@ import SqliteFormatter from 'src/languages/sqlite.formatter'; import SqlFormatter from 'src/languages/sql.formatter'; import TSqlFormatter from 'src/languages/tsql.formatter'; -import { FormatOptions } from './types'; +import type { FormatOptions } from './types'; import { isNumber } from './utils'; export const formatters = { diff --git a/src/types.ts b/src/types.ts index f1e0aad8f0..e321c58656 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,4 @@ -import { ParamItems } from './core/Params'; +import { type ParamItems } from './core/Params'; export type IndentStyle = 'standard' | 'tabularLeft' | 'tabularRight'; From 5509e311d11c9eecf6fde0c51c6d3b1958e1f334 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Mon, 30 May 2022 16:36:59 -0700 Subject: [PATCH 14/21] add webpack alias for absolute imports --- webpack.common.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/webpack.common.js b/webpack.common.js index dfe84c15f7..aba4152caa 100644 --- a/webpack.common.js +++ b/webpack.common.js @@ -10,6 +10,9 @@ module.exports = { }, resolve: { extensions: ['.js', '.ts'], + alias: { + src: path.resolve(__dirname, 'src/'), + }, }, module: { rules: [ From bbeb17924d8097e7937c543f69108fdb9894b113 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Mon, 30 May 2022 17:30:02 -0700 Subject: [PATCH 15/21] update jest config to alias absolute path --- package.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index acbdb0499f..5ed182654f 100644 --- a/package.json +++ b/package.json @@ -141,6 +141,9 @@ "collectCoverage": true, "collectCoverageFrom": [ "src/**/*.ts" - ] + ], + "moduleNameMapper": { + "src/(.*)": "/src/$1" + } } } From ba2d7a864065314af48a4ad08bc81ea8131a6267 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Mon, 30 May 2022 17:40:00 -0700 Subject: [PATCH 16/21] :recycle: --- src/core/Formatter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/Formatter.ts b/src/core/Formatter.ts index add598f945..1803c3fd6a 100644 --- a/src/core/Formatter.ts +++ b/src/core/Formatter.ts @@ -1,6 +1,6 @@ +import type { FormatOptions } from 'src/types'; import Params from './Params'; import Tokenizer from './Tokenizer'; -import type { FormatOptions } from '../types'; import formatCommaPositions from './formatCommaPositions'; import formatAliasPositions from './formatAliasPositions'; import AsTokenFactory from './AsTokenFactory'; From 5201f7c6611e12bad1e1fe1187608d7064283a51 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Sat, 4 Jun 2022 13:18:07 -0700 Subject: [PATCH 17/21] get moo test working with ts-jest --- src/core/token.ts | 6 --- src/languages/sql.formatter.ts | 8 ++-- src/lexer/tokenizer.ts | 37 ++++++++--------- test/moo.test.ts | 73 +++++++++++++++++++--------------- 4 files changed, 63 insertions(+), 61 deletions(-) diff --git a/src/core/token.ts b/src/core/token.ts index 5d5bb852dd..9e96425954 100644 --- a/src/core/token.ts +++ b/src/core/token.ts @@ -12,12 +12,6 @@ export enum TokenType { RESERVED_CASE_END = 'RESERVED_CASE_END', OPERATOR = 'OPERATOR', COMMA = 'COMMA', - OPEN_PAREN = 'OPEN_PAREN', - CLOSE_PAREN = 'CLOSE_PAREN', - OPEN_BRACKET = 'OPEN_BRACKET', - CLOSE_BRACKET = 'CLOSE_BRACKET', - CASE_START = 'CASE_START', - CASE_END = 'CASE_END', BLOCK_START = 'BLOCK_START', BLOCK_END = 'BLOCK_END', LINE_COMMENT = 'LINE_COMMENT', diff --git a/src/languages/sql.formatter.ts b/src/languages/sql.formatter.ts index a22a4077a2..20ede7f78b 100644 --- a/src/languages/sql.formatter.ts +++ b/src/languages/sql.formatter.ts @@ -9,7 +9,7 @@ import { dedupe } from 'src/utils'; * any words that are in a higher priority are removed */ // https://jakewheat.github.io/sql-overview/sql-2008-foundation-grammar.html#reserved-word -const reservedKeywords = [ +export const reservedKeywords = [ 'ABS', 'ALL', 'ALLOCATE', @@ -304,7 +304,7 @@ const reservedKeywords = [ * keywords that begin a new statement * will begin new indented block */ -const reservedCommands = [ +export const reservedCommands = [ 'ADD', 'ALTER COLUMN', 'ALTER TABLE', @@ -338,7 +338,7 @@ const reservedCommands = [ * commands that operate on two tables or subqueries * two main categories: joins and boolean set operators */ -const reservedBinaryCommands = [ +export const reservedBinaryCommands = [ // set booleans 'INTERSECT', 'INTERSECT ALL', @@ -367,7 +367,7 @@ const reservedBinaryCommands = [ * keywords that follow a previous Statement, must be attached to subsequent data * can be fully inline or on newline with optional indent */ -const reservedDependentClauses = ['WHEN', 'ELSE']; +export const reservedDependentClauses = ['WHEN', 'ELSE']; export default class SqlFormatter extends Formatter { static stringTypes: StringPatternType[] = [`""`, "''", '``']; diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts index 23a65e3473..0efda307d8 100644 --- a/src/lexer/tokenizer.ts +++ b/src/lexer/tokenizer.ts @@ -1,20 +1,21 @@ import * as moo from 'moo'; -import { TokenType } from '../core/token'; // convert to partial type import in TS 4.5 -import * as regex from '../core/mooRegexFactory'; +import { TokenType } from 'src/core/token'; +import * as regex from 'src/core/mooRegexFactory'; interface TokenizerOptions { reservedKeywords: string[]; reservedCommands: string[]; - reservedLogicalOperators: string[]; + reservedLogicalOperators?: string[]; reservedDependentClauses: string[]; reservedBinaryCommands: string[]; + reservedJoinConditions?: string[]; stringTypes: regex.StringPatternType[]; - blockStart: string[]; - blockEnd: string[]; + blockStart?: string[]; + blockEnd?: string[]; indexedPlaceholderTypes?: string[]; - namedPlaceholderTypes: string[]; - lineCommentTypes: string[]; + namedPlaceholderTypes?: string[]; + lineCommentTypes?: string[]; specialWordChars?: { prefix?: string; any?: string; suffix?: string }; operators?: string[]; } @@ -47,13 +48,11 @@ export default class Tokenizer { NL: { match: /\n/, lineBreaks: true }, [TokenType.BLOCK_COMMENT]: { match: /^(?:\/\*[^]*?(?:\*\/|$))/u, lineBreaks: true }, [TokenType.LINE_COMMENT]: { - match: regex.lineComment(cfg.lineCommentTypes), + match: regex.lineComment(cfg.lineCommentTypes ?? ['--']), }, [TokenType.COMMA]: { match: /[,]/ }, - [TokenType.OPEN_PAREN]: { match: /[(]/ }, - [TokenType.CLOSE_PAREN]: { match: /[)]/ }, - [TokenType.OPEN_BRACKET]: { match: /[[]/ }, - [TokenType.CLOSE_BRACKET]: { match: /[\]]/ }, + [TokenType.BLOCK_START]: { match: /[([]/ }, // add params later + [TokenType.BLOCK_END]: { match: /[)\]]/ }, // add params later [TokenType.OPERATOR]: { match: regex.operator('+-/*%&|^><=.;{}`:$', [ '<>', @@ -67,8 +66,8 @@ export default class Tokenizer { match: /^(?:(?:-\s*)?[0-9]+(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b/u, }, - [TokenType.CASE_START]: { match: /[Cc][Aa][Ss][Ee]/u }, - [TokenType.CASE_END]: { match: /[Ee][Nn][Dd]/u }, + [TokenType.RESERVED_CASE_START]: { match: /[Cc][Aa][Ss][Ee]/u }, + [TokenType.RESERVED_CASE_END]: { match: /[Ee][Nn][Dd]/u }, [TokenType.RESERVED_COMMAND]: { match: regex.reservedWord(cfg.reservedCommands, specialWordCharsAll), }, @@ -79,7 +78,7 @@ export default class Tokenizer { match: regex.reservedWord(cfg.reservedDependentClauses, specialWordCharsAll), }, [TokenType.RESERVED_LOGICAL_OPERATOR]: { - match: regex.reservedWord(cfg.reservedLogicalOperators, specialWordCharsAll), + match: regex.reservedWord(cfg.reservedLogicalOperators ?? [], specialWordCharsAll), }, [TokenType.RESERVED_KEYWORD]: { match: regex.reservedWord(cfg.reservedKeywords, specialWordCharsAll), @@ -87,10 +86,12 @@ export default class Tokenizer { INDEXED_PLACEHOLDER: { match: regex.placeholder(cfg.indexedPlaceholderTypes ?? [], '[0-9]*'), }, - NAMED_PLACEHOLDER: { match: regex.placeholder(cfg.namedPlaceholderTypes, '[a-zA-Z0-9._$]+') }, + NAMED_PLACEHOLDER: { + match: regex.placeholder(cfg.namedPlaceholderTypes ?? [], '[a-zA-Z0-9._$]+'), + }, STRING_PLACEHOLDER: { match: regex.placeholder( - cfg.namedPlaceholderTypes, + cfg.namedPlaceholderTypes ?? [], regex.string({ stringTypes: cfg.stringTypes }).source ), }, @@ -123,7 +124,7 @@ export default class Tokenizer { this.LEXER = moo.compile(this.LEXER_OPTIONS); } - tokenize(input: string) { + tokenize(input: string): moo.Token[] { this.LEXER.reset(input); return Array.from(this.LEXER); } diff --git a/test/moo.test.ts b/test/moo.test.ts index 48d62a4731..d422b75fb4 100644 --- a/test/moo.test.ts +++ b/test/moo.test.ts @@ -1,45 +1,52 @@ -/* eslint-disable */ -import StandardSqlFormatter from 'src/languages/sql.formatter'; +import SqlFormatter, { + reservedBinaryCommands, + reservedCommands, + reservedDependentClauses, + // reservedLogicalOperators, + reservedKeywords, +} from 'src/languages/sql.formatter'; import Tokenizer from 'src/core/Tokenizer'; import MooTokenizer from 'src/lexer/tokenizer'; -// @ts-ignore-next-line -import testSql from './test.sql'; - -// const options = { -// reservedCommands: StandardSqlFormatter.reservedCommands, -// reservedBinaryCommands: StandardSqlFormatter.reservedBinaryCommands, -// reservedDependentClauses: StandardSqlFormatter.reservedDependentClauses, -// reservedLogicalOperators: StandardSqlFormatter.reservedLogicalOperators, -// reservedKeywords: StandardSqlFormatter.reservedKeywords, -// stringTypes: StandardSqlFormatter.stringTypes, -// blockStart: StandardSqlFormatter.blockStart, -// blockEnd: StandardSqlFormatter.blockEnd, -// indexedPlaceholderTypes: StandardSqlFormatter.indexedPlaceholderTypes, -// namedPlaceholderTypes: StandardSqlFormatter.namedPlaceholderTypes, -// lineCommentTypes: StandardSqlFormatter.lineCommentTypes, -// }; +import fs from 'fs'; + +// import testSql from './test.sql'; +const testSql = fs.readFileSync('test/test.sql', 'utf8'); + +const options = { + reservedKeywords, + reservedCommands, + reservedBinaryCommands, + reservedDependentClauses, + reservedLogicalOperators: ['AND', 'OR'], + stringTypes: SqlFormatter.stringTypes as any[], + // blockStart: blockStart, + // blockEnd: blockEnd, + indexedPlaceholderTypes: ['?'], + // namedPlaceholderTypes: namedPlaceholderTypes, + // lineCommentTypes: lineCommentTypes, +}; describe('Moo', () => { console.log(testSql); - // const tokenizer = new Tokenizer(options); - // const stream = tokenizer.tokenize(testSql); + const tokenizer = new Tokenizer(options); + const stream = tokenizer.tokenize(testSql); - // const mooTokenizer = new MooTokenizer(options); - // const mooStream = mooTokenizer.tokenize(testSql); + const mooTokenizer = new MooTokenizer(options); + const mooStream = mooTokenizer.tokenize(testSql); - // const filtered = mooStream.filter(token => token.type !== 'WS' && token.type !== 'NL'); - // console.log('old:', stream.length, 'new:', filtered.length); + const filtered = mooStream.filter(token => token.type !== 'WS' && token.type !== 'NL'); + console.log('old:', stream.length, 'new:', filtered.length); - // console.log(filtered.map(({ type, text }) => ({ type, text }))); + console.log(filtered.map(({ type, text }) => ({ type, text }))); - // console.log(filtered.filter(token => token.type === 'WIP').length); + console.log(filtered.filter(token => token.type === 'WIP').length); - // it('does not have any WIP tokens', () => { - // expect(filtered).not.toContainEqual( - // expect.objectContaining({ - // type: 'WIP', - // }) - // ); - // }); + it('does not have any WIP tokens', () => { + expect(filtered).not.toContainEqual( + expect.objectContaining({ + type: 'WIP', + }) + ); + }); }); From a2f4e054d0a1dd07087f799565d2fe824ef1f82d Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Sat, 4 Jun 2022 14:09:39 -0700 Subject: [PATCH 18/21] remove ^ and \b on moo regex --- src/core/mooRegexFactory.ts | 8 ++++---- src/lexer/tokenizer.ts | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/core/mooRegexFactory.ts b/src/core/mooRegexFactory.ts index 2dd82e12fb..d3b10a2183 100644 --- a/src/core/mooRegexFactory.ts +++ b/src/core/mooRegexFactory.ts @@ -3,11 +3,11 @@ import { escapeRegExp, sortByLengthDesc } from '../utils'; export const NULL_REGEX = /(?!)/; // zero-width negative lookahead, matches nothing export const lineComment = (lineCommentTypes: string[]) => - new RegExp(`^(?:${lineCommentTypes.map(escapeRegExp).join('|')}.*?)(?:\r\n|\r|\n|$)`, 'u'); + new RegExp(`(?:${lineCommentTypes.map(escapeRegExp).join('|')}.*?)(?:\r\n|\r|\n|$)`, 'u'); export const operator = (monadOperators: string, polyadOperators: string[]) => new RegExp( - `^${sortByLengthDesc(polyadOperators).map(escapeRegExp).join('|')}|` + + `${sortByLengthDesc(polyadOperators).map(escapeRegExp).join('|')}|` + `[${monadOperators.split('').map(escapeRegExp).join('')}]`, 'u' ); @@ -42,7 +42,7 @@ export const string = ({ const stringPrefix = stringPrefixes?.length ? `(?:${stringPrefixes.join('|')})?` : ''; const stringPatternMap = createStringPattern(stringPrefix); const stringPattern = stringTypes.map(stringType => stringPatternMap[stringType]).join('|'); - return new RegExp(`^${stringPattern}`, 'u'); + return new RegExp(`${stringPattern}`, 'u'); }; export const word = (specialChars: { any?: string; suffix?: string; prefix?: string } = {}) => { @@ -86,7 +86,7 @@ export const reservedWord = (reservedKeywords: string[], specialWordChars: strin const specialCharPattern = specialWordChars.length ? `(?![${escapeRegExp(specialWordChars)}]+)` : ''; - return new RegExp(`^${reservedKeywordsPattern}${specialCharPattern}\\b`, 'iu'); + return new RegExp(`${reservedKeywordsPattern}${specialCharPattern}`, 'iu'); }; export const placeholder = (types: string[], pattern: string) => { diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts index 0efda307d8..70bafa01b1 100644 --- a/src/lexer/tokenizer.ts +++ b/src/lexer/tokenizer.ts @@ -46,7 +46,7 @@ export default class Tokenizer { this.LEXER_OPTIONS = { WS: { match: /[ \t]+/ }, NL: { match: /\n/, lineBreaks: true }, - [TokenType.BLOCK_COMMENT]: { match: /^(?:\/\*[^]*?(?:\*\/|$))/u, lineBreaks: true }, + [TokenType.BLOCK_COMMENT]: { match: /(?:\/\*[^]*?(?:\*\/|$))/, lineBreaks: true }, [TokenType.LINE_COMMENT]: { match: regex.lineComment(cfg.lineCommentTypes ?? ['--']), }, @@ -64,7 +64,7 @@ export default class Tokenizer { }, [TokenType.NUMBER]: { match: - /^(?:(?:-\s*)?[0-9]+(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b/u, + /(?:0x[0-9a-fA-F]+|0b[01]+|(?:-\s*)?[0-9]+(?:\.[0-9]*)?(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?)/, }, [TokenType.RESERVED_CASE_START]: { match: /[Cc][Aa][Ss][Ee]/u }, [TokenType.RESERVED_CASE_END]: { match: /[Ee][Nn][Dd]/u }, @@ -78,7 +78,16 @@ export default class Tokenizer { match: regex.reservedWord(cfg.reservedDependentClauses, specialWordCharsAll), }, [TokenType.RESERVED_LOGICAL_OPERATOR]: { - match: regex.reservedWord(cfg.reservedLogicalOperators ?? [], specialWordCharsAll), + match: regex.reservedWord( + cfg.reservedLogicalOperators ?? ['AND', 'OR'], + specialWordCharsAll + ), + }, + [TokenType.RESERVED_JOIN_CONDITION]: { + match: regex.reservedWord( + cfg.reservedJoinConditions ?? ['ON', 'USING'], + specialWordCharsAll + ), }, [TokenType.RESERVED_KEYWORD]: { match: regex.reservedWord(cfg.reservedKeywords, specialWordCharsAll), From cb5a6c99de0a1d85136dae11aea490321b32ede1 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Sat, 4 Jun 2022 14:54:04 -0700 Subject: [PATCH 19/21] replace spread with concat --- src/core/mooRegexFactory.ts | 2 +- src/languages/bigquery.formatter.ts | 4 ++-- src/languages/db2.formatter.ts | 4 ++-- src/languages/hive.formatter.ts | 4 ++-- src/languages/mariadb.formatter.ts | 2 +- src/languages/mysql.formatter.ts | 2 +- src/languages/n1ql.formatter.ts | 2 +- src/languages/postgresql.formatter.ts | 2 +- src/languages/redshift.formatter.ts | 4 ++-- src/languages/spark.formatter.ts | 2 +- src/languages/sqlite.formatter.ts | 2 +- src/languages/tsql.formatter.ts | 4 ++-- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/core/mooRegexFactory.ts b/src/core/mooRegexFactory.ts index d3b10a2183..e0c45141f3 100644 --- a/src/core/mooRegexFactory.ts +++ b/src/core/mooRegexFactory.ts @@ -19,7 +19,7 @@ export const operator = (monadOperators: string, polyadOperators: string[]) => // 4. single quoted string using '' or \' to escape, with optional prefix for format-specific strings // 8. PostgreSQL dollar-quoted strings (does not check for matching tags due to moo not allowing capturing groups) -const stringPrefixList = ['[Nn]', '_utf8', 'U&', 'x']; +const stringPrefixList = ['[Nn]', '_utf8', 'U&', 'x', '[Rr]', '[Bb]', 'E']; const createStringPattern = (stringPrefixes: string) => ({ '``': '(?:`[^`]*(?:$|`))+', '{}': '(?:\\{[^\\}]*(?:$|\\}))+', diff --git a/src/languages/bigquery.formatter.ts b/src/languages/bigquery.formatter.ts index 329b95be29..5129fe9213 100644 --- a/src/languages/bigquery.formatter.ts +++ b/src/languages/bigquery.formatter.ts @@ -835,8 +835,8 @@ export default class BigQueryFormatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, reservedKeywords: dedupe([ - ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []), - ...Object.values(reservedKeywords).reduce((acc, arr) => [...acc, ...arr], []), + ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []), + ...Object.values(reservedKeywords).reduce((acc, arr) => acc.concat(arr), []), ]), stringTypes: BigQueryFormatter.stringTypes, indexedPlaceholderTypes: ['?'], diff --git a/src/languages/db2.formatter.ts b/src/languages/db2.formatter.ts index c3b54ad006..4ade36c17f 100644 --- a/src/languages/db2.formatter.ts +++ b/src/languages/db2.formatter.ts @@ -868,8 +868,8 @@ export default class Db2Formatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, reservedKeywords: dedupe([ - ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []), - ...Object.values(reservedKeywords).reduce((acc, arr) => [...acc, ...arr], []), + ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []), + ...Object.values(reservedKeywords).reduce((acc, arr) => acc.concat(arr), []), ]), stringTypes: Db2Formatter.stringTypes, indexedPlaceholderTypes: ['?'], diff --git a/src/languages/hive.formatter.ts b/src/languages/hive.formatter.ts index eb0ff7d741..622eb10266 100644 --- a/src/languages/hive.formatter.ts +++ b/src/languages/hive.formatter.ts @@ -623,8 +623,8 @@ export default class HiveFormatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, reservedKeywords: dedupe([ - ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []), - ...Object.values(reservedKeywords).reduce((acc, arr) => [...acc, ...arr], []), + ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []), + ...Object.values(reservedKeywords).reduce((acc, arr) => acc.concat(arr), []), ]), stringTypes: HiveFormatter.stringTypes, indexedPlaceholderTypes: ['?'], diff --git a/src/languages/mariadb.formatter.ts b/src/languages/mariadb.formatter.ts index 4751fac06c..c22042c98b 100644 --- a/src/languages/mariadb.formatter.ts +++ b/src/languages/mariadb.formatter.ts @@ -1165,7 +1165,7 @@ export default class MariaDbFormatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, reservedLogicalOperators: ['AND', 'OR', 'XOR'], - reservedKeywords: dedupe([...reservedKeywords, ...reservedFunctions]), + reservedKeywords: dedupe(reservedKeywords.concat(reservedFunctions)), stringTypes: MariaDbFormatter.stringTypes, indexedPlaceholderTypes: ['?'], lineCommentTypes: ['--', '#'], diff --git a/src/languages/mysql.formatter.ts b/src/languages/mysql.formatter.ts index 4a5ec7b530..33042d36a5 100644 --- a/src/languages/mysql.formatter.ts +++ b/src/languages/mysql.formatter.ts @@ -1328,7 +1328,7 @@ export default class MySqlFormatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, reservedLogicalOperators: ['AND', 'OR', 'XOR'], - reservedKeywords: dedupe([...reservedKeywords, ...reservedFunctions]), + reservedKeywords: dedupe(reservedKeywords.concat(reservedFunctions)), stringTypes: MySqlFormatter.stringTypes, indexedPlaceholderTypes: ['?'], lineCommentTypes: ['--', '#'], diff --git a/src/languages/n1ql.formatter.ts b/src/languages/n1ql.formatter.ts index 5693050482..a61212d3a8 100644 --- a/src/languages/n1ql.formatter.ts +++ b/src/languages/n1ql.formatter.ts @@ -521,7 +521,7 @@ export default class N1qlFormatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, reservedLogicalOperators: ['AND', 'OR', 'XOR'], - reservedKeywords: dedupe([...reservedKeywords, ...reservedFunctions]), + reservedKeywords: dedupe(reservedKeywords.concat(reservedFunctions)), stringTypes: N1qlFormatter.stringTypes, blockStart: ['(', '[', '{'], blockEnd: [')', ']', '}'], diff --git a/src/languages/postgresql.formatter.ts b/src/languages/postgresql.formatter.ts index e459bb792a..0f54d38610 100644 --- a/src/languages/postgresql.formatter.ts +++ b/src/languages/postgresql.formatter.ts @@ -1680,7 +1680,7 @@ export default class PostgreSqlFormatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, reservedKeywords: dedupe([ - ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []), + ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []), ...reservedKeywords, ]), stringTypes: PostgreSqlFormatter.stringTypes, diff --git a/src/languages/redshift.formatter.ts b/src/languages/redshift.formatter.ts index d80a82ac10..e39c7939c8 100644 --- a/src/languages/redshift.formatter.ts +++ b/src/languages/redshift.formatter.ts @@ -725,8 +725,8 @@ export default class RedshiftFormatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, reservedKeywords: dedupe([ - ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []), - ...Object.values(reservedKeywords).reduce((acc, arr) => [...acc, ...arr], []), + ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []), + ...Object.values(reservedKeywords).reduce((acc, arr) => acc.concat(arr), []), ]), stringTypes: RedshiftFormatter.stringTypes, indexedPlaceholderTypes: ['?'], diff --git a/src/languages/spark.formatter.ts b/src/languages/spark.formatter.ts index 98590b4f3c..cf7661d8c6 100644 --- a/src/languages/spark.formatter.ts +++ b/src/languages/spark.formatter.ts @@ -788,7 +788,7 @@ export default class SparkFormatter extends Formatter { reservedDependentClauses, reservedLogicalOperators: ['AND', 'OR', 'XOR'], reservedKeywords: dedupe([ - ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []), + ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []), ...reservedKeywords, ]), stringTypes: SparkFormatter.stringTypes, diff --git a/src/languages/sqlite.formatter.ts b/src/languages/sqlite.formatter.ts index f90c9eb366..0144445ac9 100644 --- a/src/languages/sqlite.formatter.ts +++ b/src/languages/sqlite.formatter.ts @@ -431,7 +431,7 @@ export default class SqliteFormatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, // https://www.sqlite.org/lang_keywords.html - reservedKeywords: [...standardReservedWords, ...nonStandardSqliteReservedWords], + reservedKeywords: standardReservedWords.concat(nonStandardSqliteReservedWords), stringTypes: SqliteFormatter.stringTypes, // https://www.sqlite.org/lang_expr.html#parameters indexedPlaceholderTypes: ['?'], diff --git a/src/languages/tsql.formatter.ts b/src/languages/tsql.formatter.ts index 09e0358361..2ed1bd62bd 100644 --- a/src/languages/tsql.formatter.ts +++ b/src/languages/tsql.formatter.ts @@ -1241,8 +1241,8 @@ export default class TSqlFormatter extends Formatter { reservedBinaryCommands, reservedDependentClauses, reservedKeywords: dedupe([ - ...Object.values(reservedFunctions).reduce((acc, arr) => [...acc, ...arr], []), - ...Object.values(reservedKeywords).reduce((acc, arr) => [...acc, ...arr], []), + ...Object.values(reservedFunctions).reduce((acc, arr) => acc.concat(arr), []), + ...Object.values(reservedKeywords).reduce((acc, arr) => acc.concat(arr), []), ]), stringTypes: TSqlFormatter.stringTypes, namedPlaceholderTypes: ['@'], From 87f786f4ad1000e1223eeae1abae52329651e293 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Sat, 4 Jun 2022 15:37:09 -0700 Subject: [PATCH 20/21] move regexFactory to lexer --- .eslintrc | 3 ++- src/{core/mooRegexFactory.ts => lexer/regexFactory.ts} | 0 src/lexer/tokenizer.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) rename src/{core/mooRegexFactory.ts => lexer/regexFactory.ts} (100%) diff --git a/.eslintrc b/.eslintrc index a839e59681..12feaec879 100644 --- a/.eslintrc +++ b/.eslintrc @@ -28,6 +28,7 @@ "no-useless-concat": "off", "no-restricted-syntax": "off", "no-constant-condition": "off", + "no-unused-expressions": "warn", "prefer-template": "off", "default-case": "off", "prettier/prettier": ["error"], @@ -36,7 +37,7 @@ "@typescript-eslint/indent": "off", "@typescript-eslint/lines-between-class-members": "off", "@typescript-eslint/naming-convention": "error", - "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-unused-vars": "warn", "@typescript-eslint/quotes": [ "error", "single", diff --git a/src/core/mooRegexFactory.ts b/src/lexer/regexFactory.ts similarity index 100% rename from src/core/mooRegexFactory.ts rename to src/lexer/regexFactory.ts diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts index 70bafa01b1..6f653ae349 100644 --- a/src/lexer/tokenizer.ts +++ b/src/lexer/tokenizer.ts @@ -1,7 +1,7 @@ import * as moo from 'moo'; import { TokenType } from 'src/core/token'; -import * as regex from 'src/core/mooRegexFactory'; +import * as regex from 'src/lexer/regexFactory'; interface TokenizerOptions { reservedKeywords: string[]; From 436c162d47c8ce94f3a526f0962d6f308ce52830 Mon Sep 17 00:00:00 2001 From: inferrinizzard Date: Sun, 5 Jun 2022 07:52:40 -0700 Subject: [PATCH 21/21] add converter between moo tokens and bespoke tokens --- src/lexer/tokenizer.ts | 26 +++++++++++++++++++++++++- test/moo.test.ts | 30 +++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/lexer/tokenizer.ts b/src/lexer/tokenizer.ts index 6f653ae349..5eed9b31f1 100644 --- a/src/lexer/tokenizer.ts +++ b/src/lexer/tokenizer.ts @@ -1,6 +1,6 @@ import * as moo from 'moo'; -import { TokenType } from 'src/core/token'; +import { Token, TokenType } from 'src/core/token'; import * as regex from 'src/lexer/regexFactory'; interface TokenizerOptions { @@ -138,3 +138,27 @@ export default class Tokenizer { return Array.from(this.LEXER); } } + +// temporary converter for moo.Token to Token +export const tokenConverter = (tokens: moo.Token[]): Token[] => { + const outTokens = [] as Token[]; + for (let i = 0; i < tokens.length; i++) { + // collect whitespaceBefore + let whitespace = ''; + while (tokens[i].type === 'WS' || tokens[i].type === 'NL') { + whitespace += tokens[i].value; + if (!tokens[++i]) { + break; + } + } + const token = tokens[i]; + + outTokens.push({ + type: token.type as TokenType, + text: token.text, + value: token.value, + whitespaceBefore: whitespace, + }); + } + return outTokens; +}; diff --git a/test/moo.test.ts b/test/moo.test.ts index d422b75fb4..0c7bdc7eca 100644 --- a/test/moo.test.ts +++ b/test/moo.test.ts @@ -6,7 +6,7 @@ import SqlFormatter, { reservedKeywords, } from 'src/languages/sql.formatter'; import Tokenizer from 'src/core/Tokenizer'; -import MooTokenizer from 'src/lexer/tokenizer'; +import MooTokenizer, { tokenConverter } from 'src/lexer/tokenizer'; import fs from 'fs'; @@ -18,7 +18,7 @@ const options = { reservedCommands, reservedBinaryCommands, reservedDependentClauses, - reservedLogicalOperators: ['AND', 'OR'], + // reservedLogicalOperators: ['AND', 'OR'], stringTypes: SqlFormatter.stringTypes as any[], // blockStart: blockStart, // blockEnd: blockEnd, @@ -36,11 +36,31 @@ describe('Moo', () => { const mooStream = mooTokenizer.tokenize(testSql); const filtered = mooStream.filter(token => token.type !== 'WS' && token.type !== 'NL'); - console.log('old:', stream.length, 'new:', filtered.length); + // console.log('old:', stream.length, 'new:', filtered.length); - console.log(filtered.map(({ type, text }) => ({ type, text }))); + // console.log(filtered.map(({ type, text, value }) => ({ type, text, value }))); - console.log(filtered.filter(token => token.type === 'WIP').length); + // console.log(filtered.filter(token => token.type === 'WIP').length); + + console.log(stream.length, tokenConverter(mooStream).length); + + console.log( + stream.map(({ type, text, value, whitespaceBefore }) => ({ + type, + text, + value, + ws: whitespaceBefore, + })) + ); + + console.log( + tokenConverter(mooStream).map(({ type, text, value, whitespaceBefore }) => ({ + type, + text, + value, + ws: whitespaceBefore, + })) + ); it('does not have any WIP tokens', () => { expect(filtered).not.toContainEqual(