Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/parse-anplusb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ export class ANplusBParser {
*/
parse_anplusb(start: number, end: number, line: number = 1): number | null {
this.expr_end = end
this.lexer.pos = start
this.lexer.line = line
this.lexer.seek(start, line)

let b: string | null = null
let a_start = start
Expand Down
8 changes: 2 additions & 6 deletions src/parse-atrule-prelude.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,7 @@ export class AtRulePreludeParser {
this.prelude_end = end

// Position lexer at prelude start
this.lexer.pos = start
this.lexer.line = line
this.lexer.column = column
this.lexer.seek(start, line, column)

return this.parse_prelude_dispatch(at_rule_name)
}
Expand Down Expand Up @@ -745,9 +743,7 @@ export class AtRulePreludeParser {
private parse_feature_value(start: number, end: number): number[] {
// Use a temporary lexer for this range to avoid corrupting main lexer position state
let temp_lexer = new Lexer(this.source)
temp_lexer.pos = start
temp_lexer.line = this.lexer.line
temp_lexer.column = this.lexer.column
temp_lexer.seek(start, this.lexer.line, this.lexer.column)

let nodes: number[] = []
let saved_lexer = this.lexer
Expand Down
4 changes: 1 addition & 3 deletions src/parse-declaration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,7 @@ export class DeclarationParser {
parse_declaration(start: number, end: number, line: number = 1, column: number = 1): number | null {
// Create a fresh lexer instance for standalone parsing
const lexer = new Lexer(this.source)
lexer.pos = start
lexer.line = line
lexer.column = column
lexer.seek(start, line, column)
lexer.next_token_fast(true) // skip whitespace like Parser does

return this.parse_declaration_with_lexer(lexer, end)
Expand Down
8 changes: 2 additions & 6 deletions src/parse-selector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,7 @@ export class SelectorParser {
this.selector_end = end

// Position lexer at selector start
this.lexer.pos = start
this.lexer.line = line
this.lexer.column = column
this.lexer.seek(start, line, column)

// Parse selector list (comma-separated selectors)
// Returns NODE_SELECTOR_LIST directly (no wrapper)
Expand Down Expand Up @@ -827,9 +825,7 @@ export class SelectorParser {
private parse_lang_identifiers(start: number, end: number, parent_node: number): void {
// Use a temporary lexer for this range to avoid corrupting main lexer position state
let temp_lexer = new Lexer(this.source)
temp_lexer.pos = start
temp_lexer.line = this.lexer.line
temp_lexer.column = this.lexer.column
temp_lexer.seek(start, this.lexer.line, this.lexer.column)

// Save current parser state
let saved_selector_end = this.selector_end
Expand Down
4 changes: 1 addition & 3 deletions src/parse-value.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,7 @@ export class ValueParser {
this.value_end = end

// Position lexer at value start with provided line/column
this.lexer.pos = start
this.lexer.line = start_line
this.lexer.column = start_column
this.lexer.seek(start, start_line, start_column)

// Parse individual value tokens
let value_nodes = this.parse_value_tokens()
Expand Down
54 changes: 36 additions & 18 deletions src/tokenize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,16 @@ import {
is_ident_start,
is_ident_char,
is_whitespace,
is_newline,
char_types,
CHAR_DIGIT,
CHAR_WHITESPACE,
CHAR_NEWLINE,
} from './char-types'

// Local inline version for hot paths that still need it
function is_newline(ch: number): boolean {
return ch < 128 && (char_types[ch] & CHAR_NEWLINE) !== 0
}
import {
TOKEN_IDENT,
TOKEN_FUNCTION,
Expand Down Expand Up @@ -74,6 +78,7 @@ export interface LexerPosition {
pos: number
line: number
column: number
_line_offset: number
token_type: TokenType
token_start: number
token_end: number
Expand All @@ -93,8 +98,8 @@ export interface CommentInfo {
export class Lexer {
source: string
pos: number
line: number
column: number
private _line: number
private _line_offset: number
on_comment: ((info: CommentInfo) => void) | undefined
// Current token properties (avoiding object allocation)
token_type: TokenType
Expand All @@ -106,8 +111,8 @@ export class Lexer {
constructor(source: string, on_comment?: (info: CommentInfo) => void) {
this.source = source
this.pos = 0
this.line = 1
this.column = 1
this._line = 1
this._line_offset = 0
this.on_comment = on_comment
this.token_type = TOKEN_EOF
this.token_start = 0
Expand All @@ -116,6 +121,20 @@ export class Lexer {
this.token_column = 1
}

get line(): number {
return this._line
}

get column(): number {
return this.pos - this._line_offset + 1
}

seek(pos: number, line: number, column: number = 1): void {
this.pos = pos
this._line = line
this._line_offset = pos - column + 1
}

// Fast token advancing without object allocation (for internal parser use)
next_token_fast(skip_whitespace: boolean = false): TokenType {
// Fast path: skip whitespace if requested
Expand Down Expand Up @@ -597,15 +616,14 @@ export class Lexer {
let ch = this.source.charCodeAt(this.pos)
this.pos++

if (is_newline(ch)) {
// Inline newline check - only update on newline
if (ch < 128 && (char_types[ch] & CHAR_NEWLINE) !== 0) {
// Handle \r\n as single newline
if (ch === CHAR_CARRIAGE_RETURN && this.pos < this.source.length && this.source.charCodeAt(this.pos) === CHAR_LINE_FEED) {
this.pos++
}
this.line++
this.column = 1
} else {
this.column++
this._line++
this._line_offset = this.pos
}
return
}
Expand All @@ -617,16 +635,15 @@ export class Lexer {
let ch = this.source.charCodeAt(this.pos)
this.pos++

if (is_newline(ch)) {
// Inline newline check - only update on newline
if (ch < 128 && (char_types[ch] & CHAR_NEWLINE) !== 0) {
// Handle \r\n as single newline
if (ch === CHAR_CARRIAGE_RETURN && this.pos < this.source.length && this.source.charCodeAt(this.pos) === CHAR_LINE_FEED) {
this.pos++
i++ // Count \r\n as 2 characters for advance(count)
}
this.line++
this.column = 1
} else {
this.column++
this._line++
this._line_offset = this.pos
}
}
}
Expand Down Expand Up @@ -665,8 +682,9 @@ export class Lexer {
save_position(): LexerPosition {
return {
pos: this.pos,
line: this.line,
line: this._line,
column: this.column,
_line_offset: this._line_offset,
token_type: this.token_type,
token_start: this.token_start,
token_end: this.token_end,
Expand All @@ -681,8 +699,8 @@ export class Lexer {
*/
restore_position(saved: LexerPosition): void {
this.pos = saved.pos
this.line = saved.line
this.column = saved.column
this._line = saved.line
this._line_offset = saved._line_offset
this.token_type = saved.token_type
this.token_start = saved.token_start
this.token_end = saved.token_end
Expand Down