perf: inline is_ident_char() in hot tokenizer loops via CHAR_IDENT bi…

src/char-types.ts

@@ -26,6 +26,7 @@ export let CHAR_DIGIT = 1 << 1 // 2
 export let CHAR_HEX = 1 << 2 // 4
 export let CHAR_WHITESPACE = 1 << 3 // 8
 export let CHAR_NEWLINE = 1 << 4 // 16
+export let CHAR_IDENT = 1 << 5 // 32
 
 // Lookup table for ASCII characters (0-127)
 export let char_types = new Uint8Array(128)
@@ -63,6 +64,16 @@ char_types[0x0a] = CHAR_NEWLINE // \n
 char_types[0x0d] = CHAR_NEWLINE // \r
 char_types[0x0c] = CHAR_NEWLINE // \f
 
+// Initialize ident characters: letters, digits, hyphen (-), underscore (_)
+// Derived from already-populated table so it stays consistent with CHAR_ALPHA/CHAR_DIGIT
+for (let i = 0; i < 128; i++) {
+	if (char_types[i] & (CHAR_ALPHA | CHAR_DIGIT)) {
+		char_types[i] |= CHAR_IDENT
+	}
+}
+char_types[0x2d] |= CHAR_IDENT // hyphen -
+char_types[0x5f] |= CHAR_IDENT // underscore _
+
 export function is_digit(ch: number): boolean {
 	return ch < 128 && (char_types[ch] & CHAR_DIGIT) !== 0
 }

src/tokenize.ts

@@ -1,12 +1,12 @@
 import {
 	is_hex_digit,
 	is_ident_start,
-	is_ident_char,
 	is_whitespace,
 	char_types,
 	CHAR_DIGIT,
 	CHAR_WHITESPACE,
 	CHAR_NEWLINE,
+	CHAR_IDENT,
 } from './char-types'
 
 // Local inline version for hot paths that still need it
@@ -454,7 +454,10 @@ export class Lexer {
 			}
 			if (is_ident_start(ch) || (ch === CHAR_HYPHEN && is_ident_start(this.peek()))) {
 				// Unit: px, em, rem, etc
-				while (this.pos < this.source.length && is_ident_char(this.source.charCodeAt(this.pos))) {
+				// Hot path: inline ident char check in tight loop
+				while (this.pos < this.source.length) {
+					let ch = this.source.charCodeAt(this.pos)
+					if (ch < 0x80 && (char_types[ch] & CHAR_IDENT) === 0) break
 					this.advance()
 				}
 				return this.make_token(TOKEN_DIMENSION, start, this.pos, start_line, start_column)
@@ -502,8 +505,8 @@ export class Lexer {
 					// Escape any other character (except newline, already checked)
 					this.advance()
 				}
-			} else if (is_ident_char(ch)) {
-				// Normal identifier character
+			} else if (ch >= 0x80 || (char_types[ch] & CHAR_IDENT) !== 0) {
+				// Normal identifier character — Hot path: inline ident char check
 				this.advance()
 			} else {
 				// Not part of identifier
@@ -589,7 +592,10 @@ export class Lexer {
 		this.advance() // Skip @
 
 		// Consume identifier
-		while (this.pos < this.source.length && is_ident_char(this.source.charCodeAt(this.pos))) {
+		// Hot path: inline ident char check in tight loop
+		while (this.pos < this.source.length) {
+			let ch = this.source.charCodeAt(this.pos)
+			if (ch < 0x80 && (char_types[ch] & CHAR_IDENT) === 0) break
 			this.advance()
 		}
 
@@ -601,7 +607,10 @@ export class Lexer {
 		this.advance() // Skip #
 
 		// Consume identifier or hex digits
-		while (this.pos < this.source.length && is_ident_char(this.source.charCodeAt(this.pos))) {
+		// Hot path: inline ident char check in tight loop
+		while (this.pos < this.source.length) {
+			let ch = this.source.charCodeAt(this.pos)
+			if (ch < 0x80 && (char_types[ch] & CHAR_IDENT) === 0) break
 			this.advance()
 		}