Skip to content

Commit cebddbd

Browse files
authored
Commit 7: Implement POSIX collating symbol and equivalence class support
Add posixCollatingSymbol([.x.]) and posixEquivalenceClass([=x=]) predicates to ParseRegExp.qll, enabling correct tokenization of all three POSIX bracket atom types inside character classes: - [:name:] — already handled, unchanged - [.x.] — new: posixCollatingSymbol predicate - [=x=] — new: posixEquivalenceClass predicate Refactored: - charSetDelimiter: use inAnyPosixBracket helper (covers all three types) - charSet closing: use inAnyPosixBracket helper - inPosixBracket: updated to cover all three types - simpleCharacter: updated to exclude all three types - namedCharacterProperty: includes posixCollatingSymbol and posixEquivalenceClass - inAnyPosixBracket: new private helper; uses pos in [x..y-1] for QL binding Before this commit (commit 6): [[.a.]] → RegExpCharacterClass([[.a.]) + stray ] (WRONG) [[=a=]] → RegExpCharacterClass([[=a=]) + stray ] (WRONG) After this commit: [[.a.]] → RegExpCharacterClass containing RegExpNamedCharacterProperty [.a.] [[=a=]] → RegExpCharacterClass containing RegExpNamedCharacterProperty [=a=] Regenerated parse.expected and regexp.expected via codeql test run --learn (CodeQL 2.26.1); all 2 tests pass.
1 parent 9fd7e94 commit cebddbd

3 files changed

Lines changed: 76 additions & 75 deletions

File tree

cpp/ql/lib/semmle/code/cpp/regex/internal/ParseRegExp.qll

Lines changed: 66 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -102,11 +102,9 @@ class RegExp extends StringLiteral {
102102
pos =
103103
rank[index](int p |
104104
(this.nonEscapedCharAt(p) = "[" or this.nonEscapedCharAt(p) = "]") and
105-
// Brackets that art part of POSIX expressions should not count as
105+
// Brackets that are part of POSIX expressions should not count as
106106
// char-set delimiters.
107-
not exists(int x, int y |
108-
this.posixStyleNamedCharacterProperty(x, y, _) and pos >= x and pos < y
109-
)
107+
not this.inAnyPosixBracket(p)
110108
) and
111109
(
112110
this.nonEscapedCharAt(pos) = "[" and result = true
@@ -136,9 +134,7 @@ class RegExp extends StringLiteral {
136134
min(int e |
137135
e > innerStart and
138136
this.nonEscapedCharAt(e) = "]" and
139-
not exists(int x, int y |
140-
this.posixStyleNamedCharacterProperty(x, y, _) and e >= x and e < y
141-
)
137+
not this.inAnyPosixBracket(e)
142138
|
143139
e
144140
)
@@ -289,14 +285,31 @@ class RegExp extends StringLiteral {
289285
/** Matches named character properties such as `\p{Word}` and `[[:digit:]]` */
290286
predicate namedCharacterProperty(int start, int end, string name) {
291287
this.pStyleNamedCharacterProperty(start, end, name) or
292-
this.posixStyleNamedCharacterProperty(start, end, name)
288+
this.posixStyleNamedCharacterProperty(start, end, name) or
289+
this.posixCollatingSymbol(start, end, name) or
290+
this.posixEquivalenceClass(start, end, name)
293291
}
294292

295293
/** Gets the name of the character property in start,end */
296294
string getCharacterPropertyName(int start, int end) {
297295
this.namedCharacterProperty(start, end, result)
298296
}
299297

298+
/**
299+
* Holds if the position `pos` falls inside any POSIX bracket atom
300+
* (`[:name:]`, `[.x.]`, or `[=x=]`).
301+
* Used to prevent the inner brackets from being treated as charSet delimiters.
302+
*/
303+
private predicate inAnyPosixBracket(int pos) {
304+
exists(int x, int y |
305+
this.posixStyleNamedCharacterProperty(x, y, _) or
306+
this.posixCollatingSymbol(x, y, _) or
307+
this.posixEquivalenceClass(x, y, _)
308+
|
309+
pos in [x .. y - 1]
310+
)
311+
}
312+
300313
/** Matches a POSIX bracket expression such as `[:alnum:]` within a character class. */
301314
private predicate posixStyleNamedCharacterProperty(int start, int end, string name) {
302315
this.getChar(start) = "[" and
@@ -318,6 +331,42 @@ class RegExp extends StringLiteral {
318331
)
319332
}
320333

334+
/**
335+
* Matches a POSIX collating symbol such as `[.a.]` within a character class.
336+
* Example: `[[.a.]]` — the `[.a.]` atom nested inside the outer bracket.
337+
*/
338+
private predicate posixCollatingSymbol(int start, int end, string name) {
339+
this.getChar(start) = "[" and
340+
this.getChar(start + 1) = "." and
341+
end =
342+
min(int e |
343+
e > start and
344+
this.getChar(e - 2) = "." and
345+
this.getChar(e - 1) = "]"
346+
|
347+
e
348+
) and
349+
name = this.getText().substring(start + 2, end - 2)
350+
}
351+
352+
/**
353+
* Matches a POSIX equivalence class such as `[=a=]` within a character class.
354+
* Example: `[[=a=]]` — the `[=a=]` atom nested inside the outer bracket.
355+
*/
356+
private predicate posixEquivalenceClass(int start, int end, string name) {
357+
this.getChar(start) = "[" and
358+
this.getChar(start + 1) = "=" and
359+
end =
360+
min(int e |
361+
e > start and
362+
this.getChar(e - 2) = "=" and
363+
this.getChar(e - 1) = "]"
364+
|
365+
e
366+
) and
367+
name = this.getText().substring(start + 2, end - 2)
368+
}
369+
321370
/**
322371
* Matches named character properties. For example:
323372
* - `\p{Space}`
@@ -398,11 +447,17 @@ class RegExp extends StringLiteral {
398447
}
399448

400449
/**
401-
* Holds if the character at `index` is inside a posix bracket.
450+
* Holds if the character at `index` is inside a posix bracket
451+
* (`[:name:]`, `[.x.]`, or `[=x=]`).
402452
*/
403453
predicate inPosixBracket(int index) {
404454
exists(int x, int y |
405-
this.posixStyleNamedCharacterProperty(x, y, _) and index in [x + 1 .. y - 2]
455+
(
456+
this.posixStyleNamedCharacterProperty(x, y, _) or
457+
this.posixCollatingSymbol(x, y, _) or
458+
this.posixEquivalenceClass(x, y, _)
459+
) and
460+
index in [x + 1 .. y - 2]
406461
)
407462
}
408463

@@ -413,11 +468,7 @@ class RegExp extends StringLiteral {
413468
end = start + 1 and
414469
not this.charSet(start, _) and
415470
not this.charSet(_, start + 1) and
416-
not exists(int x, int y |
417-
this.posixStyleNamedCharacterProperty(x, y, _) and
418-
start >= x and
419-
end <= y
420-
) and
471+
not this.inAnyPosixBracket(start) and
421472
exists(string c | c = this.getChar(start) |
422473
exists(int x, int y, int z |
423474
this.charSet(x, z) and

cpp/ql/test/library-tests/regex/parse.expected

Lines changed: 6 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -245,45 +245,15 @@ test.cpp:
245245

246246
# 76| [RegExpNamedCharacterProperty] [:space:]
247247

248-
# 79| [RegExpCharacterClass] [[.a.]
249-
#-----| 0 -> [RegExpConstant, RegExpNormalChar] [
250-
#-----| 1 -> [RegExpConstant, RegExpNormalChar] .
251-
#-----| 2 -> [RegExpConstant, RegExpNormalChar] a
252-
#-----| 3 -> [RegExpConstant, RegExpNormalChar] .
253-
254-
# 79| [RegExpSequence] [[.a.]]
255-
#-----| 0 -> [RegExpCharacterClass] [[.a.]
256-
#-----| 1 -> [RegExpConstant, RegExpNormalChar] ]
257-
258-
# 79| [RegExpConstant, RegExpNormalChar] [
259-
260-
# 79| [RegExpConstant, RegExpNormalChar] .
261-
262-
# 79| [RegExpConstant, RegExpNormalChar] a
263-
264-
# 79| [RegExpConstant, RegExpNormalChar] .
265-
266-
# 79| [RegExpConstant, RegExpNormalChar] ]
267-
268-
# 82| [RegExpCharacterClass] [[=a=]
269-
#-----| 0 -> [RegExpConstant, RegExpNormalChar] [
270-
#-----| 1 -> [RegExpConstant, RegExpNormalChar] =
271-
#-----| 2 -> [RegExpConstant, RegExpNormalChar] a
272-
#-----| 3 -> [RegExpConstant, RegExpNormalChar] =
273-
274-
# 82| [RegExpSequence] [[=a=]]
275-
#-----| 0 -> [RegExpCharacterClass] [[=a=]
276-
#-----| 1 -> [RegExpConstant, RegExpNormalChar] ]
277-
278-
# 82| [RegExpConstant, RegExpNormalChar] [
279-
280-
# 82| [RegExpConstant, RegExpNormalChar] =
248+
# 79| [RegExpCharacterClass] [[.a.]]
249+
#-----| 0 -> [RegExpNamedCharacterProperty] [.a.]
281250

282-
# 82| [RegExpConstant, RegExpNormalChar] a
251+
# 79| [RegExpNamedCharacterProperty] [.a.]
283252

284-
# 82| [RegExpConstant, RegExpNormalChar] =
253+
# 82| [RegExpCharacterClass] [[=a=]]
254+
#-----| 0 -> [RegExpNamedCharacterProperty] [=a=]
285255

286-
# 82| [RegExpConstant, RegExpNormalChar] ]
256+
# 82| [RegExpNamedCharacterProperty] [=a=]
287257

288258
# 85| [RegExpCharacterClass] [[:alpha:]0-9]
289259
#-----| 0 -> [RegExpNamedCharacterProperty] [:alpha:]

cpp/ql/test/library-tests/regex/regexp.expected

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -100,20 +100,10 @@ term
100100
| test.cpp:76:29:76:40 | [a[:space:]] | RegExpCharacterClass |
101101
| test.cpp:76:30:76:30 | a | RegExpConstant,RegExpNormalChar |
102102
| test.cpp:76:31:76:39 | [:space:] | RegExpNamedCharacterProperty |
103-
| test.cpp:79:28:79:33 | [[.a.] | RegExpCharacterClass |
104-
| test.cpp:79:28:79:34 | [[.a.]] | RegExpSequence |
105-
| test.cpp:79:29:79:29 | [ | RegExpConstant,RegExpNormalChar |
106-
| test.cpp:79:30:79:30 | . | RegExpConstant,RegExpNormalChar |
107-
| test.cpp:79:31:79:31 | a | RegExpConstant,RegExpNormalChar |
108-
| test.cpp:79:32:79:32 | . | RegExpConstant,RegExpNormalChar |
109-
| test.cpp:79:34:79:34 | ] | RegExpConstant,RegExpNormalChar |
110-
| test.cpp:82:29:82:34 | [[=a=] | RegExpCharacterClass |
111-
| test.cpp:82:29:82:35 | [[=a=]] | RegExpSequence |
112-
| test.cpp:82:30:82:30 | [ | RegExpConstant,RegExpNormalChar |
113-
| test.cpp:82:31:82:31 | = | RegExpConstant,RegExpNormalChar |
114-
| test.cpp:82:32:82:32 | a | RegExpConstant,RegExpNormalChar |
115-
| test.cpp:82:33:82:33 | = | RegExpConstant,RegExpNormalChar |
116-
| test.cpp:82:35:82:35 | ] | RegExpConstant,RegExpNormalChar |
103+
| test.cpp:79:28:79:34 | [[.a.]] | RegExpCharacterClass |
104+
| test.cpp:79:29:79:33 | [.a.] | RegExpNamedCharacterProperty |
105+
| test.cpp:82:29:82:35 | [[=a=]] | RegExpCharacterClass |
106+
| test.cpp:82:30:82:34 | [=a=] | RegExpNamedCharacterProperty |
117107
| test.cpp:85:29:85:42 | [[:alpha:]0-9] | RegExpCharacterClass |
118108
| test.cpp:85:30:85:38 | [:alpha:] | RegExpNamedCharacterProperty |
119109
| test.cpp:85:39:85:39 | 0 | RegExpConstant,RegExpNormalChar |
@@ -252,16 +242,6 @@ regExpNormalCharValue
252242
| test.cpp:57:28:57:28 | f | f |
253243
| test.cpp:57:30:57:33 | A-F] | A-F] |
254244
| test.cpp:76:30:76:30 | a | a |
255-
| test.cpp:79:29:79:29 | [ | [ |
256-
| test.cpp:79:30:79:30 | . | . |
257-
| test.cpp:79:31:79:31 | a | a |
258-
| test.cpp:79:32:79:32 | . | . |
259-
| test.cpp:79:34:79:34 | ] | ] |
260-
| test.cpp:82:30:82:30 | [ | [ |
261-
| test.cpp:82:31:82:31 | = | = |
262-
| test.cpp:82:32:82:32 | a | a |
263-
| test.cpp:82:33:82:33 | = | = |
264-
| test.cpp:82:35:82:35 | ] | ] |
265245
| test.cpp:85:39:85:39 | 0 | 0 |
266246
| test.cpp:85:41:85:41 | 9 | 9 |
267247
| test.cpp:89:23:89:24 | \\w | w |

0 commit comments

Comments
 (0)