2121"""
2222from __future__ import annotations
2323
24+ import heapq
2425import re
2526from dataclasses import dataclass , field
2627from typing import Any , Optional
2728
2829from engraphis .core .interfaces import Edge , Node
2930
3031# ── Regex NER (ported from engraphis/engines/ingest.py — the v1 heuristic path) ──
31- # Capitalized multi-word sequences, emails, URLs/hashtags, quoted names/mentions.
32- _ENTITY_RE = re .compile (
33- r"\b([A-Z][a-z]+(?:-[A-Za-z]+)*(?:\s+[A-Z][a-z]+(?:-[A-Za-z]+)*){0,3})\b"
34- r"|([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})"
35- r"|(#[a-zA-Z][a-zA-Z0-9_-]+)"
36- r"|(@[a-zA-Z][a-zA-Z0-9_-]+)"
32+ # Capitalized multi-word sequences, emails, hashtags, and mentions. Keep the
33+ # individual recognizers unambiguous: the old all-in-one expression could take
34+ # polynomial time while backtracking over a long email-like local part.
35+ _CAPITALIZED_WORD_RE = re .compile (r"\b[A-Z][a-z]+(?:-[A-Za-z]+)*\b" )
36+ _HASHTAG_RE = re .compile (r"#[a-zA-Z][a-zA-Z0-9_-]+" )
37+ _MENTION_RE = re .compile (r"@[a-zA-Z][a-zA-Z0-9_-]+" )
38+ _EMAIL_LOCAL_CHARS = frozenset (
39+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._%+-"
40+ )
41+ _EMAIL_DOMAIN_CHARS = frozenset (
42+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-"
43+ )
44+ _EMAIL_SUFFIX_CHARS = frozenset (
45+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
3746)
3847_RELATION_RE = re .compile (
3948 r"\b(?:is|are|was|were|has|have|had|owns|works at|lives in|prefers|likes|"
7988 "My" , "Our" , "Your" , "Their" , "His" , "Her" , "Its" }
8089
8190
91+ def _iter_emails (text : str ):
92+ """Yield email-like spans in one pass without regex backtracking."""
93+
94+ index = 0
95+ length = len (text )
96+ while index < length :
97+ if text [index ] not in _EMAIL_LOCAL_CHARS :
98+ index += 1
99+ continue
100+ if index > 0 and text [index - 1 ] in _EMAIL_LOCAL_CHARS :
101+ index += 1
102+ continue
103+ start = index
104+ while index < length and text [index ] in _EMAIL_LOCAL_CHARS :
105+ index += 1
106+ if index >= length or text [index ] != "@" :
107+ continue
108+ domain_start = index + 1
109+ end = domain_start
110+ while end < length and text [end ] in _EMAIL_DOMAIN_CHARS :
111+ end += 1
112+ domain = text [domain_start :end ]
113+ dot = domain .rfind ("." )
114+ suffix = domain [dot + 1 :] if dot > 0 else ""
115+ if len (suffix ) >= 2 and all (char in _EMAIL_SUFFIX_CHARS for char in suffix ):
116+ yield start , end , text [start :end ]
117+ index = domain_start if end < length and text [end ] == "@" else end
118+
119+
82120def _defang (value : str ) -> str :
83121 return _CONTROL_RE .sub ("" , value or "" ).strip ()[:_MAX_NAME ]
84122
@@ -105,24 +143,71 @@ class GraphExtraction:
105143
106144
107145def _extract_entities (text : str ) -> list [tuple [str , str ]]:
146+ text = text or ""
147+
148+ def capitalized_candidates ():
149+ # Assemble at most four whitespace-separated capitalized words, matching the
150+ # former heuristic without placing a nested repetition over untrusted text.
151+ words = iter (_CAPITALIZED_WORD_RE .finditer (text ))
152+ current = next (words , None )
153+ while current is not None :
154+ first = last = current
155+ current = None
156+ for _ in range (3 ):
157+ following = next (words , None )
158+ if following is None :
159+ break
160+ if text [last .end ():following .start ()].isspace ():
161+ last = following
162+ continue
163+ current = following
164+ break
165+ else :
166+ current = next (words , None )
167+ yield (
168+ first .start (),
169+ 0 ,
170+ last .end (),
171+ text [first .start ():last .end ()],
172+ "person_or_concept" ,
173+ )
174+
175+ # Merge four already-position-ordered iterators. This keeps memory bounded even
176+ # for very large untrusted input and lets the entity fanout cap stop collection.
177+ candidates = heapq .merge (
178+ capitalized_candidates (),
179+ (
180+ (start , 1 , end , value , "email" )
181+ for start , end , value in _iter_emails (text )
182+ ),
183+ (
184+ (match .start (), 2 , match .end (), match .group (0 ), "hashtag" )
185+ for match in _HASHTAG_RE .finditer (text )
186+ ),
187+ (
188+ (match .start (), 3 , match .end (), match .group (0 ), "mention" )
189+ for match in _MENTION_RE .finditer (text )
190+ ),
191+ )
192+
108193 seen : set [str ] = set ()
109194 out : list [tuple [str , str ]] = []
110- for m in _ENTITY_RE .finditer (text or "" ):
111- raw = (m .group (0 ) or "" ).strip ()
195+ matched_through = 0
196+ for start , _priority , end , candidate , etype in candidates :
197+ if start < matched_through :
198+ continue
199+ matched_through = end
200+ raw = candidate .strip ()
112201 if not raw or raw .casefold () in _STOPWORD_KEYS or raw .casefold () in (
113202 "user" , "the user"
114203 ):
115204 continue
116- if raw .startswith ("#" ):
117- ent , etype = raw , "hashtag"
118- elif raw .startswith ("@" ):
119- ent , etype = raw , "mention"
120- elif "@" in raw and "." in raw :
121- ent , etype = raw , "email"
122- else :
205+ if etype == "person_or_concept" :
123206 ent , etype = _canon_concept (raw ), "person_or_concept"
124207 if len (ent ) < 2 or ent .casefold () in _STOPWORD_KEYS :
125208 continue
209+ else :
210+ ent = raw
126211 key = ent .lower ()
127212 if key not in seen :
128213 seen .add (key )
0 commit comments