From 77afd92448102a757d2692bc123997dc8d3eb1f3 Mon Sep 17 00:00:00 2001 From: danielogen Date: Sat, 4 Oct 2025 07:32:37 -0700 Subject: [PATCH 01/19] refactor: add base class and auto registry classes using lazy loading --- src/PyReprism/languages/__init__.py | 196 +++++++--------------------- src/PyReprism/languages/base.py | 68 ++++++++++ src/PyReprism/languages/registry.py | 28 ++++ 3 files changed, 146 insertions(+), 146 deletions(-) create mode 100644 src/PyReprism/languages/base.py create mode 100644 src/PyReprism/languages/registry.py diff --git a/src/PyReprism/languages/__init__.py b/src/PyReprism/languages/__init__.py index 84faf0f..9803fda 100644 --- a/src/PyReprism/languages/__init__.py +++ b/src/PyReprism/languages/__init__.py @@ -1,147 +1,51 @@ __version__ = "0.0.3" -from .python import * -from .java import * -from .c import * -from .csharp import * -from .bash import * -from .markup import * -from .perl import * -from .php import * -from .ruby import * -from .rust import * -from .abap import * -from .actionscript import * -from .ada import * -from .apacheconf import * -from .apl import * -from .applescript import * -from .arduino import * -from .arff import * -from .asciidoc import * -from .asm6502 import * -from .aspnet import * -from .autohotkey import * -from .autoit import * -from .bash import * -from .basic import * -from .batch import * -from .bison import * -from .brainfuck import * -from .bro import * -from .clike import * -from .clojure import * -from .coffeescript import * -from .cpp import * -from .crystal import * -from .csp import * -from .css_extras import * -from .css import * -from .d import * -from .dart import * -from .diff import * -from .django import * -from .docker import * -from .eiffel import * -from .elixir import * -from .erb import * -from .erlang import * -from .flow import * -from .fortran import * -from .fsharp import * -from .gedcom import * -from .gherkin import * -from .git import * -from .glsl import * -from .go import * -from .graphql import * -from .groovy import * -from .haml import * -from .handlebars import * -from .haskell import * -from .haxe import * -from .hpkp import * -from .hsts import * -from .ichigojam import * -from .icon import * -from .inform7 import * -from .ini import * -from .io import * -from .j import * -from .javascript import * -from .jolie import * -from .json import * -from .jsx import * -from .julia import * -from .keyman import * -from .kotlin import * -from .latex import * -from .less import * -from .liquid import * -from .livescript import * -from .lolcode import * -from .lua import * -from .makefile import * -from .markdown import * -from .markup_templating import * -from .markup import * -from .matlab import * -from .mel import * -from .mizar import * -from .monkey import * -from .n4js import * -from .nasm import * -from .nginx import * -from .nim import * -from .nix import * -from .nsis import * -from .objectivec import * -from .ocaml import * -from .opencl import * -from .oz import * -from .parigp import * -from .parser import * -from .pascal import * -from .perl import * -from .php_extras import * -from .plsql import * -from .powershell import * -from .processing import * -from .prolog import * -from .properties import * -from .protobuf import * -from .pug import * -from .puppet import * -from .pure import * -from .q import * -from .qore import * -from .r import * -from .reason import * -from .renpy import * -from .rest import * -from .rip import * -from .roboconf import * -from .sas import * -from .sass import * -from .scala import * -from .scheme import * -from .scss import * -from .smalltalk import * -from .smarty import * -from .soy import * -from .stylus import * -from .swift import * -from .tcl import * -from .textile import * -from .tsx import * -from .twig import * -from .typescript import * -from .vbnet import * -from .velocity import * -from .verilog import * -from .vhdl import * -from .vim import * -from .visual_basic import * -from .wasam import * -from .xeora import * -from .xojo import * -from .yaml import * + +# Minimal lazy loader / registry bridge for language modules. +import importlib +from typing import Any + +from .registry import LanguageRegistry + + +# Pre-declare an __all__ mapping of known language module names (optional). +# Keep it small and let modules register themselves via LanguageRegistry. +__all__ = [ + # common names; modules register on import + 'Python', 'JavaScript', 'CPP', 'C', 'Clike', 'Go', 'MatLab', 'Ruby', 'PHP', 'Bash' +] + + +def __getattr__(name: str) -> Any: + """Lazy-load language module attribute by name. + + Accessing e.g. `PyReprism.languages.Python` will import the submodule + `PyReprism.languages.python` and return the class object if present. + """ + # If already registered, return directly + cls = LanguageRegistry.get(name) + if cls: + return cls + + # Try to import the submodule named by lowercasing the name + mod_name = name.lower() + try: + importlib.import_module(f'.{mod_name}', __name__) + except ModuleNotFoundError: + raise AttributeError(f"module {__name__} has no attribute {name}") + + cls = LanguageRegistry.get(name) + if cls: + return cls + raise AttributeError(f"module {__name__} has no attribute {name}") + + +def get_language_by_extension(ext: str): + """Return the first registered language class that reports the given file extension.""" + for name, cls in LanguageRegistry.all().items(): + try: + if cls.file_extension() == ext: + return cls + except Exception: + continue + return None + diff --git a/src/PyReprism/languages/base.py b/src/PyReprism/languages/base.py new file mode 100644 index 0000000..2e69982 --- /dev/null +++ b/src/PyReprism/languages/base.py @@ -0,0 +1,68 @@ +import re +from functools import lru_cache +from typing import List, Pattern, Union + + +class BaseLanguage: + """Minimal base class for language implementations. + + Subclasses should provide: file_extension(), keywords(), comment_regex(), + and may override number_regex(), operator_regex(), keywords_regex(). + + This base centralizes implementations for remove_comments and remove_keywords + and caches commonly used compiled regexes. + """ + + @classmethod + def file_extension(cls) -> str: + raise NotImplementedError() + + @classmethod + def keywords(cls) -> List[str]: + return [] + + @classmethod + @lru_cache(maxsize=None) + def keywords_regex(cls) -> Pattern: + words = cls.keywords() or [] + if not words: + # match nothing + return re.compile(r"^$") + return re.compile(r'\b(' + '|'.join(words) + r')\b', re.IGNORECASE) + + @classmethod + def comment_regex(cls) -> Pattern: + """Return a compiled regex that contains named groups 'comment' and 'noncomment'.""" + raise NotImplementedError() + + @classmethod + def number_regex(cls) -> Pattern: + return re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?') + + @classmethod + def operator_regex(cls) -> Pattern: + return re.compile(r'.') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> Union[str, List[str]]: + """Strip comments using the language's comment_regex. + + The regex is expected to provide a 'noncomment' named group for text to keep. + Returns either the joined string or a list of non-comment segments when isList=True. + """ + pattern = cls.comment_regex() + result = [] + for match in pattern.finditer(source_code): + non = match.groupdict().get('noncomment') + # Append the 'noncomment' group even when it's an empty string. + # Use `is not None` to avoid skipping valid empty segments which + # may carry whitespace that should be preserved. + if non is not None: + result.append(non) + if isList: + return result + return ''.join(result) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/registry.py b/src/PyReprism/languages/registry.py new file mode 100644 index 0000000..ed296de --- /dev/null +++ b/src/PyReprism/languages/registry.py @@ -0,0 +1,28 @@ +from typing import Dict, Type + +from .base import BaseLanguage + + +class LanguageRegistry: + _registry: Dict[str, Type[BaseLanguage]] = {} + + @classmethod + def register(cls, language_cls: Type[BaseLanguage]): + cls._registry[language_cls.__name__] = language_cls + return language_cls + + @classmethod + def get(cls, name: str): + # If not registered yet, try lazy-importing the module by lowercase name + if name not in cls._registry: + try: + import importlib + importlib.import_module(f'.{name.lower()}', 'PyReprism.languages') + except Exception: + # import failed or module doesn't exist; fall through + pass + return cls._registry.get(name) + + @classmethod + def all(cls): + return dict(cls._registry) From 1f288a5fef213d4204d90a11e54a01237946e57f Mon Sep 17 00:00:00 2001 From: danielogen Date: Sat, 4 Oct 2025 07:33:57 -0700 Subject: [PATCH 02/19] refactor: configure languages to use base class and registry module --- src/PyReprism/languages/bash.py | 70 ++++++------ src/PyReprism/languages/c.py | 135 ++++++++--------------- src/PyReprism/languages/clike.py | 70 ++++++------ src/PyReprism/languages/cpp.py | 116 +++++--------------- src/PyReprism/languages/go.py | 97 ++++++----------- src/PyReprism/languages/javascript.py | 148 +++++++------------------- src/PyReprism/languages/matlab.py | 64 ++++------- src/PyReprism/languages/php.py | 137 ++++++------------------ src/PyReprism/languages/php_extras.py | 17 +++ src/PyReprism/languages/python.py | 45 ++++++++ src/PyReprism/languages/ruby.py | 70 ++++++------ 11 files changed, 344 insertions(+), 625 deletions(-) diff --git a/src/PyReprism/languages/bash.py b/src/PyReprism/languages/bash.py index add6895..ebed0a7 100644 --- a/src/PyReprism/languages/bash.py +++ b/src/PyReprism/languages/bash.py @@ -1,49 +1,39 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Bash: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Bash(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.bash - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0x[\dA-Fa-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'--?|-=|\+\+?|\+=|!=?|~|\*\*?|\*=|\/=?|%=?|<<=?|>>=?|<=?|>=?|==?|&&?|&=|\^=?|\|\|?|\|=|\?|:') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Bash.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Bash.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def keywords(cls) -> list: + return ''.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P#.*?$)|(?P'"'(\\.|[^\\'])*'"'|"(\\.|[^\\"])*"|.[^#\'\"]*)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b0x[\dA-Fa-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?') + + @classmethod + def operator_regex(cls): + return re.compile(r'--?|-=|\+\+?|\+=|!=?|~|\*\*?|\*=|\/?=|%=?|<<=?|>>=?|<=?|>=?|==?|&&?|&=|\^=?|\|\|?|\|=|\?|:') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) if isList: - return result - return ''.join(result) + return res + return ''.join(res) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Bash.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/c.py b/src/PyReprism/languages/c.py index 5d8ea85..c6f6246 100644 --- a/src/PyReprism/languages/c.py +++ b/src/PyReprism/languages/c.py @@ -1,113 +1,64 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class C: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - """ - Return the file extension used for C files. - - :return: The file extension for C files. - :rtype: str - """ +@LanguageRegistry.register +class C(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.c - @staticmethod - def keywords() -> list: - """ - Return a list of C keywords and built-in functions. + @classmethod + def keywords(cls) -> list: + return '_Alignas|_Alignof|_Atomic|_Bool|_Complex|_Generic|_Imaginary|_Noreturn|_Static_assert|_Thread_local|asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while|define|defined|elif|else|endif|error|ifdef|ifndef|if|import|include|line|pragma|undef|using|__FILE__|__LINE__|__DATE__|__TIME__|__TIMESTAMP__|__func__|EOF|NULL|SEEK_CUR|SEEK_END|SEEK_SET|stdin|stdout|stderr'.split('|') - :return: A list of C keywords and built-in function names. - :rtype: list + @classmethod + def comment_regex(cls) -> re.Pattern: """ - keyword = '_Alignas|_Alignof|_Atomic|_Bool|_Complex|_Generic|_Imaginary|_Noreturn|_Static_assert|_Thread_local|asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while|define|defined|elif|else|endif|error|ifdef|ifndef|if|import|include|line|pragma|undef|using|__FILE__|__LINE__|__DATE__|__TIME__|__TIMESTAMP__|__func__|EOF|NULL|SEEK_CUR|SEEK_END|SEEK_SET|stdin|stdout|stderr'.split('|') - return keyword + Returns a compiled regex pattern to match comments and non-comment parts in C source code. + This pattern matches both single-line (//) and multi-line (/* ... */) comments - @staticmethod - def comment_regex() -> re.Pattern: + :returns: A compiled regex pattern. + :type: re.Pattern """ - Compile and return a regular expression pattern to identify different types of comments and non-comment code in C source files. + return re.compile(r'(?P//.*?$|/\*[^*]*\*+(?:[^/*][^*]*\*+)*?/)|(?P[^/]+)', re.DOTALL | re.MULTILINE) - :return: A compiled regex pattern with named groups to match single-line comments, multiline comments, and non-comment code elements. - :rtype: re.Pattern + @classmethod + def number_regex(cls) -> re.Pattern: """ - pattern = re.compile(r'(?P//.*?$|/\*[^*]*\*+(?:[^/*][^*]*\*+)*?/)|(?P[^/]+)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex() -> re.Pattern: + Returns a compiled regex pattern to match numeric literals in C source code. + This pattern matches hexadecimal, decimal, and floating-point numbers + + :returns: A compiled regex pattern. + :type: re.Pattern """ - Compile and return a regular expression pattern to identify numeric literals in C code. + return re.compile(r'(?:\b0x[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?)[ful]*') - :return: A compiled regex pattern to match C numeric literals, including integers, floats, and complex numbers. - :rtype: re.Pattern + @classmethod + def operator_regex(cls) -> re.Pattern: """ - pattern = re.compile(r'(?:\b0x[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?)[ful]*') - return pattern - - @staticmethod - def operator_regex() -> re.Pattern: + Returns a compiled regex pattern to match operators in C source code. + This pattern matches various C operators including arithmetic, comparison, and logical operators. + + :returns: A compiled regex pattern. + :type: re.Pattern """ - Compile and return a regular expression pattern to identify C operators. + return re.compile(r'-[>-]?|\+\+?|!=?|<>?=?|==?|&&?|\|\|?|[~^%?*\/]') - :return: A compiled regex pattern to match various C operators and logical keywords. - :rtype: re.Pattern - """ - pattern = re.compile(r'-[>-]?|\+\+?|!=?|<>?=?|==?|&&?|\|\|?|[~^%?*\/]') - return pattern - - @staticmethod - def keywords_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify C keywords. - - :return: A compiled regex pattern to match C keywords. - :rtype: re.Pattern - """ - return re.compile(r'\b(' + '|'.join(C.keywords()) + r')\b') - - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify C boolean literals. - - :return: A compiled regex pattern to match C boolean literals. - :rtype: re.Pattern - """ - return re.compile(r'\b(?:true|false)\b') - - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify C and C++ delimiters. - - :return: A compiled regex pattern to match C and C++ delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@<>*&]') - @staticmethod - def remove_comments(source_code: str) -> str: - """ - Remove comments from the provided C source code string. - - :param str source_code: The C source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return C.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) + if isList: + return res + return res.strip() - @staticmethod - def remove_keywords(source: str) -> str: - """ - Remove all C keywords from the provided source code string. - - :param str source: The source code string from which to remove C keywords. - :return: The source code string with all C keywords removed. - :rtype: str - """ - return re.sub(re.compile(C.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/clike.py b/src/PyReprism/languages/clike.py index c362a94..7ea0b5f 100644 --- a/src/PyReprism/languages/clike.py +++ b/src/PyReprism/languages/clike.py @@ -1,49 +1,39 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Clike: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Clike(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.clike - @staticmethod - def keywords() -> list: - keyword = 'if|else|while|do|for|return|in|instanceof|function|new|try|throw|catch|finally|null|break|continue|true|false|class|interface|extends|implements|trait|instanceof|new'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'--?|\+\+?|!=?=?|<=?|>=?|==?=?|&&?|\|\|?|\?|\*|\/|~|\^|%') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Clike.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Clike.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def keywords(cls) -> list: + return 'if|else|while|do|for|return|in|instanceof|function|new|try|throw|catch|finally|null|break|continue|true|false|class|interface|extends|implements|trait|instanceof|new'.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r"(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^/\'\"{}]*)", re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?') + + @classmethod + def operator_regex(cls): + return re.compile(r'--?|\+\+?|!=?=?|<=?|>=?|==?=?|&&?|\|\|?|\?|\*|\/|~|\^|%') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) if isList: - return result - return ''.join(result) + return res + return ''.join(res) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Clike.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/cpp.py b/src/PyReprism/languages/cpp.py index a6c2a58..318e397 100644 --- a/src/PyReprism/languages/cpp.py +++ b/src/PyReprism/languages/cpp.py @@ -1,113 +1,51 @@ import re from PyReprism.utils import extension +from .base.base import BaseLanguage +from .registry import LanguageRegistry -class CPP: - def __init__(self): - pass - @staticmethod - def file_extension() -> str: - """ - Return the file extension used for C++ files. - - :return: The file extension for C++ files. - :rtype: str - """ +@LanguageRegistry.register +class CPP(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.cpp - @staticmethod - def keywords() -> list: - """ - Return a list of C++ keywords and built-in functions. - - :return: A list of C++ keywords and built-in function names. - :rtype: list - """ + @classmethod + def keywords(cls) -> list: keyword = 'alignas|alignof|asm|auto|bool|break|case|catch|char|char16_t|char32_t|class|compl|const|constexpr|const_cast|continue|decltype|default|delete|do|double|dynamic_cast|else|enum|explicit|export|extern|float|for|friend|goto|if|inline|int|int8_t|int16_t|int32_t|int64_t|uint8_t|uint16_t|uint32_t|uint64_t|long|mutable|namespace|new|noexcept|nullptr|operator|private|protected|public|register|reinterpret_cast|return|short|signed|sizeof|static|static_assert|static_cast|struct|switch|template|this|thread_local|throw|try|typedef|typeid|typename|union|unsigned|using|virtual|void|volatile|wchar_t|while|true|false'.split('|') return keyword - @staticmethod - def comment_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify different types of comments and non-comment code in C source files. - - :return: A compiled regex pattern with named groups to match single-line comments, multiline comments, and non-comment code elements. - :rtype: re.Pattern - """ + @classmethod + def comment_regex(cls) -> re.Pattern: pattern = re.compile(r'(?P//.*?$|/\*[^*]*\*+(?:[^/*][^*]*\*+)*?/)|(?P[^/]+)', re.DOTALL | re.MULTILINE) return pattern - @staticmethod - def number_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify numeric literals in C++ code. - - :return: A compiled regex pattern to match C++ numeric literals, including integers, floats, and complex numbers. - :rtype: re.Pattern - """ + @classmethod + def number_regex(cls) -> re.Pattern: pattern = re.compile(r'(?:\b0x[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?)[ful]*') return pattern - @staticmethod - def operator_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify C++ operators. - - :return: A compiled regex pattern to match various C++ operators and logical keywords. - :rtype: re.Pattern - """ + @classmethod + def operator_regex(cls) -> re.Pattern: pattern = re.compile(r'--?|\+\+?|!=?|<{1,2}=?|>{1,2}=?|->|:{1,2}|={1,2}|\^|~|%|&{1,2}|\|\|?|\?|\*|\/|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b') return pattern - @staticmethod - def keywords_regex() -> re.Pattern: - """ - Return a list of C++ keywords and built-in functions. - - :return: A list of C++ keywords and built-in function names. - :rtype: list - """ - return re.compile(r'\b(' + '|'.join(CPP.keywords()) + r')\b') - - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify C++ boolean literals. - - :return: A compiled regex pattern to match C++ boolean literals. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false)\b') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify C and C++ delimiters. - - :return: A compiled regex pattern to match C and C++ delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@<>*&]') - @staticmethod - def remove_comments(source_code: str) -> str: - """ - Remove comments from the provided C++ source code string. - - :param str source_code: The C++ source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return CPP.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - - @staticmethod - def remove_keywords(source: str) -> str: - """ - Remove all C++ keywords from the provided source code string. + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) + if isList: + return res + return res.strip() - :param str source: The source code string from which to remove C++ keywords. - :return: The source code string with all C++ keywords removed. - :rtype: str - """ - return re.sub(re.compile(CPP.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/go.py b/src/PyReprism/languages/go.py index 5e6e4d2..0245a05 100644 --- a/src/PyReprism/languages/go.py +++ b/src/PyReprism/languages/go.py @@ -1,82 +1,47 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Go: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.Go +@LanguageRegistry.register +class Go(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + return extension.go - @staticmethod - def keywords() -> list: - keyword = 'break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go(?:to)?|if|import|interface|map|package|range|return|select|struct|switch|type|var|bool|byte|complex(?:64|128)|error|float32|float64|rune|string|u?int(?:8|16|32|64)?|uintptr|append|cap|close|complex|copy|delete|imag|len|make|new|panic|print|println|real|recover|iota|nil|true|false'.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + return 'break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go(?:to)?|if|import|interface|map|package|range|return|select|struct|switch|type|var|bool|byte|complex(?:64|128)|error|float32|float64|rune|string|u?int(?:8|16|32|64)?|uintptr|append|cap|close|complex|copy|delete|imag|len|make|new|panic|print|println|real|recover|iota|nil|true|false'.split('|') - @staticmethod - def comment_regex() -> re.Pattern: - pattern = re.compile(r'(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r"(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^/\\'\"{}]*)", re.DOTALL | re.MULTILINE) - @staticmethod - def number_regex() -> re.Pattern: - pattern = re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?[df]?') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?[df]?') - @staticmethod - def operator_regex() -> re.Pattern: - pattern = re.compile(r'[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\.') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\.') - @staticmethod - def keywords_regex() -> re.Pattern: - return re.compile(r'\b(' + '|'.join(Go.keywords()) + r')\b') - - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Go delimiters. - - This function generates a regular expression that matches Go language delimiters, which include parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, and the ellipsis `...`. - - :return: A compiled regex pattern to match Go delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;]|\.{3}') - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Go boolean literals. - - This function generates a regular expression that matches the Go boolean literals `true`, `false`, and the special constant `nil`. - - :return: A compiled regex pattern to match Go boolean literals and `nil`. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false|nil)\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided Go source code string. - - :param str source_code: The Go source code from which to remove comments. - :param bool isList: (Optional) A flag indicating if the input is a list of source code lines. This parameter is not used in the function logic. - :return: The source code with all comments removed. - :rtype: str - """ - return Go.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in Go.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) if isList: - return result - return ''.join(result) + return res + return res.strip() - @staticmethod - def remove_keywords(source: str) -> str: - return re.sub(re.compile(Go.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/javascript.py b/src/PyReprism/languages/javascript.py index c1debb2..51ae3ba 100644 --- a/src/PyReprism/languages/javascript.py +++ b/src/PyReprism/languages/javascript.py @@ -1,122 +1,52 @@ import re from PyReprism.utils import extension +from .base.base import BaseLanguage +from .registry import LanguageRegistry -class JavaScript: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - """ - Return the file extension used for JavaScript files. - - :return: The file extension for Java files. - :rtype: str - """ +@LanguageRegistry.register +class JavaScript(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.javascript - @staticmethod - def keywords() -> list: - """ - Return a list of JavaScript keywords and built-in functions. - - :return: A list of JavaScript keywords and built-in function names. - :rtype: list - """ - keyword = 'as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield'.split('|') - return keyword - - @staticmethod - def comment_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify different types of comments and non-comment code in JavaScript source files. - - :return: A compiled regex pattern with named groups to match single-line comments, multiline comments, and non-comment code elements. - :rtype: re.Pattern - """ - pattern = re.compile(r'(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify numeric literals in JavaScript code. - - :return: A compiled regex pattern to match JavaScript numeric literals, including integers, floats, and complex numbers. - :rtype: re.Pattern - """ - pattern = re.compile(r'\b(?:0[xX][\dA-Fa-f]+|0[bB][01]+|0[oO][0-7]+|NaN|Infinity)\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee][+-]?\d+)?') - return pattern - - @staticmethod - def operator_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify JavaScript operators. - - :return: A compiled regex pattern to match various JavaScript operators and logical keywords. - :rtype: re.Pattern - """ - pattern = re.compile(r'-[-=]?|\+[+=]?|!=?=?|<>?>?=?|=(?:==?|>)?|&[&=]?|\|[|=]?|\*\*?=?|\/=?|~|\^=?|%=?|\?|\.{3}') - return pattern - - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify JavaScript delimiters. - - This function generates a regular expression that matches JavaScript delimiters, including parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, at symbols `@`, as well as angle brackets `<` and `>`. - - :return: A compiled regex pattern to match JavaScript delimiters. - :rtype: re.Pattern - """ + @classmethod + def keywords(cls) -> list: + return 'as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Match C-style // line comments and /* block comments; preserve + # string literals and other non-comment sequences as 'noncomment'. + # Do not treat braces as comments. + # Allow newlines in noncomment groups so original line breaks are + # preserved when comments are removed. + return re.compile(r"(?P//.*?$|/\*.*?\*/)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|[^/\\'\"]+)", re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b(?:0[xX][\dA-Fa-f]+|0[bB][01]+|0[oO][0-7]+|NaN|Infinity)\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee][+-]?\d+)?') + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'-[-=]?|\+[+=]?|!=?=?|<>?>?=?|=(?:==?|>)?|&[&=]?|\|[|=]?|\*\*?=?|\/?=?|~|\^=?|%=?|\?|\.{3}') + + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@<>]') - @staticmethod - def keywords_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify JavaScript keywords. - - :return: A compiled regex pattern to match JavaScript keywords. - :rtype: re.Pattern - """ - return re.compile(r'\b(' + '|'.join(JavaScript.keywords()) + r')\b') - - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify JavaScript boolean literals. - - :return: A compiled regex pattern to match JavaScript boolean literals. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false)\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided JavaScript source code string. - - :param str source_code: The JavaScript source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return JavaScript.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in JavaScript.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str) -> str: - """ - Remove all JavaScript keywords from the provided source code string. + return res + return res.strip() - :param str source: The source code string from which to remove JavaScript keywords. - :return: The source code string with all JavaScript keywords removed. - :rtype: str - """ - return re.sub(re.compile(JavaScript.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/matlab.py b/src/PyReprism/languages/matlab.py index a22e004..c87b9ee 100644 --- a/src/PyReprism/languages/matlab.py +++ b/src/PyReprism/languages/matlab.py @@ -1,49 +1,29 @@ import re from PyReprism.utils import extension +from .base.base import BaseLanguage +from .registry import LanguageRegistry -class MatLab: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class MatLab(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.matlab - @staticmethod - def keywords() -> list: - keyword = 'break|case|catch|continue|else|elseif|end|for|function|if|inf|NaN|otherwise|parfor|pause|pi|return|switch|try|while'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P%\{[\s\S]*?\}%|%.*?$)|(?P[^%]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?(?:[ij])?|\b[ij]\b', re.IGNORECASE) - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r"\.?[*^\/\\']|[+\-:@]|[<>=~]=?|&&?|\|\|?") - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(MatLab.keywords()) + r')\b', re.IGNORECASE) - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in MatLab.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(MatLab.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + return 'break|case|catch|continue|else|elseif|end|for|function|if|inf|NaN|otherwise|parfor|pause|pi|return|switch|try|while'.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P%\{[\s\S]*?\}%|%.*?$)|(?P[^%]*)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?(?:[ij])?|\b[ij]\b', re.IGNORECASE) + + @classmethod + def operator_regex(cls): + return re.compile(r"\.?[*^\/\\']|[+\-:@]|[<>=~]=?|&&?|\|\|?") + diff --git a/src/PyReprism/languages/php.py b/src/PyReprism/languages/php.py index 4957d48..e0e5594 100644 --- a/src/PyReprism/languages/php.py +++ b/src/PyReprism/languages/php.py @@ -1,124 +1,47 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class PHP: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - """ - Return the file extension used for PHP files. - - :return: The file extension for PHP files. - :rtype: str - """ +@LanguageRegistry.register +class PHP(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.php - @staticmethod - def keywords() -> list: - """ - Return a list of PHP keywords and built-in functions. - - :return: A list of PHP keywords and built-in function names. - :rtype: list - """ - keyword = 'and|or|xor|array|as|break|case|cfunction|class|const|continue|declare|default|die|do|else|elseif|enddeclare|endfor|endforeach|endif|endswitch|endwhile|extends|for|foreach|function|include|include_once|global|if|new|return|static|switch|use|require|require_once|var|while|abstract|interface|public|implements|private|protected|parent|throw|null|echo|print|trait|namespace|final|yield|goto|instanceof|finally|try|catch'.split('|') - return keyword - - @staticmethod - def comment_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify different types of comments and non-comment code in PHP source files. - - :return: A compiled regex pattern with named groups to match single-line comments, multiline comments, and non-comment code elements. - :rtype: re.Pattern - """ - pattern = re.compile(r'(?P#.*?$|//.*?$|[{}]+)|(?P/\*.*?\*/)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#/\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify numeric literals in PHP code. - - :return: A compiled regex pattern to match PHP numeric literals, including integers, floats, and complex numbers. - :rtype: re.Pattern - """ - pattern = re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?\b', re.IGNORECASE) - return pattern - - @staticmethod - def operator_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify PHP operators. + @classmethod + def keywords(cls) -> list: + return 'and|or|xor|array|as|break|case|cfunction|class|const|continue|declare|default|die|do|else|elseif|enddeclare|endfor|endforeach|endif|endswitch|endwhile|extends|for|foreach|function|include|include_once|global|if|new|return|static|switch|use|require|require_once|var|while|abstract|interface|public|implements|private|protected|parent|throw|null|echo|print|trait|namespace|final|yield|goto|instanceof|finally|try|catch'.split('|') - :return: A compiled regex pattern to match various PHP operators and logical keywords. - :rtype: re.Pattern - """ - pattern = re.compile(r'[-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]|\b(?:or|and|not)\b') - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r"""(?P#.*?$|//.*?$|[{}]+)|(?P/\*.*?\*/)|(?P'(\\.|[^\\'])*'|"(\\.|[^\\"])*"|.[^#/\'"{}]*)""", re.DOTALL | re.MULTILINE) - @staticmethod - def keywords_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify PHP keywords. + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?\b', re.IGNORECASE) - :return: A compiled regex pattern to match PHP keywords. - :rtype: re.Pattern - """ - return re.compile(r'\b(' + '|'.join(PHP.keywords()) + r')\b', re.IGNORECASE) + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[-+%=]=?|!=|\*\*?=?|//?=?|<[<=>]?|>[=>]?|[&|^~]|\b(?:or|and|not)\b') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify PHP delimiters. - - This function generates a regular expression that matches PHP delimiters, including parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, at symbols `@`, angle brackets `<` and `>`, as well as PHP-specific tokens like `$` for variables. - - :return: A compiled regex pattern to match PHP delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@<>$]') - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify PHP boolean literals. - - This function generates a regular expression that matches the PHP boolean literals `true`, `false`, and the special constant `null`. The matching is case-insensitive, as PHP boolean literals are not case-sensitive. - - :return: A compiled regex pattern to match PHP boolean literals and `null`. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false|null)\b', re.IGNORECASE) - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided PHP source code string. - - :param str source_code: The PHP source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return PHP.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in PHP.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str) -> str: - """ - Remove all PHP keywords from the provided source code string. + return res + return res.strip() - :param str source: The source code string from which to remove PHP keywords. - :return: The source code string with all PHP keywords removed. - :rtype - """ - return re.sub(re.compile(PHP.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/php_extras.py b/src/PyReprism/languages/php_extras.py index e69de29..31c6cdd 100644 --- a/src/PyReprism/languages/php_extras.py +++ b/src/PyReprism/languages/php_extras.py @@ -0,0 +1,17 @@ +import re +from .php import PHP +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class PHPExtras(PHP): + """Alias/extension for PHP-related extras. Inherits behavior from PHP.""" + + @classmethod + def file_extension(cls) -> str: + return PHP.file_extension() + + @classmethod + def keywords(cls) -> list: + # extend or reuse PHP keywords + return PHP.keywords() diff --git a/src/PyReprism/languages/python.py b/src/PyReprism/languages/python.py index 482cadb..705bff1 100644 --- a/src/PyReprism/languages/python.py +++ b/src/PyReprism/languages/python.py @@ -1,6 +1,51 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Python(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + return extension.python + + @classmethod + def keywords(cls) -> list: + return 'as|assert|async|await|break|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|nonlocal|pass|print|raise|return|try|while|with|yield|__import__|abs|all|any|apply|ascii|basestring|bin|bool|buffer|bytearray|bytes|case|callable|chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|divmod|enumerate|eval|execfile|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|list|locals|long|match|map|max|memoryview|min|next|object|oct|open|ord|pow|property|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|vars|xrange|zip|True|False|None'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Pattern captures: single-line '#', triple-quoted triple double or triple single, and noncomment segments + return re.compile(r"(?P#.*?$)|(?P\"\"\".*?\"\"\")|(?P''' .*?''')|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^#'\"]*)", re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?\b', re.IGNORECASE) + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[-+%=]=?|!=|\*\*?=?|//?=?|<[<=>]?|>[=>]?|[&|^~]|\b(?:or|and|not)\b') + + @classmethod + def delimiters_regex(cls) -> re.Pattern: + return re.compile(r'[()\[\]{}.,:;@]') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) + if isList: + return res + return res.strip() + + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) + +import re +from PyReprism.utils import extension + class Python: """ diff --git a/src/PyReprism/languages/ruby.py b/src/PyReprism/languages/ruby.py index 362c4a1..2a8866b 100644 --- a/src/PyReprism/languages/ruby.py +++ b/src/PyReprism/languages/ruby.py @@ -1,49 +1,39 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Ruby: - def __init__(self): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Ruby(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.ruby - @staticmethod - def keywords() -> list: - keyword = 'alias|and|BEGIN|begin|break|case|class|def|define_method|defined|do|each|else|elsif|END|end|ensure|false|for|if|in|module|new|next|nil|not|or|protected|private|public|raise|redo|require|rescue|retry|return|self|super|then|throw|true|undef|unless|until|when|while|yield|Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Stat|Fixnum|Float|Hash|Integer|IO|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|String|Struct|TMS|Symbol|ThreadGroup|Thread|Time|TrueClass'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$|=begin[\s\S]*?=end|=begin.*?$|^.*?=end)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#=\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = '' - return pattern - - @staticmethod - def operator_regex(): - pattern = '' - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Ruby.keywords()) + r')\b', re.IGNORECASE) - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Ruby.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def keywords(cls) -> list: + return 'alias|and|BEGIN|begin|break|case|class|def|define_method|defined|do|each|else|elsif|END|end|ensure|false|for|if|in|module|new|next|nil|not|or|protected|private|public|raise|redo|require|rescue|retry|return|self|super|then|throw|true|undef|unless|until|when|while|yield|Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Stat|Fixnum|Float|Hash|Integer|IO|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|String|Struct|TMS|Symbol|ThreadGroup|Thread|Time|TrueClass'.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r"(?P#.*?$|=begin[\s\S]*?=end|=begin.*?$|^.*?=end)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^#=\'\"]*)", re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return '' + + @classmethod + def operator_regex(cls): + return '' + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) if isList: - return result - return ''.join(result) + return res + return ''.join(res) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Ruby.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) From da170bcf548f83c56856e60cb978b39a1a81af41 Mon Sep 17 00:00:00 2001 From: danielogen Date: Sat, 4 Oct 2025 07:34:10 -0700 Subject: [PATCH 03/19] ft: add helpers class --- src/PyReprism/utils/helpers.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/PyReprism/utils/helpers.py diff --git a/src/PyReprism/utils/helpers.py b/src/PyReprism/utils/helpers.py new file mode 100644 index 0000000..e69de29 From f1746ef1a6f811c50d5fe8fbc09278a9f35ff9b7 Mon Sep 17 00:00:00 2001 From: danielogen Date: Sun, 5 Oct 2025 07:59:20 -0700 Subject: [PATCH 04/19] refactor: auto register and lazy loading --- src/PyReprism/languages/abap.py | 83 ++++++------- src/PyReprism/languages/actionscript.py | 69 +++++------ src/PyReprism/languages/ada.py | 74 +++++------ src/PyReprism/languages/apacheconf.py | 145 ++++++++++++++++------ src/PyReprism/languages/apl.py | 76 +++++------- src/PyReprism/languages/applescript.py | 92 +++++++------- src/PyReprism/languages/arduino.py | 81 ++++++------ src/PyReprism/languages/aspnet.py | 81 ++++++------ src/PyReprism/languages/brainfuck.py | 78 ++++++------ src/PyReprism/languages/cpp.py | 2 +- src/PyReprism/languages/crystal.py | 64 +++++----- src/PyReprism/languages/csharp.py | 99 ++++++--------- src/PyReprism/languages/dart.py | 103 +++++++--------- src/PyReprism/languages/graphql.py | 66 +++++----- src/PyReprism/languages/groovy.py | 80 ++++++------ src/PyReprism/languages/ini.py | 53 ++++---- src/PyReprism/languages/java.py | 157 +++++++++--------------- src/PyReprism/languages/javascript.py | 2 +- src/PyReprism/languages/jsx.py | 137 +++++++-------------- src/PyReprism/languages/julia.py | 78 ++++++------ src/PyReprism/languages/keyman.py | 45 +++++++ src/PyReprism/languages/kotlin.py | 101 ++++++--------- src/PyReprism/languages/matlab.py | 2 +- src/PyReprism/languages/nginx.py | 79 ++++++------ src/PyReprism/languages/objectivec.py | 81 ++++++------ src/PyReprism/languages/perl.py | 88 +++++++++---- src/PyReprism/languages/php.py | 11 +- src/PyReprism/languages/q.py | 27 ++++ src/PyReprism/languages/qore.py | 27 ++++ src/PyReprism/languages/r.py | 28 +++++ src/PyReprism/languages/registry.py | 26 +++- src/PyReprism/languages/rust.py | 110 +++++++---------- src/PyReprism/languages/scala.py | 97 +++++---------- src/PyReprism/languages/scheme.py | 67 +++++----- src/PyReprism/languages/smalltalk.py | 80 ++++++------ src/PyReprism/languages/stylus.py | 73 ++++++----- src/PyReprism/languages/swift.py | 102 ++++++--------- src/PyReprism/languages/tsx.py | 67 +++++----- src/PyReprism/languages/typescript.py | 101 ++++++--------- 39 files changed, 1423 insertions(+), 1409 deletions(-) diff --git a/src/PyReprism/languages/abap.py b/src/PyReprism/languages/abap.py index a70a23c..38efcab 100644 --- a/src/PyReprism/languages/abap.py +++ b/src/PyReprism/languages/abap.py @@ -1,50 +1,47 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Abap: - def __init__(self): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Abap(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.abap - @staticmethod - def keywords() -> list: - # keyword = 'SCIENTIFIC_WITH_LEADING_ZERO|SCALE_PRESERVING_SCIENTIFIC|RMC_COMMUNICATION_FAILURE|END-ENHANCEMENT-SECTION|MULTIPLY-CORRESPONDING|SUBTRACT-CORRESPONDING|VERIFICATION-MESSAGE|DIVIDE-CORRESPONDING|ENHANCEMENT-SECTION|CURRENCY_CONVERSION|RMC_SYSTEM_FAILURE|START-OF-SELECTION|MOVE-CORRESPONDING|RMC_INVALID_STATUS|CUSTOMER-FUNCTION|END-OF-DEFINITION|ENHANCEMENT-POINT|SYSTEM-EXCEPTIONS|ADD-CORRESPONDING|SCALE_PRESERVING|SELECTION-SCREEN|CURSOR-SELECTION|END-OF-SELECTION|LOAD-OF-PROGRAM|SCROLL-BOUNDARY|SELECTION-TABLE|EXCEPTION-TABLE|IMPLEMENTATIONS|PARAMETER-TABLE|RIGHT-JUSTIFIED|UNIT_CONVERSION|AUTHORITY-CHECK|LIST-PROCESSING|SIGN_AS_POSTFIX|COL_BACKGROUND|IMPLEMENTATION|INTERFACE-POOL|TRANSFORMATION|IDENTIFICATION|ENDENHANCEMENT|LINE-SELECTION|INITIALIZATION|LEFT-JUSTIFIED|SELECT-OPTIONS|SELECTION-SETS|COMMUNICATION|CORRESPONDING|DECIMAL_SHIFT|PRINT-CONTROL|VALUE-REQUEST|CHAIN-REQUEST|FUNCTION-POOL|FIELD-SYMBOLS|FUNCTIONALITY|INVERTED-DATE|SELECTION-SET|CLASS-METHODS|OUTPUT-LENGTH|CLASS-CODING|COL_NEGATIVE|ERRORMESSAGE|FIELD-GROUPS|HELP-REQUEST|NO-EXTENSION|NO-TOPOFPAGE|REDEFINITION|DISPLAY-MODE|ENDINTERFACE|EXIT-COMMAND|FIELD-SYMBOL|NO-SCROLLING|SHORTDUMP-ID|ACCESSPOLICY|CLASS-EVENTS|COL_POSITIVE|DECLARATIONS|ENHANCEMENTS|FILTER-TABLE|SWITCHSTATES|SYNTAX-CHECK|TRANSPORTING|ASYNCHRONOUS|SYNTAX-TRACE|TOKENIZATION|USER-COMMAND|WITH-HEADING|ABAP-SOURCE|BREAK-POINT|CHAIN-INPUT|COMPRESSION|FIXED-POINT|NEW-SECTION|NON-UNICODE|OCCURRENCES|RESPONSIBLE|SYSTEM-CALL|TRACE-TABLE|ABBREVIATED|CHAR-TO-HEX|END-OF-FILE|ENDFUNCTION|ENVIRONMENT|ASSOCIATION|COL_HEADING|EDITOR-CALL|END-OF-PAGE|ENGINEERING|IMPLEMENTED|INTENSIFIED|RADIOBUTTON|SYSTEM-EXIT|TOP-OF-PAGE|TRANSACTION|APPLICATION|CONCATENATE|DESTINATION|ENHANCEMENT|IMMEDIATELY|NO-GROUPING|PRECOMPILED|REPLACEMENT|TITLE-LINES|ACTIVATION|BYTE-ORDER|CLASS-POOL|CONNECTION|CONVERSION|DEFINITION|DEPARTMENT|EXPIRATION|INHERITING|MESSAGE-ID|NO-HEADING|PERFORMING|QUEUE-ONLY|RIGHTSPACE|SCIENTIFIC|STATUSINFO|STRUCTURES|SYNCPOINTS|WITH-TITLE|ATTRIBUTES|BOUNDARIES|CLASS-DATA|COL_NORMAL|DD\/MM\/YYYY|DESCENDING|INTERFACES|LINE-COUNT|MM\/DD\/YYYY|NON-UNIQUE|PRESERVING|SELECTIONS|STATEMENTS|SUBROUTINE|TRUNCATION|TYPE-POOLS|ARITHMETIC|BACKGROUND|ENDPROVIDE|EXCEPTIONS|IDENTIFIER|INDEX-LINE|OBLIGATORY|PARAMETERS|PERCENTAGE|PUSHBUTTON|RESOLUTION|COMPONENTS|DEALLOCATE|DISCONNECT|DUPLICATES|FIRST-LINE|HEAD-LINES|NO-DISPLAY|OCCURRENCE|RESPECTING|RETURNCODE|SUBMATCHES|TRACE-FILE|ASCENDING|BYPASSING|ENDMODULE|EXCEPTION|EXCLUDING|EXPORTING|INCREMENT|MATCHCODE|PARAMETER|PARTIALLY|PREFERRED|REFERENCE|REPLACING|RETURNING|SELECTION|SEPARATED|SPECIFIED|STATEMENT|TIMESTAMP|TYPE-POOL|ACCEPTING|APPENDAGE|ASSIGNING|COL_GROUP|COMPARING|CONSTANTS|DANGEROUS|IMPORTING|INSTANCES|LEFTSPACE|LOG-POINT|QUICKINFO|READ-ONLY|SCROLLING|SQLSCRIPT|STEP-LOOP|TOP-LINES|TRANSLATE|APPENDING|AUTHORITY|CHARACTER|COMPONENT|CONDITION|DIRECTORY|DUPLICATE|MESSAGING|RECEIVING|SUBSCREEN|ACCORDING|COL_TOTAL|END-LINES|ENDMETHOD|ENDSELECT|EXPANDING|EXTENSION|INCLUDING|INFOTYPES|INTERFACE|INTERVALS|LINE-SIZE|PF-STATUS|PROCEDURE|PROTECTED|REQUESTED|RESUMABLE|RIGHTPLUS|SAP-SPOOL|SECONDARY|STRUCTURE|SUBSTRING|TABLEVIEW|NUMOFCHAR|ADJACENT|ANALYSIS|ASSIGNED|BACKWARD|CHANNELS|CHECKBOX|CONTINUE|CRITICAL|DATAINFO|DD\/MM\/YY|DURATION|ENCODING|ENDCLASS|FUNCTION|LEFTPLUS|LINEFEED|MM\/DD\/YY|OVERFLOW|RECEIVED|SKIPPING|SORTABLE|STANDARD|SUBTRACT|SUPPRESS|TABSTRIP|TITLEBAR|TRUNCATE|UNASSIGN|WHENEVER|ANALYZER|COALESCE|COMMENTS|CONDENSE|DECIMALS|DEFERRED|ENDWHILE|EXPLICIT|KEYWORDS|MESSAGES|POSITION|PRIORITY|RECEIVER|RENAMING|TIMEZONE|TRAILING|ALLOCATE|CENTERED|CIRCULAR|CONTROLS|CURRENCY|DELETING|DESCRIBE|DISTANCE|ENDCATCH|EXPONENT|EXTENDED|GENERATE|IGNORING|INCLUDES|INTERNAL|MAJOR-ID|MODIFIER|NEW-LINE|OPTIONAL|PROPERTY|ROLLBACK|STARTING|SUPPLIED|ABSTRACT|CHANGING|CONTEXTS|CREATING|CUSTOMER|DATABASE|DAYLIGHT|DEFINING|DISTINCT|DIVISION|ENABLING|ENDCHAIN|ESCAPING|HARMLESS|IMPLICIT|INACTIVE|LANGUAGE|MINOR-ID|MULTIPLY|NEW-PAGE|NO-TITLE|POS_HIGH|SEPARATE|TEXTPOOL|TRANSFER|SELECTOR|DBMAXLEN|ITERATOR|SELECTOR|ARCHIVE|BIT-XOR|BYTE-CO|COLLECT|COMMENT|CURRENT|DEFAULT|DISPLAY|ENDFORM|EXTRACT|LEADING|LISTBOX|LOCATOR|MEMBERS|METHODS|NESTING|POS_LOW|PROCESS|PROVIDE|RAISING|RESERVE|SECONDS|SUMMARY|VISIBLE|BETWEEN|BIT-AND|BYTE-CS|CLEANUP|COMPUTE|CONTROL|CONVERT|DATASET|ENDCASE|FORWARD|HEADERS|HOTSPOT|INCLUDE|INVERSE|KEEPING|NO-ZERO|OBJECTS|OVERLAY|PADDING|PATTERN|PROGRAM|REFRESH|SECTION|SUMMING|TESTING|VERSION|WINDOWS|WITHOUT|BIT-NOT|BYTE-CA|BYTE-NA|CASTING|CONTEXT|COUNTRY|DYNAMIC|ENABLED|ENDLOOP|EXECUTE|FRIENDS|HANDLER|HEADING|INITIAL|\*-INPUT|LOGFILE|MAXIMUM|MINIMUM|NO-GAPS|NO-SIGN|PRAGMAS|PRIMARY|PRIVATE|REDUCED|REPLACE|REQUEST|RESULTS|UNICODE|WARNING|ALIASES|BYTE-CN|BYTE-NS|CALLING|COL_KEY|COLUMNS|CONNECT|ENDEXEC|ENTRIES|EXCLUDE|FILTERS|FURTHER|HELP-ID|LOGICAL|MAPPING|MESSAGE|NAMETAB|OPTIONS|PACKAGE|PERFORM|RECEIVE|STATICS|VARYING|BINDING|CHARLEN|GREATER|XSTRLEN|ACCEPT|APPEND|DETAIL|ELSEIF|ENDING|ENDTRY|FORMAT|FRAMES|GIVING|HASHED|HEADER|IMPORT|INSERT|MARGIN|MODULE|NATIVE|OBJECT|OFFSET|REMOTE|RESUME|SAVING|SIMPLE|SUBMIT|TABBED|TOKENS|UNIQUE|UNPACK|UPDATE|WINDOW|YELLOW|ACTUAL|ASPECT|CENTER|CURSOR|DELETE|DIALOG|DIVIDE|DURING|ERRORS|EVENTS|EXTEND|FILTER|HANDLE|HAVING|IGNORE|LITTLE|MEMORY|NO-GAP|OCCURS|OPTION|PERSON|PLACES|PUBLIC|REDUCE|REPORT|RESULT|SINGLE|SORTED|SWITCH|SYNTAX|TARGET|VALUES|WRITER|ASSERT|BLOCKS|BOUNDS|BUFFER|CHANGE|COLUMN|COMMIT|CONCAT|COPIES|CREATE|DDMMYY|DEFINE|ENDIAN|ESCAPE|EXPAND|KERNEL|LAYOUT|LEGACY|LEVELS|MMDDYY|NUMBER|OUTPUT|RANGES|READER|RETURN|SCREEN|SEARCH|SELECT|SHARED|SOURCE|STABLE|STATIC|SUBKEY|SUFFIX|TABLES|UNWIND|YYMMDD|ASSIGN|BACKUP|BEFORE|BINARY|BIT-OR|BLANKS|CLIENT|CODING|COMMON|DEMAND|DYNPRO|EXCEPT|EXISTS|EXPORT|FIELDS|GLOBAL|GROUPS|LENGTH|LOCALE|MEDIUM|METHOD|MODIFY|NESTED|OTHERS|REJECT|SCROLL|SUPPLY|SYMBOL|ENDFOR|STRLEN|ALIGN|BEGIN|BOUND|ENDAT|ENTRY|EVENT|FINAL|FLUSH|GRANT|INNER|SHORT|USING|WRITE|AFTER|BLACK|BLOCK|CLOCK|COLOR|COUNT|DUMMY|EMPTY|ENDDO|ENDON|GREEN|INDEX|INOUT|LEAVE|LEVEL|LINES|MODIF|ORDER|OUTER|RANGE|RESET|RETRY|RIGHT|SMART|SPLIT|STYLE|TABLE|THROW|UNDER|UNTIL|UPPER|UTF-8|WHERE|ALIAS|BLANK|CLEAR|CLOSE|EXACT|FETCH|FIRST|FOUND|GROUP|LLANG|LOCAL|OTHER|REGEX|SPOOL|TITLE|TYPES|VALID|WHILE|ALPHA|BOXED|CATCH|CHAIN|CHECK|CLASS|COVER|ENDIF|EQUIV|FIELD|FLOOR|FRAME|INPUT|LOWER|MATCH|NODES|PAGES|PRINT|RAISE|ROUND|SHIFT|SPACE|SPOTS|STAMP|STATE|TASKS|TIMES|TRMAC|ULINE|UNION|VALUE|WIDTH|EQUAL|LOG10|TRUNC|BLOB|CASE|CEIL|CLOB|COND|EXIT|FILE|GAPS|HOLD|INCL|INTO|KEEP|KEYS|LAST|LINE|LONG|LPAD|MAIL|MODE|OPEN|PINK|READ|ROWS|TEST|THEN|ZERO|AREA|BACK|BADI|BYTE|CAST|EDIT|EXEC|FAIL|FIND|FKEQ|FONT|FREE|GKEQ|HIDE|INIT|ITNO|LATE|LOOP|MAIN|MARK|MOVE|NEXT|NULL|RISK|ROLE|UNIT|WAIT|ZONE|BASE|CALL|CODE|DATA|DATE|FKGE|GKGE|HIGH|KIND|LEFT|LIST|MASK|MESH|NAME|NODE|PACK|PAGE|POOL|SEND|SIGN|SIZE|SOME|STOP|TASK|TEXT|TIME|USER|VARY|WITH|WORD|BLUE|CONV|COPY|DEEP|ELSE|FORM|FROM|HINT|ICON|JOIN|LIKE|LOAD|ONLY|PART|SCAN|SKIP|SORT|TYPE|UNIX|VIEW|WHEN|WORK|ACOS|ASIN|ATAN|COSH|EACH|FRAC|LESS|RTTI|SINH|SQRT|TANH|AVG|BIT|DIV|ISO|LET|OUT|PAD|SQL|ALL|CI_|CPI|END|LOB|LPI|MAX|MIN|NEW|OLE|RUN|SET|\?TO|YES|ABS|ADD|AND|BIG|FOR|HDB|JOB|LOW|NOT|SAP|TRY|VIA|XML|ANY|GET|IDS|KEY|MOD|OFF|PUT|RAW|RED|REF|SUM|TAB|XSD|CNT|COS|EXP|LOG|SIN|TAN|XOR|AT|CO|CP|DO|GT|ID|IF|NS|OR|BT|CA|CS|GE|NA|NB|EQ|IN|LT|NE|NO|OF|ON|PF|TO|AS|BY|CN|IS|LE|NP|UP|E|I|M|O|Z|C|X'.split('|') - # return keyword - pass - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P^\*.*?$|".*?$|\(\*[\s\S]*?\*\)|\(\*.*?$|^.*?\*\))|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^*"\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b\d+\b') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'(\s)(?:\*\*?|<[=>]?|>=?|\?=|[-+\/=])(?=\s)') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Abap.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str) -> str: - result = [] - for match in Abap.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - # if isList: - # return result - return ''.join(result).strip() - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Abap.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + # Keep keywords minimal for now to avoid a huge inline list. + # The original project had a very large set; add more to data/ later. + return [] + + @classmethod + def comment_regex(cls): + # Preserve original matching behavior: lines starting with *, + # double-quote inline comments, and (* ... *) block comments. + return re.compile( + r'''(?P^\*.*?$|".*?$|\(\*[\s\S]*?\*\)|\(\*.*?$|^.*?\*\))|(?P'(\\.|[^\\'])*'|"(\\.|[^\\"])*"|.[^*"']*)''', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls): + return re.compile(r"\b\d+\b") + + @classmethod + def operator_regex(cls): + return re.compile(r"(\s)(?:\*\*?|<[=>]?|>=?|\?=|[-+\/=])(?=\s)") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Delegate to BaseLanguage which uses the 'noncomment' group. + res = super().remove_comments(source_code, isList=isList) + if isList: + return res + return res.strip() + + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/actionscript.py b/src/PyReprism/languages/actionscript.py index 04ba849..91b6c0b 100644 --- a/src/PyReprism/languages/actionscript.py +++ b/src/PyReprism/languages/actionscript.py @@ -1,43 +1,40 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class ActionScript: - def __init__(self): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class ActionScript(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.actionscript - @staticmethod - def keywords() -> list: - keyword = 'as|break|case|catch|class|const|default|delete|do|else|extends|finally|for|function|if|implements|import|in|instanceof|interface|internal|is|native|new|null|package|private|protected|public|return|super|switch|this|throw|try|typeof|use|var|void|while|with|dynamic|each|final|get|include|namespace|native|override|set|static'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[^*]*\*+(?:[^/*][^*]*\*+)*?/)|(?P[^/]+)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b\d+\b') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'\+\+|--|(?:[+\-*\/%^]|&&?|\|\|?|<>?>?|[!=]=?)=?|[~?@]') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(ActionScript.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str) -> str: - return ActionScript.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(ActionScript.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + return 'as|break|case|catch|class|const|default|delete|do|else|extends|finally|for|function|if|implements|import|in|instanceof|interface|internal|is|native|new|null|package|private|protected|public|return|super|switch|this|throw|try|typeof|use|var|void|while|with|dynamic|each|final|get|include|namespace|native|override|set|static'.split('|') + + @classmethod + def comment_regex(cls): + # Keep the original pattern that captures // and /* */ comments and preserves non-comment sequences + return re.compile(r'(?P//.*?$|/\*[^*]*\*+(?:[^/*][^*]*\*+)*?/)|(?P[^/]+)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b\d+\b') + + @classmethod + def operator_regex(cls): + return re.compile(r'\+\+|--|(?:[+\-*\/%^]|&&?|\|\|?|<>?>?|[!=]=?)=?|[~?@]') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) + if isList: + return res + return res.strip() + + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/ada.py b/src/PyReprism/languages/ada.py index dddf505..ede0ce4 100644 --- a/src/PyReprism/languages/ada.py +++ b/src/PyReprism/languages/ada.py @@ -1,49 +1,43 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Ada: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Ada(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.ada - @staticmethod - def keywords() -> list: - keyword = 'abort|abs|abstract|accept|access|aliased|all|and|array|at|begin|body|case|constant|declare|delay|delta|digits|do|else|new|return|elsif|end|entry|exception|exit|for|function|generic|Adato|if|in|interface|is|limited|loop|mod|not|null|of|others|out|overriding|package|pragma|private|procedure|protected|raise|range|record|rem|renames|requeue|reverse|select|separate|some|subtype|synchronized|tagged|task|terminate|then|type|until|use|when|while|with|xor|true|false'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P--.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b\d(?:_?\d)*#[\dA-F](?:_?[\dA-F])*(?:\.[\dA-F](?:_?[\dA-F])*)?#(?:E[+-]?\d(?:_?\d)*)?') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'<[=>]?|>=?|=>?|:=|\/=?|\*\*?|[&+-]') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Ada.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Ada.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def keywords(cls) -> list: + return 'abort|abs|abstract|accept|access|aliased|all|and|array|at|begin|body|case|constant|declare|delay|delta|digits|do|else|new|return|elsif|end|entry|exception|exit|for|function|generic|Adato|if|in|interface|is|limited|loop|mod|not|null|of|others|out|overriding|package|pragma|private|procedure|protected|raise|range|record|rem|renames|requeue|reverse|select|separate|some|subtype|synchronized|tagged|task|terminate|then|type|until|use|when|while|with|xor|true|false|null'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Ada uses '--' for single-line comments. Capture strings and non-comment segments. + return re.compile( + r"(?P--.*?$)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^-']*)", + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b(?:0x[\da-f]+|\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)\b', re.IGNORECASE) + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[:=<>+\-*/&|^%]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) if isList: - return result - return ''.join(result) + return res + return res.strip() - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Ada.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/apacheconf.py b/src/PyReprism/languages/apacheconf.py index 900f5cd..fc3cd63 100644 --- a/src/PyReprism/languages/apacheconf.py +++ b/src/PyReprism/languages/apacheconf.py @@ -1,49 +1,116 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class ApacheConf: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class ApacheConf(BaseLanguage): + """Apache configuration language helper.""" + + @classmethod + def file_extension(cls) -> str: return extension.apacheconf - @staticmethod - def keywords() -> list: - keyword = 'AcceptFilter|AcceptPathInfo|AccessFileName|Action|AddAlt|AddAltByEncoding|AddAltByType|AddCharset|AddDefaultCharset|AddDescription|AddEncoding|AddHandler|AddIcon|AddIconByEncoding|AddIconByType|AddInputFilter|AddLanguage|AddModuleInfo|AddOutputFilter|AddOutputFilterByType|AddType|Alias|AliasMatch|Allow|AllowCONNECT|AllowEncodedSlashes|AllowMethods|AllowOverride|AllowOverrideList|Anonymous|Anonymous_LogEmail|Anonymous_MustGiveEmail|Anonymous_NoUserID|Anonymous_VerifyEmail|AsyncRequestWorkerFactor|AuthBasicAuthoritative|AuthBasicFake|AuthBasicProvider|AuthBasicUseDigestAlgorithm|AuthDBDUserPWQuery|AuthDBDUserRealmQuery|AuthDBMGroupFile|AuthDBMType|AuthDBMUserFile|AuthDigestAlgorithm|AuthDigestDomain|AuthDigestNonceLifetime|AuthDigestProvider|AuthDigestQop|AuthDigestShmemSize|AuthFormAuthoritative|AuthFormBody|AuthFormDisableNoStore|AuthFormFakeBasicAuth|AuthFormLocation|AuthFormLoginRequiredLocation|AuthFormLoginSuccessLocation|AuthFormLogoutLocation|AuthFormMethod|AuthFormMimetype|AuthFormPassword|AuthFormProvider|AuthFormSitePassphrase|AuthFormSize|AuthFormUsername|AuthGroupFile|AuthLDAPAuthorizePrefix|AuthLDAPBindAuthoritative|AuthLDAPBindDN|AuthLDAPBindPassword|AuthLDAPCharsetConfig|AuthLDAPCompareAsUser|AuthLDAPCompareDNOnServer|AuthLDAPDereferenceAliases|AuthLDAPGroupAttribute|AuthLDAPGroupAttributeIsDN|AuthLDAPInitialBindAsUser|AuthLDAPInitialBindPattern|AuthLDAPMaxSubGroupDepth|AuthLDAPRemoteUserAttribute|AuthLDAPRemoteUserIsDN|AuthLDAPSearchAsUser|AuthLDAPSubGroupAttribute|AuthLDAPSubGroupClass|AuthLDAPUrl|AuthMerging|AuthName|AuthnCacheContext|AuthnCacheEnable|AuthnCacheProvideFor|AuthnCacheSOCache|AuthnCacheTimeout|AuthnzFcgiCheckAuthnProvider|AuthnzFcgiDefineProvider|AuthType|AuthUserFile|AuthzDBDLoginToReferer|AuthzDBDQuery|AuthzDBDRedirectQuery|AuthzDBMType|AuthzSendForbiddenOnFailure|BalancerGrowth|BalancerInherit|BalancerMember|BalancerPersist|BrowserMatch|BrowserMatchNoCase|BufferedLogs|BufferSize|CacheDefaultExpire|CacheDetailHeader|CacheDirLength|CacheDirLevels|CacheDisable|CacheEnable|CacheFile|CacheHeader|CacheIgnoreCacheControl|CacheIgnoreHeaders|CacheIgnoreNoLastMod|CacheIgnoreQueryString|CacheIgnoreURLSessionIdentifiers|CacheKeyBaseURL|CacheLastModifiedFactor|CacheLock|CacheLockMaxAge|CacheLockPath|CacheMaxExpire|CacheMaxFileSize|CacheMinExpire|CacheMinFileSize|CacheNegotiatedDocs|CacheQuickHandler|CacheReadSize|CacheReadTime|CacheRoot|CacheSocache|CacheSocacheMaxSize|CacheSocacheMaxTime|CacheSocacheMinTime|CacheSocacheReadSize|CacheSocacheReadTime|CacheStaleOnError|CacheStoreExpired|CacheStoreNoStore|CacheStorePrivate|CGIDScriptTimeout|CGIMapExtension|CharsetDefault|CharsetOptions|CharsetSourceEnc|CheckCaseOnly|CheckSpelling|ChrootDir|ContentDigest|CookieDomain|CookieExpires|CookieName|CookieStyle|CookieTracking|CoreDumpDirectory|CustomLog|Dav|DavDepthInfinity|DavGenericLockDB|DavLockDB|DavMinTimeout|DBDExptime|DBDInitSQL|DBDKeep|DBDMax|DBDMin|DBDParams|DBDPersist|DBDPrepareSQL|DBDriver|DefaultIcon|DefaultLanguage|DefaultRuntimeDir|DefaultType|Define|DeflateBufferSize|DeflateCompressionLevel|DeflateFilterNote|DeflateInflateLimitRequestBody|DeflateInflateRatioBurst|DeflateInflateRatioLimit|DeflateMemLevel|DeflateWindowSize|Deny|DirectoryCheckHandler|DirectoryIndex|DirectoryIndexRedirect|DirectorySlash|DocumentRoot|DTracePrivileges|DumpIOInput|DumpIOOutput|EnableExceptionHook|EnableMMAP|EnableSendfile|Error|ErrorDocument|ErrorLog|ErrorLogFormat|Example|ExpiresActive|ExpiresByType|ExpiresDefault|ExtendedStatus|ExtFilterDefine|ExtFilterOptions|FallbackResource|FileETag|FilterChain|FilterDeclare|FilterProtocol|FilterProvider|FilterTrace|ForceLanguagePriority|ForceType|ForensicLog|GprofDir|GracefulShutdownTimeout|Group|Header|HeaderName|HeartbeatAddress|HeartbeatListen|HeartbeatMaxServers|HeartbeatStorage|HeartbeatStorage|HostnameLookups|IdentityCheck|IdentityCheckTimeout|ImapBase|ImapDefault|ImapMenu|Include|IncludeOptional|IndexHeadInsert|IndexIgnore|IndexIgnoreReset|IndexOptions|IndexOrderDefault|IndexStyleSheet|InputSed|ISAPIAppendLogToErrors|ISAPIAppendLogToQuery|ISAPICacheFile|ISAPIFakeAsync|ISAPILogNotSupported|ISAPIReadAheadBuffer|KeepAlive|KeepAliveTimeout|KeptBodySize|LanguagePriority|LDAPCacheEntries|LDAPCacheTTL|LDAPConnectionPoolTTL|LDAPConnectionTimeout|LDAPLibraryDebug|LDAPOpCacheEntries|LDAPOpCacheTTL|LDAPReferralHopLimit|LDAPReferrals|LDAPRetries|LDAPRetryDelay|LDAPSharedCacheFile|LDAPSharedCacheSize|LDAPTimeout|LDAPTrustedClientCert|LDAPTrustedGlobalCert|LDAPTrustedMode|LDAPVerifyServerCert|LimitInternalRecursion|LimitRequestBody|LimitRequestFields|LimitRequestFieldSize|LimitRequestLine|LimitXMLRequestBody|Listen|ListenBackLog|LoadFile|LoadModule|LogFormat|LogLevel|LogMessage|LuaAuthzProvider|LuaCodeCache|LuaHookAccessChecker|LuaHookAuthChecker|LuaHookCheckUserID|LuaHookFixups|LuaHookInsertFilter|LuaHookLog|LuaHookMapToStorage|LuaHookTranslateName|LuaHookTypeChecker|LuaInherit|LuaInputFilter|LuaMapHandler|LuaOutputFilter|LuaPackageCPath|LuaPackagePath|LuaQuickHandler|LuaRoot|LuaScope|MaxConnectionsPerChild|MaxKeepAliveRequests|MaxMemFree|MaxRangeOverlaps|MaxRangeReversals|MaxRanges|MaxRequestWorkers|MaxSpareServers|MaxSpareThreads|MaxThreads|MergeTrailers|MetaDir|MetaFiles|MetaSuffix|MimeMagicFile|MinSpareServers|MinSpareThreads|MMapFile|ModemStandard|ModMimeUsePathInfo|MultiviewsMatch|Mutex|NameVirtualHost|NoProxy|NWSSLTrustedCerts|NWSSLUpgradeable|Options|Order|OutputSed|PassEnv|PidFile|PrivilegesMode|Protocol|ProtocolEcho|ProxyAddHeaders|ProxyBadHeader|ProxyBlock|ProxyDomain|ProxyErrorOverride|ProxyExpressDBMFile|ProxyExpressDBMType|ProxyExpressEnable|ProxyFtpDirCharset|ProxyFtpEscapeWildcards|ProxyFtpListOnWildcard|ProxyHTMLBufSize|ProxyHTMLCharsetOut|ProxyHTMLDocType|ProxyHTMLEnable|ProxyHTMLEvents|ProxyHTMLExtended|ProxyHTMLFixups|ProxyHTMLInterp|ProxyHTMLLinks|ProxyHTMLMeta|ProxyHTMLStripComments|ProxyHTMLURLMap|ProxyIOBufferSize|ProxyMaxForwards|ProxyPass|ProxyPassInherit|ProxyPassInterpolateEnv|ProxyPassMatch|ProxyPassReverse|ProxyPassReverseCookieDomain|ProxyPassReverseCookiePath|ProxyPreserveHost|ProxyReceiveBufferSize|ProxyRemote|ProxyRemoteMatch|ProxyRequests|ProxySCGIInternalRedirect|ProxySCGISendfile|ProxySet|ProxySourceAddress|ProxyStatus|ProxyTimeout|ProxyVia|ReadmeName|ReceiveBufferSize|Redirect|RedirectMatch|RedirectPermanent|RedirectTemp|ReflectorHeader|RemoteIPHeader|RemoteIPInternalProxy|RemoteIPInternalProxyList|RemoteIPProxiesHeader|RemoteIPTrustedProxy|RemoteIPTrustedProxyList|RemoveCharset|RemoveEncoding|RemoveHandler|RemoveInputFilter|RemoveLanguage|RemoveOutputFilter|RemoveType|RequestHeader|RequestReadTimeout|Require|RewriteBase|RewriteCond|RewriteEngine|RewriteMap|RewriteOptions|RewriteRule|RLimitCPU|RLimitMEM|RLimitNPROC|Satisfy|ScoreBoardFile|Script|ScriptAlias|ScriptAliasMatch|ScriptInterpreterSource|ScriptLog|ScriptLogBuffer|ScriptLogLength|ScriptSock|SecureListen|SeeRequestTail|SendBufferSize|ServerAdmin|ServerAlias|ServerLimit|ServerName|ServerPath|ServerRoot|ServerSignature|ServerTokens|Session|SessionCookieName|SessionCookieName2|SessionCookieRemove|SessionCryptoCipher|SessionCryptoDriver|SessionCryptoPassphrase|SessionCryptoPassphraseFile|SessionDBDCookieName|SessionDBDCookieName2|SessionDBDCookieRemove|SessionDBDDeleteLabel|SessionDBDInsertLabel|SessionDBDPerUser|SessionDBDSelectLabel|SessionDBDUpdateLabel|SessionEnv|SessionExclude|SessionHeader|SessionInclude|SessionMaxAge|SetEnv|SetEnvIf|SetEnvIfExpr|SetEnvIfNoCase|SetHandler|SetInputFilter|SetOutputFilter|SSIEndTag|SSIErrorMsg|SSIETag|SSILastModified|SSILegacyExprParser|SSIStartTag|SSITimeFormat|SSIUndefinedEcho|SSLCACertificateFile|SSLCACertificatePath|SSLCADNRequestFile|SSLCADNRequestPath|SSLCARevocationCheck|SSLCARevocationFile|SSLCARevocationPath|SSLCertificateChainFile|SSLCertificateFile|SSLCertificateKeyFile|SSLCipherSuite|SSLCompression|SSLCryptoDevice|SSLEngine|SSLFIPS|SSLHonorCipherOrder|SSLInsecureRenegotiation|SSLOCSPDefaultResponder|SSLOCSPEnable|SSLOCSPOverrideResponder|SSLOCSPResponderTimeout|SSLOCSPResponseMaxAge|SSLOCSPResponseTimeSkew|SSLOCSPUseRequestNonce|SSLOpenSSLConfCmd|SSLOptions|SSLPassPhraseDialog|SSLProtocol|SSLProxyCACertificateFile|SSLProxyCACertificatePath|SSLProxyCARevocationCheck|SSLProxyCARevocationFile|SSLProxyCARevocationPath|SSLProxyCheckPeerCN|SSLProxyCheckPeerExpire|SSLProxyCheckPeerName|SSLProxyCipherSuite|SSLProxyEngine|SSLProxyMachineCertificateChainFile|SSLProxyMachineCertificateFile|SSLProxyMachineCertificatePath|SSLProxyProtocol|SSLProxyVerify|SSLProxyVerifyDepth|SSLRandomSeed|SSLRenegBufferSize|SSLRequire|SSLRequireSSL|SSLSessionCache|SSLSessionCacheTimeout|SSLSessionTicketKeyFile|SSLSRPUnknownUserSeed|SSLSRPVerifierFile|SSLStaplingCache|SSLStaplingErrorCacheTimeout|SSLStaplingFakeTryLater|SSLStaplingForceURL|SSLStaplingResponderTimeout|SSLStaplingResponseMaxAge|SSLStaplingResponseTimeSkew|SSLStaplingReturnResponderErrors|SSLStaplingStandardCacheTimeout|SSLStrictSNIVHostCheck|SSLUserName|SSLUseStapling|SSLVerifyClient|SSLVerifyDepth|StartServers|StartThreads|Substitute|Suexec|SuexecUserGroup|ThreadLimit|ThreadsPerChild|ThreadStackSize|TimeOut|TraceEnable|TransferLog|TypesConfig|UnDefine|UndefMacro|UnsetEnv|Use|UseCanonicalName|UseCanonicalPhysicalPort|User|UserDir|VHostCGIMode|VHostCGIPrivs|VHostGroup|VHostPrivs|VHostSecure|VHostUser|VirtualDocumentRoot|VirtualDocumentRootIP|VirtualScriptAlias|VirtualScriptAliasIP|WatchdogInterval|XBitHack|xml2EncAlias|xml2EncDefault|xml2StartParse'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$)|(?P[^#]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = '' - return pattern - - @staticmethod - def operator_regex(): - pattern = '' - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(ApacheConf.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in ApacheConf.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def keywords(cls) -> list: + # Build the long keyword list from smaller string pieces to be safe + kw = ( + 'AcceptFilter|AcceptPathInfo|AccessFileName|Action|AddAlt|AddAltByEncoding|AddAltByType|AddCharset|' + 'AddDefaultCharset|AddDescription|AddEncoding|AddHandler|AddIcon|AddIconByEncoding|AddIconByType|AddInputFilter|' + 'AddLanguage|AddModuleInfo|AddOutputFilter|AddOutputFilterByType|AddType|Alias|AliasMatch|Allow|AllowCONNECT|AllowEncodedSlashes|' + 'AllowMethods|AllowOverride|AllowOverrideList|Anonymous|Anonymous_LogEmail|Anonymous_MustGiveEmail|Anonymous_NoUserID|Anonymous_VerifyEmail|AsyncRequestWorkerFactor|' + 'AuthBasicAuthoritative|AuthBasicFake|AuthBasicProvider|AuthBasicUseDigestAlgorithm|AuthDBDUserPWQuery|AuthDBDUserRealmQuery|AuthDBMGroupFile|AuthDBMType|' + 'AuthDBMUserFile|AuthDigestAlgorithm|AuthDigestDomain|AuthDigestNonceLifetime|AuthDigestProvider|AuthDigestQop|AuthDigestShmemSize|AuthFormAuthoritative|' + 'AuthFormBody|AuthFormDisableNoStore|AuthFormFakeBasicAuth|AuthFormLocation|AuthFormLoginRequiredLocation|AuthFormLoginSuccessLocation|AuthFormLogoutLocation|AuthFormMethod|AuthFormMimetype|' + 'AuthFormPassword|AuthFormProvider|AuthFormSitePassphrase|AuthFormSize|AuthFormUsername|AuthGroupFile|AuthLDAPAuthorizePrefix|AuthLDAPBindAuthoritative|AuthLDAPBindDN|' + 'AuthLDAPBindPassword|AuthLDAPCharsetConfig|AuthLDAPCompareAsUser|AuthLDAPCompareDNOnServer|AuthLDAPDereferenceAliases|AuthLDAPGroupAttribute|AuthLDAPGroupAttributeIsDN|' + 'AuthLDAPInitialBindAsUser|AuthLDAPInitialBindPattern|AuthLDAPMaxSubGroupDepth|AuthLDAPRemoteUserAttribute|AuthLDAPRemoteUserIsDN|AuthLDAPSearchAsUser|AuthLDAPSubGroupAttribute|' + 'AuthLDAPSubGroupClass|AuthLDAPUrl|AuthMerging|AuthName|AuthnCacheContext|AuthnCacheEnable|AuthnCacheProvideFor|AuthnCacheSOCache|AuthnCacheTimeout|AuthnzFcgiCheckAuthnProvider|' + 'AuthnzFcgiDefineProvider|AuthType|AuthUserFile|AuthzDBDLoginToReferer|AuthzDBDQuery|AuthzDBDRedirectQuery|AuthzDBMType|AuthzSendForbiddenOnFailure|BalancerGrowth|BalancerInherit|' + 'BalancerMember|BalancerPersist|BrowserMatch|BrowserMatchNoCase|BufferedLogs|BufferSize|CacheDefaultExpire|CacheDetailHeader|CacheDirLength|CacheDirLevels|CacheDisable|' + 'CacheEnable|CacheFile|CacheHeader|CacheIgnoreCacheControl|CacheIgnoreHeaders|CacheIgnoreNoLastMod|CacheIgnoreQueryString|CacheIgnoreURLSessionIdentifiers|CacheKeyBaseURL|' + 'CacheLastModifiedFactor|CacheLock|CacheLockMaxAge|CacheLockPath|CacheMaxExpire|CacheMaxFileSize|CacheMinExpire|CacheMinFileSize|CacheNegotiatedDocs|CacheQuickHandler|' + 'CacheReadSize|CacheReadTime|CacheRoot|CacheSocache|CacheSocacheMaxSize|CacheSocacheMaxTime|CacheSocacheMinTime|CacheSocacheReadSize|CacheSocacheReadTime|CacheStaleOnError|' + 'CacheStoreExpired|CacheStoreNoStore|CacheStorePrivate|CGIDScriptTimeout|CGIMapExtension|CharsetDefault|CharsetOptions|CharsetSourceEnc|CheckCaseOnly|CheckSpelling|' + 'ChrootDir|ContentDigest|CookieDomain|CookieExpires|CookieName|CookieStyle|CookieTracking|CoreDumpDirectory|CustomLog|Dav|DavDepthInfinity|DavGenericLockDB|DavLockDB|DavMinTimeout|' + 'DBDExptime|DBDInitSQL|DBDKeep|DBDMax|DBDMin|DBDParams|DBDPersist|DBDPrepareSQL|DBDriver|DefaultIcon|DefaultLanguage|DefaultRuntimeDir|DefaultType|Define|' + 'DeflateBufferSize|DeflateCompressionLevel|DeflateFilterNote|DeflateInflateLimitRequestBody|DeflateInflateRatioBurst|DeflateInflateRatioLimit|DeflateMemLevel|DeflateWindowSize|' + 'Deny|DirectoryCheckHandler|DirectoryIndex|DirectoryIndexRedirect|DirectorySlash|DocumentRoot|DTracePrivileges|DumpIOInput|DumpIOOutput|EnableExceptionHook|EnableMMAP|EnableSendfile|' + 'Error|ErrorDocument|ErrorLog|ErrorLogFormat|Example|ExpiresActive|ExpiresByType|ExpiresDefault|ExtendedStatus|ExtFilterDefine|ExtFilterOptions|FallbackResource|FileETag|FilterChain|' + 'FilterDeclare|FilterProtocol|FilterProvider|FilterTrace|ForceLanguagePriority|ForceType|ForensicLog|GprofDir|GracefulShutdownTimeout|Group|Header|HeaderName|HeartbeatAddress|' + 'HeartbeatListen|HeartbeatMaxServers|HeartbeatStorage|HostnameLookups|IdentityCheck|IdentityCheckTimeout|ImapBase|ImapDefault|ImapMenu|Include|IncludeOptional|IndexHeadInsert|' + 'IndexIgnore|IndexIgnoreReset|IndexOptions|IndexOrderDefault|IndexStyleSheet|InputSed|ISAPIAppendLogToErrors|ISAPIAppendLogToQuery|ISAPICacheFile|ISAPIFakeAsync|ISAPILogNotSupported|ISAPIReadAheadBuffer|' + 'KeepAlive|KeepAliveTimeout|KeptBodySize|LanguagePriority|LDAPCacheEntries|LDAPCacheTTL|LDAPConnectionPoolTTL|LDAPConnectionTimeout|LDAPLibraryDebug|LDAPOpCacheEntries|LDAPOpCacheTTL|' + 'LDAPReferralHopLimit|LDAPReferrals|LDAPRetries|LDAPRetryDelay|LDAPSharedCacheFile|LDAPSharedCacheSize|LDAPTimeout|LDAPTrustedClientCert|LDAPTrustedGlobalCert|LDAPTrustedMode|LDAPVerifyServerCert|' + 'LimitInternalRecursion|LimitRequestBody|LimitRequestFields|LimitRequestFieldSize|LimitRequestLine|LimitXMLRequestBody|Listen|ListenBackLog|LoadFile|LoadModule|LogFormat|LogLevel|LogMessage|LuaAuthzProvider|' + 'LuaCodeCache|LuaHookAccessChecker|LuaHookAuthChecker|LuaHookCheckUserID|LuaHookFixups|LuaHookInsertFilter|LuaHookLog|LuaHookMapToStorage|LuaHookTranslateName|LuaHookTypeChecker|LuaInherit|' + 'LuaInputFilter|LuaMapHandler|LuaOutputFilter|LuaPackageCPath|LuaPackagePath|LuaQuickHandler|LuaRoot|LuaScope|MaxConnectionsPerChild|MaxKeepAliveRequests|MaxMemFree|MaxRangeOverlaps|' + 'MaxRangeReversals|MaxRanges|MaxRequestWorkers|MaxSpareServers|MaxSpareThreads|MaxThreads|MergeTrailers|MetaDir|MetaFiles|MetaSuffix|MimeMagicFile|MinSpareServers|MinSpareThreads|' + 'MMapFile|ModemStandard|ModMimeUsePathInfo|MultiviewsMatch|Mutex|NameVirtualHost|NoProxy|NWSSLTrustedCerts|NWSSLUpgradeable|Options|Order|OutputSed|PassEnv|PidFile|PrivilegesMode|Protocol|' + 'ProtocolEcho|ProxyAddHeaders|ProxyBadHeader|ProxyBlock|ProxyDomain|ProxyErrorOverride|ProxyExpressDBMFile|ProxyExpressDBMType|ProxyExpressEnable|ProxyFtpDirCharset|ProxyFtpEscapeWildcards|' + 'ProxyFtpListOnWildcard|ProxyHTMLBufSize|ProxyHTMLCharsetOut|ProxyHTMLDocType|ProxyHTMLEnable|ProxyHTMLEvents|ProxyHTMLExtended|ProxyHTMLFixups|ProxyHTMLInterp|ProxyHTMLLinks|ProxyHTMLMeta|' + 'ProxyHTMLStripComments|ProxyHTMLURLMap|ProxyIOBufferSize|ProxyMaxForwards|ProxyPass|ProxyPassInherit|ProxyPassInterpolateEnv|ProxyPassMatch|ProxyPassReverse|ProxyPassReverseCookieDomain|' + 'ProxyPassReverseCookiePath|ProxyPreserveHost|ProxyReceiveBufferSize|ProxyRemote|ProxyRemoteMatch|ProxyRequests|ProxySCGIInternalRedirect|ProxySCGISendfile|ProxySet|ProxySourceAddress|' + 'ProxyStatus|ProxyTimeout|ProxyVia|ReadmeName|ReceiveBufferSize|Redirect|RedirectMatch|RedirectPermanent|RedirectTemp|ReflectorHeader|RemoteIPHeader|RemoteIPInternalProxy|RemoteIPInternalProxyList|' + 'RemoteIPProxiesHeader|RemoteIPTrustedProxy|RemoteIPTrustedProxyList|RemoveCharset|RemoveEncoding|RemoveHandler|RemoveInputFilter|RemoveLanguage|RemoveOutputFilter|RemoveType|RequestHeader|' + 'RequestReadTimeout|Require|RewriteBase|RewriteCond|RewriteEngine|RewriteMap|RewriteOptions|RewriteRule|RLimitCPU|RLimitMEM|RLimitNPROC|Satisfy|ScoreBoardFile|Script|ScriptAlias|ScriptAliasMatch|' + 'ScriptInterpreterSource|ScriptLog|ScriptLogBuffer|ScriptLogLength|ScriptSock|SecureListen|SeeRequestTail|SendBufferSize|ServerAdmin|ServerAlias|ServerLimit|ServerName|ServerPath|ServerRoot|' + 'ServerSignature|ServerTokens|Session|SessionCookieName|SessionCookieName2|SessionCookieRemove|SessionCryptoCipher|SessionCryptoDriver|SessionCryptoPassphrase|SessionCryptoPassphraseFile|' + 'SessionDBDCookieName|SessionDBDCookieName2|SessionDBDCookieRemove|SessionDBDDeleteLabel|SessionDBDInsertLabel|SessionDBDPerUser|SessionDBDSelectLabel|SessionDBDUpdateLabel|SessionEnv|' + 'SessionExclude|SessionHeader|SessionInclude|SessionMaxAge|SetEnv|SetEnvIf|SetEnvIfExpr|SetEnvIfNoCase|SetHandler|SetInputFilter|SetOutputFilter|SSIEndTag|SSIErrorMsg|SSIETag|SSILastModified|' + 'SSILegacyExprParser|SSIStartTag|SSITTimeFormat|SSIUndefinedEcho|SSLCACertificateFile|SSLCACertificatePath|SSLCADNRequestFile|SSLCADNRequestPath|SSLCARevocationCheck|SSLCARevocationFile|' + 'SSLCARevocationPath|SSLCertificateChainFile|SSLCertificateFile|SSLCertificateKeyFile|SSLCipherSuite|SSLCompression|SSLCryptoDevice|SSLEngine|SSLFIPS|SSLHonorCipherOrder|SSLInsecureRenegotiation|' + 'SSLOCSPDefaultResponder|SSLOCSPEnable|SSLOCSPOverrideResponder|SSLOCSPResponderTimeout|SSLOCSPResponseMaxAge|SSLOCSPResponseTimeSkew|SSLOCSPUseRequestNonce|SSLOpenSSLConfCmd|SSLOptions|' + 'SSLPassPhraseDialog|SSLProtocol|SSLProxyCACertificateFile|SSLProxyCACertificatePath|SSLProxyCARevocationCheck|SSLProxyCARevocationFile|SSLProxyCARevocationPath|SSLProxyCheckPeerCN|' + 'SSLProxyCheckPeerExpire|SSLProxyCheckPeerName|SSLProxyCipherSuite|SSLProxyEngine|SSLProxyMachineCertificateChainFile|SSLProxyMachineCertificateFile|SSLProxyMachineCertificatePath|' + 'SSLProxyProtocol|SSLProxyVerify|SSLProxyVerifyDepth|SSLRandomSeed|SSLRenegBufferSize|SSLRequire|SSLRequireSSL|SSLSessionCache|SSLSessionCacheTimeout|SSLSessionTicketKeyFile|' + 'SSLSRPUnknownUserSeed|SSLSRPVerifierFile|SSLStaplingCache|SSLStaplingErrorCacheTimeout|SSLStaplingFakeTryLater|SSLStaplingForceURL|SSLStaplingResponderTimeout|SSLStaplingResponseMaxAge|' + 'SSLStaplingResponseTimeSkew|SSLStaplingReturnResponderErrors|SSLOCSPStandardCacheTimeout|SSLStrictSNIVHostCheck|SSLUserName|SSLUseStapling|SSLVerifyClient|SSLVerifyDepth|StartServers|' + 'StartThreads|Substitute|Suexec|SuexecUserGroup|ThreadLimit|ThreadsPerChild|ThreadStackSize|TimeOut|TraceEnable|TransferLog|TypesConfig|UnDefine|UndefMacro|UnsetEnv|Use|UseCanonicalName|' + 'UseCanonicalPhysicalPort|User|UserDir|VHostCGIMode|VHostCGIPrivs|VHostGroup|VHostPrivs|VHostSecure|VHostUser|VirtualDocumentRoot|VirtualDocumentRootIP|VirtualScriptAlias|VirtualScriptAliasIP|' + 'WatchdogInterval|XBitHack|xml2EncAlias|xml2EncDefault|xml2StartParse' + ) + return kw.split('|') + + @classmethod + def comment_regex(cls): + """ + ApacheConf comments start with # and go to the end of the line. + Capture strings and non-comment segments. + + :return: Compiled regex with 'comment' and 'noncomment' named groups. + :rtype: re.Pattern + """ + return re.compile(r'(?P#.*?$)|(?P[^#]*)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r"\b\d+\b") + + @classmethod + def operator_regex(cls): + return re.compile(r".") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) + + + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + res = super().remove_comments(source_code, isList=isList) if isList: - return result - return ''.join(result) + return res + return res - @staticmethod - def remove_keywords(source: str): + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) return re.sub(re.compile(ApacheConf.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/apl.py b/src/PyReprism/languages/apl.py index c72a408..09e194d 100644 --- a/src/PyReprism/languages/apl.py +++ b/src/PyReprism/languages/apl.py @@ -1,49 +1,41 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Apl: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Apl(BaseLanguage): + """APL language helper. + + This class provides minimal support for APL-like source files: file + extension metadata, comment matching, and a conservative list of common + APL primitives/operators for keyword removal. The keyword list is kept + intentionally small; we can move to a data file later if you want a + comprehensive set. + """ + + @classmethod + def file_extension(cls) -> str: return extension.apl - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P⍝.*?$)|(?P[^⍝]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = '' - return pattern - - @staticmethod - def operator_regex(): - pattern = '' - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Apl.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Apl.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Apl.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + # A conservative set of commonly-used APL symbols / primitives. + return '⍝|⍴|⍳|⌈|⌊|⌿|⍟|∘|⍎|⍕'.split('|') + + @classmethod + def comment_regex(cls): + # APL uses the '⍝' glyph to start comments. + return re.compile(r'(?P⍝.*?$)|(?P[^⍝]*)', re.MULTILINE) + + # Use BaseLanguage.number_regex and operator_regex by default. + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str): + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/applescript.py b/src/PyReprism/languages/applescript.py index 6726784..c7ab079 100644 --- a/src/PyReprism/languages/applescript.py +++ b/src/PyReprism/languages/applescript.py @@ -1,49 +1,57 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class AppleScript: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class AppleScript(BaseLanguage): + """AppleScript language helper. + + Provides file extension metadata, comment/number/operator regexes, and a + conservative keywords list for AppleScript. The comment matching supports + single-line comments starting with `--` or `#` and block comments using + `(* ... *)`. + """ + + @classmethod + def file_extension(cls) -> str: return extension.applescript - @staticmethod - def keywords() -> list: - keyword = 'about|above|after|against|apart from|around|aside from|at|back|before|beginning|behind|below|beneath|beside|between|but|by|considering|continue|copy|does|eighth|else|end|equal|error|every|exit|false|fifth|first|for|fourth|from|front|get|given|global|if|ignoring|in|instead of|into|is|it|its|last|local|me|middle|my|ninth|of|on|onto|out of|over|prop|property|put|repeat|return|returning|second|set|seventh|since|sixth|some|tell|tenth|that|the|then|third|through|thru|timeout|times|to|transaction|true|try|until|where|while|whose|with|without|alias|application|boolean|class|constant|date|file|integer|list|number|POSIX file|real|record|reference|RGB color|script|text|centimetres|centimeters|feet|inches|kilometres|kilometers|metres|meters|miles|yards|square feet|square kilometres|square kilometers|square metres|square meters|square miles|square yards|cubic centimetres|cubic centimeters|cubic feet|cubic inches|cubic metres|cubic meters|cubic yards|gallons|litres|liters|quarts|grams|kilograms|ounces|pounds|degrees Celsius|degrees Fahrenheit|degrees Kelvin)'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P--.*?$|#.*?$|\(\*[\s\S]*?\*\)|\(\*.*?$|^.*?\*\))|(?P[^#\-\(\*]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:e-?\d+)?\b') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'[&=≠≤≥*+\-\/÷^]|[<>]=?') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(AppleScript.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in AppleScript.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(AppleScript.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + # A conservative (cleaned) set of AppleScript keywords and types. This + # can be extended or moved to a data file if you want the complete set. + kw = ( + 'about|above|after|against|around|aside from|at|back|before|beginning|behind|below|' + 'beneath|beside|between|but|by|continue|copy|does|else|end|error|every|exit|false|' + 'first|for|from|get|global|if|in|into|is|it|its|last|local|me|my|of|on|out|over|prop|' + 'property|put|repeat|return|set|since|some|tell|that|the|then|through|to|true|try|until|' + 'where|while|with|without|alias|application|boolean|class|constant|date|file|integer|list|' + 'number|real|record|reference|script|text|centimetres|centimeters|feet|inches|kilometres|' + 'kilometers|metres|meters|miles|yards|square|cubic|gallons|litres|liters|quarts|grams|' + 'kilograms|ounces|pounds|degrees|Celsius|Fahrenheit|Kelvin' + ) + return kw.split('|') + + @classmethod + def comment_regex(cls): + # Support: -- comment, # comment, and block comments (* ... *) + return re.compile(r'(?P--.*?$|#.*?$|\(\*[\s\S]*?\*\))|(?P[^#\-\(\*)]+)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?\b') + + @classmethod + def operator_regex(cls): + return re.compile(r'[&=≠≤≥*+\-/÷^]|[<>]=?') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str): + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/arduino.py b/src/PyReprism/languages/arduino.py index 9bb7753..e645f81 100644 --- a/src/PyReprism/languages/arduino.py +++ b/src/PyReprism/languages/arduino.py @@ -1,49 +1,56 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Arduino: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.arduino - - @staticmethod - def keywords() -> list: - keyword = 'setup|if|else|while|do|for|return|in|instanceof|default|function|loop|goto|switch|case|new|try|throw|catch|finally|null|break|continue|boolean|bool|void|byte|word|string|String|array|int|long|integer|double|KeyboardController|MouseController|SoftwareSerial|EthernetServer|EthernetClient|LiquidCrystal|LiquidCrystal_I2C|RobotControl|GSMVoiceCall|EthernetUDP|EsploraTFT|HttpClient|RobotMotor|WiFiClient|GSMScanner|FileSystem|Scheduler|GSMServer|YunClient|YunServer|IPAddress|GSMClient|GSMModem|Keyboard|Ethernet|Console|GSMBand|Esplora|Stepper|Process|WiFiUDP|GSM_SMS|Mailbox|USBHost|Firmata|PImage|Client|Server|GSMPIN|FileIO|Bridge|Serial|EEPROM|Stream|Mouse|Audio|Servo|File|Task|GPRS|WiFi|Wire|TFT|GSM|SPI|SD|runShellCommandAsynchronously|analogWriteResolution|retrieveCallingNumber|printFirmwareVersion|analogReadResolution|sendDigitalPortPair|noListenOnLocalhost|readJoystickButton|setFirmwareVersion|readJoystickSwitch|scrollDisplayRight|getVoiceCallStatus|scrollDisplayLeft|writeMicroseconds|delayMicroseconds|beginTransmission|getSignalStrength|runAsynchronously|getAsynchronously|listenOnLocalhost|getCurrentCarrier|readAccelerometer|messageAvailable|sendDigitalPorts|lineFollowConfig|countryNameWrite|runShellCommand|readStringUntil|rewindDirectory|readTemperature|setClockDivider|readLightSensor|endTransmission|analogReference|detachInterrupt|countryNameRead|attachInterrupt|encryptionType|readBytesUntil|robotNameWrite|readMicrophone|robotNameRead|cityNameWrite|userNameWrite|readJoystickY|readJoystickX|mouseReleased|openNextFile|scanNetworks|noInterrupts|digitalWrite|beginSpeaker|mousePressed|isActionDone|mouseDragged|displayLogos|noAutoscroll|addParameter|remoteNumber|getModifiers|keyboardRead|userNameRead|waitContinue|processInput|parseCommand|printVersion|readNetworks|writeMessage|blinkVersion|cityNameRead|readMessage|setDataMode|parsePacket|isListening|setBitOrder|beginPacket|isDirectory|motorsWrite|drawCompass|digitalRead|clearScreen|serialEvent|rightToLeft|setTextSize|leftToRight|requestFrom|keyReleased|compassRead|analogWrite|interrupts|WiFiServer|disconnect|playMelody|parseFloat|autoscroll|getPINUsed|setPINUsed|setTimeout|sendAnalog|readSlider|analogRead|beginWrite|createChar|motorsStop|keyPressed|tempoWrite|readButton|subnetMask|debugPrint|macAddress|writeGreen|randomSeed|attachGPRS|readString|sendString|remotePort|releaseAll|mouseMoved|background|getXChange|getYChange|answerCall|getResult|voiceCall|endPacket|constrain|getSocket|writeJSON|getButton|available|connected|findUntil|readBytes|exitValue|readGreen|writeBlue|startLoop|IPAddress|isPressed|sendSysex|pauseMode|gatewayIP|setCursor|getOemKey|tuneWrite|noDisplay|loadImage|switchPIN|onRequest|onReceive|changePIN|playFile|noBuffer|parseInt|overflow|checkPIN|knobRead|beginTFT|bitClear|updateIR|bitWrite|position|writeRGB|highByte|writeRed|setSpeed|readBlue|noStroke|remoteIP|transfer|shutdown|hangCall|beginSMS|endWrite|attached|maintain|noCursor|checkReg|checkPUK|shiftOut|isValid|shiftIn|pulseIn|connect|println|localIP|pinMode|getIMEI|display|noBlink|process|getBand|running|beginSD|drawBMP|lowByte|setBand|release|bitRead|prepare|pointTo|readRed|setMode|noFill|remove|listen|stroke|detach|attach|noTone|exists|buffer|height|bitSet|circle|config|cursor|random|IRread|setDNS|endSMS|getKey|micros|millis|begin|print|write|ready|flush|width|isPIN|blink|clear|press|mkdir|rmdir|close|point|yield|image|BSSID|click|delay|read|text|move|peek|beep|rect|line|open|seek|fill|size|turn|stop|home|find|step|tone|sqrt|RSSI|SSID|end|bit|tan|cos|sin|pow|map|abs|max|min|get|run|put'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern +@LanguageRegistry.register +class Arduino(BaseLanguage): + """Arduino language helpers (migrated to BaseLanguage).""" - @staticmethod - def number_regex(): - pattern = re.compile(r'(?:\b0x[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?)[ful]*') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'--?|\+\+?|!=?|<{1,2}=?|>{1,2}=?|->|:{1,2}|={1,2}|\^|~|%|&{1,2}|\|\|?|\?|\*|\/|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Arduino.keywords()) + r')\b') + @classmethod + def file_extension(cls) -> str: + return extension.arduino - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def keywords(cls) -> list: + # Preserve the long keyword list from the original implementation. + return 'setup|if|else|while|do|for|return|in|instanceof|default|function|loop|goto|switch|case|new|try|throw|catch|finally|null|break|continue|boolean|bool|void|byte|word|string|String|array|int|long|integer|double|KeyboardController|MouseController|SoftwareSerial|EthernetServer|EthernetClient|LiquidCrystal|LiquidCrystal_I2C|RobotControl|GSMVoiceCall|EthernetUDP|EsploraTFT|HttpClient|RobotMotor|WiFiClient|GSMScanner|FileSystem|Scheduler|GSMServer|YunClient|YunServer|IPAddress|GSMClient|GSMModem|Keyboard|Ethernet|Console|GSMBand|Esplora|Stepper|Process|WiFiUDP|GSM_SMS|Mailbox|USBHost|Firmata|PImage|Client|Server|GSMPIN|FileIO|Bridge|Serial|EEPROM|Stream|Mouse|Audio|Servo|File|Task|GPRS|WiFi|Wire|TFT|GSM|SPI|SD|runShellCommandAsynchronously|analogWriteResolution|retrieveCallingNumber|printFirmwareVersion|analogReadResolution|sendDigitalPortPair|noListenOnLocalhost|readJoystickButton|setFirmwareVersion|readJoystickSwitch|scrollDisplayRight|getVoiceCallStatus|scrollDisplayLeft|writeMicroseconds|delayMicroseconds|beginTransmission|getSignalStrength|runAsynchronously|getAsynchronously|listenOnLocalhost|getCurrentCarrier|readAccelerometer|messageAvailable|sendDigitalPorts|lineFollowConfig|countryNameWrite|runShellCommand|readStringUntil|rewindDirectory|readTemperature|setClockDivider|readLightSensor|endTransmission|analogReference|detachInterrupt|countryNameRead|attachInterrupt|encryptionType|readBytesUntil|robotNameWrite|readMicrophone|robotNameRead|cityNameWrite|userNameWrite|readJoystickY|readJoystickX|mouseReleased|openNextFile|scanNetworks|noInterrupts|digitalWrite|beginSpeaker|mousePressed|isActionDone|mouseDragged|displayLogos|noAutoscroll|addParameter|remoteNumber|getModifiers|keyboardRead|userNameRead|waitContinue|processInput|parseCommand|printVersion|readNetworks|writeMessage|blinkVersion|cityNameRead|readMessage|setDataMode|parsePacket|isListening|setBitOrder|beginPacket|isDirectory|motorsWrite|drawCompass|digitalRead|clearScreen|serialEvent|rightToLeft|setTextSize|leftToRight|requestFrom|keyReleased|compassRead|analogWrite|interrupts|WiFiServer|disconnect|playMelody|parseFloat|autoscroll|getPINUsed|setPINUsed|setTimeout|sendAnalog|readSlider|analogRead|beginWrite|createChar|motorsStop|keyPressed|tempoWrite|readButton|subnetMask|debugPrint|macAddress|writeGreen|randomSeed|attachGPRS|readString|sendString|remotePort|releaseAll|mouseMoved|background|getXChange|getYChange|answerCall|getResult|voiceCall|endPacket|constrain|getSocket|writeJSON|getButton|available|connected|findUntil|readBytes|exitValue|readGreen|writeBlue|startLoop|IPAddress|isPressed|sendSysex|pauseMode|gatewayIP|setCursor|getOemKey|tuneWrite|noDisplay|loadImage|switchPIN|onRequest|onReceive|changePIN|playFile|noBuffer|parseInt|overflow|checkPIN|knobRead|beginTFT|bitClear|updateIR|bitWrite|position|writeRGB|highByte|writeRed|setSpeed|readBlue|noStroke|remoteIP|transfer|shutdown|hangCall|beginSMS|endWrite|attached|maintain|noCursor|checkReg|checkPUK|shiftOut|isValid|shiftIn|pulseIn|connect|println|localIP|pinMode|getIMEI|display|noBlink|process|getBand|running|beginSD|drawBMP|lowByte|setBand|release|bitRead|prepare|pointTo|readRed|setMode|noFill|remove|listen|stroke|detach|attach|noTone|exists|buffer|height|bitSet|circle|config|cursor|random|IRread|setDNS|endSMS|getKey|micros|millis|begin|print|write|ready|flush|width|isPIN|blink|clear|press|mkdir|rmdir|close|point|yield|image|BSSID|click|delay|read|text|move|peek|beep|rect|line|open|seek|fill|size|turn|stop|home|find|step|tone|sqrt|RSSI|SSID|end|bit|tan|cos|sin|pow|map|abs|max|min|get|run|put'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Preserve original behavior: capture // line comments and C-style block + # comments. Keep strings in 'noncomment'. + return re.compile( + r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\'\"\{\}]+)", + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?:\b0x[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?)[ful]*', re.IGNORECASE) + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'--?|\+\+?|!=?|<{1,2}=?|>{1,2}=?|->|:{1,2}|={1,2}|\^|~|%|&{1,2}|\|\|?|\?|\*|\/|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b') + + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Preserve original behavior: append only truthy noncomment groups. result = [] - for match in Arduino.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Arduino.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/aspnet.py b/src/PyReprism/languages/aspnet.py index 6d25387..09d9551 100644 --- a/src/PyReprism/languages/aspnet.py +++ b/src/PyReprism/languages/aspnet.py @@ -1,49 +1,56 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Aspnet: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.aspnet - - @staticmethod - def keywords() -> list: - keyword = 'Assembly|Control|Implements|Import|Master(?:Type)?|OutputCache|Page|PreviousPageType|Reference|Register'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/||/\*.*?$|^.*?\*/|)|(?P[^/ str: + return extension.aspnet - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def keywords(cls) -> list: + return 'Assembly|Control|Implements|Import|Master(?:Type)?|OutputCache|Page|PreviousPageType|Reference|Register'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Match C-style block comments, // line comments and HTML comments + return re.compile( + r"(?P//.*?$|/\*[\s\S]*?\*/|)|(?P[^/ re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Preserve original behavior: only append truthy 'noncomment' groups. result = [] - for match in Aspnet.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Aspnet.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/brainfuck.py b/src/PyReprism/languages/brainfuck.py index 6e84e48..178ec04 100644 --- a/src/PyReprism/languages/brainfuck.py +++ b/src/PyReprism/languages/brainfuck.py @@ -1,50 +1,42 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class BrainFuck: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class BrainFuck(BaseLanguage): + """Brainf*ck language helper. + + Brainf*ck doesn't have textual keywords; it uses single-character operators. + This class focuses on preserving the operators and stripping all other + non-operator characters (treated as comments/non-code in many contexts). + """ + + @classmethod + def file_extension(cls) -> str: return extension.brainfuck - @staticmethod - def keywords() -> list: - # keyword = '\S+'.split('|') - # return keyword - pass - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P[^><+\-.,[\]]+)|(?P[><+\-.,[\]])') - return pattern - - @staticmethod - def number_regex(): - pattern = '' - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'[.,]') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(BrainFuck.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in BrainFuck.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(BrainFuck.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + # No keywords in Brainfuck; return empty list so keywords_regex matches nothing. + return [] + + @classmethod + def comment_regex(cls): + # Keep operator characters in 'noncomment' group and treat everything else as 'comment'. + return re.compile(r'(?P[^><+\-.,\[\]]+)|(?P[><+\-.,\[\]])') + + @classmethod + def operator_regex(cls): + return re.compile(r'[><+\-.,\[\]]') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str): + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/cpp.py b/src/PyReprism/languages/cpp.py index 318e397..7dbc676 100644 --- a/src/PyReprism/languages/cpp.py +++ b/src/PyReprism/languages/cpp.py @@ -1,7 +1,7 @@ import re from PyReprism.utils import extension -from .base.base import BaseLanguage +from .base import BaseLanguage from .registry import LanguageRegistry diff --git a/src/PyReprism/languages/crystal.py b/src/PyReprism/languages/crystal.py index 579e15a..d731e79 100644 --- a/src/PyReprism/languages/crystal.py +++ b/src/PyReprism/languages/crystal.py @@ -1,49 +1,49 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Crystal: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Crystal(BaseLanguage): + """Crystal language helper.""" + + @classmethod + def file_extension(cls) -> str: return extension.crystal - @staticmethod - def keywords() -> list: - keyword = 'abstract|alias|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|rescue|return|require|select|self|sizeof|struct|super|then|type|typeof|uninitialized|union|unless|until|when|while|with|yield|__DIR__|__END_LINE__|__FILE__|__LINE__'.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + return 'abstract|alias|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|rescue|return|require|select|self|sizeof|struct|super|then|type|typeof|uninitialized|union|unless|until|when|while|with|yield|__DIR__|__END_LINE__|__FILE__|__LINE__'.split('|') - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$|=begin[\s\S]*?=end|=begin.*?$|^.*?=end)|(?P[^#=]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P#.*?$|=begin[\s\S]*?=end)|(?P[^#=\n]+)', re.DOTALL | re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def operator_regex(): - pattern = re.compile(r'\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Crystal.keywords()) + r')\b') + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: result = [] - for match in Crystal.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Crystal.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/csharp.py b/src/PyReprism/languages/csharp.py index 4b62d4a..281ad2b 100644 --- a/src/PyReprism/languages/csharp.py +++ b/src/PyReprism/languages/csharp.py @@ -1,82 +1,53 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class CSharp: - def __init__(): - pass +@LanguageRegistry.register +class CSharp(BaseLanguage): + """C# language helper.""" - @staticmethod - def file_extension() -> str: + @classmethod + def file_extension(cls) -> str: return extension.csharp - @staticmethod - def keywords() -> list: - keyword = 'abstract|add|alias|as|ascending|async|await|base|bool|break|byte|case|catch|char|checked|class|const|continue|decimal|default|delegate|descending|do|double|dynamic|else|enum|event|explicit|extern|false|finally|fixed|float|for|foreach|from|get|global|goto|group|if|implicit|in|int|interface|internal|into|is|join|let|lock|long|namespace|new|null|object|operator|orderby|out|override|params|partial|private|protected|public|readonly|ref|remove|return|sbyte|sealed|select|set|short|sizeof|stackalloc|static|string|struct|switch|this|throw|true|try|typeof|uint|ulong|unchecked|unsafe|ushort|using|value|var|virtual|void|volatile|where|while|yield|warning|define|elif|endif|endregion|error|if|line|pragma|region|undef'.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + return 'abstract|add|alias|as|ascending|async|await|base|bool|break|byte|case|catch|char|checked|class|const|continue|decimal|default|delegate|descending|do|double|dynamic|else|enum|event|explicit|extern|false|finally|fixed|float|for|foreach|from|get|global|goto|group|if|implicit|in|int|interface|internal|into|is|join|let|lock|long|namespace|new|null|object|operator|orderby|out|override|params|partial|private|protected|public|readonly|ref|remove|return|sbyte|sealed|select|set|short|sizeof|stackalloc|static|string|struct|switch|this|throw|true|try|typeof|uint|ulong|unchecked|unsafe|ushort|using|value|var|virtual|void|volatile|where|while|yield|warning|define|elif|endif|endregion|error|if|line|pragma|region|undef'.split('|') - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\n'\"]+)", re.DOTALL | re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)f?') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)f?') - @staticmethod - def operator_regex(): - pattern = re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/=?|%=?|\^=?|[?:~])') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/?=|%=?|\^=?|[?:~])') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(CSharp.keywords()) + r')\b') + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify C# boolean literals. - - This function generates a regular expression that matches the C# boolean literals `true`, `false`, and the special constant `null`. - - :return: A compiled regex pattern to match C# boolean literals and `null`. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false|null)\b', re.IGNORECASE) - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify C# delimiters. - - This function generates a regular expression that matches C# language delimiters, which include parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, at symbols `@`, angle brackets `<`, `>`, and the question mark `?`. - - :return: A compiled regex pattern to match C# delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@<>?]') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided Python source code string. - - :param str source_code: The Python source code from which to remove comments. - :param bool isList: (Optional) A flag indicating if the input is a list of source code lines. This parameter is not used in the function logic. - :return: The source code with all comments removed. - :rtype: str - """ - return CSharp.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in CSharp.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Preserve legacy scalar behavior: substitution + strip + out = cls.comment_regex().sub(lambda m: m.groupdict().get('noncomment') or '', source_code).strip() if isList: - return result - return ''.join(result) + return [out] + return out - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(CSharp.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/dart.py b/src/PyReprism/languages/dart.py index 1449366..0953bad 100644 --- a/src/PyReprism/languages/dart.py +++ b/src/PyReprism/languages/dart.py @@ -1,74 +1,55 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Dart: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.dart - - @staticmethod - def keywords() -> list: - keyword = 'abstract|assert|async|await|break|case|catch|class|const|continue|default|deferred|do|dynamic|else|enum|export|external|extends|factory|final|finally|for|get|if|implements|import|in|library|new|null|operator|part|rethrow|return|set|static|super|switch|this|throw|try|typedef|var|void|while|with|yield|sync'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/|///.*?$)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'\bis!|\b(?:as|is)\b|\+\+|--|&&|\|\||<<=?|>>=?|~(?:\/=?)?|[+\-*\/%&^|=!<>]=?|\?') - return pattern +@LanguageRegistry.register +class Dart(BaseLanguage): + """Dart language helper migrated to BaseLanguage pattern.""" - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Dart.keywords()) + r')\b') - - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Dart language delimiters. - - This function generates a regular expression that matches Dart language delimiters, which include parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, and angle brackets `<`, `>`. + @classmethod + def file_extension(cls) -> str: + return extension.dart - :return: A compiled regex pattern to match Dart delimiters. - :rtype: re.Pattern - """ + @classmethod + def keywords(cls) -> list: + return 'abstract|assert|async|await|break|case|catch|class|const|continue|default|deferred|do|dynamic|else|enum|export|external|extends|factory|final|finally|for|get|if|implements|import|in|library|new|null|operator|part|rethrow|return|set|static|super|switch|this|throw|try|typedef|var|void|while|with|yield|sync'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Safe comment regex: capture C-style block comments and line comments + # in the 'comment' group and string/non-comment fragments in 'noncomment'. + return re.compile( + r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\'\"\n]+)", + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?') + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'\bis!|\b(?:as|is)\b|\+\+|--|&&|\|\||<<=?|>>=?|~(?:/=?)?|[+\-*/%&^|=!<>]=?|\?') + + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;<>]') - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Dart boolean literals. - - This function generates a regular expression that matches the Dart boolean literals `true`, `false`, and the special constant `null`. - - :return: A compiled regex pattern to match Dart boolean literals and `null`. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false|null)\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - return Dart.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in Dart.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Preserve original scalar behavior: substitution + strip. if isList: - return result - return ''.join(result) + return super().remove_comments(source_code, isList=True) + pattern = cls.comment_regex() + return pattern.sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Dart.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/graphql.py b/src/PyReprism/languages/graphql.py index 5307b25..b3666be 100644 --- a/src/PyReprism/languages/graphql.py +++ b/src/PyReprism/languages/graphql.py @@ -1,49 +1,51 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class GraphSql: - def __init__(): - pass +@LanguageRegistry.register +class GraphSql(BaseLanguage): + """GraphQL helper (minimal).""" - @staticmethod - def file_extension() -> str: + @classmethod + def file_extension(cls) -> str: return extension.graphql - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + return [] - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$)|(?P[^#]*)', re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P#.*?$)|(?P[^#]+)', re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(GraphSql.keywords()) + r')\b') + @classmethod + def keywords_regex(cls) -> re.Pattern: + kws = cls.keywords() + if not kws: + return re.compile(r'(?!x)x') + return re.compile(r'\b(' + '|'.join(kws) + r')\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in GraphSql.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(GraphSql.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/groovy.py b/src/PyReprism/languages/groovy.py index ad958a5..15f622a 100644 --- a/src/PyReprism/languages/groovy.py +++ b/src/PyReprism/languages/groovy.py @@ -1,49 +1,55 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Groovy: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.groovy - - @staticmethod - def keywords() -> list: - keyword = 'as|def|in|abstract|assert|boolean|break|byte|case|catch|char|class|const|continue|default|do|double|else|enum|extends|final|finally|float|for|goto|if|implements|import|instanceof|int|interface|long|native|new|package|private|protected|public|return|short|static|strictfp|super|switch|synchronized|this|throw|throws|trait|transient|try|void|volatile|while|setup|given|when|then|and|cleanup|expect|where'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern +@LanguageRegistry.register +class Groovy(BaseLanguage): + """Groovy language helper.""" - @staticmethod - def number_regex(): - pattern = re.compile(r'\b(?:0b[01_]+|0x[\da-f_]+(?:\.[\da-f_p\-]+)?|[\d_]+(?:\.[\d_]+)?(?:e[+-]?[\d]+)?)[glidf]?\b') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'(^|[^.])(?:~|==?~?|\?[.:]?|\*(?:[.=]|\*=?)?|\.[@&]|\.\.<|\.{1,2}(?!\.)|-[-=>]?|\+[+=]?|!=?|<(?:<=?|=>?)?|>(?:>>?=?|=)?|&[&=]?|\|[|=]?|\/=?|\^=?|%=?)') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Groovy.keywords()) + r')\b') + @classmethod + def file_extension(cls) -> str: + return extension.groovy - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def keywords(cls) -> list: + return 'as|def|in|abstract|assert|boolean|break|byte|case|catch|char|class|const|continue|default|do|double|else|enum|extends|final|finally|float|for|goto|if|implements|import|instanceof|int|interface|long|native|new|package|private|protected|public|return|short|static|strictfp|super|switch|synchronized|this|throw|throws|trait|transient|try|void|volatile|while|setup|given|when|then|and|cleanup|expect|where'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Safer, simpler pattern: match // line comments and /* */ block comments + # and treat quoted strings as noncomment segments. + return re.compile( + r"(?P//.*?$|/\*[\s\S]*?\*/)|" + r"(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\n'\"]+)", + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b(?:0b[01_]+|0x[\da-f_]+(?:\.[\da-f_p\-]+)?|[\d_]+(?:\.[\d_]+)?(?:e[+-]?[\d]+)?)[glidf]?\b') + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(^|[^.])(?:~|==?~?|\?[.:]?|\*(?:[.=]|\*=?)?|\.[@&]|\.\.<|\.{1,2}(?!\.)|-[-=>]?|\+[+=]?|!=?|<(?:<=?|=>?)?|>(?:>>?=?|=)?|&[&=]?|\|[|=]?|\/?=?|\^=?|%=?)') + + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: result = [] - for match in Groovy.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Groovy.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/ini.py b/src/PyReprism/languages/ini.py index 910a2c9..32fc74c 100644 --- a/src/PyReprism/languages/ini.py +++ b/src/PyReprism/languages/ini.py @@ -1,40 +1,39 @@ + +from .base import BaseLanguage +from .registry import LanguageRegistry import re from PyReprism.utils import extension -class INI: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class INI(BaseLanguage): + """INI / .ini / Windows INI style files.""" + + @classmethod + def file_extension(cls) -> str: return extension.ini - @staticmethod - def keywords() -> list: - pass + @classmethod + def keywords(cls) -> list: + # INI files generally don't have a language keyword set; keep empty. + return [] + + @classmethod + def comment_regex(cls): + # Match either a full-line comment starting with ';' or capture non-comment segments. + # Provide named groups 'comment' and 'noncomment' as required by BaseLanguage. + return re.compile(r'(?P^[ \t]*;.*$)|(?P[^;\n]*(?:\n|$))', re.MULTILINE) - @staticmethod - def comment_regex(): - pattern = re.compile(r'^[ \t]*;.*$', re.MULTILINE) - return pattern + # number_regex and operator_regex default to BaseLanguage implementations - @staticmethod - def number_regex(): - pass + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) - @staticmethod - def operator_regex(): - pass + @classmethod + def remove_keywords(cls, source: str): + return super().remove_keywords(source) - @staticmethod - def keywords_regex(): - pass - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - return re.sub(re.compile(INI.keywords_regex()), '', source_code) - @staticmethod - def remove_keywords(source: str): - pass diff --git a/src/PyReprism/languages/java.py b/src/PyReprism/languages/java.py index 43344a5..d9f3231 100644 --- a/src/PyReprism/languages/java.py +++ b/src/PyReprism/languages/java.py @@ -1,113 +1,68 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Java: - def __init__(self): - pass - @staticmethod - def file_extension() -> str: - """ - Return the file extension used for Java files. - - :return: The file extension for Java files. - :rtype: str - """ - return extension.java - - @staticmethod - def keywords() -> list: - """ - Return a list of Java keywords and built-in functions. - - :return: A list of Java keywords and built-in function names. - :rtype: list - """ - keyword = 'abstract|continue|for|new|switch|assert|default|goto|package|synchronized|boolean|do|if|private|this|break|double|implements|protected|throw|byte|else|import|public|throws|case|enum|instanceof|return|transient|catch|extends|int|short|try|char|final|interface|static|void|class|finally|long|strictfp|volatile|const|float|native|super|while'.split('|') - return keyword - - @staticmethod - def comment_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify different types of comments and non-comment code in Java source files. - - :return: A compiled regex pattern with named groups to match single-line comments, multiline comments, and non-comment code elements. - :rtype: re.Pattern - """ - pattern = re.compile(r'(?P//.*?$|/\*[^*]*\*+(?:[^/*][^*]*\*+)*?/)|(?P[^/]+)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify numeric literals in Java code. - - :return: A compiled regex pattern to match Java numeric literals, including integers, floats, and complex numbers. - :rtype: re.Pattern - """ - pattern = re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?[df]?') - return pattern - - @staticmethod - def operator_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Java operators. +@LanguageRegistry.register +class Java(BaseLanguage): + """Java language helper; migrated to BaseLanguage pattern. - :return: A compiled regex pattern to match various Java operators and logical keywords. - :rtype: re.Pattern - """ - pattern = re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/=?|%=?|\^=?|[?:~])') - return pattern - - @staticmethod - def keywords_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Java keywords. - - :return: A compiled regex pattern to match Java keywords. - :rtype: re.Pattern - """ - return re.compile(r'\b(' + '|'.join(Java.keywords()) + r')\b') + Provides regex helpers and delegates comment/keyword removal to BaseLanguage. + """ - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Java boolean literals. + @classmethod + def file_extension(cls) -> str: + """Return the file extension used for Java files.""" + return extension.java - :return: A compiled regex pattern to match Java boolean literals. - :rtype: re.Pattern - """ + @classmethod + def keywords(cls) -> list: + """Return a list of Java keywords and built-in functions.""" + return 'abstract|continue|for|new|switch|assert|default|goto|package|synchronized|boolean|do|if|private|this|break|double|implements|protected|throw|byte|else|import|public|throws|case|enum|instanceof|return|transient|catch|extends|int|short|try|char|final|interface|static|void|class|finally|long|strictfp|volatile|const|float|native|super|while'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Regex to capture comments (group 'comment') and non-comment fragments (group 'noncomment').""" + return re.compile(r'(?P//.*?$|/\*[^*]*\*+(?:[^/*][^*]*\*+)*?/)|(?P[^/]+)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + """Regex for numeric literals.""" + return re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?[df]?') + + @classmethod + def operator_regex(cls) -> re.Pattern: + """Regex for Java operators.""" + return re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/?=|%=?|\^=?|[?:~])') + + @classmethod + def keywords_regex(cls) -> re.Pattern: + """Compile and return a regex that matches Java keywords.""" + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') + + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false)\b') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Java delimiters. - - :return: A compiled regex pattern to match Java delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@<>]') - @staticmethod - def remove_comments(source_code: str) -> str: - """ - Remove comments from the provided Java source code string. - - :param str source_code: The Java source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return Java.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - - @staticmethod - def remove_keywords(source: str) -> str: - """ - Remove all Java keywords from the provided source code string. - - :param str source: The source code string from which to remove Java keywords. - :return: The source code string with all Java keywords removed. - :rtype: str - """ - return re.sub(re.compile(Java.keywords_regex()), '', source).strip() + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Preserve original Java behavior: for scalar output perform a + substitution then strip (as older implementation did). If the + caller requests a list, fall back to BaseLanguage's behavior. + """ + if isList: + return super().remove_comments(source_code, isList=True) + pattern = cls.comment_regex() + # replicate original substitution behavior then strip the result + return pattern.sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() + + @classmethod + def remove_keywords(cls, source: str) -> str: + """Delegate keyword removal to BaseLanguage.""" + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/javascript.py b/src/PyReprism/languages/javascript.py index 51ae3ba..d50763a 100644 --- a/src/PyReprism/languages/javascript.py +++ b/src/PyReprism/languages/javascript.py @@ -1,7 +1,7 @@ import re from PyReprism.utils import extension -from .base.base import BaseLanguage +from .base import BaseLanguage from .registry import LanguageRegistry diff --git a/src/PyReprism/languages/jsx.py b/src/PyReprism/languages/jsx.py index 8d60bac..5c6d663 100644 --- a/src/PyReprism/languages/jsx.py +++ b/src/PyReprism/languages/jsx.py @@ -1,115 +1,66 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Jsx: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - """ - Return the file extension used for JSX files. +@LanguageRegistry.register +class Jsx(BaseLanguage): + """JSX/React-like language helper migrated to BaseLanguage. - :return: The file extension for JSX files. - :rtype: str - """ - return extension.jsx - - @staticmethod - def keywords() -> list: - """ - Return a list of JSX keywords and built-in functions. - - :return: A list of JSX keywords and built-in function names. - :rtype: list - """ - keyword = 'as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield'.split('|') - return keyword - - @staticmethod - def comment_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify different types of comments and non-comment code in JSX source files. - - :return: A compiled regex pattern with named groups to match single-line comments, multiline comments, and non-comment code elements. - :rtype: re.Pattern - """ - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|\{/\*[\s\S]*?\*/\}|/\*.*?$|^.*?\*/|/\*\*[\s\S]*?\*/)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"]*)', re.DOTALL | re.MULTILINE) - return pattern + Keeps comment-removal semantics from the original implementation + (only append non-empty 'noncomment' matches). Keywords removal is + delegated to the base class. + """ - @staticmethod - def number_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify numeric literals in JSX code. - - :return: A compiled regex pattern to match JSX numeric literals, including integers, floats, and complex numbers. - :rtype: re.Pattern - """ - pattern = re.compile(r'\b(?:0[xX][\dA-Fa-f]+|0[bB][01]+|0[oO][0-7]+|NaN|Infinity)\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee][+-]?\d+)?') - return pattern + @classmethod + def file_extension(cls) -> str: + return extension.jsx - @staticmethod - def operator_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify JSX operators. + @classmethod + def keywords(cls) -> list: + return 'as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield'.split('|') - :return: A compiled regex pattern to match various JSX operators and logical keywords. - :rtype: re.Pattern - """ - pattern = re.compile(r'-[-=]?|\+[+=]?|!=?=?|<>?>?=?|=(?:==?|>)?|&[&=]?|\|[|=]?|\*\*?=?|\/=?|~|\^=?|%=?|\?|\.{3}') - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + # Line comments, C-style block comments, and JSX-style {/* ... */} blocks + return re.compile( + r"(?P//.*?$|/\*[\s\S]*?\*/|\{/\*[\s\S]*?\*/\})|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\'\"\n]+)", + re.DOTALL | re.MULTILINE, + ) - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Jsx.keywords()) + r')\b') + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b(?:0[xX][\dA-Fa-f]+|0[bB][01]+|0[oO][0-7]+|NaN|Infinity)\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee][+-]?\d+)?') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify JSX delimiters. + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'-[-=]?|\+[+=]?|!=?=?|<>?>?=?|=(?:==?|>)?|&[&=]?|\|[|=]?|\*\*?=?|\/?=|~|\^=?|%=?|\?|\.\.\.') - This function generates a regular expression that matches JSX delimiters, including parentheses `(`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, at symbols `@`, as well as angle brackets `<` and `>`. + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') - :return: A compiled regex pattern to match JSX delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@<>]') - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify JSX boolean literals. - - :return: A compiled regex pattern to match JSX boolean literals. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false|null)\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided JSX source code string. - - :param str source_code: The JSX source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in Jsx.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str) -> str: - """ - Remove all JSX keywords from the provided source code string. - - :param str source: The source code string from which to remove JSX keywords. - :return: The source code string with all JSX keywords removed. - :rtype: str - """ - return re.sub(re.compile(Jsx.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/julia.py b/src/PyReprism/languages/julia.py index 0ce2d59..a6d58a4 100644 --- a/src/PyReprism/languages/julia.py +++ b/src/PyReprism/languages/julia.py @@ -1,49 +1,43 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Julia: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Julia(BaseLanguage): + """Julia language helper. + + Handles Julia comments (`#` and block comments `#= ... =#`), numbers, + operators and a conservative keyword list. + """ + + @classmethod + def file_extension(cls) -> str: return extension.julia - @staticmethod - def keywords() -> list: - keyword = 'abstract|baremodule|begin|bitstype|break|catch|ccall|const|continue|do|else|elseif|end|export|finally|for|function|global|if|immutable|import|importall|let|local|macro|module|print|println|quote|return|try|type|typealias|using|while|true|false'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$|#=[\s\S]*?=#|#=.*?$|^.*?=#)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[box])?(?:[\da-f]+\.?\d*|\.\d+)(?:[efp][+-]?\d+)?j?') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'[-+*^%÷&$\\]=?|\/[\/=]?|!=?=?|\|[=>]?|<(?:<=?|[=:])?|>(?:=|>>?=?)?|==?=?|[~≠≤≥]') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Julia.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Julia.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Julia.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + return 'abstract|baremodule|begin|bitstype|break|catch|ccall|const|continue|do|else|elseif|end|export|finally|for|function|global|if|immutable|import|importall|let|local|macro|module|print|println|quote|return|try|type|typealias|using|while|true|false'.split('|') + + @classmethod + def comment_regex(cls): + # Support single-line '#' and block comments '#= ... =#' + return re.compile(r'(?P#=.*?$|#=[\s\S]*?=#|#.*?$)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|[^#\'\"]+)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[box])?(?:[\da-f]+\.?\d*|\.\d+)(?:[efp][+-]?\d+)?j?') + + @classmethod + def operator_regex(cls): + return re.compile(r'[-+*^%÷&$\\]=?|\/[\/=]?|!=?=?|\|[=>]?|<(?:<=?|[=:])?|>(?:=|>>?=?)?|==?=?|[~≠≤≥]') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str): + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/keyman.py b/src/PyReprism/languages/keyman.py index e69de29..9861767 100644 --- a/src/PyReprism/languages/keyman.py +++ b/src/PyReprism/languages/keyman.py @@ -0,0 +1,45 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Keyman(BaseLanguage): + """Keyman keyboard mapping language (.kmn). + + This is a minimal implementation: Keyman files use simple line comments in + a few styles; we provide a forgiving comment regex and basic helpers so + the rest of the toolchain can operate consistently. + """ + + @classmethod + def file_extension(cls) -> str: + return extension.keyman + + @classmethod + def keywords(cls) -> list: + # Keyman files are mostly data; keep keywords list empty for now. + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Return a regex with named groups 'comment' and 'noncomment'. + + Accept common single-line comment forms used in keyboard mapping files + (leading 'c ' lines, semicolon, hash, and C-style //). There are no + widely used block-comments in Keyman, so we keep the pattern simple. + """ + return re.compile( + r"(?P//.*?$|#.*?$|;.*?$|^c\s.*?$)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/;#'\"\n]+)", + re.MULTILINE, + ) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/kotlin.py b/src/PyReprism/languages/kotlin.py index a8e34e9..5c05627 100644 --- a/src/PyReprism/languages/kotlin.py +++ b/src/PyReprism/languages/kotlin.py @@ -1,81 +1,56 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry +import re +from PyReprism.utils import extension -class Kotlin: - def __init__(): - pass +from .base import BaseLanguage +from .registry import LanguageRegistry - @staticmethod - def file_extension() -> str: - return extension.kotlin - @staticmethod - def keywords() -> list: - keyword = 'abstract|annotation|as|break|by|catch|class|companion|const|constructor|continue|crossinline|data|do|else|enum|final|finally|for|fun|get|if|import|in|init|inline|inner|interface|internal|is|lateinit|noinline|null|object|open|out|override|package|private|protected|public|reified|return|sealed|set|super|tailrec|this|throw|to|try|val|var|when|where|while'.split('|') - return keyword +@LanguageRegistry.register +class Kotlin(BaseLanguage): + """Kotlin language helper. - @staticmethod - def comment_regex() -> re.Pattern: - pattern = re.compile(r'(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern + Implements comment, number, operator, delimiter and keyword helpers for + Kotlin. Delegates removal logic to BaseLanguage. + """ - @staticmethod - def number_regex() -> re.Pattern: - pattern = re.compile(r'\b(?:0[bx][\da-fA-F]+|\d+(?:\.\d+)?(?:e[+-]?\d+)?[fFL]?)\b') - return pattern + @classmethod + def file_extension(cls) -> str: + return extension.kotlin - @staticmethod - def operator_regex() -> re.Pattern: - pattern = re.compile(r'+[+=]?|-[-=>]?|==?=?|!(?:!|==?)?|[\/*%<>]=?|[?:]:?|\.\.|&&|\|\||\b(?:and|inv|or|shl|shr|ushr|xor)\b') - return pattern + @classmethod + def keywords(cls) -> list: + return 'abstract|annotation|as|break|by|catch|class|companion|const|constructor|continue|crossinline|data|do|else|enum|final|finally|for|fun|get|if|import|in|init|inline|inner|interface|internal|is|lateinit|noinline|null|object|open|out|override|package|private|protected|public|reified|return|sealed|set|super|tailrec|this|throw|to|try|val|var|when|where|while'.split('|') - @staticmethod - def keywords_regex() -> re.Pattern: - return re.compile(r'\b(' + '|'.join(Kotlin.keywords()) + r')\b') + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\'\"{}]+)", re.DOTALL | re.MULTILINE) - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Kotlin language delimiters. + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b(?:0[bx][\da-fA-F]+|\d+(?:\.\d+)?(?:e[+-]?\d+)?[fFL]?)\b') - This function generates a regular expression that matches Kotlin language delimiters, which include parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, angle brackets `<`, `>`, and the question mark `?`. + @classmethod + def operator_regex(cls) -> re.Pattern: + # Conservative operator regex: symbols and common word-operators + return re.compile(r'\+[+=]?|-[-=]?|==?=?|!=?=?|[/\*%<>]=?|\.|&&|\|\||\b(?:and|or|xor|shl|shr|ushr)\b') - :return: A compiled regex pattern to match Kotlin delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;<>?]') - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Kotlin boolean literals. - - This function generates a regular expression that matches the Kotlin boolean literals `true`, `false`, and the special constant `null`. - - :return: A compiled regex pattern to match Kotlin boolean literals and `null`. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false|null)\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided Kotlin source code string. - - :param str source_code: The Kotlin source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return Kotlin.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in Kotlin.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) - @staticmethod - def remove_keywords(source: str) -> str: - return re.sub(re.compile(Kotlin.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/matlab.py b/src/PyReprism/languages/matlab.py index c87b9ee..80bca1c 100644 --- a/src/PyReprism/languages/matlab.py +++ b/src/PyReprism/languages/matlab.py @@ -1,7 +1,7 @@ import re from PyReprism.utils import extension -from .base.base import BaseLanguage +from .base import BaseLanguage from .registry import LanguageRegistry diff --git a/src/PyReprism/languages/nginx.py b/src/PyReprism/languages/nginx.py index 3f06ae4..0ce2056 100644 --- a/src/PyReprism/languages/nginx.py +++ b/src/PyReprism/languages/nginx.py @@ -1,49 +1,44 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Nginx: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Nginx(BaseLanguage): + """Nginx configuration language helper. + + Keeps comment handling for `#` comments and preserves text outside + comments in the 'noncomment' group required by BaseLanguage. + """ + + @classmethod + def file_extension(cls) -> str: return extension.nginx - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$)|(?P[^#\n]*[^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Nginx.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Nginx.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Nginx.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + # Nginx uses directives rather than traditional keywords; keep empty + # for now and expand later if you want directive-aware removal. + return [] + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P#.*?$)|(?P[^#\n]*(?:\n|$))', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b\d+\b') + + @classmethod + def operator_regex(cls): + return re.compile(r'[{};=]') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str): + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/objectivec.py b/src/PyReprism/languages/objectivec.py index e2ce7e4..5941bc5 100644 --- a/src/PyReprism/languages/objectivec.py +++ b/src/PyReprism/languages/objectivec.py @@ -1,49 +1,58 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class ObjectiveC: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.objectivec - - @staticmethod - def keywords() -> list: - keyword = 'asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while|in|self|super)\b|(?:@interface|@end|@implementation|@protocol|@class|@public|@protected|@private|@property|@try|@catch|@finally|@throw|@synthesize|@dynamic|@selector'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P[^/]*[^\n]*)', re.MULTILINE) - return pattern +@LanguageRegistry.register +class ObjectiveC(BaseLanguage): + """Objective-C language helpers (minimal migration to BaseLanguage). - @staticmethod - def number_regex(): - pattern = re.compile(r'(?:\b0x[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?)[ful]*', re.IGNORECASE) - return pattern + Preserves the original comment-removal semantics (scalar vs list) and + provides compiled regex helpers for numbers/operators/delimiters. + """ - @staticmethod - def operator_regex(): - pattern = re.compile(r'-[->]?|\+\+?|!=?|<>?=?|==?|&&?|\|\|?|[~^%?*\/@]') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(ObjectiveC.keywords()) + r')\b', re.IGNORECASE) + @classmethod + def file_extension(cls) -> str: + return extension.objectivec - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def keywords(cls) -> list: + # Keep C-family keywords; Objective-C @-directives are not suitable + # for word-boundary keyword removal (they include the '@' prefix). + return 'asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while|in|self|super'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Capture // line comments and C-style block comments; provide a + # 'noncomment' group with the text to keep. + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P[^/]*[^\n]*)', re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?:\b0x[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?)[ful]*', re.IGNORECASE) + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'-[->]?|\+\+?|!=?|<>?=?|==?|&&?|\|\|?|[~^%?*\/@]') + + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b', re.IGNORECASE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Preserve original behavior: only append non-empty noncomment groups. result = [] - for match in ObjectiveC.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(ObjectiveC.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/perl.py b/src/PyReprism/languages/perl.py index 2aabffc..b0ea35d 100644 --- a/src/PyReprism/languages/perl.py +++ b/src/PyReprism/languages/perl.py @@ -2,38 +2,76 @@ from PyReprism.utils import extension -class Perl: - def __init__(): - pass +import re +from PyReprism.utils import extension - @staticmethod - def file_extension() -> str: - return extension.perl +from .base import BaseLanguage +from .registry import LanguageRegistry - @staticmethod - def keywords() -> list: - keyword = 'any|break|continue|default|delete|die|do|else|elsif|eval|for|foreach|given|goto|if|last|local|my|next|our|package|print|redo|require|say|state|sub|switch|undef|unless|until|use|when|while'.split('|') - return keyword - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$|[{}]+)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern +@LanguageRegistry.register +class Perl(BaseLanguage): + """Perl language helper.""" - @staticmethod - def number_regex(): - pattern = re.compile(r'\b(?:0x[\dA-Fa-f](?:_?[\dA-Fa-f])*|0b[01](?:_?[01])*|(?:\d(?:_?\d)*)?\.?\d(?:_?\d)*(?:[Ee][+-]?\d+)?)\b', re.IGNORECASE) - return pattern + @classmethod + def file_extension(cls) -> str: + return extension.pl - @staticmethod - def operator_regex(): - pattern = re.compile(r'-[rwxoRWXOezsfdlpSbctugkTBMAC]\b|\+[+=]?|-[-=>]?|\*\*?=?|\/\/?=?|=[=~>]?|~[~=]?|\|\|?=?|&&?=?|<(?:=>?|<=?)?|>>?=?|![~=]?|[%^]=?|\.(?:=|\.\.?)?|[\\?]|\bx(?:=|\b)|\b(?:lt|gt|le|ge|eq|ne|cmp|not|and|or|xor)\b') - return pattern + @classmethod + def keywords(cls) -> list: + return sorted([ + 'and', 'cmp', 'continue', 'or', 'eq', 'ne', 'xor', 'not', 'm', 's', 'y', 'tr', + 'BEGIN', 'END', 'INIT', 'CHECK', 'DESTROY', 'AUTOLOAD', 'scalar', 'int', 'pack', + 'unpack', 'join', 'split', 'reverse', 'defined', 'undef', 'length', 'substr', 'vec', + 'index', 'rindex', 'abs', 'atan2', 'cos', 'sin', 'exp', 'log', 'sqrt', 'srand', 'rand', + 'time', 'localtime', 'gmtime', 'caller', 'chdir', 'chmod', 'chown', 'chroot', 'closedir', + 'connect', 'dbmclose', 'dbmopen', 'die', 'dump', 'each', 'eof', 'eval', 'exec', 'exists', + 'exit', 'fcntl', 'fileno', 'flock', 'fork', 'format', 'getc', 'getgrent', 'getgrgid', + 'getgrnam', 'gethostbyaddr', 'gethostbyname', 'gethostent', 'getlogin', 'getnetbyaddr', + 'getnetbyname', 'getnetent', 'getpeername', 'getpgrp', 'getppid', 'getpriority', + 'getprotobyname', 'getprotobynumber', 'getprotoent', 'getpwent', 'getpwnam', 'getpwuid', + 'getservbyname', 'getservbyport', 'getservent', 'getsockname', 'getsockopt', 'glob', + 'goto', 'grep', 'hex', 'import', 'ioctl', 'join', 'keys', 'kill', 'lc', 'lcfirst', 'link', + 'listen', 'local', 'localtime', 'lstat', 'map', 'mkdir', 'msgctl', 'msgget', 'msgsnd', + 'my', 'next', 'no', 'oct', 'open', 'opendir', 'ord', 'our', 'pipe', 'pop', 'pos', 'print', + 'printf', 'push', 'q', 'qq', 'qr', 'quotemeta', 'rand', 'read', 'readdir', 'readlink', + 'readpipe', 'recv', 'redo', 'ref', 'rename', 'require', 'reset', 'return', 'rewinddir', + 'rmdir', 'say', 'seek', 'seekdir', 'select', 'semctl', 'semget', 'semop', 'send', 'setgrent', + 'sethostent', 'setnetent', 'setpgrp', 'setpriority', 'setprotoent', 'setpwent', 'setservent', + 'setsockopt', 'shift', 'shmctl', 'shmget', 'shmread', 'shmwrite', 'shutdown', 'sin', 'sleep', + 'socket', 'socketpair', 'sort', 'splice', 'sprintf', 'srand', 'stat', 'state', 'study', 'sub', + 'syscall', 'sysopen', 'sysread', 'sysseek', 'system', 'syswrite', 'tell', 'telldir', 'tie', + 'tied', 'times', 'truncate', 'uc', 'ucfirst', 'umask', 'undef', 'unshift', 'untie', 'use', + 'utime', 'values', 'wait', 'waitpid', 'wantarray', 'write', 'y' + ]) - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Perl.keywords()) + r')\b', re.IGNORECASE) + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r"(?P#.*?$)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^#\'\"\n]+)", re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?') + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/?=|%=?|\^=?|[?:~])') + + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Preserve legacy scalar behavior: substitution + strip + out = re.sub(cls.comment_regex(), lambda m: m.groupdict().get('noncomment') or '', source_code) + if isList: + return [out] + return out.strip() + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) @staticmethod def remove_comments(source_code: str, isList: bool = False) -> str: result = [] diff --git a/src/PyReprism/languages/php.py b/src/PyReprism/languages/php.py index e0e5594..1251411 100644 --- a/src/PyReprism/languages/php.py +++ b/src/PyReprism/languages/php.py @@ -7,6 +7,12 @@ @LanguageRegistry.register class PHP(BaseLanguage): + """PHP language helper. + + Provides PHP-specific regexes for comments, numbers, operators, delimiters, + and a conservative keyword list. Delegates comment/keyword removal to + BaseLanguage so behavior is consistent across languages. + """ @classmethod def file_extension(cls) -> str: return extension.php @@ -17,7 +23,10 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls) -> re.Pattern: - return re.compile(r"""(?P#.*?$|//.*?$|[{}]+)|(?P/\*.*?\*/)|(?P'(\\.|[^\\'])*'|"(\\.|[^\\"])*"|.[^#/\'"{}]*)""", re.DOTALL | re.MULTILINE) + # Capture single-line comments (#, //) and block comments (/* ... */) + # in the same 'comment' group so BaseLanguage.remove_comments can + # uniformly extract the 'noncomment' group. + return re.compile(r"""(?P#.*?$|//.*?$|/\*[\s\S]*?\*/|[{}]+)|(?P'(\\.|[^\\'])*'|"(\\.|[^\\"])*"|.[^#/\'"{}]*)""", re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls) -> re.Pattern: diff --git a/src/PyReprism/languages/q.py b/src/PyReprism/languages/q.py index e69de29..10755a3 100644 --- a/src/PyReprism/languages/q.py +++ b/src/PyReprism/languages/q.py @@ -0,0 +1,27 @@ + +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Q(BaseLanguage): + """Minimal stub for q language (kdb+).""" + + @classmethod + def file_extension(cls) -> str: + return extension.q + + @classmethod + def keywords(cls) -> list: + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Conservative comment matcher: supports //, /* */, and # comments + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|#.*?$)|(?P' + r"'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\#\n'\"]+)", + re.DOTALL | re.MULTILINE) + diff --git a/src/PyReprism/languages/qore.py b/src/PyReprism/languages/qore.py index e69de29..98450f3 100644 --- a/src/PyReprism/languages/qore.py +++ b/src/PyReprism/languages/qore.py @@ -0,0 +1,27 @@ + +import re +try: + from PyReprism.utils import extension +except Exception: + from PyRePrism.utils import extension # fallback + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Qore(BaseLanguage): + """Minimal stub for Qore language.""" + + @classmethod + def file_extension(cls) -> str: + return extension.qore + + @classmethod + def keywords(cls) -> list: + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P[^/\n]+)', re.DOTALL | re.MULTILINE) + diff --git a/src/PyReprism/languages/r.py b/src/PyReprism/languages/r.py index e69de29..37539a4 100644 --- a/src/PyReprism/languages/r.py +++ b/src/PyReprism/languages/r.py @@ -0,0 +1,28 @@ + +import re +try: + from PyReprism.utils import extension +except Exception: + from PyRePrism.utils import extension # fallback + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class R(BaseLanguage): + """R language helper (minimal).""" + + @classmethod + def file_extension(cls) -> str: + return extension.r + + @classmethod + def keywords(cls) -> list: + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + # R uses '#' for line comments; support strings as noncomment + return re.compile(r"(?P#.*?$)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^#'\"\n]+)", re.DOTALL | re.MULTILINE) + diff --git a/src/PyReprism/languages/registry.py b/src/PyReprism/languages/registry.py index ed296de..77cb689 100644 --- a/src/PyReprism/languages/registry.py +++ b/src/PyReprism/languages/registry.py @@ -17,7 +17,31 @@ def get(cls, name: str): if name not in cls._registry: try: import importlib - importlib.import_module(f'.{name.lower()}', 'PyReprism.languages') + # Try importing using this module's package first. On case-insensitive + # filesystems the package may be imported with different capitalization + # (e.g. PyRePrism vs PyReprism). Attempt the package recorded in + # __package__, and fall back to the canonical on-disk package name. + pkg = __package__ or 'PyRePrism.languages' + # First try a relative import using this module's package + try: + importlib.import_module(f'.{name.lower()}', pkg) + except Exception: + # Fall back to importing using the actual on-disk package + # directory name. This avoids issues where the package was + # installed or imported with different capitalization. + try: + import os + # languages package is two levels up from this file + package_dir = os.path.basename(os.path.dirname(os.path.dirname(__file__))) + importlib.import_module(f'{package_dir}.languages.{name.lower()}') + except Exception: + # As a last resort, try a couple of common casing variants + for candidate in ('PyReprism', 'PyRePrism'): + try: + importlib.import_module(f'{candidate}.languages.{name.lower()}') + break + except Exception: + continue except Exception: # import failed or module doesn't exist; fall through pass diff --git a/src/PyReprism/languages/rust.py b/src/PyReprism/languages/rust.py index 4d4bbca..9be3c44 100644 --- a/src/PyReprism/languages/rust.py +++ b/src/PyReprism/languages/rust.py @@ -1,81 +1,55 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Rust: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.rust - - @staticmethod - def keywords() -> list: - keyword = 'abstract|alignof|as|be|box|break|const|continue|crate|do|else|enum|extern|false|final|fn|for|if|impl|in|let|loop|match|mod|move|mut|offsetof|once|override|priv|pub|pure|ref|return|sizeof|static|self|struct|super|true|trait|type|typeof|unsafe|unsized|use|virtual|where|while|yield'.split('|') - return keyword - - @staticmethod - def comment_regex() -> re.Pattern: - pattern = re.compile(r'(?P//.*?$|///.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex() -> re.Pattern: - pattern = re.compile(r'\b(?:0x[\dA-Fa-f](?:_?[\dA-Fa-f])*|0o[0-7](?:_?[0-7])*|0b[01](?:_?[01])*|(\d(?:_?\d)*)?\.?\d(?:_?\d)*(?:[Ee][+-]?\d+)?)(?:_?(?:[iu](?:8|16|32|64)?|f32|f64))?\b') - return pattern - - @staticmethod - def operator_regex() -> re.Pattern: - pattern = re.compile(r'[-+*\/%!^]=?|=[=>]?|@|&[&=]?|\|[|=]?|<>?=?') - return pattern +@LanguageRegistry.register +class Rust(BaseLanguage): + """Rust language helper. - @staticmethod - def keywords_regex() -> re.Pattern: - return re.compile(r'\b(' + '|'.join(Rust.keywords()) + r')\b') + Provides comment/number/operator regexes and a list of Rust keywords. + """ - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Rust boolean literals. - - This function generates a regular expression that matches the Rust boolean literals `true`, `false`, and the special constant `None`. + @classmethod + def file_extension(cls) -> str: + return extension.rust - :return: A compiled regex pattern to match Rust boolean literals and `None`. - :rtype: re.Pattern - """ + @classmethod + def keywords(cls) -> list: + return 'abstract|alignof|as|be|box|break|const|continue|crate|do|else|enum|extern|false|final|fn|for|if|impl|in|let|loop|match|mod|move|mut|offsetof|once|override|priv|pub|pure|ref|return|sizeof|static|self|struct|super|true|trait|type|typeof|unsafe|unsized|use|virtual|where|while|yield'.split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Capture block and line comments as 'comment'. Preserve strings and + # other non-comment content in 'noncomment'. This is conservative but + # practical for removing comments while keeping code and string literals. + return re.compile( + r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P\'"""(\\.|[^\\\'])*\'"""|\"(\\.|[^\\\"])*\"|[^/\'\"]+|/[^/*][^/\'\"]*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b(?:0x[\dA-Fa-f](?:_?[\dA-Fa-f])*|0o[0-7](?:_?[0-7])*|0b[01](?:_?[01])*|(\d(?:_?\d)*)?\.?\d(?:_?\d)*(?:[Ee][+-]?\d+)?)(?:_?(?:[iu](?:8|16|32|64)?|f32|f64))?\b') + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[-+*\/%!^]=?|=[=>]?|@|&[&=]?|\|[|=]?|<>?=?') + + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false|None)\b') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Rust language delimiters. - - This function generates a regular expression that matches Rust language delimiters, which include parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, angle brackets `<`, `>`, and the question mark `?`. - - :return: A compiled regex pattern to match Rust delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;<>?]') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided Rust source code string. - - :param str source_code: The Rust source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return Rust.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in Rust.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) - @staticmethod - def remove_keywords(source: str) -> str: - return re.sub(re.compile(Rust.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/scala.py b/src/PyReprism/languages/scala.py index d1e57c1..378e6b3 100644 --- a/src/PyReprism/languages/scala.py +++ b/src/PyReprism/languages/scala.py @@ -1,81 +1,50 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Scala: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.scala - - @staticmethod - def keywords() -> list: - keyword = 'abstract|case|catch|class|def|do|else|extends|final|finally|for|forSome|if|implicit|import|lazy|match|new|null|object|override|package|private|protected|return|sealed|self|super|this|throw|trait|try|type|val|var|while|with|yield|String|Int|Long|Short|Byte|Boolean|Double|Float|Char|Any|AnyRef|AnyVal|Unit|Nothing'.split('|') - return keyword +@LanguageRegistry.register +class Scala(BaseLanguage): + """Scala language helper. - @staticmethod - def comment_regex() -> re.Pattern: - pattern = re.compile(r'(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"{}]*)', re.DOTALL | re.MULTILINE) - return pattern + Provides keyword list and regex helpers for comment, number, operator, and + delimiter handling. Delegates removal logic to BaseLanguage. + """ - @staticmethod - def number_regex() -> re.Pattern: - pattern = re.compile(r'\b0x[\da-f]*\.?[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e\d+)?[dfl]?') - return pattern + @classmethod + def file_extension(cls) -> str: + return extension.scala - @staticmethod - def operator_regex() -> re.Pattern: - pattern = re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/=?|%=?|\^=?|[?:~])') - return pattern + @classmethod + def keywords(cls) -> list: + return 'abstract|case|catch|class|def|do|else|extends|final|finally|for|forSome|if|implicit|import|lazy|match|new|null|object|override|package|private|protected|return|sealed|self|super|this|throw|trait|try|type|val|var|while|with|yield|String|Int|Long|Short|Byte|Boolean|Double|Float|Char|Any|AnyRef|AnyVal|Unit|Nothing'.split('|') - @staticmethod - def keywords_regex() -> re.Pattern: - return re.compile(r'\b(' + '|'.join(Scala.keywords()) + r')\b') + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|[^/\'"{}]+|/[^/*][^/\'"{}]*)', re.DOTALL | re.MULTILINE) - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Scala boolean literals. + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b0x[\da-f]*\.?[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e\d+)?[dfl]?') - This function generates a regular expression that matches the Scala boolean literals `true`, `false`, and the special constant `null`. + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/?=|%=?|\^=?|[?:~])') - :return: A compiled regex pattern to match Scala boolean literals and `null`. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false|null)\b') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Scala language delimiters. - - This function generates a regular expression that matches Scala language delimiters, which include parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, angle brackets `<`, `>`, the question mark `?`, and the underscore `_`. - - :return: A compiled regex pattern to match Scala delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;<>?_]') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided Java source code string. - - :param str source_code: The Java source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return Scala.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in Scala.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) - @staticmethod - def remove_keywords(source: str) -> str: - return re.sub(re.compile(Scala.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/scheme.py b/src/PyReprism/languages/scheme.py index 3f33c94..020646a 100644 --- a/src/PyReprism/languages/scheme.py +++ b/src/PyReprism/languages/scheme.py @@ -1,49 +1,52 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Scheme: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Scheme(BaseLanguage): + """Scheme/Racket-like language helpers.""" + + @classmethod + def file_extension(cls) -> str: return extension.scheme - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + # No keywords defined previously + return [] - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P;.*?$|#\|[\s\S]*?\|#|#\|.*?$|^.*?\|#)|(?P[^;#]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + # Scheme: ; line comments, and block comments #| ... |# + return re.compile(r"(?P;.*?$|#\|[\s\S]*?\|#)|(?P[^;#\n]+)", re.DOTALL | re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + # No number regex previously; return a noop pattern + return re.compile(r'^$') - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'^$') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Scheme.keywords()) + r')\b') + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r"^$") - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in Scheme.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Scheme.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/smalltalk.py b/src/PyReprism/languages/smalltalk.py index eb77a44..2063054 100644 --- a/src/PyReprism/languages/smalltalk.py +++ b/src/PyReprism/languages/smalltalk.py @@ -1,49 +1,55 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class SmallTalk: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.smalltalk - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P\".*?\"|\".*?$|^.*?\")|(?P[^"]*)', re.DOTALL | re.MULTILINE) - return pattern +@LanguageRegistry.register +class SmallTalk(BaseLanguage): + """Smalltalk language helper.""" - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(SmallTalk.keywords()) + r')\b') + @classmethod + def file_extension(cls) -> str: + return extension.smalltalk - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def keywords(cls) -> list: + # Smalltalk has no keyword list here + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P".*?"|".*?$|^.*?")|(?P[^\"]*)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + # No number pattern defined; return a no-match pattern + return re.compile(r'(?!x)x') + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') + + @classmethod + def keywords_regex(cls) -> re.Pattern: + kws = cls.keywords() + if not kws: + return re.compile(r'(?!x)x') + return re.compile(r'\b(' + '|'.join(kws) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Preserve previous behavior: collect noncomment matches when truthy result = [] - for match in SmallTalk.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(SmallTalk.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/stylus.py b/src/PyReprism/languages/stylus.py index dc1c6e1..dbac2e5 100644 --- a/src/PyReprism/languages/stylus.py +++ b/src/PyReprism/languages/stylus.py @@ -1,49 +1,56 @@ import re -from PyReprism.utils import extension +try: + from PyReprism.utils import extension +except Exception: + # Support alternate casing that occurs in some import paths + from PyReprism.utils import extension # type: ignore +from .base import BaseLanguage +from .registry import LanguageRegistry -class Stylus: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Stylus(BaseLanguage): + """Stylus language helper.""" + + @classmethod + def file_extension(cls) -> str: return extension.stylus - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + return [] - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P[^/\n][^\n]*)', re.DOTALL | re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Stylus.keywords()) + r')\b') + @classmethod + def keywords_regex(cls) -> re.Pattern: + kws = cls.keywords() + if not kws: + return re.compile(r'(?!x)x') + return re.compile(r'\b(' + '|'.join(kws) + r')\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in Stylus.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Stylus.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/swift.py b/src/PyReprism/languages/swift.py index 05247c7..307e389 100644 --- a/src/PyReprism/languages/swift.py +++ b/src/PyReprism/languages/swift.py @@ -1,81 +1,57 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Swift: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.swift - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword +@LanguageRegistry.register +class Swift(BaseLanguage): + """Swift language helper.""" - @staticmethod - def comment_regex() -> re.Pattern: - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/\n]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern + @classmethod + def file_extension(cls) -> str: + return extension.swift - @staticmethod - def number_regex() -> re.Pattern: - pattern = re.compile(r'') - return pattern + @classmethod + def keywords(cls) -> list: + return [] - @staticmethod - def operator_regex() -> re.Pattern: - pattern = re.compile(r'') - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P[^/\n]*[^\n]*)', re.DOTALL | re.MULTILINE) - @staticmethod - def keywords_regex() -> re.Pattern: - return re.compile(r'\b(' + '|'.join(Swift.keywords()) + r')\b') + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Swift boolean literals. + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - This function generates a regular expression that matches the Swift boolean literals `true`, `false`, and the special constant `nil`. + @classmethod + def keywords_regex(cls) -> re.Pattern: + kws = cls.keywords() + if not kws: + return re.compile(r'(?!x)x') + return re.compile(r'\b(' + '|'.join(kws) + r')\b') - :return: A compiled regex pattern to match Swift boolean literals and `nil`. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false|nil)\b') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Swift language delimiters. - - This function generates a regular expression that matches Swift language delimiters, which include parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, angle brackets `<`, `>`, the question mark `?`, and the exclamation mark `!`. - - :return: A compiled regex pattern to match Swift delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;<>?!]') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided Rust source code string. - - :param str source_code: The Rust source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return Swift.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in Swift.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Preserve legacy scalar behavior (substitution + strip) + out = cls.comment_regex().sub(lambda match: match.groupdict().get('noncomment') or '', source_code).strip() if isList: - return result - return ''.join(result) + return [out] + return out - @staticmethod - def remove_keywords(source: str) -> str: - return re.sub(re.compile(Swift.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/tsx.py b/src/PyReprism/languages/tsx.py index 1e4c4c5..731c1b5 100644 --- a/src/PyReprism/languages/tsx.py +++ b/src/PyReprism/languages/tsx.py @@ -1,49 +1,52 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Tsx: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Tsx(BaseLanguage): + """TSX (TypeScript + JSX) helper.""" + + @classmethod + def file_extension(cls) -> str: return extension.tsx - @staticmethod - def keywords() -> list: - keyword = 'as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield|module|declare|constructor|namespace|abstract|require|type|string|Function|any|number|boolean|Array|symbol|console'.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + return 'as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield|module|declare|constructor|namespace|abstract|require|type|string|Function|any|number|boolean|Array|symbol|console'.split('|') - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/|/\*\*[\s\S]*?\*/)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"]*)', re.DOTALL | re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile( + r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\'\"\n]+)", + re.DOTALL | re.MULTILINE, + ) - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?[df]?') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?[df]?') - @staticmethod - def operator_regex(): - pattern = re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/=?|%=?|\^=?|[?:~])') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/?=|%=?|\^=?|[?:~])') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Tsx.keywords()) + r')\b') + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in Tsx.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Tsx.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/typescript.py b/src/PyReprism/languages/typescript.py index f23f5ae..7dd8b41 100644 --- a/src/PyReprism/languages/typescript.py +++ b/src/PyReprism/languages/typescript.py @@ -1,79 +1,58 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class TypeScript: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.typescript - - @staticmethod - def keywords() -> list: - keyword = 'as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield|module|declare|constructor|namespace|abstract|require|type|string|Function|any|number|boolean|Array|symbol|console'.split('|') - return keyword +@LanguageRegistry.register +class TypeScript(BaseLanguage): + """TypeScript language helper migrated to BaseLanguage.""" - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/|/\*\*[\s\S]*?\*/)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"]*)', re.DOTALL | re.MULTILINE) - return pattern + @classmethod + def file_extension(cls) -> str: + return extension.typescript - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?[df]?') - return pattern + @classmethod + def keywords(cls) -> list: + return 'as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield|module|declare|constructor|namespace|abstract|require|type|string|Function|any|number|boolean|Array|symbol|console'.split('|') - @staticmethod - def operator_regex(): - pattern = re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/=?|%=?|\^=?|[?:~])') - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + # Capture // and /* */ comments; preserve string literals in 'noncomment'. + return re.compile( + r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\'\"\n]+)", + re.DOTALL | re.MULTILINE, + ) - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(TypeScript.keywords()) + r')\b') + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?[df]?') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify TypeScript delimiters. + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/?=|%=?|\^=?|[?:~])') - This function generates a regular expression that matches TypeScript delimiters, including parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, at symbols `@`, as well as angle brackets `<` and `>`. + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') - :return: A compiled regex pattern to match TypeScript delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@<>]') - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify TypeScript boolean literals. - - :return: A compiled regex pattern to match TypeScript boolean literals. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:true|false)\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - """ - Remove comments from the provided TypeScript source code string. - - :param str source_code: The TypeScript source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return TypeScript.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in TypeScript.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Maintain original scalar behavior: substitution + strip. if isList: - return result - return ''.join(result) + return super().remove_comments(source_code, isList=True) + pattern = cls.comment_regex() + return pattern.sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(TypeScript.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) From 772ded58af4bee2f593cf148cd5f3e251623efc5 Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 6 Oct 2025 08:55:39 -0700 Subject: [PATCH 05/19] refactor: register language classes and add doc comments --- src/PyReprism/__init__.py | 2 + src/PyReprism/languages/abap.py | 70 ++++++++++-- src/PyReprism/languages/actionscript.py | 42 +++++++ src/PyReprism/languages/ada.py | 37 +++++++ src/PyReprism/languages/apl.py | 26 +++++ src/PyReprism/languages/arduino.py | 41 +++++++ src/PyReprism/languages/arff.py | 91 +++++++++++----- src/PyReprism/languages/asciidoc.py | 113 ++++++++++++------- src/PyReprism/languages/asm6502.py | 123 ++++++++++++++------- src/PyReprism/languages/aspnet.py | 41 +++++++ src/PyReprism/languages/autohotkey.py | 55 +++++----- src/PyReprism/languages/autoit.py | 55 +++++----- src/PyReprism/languages/basic.py | 71 ++++++------ src/PyReprism/languages/batch.py | 71 ++++++------ src/PyReprism/languages/bison.py | 81 +++++++------- src/PyReprism/languages/bro.py | 81 +++++++------- src/PyReprism/languages/clojure.py | 86 +++++++-------- src/PyReprism/languages/coffeescript.py | 71 ++++++------ src/PyReprism/languages/csp.py | 83 +++++++------- src/PyReprism/languages/diff.py | 63 +++++------ src/PyReprism/languages/django.py | 139 +++++++----------------- src/PyReprism/languages/docker.py | 62 +++++------ src/PyReprism/languages/eiffel.py | 65 +++++------ src/PyReprism/languages/elixir.py | 79 +++++++------- src/PyReprism/languages/erb.py | 30 +++++ src/PyReprism/languages/erlang.py | 66 +++++------ src/PyReprism/languages/flow.py | 30 +++++ src/PyReprism/languages/fortran.py | 66 +++++------ src/PyReprism/languages/fsharp.py | 67 ++++++------ src/PyReprism/languages/gedcom.py | 33 ++++++ src/PyReprism/languages/gherkin.py | 31 ++++++ src/PyReprism/languages/git.py | 32 ++++++ src/PyReprism/languages/glsl.py | 32 ++++++ src/PyReprism/languages/jsx.py | 52 +++++++++ src/PyReprism/languages/julia.py | 38 ++++++- src/PyReprism/languages/keyman.py | 24 ++++ src/PyReprism/languages/kotlin.py | 6 - src/PyReprism/languages/latex.py | 82 +++++++------- src/PyReprism/languages/less.py | 82 +++++++------- src/PyReprism/languages/liquid.py | 82 +++++++------- src/PyReprism/languages/livescript.py | 82 +++++++------- src/PyReprism/languages/python.py | 124 +++++---------------- 42 files changed, 1516 insertions(+), 1091 deletions(-) diff --git a/src/PyReprism/__init__.py b/src/PyReprism/__init__.py index 81f0fde..b86c6c6 100644 --- a/src/PyReprism/__init__.py +++ b/src/PyReprism/__init__.py @@ -1 +1,3 @@ +"""PyRePrism package initializer.""" + __version__ = "0.0.4" diff --git a/src/PyReprism/languages/abap.py b/src/PyReprism/languages/abap.py index 38efcab..a898c0f 100644 --- a/src/PyReprism/languages/abap.py +++ b/src/PyReprism/languages/abap.py @@ -7,36 +7,82 @@ @LanguageRegistry.register class Abap(BaseLanguage): + """ABAP language adapter. + + Provides basic patterns for comment removal and simple token matching. + This implementation intentionally keeps the keyword list minimal; a + more complete list can be provided later from a data file. + """ + @classmethod def file_extension(cls) -> str: + """Return the file extension used for ABAP files. + + :rtype: str + """ return extension.abap @classmethod def keywords(cls) -> list: - # Keep keywords minimal for now to avoid a huge inline list. - # The original project had a very large set; add more to data/ later. + """Return a (possibly empty) list of ABAP keywords. + + Keep this minimal to avoid large inline lists in the source. A fuller + list can be loaded from a data file if needed. + + :rtype: list + """ return [] @classmethod - def comment_regex(cls): - # Preserve original matching behavior: lines starting with *, - # double-quote inline comments, and (* ... *) block comments. + def comment_regex(cls) -> re.Pattern: + """Compile and return a regex that separates comments from code. + + The pattern provides a named capture group ``comment`` for comment + fragments and ``noncomment`` for code fragments. ``BaseLanguage.remove_comments`` + expects the ``noncomment`` group to extract non-comment content. + + ABAP comment forms handled: + - Lines that start with an asterisk (*...) + - Inline comments that start with a double-quote (") + - Block comments using (* ... *) + + :rtype: re.Pattern + """ return re.compile( r'''(?P^\*.*?$|".*?$|\(\*[\s\S]*?\*\)|\(\*.*?$|^.*?\*\))|(?P'(\\.|[^\\'])*'|"(\\.|[^\\"])*"|.[^*"']*)''', re.DOTALL | re.MULTILINE, ) @classmethod - def number_regex(cls): + def number_regex(cls) -> re.Pattern: + """Return a regex matching integer-like numbers in ABAP source. + + :rtype: re.Pattern + """ return re.compile(r"\b\d+\b") @classmethod - def operator_regex(cls): - return re.compile(r"(\s)(?:\*\*?|<[=>]?|>=?|\?=|[-+\/=])(?=\s)") + def operator_regex(cls) -> re.Pattern: + """Return a regex for ABAP operators (basic/heuristic). + + :rtype: re.Pattern + """ + return re.compile(r"(\s)(?:\*\*?|<[=>]?|>=?|\?=|[-+\\/=])(?=\s)") @classmethod def remove_comments(cls, source_code: str, isList: bool = False): - # Delegate to BaseLanguage which uses the 'noncomment' group. + """Remove comments from ABAP source. + + Delegates to :meth:`BaseLanguage.remove_comments`, which uses the + ``noncomment`` named group from :meth:`comment_regex` to build the result. + + :param source_code: the ABAP source to strip comments from + :type source_code: str + :param isList: if True return a list of non-comment fragments; otherwise + return a single trimmed string + :type isList: bool + :rtype: list[str] or str + """ res = super().remove_comments(source_code, isList=isList) if isList: return res @@ -44,4 +90,10 @@ def remove_comments(cls, source_code: str, isList: bool = False): @classmethod def remove_keywords(cls, source: str) -> str: + """Remove known ABAP keywords from ``source`` using the compiled keywords regex. + + :param source: the source text to remove keywords from + :type source: str + :rtype: str + """ return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/actionscript.py b/src/PyReprism/languages/actionscript.py index 91b6c0b..36a0689 100644 --- a/src/PyReprism/languages/actionscript.py +++ b/src/PyReprism/languages/actionscript.py @@ -9,27 +9,63 @@ class ActionScript(BaseLanguage): @classmethod def file_extension(cls) -> str: + """Return the file extension used for ActionScript files. + + :rtype: str + """ return extension.actionscript @classmethod def keywords(cls) -> list: + """Return the list of ActionScript keywords and reserved words. + + :rtype: list + """ return 'as|break|case|catch|class|const|default|delete|do|else|extends|finally|for|function|if|implements|import|in|instanceof|interface|internal|is|native|new|null|package|private|protected|public|return|super|switch|this|throw|try|typeof|use|var|void|while|with|dynamic|each|final|get|include|namespace|native|override|set|static'.split('|') @classmethod def comment_regex(cls): + """Compile and return a regex that separates comments from non-comment code. + + The pattern provides a named capture group ``comment`` for comment + fragments and ``noncomment`` for code fragments. + + :rtype: re.Pattern + """ # Keep the original pattern that captures // and /* */ comments and preserves non-comment sequences return re.compile(r'(?P//.*?$|/\*[^*]*\*+(?:[^/*][^*]*\*+)*?/)|(?P[^/]+)', re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls): + """Return a regex that matches basic numeric literals. + + :rtype: re.Pattern + """ return re.compile(r'\b\d+\b') @classmethod def operator_regex(cls): + """Return a regex for common ActionScript operators. + + :rtype: re.Pattern + """ return re.compile(r'\+\+|--|(?:[+\-*\/%^]|&&?|\|\|?|<>?>?|[!=]=?)=?|[~?@]') @classmethod def remove_comments(cls, source_code: str, isList: bool = False) -> str: + """Remove comments from the provided ActionScript source. + + Delegates to :meth:`BaseLanguage.remove_comments` which uses the + ``noncomment`` named group from :meth:`comment_regex` to assemble the + non-comment fragments. + + :param source_code: the ActionScript source to strip comments from + :type source_code: str + :param isList: if True return a list of non-comment fragments; otherwise + return a single trimmed string + :type isList: bool + :rtype: list[str] or str + """ res = super().remove_comments(source_code, isList=isList) if isList: return res @@ -37,4 +73,10 @@ def remove_comments(cls, source_code: str, isList: bool = False) -> str: @classmethod def remove_keywords(cls, source: str) -> str: + """Remove known ActionScript keywords from ``source`` using the compiled keywords regex. + + :param source: the source text to remove keywords from + :type source: str + :rtype: str + """ return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/ada.py b/src/PyReprism/languages/ada.py index ede0ce4..0e81908 100644 --- a/src/PyReprism/languages/ada.py +++ b/src/PyReprism/languages/ada.py @@ -9,14 +9,29 @@ class Ada(BaseLanguage): @classmethod def file_extension(cls) -> str: + """Return the file extension used for Ada source files. + + :rtype: str + """ return extension.ada @classmethod def keywords(cls) -> list: + """Return a list of Ada keywords and reserved words. + + :rtype: list + """ return 'abort|abs|abstract|accept|access|aliased|all|and|array|at|begin|body|case|constant|declare|delay|delta|digits|do|else|new|return|elsif|end|entry|exception|exit|for|function|generic|Adato|if|in|interface|is|limited|loop|mod|not|null|of|others|out|overriding|package|pragma|private|procedure|protected|raise|range|record|rem|renames|requeue|reverse|select|separate|some|subtype|synchronized|tagged|task|terminate|then|type|until|use|when|while|with|xor|true|false|null'.split('|') @classmethod def comment_regex(cls) -> re.Pattern: + """Compile and return a regex separating Ada comments and non-comment fragments. + + Ada uses ``--`` for single-line comments. The pattern also preserves + string literals in the ``noncomment`` group. + + :rtype: re.Pattern + """ # Ada uses '--' for single-line comments. Capture strings and non-comment segments. return re.compile( r"(?P--.*?$)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^-']*)", @@ -25,14 +40,30 @@ def comment_regex(cls) -> re.Pattern: @classmethod def number_regex(cls) -> re.Pattern: + """Return a regex matching integer, float and hex numeric literals. + + :rtype: re.Pattern + """ return re.compile(r'\b(?:0x[\da-f]+|\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)\b', re.IGNORECASE) @classmethod def operator_regex(cls) -> re.Pattern: + """Return a regex matching Ada operator characters. + + :rtype: re.Pattern + """ return re.compile(r'[:=<>+\-*/&|^%]+') @classmethod def remove_comments(cls, source_code: str, isList: bool = False) -> str: + """Remove comments from Ada source. + + :param source_code: the Ada source text + :type source_code: str + :param isList: if True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ res = super().remove_comments(source_code, isList=isList) if isList: return res @@ -40,4 +71,10 @@ def remove_comments(cls, source_code: str, isList: bool = False) -> str: @classmethod def remove_keywords(cls, source: str) -> str: + """Remove Ada keywords from the given source text. + + :param source: input source + :type source: str + :rtype: str + """ return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/apl.py b/src/PyReprism/languages/apl.py index 09e194d..8b4f4f8 100644 --- a/src/PyReprism/languages/apl.py +++ b/src/PyReprism/languages/apl.py @@ -18,15 +18,27 @@ class Apl(BaseLanguage): @classmethod def file_extension(cls) -> str: + """Return the file extension used for APL files. + + :rtype: str + """ return extension.apl @classmethod def keywords(cls) -> list: + """Return a conservative list of APL primitives and symbols. + + :rtype: list + """ # A conservative set of commonly-used APL symbols / primitives. return '⍝|⍴|⍳|⌈|⌊|⌿|⍟|∘|⍎|⍕'.split('|') @classmethod def comment_regex(cls): + """Compile and return a regex for APL line comments (started by '⍝'). + + :rtype: re.Pattern + """ # APL uses the '⍝' glyph to start comments. return re.compile(r'(?P⍝.*?$)|(?P[^⍝]*)', re.MULTILINE) @@ -34,8 +46,22 @@ def comment_regex(cls): @classmethod def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments from APL source. + + :param source_code: the APL source text + :type source_code: str + :param isList: if True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ return super().remove_comments(source_code, isList=isList) @classmethod def remove_keywords(cls, source: str): + """Remove known APL keywords from the source. + + :param source: input text + :type source: str + :rtype: str + """ return super().remove_keywords(source) diff --git a/src/PyReprism/languages/arduino.py b/src/PyReprism/languages/arduino.py index e645f81..e6e164f 100644 --- a/src/PyReprism/languages/arduino.py +++ b/src/PyReprism/languages/arduino.py @@ -11,15 +11,27 @@ class Arduino(BaseLanguage): @classmethod def file_extension(cls) -> str: + """Return the file extension used for Arduino sketches. + + :rtype: str + """ return extension.arduino @classmethod def keywords(cls) -> list: + """Return a (very long) list of Arduino keywords and helper names. + + :rtype: list + """ # Preserve the long keyword list from the original implementation. return 'setup|if|else|while|do|for|return|in|instanceof|default|function|loop|goto|switch|case|new|try|throw|catch|finally|null|break|continue|boolean|bool|void|byte|word|string|String|array|int|long|integer|double|KeyboardController|MouseController|SoftwareSerial|EthernetServer|EthernetClient|LiquidCrystal|LiquidCrystal_I2C|RobotControl|GSMVoiceCall|EthernetUDP|EsploraTFT|HttpClient|RobotMotor|WiFiClient|GSMScanner|FileSystem|Scheduler|GSMServer|YunClient|YunServer|IPAddress|GSMClient|GSMModem|Keyboard|Ethernet|Console|GSMBand|Esplora|Stepper|Process|WiFiUDP|GSM_SMS|Mailbox|USBHost|Firmata|PImage|Client|Server|GSMPIN|FileIO|Bridge|Serial|EEPROM|Stream|Mouse|Audio|Servo|File|Task|GPRS|WiFi|Wire|TFT|GSM|SPI|SD|runShellCommandAsynchronously|analogWriteResolution|retrieveCallingNumber|printFirmwareVersion|analogReadResolution|sendDigitalPortPair|noListenOnLocalhost|readJoystickButton|setFirmwareVersion|readJoystickSwitch|scrollDisplayRight|getVoiceCallStatus|scrollDisplayLeft|writeMicroseconds|delayMicroseconds|beginTransmission|getSignalStrength|runAsynchronously|getAsynchronously|listenOnLocalhost|getCurrentCarrier|readAccelerometer|messageAvailable|sendDigitalPorts|lineFollowConfig|countryNameWrite|runShellCommand|readStringUntil|rewindDirectory|readTemperature|setClockDivider|readLightSensor|endTransmission|analogReference|detachInterrupt|countryNameRead|attachInterrupt|encryptionType|readBytesUntil|robotNameWrite|readMicrophone|robotNameRead|cityNameWrite|userNameWrite|readJoystickY|readJoystickX|mouseReleased|openNextFile|scanNetworks|noInterrupts|digitalWrite|beginSpeaker|mousePressed|isActionDone|mouseDragged|displayLogos|noAutoscroll|addParameter|remoteNumber|getModifiers|keyboardRead|userNameRead|waitContinue|processInput|parseCommand|printVersion|readNetworks|writeMessage|blinkVersion|cityNameRead|readMessage|setDataMode|parsePacket|isListening|setBitOrder|beginPacket|isDirectory|motorsWrite|drawCompass|digitalRead|clearScreen|serialEvent|rightToLeft|setTextSize|leftToRight|requestFrom|keyReleased|compassRead|analogWrite|interrupts|WiFiServer|disconnect|playMelody|parseFloat|autoscroll|getPINUsed|setPINUsed|setTimeout|sendAnalog|readSlider|analogRead|beginWrite|createChar|motorsStop|keyPressed|tempoWrite|readButton|subnetMask|debugPrint|macAddress|writeGreen|randomSeed|attachGPRS|readString|sendString|remotePort|releaseAll|mouseMoved|background|getXChange|getYChange|answerCall|getResult|voiceCall|endPacket|constrain|getSocket|writeJSON|getButton|available|connected|findUntil|readBytes|exitValue|readGreen|writeBlue|startLoop|IPAddress|isPressed|sendSysex|pauseMode|gatewayIP|setCursor|getOemKey|tuneWrite|noDisplay|loadImage|switchPIN|onRequest|onReceive|changePIN|playFile|noBuffer|parseInt|overflow|checkPIN|knobRead|beginTFT|bitClear|updateIR|bitWrite|position|writeRGB|highByte|writeRed|setSpeed|readBlue|noStroke|remoteIP|transfer|shutdown|hangCall|beginSMS|endWrite|attached|maintain|noCursor|checkReg|checkPUK|shiftOut|isValid|shiftIn|pulseIn|connect|println|localIP|pinMode|getIMEI|display|noBlink|process|getBand|running|beginSD|drawBMP|lowByte|setBand|release|bitRead|prepare|pointTo|readRed|setMode|noFill|remove|listen|stroke|detach|attach|noTone|exists|buffer|height|bitSet|circle|config|cursor|random|IRread|setDNS|endSMS|getKey|micros|millis|begin|print|write|ready|flush|width|isPIN|blink|clear|press|mkdir|rmdir|close|point|yield|image|BSSID|click|delay|read|text|move|peek|beep|rect|line|open|seek|fill|size|turn|stop|home|find|step|tone|sqrt|RSSI|SSID|end|bit|tan|cos|sin|pow|map|abs|max|min|get|run|put'.split('|') @classmethod def comment_regex(cls) -> re.Pattern: + """Compile and return a regex that finds line and block comments while preserving strings as non-comment fragments. + + :rtype: re.Pattern + """ # Preserve original behavior: capture // line comments and C-style block # comments. Keep strings in 'noncomment'. return re.compile( @@ -29,18 +41,41 @@ def comment_regex(cls) -> re.Pattern: @classmethod def number_regex(cls) -> re.Pattern: + """Return a regex that matches numeric literals (integers, floats, hex) in Arduino/C-like code. + + :rtype: re.Pattern + """ return re.compile(r'(?:\b0x[\da-f]+|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?)[ful]*', re.IGNORECASE) @classmethod def operator_regex(cls) -> re.Pattern: + """Return a regex that matches common operators and words like 'and', 'or'. + + :rtype: re.Pattern + """ return re.compile(r'--?|\+\+?|!=?|<{1,2}=?|>{1,2}=?|->|:{1,2}|={1,2}|\^|~|%|&{1,2}|\|\|?|\?|\*|\/|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b') @classmethod def keywords_regex(cls) -> re.Pattern: + """Compile and return the keywords regex for Arduino. + + :rtype: re.Pattern + """ return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') @classmethod def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments from Arduino/C-like source text. + + This preserves string literals and returns either a list of non-comment + fragments or a combined string depending on ``isList``. + + :param source_code: input source text + :type source_code: str + :param isList: if True, return list of fragments + :type isList: bool + :rtype: list[str] or str + """ # Preserve original behavior: append only truthy noncomment groups. result = [] for match in cls.comment_regex().finditer(source_code): @@ -53,4 +88,10 @@ def remove_comments(cls, source_code: str, isList: bool = False): @classmethod def remove_keywords(cls, source: str) -> str: + """Remove keywords from source using the compiled keywords regex. + + :param source: the input source + :type source: str + :rtype: str + """ return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/arff.py b/src/PyReprism/languages/arff.py index 9880ed2..379e0ad 100644 --- a/src/PyReprism/languages/arff.py +++ b/src/PyReprism/languages/arff.py @@ -1,49 +1,84 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Arff: - def __init__(): - pass +@LanguageRegistry.register +class Arff(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + """Return the file extension used for ARFF files. - @staticmethod - def file_extension() -> str: + :rtype: str + """ return extension.arff - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: + """Return ARFF control keywords. + + :rtype: list + """ keyword = 'attribute|data|end|relation'.split('|') return keyword - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P%.*?$)|(?P[^%]*)', re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls): + """Compile and return a regex that splits ARFF comments and data. + + :rtype: re.Pattern + """ + return re.compile(r'(?P%.*?$)|(?P[^%]*)', re.MULTILINE) + + @classmethod + def number_regex(cls): + """Return regex matching numeric literals (ints and floats). + + :rtype: re.Pattern + """ + return re.compile(r'\b\d+(?:\.\d+)?\b') - @staticmethod - def number_regex(): - pattern = re.compile(r'\b\d+(?:\.\d+)?\b') - return pattern + @classmethod + def operator_regex(cls): + """Return a placeholder operator regex. - @staticmethod - def operator_regex(): - pattern = '' - return pattern + :rtype: re.Pattern + """ + return re.compile(r'') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Arff.keywords()) + r')\b') + @classmethod + def keywords_regex(cls): + """Compile and return the keywords regex for ARFF. - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + :rtype: re.Pattern + """ + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comment lines starting with '%' from ARFF content. + + :param source_code: input ARFF text + :type source_code: str + :param isList: if True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ result = [] - for match in Arff.comment_regex().finditer(source_code): + for match in cls.comment_regex().finditer(source_code): if match.group('noncomment'): result.append(match.group('noncomment')) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Arff.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str): + """Remove ARFF keywords from the provided source string. + + :param source: input string + :type source: str + :rtype: str + """ + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/asciidoc.py b/src/PyReprism/languages/asciidoc.py index e73311e..586064b 100644 --- a/src/PyReprism/languages/asciidoc.py +++ b/src/PyReprism/languages/asciidoc.py @@ -1,49 +1,78 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Asciidoc: - def __init__(): - pass +@LanguageRegistry.register +class Asciidoc(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + """Return the file extension used for Asciidoc files. - @staticmethod - def file_extension() -> str: + :rtype: str + """ return extension.asciidoc - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|////[\s\S]*?////|////.*?$|^.*?////)|(?P[^/]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = '' - return pattern - - @staticmethod - def operator_regex(): - pattern = '' - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Asciidoc.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Asciidoc.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Asciidoc.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """Return a minimal keyword list for Asciidoc (currently empty). + + :rtype: list + """ + return [] + + @classmethod + def comment_regex(cls): + """Compile and return a regex that captures Asciidoc comment forms. + + :rtype: re.Pattern + """ + # Match // single-line and //// delimited blocks conservatively + return re.compile(r'(?P//.*?$|////[\s\S]*?////)|(?P[^/\n][^\n]*)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + """Return a placeholder regex for numeric literals. + + :rtype: re.Pattern + """ + return re.compile(r'') + + @classmethod + def operator_regex(cls): + """Return a placeholder regex for operators. + + :rtype: re.Pattern + """ + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + """Compile and return keywords regex. + + :rtype: re.Pattern + """ + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments from Asciidoc source. + + :param source_code: the Asciidoc text + :type source_code: str + :param isList: if True return a list of fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + """Remove keywords from the input string. + + :param source: input string + :type source: str + :rtype: str + """ + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/asm6502.py b/src/PyReprism/languages/asm6502.py index 0a5d357..2df0d05 100644 --- a/src/PyReprism/languages/asm6502.py +++ b/src/PyReprism/languages/asm6502.py @@ -1,49 +1,88 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Asm6502: - def __init__(): - pass +@LanguageRegistry.register +class Asm6502(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + """Return the file extension used for 6502 assembly files. - @staticmethod - def file_extension() -> str: + :rtype: str + """ return extension.asm6502 - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P;.*?$)|(?P[^;]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = '' - return pattern - - @staticmethod - def operator_regex(): - pattern = '' - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Asm6502.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Asm6502.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Asm6502.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """Return a list of common 6502 mnemonics. + + The returned list is used to construct a case-insensitive regex when + matching keywords. + + :rtype: list + """ + # Common 6502 mnemonics (uppercase preferred); regex will be case-insensitive + return [ + 'ADC','AND','ASL','BCC','BCS','BEQ','BIT','BMI','BNE','BPL','BRK','BVC','BVS', + 'CLC','CLD','CLI','CLV','CMP','CPX','CPY','DEC','DEX','DEY','EOR','INC','INX','INY', + 'JMP','JSR','LDA','LDX','LDY','LSR','NOP','ORA','PHA','PHP','PLA','PLP','ROL','ROR', + 'RTI','RTS','SBC','SEC','SED','SEI','STA','STX','STY','TAX','TAY','TSX','TXA','TXS','TYA', + 'INX','DEX','INY','DEY','INC','DEC' + ] + + @classmethod + def comment_regex(cls): + """Compile and return a regex that captures semicolon comments and non-comment text. + + :rtype: re.Pattern + """ + return re.compile(r'(?P;.*?$)|(?P[^;]*)', re.MULTILINE) + + @classmethod + def number_regex(cls): + """Return a regex for numeric literals (placeholder). + + :rtype: re.Pattern + """ + return re.compile(r'') + + @classmethod + def operator_regex(cls): + """Return a regex matching operators (placeholder). + + :rtype: re.Pattern + """ + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + """Compile and return the keywords regex (case-insensitive). + + :rtype: re.Pattern + """ + # Make keyword matching case-insensitive since assembly is often lowercase + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b", re.IGNORECASE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments from 6502 assembly source. + + :param source_code: assembly source text + :type source_code: str + :param isList: if True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + """Remove keywords from the given source string. + + :param source: input source string + :type source: str + :rtype: str + """ + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/aspnet.py b/src/PyReprism/languages/aspnet.py index 09d9551..7f91329 100644 --- a/src/PyReprism/languages/aspnet.py +++ b/src/PyReprism/languages/aspnet.py @@ -11,14 +11,26 @@ class Aspnet(BaseLanguage): @classmethod def file_extension(cls) -> str: + """Return the file extension used for ASP.NET files. + + :rtype: str + """ return extension.aspnet @classmethod def keywords(cls) -> list: + """Return a small set of ASP.NET-specific directive keywords. + + :rtype: list + """ return 'Assembly|Control|Implements|Import|Master(?:Type)?|OutputCache|Page|PreviousPageType|Reference|Register'.split('|') @classmethod def comment_regex(cls) -> re.Pattern: + """Compile and return a regex that matches C-style, '//' and HTML comments. + + :rtype: re.Pattern + """ # Match C-style block comments, // line comments and HTML comments return re.compile( r"(?P//.*?$|/\*[\s\S]*?\*/|)|(?P[^/ re.Pattern: @classmethod def number_regex(cls): + """Return a regex matching no numbers (placeholder). + + :rtype: re.Pattern + """ # No number regex defined previously; return a regex that matches nothing return re.compile(r'^$') @classmethod def operator_regex(cls): + """Return a placeholder operator regex (matches nothing). + + :rtype: re.Pattern + """ # No operator regex defined previously; return a regex that matches nothing return re.compile(r'^$') @classmethod def keywords_regex(cls) -> re.Pattern: + """Compile and return the keywords regex for ASP.NET. + + :rtype: re.Pattern + """ return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') @classmethod def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments from ASP.NET/HTML-like content. + + Preserves non-comment fragments (including strings) and returns either + a list of fragments or a joined string depending on ``isList``. + + :param source_code: input source text + :type source_code: str + :param isList: if True return a list of fragments + :type isList: bool + :rtype: list[str] or str + """ # Preserve original behavior: only append truthy 'noncomment' groups. result = [] for match in cls.comment_regex().finditer(source_code): @@ -53,4 +88,10 @@ def remove_comments(cls, source_code: str, isList: bool = False): @classmethod def remove_keywords(cls, source: str) -> str: + """Remove keywords from the input source text. + + :param source: input source + :type source: str + :rtype: str + """ return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/autohotkey.py b/src/PyReprism/languages/autohotkey.py index e953937..5ee863c 100644 --- a/src/PyReprism/languages/autohotkey.py +++ b/src/PyReprism/languages/autohotkey.py @@ -1,49 +1,46 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class AutoHotKey: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class AutoHotKey(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.autohotkey - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: keyword = 'Abort|AboveNormal|Add|ahk_class|ahk_group|ahk_id|ahk_pid|All|Alnum|Alpha|AltSubmit|AltTab|AltTabAndMenu|AltTabMenu|AltTabMenuDismiss|AlwaysOnTop|AutoSize|Background|BackgroundTrans|BelowNormal|between|BitAnd|BitNot|BitOr|BitShiftLeft|BitShiftRight|BitXOr|Bold|Border|Button|ByRef|Checkbox|Checked|CheckedGray|Choose|ChooseString|Close|Color|ComboBox|Contains|ControlList|Count|Date|DateTime|Days|DDL|Default|DeleteAll|Delimiter|Deref|Destroy|Digit|Disable|Disabled|DropDownList|Edit|Eject|Else|Enable|Enabled|Error|Exist|Expand|ExStyle|FileSystem|First|Flash|Float|FloatFast|Focus|Font|for|global|Grid|Group|GroupBox|GuiClose|GuiContextMenu|GuiDropFiles|GuiEscape|GuiSize|Hdr|Hidden|Hide|High|HKCC|HKCR|HKCU|HKEY_CLASSES_ROOT|HKEY_CURRENT_CONFIG|HKEY_CURRENT_USER|HKEY_LOCAL_MACHINE|HKEY_USERS|HKLM|HKU|Hours|HScroll|Icon|IconSmall|ID|IDLast|If|IfEqual|IfExist|IfGreater|IfGreaterOrEqual|IfInString|IfLess|IfLessOrEqual|IfMsgBox|IfNotEqual|IfNotExist|IfNotInString|IfWinActive|IfWinExist|IfWinNotActive|IfWinNotExist|Ignore|ImageList|in|Integer|IntegerFast|Interrupt|is|italic|Join|Label|LastFound|LastFoundExist|Limit|Lines|List|ListBox|ListView|local|Lock|Logoff|Low|Lower|Lowercase|MainWindow|Margin|Maximize|MaximizeBox|MaxSize|Minimize|MinimizeBox|MinMax|MinSize|Minutes|MonthCal|Mouse|Move|Multi|NA|No|NoActivate|NoDefault|NoHide|NoIcon|NoMainWindow|norm|Normal|NoSort|NoSortHdr|NoStandard|Not|NoTab|NoTimers|Number|Off|Ok|On|OwnDialogs|Owner|Parse|Password|Picture|Pixel|Pos|Pow|Priority|ProcessName|Radio|Range|Read|ReadOnly|Realtime|Redraw|REG_BINARY|REG_DWORD|REG_EXPAND_SZ|REG_MULTI_SZ|REG_SZ|Region|Relative|Rename|Report|Resize|Restore|Retry|RGB|Screen|Seconds|Section|Serial|SetLabel|ShiftAltTab|Show|Single|Slider|SortDesc|Standard|static|Status|StatusBar|StatusCD|strike|Style|Submit|SysMenu|Tab2|TabStop|Text|Theme|Tile|ToggleCheck|ToggleEnable|ToolWindow|Top|Topmost|TransColor|Transparent|Tray|TreeView|TryAgain|Type|UnCheck|underline|Unicode|Unlock|UpDown|Upper|Uppercase|UseErrorLevel|Vis|VisFirst|Visible|VScroll|Wait|WaitClose|WantCtrlA|WantF2|WantReturn|While|Wrap|Xdigit|xm|xp|xs|Yes|ym|yp|ys|abs|acos|asc|asin|atan|ceil|chr|class|cos|dllcall|exp|fileexist|Fileopen|floor|il_add|il_create|il_destroy|instr|substr|isfunc|islabel|IsObject|ln|log|lv_add|lv_delete|lv_deletecol|lv_getcount|lv_getnext|lv_gettext|lv_insert|lv_insertcol|lv_modify|lv_modifycol|lv_setimagelist|mod|onmessage|numget|numput|registercallback|regexmatch|regexreplace|round|sin|tan|sqrt|strlen|sb_seticon|sb_setparts|sb_settext|strsplit|tv_add|tv_delete|tv_getchild|tv_getcount|tv_getnext|tv_get|tv_getparent|tv_getprev|tv_getselection|tv_gettext|tv_modify|varsetcapacity|winactive|winexist|__New|__Call|__Get|__Set'.split('|') return keyword - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P;.*?$)|(?P[^;]*)', re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls): + return re.compile(r'(?P;.*?$)|(?P[^;]*)', re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0x[\dA-Fa-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?') - return pattern + @classmethod + def number_regex(cls): + return re.compile(r'\b0x[\dA-Fa-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?') - @staticmethod - def operator_regex(): - pattern = re.compile(r'\?|\/\/?=?|:=|\|[=|]?|&[=&]?|\+[=+]?|-[=-]?|\*[=*]?|<(?:<=?|>|=)?|>>?=?|[.^!=~]=?|\b(?:AND|NOT|OR)\b') - return pattern + @classmethod + def operator_regex(cls): + return re.compile(r'\?|\/\/=?|:=|\|[=|]?|&[=&]?|\+[=+]?|-[=-]?|\*[=*]?|<(?:<=?|>|=)?|>>?=?|[.^!=~]=?|\b(?:AND|NOT|OR)\b') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(AutoHotKey.keywords()) + r')\b') + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in AutoHotKey.comment_regex().finditer(source_code): + for match in cls.comment_regex().finditer(source_code): if match.group('noncomment'): result.append(match.group('noncomment')) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(AutoHotKey.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/autoit.py b/src/PyReprism/languages/autoit.py index fda7750..d2f28df 100644 --- a/src/PyReprism/languages/autoit.py +++ b/src/PyReprism/languages/autoit.py @@ -1,49 +1,46 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Autoit: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Autoit(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.autoit - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: keyword = 'Case|Const|Continue(?:Case|Loop)|Default|Dim|Do|Else(?:If)?|End(?:Func|If|Select|Switch|With)|Enum|Exit(?:Loop)?|For|Func|Global|If|In|Local|Next|Null|ReDim|Select|Static|Step|Switch|Then|To|Until|Volatile|WEnd|While|With|True|False'.split('|') return keyword - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P;.*?$|#cs[\s\S]*?#ce|#cs.*?$|^.*?#ce)|(?P[^;#]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls): + return re.compile(r'(?P;.*?$|#cs[\s\S]*?#ce|#cs.*?$|^.*?#ce)|(?P[^;#]*[^\n]*)', re.DOTALL | re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'\b(?:0x[\da-f]+|\d+(?:\.\d+)?(?:e[+-]?\d+)?)\b') - return pattern + @classmethod + def number_regex(cls): + return re.compile(r'\b(?:0x[\da-f]+|\d+(?:\.\d+)?(?:e[+-]?\d+)?)\b') - @staticmethod - def operator_regex(): - pattern = re.compile(r'<[=>]?|[-+*\/=&>]=?|[?^]|\b(?:And|Or|Not)\b') - return pattern + @classmethod + def operator_regex(cls): + return re.compile(r'<[=>]?|[-+*\/=&>]=?|[?^]|\b(?:And|Or|Not)\b') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Autoit.keywords()) + r')\b') + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in Autoit.comment_regex().finditer(source_code): + for match in cls.comment_regex().finditer(source_code): if match.group('noncomment'): result.append(match.group('noncomment')) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Autoit.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/basic.py b/src/PyReprism/languages/basic.py index 795a82f..bf2db66 100644 --- a/src/PyReprism/languages/basic.py +++ b/src/PyReprism/languages/basic.py @@ -1,49 +1,40 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Basic: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Basic(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.basic - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: keyword = 'AS|BEEP|BLOAD|BSAVE|CALL(?: ABSOLUTE)?|CASE|CHAIN|CHDIR|CLEAR|CLOSE|CLS|COM|COMMON|CONST|DATA|DECLARE|DEF(?: FN| SEG|DBL|INT|LNG|SNG|STR)|DIM|DO|DOUBLE|ELSE|ELSEIF|END|ENVIRON|ERASE|ERROR|EXIT|FIELD|FILES|FOR|FUNCTION|GET|GOSUB|GOTO|IF|INPUT|INTEGER|IOCTL|KEY|KILL|LINE INPUT|LOCATE|LOCK|LONG|LOOP|LSET|MKDIR|NAME|NEXT|OFF|ON(?: COM| ERROR| KEY| TIMER)?|OPEN|OPTION BASE|OUT|POKE|PUT|READ|REDIM|REM|RESTORE|RESUME|RETURN|RMDIR|RSET|RUN|SHARED|SINGLE|SELECT CASE|SHELL|SLEEP|STATIC|STEP|STOP|STRING|SUB|SWAP|SYSTEM|THEN|TIMER|TO|TROFF|TRON|TYPE|UNLOCK|UNTIL|USING|VIEW PRINT|WAIT|WEND|WHILE|WRITE|ABS|ACCESS|ACOS|ANGLE|AREA|ARITHMETIC|ARRAY|ASIN|ASK|AT|ATN|BASE|BEGIN|BREAK|CAUSE|CEIL|CHR|CLIP|COLLATE|COLOR|CON|COS|COSH|COT|CSC|DATE|DATUM|DEBUG|DECIMAL|DEF|DEG|DEGREES|DELETE|DET|DEVICE|DISPLAY|DOT|ELAPSED|EPS|ERASABLE|EXLINE|EXP|EXTERNAL|EXTYPE|FILETYPE|FIXED|FP|GO|GRAPH|HANDLER|IDN|IMAGE|IN|INT|INTERNAL|IP|IS|KEYED|LBOUND|LCASE|LEFT|LEN|LENGTH|LET|LINE|LINES|LOG|LOG10|LOG2|LTRIM|MARGIN|MAT|MAX|MAXNUM|MID|MIN|MISSING|MOD|NATIVE|NUL|NUMERIC|OF|OPTION|ORD|ORGANIZATION|OUTIN|OUTPUT|PI|POINT|POINTER|POINTS|POS|PRINT|PROGRAM|PROMPT|RAD|RADIANS|RANDOMIZE|RECORD|RECSIZE|RECTYPE|RELATIVE|REMAINDER|REPEAT|REST|RETRY|REWRITE|RIGHT|RND|ROUND|RTRIM|SAME|SEC|SELECT|SEQUENTIAL|SET|SETTER|SGN|SIN|SINH|SIZE|SKIP|SQR|STANDARD|STATUS|STR|STREAM|STYLE|TAB|TAN|TANH|TEMPLATE|TEXT|THERE|TIME|TIMEOUT|TRACE|TRANSFORM|TRUNCATE|UBOUND|UCASE|USE|VAL|VARIABLE|VIEWPORT|WHEN|WINDOW|WITH|ZER|ZONEWIDTH'.split('|') return keyword - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?PREM.*?$|\'.*?$)|(?P[^\'R]*[^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:E[+-]?\d+)?') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'<[=>]?|>=?|[+\-*\/^=&]|\b(?:AND|EQV|IMP|NOT|OR|XOR)\b') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Basic.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Basic.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Basic.keywords_regex()), '', source) + @classmethod + def comment_regex(cls): + return re.compile(r"(?PREM.*?$|'.*?$)|(?P[^'R]*[^\n]*)", re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:E[+-]?\d+)?') + + @classmethod + def operator_regex(cls): + return re.compile(r'<[=>]?|>=?|[+\-*\/^=&]|\b(?:AND|EQV|IMP|NOT|OR|XOR)\b', re.IGNORECASE) + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b", re.IGNORECASE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/batch.py b/src/PyReprism/languages/batch.py index ecc1d48..228d845 100644 --- a/src/PyReprism/languages/batch.py +++ b/src/PyReprism/languages/batch.py @@ -1,49 +1,40 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Batch: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Batch(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.batch - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: keyword = 'not|cmdextversion|defined|errorlevel|exist|echo|set'.split('|') return keyword - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?PREM.*?$|::.*?$)|(?P[^:R]*[^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = '' - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'\^|==|\b(?:equ|neq|lss|leq|gtr|geq)\b|([*\/%+\-&^|]=?|<<=?|>>=?|[!~_=])') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Batch.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Batch.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Batch.keywords_regex()), '', source) + @classmethod + def comment_regex(cls): + return re.compile(r'(?PREM.*?$|::.*?$)|(?P[^:R]*[^\n]*)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'') + + @classmethod + def operator_regex(cls): + return re.compile(r'\^|==|\b(?:equ|neq|lss|leq|gtr|geq)\b|([*\/%%+\-&^|]=?|<<=?|>>=?|[!~_=])', re.IGNORECASE) + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b", re.IGNORECASE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/bison.py b/src/PyReprism/languages/bison.py index 83f9105..0811696 100644 --- a/src/PyReprism/languages/bison.py +++ b/src/PyReprism/languages/bison.py @@ -1,50 +1,43 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Bison: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Bison(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.bison - @staticmethod - def keywords() -> list: - # keyword = '%\w+'.split('|') - # return keyword - pass - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'(^|[^@])\b(?:0x[\da-f]+|\d+)') - return pattern - - @staticmethod - def operator_regex(): - pattern = '' - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Bison.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Bison.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Bison.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + # Bison/Yacc directives typically start with % (e.g. %token, %start) + return ['%token', '%left', '%right', '%nonassoc', '%start', '%union', '%type'] + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P[^/\n][^\n]*)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'(^|[^@])\b(?:0x[\da-f]+|\d+)') + + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + kws = cls.keywords() or [] + if not kws: + return re.compile(r'\b\B') + return re.compile(r"\b(" + "|".join(re.escape(k) for k in kws) + r")\b", re.IGNORECASE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/bro.py b/src/PyReprism/languages/bro.py index e258291..de9690c 100644 --- a/src/PyReprism/languages/bro.py +++ b/src/PyReprism/languages/bro.py @@ -1,49 +1,44 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Bro: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Bro(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.bro - @staticmethod - def keywords() -> list: - keyword = 'break|next|continue|alarm|using|of|add|delete|export|print|return|schedule|when|timeout|addr|any|bool|count|double|enum|file|int|interval|pattern|opaque|port|record|set|string|subnet|table|time|vector|for|if|else|in|module|function|load(?:-(?:sigs|plugin))?|unload|prefixes|ifn?def|else|(?:end)?if|DIR|FILENAME))|(?:&?(?:redef|priority|log|optional|default|add_func|delete_func|expire_func|read_expire|write_expire|create_expire|synchronized|persistent|rotate_interval|rotate_size|encrypt|raw_output|mergeable|group|error_handler|type_column'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$)|(?P[^#]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'--?|\+\+?|!=?=?|<=?|>=?|==?=?|&&|\|\|?|\?|\*|\/|~|\^|%') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Bro.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Bro.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Bro.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + # Conservative, common Bro/Zeek keywords and types + return [ + 'break','next','continue','alarm','using','of','add','delete','export','print','return','schedule','when','timeout', + 'addr','any','bool','count','double','enum','file','int','interval','pattern','opaque','port','record','set','string','subnet','table','time','vector', + 'for','if','else','in','module','function','load','unload','prefixes','ifdef','ifndef','DIR','FILENAME','redef','priority','log','optional','default' + ] + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P#.*?$)|(?P[^#\n].*?$)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b0x[\da-fA-F]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?') + + @classmethod + def operator_regex(cls): + return re.compile(r'--?|\+\+?|!=?=?|<=?|>=?|==?=?|&&|\|\|?|\?|\*|\/|~|\^|%') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(re.escape(k) for k in cls.keywords()) + r")\b", re.IGNORECASE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/clojure.py b/src/PyReprism/languages/clojure.py index 66e00ec..885b5b7 100644 --- a/src/PyReprism/languages/clojure.py +++ b/src/PyReprism/languages/clojure.py @@ -1,50 +1,50 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Clojure: - def __init__(): - pass +@LanguageRegistry.register +class Clojure(BaseLanguage): + """Clojure language adapter implementing the BaseLanguage contract. - @staticmethod - def file_extension() -> str: + This implementation provides a conservative set of core forms, + comment handling, and simple numeric/keyword patterns. + """ + + @classmethod + def file_extension(cls) -> str: return extension.clojure - @staticmethod - def keywords() -> list: - # keyword = 'def|if|do|let|\.\.|quote|var|->>|->|fn|loop|recur|throw|try|monitor-enter|\.|new|set!|def\-|defn|defn\-|defmacro|defmulti|defmethod|defstruct|defonce|declare|definline|definterface|defprotocol|==|defrecord|>=|deftype|<=|defproject|ns|\*|\+|\-|\/|<|=|>|accessor|agent|agent-errors|aget|alength|all-ns|alter|and|append-child|apply|array-map|aset|aset-boolean|aset-byte|aset-char|aset-double|aset-float|aset-int|aset-long|aset-short|assert|assoc|await|await-for|bean|binding|bit-and|bit-not|bit-or|bit-shift-left|bit-shift-right|bit-xor|boolean|branch|butlast|byte|cast|char|children|class|clear-agent-errors|comment|commute|comp|comparator|complement|concat|conj|cons|constantly|cond|if-not|construct-proxy|contains|count|create-ns|create-struct|cycle|dec|deref|difference|disj|dissoc|distinct|doall|doc|dorun|doseq|dosync|dotimes|doto|double|down|drop|drop-while|edit|end|ensure|eval|every|false|ffirst|file-seq|filter|find|find-doc|find-ns|find-var|first|float|flush|for|fnseq|frest|gensym|get-proxy-class|get|hash-map|hash-set|identical|identity|if-let|import|in-ns|inc|index|insert-child|insert-left|insert-right|inspect-table|inspect-tree|instance|int|interleave|intersection|into|into-array|iterate|join|key|keys|keyword|keyword|last|lazy-cat|lazy-cons|left|lefts|line-seq|list\*|list|load|load-file|locking|long|loop|macroexpand|macroexpand-1|make-array|make-node|map|map-invert|map|mapcat|max|max-key|memfn|merge|merge-with|meta|min|min-key|name|namespace|neg|new|newline|next|nil|node|not|not-any|not-every|not=|ns-imports|ns-interns|ns-map|ns-name|ns-publics|ns-refers|ns-resolve|ns-unmap|nth|nthrest|or|parse|partial|path|peek|pop|pos|pr|pr-str|print|print-str|println|println-str|prn|prn-str|project|proxy|proxy-mappings|quot|rand|rand-int|range|re-find|re-groups|re-matcher|re-matches|re-pattern|re-seq|read|read-line|reduce|ref|ref-set|refer|rem|remove|remove-method|remove-ns|rename|rename-keys|repeat|replace|replicate|resolve|rest|resultset-seq|reverse|rfirst|right|rights|root|rrest|rseq|second|select|select-keys|send|send-off|seq|seq-zip|seq|set|short|slurp|some|sort|sort-by|sorted-map|sorted-map-by|sorted-set|special-symbol|split-at|split-with|str|string|struct|struct-map|subs|subvec|symbol|symbol|sync|take|take-nth|take-while|test|time|to-array|to-array-2d|tree-seq|true|union|up|update-proxy|val|vals|var-get|var-set|var|vector|vector-zip|vector|when|when-first|when-let|when-not|with-local-vars|with-meta|with-open|with-out-str|xml-seq|xml-zip|zero|zipmap|zipper|true|false|nil'.split('|') - pass - # return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P;.*?$)|(?P[^;]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b[0-9A-Fa-f]+\b') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r"(?:::|[:|'])\b[a-z][\w*+!?-]*\b") - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Clojure.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Clojure.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Clojure.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + # A conservative set of core Clojure forms and special symbols + return [ + 'def', 'defn', 'if', 'do', 'let', 'quote', 'var', 'fn', 'loop', 'recur', + 'throw', 'try', 'monitor-enter', 'new', 'set!', 'defmacro', 'defmulti', + 'defmethod', 'defstruct', 'defonce', 'declare', 'true', 'false', 'nil', + 'and', 'or', 'not', 'first', 'rest', 'cons', 'conj', 'map', 'filter', + 'reduce', 'apply', 'seq', 'concat', '->', '->>', 'when', 'when-not', + 'doseq', 'dotimes', 'for' + ] + + @classmethod + def comment_regex(cls): + # semicolon begins a line or inline comment in Clojure + return re.compile(r'(?P;.*?$)|(?P[^;\n].*?$)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b\d+(?:\.\d+)?\b') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(re.escape(k) for k in cls.keywords()) + r")\b", re.IGNORECASE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/coffeescript.py b/src/PyReprism/languages/coffeescript.py index dc67e71..f66ac35 100644 --- a/src/PyReprism/languages/coffeescript.py +++ b/src/PyReprism/languages/coffeescript.py @@ -1,49 +1,40 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class CoffeeScript: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class CoffeeScript(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.coffeescript - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: keyword = 'and|break|by|catch|class|continue|debugger|delete|do|each|else|extend|extends|false|finally|for|if|in|instanceof|is|isnt|let|loop|namespace|new|no|not|null|of|off|on|or|own|return|super|switch|then|this|throw|true|try|typeof|undefined|unless|until|when|while|window|with|yes|yield'.split('|') return keyword - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$|###.*?###|###.*?$|^.*?###)|(?P[^#]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r' ') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r" ") - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(CoffeeScript.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in CoffeeScript.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(CoffeeScript.keywords_regex()), '', source) + @classmethod + def comment_regex(cls): + return re.compile(r'(?P#.*?$|###.*?###)|(?P[^#\n].*?$)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'') + + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(re.escape(k) for k in cls.keywords()) + r")\b", re.IGNORECASE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/csp.py b/src/PyReprism/languages/csp.py index 250ddb5..d340483 100644 --- a/src/PyReprism/languages/csp.py +++ b/src/PyReprism/languages/csp.py @@ -1,49 +1,44 @@ import re -from PyReprism.utils import extension +from PyRePrism.utils import extension +from PyRePrism.languages.base import BaseLanguage +from PyRePrism.languages.registry import LanguageRegistry -class CSP: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class CSP(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.csp - @staticmethod - def keywords() -> list: - keyword = 'base-uri|form-action|frame-ancestors|plugin-types|referrer|reflected-xss|report-to|report-uri|require-sri-for|sandbox) |(?:block-all-mixed-content|disown-opener|upgrade-insecure-requests)(?: |;)|(?:child|connect|default|font|frame|img|manifest|media|object|script|style|worker)-src'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P--.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/\-]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(CSP.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in CSP.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(CSP.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + # Common CSP directives + return [ + 'default-src','script-src','style-src','img-src','connect-src','font-src','object-src','media-src','frame-src', + 'base-uri','form-action','frame-ancestors','report-uri','report-to','require-sri-for','sandbox','block-all-mixed-content','upgrade-insecure-requests' + ] + + @classmethod + def comment_regex(cls): + # CSP is typically header text — use a conservative non-comment match + return re.compile(r'(?P.+)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'') + + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(re.escape(k) for k in cls.keywords()) + r")\b", re.IGNORECASE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) diff --git a/src/PyReprism/languages/diff.py b/src/PyReprism/languages/diff.py index f204b79..5670929 100644 --- a/src/PyReprism/languages/diff.py +++ b/src/PyReprism/languages/diff.py @@ -1,49 +1,42 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Diff: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Diff(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.diff - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: keyword = ''.split('|') return keyword - @staticmethod - def comment_regex(): + @classmethod + def comment_regex(cls): + # line-oriented diff comments (lines starting with #) vs non-comment lines pattern = re.compile(r'(?P^#.*?$)|(?P^[^#\n].*?$)', re.MULTILINE) return pattern - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def number_regex(cls): + return re.compile(r'') - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Diff.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Diff.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Diff.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/django.py b/src/PyReprism/languages/django.py index f65b98d..77e1e1e 100644 --- a/src/PyReprism/languages/django.py +++ b/src/PyReprism/languages/django.py @@ -1,124 +1,59 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Django: - def __init__(self): - pass - - @staticmethod - def file_extension() -> str: - """ - Return the file extension used for Python files. - - :return: The file extension for Python files. - :rtype: str - """ +@LanguageRegistry.register +class Django(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.django - @staticmethod - def keywords() -> list: - """ - Return a list of Python keywords and built-in functions. - - :return: A list of Python keywords and built-in function names. - :rtype: list - """ + @classmethod + def keywords(cls) -> list: keyword = 'load|verbatim|widthratio|ssi|firstof|for|url|ifchanged|csrf_token|lorem|ifnotequal|autoescape|now|templatetag|debug|cycle|ifequal|regroup|comment|filter|endfilter|if|spaceless|with|extends|block|include|else|empty|endif|endfor|as|endblock|endautoescape|endverbatim|trans|endtrans|[Tt]rue|[Ff]alse|[Nn]one|in|is|static|macro|endmacro|call|endcall|set|endset|raw|endraw'.split('|') return keyword - @staticmethod - def functions() -> list: - """ - Return a list of Python keywords and built-in functions. - - :return: A list of Python keywords and built-in function names. - :rtype: list - """ + @classmethod + def functions(cls) -> list: func = 'abs|add|addslashes|attr|batch|callable|capfirst|capitalize|center|count|cut|d|date|default|default_if_none|defined|dictsort|dictsortreversed|divisibleby|e|equalto|escape|escaped|escapejs|even|filesizeformat|first|float|floatformat|force_escape|forceescape|format|get_digit|groupby|indent|int|iriencode|iterable|join|last|length|length_is|linebreaks|linebreaksbr|linenumbers|list|ljust|lower|make_list|map|mapping|number|odd|phone2numeric|pluralize|pprint|random|reject|rejectattr|removetags|replace|reverse|rjust|round|safe|safeseq|sameas|select|selectattr|sequence|slice|slugify|sort|string|stringformat|striptags|sum|time|timesince|timeuntil|title|trim|truncate|truncatechars|truncatechars_html|truncatewords|truncatewords_html|undefined|unordered_list|upper|urlencode|urlize|urlizetrunc|wordcount|wordwrap|xmlattr|yesno'.split('|') return func - @staticmethod - def comment_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify different types of comments and non-comment code in Python source files. - - :return: A compiled regex pattern with named groups to match single-line comments, multiline comments, and non-comment code elements. - :rtype: re.Pattern - """ - pattern = re.compile(r'(?P#.*?$)|'r'(?P""".*?""")|'r'(?P\'\'\'.*?\'\'\')|'r'(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify numeric literals in Python code. - - :return: A compiled regex pattern to match Python numeric literals, including integers, floats, and complex numbers. - :rtype: re.Pattern - """ - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Python operators. - - :return: Regex pattern to match various Python operators and logical keywords. - :rtype: re.Pattern - """ - pattern = re.compile(r'') + @classmethod + def comment_regex(cls) -> re.Pattern: + # Match single-line comments and triple-quoted strings as multiline comments. + pattern = re.compile( + r"(?P#.*?$)|(?P\"\"\".*?\"\"\")|(?P'''(.*?)''')|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|[^#'\"]+)", + re.DOTALL | re.MULTILINE, + ) return pattern - @staticmethod - def keywords_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Python keywords. + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'') - :return: A compiled regex pattern to match Python keywords. - :rtype: re.Pattern - """ - return re.compile(r'\b(' + '|'.join(Django.keywords()) + r')\b') + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'') - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Python boolean literals. + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") - :return: A compiled regex pattern to match Python boolean literals and `None`. - :rtype: re.Pattern - """ + @classmethod + def boolean_regex(cls) -> re.Pattern: return re.compile(r'\b(?:True|False|None)\b') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Python delimiters. - - :return: A compiled regex pattern to match Python delimiters. - :rtype: re.Pattern - """ + @classmethod + def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@]') - @staticmethod - def remove_comments(source_code: str) -> str: - """ - Remove comments from the provided Python source code string. - - :param str source_code: The Python source code from which to remove comments. - :return: The source code with all comments removed. - :rtype: str - """ - return Django.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - - @staticmethod - def remove_keywords(source: str) -> str: - """ - Remove all Python keywords from the provided source code string. + @classmethod + def remove_comments(cls, source_code: str) -> str: + # Preserve original scalar behavior (return stripped string) + return cls.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - :param str source: The source code string from which to remove Python keywords. - :return: The source code string with all Python keywords removed. - :rtype: str - """ - return re.sub(re.compile(Django.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/docker.py b/src/PyReprism/languages/docker.py index dea85a6..c0cd5fb 100644 --- a/src/PyReprism/languages/docker.py +++ b/src/PyReprism/languages/docker.py @@ -1,49 +1,41 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Docker: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Docker(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.docker - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: keyword = 'ADD|ARG|CMD|COPY|ENTRYPOINT|ENV|EXPOSE|FROM|HEALTHCHECK|LABEL|MAINTAINER|ONBUILD|RUN|SHELL|STOPSIGNAL|USER|VOLUME|WORKDIR'.split('|') return keyword - @staticmethod - def comment_regex(): + @classmethod + def comment_regex(cls): pattern = re.compile(r'(?P#.*?$)|(?P[^#]*)', re.MULTILINE) return pattern - @staticmethod - def number_regex(): - pattern = '' - return pattern + @classmethod + def number_regex(cls): + return re.compile(r'') - @staticmethod - def operator_regex(): - pattern = '' - return pattern + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Docker.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Docker.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Docker.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/eiffel.py b/src/PyReprism/languages/eiffel.py index 88b0960..de4e58a 100644 --- a/src/PyReprism/languages/eiffel.py +++ b/src/PyReprism/languages/eiffel.py @@ -1,49 +1,42 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Eiffel: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Eiffel(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.eiffel - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: keyword = 'across|agent|alias|all|and|attached|as|assign|attribute|check|class|convert|create|Current|debug|deferred|detachable|do|else|elseif|end|ensure|expanded|export|external|feature|from|frozen|if|implies|inherit|inspect|invariant|like|local|loop|not|note|obsolete|old|once|or|Precursor|redefine|rename|require|rescue|Result|retry|select|separate|some|then|undefine|until|variant|Void|when|xor|True|False'.split('|') return keyword - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P--.*?$|\{[\s\S]*?\}|\{.*?$|^.*?\})|(?P[^-{]*[^\n]*)', re.DOTALL | re.MULTILINE) + @classmethod + def comment_regex(cls): + # Eiffel uses -- for line comments and { ... } block comments; conservative pattern + pattern = re.compile(r'(?P--.*?$|\{[\s\S]*?\})|(?P[^\n]+)', re.DOTALL | re.MULTILINE) return pattern - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0[xcb][\da-f](?:_*[\da-f])*\b|(?:\d(?:_*\d)*)?\.(?:(?:\d(?:_*\d)*)?e[+-]?)?\d(?:_*\d)*|\d(?:_*\d)*\.?') - return pattern + @classmethod + def number_regex(cls): + return re.compile(r'\b0[xcb][\da-f](?:_*[\da-f])*\b|(?:\d(?:_*\d)*)?\.(?:(?:\d(?:_*\d)*)?e[+-]?)?\d(?:_*\d)*|\d(?:_*\d)*\.?') - @staticmethod - def operator_regex(): - pattern = re.compile(r'\\\\|\|\.\.\||\.\.|\/[~\/=]?|[><]=?|[-+*^=~]') - return pattern + @classmethod + def operator_regex(cls): + return re.compile(r'\\\\|\|\.\.\||\.\.|\/[~\/=]?|[><]=?|[-+*^=~]') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Eiffel.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Eiffel.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Eiffel.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/elixir.py b/src/PyReprism/languages/elixir.py index df5e40e..4309622 100644 --- a/src/PyReprism/languages/elixir.py +++ b/src/PyReprism/languages/elixir.py @@ -1,49 +1,54 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Elixir: - def __init__(): - pass +@LanguageRegistry.register +class Elixir(BaseLanguage): + """Elixir language helper (minimal).""" - @staticmethod - def file_extension() -> str: + @classmethod + def file_extension(cls) -> str: return extension.elixir - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$|"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\')|(?P[^#\'"\n]*[^\n]*)', re.MULTILINE | re.DOTALL) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Elixir.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def keywords(cls) -> list: + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Use a conservative line-comment matcher for Elixir (#). Avoid + # embedding triple-quoted literals in the source regex to keep the + # implementation simple and safe. + return re.compile(r'(?P#.*?$)|(?P[^#\n]+)', re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') + + @classmethod + def keywords_regex(cls) -> re.Pattern: + kws = cls.keywords() + if not kws: + return re.compile(r'(?!x)x') + return re.compile(r'\b(' + '|'.join(kws) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in Elixir.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Elixir.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/erb.py b/src/PyReprism/languages/erb.py index e69de29..1948f78 100644 --- a/src/PyReprism/languages/erb.py +++ b/src/PyReprism/languages/erb.py @@ -0,0 +1,30 @@ +import re +from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Erb(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + return extension.erb + + @classmethod + def keywords(cls) -> list: + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P|#.*?$)|(?P[^#<\n]+)', re.DOTALL | re.MULTILINE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + result = [] + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) + if isList: + return result + return ''.join(result) diff --git a/src/PyReprism/languages/erlang.py b/src/PyReprism/languages/erlang.py index ffde1f6..2065187 100644 --- a/src/PyReprism/languages/erlang.py +++ b/src/PyReprism/languages/erlang.py @@ -1,49 +1,51 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class ErLang: - def __init__(): - pass +@LanguageRegistry.register +class ErLang(BaseLanguage): + """Erlang language helper.""" - @staticmethod - def file_extension() -> str: + @classmethod + def file_extension(cls) -> str: return extension.erlang - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + return [] - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P%.*?$)|(?P[^%\n]*[^\n]*)', re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P%.*?$)|(?P[^%\n]+)', re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(ErLang.keywords()) + r')\b') + @classmethod + def keywords_regex(cls) -> re.Pattern: + kws = cls.keywords() + if not kws: + return re.compile(r'(?!x)x') + return re.compile(r'\b(' + '|'.join(kws) + r')\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in ErLang.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(ErLang.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/flow.py b/src/PyReprism/languages/flow.py index e69de29..c630530 100644 --- a/src/PyReprism/languages/flow.py +++ b/src/PyReprism/languages/flow.py @@ -0,0 +1,30 @@ +import re +from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Flow(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + return extension.flow + + @classmethod + def keywords(cls) -> list: + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P[^/\n]+)', re.DOTALL | re.MULTILINE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + result = [] + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) + if isList: + return result + return ''.join(result) diff --git a/src/PyReprism/languages/fortran.py b/src/PyReprism/languages/fortran.py index d9df95d..966a123 100644 --- a/src/PyReprism/languages/fortran.py +++ b/src/PyReprism/languages/fortran.py @@ -1,49 +1,51 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class ForTran: - def __init__(): - pass +@LanguageRegistry.register +class ForTran(BaseLanguage): + """Fortran language helper (minimal).""" - @staticmethod - def file_extension() -> str: + @classmethod + def file_extension(cls) -> str: return extension.fortran - @staticmethod - def keywords() -> list: - keyword = 'INTEGER|REAL|DOUBLE|PRECISION|COMPLEX|CHARACTER|LOGICAL|/\b(?:END ?)?(?:BLOCK ?DATA|DO|FILE|FORALL|FUNCTION|IF|INTERFACE|MODULE(?! PROCEDURE)|PROGRAM|SELECT|SUBROUTINE|TYPE|WHERE|ALLOCATABLE|ALLOCATE|BACKSPACE|CALL|CASE|CLOSE|COMMON|CONTAINS|CONTINUE|CYCLE|DATA|DEALLOCATE|DIMENSION|DO|END|EQUIVALENCE|EXIT|EXTERNAL|FORMAT|GO ?TO|IMPLICIT(?: NONE)?|INQUIRE|INTENT|INTRINSIC|MODULE PROCEDURE|NAMELIST|NULLIFY|OPEN|OPTIONAL|PARAMETER|POINTER|PRINT|PRIVATE|PUBLIC|READ|RETURN|REWIND|SAVE|SELECT|STOP|TARGET|WHILE|WRITE|ASS'.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + # Keep the original token list but guard against mis-formed entries + return [ + 'INTEGER', 'REAL', 'DOUBLE', 'PRECISION', 'COMPLEX', 'CHARACTER', 'LOGICAL' + ] - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P!.*?$)|(?P[^!]*[^\n]*)', re.MULTILINE) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + return re.compile(r'(?P!.*?$)|(?P[^!\n]+)', re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[ED][+-]?\d+)?(?:_\w+)?') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[ED][+-]?\d+)?(?:_\w+)?') - @staticmethod - def operator_regex(): - pattern = re.compile(r'\*\*|\/\/|=>|[=\/]=|[<>]=?|::|[+\-*=%]|\.(?:EQ|NE|LT|LE|GT|GE|NOT|AND|OR|EQV|NEQV)\.|\.[A-Z]+\.') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'\*\*|\/\/|=>|[=\/]=|[<>]=?|::|[+\-*=%]|\.(?:EQ|NE|LT|LE|GT|GE|NOT|AND|OR|EQV|NEQV)\.|\.[A-Z]+\.') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(ForTran.keywords()) + r')\b') + @classmethod + def keywords_regex(cls) -> re.Pattern: + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in ForTran.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(ForTran.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/fsharp.py b/src/PyReprism/languages/fsharp.py index 6efb249..8502301 100644 --- a/src/PyReprism/languages/fsharp.py +++ b/src/PyReprism/languages/fsharp.py @@ -1,49 +1,52 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class FSharp: - def __init__(): - pass +@LanguageRegistry.register +class FSharp(BaseLanguage): + """F# language helper.""" - @staticmethod - def file_extension() -> str: + @classmethod + def file_extension(cls) -> str: return extension.fsharp - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword + @classmethod + def keywords(cls) -> list: + return [] - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|\(\*[\s\S]*?\*\)|\(\*.*?$|^.*?\*\))|(?P[^/(\*]*[^\n]*)', re.MULTILINE | re.DOTALL) - return pattern + @classmethod + def comment_regex(cls) -> re.Pattern: + # Support // line comments and (* ... *) block comments + return re.compile(r"(?P//.*?$|\(\*[\s\S]*?\*\))|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^(/\n'\"]+)", re.DOTALL | re.MULTILINE) - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def number_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'(?!x)x') - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(FSharp.keywords()) + r')\b') + @classmethod + def keywords_regex(cls) -> re.Pattern: + kws = cls.keywords() + if not kws: + return re.compile(r'(?!x)x') + return re.compile(r'\b(' + '|'.join(kws) + r')\b') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): result = [] - for match in FSharp.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + for match in cls.comment_regex().finditer(source_code): + non = match.groupdict().get('noncomment') + if non: + result.append(non) if isList: return result return ''.join(result) - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(FSharp.keywords_regex()), '', source) + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/gedcom.py b/src/PyReprism/languages/gedcom.py index e69de29..b714ba5 100644 --- a/src/PyReprism/languages/gedcom.py +++ b/src/PyReprism/languages/gedcom.py @@ -0,0 +1,33 @@ +"""GEDCOM language support for PyRePrism. + +This module provides a minimal language class that integrates with the +BaseLanguage + LanguageRegistry architecture. The implementation is +conservative: it recognizes line comments starting with 0 or 1 and +provides empty keyword lists. +""" + +import re +from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Gedcom(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + return extension.gedcom + + @classmethod + def keywords(cls) -> list: + return [] + + @classmethod + def comment_regex(cls): + # GEDCOM has numeric-level lines; no formal comment token, but accept lines starting with "0" or "1" as data; treat lines starting with "#" as comment for safety + return re.compile(r'(?P^#.*?$)|(?P^[^#\n].*?$)', re.MULTILINE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + diff --git a/src/PyReprism/languages/gherkin.py b/src/PyReprism/languages/gherkin.py index e69de29..fec9c49 100644 --- a/src/PyReprism/languages/gherkin.py +++ b/src/PyReprism/languages/gherkin.py @@ -0,0 +1,31 @@ + +"""Gherkin (Cucumber) language minimal support. + +This file provides a conservative implementation suitable for initial +migration to BaseLanguage. It recognizes # comments and basic keywords. +""" + +import re +from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Gherkin(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + return extension.gherkin + + @classmethod + def keywords(cls) -> list: + return ['Feature', 'Scenario', 'Given', 'When', 'Then', 'And', 'But'] + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P^#.*?$)|(?P^[^#\n].*?$)', re.MULTILINE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + diff --git a/src/PyReprism/languages/git.py b/src/PyReprism/languages/git.py index e69de29..fc857f7 100644 --- a/src/PyReprism/languages/git.py +++ b/src/PyReprism/languages/git.py @@ -0,0 +1,32 @@ + +"""Minimal Git commit/patch language support. + +This class handles simple comment removal for git-related files +(commit messages, patch fragments)." +""" + +import re +from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Git(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + return extension.git + + @classmethod + def keywords(cls) -> list: + return [] + + @classmethod + def comment_regex(cls): + # Git commit files often use # for comments; accept ; as well + return re.compile(r'(?P^[#;].*?$)|(?P^[^#;\n].*?$)', re.MULTILINE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + diff --git a/src/PyReprism/languages/glsl.py b/src/PyReprism/languages/glsl.py index e69de29..25dbeb0 100644 --- a/src/PyReprism/languages/glsl.py +++ b/src/PyReprism/languages/glsl.py @@ -0,0 +1,32 @@ + +"""GLSL shader language minimal support. + +Handles C-style single-line and block comments conservatively. +""" + +import re +from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Glsl(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + return extension.glsl + + @classmethod + def keywords(cls) -> list: + # keep empty for now; can be extended later + return [] + + @classmethod + def comment_regex(cls): + # Match // single-line and /* ... */ block comments; capture non-comment otherwise + return re.compile(r'(?P//.*?$)|(?P/\*[\s\S]*?\*/)|(?P[^/\n][^\n]*)', re.DOTALL | re.MULTILINE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + diff --git a/src/PyReprism/languages/jsx.py b/src/PyReprism/languages/jsx.py index 5c6d663..42809c3 100644 --- a/src/PyReprism/languages/jsx.py +++ b/src/PyReprism/languages/jsx.py @@ -16,14 +16,31 @@ class Jsx(BaseLanguage): @classmethod def file_extension(cls) -> str: + """Return the file extension used for JSX/TSX files. + + :rtype: str + """ return extension.jsx @classmethod def keywords(cls) -> list: + """Return a list of JavaScript/JSX keywords used for token filtering. + + :rtype: list + """ return 'as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield'.split('|') @classmethod def comment_regex(cls) -> re.Pattern: + """Compile and return a regex that captures comments and non-comment + fragments in JSX-like source. + + The pattern includes single-line // comments, C-style /* ... */ block + comments, and JSX fragments like {/* ... */}. It provides named groups + ``comment`` and ``noncomment`` as required by BaseLanguage helpers. + + :rtype: re.Pattern + """ # Line comments, C-style block comments, and JSX-style {/* ... */} blocks return re.compile( r"(?P//.*?$|/\*[\s\S]*?\*/|\{/\*[\s\S]*?\*/\})|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\'\"\n]+)", @@ -32,26 +49,55 @@ def comment_regex(cls) -> re.Pattern: @classmethod def number_regex(cls) -> re.Pattern: + """Return a regex matching numeric literals used in JSX/JavaScript. + + :rtype: re.Pattern + """ return re.compile(r'\b(?:0[xX][\dA-Fa-f]+|0[bB][01]+|0[oO][0-7]+|NaN|Infinity)\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee][+-]?\d+)?') @classmethod def operator_regex(cls) -> re.Pattern: + """Return a regex matching common JavaScript operators. + + :rtype: re.Pattern + """ return re.compile(r'-[-=]?|\+[+=]?|!=?=?|<>?>?=?|=(?:==?|>)?|&[&=]?|\|[|=]?|\*\*?=?|\/?=|~|\^=?|%=?|\?|\.\.\.') @classmethod def keywords_regex(cls) -> re.Pattern: + """Compile a regex that matches any of the language keywords. + + :rtype: re.Pattern + """ return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') @classmethod def delimiters_regex(cls) -> re.Pattern: + """Return a regex matching delimiter characters used in JSX/JS. + + :rtype: re.Pattern + """ return re.compile(r'[()\[\]{}.,:;@<>]') @classmethod def boolean_regex(cls) -> re.Pattern: + """Return a regex matching boolean-like literals. + + :rtype: re.Pattern + """ return re.compile(r'\b(?:true|false|null)\b') @classmethod def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments from source, returning either the joined string + or a list of non-comment fragments when isList is True. + + :param source_code: the source text to process + :type source_code: str + :param isList: if True return list of fragments instead of string + :type isList: bool + :rtype: list[str] or str + """ result = [] for match in cls.comment_regex().finditer(source_code): non = match.groupdict().get('noncomment') @@ -63,4 +109,10 @@ def remove_comments(cls, source_code: str, isList: bool = False): @classmethod def remove_keywords(cls, source: str) -> str: + """Remove known keywords from the source using BaseLanguage helper. + + :param source: input source text + :type source: str + :rtype: str + """ return super().remove_keywords(source) diff --git a/src/PyReprism/languages/julia.py b/src/PyReprism/languages/julia.py index a6d58a4..e7b3eb3 100644 --- a/src/PyReprism/languages/julia.py +++ b/src/PyReprism/languages/julia.py @@ -15,19 +15,41 @@ class Julia(BaseLanguage): @classmethod def file_extension(cls) -> str: + """Return the file extension used for Julia source files. + + :rtype: str + """ return extension.julia @classmethod def keywords(cls) -> list: + """Return a conservative list of Julia keywords. + + :rtype: list + """ return 'abstract|baremodule|begin|bitstype|break|catch|ccall|const|continue|do|else|elseif|end|export|finally|for|function|global|if|immutable|import|importall|let|local|macro|module|print|println|quote|return|try|type|typealias|using|while|true|false'.split('|') @classmethod def comment_regex(cls): + """Compile and return a regex that captures Julia comments and non-comment code. + + Supports single-line comments starting with ``#`` and block comments + delimited with ``#= ... =#``. The pattern provides named groups + ``comment`` and ``noncomment`` so BaseLanguage helpers can extract + non-comment fragments. + + :rtype: re.Pattern + """ # Support single-line '#' and block comments '#= ... =#' - return re.compile(r'(?P#=.*?$|#=[\s\S]*?=#|#.*?$)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|[^#\'\"]+)', re.DOTALL | re.MULTILINE) + pattern = r'''(?P#=.*?$|#=[\s\S]*?=#|#.*?$)|(?P'(\\.|[^\\'])*'|"(\\.|[^\\"])*"|[^#'\"]+)''' + return re.compile(pattern, re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls): + """Return a regex that matches Julia numeric literals (heuristic). + + :rtype: re.Pattern + """ return re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[box])?(?:[\da-f]+\.?\d*|\.\d+)(?:[efp][+-]?\d+)?j?') @classmethod @@ -36,8 +58,22 @@ def operator_regex(cls): @classmethod def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments from Julia source using BaseLanguage helper. + + :param source_code: Julia source text + :type source_code: str + :param isList: if True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ return super().remove_comments(source_code, isList=isList) @classmethod def remove_keywords(cls, source: str): + """Remove known Julia keywords from the provided source. + + :param source: input source text + :type source: str + :rtype: str + """ return super().remove_keywords(source) diff --git a/src/PyReprism/languages/keyman.py b/src/PyReprism/languages/keyman.py index 9861767..0630e2a 100644 --- a/src/PyReprism/languages/keyman.py +++ b/src/PyReprism/languages/keyman.py @@ -16,11 +16,19 @@ class Keyman(BaseLanguage): @classmethod def file_extension(cls) -> str: + """Return the file extension used for Keyman keyboard files. + + :rtype: str + """ return extension.keyman @classmethod def keywords(cls) -> list: # Keyman files are mostly data; keep keywords list empty for now. + """Return a list of language keywords (empty for Keyman). + + :rtype: list + """ return [] @classmethod @@ -30,6 +38,8 @@ def comment_regex(cls) -> re.Pattern: Accept common single-line comment forms used in keyboard mapping files (leading 'c ' lines, semicolon, hash, and C-style //). There are no widely used block-comments in Keyman, so we keep the pattern simple. + + :rtype: re.Pattern """ return re.compile( r"(?P//.*?$|#.*?$|;.*?$|^c\s.*?$)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/;#'\"\n]+)", @@ -38,8 +48,22 @@ def comment_regex(cls) -> re.Pattern: @classmethod def remove_comments(cls, source_code: str, isList: bool = False): + """Delegate to BaseLanguage.remove_comments to strip comment text. + + :param source_code: the Keyman source text + :type source_code: str + :param isList: if True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ return super().remove_comments(source_code, isList=isList) @classmethod def remove_keywords(cls, source: str) -> str: + """Delegate keyword removal to BaseLanguage (no-op for Keyman). + + :param source: input source text + :type source: str + :rtype: str + """ return super().remove_keywords(source) diff --git a/src/PyReprism/languages/kotlin.py b/src/PyReprism/languages/kotlin.py index 5c05627..884d316 100644 --- a/src/PyReprism/languages/kotlin.py +++ b/src/PyReprism/languages/kotlin.py @@ -1,11 +1,5 @@ import re from PyReprism.utils import extension - -from .base import BaseLanguage -from .registry import LanguageRegistry -import re -from PyReprism.utils import extension - from .base import BaseLanguage from .registry import LanguageRegistry diff --git a/src/PyReprism/languages/latex.py b/src/PyReprism/languages/latex.py index 0bb45ce..416a33f 100644 --- a/src/PyReprism/languages/latex.py +++ b/src/PyReprism/languages/latex.py @@ -1,49 +1,45 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Latex: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Latex(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.latex - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P%.*?$)|(?P[^%\n]*[^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Latex.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Latex.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Latex.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + return ''.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P%.*?$)|(?P[^%\n]*[^\n]*)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'') + + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + keys = cls.keywords() + if not keys: + return re.compile(r'$^') + return re.compile(r'\b(' + '|'.join(re.escape(k) for k in keys) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + kr = cls.keywords_regex() + if kr.pattern == '$^': + return source + return re.sub(kr, '', source) diff --git a/src/PyReprism/languages/less.py b/src/PyReprism/languages/less.py index 2186ad7..1e9ff62 100644 --- a/src/PyReprism/languages/less.py +++ b/src/PyReprism/languages/less.py @@ -1,49 +1,45 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Less: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Less(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.less - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/*]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Less.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Less.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Less.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + return ''.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/*]*[^\n]*)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'') + + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + keys = cls.keywords() + if not keys: + return re.compile(r'$^') + return re.compile(r'\b(' + '|'.join(re.escape(k) for k in keys) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + kr = cls.keywords_regex() + if kr.pattern == '$^': + return source + return re.sub(kr, '', source) diff --git a/src/PyReprism/languages/liquid.py b/src/PyReprism/languages/liquid.py index cdab183..c94bb12 100644 --- a/src/PyReprism/languages/liquid.py +++ b/src/PyReprism/languages/liquid.py @@ -1,49 +1,45 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Liquid: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Liquid(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.liquid - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P\{% comment %\}[\s\S]*?\{% endcomment %\}|{% #.*? %})|(?P[^{%]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Liquid.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Liquid.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Liquid.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + return ''.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P\{% comment %\}[\s\S]*?\{% endcomment %\}|\{\% #.*? \%\})|(?P[^\{%]*)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'') + + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + keys = cls.keywords() + if not keys: + return re.compile(r'$^') + return re.compile(r'\b(' + '|'.join(re.escape(k) for k in keys) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + kr = cls.keywords_regex() + if kr.pattern == '$^': + return source + return re.sub(kr, '', source) diff --git a/src/PyReprism/languages/livescript.py b/src/PyReprism/languages/livescript.py index dd9e7a4..ea901eb 100644 --- a/src/PyReprism/languages/livescript.py +++ b/src/PyReprism/languages/livescript.py @@ -1,49 +1,45 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class LiveScript: - def __init__(): - pass - - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class LiveScript(BaseLanguage): + @classmethod + def file_extension(cls) -> str: return extension.livescript - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^#/*]*[^\n]*)', re.MULTILINE | re.DOTALL) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(LiveScript.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in LiveScript.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(LiveScript.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + return ''.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P#.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^#/*]*[^\n]*)', re.MULTILINE | re.DOTALL) + + @classmethod + def number_regex(cls): + return re.compile(r'') + + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + keys = cls.keywords() + if not keys: + return re.compile(r'$^') + return re.compile(r'\b(' + '|'.join(re.escape(k) for k in keys) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + kr = cls.keywords_regex() + if kr.pattern == '$^': + return source + return re.sub(kr, '', source) diff --git a/src/PyReprism/languages/python.py b/src/PyReprism/languages/python.py index 705bff1..7f3403a 100644 --- a/src/PyReprism/languages/python.py +++ b/src/PyReprism/languages/python.py @@ -7,55 +7,11 @@ @LanguageRegistry.register class Python(BaseLanguage): - @classmethod - def file_extension(cls) -> str: - return extension.python - - @classmethod - def keywords(cls) -> list: - return 'as|assert|async|await|break|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|nonlocal|pass|print|raise|return|try|while|with|yield|__import__|abs|all|any|apply|ascii|basestring|bin|bool|buffer|bytearray|bytes|case|callable|chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|divmod|enumerate|eval|execfile|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|list|locals|long|match|map|max|memoryview|min|next|object|oct|open|ord|pow|property|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|vars|xrange|zip|True|False|None'.split('|') - - @classmethod - def comment_regex(cls) -> re.Pattern: - # Pattern captures: single-line '#', triple-quoted triple double or triple single, and noncomment segments - return re.compile(r"(?P#.*?$)|(?P\"\"\".*?\"\"\")|(?P''' .*?''')|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^#'\"]*)", re.DOTALL | re.MULTILINE) - - @classmethod - def number_regex(cls) -> re.Pattern: - return re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?\b', re.IGNORECASE) - - @classmethod - def operator_regex(cls) -> re.Pattern: - return re.compile(r'[-+%=]=?|!=|\*\*?=?|//?=?|<[<=>]?|>[=>]?|[&|^~]|\b(?:or|and|not)\b') - - @classmethod - def delimiters_regex(cls) -> re.Pattern: - return re.compile(r'[()\[\]{}.,:;@]') - - @classmethod - def remove_comments(cls, source_code: str, isList: bool = False) -> str: - res = super().remove_comments(source_code, isList=isList) - if isList: - return res - return res.strip() - - @classmethod - def remove_keywords(cls, source: str) -> str: - return re.sub(cls.keywords_regex(), '', source) - -import re -from PyReprism.utils import extension - - -class Python: """ This is the class for processing Python source code """ - def __init__(self): - pass - - @staticmethod - def file_extension() -> str: + @classmethod + def file_extension(cls) -> str: """ Return the file extension used for Python files. @@ -64,82 +20,58 @@ def file_extension() -> str: """ return extension.python - @staticmethod - def keywords() -> list: + @classmethod + def keywords(cls) -> list: """ Return a list of Python keywords and built-in functions. :return: A list of Python keywords and built-in function names. :rtype: list """ - keyword = 'as|assert|async|await|break|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|nonlocal|pass|print|raise|return|try|while|with|yield|__import__|abs|all|any|apply|ascii|basestring|bin|bool|buffer|bytearray|bytes|case|callable|chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|divmod|enumerate|eval|execfile|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|list|locals|long|match|map|max|memoryview|min|next|object|oct|open|ord|pow|property|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|vars|xrange|zip|True|False|None'.split('|') - return keyword + return 'as|assert|async|await|break|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|nonlocal|pass|print|raise|return|try|while|with|yield|__import__|abs|all|any|apply|ascii|basestring|bin|bool|buffer|bytearray|bytes|case|callable|chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|divmod|enumerate|eval|execfile|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|list|locals|long|match|map|max|memoryview|min|next|object|oct|open|ord|pow|property|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|vars|xrange|zip|True|False|None'.split('|') - @staticmethod - def comment_regex() -> re.Pattern: + @classmethod + def comment_regex(cls) -> re.Pattern: """ Compile and return a regular expression pattern to identify different types of comments and non-comment code in Python source files. :return: A compiled regex pattern with named groups to match single-line comments, multiline comments, and non-comment code elements. :rtype: re.Pattern """ - pattern = re.compile(r'(?P#.*?$)|'r'(?P""".*?""")|'r'(?P\'\'\'.*?\'\'\')|'r'(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#\'"]*)', re.DOTALL | re.MULTILINE) - return pattern + return re.compile(r"(?P#.*?$)|(?P\"\"\".*?\"\"\")|(?P''' .*?''')|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^#'\"]*)", re.DOTALL | re.MULTILINE) - @staticmethod - def number_regex() -> re.Pattern: + @classmethod + def number_regex(cls) -> re.Pattern: """ Compile and return a regular expression pattern to identify numeric literals in Python code. :return: A compiled regex pattern to match Python numeric literals, including integers, floats, and complex numbers. :rtype: re.Pattern """ - pattern = re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?\b', re.IGNORECASE) - return pattern + return re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?\b', re.IGNORECASE) - @staticmethod - def operator_regex() -> re.Pattern: + @classmethod + def operator_regex(cls) -> re.Pattern: """ Compile and return a regular expression pattern to identify Python operators. :return: A compiled regex pattern to match various Python operators and logical keywords. :rtype: re.Pattern """ - pattern = re.compile(r'[-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]|\b(?:or|and|not)\b') - return pattern - - @staticmethod - def keywords_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Python keywords. - - :return: A compiled regex pattern to match Python keywords. - :rtype: re.Pattern - """ - return re.compile(r'\b(' + '|'.join(Python.keywords()) + r')\b') - - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify Python boolean literals. - - :return: A compiled regex pattern to match Python boolean literals and `None`. - :rtype: re.Pattern - """ - return re.compile(r'\b(?:True|False|None)\b') + return re.compile(r'[-+%=]=?|!=|\*\*?=?|//?=?|<[<=>]?|>[=>]?|[&|^~]|\b(?:or|and|not)\b') - @staticmethod - def delimiters_regex() -> re.Pattern: + @classmethod + def delimiters_regex(cls) -> re.Pattern: """ Compile and return a regular expression pattern to identify Python delimiters. :return: A compiled regex pattern to match Python delimiters. :rtype: re.Pattern - """ + """ return re.compile(r'[()\[\]{}.,:;@]') - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: """ Remove comments from the provided Python source code string. @@ -148,22 +80,18 @@ def remove_comments(source_code: str, isList: bool = False) -> str: :return: The source code with all comments removed. :rtype: str """ - return Python.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in Python.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) + res = super().remove_comments(source_code, isList=isList) if isList: - return result - return ''.join(result) + return res + return res.strip() - @staticmethod - def remove_keywords(source: str) -> str: + @classmethod + def remove_keywords(cls, source: str) -> str: """ Remove all Python keywords from the provided source code string. :param str source: The source code string from which to remove Python keywords. :return: The source code string with all Python keywords removed. :rtype: str - """ - return re.sub(re.compile(Python.keywords_regex()), '', source) + """ + return re.sub(cls.keywords_regex(), '', source) \ No newline at end of file From 3dbc04ed66e9421d3807655943f9ccf043e4d5a1 Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 6 Oct 2025 18:19:35 -0700 Subject: [PATCH 06/19] ft/refactor: add implementation for new file extension --- src/PyReprism/languages/qore.py | 6 +- src/PyReprism/languages/r.py | 6 +- src/PyReprism/languages/wasam.py | 96 ++++++++++++++++++++++++++++++++ src/PyReprism/languages/xeora.py | 67 ++++++++++++++++++++++ src/PyReprism/languages/xojo.py | 87 +++++++++++++++++++++++++++++ 5 files changed, 252 insertions(+), 10 deletions(-) diff --git a/src/PyReprism/languages/qore.py b/src/PyReprism/languages/qore.py index 98450f3..3cd95e3 100644 --- a/src/PyReprism/languages/qore.py +++ b/src/PyReprism/languages/qore.py @@ -1,10 +1,6 @@ import re -try: - from PyReprism.utils import extension -except Exception: - from PyRePrism.utils import extension # fallback - +from PyReprism.utils import extension from .base import BaseLanguage from .registry import LanguageRegistry diff --git a/src/PyReprism/languages/r.py b/src/PyReprism/languages/r.py index 37539a4..3e2c5e4 100644 --- a/src/PyReprism/languages/r.py +++ b/src/PyReprism/languages/r.py @@ -1,10 +1,6 @@ import re -try: - from PyReprism.utils import extension -except Exception: - from PyRePrism.utils import extension # fallback - +from PyReprism.utils import extension from .base import BaseLanguage from .registry import LanguageRegistry diff --git a/src/PyReprism/languages/wasam.py b/src/PyReprism/languages/wasam.py index e69de29..4f4fc03 100644 --- a/src/PyReprism/languages/wasam.py +++ b/src/PyReprism/languages/wasam.py @@ -0,0 +1,96 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Wasm(BaseLanguage): + """WebAssembly helper (heuristics for .wasm/.wat text). + + Provides comment removal and simple token regexes suitable for the + textual WebAssembly formats. This is intentionally conservative — the + implementation focuses on reliably extracting non-comment fragments so + downstream processors can operate without language-specific parsing. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the file extension used for WebAssembly files. + + :rtype: str + """ + return extension.wasm + + @classmethod + def keywords(cls) -> list: + """Return a short, conservative list of WebAssembly text keywords. + + :rtype: list + """ + return [ + "module", + "func", + "memory", + "import", + "export", + "global", + "local", + "param", + "result", + "type", + ] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Compile and return a regex that captures WebAssembly comments. + + Supports WebAssembly text-style line comments that start with ``;;`` + as well as single-semicolon lines and block comments written as + ``(; ... ;)``. The pattern provides named groups ``comment`` and + ``noncomment`` as required by BaseLanguage helpers. + + :rtype: re.Pattern + """ + pattern = r'''(?P;;.*?$|\(;[\s\S]*?;\)|;.*?$)|(?P'(?:\\.|[^\\'])*'|"(?:\\.|[^\\\"])*"|[^;\(\)'"\n]+)''' + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + """Return a regex matching numeric literals (heuristic). + + :rtype: re.Pattern + """ + return re.compile(r'(?:\b0x[0-9A-Fa-f]+|\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?') + + @classmethod + def operator_regex(cls) -> re.Pattern: + """Return a simple operator regex suitable for tokenization. + + :rtype: re.Pattern + """ + return re.compile(r'[=+\-*/<>!&|%]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments using the BaseLanguage helper. + + :param source_code: the WebAssembly source text + :type source_code: str + :param isList: if True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + """Remove known keywords from the source using BaseLanguage. + + :param source: input source text + :type source: str + :rtype: str + """ + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/xeora.py b/src/PyReprism/languages/xeora.py index e69de29..34020d3 100644 --- a/src/PyReprism/languages/xeora.py +++ b/src/PyReprism/languages/xeora.py @@ -0,0 +1,67 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Xeora(BaseLanguage): + """Xeora language helper (.xeora template files). + + Xeora is a templating language used in some systems; this implementation + focuses on comment removal and a minimal set of helpers to allow + normalization and keyword stripping in the rest of the toolchain. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the file extension for Xeora files. + + :rtype: str + """ + return extension.xeora + + @classmethod + def keywords(cls) -> list: + """Return a small set of Xeora directives/keywords. + + :rtype: list + """ + return ["if", "else", "end", "for", "foreach", "import", "include"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Compile and return a regex capturing Xeora comments. + + Many template languages use ``//`` or ``#`` for single-line comments + and ``/* ... */`` for block comments. Provide a forgiving pattern that + captures these forms and returns a 'noncomment' group for text to keep. + + :rtype: re.Pattern + """ + pattern = r"(?P//.*?$|#.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/#'\"\n]+)" + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments and return either joined text or list of fragments. + + :param source_code: the template source + :type source_code: str + :param isList: when True return list of fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + """Strip known Xeora keywords using BaseLanguage helper. + + :param source: input text + :type source: str + :rtype: str + """ + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/xojo.py b/src/PyReprism/languages/xojo.py index e69de29..2f8e7ff 100644 --- a/src/PyReprism/languages/xojo.py +++ b/src/PyReprism/languages/xojo.py @@ -0,0 +1,87 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Xojo(BaseLanguage): + """Xojo (formerly REALbasic) language helper. + + Provides conservative comment handling and a minimal keyword list suitable + for normalization tasks. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the file extension used for Xojo/REALbasic files. + + :rtype: str + """ + return extension.xojo + + @classmethod + def keywords(cls) -> list: + """Return a conservative list of Xojo keywords. + + :rtype: list + """ + return [ + "Sub", + "Function", + "Dim", + "As", + "If", + "Then", + "Else", + "End", + "For", + "While", + "Return", + "Module", + ] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Return a regex capturing Xojo comments and non-comment fragments. + + Xojo supports single-line comments starting with ``'`` or ``//`` and + block comments using ``/* ... */`` in some contexts. Provide a forgiving + pattern with named groups 'comment' and 'noncomment'. + + :rtype: re.Pattern + """ + pattern = r"(?P//.*?$|'.*?$|/\*[\s\S]*?\*/)|(?P\"(?:\\.|[^\\\"])*\"|'(?:\\.|[^\\'])*'|[^/\'\"\n]+)" + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + """Return a basic number literal regex. + + :rtype: re.Pattern + """ + return re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments returning text or fragment list. + + :param source_code: source text + :type source_code: str + :param isList: when True return list of fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + """Remove known Xojo keywords. + + :param source: input text + :type source: str + :rtype: str + """ + return super().remove_keywords(source) + From 6c1e4c58c649bcc3c629665358f791b3dd1e2410 Mon Sep 17 00:00:00 2001 From: danielogen Date: Tue, 7 Oct 2025 05:52:27 -0700 Subject: [PATCH 07/19] refactor/add: add new programming language and refactoring existing code --- src/PyReprism/languages/docker.py | 72 ++++++++++++++++++-- src/PyReprism/languages/handlebars.py | 41 ++++++++++++ src/PyReprism/languages/haxe.py | 57 ++++++++++++++++ src/PyReprism/languages/hpkp.py | 40 ++++++++++++ src/PyReprism/languages/hsts.py | 40 ++++++++++++ src/PyReprism/languages/ichigojam.py | 55 ++++++++++++++++ src/PyReprism/languages/icon.py | 59 +++++++++++++++++ src/PyReprism/languages/inform7.py | 83 +++++++++++++++++++++++ src/PyReprism/languages/tcl.py | 40 ++++++++++++ src/PyReprism/languages/textile.py | 42 ++++++++++++ src/PyReprism/languages/twig.py | 77 ++++++++++++++++++++++ src/PyReprism/languages/verilog.py | 94 +++++++++++++++++++++++++++ src/PyReprism/languages/vhdl.py | 83 +++++++++++++++++++++++ src/PyReprism/languages/xeora.py | 11 ++++ src/PyReprism/languages/xojo.py | 8 +++ 15 files changed, 796 insertions(+), 6 deletions(-) diff --git a/src/PyReprism/languages/docker.py b/src/PyReprism/languages/docker.py index c0cd5fb..29e081c 100644 --- a/src/PyReprism/languages/docker.py +++ b/src/PyReprism/languages/docker.py @@ -6,36 +6,96 @@ @LanguageRegistry.register class Docker(BaseLanguage): + """Dockerfile language helper. + + Provides comment removal for Dockerfiles (lines starting with ``#``), a + basic keywords list for common Dockerfile directives, and simple regexes + for numbers and operators used by normalization utilities. + """ + @classmethod def file_extension(cls) -> str: + """Return the file extension/name used for Dockerfiles. + + Dockerfiles are often named "Dockerfile" rather than by extension, + but we keep the canonical name from the project's extensions map. + + :rtype: str + """ return extension.docker @classmethod def keywords(cls) -> list: + """Return Dockerfile directive keywords used for token filtering. + + :rtype: list + """ keyword = 'ADD|ARG|CMD|COPY|ENTRYPOINT|ENV|EXPOSE|FROM|HEALTHCHECK|LABEL|MAINTAINER|ONBUILD|RUN|SHELL|STOPSIGNAL|USER|VOLUME|WORKDIR'.split('|') return keyword @classmethod def comment_regex(cls): - pattern = re.compile(r'(?P#.*?$)|(?P[^#]*)', re.MULTILINE) + """Return a regex capturing Dockerfile comments and non-comment text. + + Matches lines starting with ``#`` as comments. The compiled pattern + provides named groups ``comment`` and ``noncomment`` required by + BaseLanguage.remove_comments. + + :rtype: re.Pattern + """ + pattern = re.compile(r'(?P#.*?$)|(?P[^#\n]+)', re.MULTILINE) return pattern @classmethod def number_regex(cls): - return re.compile(r'') + """Return a regex matching numeric literals (heuristic). + + Dockerfiles commonly contain ports and numeric arguments; this + pattern is intentionally simple. + + :rtype: re.Pattern + """ + return re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?') @classmethod def operator_regex(cls): - return re.compile(r'') + """Return a regex matching simple operators and punctuation. + + This is used for token splitting during normalization. + + :rtype: re.Pattern + """ + return re.compile(r'[=+\-*/<>!&|%:]+') @classmethod def keywords_regex(cls): - return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + """Compile a regex that matches any Dockerfile directive keyword. + + :rtype: re.Pattern + """ + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b", re.IGNORECASE) @classmethod def remove_comments(cls, source_code: str, isList: bool = False): - return super().remove_comments(source_code, isList) + """Remove comments from Dockerfile source. + + :param source_code: Dockerfile source text + :type source_code: str + :param isList: when True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList=isList) @classmethod def remove_keywords(cls, source: str): - return re.sub(re.compile(cls.keywords_regex()), '', source) + """Remove known Dockerfile directive keywords from source. + + Delegates to BaseLanguage.remove_keywords which uses the + cached keywords_regex implementation. + + :param source: input source text + :type source: str + :rtype: str + """ + return super().remove_keywords(source) diff --git a/src/PyReprism/languages/handlebars.py b/src/PyReprism/languages/handlebars.py index e69de29..0637a5e 100644 --- a/src/PyReprism/languages/handlebars.py +++ b/src/PyReprism/languages/handlebars.py @@ -0,0 +1,41 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Handlebars(BaseLanguage): + """Handlebars/Mustache templating helper. + + Focuses on removing template comments (``{{!-- ... --}}`` and + ``{{! ... }}``) and providing token regexes useful for normalization. + """ + + @classmethod + def file_extension(cls) -> str: + return extension.handlebars + + @classmethod + def keywords(cls) -> list: + return ["if", "else", "each", "with", "unless", "partial"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Capture Handlebars comments {{!-- ... --}} and {{! ... }} as comments + pattern = r"(?P\{\{!--[\s\S]*?--\}\}|\{\{![\s\S]*?\}\})|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^\{\'\"\n]+)" + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[=+\-*/<>!&|%:]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/haxe.py b/src/PyReprism/languages/haxe.py index e69de29..9c6e34e 100644 --- a/src/PyReprism/languages/haxe.py +++ b/src/PyReprism/languages/haxe.py @@ -0,0 +1,57 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Haxe(BaseLanguage): + """Haxe language helper. + + Provides conservative comment removal and basic token regexes used by + normalization utilities. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the file extension for Haxe source files. + + :rtype: str + """ + return extension.haxe + + @classmethod + def keywords(cls) -> list: + """Return a conservative list of Haxe keywords. + + :rtype: list + """ + return ["class", "extends", "implements", "var", "function", "if", "else", "return", "package", "import", "new", "static"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Return a regex capturing Haxe comments and non-comment fragments. + + Supports C-style block comments and line comments starting with ``//``. + :rtype: re.Pattern + """ + pattern = r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\'\"\n]+)" + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def operator_regex(cls) -> re.Pattern: + """Return an operator regex for Haxe tokenization. + + :rtype: re.Pattern + """ + return re.compile(r'[+\-*/%=<>!&|:^~]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/hpkp.py b/src/PyReprism/languages/hpkp.py index e69de29..2f8708f 100644 --- a/src/PyReprism/languages/hpkp.py +++ b/src/PyReprism/languages/hpkp.py @@ -0,0 +1,40 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Hpkp(BaseLanguage): + """Helper for HPKP (HTTP Public Key Pinning) header snippets. + + This is a simple helper focused on comment removal and basic token + regexes for normalization. + """ + + @classmethod + def file_extension(cls) -> str: + return extension.hpkp + + @classmethod + def keywords(cls) -> list: + return ["pin-sha256", "max-age", "includeSubDomains", "report-uri"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + pattern = re.compile(r'(?P#.*?$)|(?P[^#\n]+)', re.MULTILINE) + return pattern + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[=;:,]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/hsts.py b/src/PyReprism/languages/hsts.py index e69de29..f60cb2c 100644 --- a/src/PyReprism/languages/hsts.py +++ b/src/PyReprism/languages/hsts.py @@ -0,0 +1,40 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Hsts(BaseLanguage): + """Helper for HSTS (HTTP Strict Transport Security) policy snippets. + + HSTS snippets are small and mostly header-like; the class provides + comment removal and token regexes to help normalization utilities. + """ + + @classmethod + def file_extension(cls) -> str: + return extension.hsts + + @classmethod + def keywords(cls) -> list: + return ["max-age", "includeSubDomains", "preload"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + pattern = re.compile(r'(?P#.*?$)|(?P[^#\n]+)', re.MULTILINE) + return pattern + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[=;:,]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/ichigojam.py b/src/PyReprism/languages/ichigojam.py index e69de29..6634286 100644 --- a/src/PyReprism/languages/ichigojam.py +++ b/src/PyReprism/languages/ichigojam.py @@ -0,0 +1,55 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class IchigoJam(BaseLanguage): + """IchigoJam BASIC-like language helper. + + Provides comment removal (remarks with ``'`` or line comments) and + conservative regexes to assist normalization. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the file extension used for IchigoJam files. + + :rtype: str + """ + return extension.ichigojam + + @classmethod + def keywords(cls) -> list: + """Return a small list of BASIC-like keywords. + + :rtype: list + """ + return ["LET", "PRINT", "GOTO", "IF", "THEN", "FOR", "NEXT"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Return a regex capturing comments and non-comment fragments. + + IchigoJam uses single-line comments starting with ``'`` and often + shell-style comments; capture both forms. + + :rtype: re.Pattern + """ + pattern = re.compile(r"(?P'[^\n]*$|#.*?$)|(?P[^'\#\n]+)", re.MULTILINE) + return pattern + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[+\-*/=<>!&|%]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/icon.py b/src/PyReprism/languages/icon.py index e69de29..d243eda 100644 --- a/src/PyReprism/languages/icon.py +++ b/src/PyReprism/languages/icon.py @@ -0,0 +1,59 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Icon(BaseLanguage): + """Icon programming language helper. + + Provides a simple set of helpers for comment removal and basic token + recognition for Icon source files. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the file extension for Icon files. + + :rtype: str + """ + return extension.icon + + @classmethod + def keywords(cls) -> list: + """Return a conservative list of Icon keywords. + + :rtype: list + """ + return ["procedure", "if", "then", "else", "every", "return"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Return a regex capturing Icon comments and non-comment fragments. + + Icon uses single-line comments starting with "#". Provide a forgiving + pattern returning named groups 'comment' and 'noncomment'. + + :rtype: re.Pattern + """ + pattern = re.compile(r'(?P#.*?$)|(?P[^#\n]+)', re.MULTILINE) + return pattern + + @classmethod + def operator_regex(cls) -> re.Pattern: + """Return an operator regex for Icon. + + :rtype: re.Pattern + """ + return re.compile(r'[+\-*/=<>!&|%~:^]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/inform7.py b/src/PyReprism/languages/inform7.py index e69de29..c0290f5 100644 --- a/src/PyReprism/languages/inform7.py +++ b/src/PyReprism/languages/inform7.py @@ -0,0 +1,83 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Inform7(BaseLanguage): + """Inform 7 language helper (authoring language for interactive fiction). + + This implementation is intentionally conservative: it focuses on + extracting non-comment fragments and provides simple token regexes used + by normalization utilities. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the file extension for Inform 7 source files. + + :rtype: str + """ + return extension.inform7 + + @classmethod + def keywords(cls) -> list: + """Return a small set of Inform 7 control words. + + :rtype: list + """ + return ["if", "then", "else", "rule", "when", "instead", "carry"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Return a regex capturing comments and non-comment fragments. + + Uses a forgiving set of single-line comment markers (``//``, ``#``, + ``;``) and C-style block comments. Provides named groups + ``comment`` and ``noncomment``. + + :rtype: re.Pattern + """ + pattern = r"(?P//.*?$|#.*?$|;.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/;#'\"\n]+)" + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + """Return a basic numeric literal regex. + + :rtype: re.Pattern + """ + return re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?') + + @classmethod + def operator_regex(cls) -> re.Pattern: + """Return a regex matching common operators. + + :rtype: re.Pattern + """ + return re.compile(r'[=+\-*/<>!&|%:^~]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Strip comments and return either joined text or list of fragments. + + :param source_code: input text + :type source_code: str + :param isList: when True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + """Remove known keywords using BaseLanguage helper. + + :param source: input text + :type source: str + :rtype: str + """ + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/tcl.py b/src/PyReprism/languages/tcl.py index e69de29..fc2fa23 100644 --- a/src/PyReprism/languages/tcl.py +++ b/src/PyReprism/languages/tcl.py @@ -0,0 +1,40 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Tcl(BaseLanguage): + """Tcl language helper. + + Provides comment removal for Tcl (``#`` single-line) and basic + token regexes. + """ + + @classmethod + def file_extension(cls) -> str: + return extension.tcl + + @classmethod + def keywords(cls) -> list: + return ["proc", "set", "if", "else", "foreach", "while", "return"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + pattern = re.compile(r'(?P#.*?$)|(?P[^#\n]+)', re.MULTILINE) + return pattern + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[+\-*/=<>!&|%:]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/textile.py b/src/PyReprism/languages/textile.py index e69de29..4a97dba 100644 --- a/src/PyReprism/languages/textile.py +++ b/src/PyReprism/languages/textile.py @@ -0,0 +1,42 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Textile(BaseLanguage): + """Textile markup helper. + + Provides comment removal for Textile and simple token helpers for + normalization. + """ + + @classmethod + def file_extension(cls) -> str: + return extension.textile + + @classmethod + def keywords(cls) -> list: + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + # Textile comments are often HTML-style or line-prefixed; support + # common cases like and lines starting with "#" + pattern = r"(?P|#.*?$)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^<#'\"\n]+)" + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def operator_regex(cls) -> re.Pattern: + return re.compile(r'[=+\-*/<>!&|%:]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/twig.py b/src/PyReprism/languages/twig.py index e69de29..080a1b6 100644 --- a/src/PyReprism/languages/twig.py +++ b/src/PyReprism/languages/twig.py @@ -0,0 +1,77 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Twig(BaseLanguage): + """Twig templating language helper. + + Focuses on comment removal for Twig templates (``{# ... #}``) and + embedded languages while providing small helpers for keywords and + token regexes used by normalization routines. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the file extension for Twig templates. + + :rtype: str + """ + return extension.twig + + @classmethod + def keywords(cls) -> list: + """Return a small list of Twig control keywords. + + :rtype: list + """ + return ["if", "else", "endif", "for", "endfor", "set", "block", "endblock", "include"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Compile a regex capturing Twig comments and non-comment content. + + Twig comments are delimited with ``{# ... #}``. The pattern also + captures quoted strings and returns a 'noncomment' group for other + content. + + :rtype: re.Pattern + """ + # Capture Twig comments {# ... #}, quoted strings, and treat the + # remainder as non-comment fragments. Keep the pattern simple to + # avoid quoting fragility when patches are applied. + pattern = r"(?P\{#[\s\S]*?#\})|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^\{\'\"\n]+)" + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Remove Twig comments and return text or fragments. + + :param source_code: Twig template text + :type source_code: str + :param isList: when True return list of fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + """Remove Twig keywords from the provided source. + + :param source: input text + :type source: str + :rtype: str + """ + return super().remove_keywords(source) + + @classmethod + def operator_regex(cls) -> re.Pattern: + """Return a regex matching common operators used inside templates. + + :rtype: re.Pattern + """ + return re.compile(r'[=+\-*/<>!&|%\?:]+') diff --git a/src/PyReprism/languages/verilog.py b/src/PyReprism/languages/verilog.py index e69de29..5fdcbd7 100644 --- a/src/PyReprism/languages/verilog.py +++ b/src/PyReprism/languages/verilog.py @@ -0,0 +1,94 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Verilog(BaseLanguage): + """Verilog/SystemVerilog language helper. + + Implements conservative comment removal (C-style // and /* */) and + provides basic regex helpers for numbers and operators suitable for + normalization tasks. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the typical Verilog file extension. + + :rtype: str + """ + return extension.verilog + + @classmethod + def keywords(cls) -> list: + """Return a conservative list of Verilog/SystemVerilog keywords. + + :rtype: list + """ + return [ + "module", + "endmodule", + "input", + "output", + "wire", + "reg", + "always", + "assign", + "begin", + "end", + ] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Return a regex that captures comments and non-comment fragments. + + Supports single-line ``//`` and C-style block comments ``/* ... */``, + and returns named groups ``comment`` and ``noncomment`` for use by + BaseLanguage.remove_comments. + + :rtype: re.Pattern + """ + pattern = r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^/\'\"\n]+)" + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + """Return a regex matching numeric literals commonly found in Verilog. + + :rtype: re.Pattern + """ + return re.compile(r'\b(?:\d+\'?[duh]?\d*|\d+|\B\.\d+)') + + @classmethod + def operator_regex(cls) -> re.Pattern: + """Simple operator regex used for token splitting. + + :rtype: re.Pattern + """ + return re.compile(r'[+\-*/%=<>!&|:^~]+') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Strip comments and return joined text or list of fragments. + + :param source_code: Verilog source text + :type source_code: str + :param isList: if True return list of non-comment fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + """Remove known Verilog keywords using BaseLanguage helper. + + :param source: input text + :type source: str + :rtype: str + """ + return super().remove_keywords(source) + diff --git a/src/PyReprism/languages/vhdl.py b/src/PyReprism/languages/vhdl.py index e69de29..8a3b798 100644 --- a/src/PyReprism/languages/vhdl.py +++ b/src/PyReprism/languages/vhdl.py @@ -0,0 +1,83 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Vhdl(BaseLanguage): + """VHDL language helper. + + Provides comment removal (``--`` single-line comments and C-style + block comments when present), a small keyword set, and basic numeric + and operator regexes for normalization. + """ + + @classmethod + def file_extension(cls) -> str: + """Return the file extension used for VHDL files. + + :rtype: str + """ + return extension.vhdl + + @classmethod + def keywords(cls) -> list: + """Return a conservative list of VHDL keywords. + + :rtype: list + """ + return ["library", "use", "entity", "architecture", "begin", "end", "process", "signal", "port"] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """Return a regex capturing VHDL comments and non-comment text. + + VHDL single-line comments start with ``--``. We also include a + forgiving capture for C-style block comments (``/* ... */``) + which may appear in mixed sources. + + :rtype: re.Pattern + """ + pattern = r"(?P--.*?$|/\*[\s\S]*?\*/)|(?P'(?:\\.|[^\\'])*'|\"(?:\\.|[^\\\"])*\"|[^-/'\"\n]+)" + return re.compile(pattern, re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls) -> re.Pattern: + """Return a regex for numeric literals (heuristic). + + :rtype: re.Pattern + """ + return re.compile(r'(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments and return text or fragments list. + + :param source_code: VHDL source text + :type source_code: str + :param isList: when True return list of fragments + :type isList: bool + :rtype: list[str] or str + """ + return super().remove_comments(source_code, isList=isList) + + @classmethod + def remove_keywords(cls, source: str) -> str: + """Remove known VHDL keywords from the provided source. + + :param source: input text + :type source: str + :rtype: str + """ + return super().remove_keywords(source) + + @classmethod + def operator_regex(cls) -> re.Pattern: + """Return a regex matching VHDL operators and punctuation. + + :rtype: re.Pattern + """ + return re.compile(r'[+\-*/=<>:&|]+') + diff --git a/src/PyReprism/languages/xeora.py b/src/PyReprism/languages/xeora.py index 34020d3..e81a407 100644 --- a/src/PyReprism/languages/xeora.py +++ b/src/PyReprism/languages/xeora.py @@ -65,3 +65,14 @@ def remove_keywords(cls, source: str) -> str: """ return super().remove_keywords(source) + @classmethod + def operator_regex(cls) -> re.Pattern: + """Return a regex matching common operators and punctuation in templates. + + This is a conservative pattern intended for token splitting and + normalization tasks. + + :rtype: re.Pattern + """ + return re.compile(r'[=+\-*/<>!&|%^~]+') + diff --git a/src/PyReprism/languages/xojo.py b/src/PyReprism/languages/xojo.py index 2f8e7ff..80d7e1c 100644 --- a/src/PyReprism/languages/xojo.py +++ b/src/PyReprism/languages/xojo.py @@ -85,3 +85,11 @@ def remove_keywords(cls, source: str) -> str: """ return super().remove_keywords(source) + @classmethod + def operator_regex(cls) -> re.Pattern: + """Return an operator regex for Xojo source tokenization. + + :rtype: re.Pattern + """ + return re.compile(r'[+\-*/%=<>!&|:^~]+') + From 10a76cc6578402bef7bedd0a1ebbbe5385550979 Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 6 Jul 2026 19:23:38 -0700 Subject: [PATCH 08/19] update: major updates --- .flake8 | 10 +- .github/workflows/ci.yml | 23 +-- .gitignore | 5 +- README.md | 4 +- pyproject.toml | 51 ++++--- pytest.ini | 4 - requirements.txt | 9 +- src/PyReprism/languages/__init__.py | 106 ++++++++------ src/PyReprism/languages/apacheconf.py | 14 -- src/PyReprism/languages/aspnet.py | 2 +- src/PyReprism/languages/autoit.py | 2 +- src/PyReprism/languages/basic.py | 2 +- src/PyReprism/languages/batch.py | 2 +- src/PyReprism/languages/clike.py | 2 +- src/PyReprism/languages/csp.py | 7 +- src/PyReprism/languages/css.py | 63 +++----- src/PyReprism/languages/css_extras.py | 63 +++----- src/PyReprism/languages/d.py | 89 ++++++------ src/PyReprism/languages/django.py | 7 +- src/PyReprism/languages/erb.py | 16 +++ src/PyReprism/languages/erlang.py | 4 + src/PyReprism/languages/glsl.py | 22 +-- src/PyReprism/languages/go.py | 2 +- src/PyReprism/languages/haml.py | 64 +++------ src/PyReprism/languages/haskell.py | 72 ++++------ src/PyReprism/languages/html.py | 69 +++++---- src/PyReprism/languages/io.py | 43 ++++++ src/PyReprism/languages/j.py | 63 +++----- src/PyReprism/languages/java.py | 47 +++++- src/PyReprism/languages/jolie.py | 64 +++------ src/PyReprism/languages/json.py | 64 +++------ src/PyReprism/languages/latex.py | 2 +- src/PyReprism/languages/less.py | 2 +- src/PyReprism/languages/livescript.py | 2 +- src/PyReprism/languages/lolcode.py | 73 ++++++++++ src/PyReprism/languages/lua.py | 42 ++++++ src/PyReprism/languages/makefile.py | 66 ++++----- src/PyReprism/languages/markdown.py | 42 +++--- src/PyReprism/languages/markup.py | 64 +++------ src/PyReprism/languages/markup_templating.py | 32 +++++ src/PyReprism/languages/matlab.py | 11 ++ src/PyReprism/languages/mel.py | 40 ++++++ src/PyReprism/languages/mizar.py | 32 +++++ src/PyReprism/languages/monkey.py | 35 +++++ src/PyReprism/languages/n4js.py | 70 ++++----- src/PyReprism/languages/nasm.py | 44 ++++++ src/PyReprism/languages/nginx.py | 2 +- src/PyReprism/languages/nim.py | 54 +++++++ src/PyReprism/languages/nix.py | 39 +++++ src/PyReprism/languages/nsis.py | 34 +++++ src/PyReprism/languages/objectivec.py | 2 +- src/PyReprism/languages/ocaml.py | 69 ++++----- src/PyReprism/languages/opencl.py | 40 ++++++ src/PyReprism/languages/oz.py | 33 +++++ src/PyReprism/languages/parigp.py | 32 +++++ src/PyReprism/languages/parser.py | 32 +++++ src/PyReprism/languages/pascal.py | 83 +++++------ src/PyReprism/languages/perl.py | 19 +-- src/PyReprism/languages/php_extras.py | 1 - src/PyReprism/languages/plsql.py | 43 ++++++ src/PyReprism/languages/powershell.py | 33 +++++ src/PyReprism/languages/processing.py | 40 ++++++ src/PyReprism/languages/prolog.py | 66 ++++----- src/PyReprism/languages/properties.py | 28 ++++ src/PyReprism/languages/protobuf.py | 40 ++++++ src/PyReprism/languages/pug.py | 31 ++++ src/PyReprism/languages/puppet.py | 33 +++++ src/PyReprism/languages/pure.py | 33 +++++ src/PyReprism/languages/python.py | 2 +- src/PyReprism/languages/reason.py | 34 +++++ src/PyReprism/languages/renpy.py | 33 +++++ src/PyReprism/languages/rest.py | 32 +++++ src/PyReprism/languages/rip.py | 32 +++++ src/PyReprism/languages/roboconf.py | 31 ++++ src/PyReprism/languages/sas.py | 45 ++++++ src/PyReprism/languages/sass.py | 33 +++++ src/PyReprism/languages/scss.py | 67 ++++----- src/PyReprism/languages/smarty.py | 32 +++++ src/PyReprism/languages/soy.py | 33 +++++ src/PyReprism/languages/sql.py | 144 +++++++++++++++++++ src/PyReprism/languages/swift.py | 2 +- src/PyReprism/languages/vbnet.py | 111 +++++++------- src/PyReprism/languages/velocity.py | 66 ++++----- src/PyReprism/languages/vim.py | 68 ++++----- src/PyReprism/languages/visual_basic.py | 77 +++++----- src/PyReprism/languages/yaml.py | 64 +++------ src/PyReprism/{utils/helpers.py => py.typed} | 0 src/PyReprism/utils/extension.py | 1 + src/PyReprism/utils/normalizer.py | 3 +- src/tests/languages/test_new_languages.py | 98 +++++++++++++ src/tests/test_registry.py | 64 +++++++++ 91 files changed, 2346 insertions(+), 1065 deletions(-) delete mode 100644 pytest.ini create mode 100644 src/PyReprism/languages/sql.py rename src/PyReprism/{utils/helpers.py => py.typed} (100%) create mode 100644 src/tests/languages/test_new_languages.py create mode 100644 src/tests/test_registry.py diff --git a/.flake8 b/.flake8 index 4498afd..9fc97bc 100644 --- a/.flake8 +++ b/.flake8 @@ -2,6 +2,10 @@ show-source = True count = True statistics = True -; max-line-length = 150 -ignore = E501 -exclude = .github,venv,build,dist,docs,.venv,src/tests,src/PyReprism/languages/__init__.py,__test__.py \ No newline at end of file +max-line-length = 120 +# E501: long lines are common in this project's regex/keyword tables. +# The W1/W2/W3 and E1/E2/E3 codes below are pre-existing whitespace/indentation +# style debt across the legacy language modules; they are ignored so flake8 can +# focus on real correctness issues (pyflakes F-codes, undefined names, etc.). +extend-ignore = E501, W191, W291, W292, W293, W391, E101, E117, E128, E231, E302, E303, E265, E266 +exclude = .git,.github,venv,.venv,build,dist,docs,htmlcov,src/tests,src/PyReprism/languages/__init__.py,__test__.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e46f9f8..9e83635 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,34 +8,39 @@ on: - main jobs: - publish: - name: "Lint, test and upload to codecov" + test: + name: "Lint & test (Python ${{ matrix.python-version }})" runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - name: Check out the code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest pytest-cov + pip install -e ".[dev]" - name: Lint with flake8 run: | - flake8 . + flake8 src/PyReprism - name: Run tests with pytest run: | - pytest --cov=src/ --cov-report=xml + pytest --cov=src/PyReprism --cov-report=xml - name: Upload coverage to Codecov + if: matrix.python-version == '3.12' uses: codecov/codecov-action@v4 with: verbose: true env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index a9cfef5..6639b20 100644 --- a/.gitignore +++ b/.gitignore @@ -163,4 +163,7 @@ cython_debug/ #.idea/ # others -__test__.py \ No newline at end of file +__test__.py + +# macOS +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index ddd0ef6..d9557c3 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,10 @@ $ python3 -m venv venv $ source venv/bin/activate ``` -Then, install the requirements: +Then, install the package in editable mode with its development tooling: ```shell -$ pip install -r requirements.txt +$ pip install -e ".[dev]" ``` For more information on how to contribute, read our [contributing guidelines](CONTRIBUTING.md). diff --git a/pyproject.toml b/pyproject.toml index 20f7c20..a229ab5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,47 +1,64 @@ [build-system] -requires = ["hatchling", "hatch_vcs"] +requires = ["hatchling"] build-backend = "hatchling.build" [project] name = "PyReprism" version = "0.0.4" authors = [ - { name= "UNLV EVOL LAB", email="ogenrwot@unlv.nevada.edu" }, + { name = "UNLV EVOL LAB", email = "ogenrwot@unlv.nevada.edu" }, ] - -license={file="LICENSE" } -python_requires = ">=3.5" +license = { file = "LICENSE" } description = "Framework for Source Code Preprocessing" readme = "docs/intro.rst" -requires-python = ">=3.6" +requires-python = ">=3.8" classifiers = [ "Environment :: Console", - "Development Status :: 5 - Production/Stable", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] -[tool.hatch.build.targets.wheel] -packages = ["src/PyReprism"] - -[tool.hatch.build] -exclude = [ - "docs", - "tests", - "venv" -] [project.optional-dependencies] dev = [ "flake8", "pytest", "pytest-cov", ] +docs = [ + "sphinx", + "sphinx_rtd_theme", +] [project.urls] Homepage = "https://github.com/unlv-evol/PyReprism" Documentation = "https://pyreprism.readthedocs.io" -Issues = "https://github.com/unlv-evol/PyReprism/issues" \ No newline at end of file +Issues = "https://github.com/unlv-evol/PyReprism/issues" + +[tool.hatch.build.targets.wheel] +packages = ["src/PyReprism"] + +[tool.hatch.build] +exclude = [ + "docs", + "tests", + "venv", +] + +[tool.pytest.ini_options] +testpaths = ["src/tests"] +addopts = "--cov=src/PyReprism --cov-report=term" + +[tool.coverage.run] +source = ["src/PyReprism"] + +[tool.coverage.report] +show_missing = true diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 48b18ce..0000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -testpaths = src/tests -addopts = --cov=src/PyReprism --cov-report=html --cov-report=term -; --cov-fail-under=80 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index df595aa..ffea41e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -pytest -flake8 -pytest-cov \ No newline at end of file +# PyReprism has no runtime dependencies (Python standard library only). +# +# For development tooling (tests, linting) install the project with its +# optional "dev" extras instead: +# +# pip install -e ".[dev]" diff --git a/src/PyReprism/languages/__init__.py b/src/PyReprism/languages/__init__.py index 9803fda..5e7dbdb 100644 --- a/src/PyReprism/languages/__init__.py +++ b/src/PyReprism/languages/__init__.py @@ -1,51 +1,77 @@ -__version__ = "0.0.3" +"""Language registry bridge and lazy loader for PyReprism language modules. -# Minimal lazy loader / registry bridge for language modules. +Each language lives in ``PyReprism/languages/.py`` and self-registers with +:class:`~PyReprism.languages.registry.LanguageRegistry` on import. Language classes +can be accessed lazily as attributes, e.g. ``PyReprism.languages.Python``. +""" import importlib -from typing import Any +import os +import pkgutil +from typing import Any, Optional, Type +from .. import __version__ # single source of truth for the package version +from .base import BaseLanguage from .registry import LanguageRegistry - -# Pre-declare an __all__ mapping of known language module names (optional). -# Keep it small and let modules register themselves via LanguageRegistry. +# Common languages surfaced for discoverability. Any registered language is +# importable by name regardless of whether it appears here (see ``__getattr__``). __all__ = [ - # common names; modules register on import - 'Python', 'JavaScript', 'CPP', 'C', 'Clike', 'Go', 'MatLab', 'Ruby', 'PHP', 'Bash' + 'Python', 'JavaScript', 'CPP', 'C', 'Clike', 'Go', 'MatLab', 'Ruby', 'PHP', 'Bash', + 'get_language_by_extension', ] def __getattr__(name: str) -> Any: - """Lazy-load language module attribute by name. - - Accessing e.g. `PyReprism.languages.Python` will import the submodule - `PyReprism.languages.python` and return the class object if present. - """ - # If already registered, return directly - cls = LanguageRegistry.get(name) - if cls: - return cls - - # Try to import the submodule named by lowercasing the name - mod_name = name.lower() - try: - importlib.import_module(f'.{mod_name}', __name__) - except ModuleNotFoundError: - raise AttributeError(f"module {__name__} has no attribute {name}") - - cls = LanguageRegistry.get(name) - if cls: - return cls - raise AttributeError(f"module {__name__} has no attribute {name}") - - -def get_language_by_extension(ext: str): - """Return the first registered language class that reports the given file extension.""" - for name, cls in LanguageRegistry.all().items(): - try: - if cls.file_extension() == ext: - return cls - except Exception: - continue - return None + """Lazy-load a language class by attribute name. + + Accessing e.g. ``PyReprism.languages.Python`` imports the submodule + ``PyReprism.languages.python`` and returns the registered class. + """ + cls = LanguageRegistry.get(name) + if cls: + return cls + + try: + importlib.import_module(f'.{name.lower()}', __name__) + except ModuleNotFoundError: + raise AttributeError(f"module {__name__} has no attribute {name}") + + cls = LanguageRegistry.get(name) + if cls: + return cls + raise AttributeError(f"module {__name__} has no attribute {name}") + + +def _load_all_languages() -> None: + """Import every language submodule so the registry is fully populated. + + Language modules register themselves on import; without importing them the + registry only knows about classes that have already been accessed. + """ + package_dir = os.path.dirname(__file__) + for module in pkgutil.iter_modules([package_dir]): + name = module.name + if name.startswith('_') or name in ('base', 'registry'): + continue + try: + importlib.import_module(f'.{name}', __name__) + except Exception: + # A broken/optional language module should not break lookups. + continue + + +def get_language_by_extension(ext: str) -> Optional[Type[BaseLanguage]]: + """Return the first registered language class whose ``file_extension()`` matches ``ext``. + All language modules are imported on first call so cold lookups succeed. Some + extensions are shared by several languages (``.py`` -> Python/Django, ``.m`` -> + MatLab/ObjectiveC); in those cases the first registered match is returned. + """ + _load_all_languages() + for cls in LanguageRegistry.all().values(): + try: + if cls.file_extension() == ext: + return cls + except Exception: + continue + return None diff --git a/src/PyReprism/languages/apacheconf.py b/src/PyReprism/languages/apacheconf.py index fc3cd63..a4c3fe9 100644 --- a/src/PyReprism/languages/apacheconf.py +++ b/src/PyReprism/languages/apacheconf.py @@ -100,17 +100,3 @@ def remove_comments(cls, source_code: str, isList: bool = False): @classmethod def remove_keywords(cls, source: str): return re.sub(cls.keywords_regex(), '', source) - - - - @classmethod - def remove_comments(cls, source_code: str, isList: bool = False): - res = super().remove_comments(source_code, isList=isList) - if isList: - return res - return res - - @classmethod - def remove_keywords(cls, source: str): - return re.sub(cls.keywords_regex(), '', source) - return re.sub(re.compile(ApacheConf.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/aspnet.py b/src/PyReprism/languages/aspnet.py index 7f91329..5ce16b8 100644 --- a/src/PyReprism/languages/aspnet.py +++ b/src/PyReprism/languages/aspnet.py @@ -33,7 +33,7 @@ def comment_regex(cls) -> re.Pattern: """ # Match C-style block comments, // line comments and HTML comments return re.compile( - r"(?P//.*?$|/\*[\s\S]*?\*/|)|(?P[^///.*?$|/\*[\s\S]*?\*/||<%--[\s\S]*?--%>)|(?P.[^/<]*)", re.DOTALL | re.MULTILINE, ) diff --git a/src/PyReprism/languages/autoit.py b/src/PyReprism/languages/autoit.py index d2f28df..ac70537 100644 --- a/src/PyReprism/languages/autoit.py +++ b/src/PyReprism/languages/autoit.py @@ -17,7 +17,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls): - return re.compile(r'(?P;.*?$|#cs[\s\S]*?#ce|#cs.*?$|^.*?#ce)|(?P[^;#]*[^\n]*)', re.DOTALL | re.MULTILINE) + return re.compile(r'(?P;.*?$|#(?:cs|comments-start)[\s\S]*?#(?:ce|comments-end))|(?P.[^;#]*)', re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls): diff --git a/src/PyReprism/languages/basic.py b/src/PyReprism/languages/basic.py index bf2db66..65d2eff 100644 --- a/src/PyReprism/languages/basic.py +++ b/src/PyReprism/languages/basic.py @@ -17,7 +17,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls): - return re.compile(r"(?PREM.*?$|'.*?$)|(?P[^'R]*[^\n]*)", re.MULTILINE) + return re.compile(r"(?P\bREM\b.*?$|'.*?$)|(?P.[^'R]*)", re.DOTALL | re.MULTILINE | re.IGNORECASE) @classmethod def number_regex(cls): diff --git a/src/PyReprism/languages/batch.py b/src/PyReprism/languages/batch.py index 228d845..228a450 100644 --- a/src/PyReprism/languages/batch.py +++ b/src/PyReprism/languages/batch.py @@ -17,7 +17,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls): - return re.compile(r'(?PREM.*?$|::.*?$)|(?P[^:R]*[^\n]*)', re.MULTILINE) + return re.compile(r'(?P\bREM\b.*?$|::.*?$)|(?P.[^:R]*)', re.DOTALL | re.MULTILINE | re.IGNORECASE) @classmethod def number_regex(cls): diff --git a/src/PyReprism/languages/clike.py b/src/PyReprism/languages/clike.py index 7ea0b5f..3645b5d 100644 --- a/src/PyReprism/languages/clike.py +++ b/src/PyReprism/languages/clike.py @@ -17,7 +17,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls): - return re.compile(r"(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^/\'\"{}]*)", re.DOTALL | re.MULTILINE) + return re.compile(r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^/\'\"]*)", re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls): diff --git a/src/PyReprism/languages/csp.py b/src/PyReprism/languages/csp.py index d340483..9d76343 100644 --- a/src/PyReprism/languages/csp.py +++ b/src/PyReprism/languages/csp.py @@ -1,7 +1,8 @@ import re -from PyRePrism.utils import extension -from PyRePrism.languages.base import BaseLanguage -from PyRePrism.languages.registry import LanguageRegistry +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry @LanguageRegistry.register diff --git a/src/PyReprism/languages/css.py b/src/PyReprism/languages/css.py index 4c94712..4b9bfab 100644 --- a/src/PyReprism/languages/css.py +++ b/src/PyReprism/languages/css.py @@ -1,49 +1,28 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class CSS: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.css - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/*]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class CSS(BaseLanguage): + """CSS support (``/* */`` block comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(CSS.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in CSS.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.css - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(CSS.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P/\*[\s\S]*?\*/)|(?P.[^/]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/css_extras.py b/src/PyReprism/languages/css_extras.py index 2c96867..29544f4 100644 --- a/src/PyReprism/languages/css_extras.py +++ b/src/PyReprism/languages/css_extras.py @@ -1,49 +1,28 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class CssExtras: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.css_extras - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/*]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class CssExtras(BaseLanguage): + """CSS Extras support (``/* */`` block comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(CssExtras.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in CssExtras.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.css_extras - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(CssExtras.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P/\*[\s\S]*?\*/)|(?P.[^/]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/d.py b/src/PyReprism/languages/d.py index 34e1163..d8cb347 100644 --- a/src/PyReprism/languages/d.py +++ b/src/PyReprism/languages/d.py @@ -1,49 +1,54 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class D: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class D(BaseLanguage): + """D language support (``//``, ``/* */`` and ``/+ +/`` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" return extension.d - @staticmethod - def keywords() -> list: - keyword = 'abstract|alias|align|asm|assert|auto|body|bool|break|byte|case|cast|catch|cdouble|cent|cfloat|char|class|const|continue|creal|dchar|debug|default|delegate|delete|deprecated|do|double|else|enum|export|extern|false|final|finally|float|for|foreach|foreach_reverse|function|goto|idouble|if|ifloat|immutable|import|inout|int|interface|invariant|ireal|lazy|long|macro|mixin|module|new|nothrow|null|out|override|package|pragma|private|protected|public|pure|real|ref|return|scope|shared|short|static|struct|super|switch|synchronized|template|this|throw|true|try|typedef|typeid|typeof|ubyte|ucent|uint|ulong|union|unittest|ushort|version|void|volatile|wchar|while|with|__(?:(?:FILE|MODULE|LINE|FUNCTION|PRETTY_FUNCTION|DATE|EOF|TIME|TIMESTAMP|VENDOR|VERSION)__|gshared|traits|vector|parameters)|string|wstring|dstring|size_t|ptrdiff_t'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/|/\+[\s\S]*?\+/|/\+.*?$|^.*?\+/)|(?P[^/]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b0x\.?[a-f\d_]+(?:(?!\.\.)\.[a-f\d_]*)?(?:p[+-]?[a-f\d_]+)?[ulfi]*') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'\|[|=]?|&[&=]?|\+[+=]?|-[-=]?|\.?\.\.|=[>=]?|!(?:i[ns]\b|<>?=?|>=?|=)?|\bi[ns]\b|(?:<[<>]?|>>?>?|\^\^|[*\/%^~])=?') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(D.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in D.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(D.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'abstract|alias|align|asm|assert|auto|body|bool|break|byte|case|cast|catch|cdouble|' + 'cent|cfloat|char|class|const|continue|creal|dchar|debug|default|delegate|delete|' + 'deprecated|do|double|else|enum|export|extern|false|final|finally|float|for|foreach|' + 'foreach_reverse|function|goto|idouble|if|ifloat|immutable|import|inout|int|interface|' + 'invariant|ireal|lazy|long|macro|mixin|module|new|nothrow|null|out|override|package|' + 'pragma|private|protected|public|pure|real|ref|return|scope|shared|short|static|struct|' + 'super|switch|synchronized|template|this|throw|true|try|typedef|typeid|typeof|ubyte|' + 'ucent|uint|ulong|union|unittest|ushort|version|void|volatile|wchar|while|with|' + 'string|wstring|dstring|size_t|ptrdiff_t' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*[\s\S]*?\*/|/\+[\s\S]*?\+/)|' + r'(?P"(\\.|[^\\"])*"|.[^/"]*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'\b0x\.?[a-f\d_]+(?:(?!\.\.)\.[a-f\d_]*)?(?:p[+-]?[a-f\d_]+)?[ulfi]*' + ) + + @classmethod + def operator_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'\|[|=]?|&[&=]?|\+[+=]?|-[-=]?|\.?\.\.|=[>=]?|!(?:i[ns]\b|<>?=?|>=?|=)?|' + r'\bi[ns]\b|(?:<[<>]?|>>?>?|\^\^|[*\/%^~])=?' + ) diff --git a/src/PyReprism/languages/django.py b/src/PyReprism/languages/django.py index 77e1e1e..95c0246 100644 --- a/src/PyReprism/languages/django.py +++ b/src/PyReprism/languages/django.py @@ -50,8 +50,11 @@ def delimiters_regex(cls) -> re.Pattern: return re.compile(r'[()\[\]{}.,:;@]') @classmethod - def remove_comments(cls, source_code: str) -> str: - # Preserve original scalar behavior (return stripped string) + def remove_comments(cls, source_code: str, isList: bool = False): + # Preserve original scalar behavior (return stripped string); support + # the list form via BaseLanguage for signature consistency. + if isList: + return super().remove_comments(source_code, isList=True) return cls.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() @classmethod diff --git a/src/PyReprism/languages/erb.py b/src/PyReprism/languages/erb.py index 1948f78..4bd60bb 100644 --- a/src/PyReprism/languages/erb.py +++ b/src/PyReprism/languages/erb.py @@ -6,8 +6,14 @@ @LanguageRegistry.register class Erb(BaseLanguage): + """Minimal stub for ERB (Embedded Ruby) language. Provides comment removal and basic token regexes. + """ @classmethod def file_extension(cls) -> str: + """Return the file extension for ERB files. + + :rtype: str + """ return extension.erb @classmethod @@ -16,10 +22,20 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls) -> re.Pattern: + """Return a regex to match ERB comments and non-comment text. + + :rtype: re.Pattern + """ return re.compile(r'(?P|#.*?$)|(?P[^#<\n]+)', re.DOTALL | re.MULTILINE) @classmethod def remove_comments(cls, source_code: str, isList: bool = False): + """Remove comments from the given ERB source code. + + :param source_code: The ERB source code to process. + :param isList: If True, return a list of non-comment segments; otherwise, return a single string. + :rtype: str or list[str] + """ result = [] for match in cls.comment_regex().finditer(source_code): non = match.groupdict().get('noncomment') diff --git a/src/PyReprism/languages/erlang.py b/src/PyReprism/languages/erlang.py index 2065187..addd843 100644 --- a/src/PyReprism/languages/erlang.py +++ b/src/PyReprism/languages/erlang.py @@ -14,6 +14,10 @@ def file_extension(cls) -> str: @classmethod def keywords(cls) -> list: + """Return a list of Erlang keywords. + + :rtype: list[str] + """ return [] @classmethod diff --git a/src/PyReprism/languages/glsl.py b/src/PyReprism/languages/glsl.py index 25dbeb0..ab70d03 100644 --- a/src/PyReprism/languages/glsl.py +++ b/src/PyReprism/languages/glsl.py @@ -1,9 +1,3 @@ - -"""GLSL shader language minimal support. - -Handles C-style single-line and block comments conservatively. -""" - import re from PyReprism.utils import extension from .base import BaseLanguage @@ -12,19 +6,29 @@ @LanguageRegistry.register class Glsl(BaseLanguage): + """GLSL shader language minimal support. + + Handles C-style single-line and block comments conservatively. + """ @classmethod def file_extension(cls) -> str: return extension.glsl @classmethod def keywords(cls) -> list: - # keep empty for now; can be extended later - return [] + return ( + 'attribute|const|uniform|varying|buffer|shared|coherent|volatile|restrict|' + 'readonly|writeonly|layout|centroid|flat|smooth|noperspective|patch|sample|' + 'break|continue|do|for|while|switch|case|default|if|else|in|out|inout|' + 'void|bool|true|false|int|uint|float|double|discard|return|struct|precision|' + 'highp|mediump|lowp|vec2|vec3|vec4|ivec2|ivec3|ivec4|bvec2|bvec3|bvec4|' + 'mat2|mat3|mat4|sampler2D|sampler3D|samplerCube' + ).split('|') @classmethod def comment_regex(cls): # Match // single-line and /* ... */ block comments; capture non-comment otherwise - return re.compile(r'(?P//.*?$)|(?P/\*[\s\S]*?\*/)|(?P[^/\n][^\n]*)', re.DOTALL | re.MULTILINE) + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P.[^/]*)', re.DOTALL | re.MULTILINE) @classmethod def remove_comments(cls, source_code: str, isList: bool = False): diff --git a/src/PyReprism/languages/go.py b/src/PyReprism/languages/go.py index 0245a05..634cca5 100644 --- a/src/PyReprism/languages/go.py +++ b/src/PyReprism/languages/go.py @@ -17,7 +17,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls) -> re.Pattern: - return re.compile(r"(?P//.*?$|/\*.*?\*/|/\*.*?$|^.*?\*/|[{}]+)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^/\\'\"{}]*)", re.DOTALL | re.MULTILINE) + return re.compile(r"(?P//.*?$|/\*[\s\S]*?\*/)|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^/\\'\"]*)", re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls) -> re.Pattern: diff --git a/src/PyReprism/languages/haml.py b/src/PyReprism/languages/haml.py index c346c3a..23e935d 100644 --- a/src/PyReprism/languages/haml.py +++ b/src/PyReprism/languages/haml.py @@ -1,49 +1,29 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Haml: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.haml - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P^-#.*?$|^-#[\s\S]*?(?=\n\S)|^-#[\s\S]*?$)|(?P[^\n]*?(?=\n\S|$))', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Haml(BaseLanguage): + """Haml template support (``-#`` comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Haml.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Haml.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.haml - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Haml.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P^-#.*?$|^-#[\s\S]*?(?=\n\S)|^-#[\s\S]*?$)|' + r'(?P[^\n]*?(?=\n\S|$))', + re.MULTILINE, + ) diff --git a/src/PyReprism/languages/haskell.py b/src/PyReprism/languages/haskell.py index 05eda4b..488e99c 100644 --- a/src/PyReprism/languages/haskell.py +++ b/src/PyReprism/languages/haskell.py @@ -1,49 +1,37 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Haskell: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.haskell - - @staticmethod - def keywords() -> list: - keyword = 'case|class|data|deriving|do|else|if|in|infixl|infixr|instance|let|module|newtype|of|primitive|then|type|where|abs|acos|acosh|all|and|any|appendFile|approxRational|asTypeOf|asin|asinh|atan|atan2|atanh|basicIORun|break|catch|ceiling|chr|compare|concat|concatMap|const|cos|cosh|curry|cycle|decodeFloat|denominator|digitToInt|div|divMod|drop|dropWhile|either|elem|encodeFloat|enumFrom|enumFromThen|enumFromThenTo|enumFromTo|error|even|exp|exponent|fail|filter|flip|floatDigits|floatRadix|floatRange|floor|fmap|foldl|foldl1|foldr|foldr1|fromDouble|fromEnum|fromInt|fromInteger|fromIntegral|fromRational|fst|gcd|getChar|getContents|getLine|group|head|id|inRange|index|init|intToDigit|interact|ioError|isAlpha|isAlphaNum|isAscii|isControl|isDenormalized|isDigit|isHexDigit|isIEEE|isInfinite|isLower|isNaN|isNegativeZero|isOctDigit|isPrint|isSpace|isUpper|iterate|last|lcm|length|lex|lexDigits|lexLitChar|lines|log|logBase|lookup|map|mapM|mapM_|max|maxBound|maximum|maybe|min|minBound|minimum|mod|negate|not|notElem|null|numerator|odd|or|ord|otherwise|pack|pi|pred|primExitWith|print|product|properFraction|putChar|putStr|putStrLn|quot|quotRem|range|rangeSize|read|readDec|readFile|readFloat|readHex|readIO|readInt|readList|readLitChar|readLn|readOct|readParen|readSigned|reads|readsPrec|realToFrac|recip|rem|repeat|replicate|return|reverse|round|scaleFloat|scanl|scanl1|scanr|scanr1|seq|sequence|sequence_|show|showChar|showInt|showList|showLitChar|showParen|showSigned|showString|shows|showsPrec|significand|signum|sin|sinh|snd|sort|span|splitAt|sqrt|subtract|succ|sum|tail|take|takeWhile|tan|tanh|threadToIOResult|toEnum|toInt|toInteger|toLower|toRational|toUpper|truncate|uncurry|undefined|unlines|until|unwords|unzip|unzip3|userError|words|writeFile|zip|zip3|zipWith|zipWith3)'.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P--.*?$|\{-[\s\S]*?-\}|\{-.*?$|^.*?-\})|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^-\{\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'\b(?:\d+(?:\.\d+)?(?:e[+-]?\d+)?|0o[0-7]+|0x[0-9a-f]+)\b') - return pattern +@LanguageRegistry.register +class Haskell(BaseLanguage): + """Haskell support (``--`` line and ``{- -}`` block comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r"\s\.\s|[-!#$%*+=?&@|~.:<>^\\\/]*\.[-!#$%*+=?&@|~.:<>^\\\/]+|[-!#$%*+=?&@|~.:<>^\\\/]+\.[-!#$%*+=?&@|~.:<>^\\\/]*|[-!#$%*+=?&@|~:<>^\\\/]+|`([A-Z][\w']*\.)*[_a-z][\w']*`") - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Haskell.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Haskell.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.haskell - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Haskell.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'case|class|data|deriving|do|else|if|in|infixl|infixr|instance|let|module|newtype|' + 'of|primitive|then|type|where' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P--.*?$|\{-[\s\S]*?-\}|\{-.*?$|^.*?-\})|' + r'(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^-\{\'"]*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile(r'\b(?:\d+(?:\.\d+)?(?:e[+-]?\d+)?|0o[0-7]+|0x[0-9a-f]+)\b') diff --git a/src/PyReprism/languages/html.py b/src/PyReprism/languages/html.py index 33ce64c..b96446b 100644 --- a/src/PyReprism/languages/html.py +++ b/src/PyReprism/languages/html.py @@ -1,43 +1,40 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class HTML: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class HTML(BaseLanguage): + """HTML support (```` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" return extension.html - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P)|(?P[^<]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(HTML.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str) -> str: - return HTML.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(HTML.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P)|(?P.[^<]*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Strip HTML comments. + + :param source_code: The HTML source to process. + :param isList: If True, return a list of non-comment segments. + :rtype: str or list[str] + """ + if isList: + return super().remove_comments(source_code, isList=True) + return super().remove_comments(source_code).strip() diff --git a/src/PyReprism/languages/io.py b/src/PyReprism/languages/io.py index e69de29..2462e7a 100644 --- a/src/PyReprism/languages/io.py +++ b/src/PyReprism/languages/io.py @@ -0,0 +1,43 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + +@LanguageRegistry.register +class IO(BaseLanguage): + """IO language minimal support. + Handles line-oriented comments (lines starting with #) vs non-comment lines. + """ + + @classmethod + def file_extension(cls) -> str: + return extension.io + + @classmethod + def keywords(cls) -> list: + return ''.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P^#.*?$)|(?P^[^#\n].*?$)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'') + + @classmethod + def operator_regex(cls): + return re.compile(r'') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) \ No newline at end of file diff --git a/src/PyReprism/languages/j.py b/src/PyReprism/languages/j.py index 58a631f..e4fd3de 100644 --- a/src/PyReprism/languages/j.py +++ b/src/PyReprism/languages/j.py @@ -1,49 +1,28 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class J: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.j - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?PNB\..*$)|(?P[^N][^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class J(BaseLanguage): + """J language support (``NB.`` line comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(J.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in J.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.j - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(J.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?PNB\..*?$)|(?P.[^N]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/java.py b/src/PyReprism/languages/java.py index d9f3231..89b266b 100644 --- a/src/PyReprism/languages/java.py +++ b/src/PyReprism/languages/java.py @@ -14,40 +14,65 @@ class Java(BaseLanguage): @classmethod def file_extension(cls) -> str: - """Return the file extension used for Java files.""" + """Return the file extension used for Java files. + :rtype: str + """ return extension.java @classmethod def keywords(cls) -> list: - """Return a list of Java keywords and built-in functions.""" + """Return a list of Java keywords and built-in functions. + :rtype: list[str] + """ return 'abstract|continue|for|new|switch|assert|default|goto|package|synchronized|boolean|do|if|private|this|break|double|implements|protected|throw|byte|else|import|public|throws|case|enum|instanceof|return|transient|catch|extends|int|short|try|char|final|interface|static|void|class|finally|long|strictfp|volatile|const|float|native|super|while'.split('|') @classmethod def comment_regex(cls) -> re.Pattern: - """Regex to capture comments (group 'comment') and non-comment fragments (group 'noncomment').""" + """Regex to capture comments (group 'comment') and non-comment fragments (group 'noncomment'). + + :rtype: re.Pattern + """ return re.compile(r'(?P//.*?$|/\*[^*]*\*+(?:[^/*][^*]*\*+)*?/)|(?P[^/]+)', re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls) -> re.Pattern: - """Regex for numeric literals.""" + """Regex for numeric literals. + + :rtype: re.Pattern + """ return re.compile(r'\b0b[01]+\b|\b0x[\da-f]*\.?[\da-fp-]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?[df]?') @classmethod def operator_regex(cls) -> re.Pattern: - """Regex for Java operators.""" + """Regex for Java operators. + + :rtype: re.Pattern + """ return re.compile(r'(^|[^.])(?:\+[+=]?|-[-=]?|!=?|<<>?>?=?|==?|&[&=]?|\|[|=]?|\*=?|\/?=|%=?|\^=?|[?:~])') @classmethod def keywords_regex(cls) -> re.Pattern: - """Compile and return a regex that matches Java keywords.""" + """Compile and return a regex that matches Java keywords. + + :rtype: re.Pattern + """ return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b') @classmethod def boolean_regex(cls) -> re.Pattern: + """Regex for Java boolean literals. + + :param source: The source code string from which to remove Java keywords. + :rtype: re.Pattern + """ return re.compile(r'\b(?:true|false)\b') @classmethod def delimiters_regex(cls) -> re.Pattern: + """Regex for Java delimiters. + + :rtype: re.Pattern + """ return re.compile(r'[()\[\]{}.,:;@<>]') @classmethod @@ -55,6 +80,10 @@ def remove_comments(cls, source_code: str, isList: bool = False): """Preserve original Java behavior: for scalar output perform a substitution then strip (as older implementation did). If the caller requests a list, fall back to BaseLanguage's behavior. + + :param source_code: The Java source code to process. + :param isList: If True, return a list of non-comment segments; otherwise, return a single string. + :rtype: str or list[str] """ if isList: return super().remove_comments(source_code, isList=True) @@ -64,5 +93,9 @@ def remove_comments(cls, source_code: str, isList: bool = False): @classmethod def remove_keywords(cls, source: str) -> str: - """Delegate keyword removal to BaseLanguage.""" + """Delegate keyword removal to BaseLanguage. + + :param source: The source code string from which to remove Java keywords. + :rtype: str + """ return super().remove_keywords(source) diff --git a/src/PyReprism/languages/jolie.py b/src/PyReprism/languages/jolie.py index 0203725..cac0457 100644 --- a/src/PyReprism/languages/jolie.py +++ b/src/PyReprism/languages/jolie.py @@ -1,49 +1,29 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Jolie: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.jolie - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/\n]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Jolie(BaseLanguage): + """Jolie support (``//`` and ``/* */`` comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Jolie.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Jolie.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.jolie - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Jolie.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*[\s\S]*?\*/)|' + r'(?P"(\\.|[^\\"])*"|.[^/"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/json.py b/src/PyReprism/languages/json.py index db9d088..82ab895 100644 --- a/src/PyReprism/languages/json.py +++ b/src/PyReprism/languages/json.py @@ -1,49 +1,29 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Json: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.json - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/\n]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Json(BaseLanguage): + """JSON support (JSONC-style ``//`` and ``/* */`` comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Json.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Json.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.json - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Json.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ['true', 'false', 'null'] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*[\s\S]*?\*/)|' + r'(?P"(\\.|[^\\"])*"|.[^/"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/latex.py b/src/PyReprism/languages/latex.py index 416a33f..9a5c387 100644 --- a/src/PyReprism/languages/latex.py +++ b/src/PyReprism/languages/latex.py @@ -16,7 +16,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls): - return re.compile(r'(?P%.*?$)|(?P[^%\n]*[^\n]*)', re.MULTILINE) + return re.compile(r'(?P%.*?$)|(?P.[^%]*)', re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls): diff --git a/src/PyReprism/languages/less.py b/src/PyReprism/languages/less.py index 1e9ff62..94e129c 100644 --- a/src/PyReprism/languages/less.py +++ b/src/PyReprism/languages/less.py @@ -16,7 +16,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls): - return re.compile(r'(?P/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/*]*[^\n]*)', re.DOTALL | re.MULTILINE) + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P"(\\.|[^\\"])*"|.[^/"]*)', re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls): diff --git a/src/PyReprism/languages/livescript.py b/src/PyReprism/languages/livescript.py index ea901eb..67b0000 100644 --- a/src/PyReprism/languages/livescript.py +++ b/src/PyReprism/languages/livescript.py @@ -16,7 +16,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls): - return re.compile(r'(?P#.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^#/*]*[^\n]*)', re.MULTILINE | re.DOTALL) + return re.compile(r'(?P#.*?$|/\*[\s\S]*?\*/)|(?P.[^#/]*)', re.MULTILINE | re.DOTALL) @classmethod def number_regex(cls): diff --git a/src/PyReprism/languages/lolcode.py b/src/PyReprism/languages/lolcode.py index e69de29..b766eb6 100644 --- a/src/PyReprism/languages/lolcode.py +++ b/src/PyReprism/languages/lolcode.py @@ -0,0 +1,73 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class LOLCODE(BaseLanguage): + """LOLCODE language minimal support. + Handles line-oriented comments (lines starting with BTW) and block comments (OBTW ... TLDR). + """ + @classmethod + def file_extension(cls) -> str: + """Return the file extension for LOLCODE source files. + + :rtype: str + """ + return extension.lolcode + + @classmethod + def keywords(cls) -> list: + """Return a conservative list of LOLCODE keywords. + + :rtype: list""" + return r'HAI|KTHXBYE|CAN HAS|I HAS A|ITZ|R|SUM OF|DIFF OF|PRODUKT OF|QUOSHUNT OF|MOD OF|BIGGR OF|SMALLR OF|BOTH OF|EITHER OF|WON OF|NOT|ALL OF|ANY OF|O RLY\?|YA RLY|MEBBE|NO WAI|OIC|MKIH|RLY\?|WTF\?|OMG|OMGWTF|GTFO'.split('|') + + @classmethod + def comment_regex(cls): + """Return a regex capturing LOLCODE comments and non-comment fragments. + + Supports line comments starting with BTW and block comments between OBTW and TLDR. + + :rtype: re.Pattern + """ + return re.compile(r'(?PBTW.*?$|OBTW[\s\S]*?TLDR)|(?P[^B]*(?:B(?!T(?:W|\s*TW))[^B]*)*)', re.MULTILINE) + + @classmethod + def number_regex(cls): + """Return a regex capturing LOLCODE numeric literals. + + :rtype: re.Pattern + """ + return re.compile(r'\b-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?\b') + + @classmethod + def operator_regex(cls): + """Return a regex matching common LOLCODE operators. + + :rtype: re.Pattern + """ + return re.compile(r'\+|\-|\*|\/|%|\^|!|&&?|\|\|?') + + @classmethod + def keywords_regex(cls): + """Return a regex matching LOLCODE keywords. + + :rtype: re.Pattern + """ + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + """Remove comments from the given LOLCODE source code. + + :param source_code: The LOLCODE source code to process. + :param isList: If True, return a list of non-comment segments; otherwise, return a single string. + :rtype: str or list[str] + """ + res = super().remove_comments(source_code, isList=isList) + if isList: + return res + return ''.join(res) \ No newline at end of file diff --git a/src/PyReprism/languages/lua.py b/src/PyReprism/languages/lua.py index e69de29..def9a74 100644 --- a/src/PyReprism/languages/lua.py +++ b/src/PyReprism/languages/lua.py @@ -0,0 +1,42 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + +@LanguageRegistry.register +class LUA(BaseLanguage): + @classmethod + def file_extension(cls) -> str: + return extension.lua + + @classmethod + def keywords(cls) -> list: + return 'and|break|do|else|elseif|end|false|for|function|if|in|local|nil|not|or|repeat|return|then|true|until|while'.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P--\[\[.*?\]\]|--.*?$)|(?P[^-\n]+)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b0x[\dA-Fa-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?') + + @classmethod + def operator_regex(cls): + return re.compile(r'--?|-=|\+\+?|\+=|!=?|~|\*\*?|\*=|\/?=|%=?|<<=?|>>=?|<=?|>=?|==?|&&?|&=|\^=?|\|\|?|\|=|\?|:') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) + if isList: + return res + return ''.join(res) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), '', source) \ No newline at end of file diff --git a/src/PyReprism/languages/makefile.py b/src/PyReprism/languages/makefile.py index e09ecf2..14da20f 100644 --- a/src/PyReprism/languages/makefile.py +++ b/src/PyReprism/languages/makefile.py @@ -1,49 +1,31 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class MakeFile: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.makefile - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$)|(?P[^#\n]*[^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class MakeFile(BaseLanguage): + """Makefile support (``#`` line comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(MakeFile.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in MakeFile.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.makefile - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(MakeFile.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'ifeq|ifneq|ifdef|ifndef|else|endif|include|define|endef|override|export|' + 'unexport|vpath' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P#.*?$)|(?P.[^#]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/markdown.py b/src/PyReprism/languages/markdown.py index 9ab913a..f65c423 100644 --- a/src/PyReprism/languages/markdown.py +++ b/src/PyReprism/languages/markdown.py @@ -1,30 +1,28 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class MarkDown: - def __init__(): - pass - @staticmethod - def comment(): +@LanguageRegistry.register +class MarkDown(BaseLanguage): + """Markdown support (embedded HTML ```` comments).""" - return re.compile(r'(?P)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"]*)', re.DOTALL | re.MULTILINE) - - @staticmethod - def remove_comments(source: str): - return re.sub(MarkDown.comment, '', source) - - @staticmethod - def file_extension(): + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" return extension.markdown - @staticmethod - def keywords() -> list: - pass - - @staticmethod - def remove_keywords(source: str): - keywords = MarkDown.keywords() - pattern = r'\b(' + '|'.join(keywords) + r')\b' - return re.sub(re.compile(pattern), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P)|(?P.[^<]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/markup.py b/src/PyReprism/languages/markup.py index 9e8c9e9..fef064e 100644 --- a/src/PyReprism/languages/markup.py +++ b/src/PyReprism/languages/markup.py @@ -1,49 +1,29 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class MarkUp: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.markup - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^/\'"]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = '' - return pattern +@LanguageRegistry.register +class MarkUp(BaseLanguage): + """Generic markup (XML/HTML) support (```` comments).""" - @staticmethod - def operator_regex(): - pattern = '' - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(MarkUp.keywords()) + r')\b', re.IGNORECASE) - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in MarkUp.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.markup - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(MarkUp.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P)|' + r'(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^<\'"]*|<)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/markup_templating.py b/src/PyReprism/languages/markup_templating.py index e69de29..1a9a3dd 100644 --- a/src/PyReprism/languages/markup_templating.py +++ b/src/PyReprism/languages/markup_templating.py @@ -0,0 +1,32 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class MarkupTemplating(BaseLanguage): + """Generic markup templating support (HTML ```` comments). + + This is a base helper for template languages embedded in markup; it strips + standard XML/HTML comments. + """ + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.markup_templating + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P)|(?P.[^<]*|<)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/matlab.py b/src/PyReprism/languages/matlab.py index 80bca1c..e8983b1 100644 --- a/src/PyReprism/languages/matlab.py +++ b/src/PyReprism/languages/matlab.py @@ -26,4 +26,15 @@ def number_regex(cls): @classmethod def operator_regex(cls): return re.compile(r"\.?[*^\/\\']|[+\-:@]|[<>=~]=?|&&?|\|\|?") + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) + if isList: + return res + return ''.join(res) diff --git a/src/PyReprism/languages/mel.py b/src/PyReprism/languages/mel.py index e69de29..c8bc071 100644 --- a/src/PyReprism/languages/mel.py +++ b/src/PyReprism/languages/mel.py @@ -0,0 +1,40 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + +@LanguageRegistry.register +class MEL(BaseLanguage): + """MEL (Maya Embedded Language) minimal support. + Handles C-style single-line comments (//) vs non-comment lines.""" + @classmethod + def file_extension(cls) -> str: + return extension.mel + + @classmethod + def keywords(cls) -> list: + return 'break|case|catch|continue|default|do|else|elseif|end|for|global|if|in|local|proc|return|switch|then|while'.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P//.*?$)|(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^/\'"]*)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b0x[\dA-Fa-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?') + + @classmethod + def operator_regex(cls): + return re.compile(r'--?|-=|\+\+?|\+=|!=?|~|\*\*?|\*=|\/?=|%=?|<<=?|>>=?|<=?|>=?|==?|&&?|&=|\^=?|\|\|?|\|=|\?|:') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False) -> str: + res = super().remove_comments(source_code, isList=isList) + if isList: + return res + return ''.join(res) \ No newline at end of file diff --git a/src/PyReprism/languages/mizar.py b/src/PyReprism/languages/mizar.py index e69de29..5966ef7 100644 --- a/src/PyReprism/languages/mizar.py +++ b/src/PyReprism/languages/mizar.py @@ -0,0 +1,32 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Mizar(BaseLanguage): + """Mizar support (``::`` line comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.mizar + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'environ|begin|theorem|scheme|proof|end|for|ex|being|such|that|holds|st|' + 'let|assume|thus|hence|then|by|from|def|definition|redefine|means|equals|' + 'reserve|consider|take|per|cases|suppose|now|and|or|not|implies|iff' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P::.*?$)|(?P.[^:]*|:)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/monkey.py b/src/PyReprism/languages/monkey.py index e69de29..21d847c 100644 --- a/src/PyReprism/languages/monkey.py +++ b/src/PyReprism/languages/monkey.py @@ -0,0 +1,35 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Monkey(BaseLanguage): + """Monkey X support (``'`` line and ``#Rem ... #End`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.monkey + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'Function|Method|Class|Interface|Extends|Implements|Import|Module|Strict|Public|' + 'Private|Property|Field|Global|Local|Const|Return|New|Self|Super|Extern|' + 'If|Then|Else|ElseIf|EndIf|While|Wend|Repeat|Until|Forever|For|To|Step|Next|' + 'Select|Case|Default|End|Throw|Try|Catch|Print|And|Or|Not|Mod|Shl|Shr|' + 'True|False|Null|Void|Int|Float|String|Bool|Array|Object|Continue|Exit|Eachin' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P#[Rr]em[\s\S]*?#[Ee]nd|\'.*?$)|' + r'(?P"(\\.|[^\\"])*"|.[^\'#"]*|[#\'])', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/n4js.py b/src/PyReprism/languages/n4js.py index 22f454f..e48ebaf 100644 --- a/src/PyReprism/languages/n4js.py +++ b/src/PyReprism/languages/n4js.py @@ -1,49 +1,35 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class N4js: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.n4js - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/*]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class N4js(BaseLanguage): + """N4JS support (``//`` and ``/* */`` comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(N4js.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in N4js.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.n4js - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(N4js.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|' + 'extends|finally|for|function|if|import|in|instanceof|new|null|return|super|switch|' + 'this|throw|try|typeof|var|void|while|with|yield|let|await|async|' + 'implements|interface|package|private|protected|public|static|get|set|' + 'true|false|abstract' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*[\s\S]*?\*/)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^/\'"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/nasm.py b/src/PyReprism/languages/nasm.py index e69de29..fd58b98 100644 --- a/src/PyReprism/languages/nasm.py +++ b/src/PyReprism/languages/nasm.py @@ -0,0 +1,44 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + +@LanguageRegistry.register +class NASM(BaseLanguage): + """NASM assembly language minimal support. + Handles line-oriented comments (lines starting with ;) vs non-comment lines. + """ + + @classmethod + def file_extension(cls) -> str: + return extension.nasm + + @classmethod + def keywords(cls) -> list: + return ''.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P^;.*?$)|(?P^[^;\n].*?$)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b0x[\dA-Fa-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?|\b\d+[bwdq]?\b') + + @classmethod + def operator_regex(cls): + return re.compile(r'--?|-=|\+\+?|\+=|!=?|~|\*\*?|\*=|\/?=|%=?|<<=?|>>=?|<=?|>=?|==?|&&?|&=|\^=?|\|\|?|\|=|\?|:') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(re.compile(cls.keywords_regex()), '', source) + \ No newline at end of file diff --git a/src/PyReprism/languages/nginx.py b/src/PyReprism/languages/nginx.py index 0ce2056..66d7b6b 100644 --- a/src/PyReprism/languages/nginx.py +++ b/src/PyReprism/languages/nginx.py @@ -25,7 +25,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls): - return re.compile(r'(?P#.*?$)|(?P[^#\n]*(?:\n|$))', re.MULTILINE) + return re.compile(r'(?P#.*?$)|(?P.[^#]*)', re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls): diff --git a/src/PyReprism/languages/nim.py b/src/PyReprism/languages/nim.py index e69de29..df6308e 100644 --- a/src/PyReprism/languages/nim.py +++ b/src/PyReprism/languages/nim.py @@ -0,0 +1,54 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class NIM(BaseLanguage): + """Nim language support (``#`` line and ``#[ ]#`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + return extension.nim + + @classmethod + def keywords(cls) -> list: + return ( + 'addr|and|as|asm|bind|block|break|case|cast|concept|const|continue|converter|' + 'defer|discard|distinct|div|do|elif|else|end|enum|except|export|finally|for|from|' + 'func|if|import|in|include|interface|is|isnot|iterator|let|macro|method|mixin|mod|' + 'nil|not|notin|object|of|or|out|proc|ptr|raise|ref|return|shl|shr|static|template|' + 'try|tuple|type|using|var|when|while|xor|yield|true|false|result|echo' + ).split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P#\[[\s\S]*?\]#|#.*?$)|(?P"(\\.|[^\\"])*"|.[^#"]*)', re.DOTALL | re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b0x[\dA-Fa-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?') + + @classmethod + def operator_regex(cls): + return re.compile(r'--?|-=|\+\+?|\+=|!=?|~|\*\*?|\*=|\/?=|%=?|<<=?|>>=?|<=?|>=?|==?|&&?|&=|\^=?|\|\|?|\|=|\?|:') + + @classmethod + def keywords_regex(cls): + keys = cls.keywords() + if not keys: + return re.compile(r'$^') + return re.compile(r'\b(' + '|'.join(re.escape(k) for k in keys) + r')\b') + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + kr = cls.keywords_regex() + if kr.pattern == '$^': + return source + return re.sub(kr, '', source) \ No newline at end of file diff --git a/src/PyReprism/languages/nix.py b/src/PyReprism/languages/nix.py index e69de29..5d403f0 100644 --- a/src/PyReprism/languages/nix.py +++ b/src/PyReprism/languages/nix.py @@ -0,0 +1,39 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + +@LanguageRegistry.register +class NIX(BaseLanguage): + """Nix language minimal support. + Handles line-oriented comments (lines starting with #) vs non-comment lines. + """ + + @classmethod + def file_extension(cls) -> str: + return extension.nix + + @classmethod + def keywords(cls) -> list: + return 'let|in|if|then|else|rec|with|inherit|assert|or|and|import|abort|baseNameOf|dirOf|builtins'.split('|') + + @classmethod + def comment_regex(cls): + return re.compile(r'(?P^#.*?$)|(?P^[^#\n].*?$)', re.MULTILINE) + + @classmethod + def number_regex(cls): + return re.compile(r'\b0x[\dA-Fa-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?') + + @classmethod + def operator_regex(cls): + return re.compile(r'--?|-=|\+\+?|\+=|!=?|~|\*\*?|\*=|\/?=|%=?|<<=?|>>=?|<=?|>=?|==?|&&?|&=|\^=?|\|\|?|\|=|\?|:') + + @classmethod + def keywords_regex(cls): + return re.compile(r"\b(" + "|".join(cls.keywords()) + r")\b") + + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + return super().remove_comments(source_code, isList) \ No newline at end of file diff --git a/src/PyReprism/languages/nsis.py b/src/PyReprism/languages/nsis.py index e69de29..7e05b82 100644 --- a/src/PyReprism/languages/nsis.py +++ b/src/PyReprism/languages/nsis.py @@ -0,0 +1,34 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class NSIS(BaseLanguage): + """NSIS installer script support (``;`` / ``#`` line and ``/* */`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.nsis + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'Section|SectionEnd|SectionGroup|SectionGroupEnd|Function|FunctionEnd|' + 'Name|OutFile|InstallDir|Page|UninstPage|SetOutPath|File|WriteRegStr|' + 'WriteUninstaller|Delete|RMDir|CreateDirectory|CreateShortCut|MessageBox|' + 'Goto|Return|Call|Push|Pop|StrCmp|IntCmp|Abort|Quit|ExecWait|Exec' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P[;#].*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|.[^;#/"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/objectivec.py b/src/PyReprism/languages/objectivec.py index 5941bc5..cd256ea 100644 --- a/src/PyReprism/languages/objectivec.py +++ b/src/PyReprism/languages/objectivec.py @@ -27,7 +27,7 @@ def keywords(cls) -> list: def comment_regex(cls) -> re.Pattern: # Capture // line comments and C-style block comments; provide a # 'noncomment' group with the text to keep. - return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P[^/]*[^\n]*)', re.MULTILINE) + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P"(\\.|[^\\"])*"|.[^/"]*)', re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls) -> re.Pattern: diff --git a/src/PyReprism/languages/ocaml.py b/src/PyReprism/languages/ocaml.py index 3656c4d..e3c6d22 100644 --- a/src/PyReprism/languages/ocaml.py +++ b/src/PyReprism/languages/ocaml.py @@ -1,49 +1,34 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Ocaml: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.ocaml - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P(\(\*[\s\S]*?\*\)|;.*?$|;;.*?$))|(?P[^;(*]*[^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Ocaml(BaseLanguage): + """OCaml support (``(* *)`` comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Ocaml.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Ocaml.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.ocaml - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Ocaml.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'and|as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|' + 'for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|match|method|' + 'module|mutable|new|nonrec|object|of|open|or|private|rec|sig|struct|then|to|try|' + 'type|val|virtual|when|while|with|true|false' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P\(\*[\s\S]*?\*\))|' + r'(?P"(\\.|[^\\"])*"|.[^(*"]*|[(*])', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/opencl.py b/src/PyReprism/languages/opencl.py index e69de29..c220cfa 100644 --- a/src/PyReprism/languages/opencl.py +++ b/src/PyReprism/languages/opencl.py @@ -0,0 +1,40 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class OpenCL(BaseLanguage): + """OpenCL C language support (C-style ``//`` and ``/* */`` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.opencl + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + '__kernel|kernel|__global|global|__local|local|__constant|constant|__private|private|' + '__read_only|__write_only|__read_write|read_only|write_only|read_write|' + 'if|else|for|while|do|switch|case|default|break|continue|return|goto|' + 'void|char|uchar|short|ushort|int|uint|long|ulong|float|double|half|bool|size_t|' + 'const|volatile|restrict|static|inline|sizeof|struct|union|enum|typedef' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^/\'"]*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile(r'\b0x[\da-fA-F]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?[fFuUlL]*') diff --git a/src/PyReprism/languages/oz.py b/src/PyReprism/languages/oz.py index e69de29..83375ec 100644 --- a/src/PyReprism/languages/oz.py +++ b/src/PyReprism/languages/oz.py @@ -0,0 +1,33 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Oz(BaseLanguage): + """Oz language support (``%`` line and ``/* */`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.oz + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'declare|local|in|end|proc|fun|functor|class|meth|attr|feat|from|prop|' + 'if|then|else|elseif|elsecase|case|of|for|do|while|try|catch|finally|raise|' + 'thread|lock|or|dis|choice|not|cond|andthen|orelse|div|mod|true|false|unit|nil' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P%.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^%/\'"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/parigp.py b/src/PyReprism/languages/parigp.py index e69de29..22f05e5 100644 --- a/src/PyReprism/languages/parigp.py +++ b/src/PyReprism/languages/parigp.py @@ -0,0 +1,32 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class PariGP(BaseLanguage): + r"""PARI/GP support (``\\`` line and ``/* */`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.parigp + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'breakpoint|break|dbg_down|dbg_err|dbg_up|dbg_x|forcomposite|fordiv|forell|' + 'forpart|forprime|forstep|forsubgroup|forvec|for|if|iferr|next|return|until|while' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + r""":rtype: re.Pattern""" + return re.compile( + r'(?P\\\\.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|.[^\\/"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/parser.py b/src/PyReprism/languages/parser.py index e69de29..f2cb51b 100644 --- a/src/PyReprism/languages/parser.py +++ b/src/PyReprism/languages/parser.py @@ -0,0 +1,32 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Parser(BaseLanguage): + """Parser (Parser 3) support (``#`` line and ``/* */`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.parser + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'if|else|elsif|switch|case|default|while|break|continue|return|' + 'true|false|def|class|method|import' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P#.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|.[^#/"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/pascal.py b/src/PyReprism/languages/pascal.py index 849b02b..90f8cb4 100644 --- a/src/PyReprism/languages/pascal.py +++ b/src/PyReprism/languages/pascal.py @@ -1,48 +1,49 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Pascal: - def __init__(self): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class Pascal(BaseLanguage): + """Pascal support (``(* *)``, ``{ }`` and ``//`` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" return extension.pascal - @staticmethod - def keywords() -> list: - keyword = 'absolute|array|asm|begin|case|const|constructor|destructor|do|downto|else|end|file|for|function|goto|if|implementation|inherited|inline|interface|label|nil|object|of|operator|packed|procedure|program|record|reintroduce|repeat|self|set|string|then|to|type|unit|until|uses|var|while|with|dispose|exit|false|new|true|class|dispinterface|except|exports|finalization|finally|initialization|inline|library|on|out|packed|property|raise|resourcestring|threadvar|try|absolute|abstract|alias|assembler|bitpacked|break|cdecl|continue|cppdecl|cvar|default|deprecated|dynamic|enumerator|experimental|export|external|far|far16|forward|generic|helper|implements|index|interrupt|iochecks|local|message|name|near|nodefault|noreturn|nostackframe|oldfpccall|otherwise|overload|override|pascal|platform|private|protected|public|published|read|register|reintroduce|result|safecall|saveregisters|softfloat|specialize|static|stdcall|stored|strict|unaligned|unimplemented|varargs|virtual|write'.split('|') - return keyword - - @staticmethod - def comment_regex(): - return re.compile(r'(?P#.*?$)|(?P\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#\'"]*)', re.DOTALL | re.MULTILINE) - - @staticmethod - def number_regex(): - pattern = re.compile(r'(?:\b(?=\d)|\B(?=\.))(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?\b', re.IGNORECASE) - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'[-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]|\b(?:or|and|not)\b') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Pascal.keywords()) + r')\b', re.IGNORECASE) - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Pascal.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Pascal.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'absolute|array|asm|begin|case|const|constructor|destructor|do|downto|else|end|file|' + 'for|function|goto|if|implementation|inherited|inline|interface|label|nil|object|of|' + 'operator|packed|procedure|program|record|reintroduce|repeat|self|set|string|then|to|' + 'type|unit|until|uses|var|while|with|dispose|exit|false|new|true|class|except|exports|' + 'finalization|finally|initialization|library|on|out|property|raise|threadvar|try|' + 'abstract|private|protected|public|published|virtual|override|overload' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|\(\*[\s\S]*?\*\)|\{[\s\S]*?\})|' + r'(?P\'(\\.|\'\'|[^\'])*\'|.[^/({\']*|[/({])', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile(r'\$[0-9a-f]+|\b\d+(?:\.\d+)?(?:e[+-]?\d+)?', re.IGNORECASE) + + @classmethod + def keywords_regex(cls) -> re.Pattern: + """Case-insensitive keyword matcher (Pascal is case-insensitive). + + :rtype: re.Pattern + """ + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b', re.IGNORECASE) diff --git a/src/PyReprism/languages/perl.py b/src/PyReprism/languages/perl.py index b0ea35d..bd0dcfb 100644 --- a/src/PyReprism/languages/perl.py +++ b/src/PyReprism/languages/perl.py @@ -1,10 +1,6 @@ import re from PyReprism.utils import extension - -import re -from PyReprism.utils import extension - from .base import BaseLanguage from .registry import LanguageRegistry @@ -15,7 +11,7 @@ class Perl(BaseLanguage): @classmethod def file_extension(cls) -> str: - return extension.pl + return extension.perl @classmethod def keywords(cls) -> list: @@ -72,16 +68,3 @@ def remove_comments(cls, source_code: str, isList: bool = False): @classmethod def remove_keywords(cls, source: str) -> str: return super().remove_keywords(source) - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Perl.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Perl.keywords_regex()), '', source) diff --git a/src/PyReprism/languages/php_extras.py b/src/PyReprism/languages/php_extras.py index 31c6cdd..a98baa3 100644 --- a/src/PyReprism/languages/php_extras.py +++ b/src/PyReprism/languages/php_extras.py @@ -1,4 +1,3 @@ -import re from .php import PHP from .registry import LanguageRegistry diff --git a/src/PyReprism/languages/plsql.py b/src/PyReprism/languages/plsql.py index e69de29..d5ad5ef 100644 --- a/src/PyReprism/languages/plsql.py +++ b/src/PyReprism/languages/plsql.py @@ -0,0 +1,43 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class PLSQL(BaseLanguage): + """Oracle PL/SQL support (``--`` line and ``/* */`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.plsql + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'BEGIN|END|DECLARE|PROCEDURE|FUNCTION|PACKAGE|BODY|TRIGGER|CURSOR|LOOP|WHILE|FOR|' + 'IF|ELSIF|ELSE|THEN|CASE|WHEN|EXIT|RETURN|EXCEPTION|RAISE|IS|AS|IN|OUT|NOCOPY|' + 'SELECT|INSERT|UPDATE|DELETE|MERGE|FROM|WHERE|INTO|VALUES|SET|AND|OR|NOT|NULL|' + 'TABLE|VIEW|INDEX|TYPE|RECORD|VARCHAR2|NUMBER|DATE|BOOLEAN|INTEGER|PLS_INTEGER|' + 'COMMIT|ROLLBACK|SAVEPOINT|GRANT|REVOKE|CREATE|REPLACE|DROP|ALTER' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P--.*?$|/\*.*?\*/)|' + r'(?P\'(\\.|\'\'|[^\'])*\'|.[^-/\']*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def keywords_regex(cls) -> re.Pattern: + """Case-insensitive keyword matcher. + + :rtype: re.Pattern + """ + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b', re.IGNORECASE) diff --git a/src/PyReprism/languages/powershell.py b/src/PyReprism/languages/powershell.py index e69de29..2ddfe36 100644 --- a/src/PyReprism/languages/powershell.py +++ b/src/PyReprism/languages/powershell.py @@ -0,0 +1,33 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class PowerShell(BaseLanguage): + """PowerShell support (``#`` line and ``<# #>`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.powershell + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'begin|break|catch|class|continue|data|define|do|dynamicparam|else|elseif|end|' + 'exit|filter|finally|for|foreach|from|function|if|in|param|process|return|switch|' + 'throw|trap|try|until|using|var|while|workflow' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P<#.*?#>|#.*?$)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^#<\'"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/processing.py b/src/PyReprism/languages/processing.py index e69de29..8a1c5bc 100644 --- a/src/PyReprism/languages/processing.py +++ b/src/PyReprism/languages/processing.py @@ -0,0 +1,40 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Processing(BaseLanguage): + """Processing language support (Java-like ``//`` and ``/* */`` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.processing + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'if|else|for|while|do|switch|case|default|break|continue|return|new|' + 'class|extends|implements|interface|public|private|protected|static|final|' + 'abstract|void|boolean|byte|char|short|int|long|float|double|color|' + 'String|true|false|null|this|super|import|try|catch|finally|throw|throws|' + 'setup|draw|size|background|fill|stroke|noFill|noStroke' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^/\'"]*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile(r'\b0x[\da-fA-F]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?[fFdDlL]?') diff --git a/src/PyReprism/languages/prolog.py b/src/PyReprism/languages/prolog.py index 0bcbb75..5ded7de 100644 --- a/src/PyReprism/languages/prolog.py +++ b/src/PyReprism/languages/prolog.py @@ -1,49 +1,31 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Prolog: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.prolog - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P%.*?$|/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^%\n/*]*[^\n]*)', re.MULTILINE | re.DOTALL) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Prolog(BaseLanguage): + """Prolog support (``%`` line and ``/* */`` block comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Prolog.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Prolog.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.prolog - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Prolog.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'is|mod|rem|xor|div|true|false|fail|halt|assert|asserta|assertz|retract|' + 'findall|bagof|setof|forall' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P%.*?$|/\*[\s\S]*?\*/)|(?P.[^%/]*)', + re.MULTILINE | re.DOTALL, + ) diff --git a/src/PyReprism/languages/properties.py b/src/PyReprism/languages/properties.py index e69de29..ed2b6b3 100644 --- a/src/PyReprism/languages/properties.py +++ b/src/PyReprism/languages/properties.py @@ -0,0 +1,28 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Properties(BaseLanguage): + """.properties files support (``#`` and ``!`` line comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.properties + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P[#!].*?$)|(?P.[^#!]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/protobuf.py b/src/PyReprism/languages/protobuf.py index e69de29..cfea12f 100644 --- a/src/PyReprism/languages/protobuf.py +++ b/src/PyReprism/languages/protobuf.py @@ -0,0 +1,40 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Protobuf(BaseLanguage): + """Protocol Buffers (.proto) support (C-style ``//`` and ``/* */`` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.protobuf + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'syntax|package|import|public|option|message|enum|service|rpc|returns|' + 'oneof|map|reserved|extend|extensions|group|to|max|weak|' + 'required|optional|repeated|default|' + 'double|float|int32|int64|uint32|uint64|sint32|sint64|fixed32|fixed64|' + 'sfixed32|sfixed64|bool|string|bytes|true|false' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^/\'"]*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def number_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile(r'\b0x[\da-fA-F]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?') diff --git a/src/PyReprism/languages/pug.py b/src/PyReprism/languages/pug.py index e69de29..19a7a46 100644 --- a/src/PyReprism/languages/pug.py +++ b/src/PyReprism/languages/pug.py @@ -0,0 +1,31 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Pug(BaseLanguage): + """Pug (Jade) template support (``//`` and ``//-`` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.pug + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'if|else|unless|case|when|default|each|while|for|in|block|extends|include|' + 'append|prepend|mixin|append|doctype|yield' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//-?.*?$)|(?P.[^/]*|/)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/puppet.py b/src/PyReprism/languages/puppet.py index e69de29..0cd2ac0 100644 --- a/src/PyReprism/languages/puppet.py +++ b/src/PyReprism/languages/puppet.py @@ -0,0 +1,33 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Puppet(BaseLanguage): + """Puppet manifest support (``#`` line and ``/* */`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.puppet + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'class|define|node|inherits|if|elsif|else|case|default|and|or|in|import|' + 'include|require|contain|create_resources|unless|type|attr|function|' + 'true|false|undef' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P#.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^#/\'"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/pure.py b/src/PyReprism/languages/pure.py index e69de29..80ed389 100644 --- a/src/PyReprism/languages/pure.py +++ b/src/PyReprism/languages/pure.py @@ -0,0 +1,33 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Pure(BaseLanguage): + """Pure language support (``//`` line and ``/* */`` block comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.pure + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'ans|break|bt|case|const|def|else|end|extern|false|force|if|infix|infixl|infixr|' + 'interface|let|namespace|nonfix|of|otherwise|outfix|postfix|prefix|private|public|' + 'quote|then|true|type|using|when|with' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*.*?\*/|#!.*?$)|' + r'(?P"(\\.|[^\\"])*"|.[^/#"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/python.py b/src/PyReprism/languages/python.py index 7f3403a..862cea1 100644 --- a/src/PyReprism/languages/python.py +++ b/src/PyReprism/languages/python.py @@ -38,7 +38,7 @@ def comment_regex(cls) -> re.Pattern: :return: A compiled regex pattern with named groups to match single-line comments, multiline comments, and non-comment code elements. :rtype: re.Pattern """ - return re.compile(r"(?P#.*?$)|(?P\"\"\".*?\"\"\")|(?P''' .*?''')|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^#'\"]*)", re.DOTALL | re.MULTILINE) + return re.compile(r"(?P#.*?$)|(?P\"\"\".*?\"\"\")|(?P'''.*?''')|(?P'(\\.|[^\\'])*'|\"(\\.|[^\\\"])*\"|.[^#'\"]*)", re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls) -> re.Pattern: diff --git a/src/PyReprism/languages/reason.py b/src/PyReprism/languages/reason.py index e69de29..55a0195 100644 --- a/src/PyReprism/languages/reason.py +++ b/src/PyReprism/languages/reason.py @@ -0,0 +1,34 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Reason(BaseLanguage): + """ReasonML support (``/* */`` block and ``//`` line comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.reason + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'and|as|assert|begin|class|constraint|else|end|exception|external|for|fun|function|' + 'functor|if|in|include|inherit|initializer|lazy|let|method|module|mutable|new|' + 'nonrec|object|of|open|or|pri|pub|rec|switch|then|to|try|type|val|virtual|when|' + 'while|with|true|false' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|.[^/"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/renpy.py b/src/PyReprism/languages/renpy.py index e69de29..c9b043e 100644 --- a/src/PyReprism/languages/renpy.py +++ b/src/PyReprism/languages/renpy.py @@ -0,0 +1,33 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class RenPy(BaseLanguage): + """Ren'Py script support (``#`` line comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.renpy + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'label|menu|scene|show|hide|with|play|stop|queue|pause|jump|call|return|' + 'define|default|image|transform|screen|python|init|if|elif|else|while|for|in|' + 'pass|at|as|expression|onlayer|zorder|behind|nvl|window|voice|text|True|False|None' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P#.*?$)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^#\'"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/rest.py b/src/PyReprism/languages/rest.py index e69de29..fe20bf4 100644 --- a/src/PyReprism/languages/rest.py +++ b/src/PyReprism/languages/rest.py @@ -0,0 +1,32 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Rest(BaseLanguage): + """reStructuredText support (``..`` explicit-markup comment lines).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.rest + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return [] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """A reST comment is a ``..`` block not followed by a directive/target. + + :rtype: re.Pattern + """ + return re.compile( + r'(?P^\.\.[ \t]+(?![\w.-]+::|_|\[|\|)[^\n]*$)|' + r'(?P.[^\n]*|\n)', + re.MULTILINE, + ) diff --git a/src/PyReprism/languages/rip.py b/src/PyReprism/languages/rip.py index e69de29..62a458b 100644 --- a/src/PyReprism/languages/rip.py +++ b/src/PyReprism/languages/rip.py @@ -0,0 +1,32 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Rip(BaseLanguage): + """Rip language support (``#`` line comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.rip + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'case|catch|class|else|exit|finally|if|switch|try|throw|' + 'true|false|nil|return' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P#.*?$)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^#\'"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/roboconf.py b/src/PyReprism/languages/roboconf.py index e69de29..b9084a9 100644 --- a/src/PyReprism/languages/roboconf.py +++ b/src/PyReprism/languages/roboconf.py @@ -0,0 +1,31 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Roboconf(BaseLanguage): + """Roboconf graph/instances support (``#`` line comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.roboconf + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'import|facet|instance of|installer|children|exports|imports|name|count|' + 'extends|external' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P#.*?$)|(?P.[^#]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/sas.py b/src/PyReprism/languages/sas.py index e69de29..5487f37 100644 --- a/src/PyReprism/languages/sas.py +++ b/src/PyReprism/languages/sas.py @@ -0,0 +1,45 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class SAS(BaseLanguage): + """SAS support (``/* */`` block comments). + + Note: SAS also has ``*...;`` statement comments, but those are ambiguous with + the multiplication operator and are intentionally not stripped here. + """ + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.sas + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'data|run|proc|quit|set|merge|by|where|if|then|else|do|end|output|keep|drop|' + 'rename|retain|array|format|informat|label|length|input|infile|put|file|' + 'libname|filename|options|title|footnote|and|or|not|in|eq|ne|gt|lt|ge|le' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P/\*.*?\*/)|' + r'(?P\'(\\.|\'\'|[^\'])*\'|"(\\.|""|[^"])*"|.[^/\'"]*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def keywords_regex(cls) -> re.Pattern: + """Case-insensitive keyword matcher. + + :rtype: re.Pattern + """ + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b', re.IGNORECASE) diff --git a/src/PyReprism/languages/sass.py b/src/PyReprism/languages/sass.py index e69de29..db1d8a7 100644 --- a/src/PyReprism/languages/sass.py +++ b/src/PyReprism/languages/sass.py @@ -0,0 +1,33 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Sass(BaseLanguage): + """Indented Sass support (``//`` silent and ``/* */`` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.sass + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + '@import|@mixin|@include|@extend|@function|@return|@if|@else|@each|@for|@while|' + '@media|@content|@use|@forward|@at-root|@debug|@warn|@error|!default|!important|' + 'from|through|to|in|and|or|not' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^/\'"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/scss.py b/src/PyReprism/languages/scss.py index 2882479..200b026 100644 --- a/src/PyReprism/languages/scss.py +++ b/src/PyReprism/languages/scss.py @@ -1,50 +1,31 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Scss: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.scss - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P/\*[\s\S]*?\*/|/\*.*?$|^.*?\*/)|(?P[^/*]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Scss(BaseLanguage): + """SCSS support (``//`` and ``/* */`` comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Scss.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Scss.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - data = ''.join(result) - return data + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.scss - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Scss.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + '@import|@mixin|@include|@extend|@function|@return|@if|@else|@each|@for|@while|' + '@media|@content|@use|@forward|@at-root|@debug|@warn|@error|!default|!important' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P.[^/]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/smarty.py b/src/PyReprism/languages/smarty.py index e69de29..efb0c84 100644 --- a/src/PyReprism/languages/smarty.py +++ b/src/PyReprism/languages/smarty.py @@ -0,0 +1,32 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Smarty(BaseLanguage): + """Smarty template support (``{* ... *}`` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.smarty + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'if|elseif|else|foreach|foreachelse|for|while|section|sectionelse|' + 'literal|capture|include|assign|block|function|nocache|ldelim|rdelim|' + 'true|false|null|and|or|not|eq|neq|gt|lt|gte|lte' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P\{\*.*?\*\})|(?P.[^{]*|\{)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/soy.py b/src/PyReprism/languages/soy.py index e69de29..2ba6f1a 100644 --- a/src/PyReprism/languages/soy.py +++ b/src/PyReprism/languages/soy.py @@ -0,0 +1,33 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class Soy(BaseLanguage): + """Closure Templates (Soy) support (``//`` and ``/* */`` comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.soy + + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'namespace|template|param|call|delcall|deltemplate|delpackage|' + 'if|elseif|else|switch|case|default|foreach|ifempty|for|let|literal|' + 'msg|fallbackmsg|print|css|xid|as|and|or|not|in|nil|true|false' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P//.*?$|/\*.*?\*/)|' + r'(?P"(\\.|[^\\"])*"|\'(\\.|[^\\\'])*\'|.[^/\'"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/sql.py b/src/PyReprism/languages/sql.py new file mode 100644 index 0000000..2f9e2c9 --- /dev/null +++ b/src/PyReprism/languages/sql.py @@ -0,0 +1,144 @@ +import re +from PyReprism.utils import extension + +from .base import BaseLanguage +from .registry import LanguageRegistry + + +@LanguageRegistry.register +class SQL(BaseLanguage): + """ + SQL language support approximating the Prism.js SQL grammar: + - C/line comments: /* */ , -- , // , # + - Variables: @name, @'quoted', @"quoted", @`quoted` + - Strings: single/double with escapes and doubled quotes + - Functions (followed by '(') + - Extensive keyword list (case-insensitive) + - Booleans/NULL, numbers, operators, punctuation + """ + + # --------------------------- + # Meta + # --------------------------- + @classmethod + def file_extension(cls) -> str: + return extension.sql + + # --------------------------- + # Keyword / Function sets + # --------------------------- + @classmethod + def functions(cls) -> list: + # Prism "function" group (only highlighted when followed by '(') + return "AVG|COUNT|FIRST|FORMAT|LAST|LCASE|LEN|MAX|MID|MIN|MOD|NOW|ROUND|SUM|UCASE".split("|") + + @classmethod + def keywords(cls) -> list: + """ + Return a list of SQL keywords. + + :rtype: list[str] + """ + # Copied/normalized from the Prism keyword list + kw = ( + "ACTION|ADD|AFTER|ALGORITHM|ALL|ALTER|ANALYZE|ANY|APPLY|AS|ASC|AUTHORIZATION|AUTO_INCREMENT|" + "BACKUP|BDB|BEGIN|BERKELEYDB|BIGINT|BINARY|BIT|BLOB|BOOL|BOOLEAN|BREAK|BROWSE|BTREE|BULK|BY|" + "CALL|CASCADE|CASCADED|CASE|CHAIN|CHAR|CHARACTER|CHARSET|CHECK|CHECKPOINT|CLOSE|CLUSTERED|" + "COALESCE|COLLATE|COLUMN|COLUMNS|COMMENT|COMMIT|COMMITTED|COMPUTE|CONNECT|CONSISTENT|" + "CONSTRAINT|CONTAINS|CONTAINSTABLE|CONTINUE|CONVERT|CREATE|CROSS|CURRENT|CURRENT_DATE|" + "CURRENT_TIME|CURRENT_TIMESTAMP|CURRENT_USER|CURSOR|CYCLE|DATA|DATABASE|DATABASES|DATETIME|" + "DAY|DBCC|DEALLOCATE|DEC|DECIMAL|DECLARE|DEFAULT|DEFINER|DELAYED|DELETE|DELIMITER|DELIMITERS|" + "DENY|DESC|DESCRIBE|DETERMINISTIC|DISABLE|DISCARD|DISK|DISTINCT|DISTINCTROW|DISTRIBUTED|DO|" + "DOUBLE|DROP|DUMMY|DUMP|DUMPFILE|DUPLICATE|ELSE|ELSEIF|ENABLE|ENCLOSED|END|ENGINE|ENUM|ERRLVL|" + "ERRORS|ESCAPE|ESCAPED|EXCEPT|EXEC|EXECUTE|EXISTS|EXIT|EXPLAIN|EXTENDED|FETCH|FIELDS|FILE|" + "FILLFACTOR|FIRST|FIXED|FLOAT|FOLLOWING|FOR|FOR EACH ROW|FORCE|FOREIGN|FREETEXT|FREETEXTTABLE|" + "FROM|FULL|FUNCTION|GEOMETRY|GEOMETRYCOLLECTION|GLOBAL|GOTO|GRANT|GROUP|HANDLER|HASH|HAVING|" + "HOLDLOCK|HOUR|IDENTITY|IDENTITY_INSERT|IDENTITYCOL|IF|IGNORE|IMPORT|INDEX|INFILE|INNER|INNODB|" + "INOUT|INSERT|INT|INTEGER|INTERSECT|INTERVAL|INTO|INVOKER|ISOLATION|ITERATE|JOIN|KEY|KEYS|KILL|" + "LANGUAGE|LAST|LEAVE|LEFT|LEVEL|LIMIT|LINENO|LINES|LINESTRING|LOAD|LOCAL|LOCK|LONGBLOB|LONGTEXT|" + "LOOP|MATCH|MATCHED|MEDIUMBLOB|MEDIUMINT|MEDIUMTEXT|MERGE|MIDDLEINT|MINUTE|MODE|MODIFIES|MODIFY|" + "MONTH|MULTILINESTRING|MULTIPOINT|MULTIPOLYGON|NATIONAL|NATURAL|NCHAR|NEXT|NO|NONCLUSTERED|NULLIF|" + "NUMERIC|OF|OFF|OFFSET|OFFSETS|ON|OPEN|OPENDATASOURCE|OPENQUERY|OPENROWSET|OPTIMIZE|OPTION|" + "OPTIONALLY|ORDER|OUT|OUTER|OUTFILE|OVER|PARTIAL|PARTITION|PERCENT|PIVOT|PLAN|POINT|POLYGON|" + "PRECEDING|PRECISION|PREPARE|PREV|PRIMARY|PRINT|PRIVILEGES|PROC|PROCEDURE|PUBLIC|PURGE|QUICK|" + "RAISERROR|READ|READS|REAL|RECONFIGURE|REFERENCES|RELEASE|RENAME|REPEAT|REPEATABLE|REPLACE|" + "REPLICATION|REQUIRE|RESIGNAL|RESTORE|RESTRICT|RETURN|RETURNS|REVOKE|RIGHT|ROLLBACK|ROUTINE|ROW|" + "ROWCOUNT|ROWGUIDCOL|ROWS|RTREE|RULE|SAVE|SAVEPOINT|SCHEMA|SECOND|SELECT|SERIAL|SERIALIZABLE|" + "SESSION|SESSION_USER|SET|SETUSER|SHARE|SHOW|SHUTDOWN|SIMPLE|SMALLINT|SNAPSHOT|SOME|SONAME|SQL|" + "START|STARTING|STATISTICS|STATUS|STRIPED|SYSTEM_USER|TABLE|TABLES|TABLESPACE|TEMP|TEMPORARY|" + "TEMPTABLE|TERMINATED|TEXT|TEXTSIZE|THEN|TIME|TIMESTAMP|TINYBLOB|TINYINT|TINYTEXT|TOP|TRAN|" + "TRANSACTION|TRANSACTIONS|TRIGGER|TRUNCATE|TSEQUAL|TYPE|TYPES|UNBOUNDED|UNCOMMITTED|UNDEFINED|" + "UNION|UNIQUE|UNLOCK|UNPIVOT|UNSIGNED|UPDATE|UPDATETEXT|USAGE|USE|USER|USING|VALUE|VALUES|" + "VARBINARY|VARCHAR|VARCHARACTER|VARYING|VIEW|WAITFOR|WARNINGS|WHEN|WHERE|WHILE|WITH|WITHIN|" + "WITH ROLLUP|WORK|WRITETEXT|YEAR" + ) + return kw.split("|") + + @classmethod + def boolean_literals(cls) -> list: + return ["TRUE", "FALSE", "NULL"] + + # --------------------------- + # Regexes (compiled) + # --------------------------- + @classmethod + def comment_regex(cls): + # Accepts /* ... */, --..., //..., #... (line-based) + return re.compile( + r"(?P/\*[\s\S]*?\*/|--.*?$|//.*?$|#.*?$)" + r"|(?P\"(\\.|\"\"|[^\"\\])*\"|'(\\.|''|[^'\\])*'|`(\\.|[^`\\])*`|.[^/\-#\"]*)", + re.DOTALL | re.MULTILINE + ) + + @classmethod + def variable_regex(cls): + # @'quoted' | @"quoted" | @`quoted` | @name (allows dots and $ per Prism) + return re.compile(r'@(["\'`])(?:\\.|(?!\1)[^\\])+\1|@[\w.$]+') + + @classmethod + def string_regex(cls): + # Single or double quotes; supports escapes and doubled quotes + return re.compile(r'"(?:\\.|""|[^"\\])*"|\'(?:\\.|\'\'|[^\'\\])*\'', re.DOTALL) + + @classmethod + def function_regex(cls): + # Functions only when followed by '(' + return re.compile(r"\b(" + "|".join(cls.functions()) + r")\b(?=\s*\()", re.IGNORECASE) + + @classmethod + def keywords_regex(cls): + # Big keyword set (case-insensitive) + return re.compile(r"\b(" + "|".join(map(re.escape, cls.keywords())) + r")\b", re.IGNORECASE) + + @classmethod + def boolean_regex(cls): + return re.compile(r"\b(" + "|".join(cls.boolean_literals()) + r")\b", re.IGNORECASE) + + @classmethod + def number_regex(cls): + # Hex, ints, floats, leading dot floats; optional scientific notation + return re.compile(r"\b0x[\da-fA-F]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:[Ee]-?\d+)?") + + @classmethod + def operator_regex(cls): + # Symbol operators + word operators like AND/OR/LIKE/BETWEEN/... + word_ops = r"\b(?:AND|BETWEEN|IN|LIKE|NOT|OR|IS|DIV|REGEXP|RLIKE|SOUNDS LIKE|XOR)\b" + sym_ops = r"[-+*/=%^~]|&&?|\|\|?|!=?|<(?:=>?|<|>)?|>[>=]?" + return re.compile(f"(?:{word_ops})|(?:{sym_ops})", re.IGNORECASE) + + @classmethod + def punctuation_regex(cls): + # Matches Prism's punctuation set: ; [ ] ( ) ` , . + return re.compile(r"[;\[\]\(\)`,.]") + + # --------------------------- + # Helpers + # --------------------------- + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + # Defer to BaseLanguage’s implementation if it expects (comment|noncomment) groups. + return super().remove_comments(source_code, isList) + + @classmethod + def remove_keywords(cls, source: str): + return re.sub(cls.keywords_regex(), "", source) diff --git a/src/PyReprism/languages/swift.py b/src/PyReprism/languages/swift.py index 307e389..40b68cb 100644 --- a/src/PyReprism/languages/swift.py +++ b/src/PyReprism/languages/swift.py @@ -19,7 +19,7 @@ def keywords(cls) -> list: @classmethod def comment_regex(cls) -> re.Pattern: - return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P[^/\n]*[^\n]*)', re.DOTALL | re.MULTILINE) + return re.compile(r'(?P//.*?$|/\*[\s\S]*?\*/)|(?P"(\\.|[^\\"])*"|.[^/"]*)', re.DOTALL | re.MULTILINE) @classmethod def number_regex(cls) -> re.Pattern: diff --git a/src/PyReprism/languages/vbnet.py b/src/PyReprism/languages/vbnet.py index 50db130..841ebd3 100644 --- a/src/PyReprism/languages/vbnet.py +++ b/src/PyReprism/languages/vbnet.py @@ -1,74 +1,69 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Vbnet: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.vbnet - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex() -> re.Pattern: - pattern = re.compile(r'(?P\'.*?$)|(?P[^\'\n]*[^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex() -> re.Pattern: - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Vbnet(BaseLanguage): + """VB.NET support (``'`` line comments).""" - @staticmethod - def operator_regex() -> re.Pattern: - pattern = re.compile(r'') - return pattern + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.vbnet - @staticmethod - def keywords_regex() -> re.Pattern: - return re.compile(r'\b(' + '|'.join(Vbnet.keywords()) + r')\b') + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'AddHandler|AddressOf|Alias|And|AndAlso|As|Boolean|ByRef|Byte|ByVal|Call|Case|Catch|' + 'Class|Const|Continue|Date|Decimal|Declare|Default|Delegate|Dim|DirectCast|Do|Double|' + 'Each|Else|ElseIf|End|EndIf|Enum|Erase|Error|Event|Exit|Finally|For|Friend|Function|' + 'Get|GetType|GoTo|Handles|If|Implements|Imports|In|Inherits|Integer|Interface|Is|' + 'Let|Lib|Like|Long|Loop|Me|Mod|Module|MustInherit|MustOverride|MyBase|MyClass|' + 'Namespace|New|Next|Not|Nothing|Object|Of|On|Operator|Option|Optional|Or|OrElse|' + 'Overloads|Overridable|Overrides|ParamArray|Partial|Private|Property|Protected|' + 'Public|RaiseEvent|ReadOnly|ReDim|RemoveHandler|Resume|Return|Select|Set|Shadows|' + 'Shared|Short|Single|Static|Step|Stop|String|Structure|Sub|SyncLock|Then|Throw|To|' + 'Try|TypeOf|Until|Variant|Wend|When|While|With|WithEvents|WriteOnly|Xor|True|False' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P\'.*?$)|(?P.[^\']*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def delimiters_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile(r'[()\[\]{}.,:;<>&]') - @staticmethod - def delimiters_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify VB.NET language delimiters. + @classmethod + def boolean_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile(r'\b(?:True|False|Nothing)\b', re.IGNORECASE) - This function generates a regular expression that matches VB.NET language delimiters, which include parentheses `()`, brackets `[]`, braces `{}`, commas `,`, colons `:`, periods `.`, semicolons `;`, and angle brackets `<`, `>`, as well as the ampersand `&`. + @classmethod + def keywords_regex(cls) -> re.Pattern: + """Case-insensitive keyword matcher (VB.NET is case-insensitive). - :return: A compiled regex pattern to match VB.NET delimiters. :rtype: re.Pattern """ - return re.compile(r'[()\[\]{}.,:;<>&]') - - @staticmethod - def boolean_regex() -> re.Pattern: - """ - Compile and return a regular expression pattern to identify VB.NET boolean literals. + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b', re.IGNORECASE) - This function generates a regular expression that matches the VB.NET boolean literals `True`, `False`, and the special constant `Nothing`. + @classmethod + def remove_comments(cls, source_code: str, isList: bool = False): + """Strip VB.NET comments. - :return: A compiled regex pattern to match VB.NET boolean literals and `Nothing`. - :rtype: re.Pattern + :param source_code: The VB.NET source to process. + :param isList: If True, return a list of non-comment segments. + :rtype: str or list[str] """ - return re.compile(r'\b(?:True|False|Nothing)\b', re.IGNORECASE) - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - return Vbnet.comment_regex().sub(lambda match: match.group('noncomment') if match.group('noncomment') else '', source_code).strip() - result = [] - for match in Vbnet.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str) -> str: - return re.sub(re.compile(Vbnet.keywords_regex()), '', source) + return super().remove_comments(source_code, isList=True) + return super().remove_comments(source_code).strip() diff --git a/src/PyReprism/languages/velocity.py b/src/PyReprism/languages/velocity.py index a4605b9..95ed177 100644 --- a/src/PyReprism/languages/velocity.py +++ b/src/PyReprism/languages/velocity.py @@ -1,49 +1,31 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Velocity: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.velocity - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P##.*?$|#\*[\s\S]*?\*#|#\*.*?$|^.*?\*#)|(?P[^#]*[^\n]*)', re.DOTALL | re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Velocity(BaseLanguage): + """Apache Velocity support (``##`` line and ``#* *#`` block comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Velocity.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Velocity.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.velocity - @staticmethod - def remove_keywords(source: str): - return re.sub(re.compile(Velocity.keywords_regex()), '', source) + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + '#set|#if|#elseif|#else|#end|#foreach|#include|#parse|#macro|#break|#stop|' + '#evaluate|#define' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P##.*?$|#\*[\s\S]*?\*#)|(?P.[^#]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/vim.py b/src/PyReprism/languages/vim.py index 90c2f71..1c71a48 100644 --- a/src/PyReprism/languages/vim.py +++ b/src/PyReprism/languages/vim.py @@ -1,50 +1,32 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Vim: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.vim - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P".*?$)|(?P[^"\n]*[^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Vim(BaseLanguage): + """Vim script support (``"`` line comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Vim.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Vim.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.vim - @staticmethod - def remove_keywords(source: str): - pattern = re.sub(re.compile(Vim.keywords_regex()), '', source) - return pattern + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'function|endfunction|if|elseif|else|endif|while|endwhile|for|endfor|in|return|' + 'let|unlet|call|execute|echo|echom|set|setlocal|autocmd|augroup|command|' + 'try|catch|finally|endtry|throw|break|continue' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P".*?$)|(?P.[^"]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/languages/visual_basic.py b/src/PyReprism/languages/visual_basic.py index 9b082fc..7fd13f4 100644 --- a/src/PyReprism/languages/visual_basic.py +++ b/src/PyReprism/languages/visual_basic.py @@ -1,50 +1,41 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class VisualBasic: - def __init__(): - pass - @staticmethod - def file_extension() -> str: +@LanguageRegistry.register +class VisualBasic(BaseLanguage): + """Visual Basic support (``'`` line comments).""" + + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" return extension.visual_basic - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P\'.*?$|REM.*?$)|(?P[^\'\nR]*[^\n]*)', re.MULTILINE | re.IGNORECASE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(VisualBasic.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in VisualBasic.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) - - @staticmethod - def remove_keywords(source: str): - pattern = re.sub(re.compile(VisualBasic.keywords_regex()), '', source) - return pattern + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ( + 'Dim|Const|As|If|Then|Else|ElseIf|End|Select|Case|For|Each|Next|To|Step|While|Wend|' + 'Do|Loop|Until|Function|Sub|Return|Exit|Call|Class|Module|Property|Get|Set|New|' + 'Nothing|ByVal|ByRef|Optional|Public|Private|Protected|Friend|Shared|Static|' + 'And|Or|Not|Xor|Mod|Is|True|False' + ).split('|') + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P\'.*?$)|(?P.[^\']*)', + re.DOTALL | re.MULTILINE, + ) + + @classmethod + def keywords_regex(cls) -> re.Pattern: + """Case-insensitive keyword matcher (VB is case-insensitive). + + :rtype: re.Pattern + """ + return re.compile(r'\b(' + '|'.join(cls.keywords()) + r')\b', re.IGNORECASE) diff --git a/src/PyReprism/languages/yaml.py b/src/PyReprism/languages/yaml.py index 8f61a1a..f6df99a 100644 --- a/src/PyReprism/languages/yaml.py +++ b/src/PyReprism/languages/yaml.py @@ -1,50 +1,28 @@ import re from PyReprism.utils import extension +from .base import BaseLanguage +from .registry import LanguageRegistry -class Yaml: - def __init__(): - pass - @staticmethod - def file_extension() -> str: - return extension.yaml - - @staticmethod - def keywords() -> list: - keyword = ''.split('|') - return keyword - - @staticmethod - def comment_regex(): - pattern = re.compile(r'(?P#.*?$)|(?P[^#\n]*[^\n]*)', re.MULTILINE) - return pattern - - @staticmethod - def number_regex(): - pattern = re.compile(r'') - return pattern +@LanguageRegistry.register +class Yaml(BaseLanguage): + """YAML support (``#`` line comments).""" - @staticmethod - def operator_regex(): - pattern = re.compile(r'') - return pattern - - @staticmethod - def keywords_regex(): - return re.compile(r'\b(' + '|'.join(Yaml.keywords()) + r')\b') - - @staticmethod - def remove_comments(source_code: str, isList: bool = False) -> str: - result = [] - for match in Yaml.comment_regex().finditer(source_code): - if match.group('noncomment'): - result.append(match.group('noncomment')) - if isList: - return result - return ''.join(result) + @classmethod + def file_extension(cls) -> str: + """:rtype: str""" + return extension.yaml - @staticmethod - def remove_keywords(source: str): - pattern = re.sub(re.compile(Yaml.keywords_regex()), '', source) - return pattern + @classmethod + def keywords(cls) -> list: + """:rtype: list[str]""" + return ['true', 'false', 'null', 'yes', 'no', 'on', 'off'] + + @classmethod + def comment_regex(cls) -> re.Pattern: + """:rtype: re.Pattern""" + return re.compile( + r'(?P#.*?$)|(?P.[^#]*)', + re.DOTALL | re.MULTILINE, + ) diff --git a/src/PyReprism/utils/helpers.py b/src/PyReprism/py.typed similarity index 100% rename from src/PyReprism/utils/helpers.py rename to src/PyReprism/py.typed diff --git a/src/PyReprism/utils/extension.py b/src/PyReprism/utils/extension.py index dd7ec5a..682afbf 100644 --- a/src/PyReprism/utils/extension.py +++ b/src/PyReprism/utils/extension.py @@ -128,6 +128,7 @@ smalltalk: str = ".st" smarty: str = ".tpl" soy: str = ".soy" +sql: str = ".sql" stylus: str = ".styl" swift: str = ".swift" tcl: str = ".tcl" diff --git a/src/PyReprism/utils/normalizer.py b/src/PyReprism/utils/normalizer.py index 19ad300..5857328 100644 --- a/src/PyReprism/utils/normalizer.py +++ b/src/PyReprism/utils/normalizer.py @@ -2,8 +2,7 @@ class Normalizer: - def __init__(): - pass + """Static helpers for normalizing whitespace in source code.""" @staticmethod def whitespaces_regex() -> re.Pattern: diff --git a/src/tests/languages/test_new_languages.py b/src/tests/languages/test_new_languages.py new file mode 100644 index 0000000..6b6a88c --- /dev/null +++ b/src/tests/languages/test_new_languages.py @@ -0,0 +1,98 @@ +"""Behavioral tests for newly implemented and migrated languages. + +Each case asserts that comment markers are removed while the surrounding code +is preserved. Comparisons are whitespace-insensitive to stay robust across the +slightly different comment-stripping strategies used per language. +""" +import pytest + +from PyReprism.languages.registry import LanguageRegistry +# Ensure the modules under test are imported and registered. +from PyReprism.languages import ( # noqa: F401 + html, json, yaml, pascal, prolog, protobuf, powershell, puppet, sql, sass, + smarty, properties, opencl, makefile, markdown, vbnet, visual_basic, +) + + +def _norm(text): + return ' '.join(text.split()) + + +# (registry name, source, substrings that must remain, substrings that must be gone) +CASES = [ + ('HTML', '

hi

', ['

hi

'], ['secret']), + ('MarkDown', '# Title\n\ntext', ['Title', 'text'], ['note']), + ('Json', '{"a": 1} // trailing\n/* block */', ['"a": 1'], ['trailing', 'block']), + ('Yaml', 'key: value # inline comment', ['key: value'], ['inline comment']), + ('Protobuf', 'message M {} // c\n/* b */', ['message M {}'], ['// c', 'b ']), + ('OpenCL', '__kernel void f() {} // gpu\n/* blk */', ['void f()'], ['gpu', 'blk']), + ('Puppet', 'class x { } # note\n/* blk */', ['class x'], ['note', 'blk']), + ('PowerShell', '$x = 1 # note\n<# block #>', ['$x = 1'], ['note', 'block']), + ('SQL', 'SELECT 1 -- c\n/* b */ # h', ['SELECT 1'], ['-- c', 'b ', 'h']), + ('PLSQL', 'BEGIN NULL; END; -- c\n/* b */', ['BEGIN', 'END'], ['-- c', 'b ']), + ('Pascal', 'x := 1; // c\n(* blk *)\n{ brace }', ['x := 1;'], ['// c', 'blk', 'brace']), + ('Prolog', 'foo :- bar. % note\n/* blk */', ['foo :- bar.'], ['note', 'blk']), + ('Smarty', 'Hello {* hidden *} World', ['Hello', 'World'], ['hidden']), + ('Properties', 'a=b\n# commented\n! also', ['a=b'], ['commented', 'also']), + ('MakeFile', 'all:\n\techo hi # note', ['echo hi'], ['note']), + ('Vbnet', "Dim x = 1 ' note", ['Dim x = 1'], ['note']), + ('VisualBasic', "x = 1 ' note", ['x = 1'], ['note']), + ('Sass', '.a { color: red } // note\n/* blk */', ['color: red'], ['note', 'blk']), +] + + +# Regression cases for languages whose comment-stripping was previously broken +# (line comments not stripped, or block comments eating surrounding code). +FIXED_REGRESSIONS = [ + ('Aspnet', 'A\n<%-- x --%>\nB', ['A', 'B'], ['x']), + ('Aspnet', 'A\n\nB', ['A', 'B'], ['x']), + ('Autoit', 'a = 1 ; note\nb = 2', ['a = 1', 'b = 2'], ['note']), + ('Batch', 'echo hi\nREM note\n:: also', ['echo hi'], ['note', 'also']), + ('Basic', "x = 1 ' note\nPRINT x", ['x = 1', 'PRINT x'], ['note']), + ('Latex', r'\a{x} % note' + '\nB', ['x', 'B'], ['note']), + ('LiveScript', 'a = 1 # note\nb = 2', ['a = 1', 'b = 2'], ['note']), + ('LiveScript', 'A\n/* note */\nB', ['A', 'B'], ['note']), + ('NIM', 'var x = 1 # note\necho x', ['var x = 1', 'echo x'], ['note']), + ('NIM', 'A\n#[ note ]#\nB', ['A', 'B'], ['note']), + ('ObjectiveC', 'int x; // note\n/*b*/\nreturn x;', ['int x;', 'return x;'], ['note', 'b ']), + ('Swift', 'let x = 1 // note\n/*b*/\nprint(x)', ['let x = 1', 'print(x)'], ['note', 'b ']), + ('Less', '.a { color: red } // note\n/* blk */', ['color: red'], ['note', 'blk']), + ('Clike', 'int x;\n/* note */\nreturn x;', ['int x;', 'return x;'], ['note']), + ('Go', 'var x = 1\n/* note */\nreturn x', ['var x = 1', 'return x'], ['note']), + ('Nginx', 'server 1;\nlisten 80; # note', ['server 1;', 'listen 80;'], ['note']), + ('Ocaml', 'let a = 1\n(* note *)\nlet b = 2', ['let a = 1', 'let b = 2'], ['note']), +] + + +@pytest.mark.parametrize('name, source, keep, gone', CASES + FIXED_REGRESSIONS) +def test_remove_comments(name, source, keep, gone): + cls = LanguageRegistry.get(name) + assert cls is not None, f"{name} is not registered" + out = cls.remove_comments(source) + normalized = _norm(out) + for fragment in keep: + assert _norm(fragment) in normalized, f"{name}: expected to keep {fragment!r} in {out!r}" + for fragment in gone: + assert fragment not in out, f"{name}: expected to strip {fragment!r} from {out!r}" + + +# Cases migrated from the former root-level _test_migrated_languages.py +# (verified against the pre-existing language implementations). +MIGRATED = [ + ('MatLab', 'a = 1; % comment\nb = 2; % another', 'a = 1; b = 2;', 'if x = 1; end', 'x = 1;'), + ('CPP', 'int x; // cmt\n/*b*/\nreturn x;', 'int x; return x;', 'if (x) return x; else x = 1;', '(x) x; x = 1;'), + ('C', 'int x; // c\n/*b*/\nreturn 0;', 'int x; return 0;', 'if (x) return x;', '(x) x;'), + ('Go', 'var x = 1 // c\n/*b*/\nreturn x', 'var x = 1 return x', 'if true { return }', '{ }'), + ('Python', 'x = 1 # c\n"""doc"""\nprint(x)', 'x = 1 print(x)', 'if True: return None', ':'), + ('Ruby', "x = 1 # c\n=begin\nblock\n=end\nputs x", 'puts x', 'if true then return end', ''), + ('Bash', "x=1 # c\necho $x", 'x=1 echo $x', None, None), +] + + +@pytest.mark.parametrize('name, src, expect_no_comments, kw_src, kw_expected', MIGRATED) +def test_migrated_languages(name, src, expect_no_comments, kw_src, kw_expected): + cls = LanguageRegistry.get(name) + assert cls is not None, f"{name} is not registered" + assert _norm(cls.remove_comments(src)) == _norm(expect_no_comments) + if kw_src is not None: + assert _norm(cls.remove_keywords(kw_src)) == _norm(kw_expected) diff --git a/src/tests/test_registry.py b/src/tests/test_registry.py new file mode 100644 index 0000000..a93c3be --- /dev/null +++ b/src/tests/test_registry.py @@ -0,0 +1,64 @@ +"""Registry-wide smoke tests. + +These guard every registered language cheaply: each must expose a file +extension and be able to strip comments and keywords without raising. This +prevents regressions such as empty/broken language stubs or classes that fail +to register (which would make them unreachable through the public API). +""" +import pytest + +from PyReprism.languages import _load_all_languages +from PyReprism.languages.base import BaseLanguage +from PyReprism.languages.registry import LanguageRegistry + +# Import every language module so the registry is fully populated. +_load_all_languages() + +ALL_LANGUAGES = sorted(LanguageRegistry.all().items()) + +SAMPLE = ( + 'value = 1 // c-line\n' + '# hash comment\n' + '-- dash comment\n' + '/* block comment */\n' + "name = 'text' \"more\"\n" + '\n' + '{* smarty comment *}\n' +) + + +def test_registry_is_populated(): + assert len(ALL_LANGUAGES) > 100 + + +@pytest.mark.parametrize('name, cls', ALL_LANGUAGES) +class TestEveryLanguage: + def test_is_base_language_subclass(self, name, cls): + assert issubclass(cls, BaseLanguage) + + def test_has_file_extension(self, name, cls): + ext = cls.file_extension() + assert isinstance(ext, str) + assert ext != '' + + def test_keywords_is_list(self, name, cls): + assert isinstance(cls.keywords(), list) + + def test_remove_comments_returns_str(self, name, cls): + assert isinstance(cls.remove_comments(SAMPLE), str) + + def test_remove_comments_islist_returns_list(self, name, cls): + assert isinstance(cls.remove_comments(SAMPLE, isList=True), list) + + def test_remove_comments_is_idempotent(self, name, cls): + once = cls.remove_comments(SAMPLE) + assert cls.remove_comments(once) == once + + def test_remove_keywords_returns_str(self, name, cls): + assert isinstance(cls.remove_keywords('if while for return class def end'), str) + + def test_public_api_import(self, name, cls): + import PyReprism.languages as languages + # Names that survive a round-trip through lower() are reachable lazily. + if name.lower() == cls.__module__.rsplit('.', 1)[-1]: + assert getattr(languages, name) is cls From 664c0acc1ab8f8d332897b6330fc02791ce4fbe4 Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 6 Jul 2026 21:41:36 -0700 Subject: [PATCH 09/19] ft: add pre-commit-hooks --- .pre-commit-hooks.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .pre-commit-hooks.yaml diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 0000000..a4e36c8 --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,27 @@ +# PyReprism pre-commit hooks. +# +# Add to your .pre-commit-config.yaml, e.g.: +# +# - repo: https://github.com/unlv-evol/PyReprism +# rev: v0.0.4 +# hooks: +# - id: pyreprism-strip-comments +# files: ^generated/ # scope it! these hooks rewrite files +# +# NOTE: these hooks modify source in place. Scope them with `files:`/`exclude:` +# to directories where stripping comments/whitespace is intended (e.g. generated +# or vendored code), not your whole repository. + +- id: pyreprism-strip-comments + name: PyReprism – strip comments + description: Remove comments in place (language auto-detected from file extension). + entry: pyreprism remove comments --in-place + language: python + types: [text] + +- id: pyreprism-normalize-whitespace + name: PyReprism – normalize whitespace + description: Collapse insignificant whitespace in place. + entry: pyreprism preprocess --steps whitespace --in-place + language: python + types: [text] From 9478d8a1a5e9a6de870e26f7c8b21b60c893b88b Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 6 Jul 2026 21:43:55 -0700 Subject: [PATCH 10/19] update: deployment --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a229ab5..07c7bb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ docs = [ "sphinx_rtd_theme", ] +[project.scripts] +pyreprism = "PyReprism.cli:main" + [project.urls] Homepage = "https://github.com/unlv-evol/PyReprism" Documentation = "https://pyreprism.readthedocs.io" From dcab80ffea687ce209ed31a4e5890bbe5b180dc9 Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 6 Jul 2026 21:45:55 -0700 Subject: [PATCH 11/19] ft: add lightweight tokenizer and cli --- src/PyReprism/__init__.py | 213 ++++++++++++++++++++- src/PyReprism/cli.py | 259 +++++++++++++++++++++++++ src/PyReprism/languages/base.py | 327 ++++++++++++++++++++++++++++++-- src/PyReprism/tokens.py | 37 ++++ 4 files changed, 821 insertions(+), 15 deletions(-) create mode 100644 src/PyReprism/cli.py create mode 100644 src/PyReprism/tokens.py diff --git a/src/PyReprism/__init__.py b/src/PyReprism/__init__.py index b86c6c6..dc363f1 100644 --- a/src/PyReprism/__init__.py +++ b/src/PyReprism/__init__.py @@ -1,3 +1,214 @@ -"""PyRePrism package initializer.""" +"""PyReprism: a framework for source-code preprocessing. + +High-level convenience API:: + + import PyReprism as pr + + pr.remove_comments(src, lang="python") + pr.extract_comments(src, lang="python") + pr.count_comments(src, lang="python") + pr.tokenize(src, lang="python") + pr.preprocess(src, lang="java", steps=["comments", "strings", "whitespace"]) + +``lang`` may be a language name (``"python"``), a file extension (``".py"``), or a +language class. Use :func:`detect_language` to infer it from a filename. +""" +import importlib +import os +from typing import List, Optional, Sequence, Type, Union + +from .metrics import CodeStats +from .tokens import Token, TokenType +from .utils.normalizer import Normalizer __version__ = "0.0.4" + +LanguageLike = Union[str, "type"] + + +def get_language(lang: LanguageLike) -> Type: + """Resolve ``lang`` to a registered language class. + + Accepts a language class, a registry name (``"Python"``/``"python"``), or a + file extension (``".py"``). Raises :class:`ValueError` if it cannot be resolved. + """ + from .languages import _load_all_languages, get_language_by_extension + from .languages.base import BaseLanguage + from .languages.registry import LanguageRegistry + + if isinstance(lang, type) and issubclass(lang, BaseLanguage): + return lang + if isinstance(lang, str): + name = lang.strip() + cls = LanguageRegistry.get(name) + if cls: + return cls + if name.startswith('.') or '.' in os.path.basename(name): + cls = get_language_by_extension(name if name.startswith('.') else os.path.splitext(name)[1]) + if cls: + return cls + try: + mod = importlib.import_module(f'.languages.{name.lower()}', __name__) + for value in vars(mod).values(): + if (isinstance(value, type) and issubclass(value, BaseLanguage) + and value.__module__ == mod.__name__): + return value + except ModuleNotFoundError: + pass + _load_all_languages() + cls = (LanguageRegistry.get(name) or LanguageRegistry.get(name.capitalize()) + or LanguageRegistry.get(name.upper())) + if cls: + return cls + raise ValueError(f"Unknown language: {lang!r}") + + +def detect_language(filename: Optional[str] = None, source: Optional[str] = None) -> Optional[Type]: + """Infer a language class from ``filename`` (by extension or basename). + + ``source``-based detection is not yet implemented and is accepted for + forward compatibility. Returns ``None`` when no language matches. + """ + from .languages import get_language_by_extension + + if filename: + base = os.path.basename(filename) + _, ext = os.path.splitext(base) + return get_language_by_extension(ext or base) + return None + + +# --------------------------------------------------------------------- helpers +def _resolve(lang: LanguageLike) -> Type: + return get_language(lang) + + +def remove_comments(source: str, lang: LanguageLike, isList: bool = False): + """Remove comments from ``source`` for the given language.""" + return _resolve(lang).remove_comments(source, isList=isList) + + +def extract_comments(source: str, lang: LanguageLike) -> List[str]: + return _resolve(lang).extract_comments(source) + + +def count_comments(source: str, lang: LanguageLike) -> int: + return _resolve(lang).count_comments(source) + + +def match_comments(source: str, lang: LanguageLike) -> List[Token]: + return _resolve(lang).match_comments(source) + + +def remove_keywords(source: str, lang: LanguageLike) -> str: + return _resolve(lang).remove_keywords(source) + + +def extract_keywords(source: str, lang: LanguageLike) -> List[str]: + return _resolve(lang).extract_keywords(source) + + +def count_keywords(source: str, lang: LanguageLike) -> int: + return _resolve(lang).count_keywords(source) + + +def remove_numbers(source: str, lang: LanguageLike) -> str: + return _resolve(lang).remove_numbers(source) + + +def extract_numbers(source: str, lang: LanguageLike) -> List[str]: + return _resolve(lang).extract_numbers(source) + + +def count_numbers(source: str, lang: LanguageLike) -> int: + return _resolve(lang).count_numbers(source) + + +def remove_operators(source: str, lang: LanguageLike) -> str: + return _resolve(lang).remove_operators(source) + + +def extract_operators(source: str, lang: LanguageLike) -> List[str]: + return _resolve(lang).extract_operators(source) + + +def remove_strings(source: str, lang: LanguageLike) -> str: + return _resolve(lang).remove_strings(source) + + +def extract_strings(source: str, lang: LanguageLike) -> List[str]: + return _resolve(lang).extract_strings(source) + + +def extract_identifiers(source: str, lang: LanguageLike) -> List[str]: + return _resolve(lang).extract_identifiers(source) + + +def remove_whitespaces(source: str) -> str: + """Collapse insignificant whitespace (language-independent).""" + return Normalizer.remove_whitespaces(source) + + +def tokenize(source: str, lang: LanguageLike) -> List[Token]: + """Return the flat list of typed tokens for ``source``.""" + return _resolve(lang).tokenize(source) + + +def blank_comments(source: str, lang: LanguageLike, replacement: str = ' ') -> str: + """Remove comment content while preserving line numbers.""" + return _resolve(lang).blank_comments(source, replacement=replacement) + + +def stats(source: str, lang: LanguageLike) -> CodeStats: + """Return line- and token-level :class:`CodeStats` for ``source``.""" + return _resolve(lang).stats(source) + + +def normalize(source: str, lang: LanguageLike, **options) -> str: + """Return a canonicalized form of ``source`` for ML / clone detection. + + See :meth:`PyReprism.languages.base.BaseLanguage.normalize` for options. + """ + return _resolve(lang).normalize(source, **options) + + +_STEPS = { + 'comments': lambda cls, s: cls.remove_comments(s), + 'strings': lambda cls, s: cls.remove_strings(s), + 'numbers': lambda cls, s: cls.remove_numbers(s), + 'operators': lambda cls, s: cls.remove_operators(s), + 'keywords': lambda cls, s: cls.remove_keywords(s), + 'whitespace': lambda cls, s: Normalizer.remove_whitespaces(s), +} + + +def preprocess(source: str, lang: LanguageLike, steps: Sequence[str] = ('comments',)) -> str: + """Apply a sequence of removal ``steps`` in order and return the result. + + Valid steps: ``comments``, ``strings``, ``numbers``, ``operators``, + ``keywords``, ``whitespace``. + """ + cls = _resolve(lang) + out = source + for step in steps: + fn = _STEPS.get(step) + if fn is None: + raise ValueError(f"Unknown preprocessing step: {step!r}. " + f"Valid steps: {', '.join(sorted(_STEPS))}") + out = fn(cls, out) + return out + + +__all__ = [ + '__version__', + 'Token', 'TokenType', 'CodeStats', 'Normalizer', + 'get_language', 'detect_language', + 'remove_comments', 'extract_comments', 'count_comments', 'match_comments', + 'remove_keywords', 'extract_keywords', 'count_keywords', + 'remove_numbers', 'extract_numbers', 'count_numbers', + 'remove_operators', 'extract_operators', + 'remove_strings', 'extract_strings', + 'extract_identifiers', 'remove_whitespaces', + 'blank_comments', 'stats', 'normalize', + 'tokenize', 'preprocess', +] diff --git a/src/PyReprism/cli.py b/src/PyReprism/cli.py new file mode 100644 index 0000000..88d6a92 --- /dev/null +++ b/src/PyReprism/cli.py @@ -0,0 +1,259 @@ +"""Command-line interface for PyReprism. + +Usage examples:: + + pyreprism remove comments file.py + cat file.go | pyreprism remove comments --lang go + pyreprism extract comments src/**/*.py + pyreprism count comments --lang python file.py + pyreprism preprocess --steps comments,strings,whitespace file.java + pyreprism tokenize --json file.py + pyreprism languages +""" +import argparse +import glob +import json +import sys +from typing import List, Optional + +from . import __version__, detect_language, get_language, preprocess + +CONSTRUCTS = ['comments', 'keywords', 'numbers', 'operators', 'strings', 'identifiers'] + + +def _iter_inputs(paths: List[str]): + """Yield ``(label, text, filename)`` for each input; filename is None for stdin.""" + if not paths: + yield ('', sys.stdin.read(), None) + return + for pattern in paths: + matches = glob.glob(pattern, recursive=True) if any(c in pattern for c in '*?[') else [pattern] + if not matches: + print(f"pyreprism: no such file: {pattern}", file=sys.stderr) + continue + for filepath in matches: + try: + with open(filepath, 'r', encoding='utf-8') as handle: + yield (filepath, handle.read(), filepath) + except OSError as exc: + print(f"pyreprism: {exc}", file=sys.stderr) + + +def _resolve(lang: Optional[str], filename: Optional[str]): + if lang: + return get_language(lang) + if filename: + cls = detect_language(filename=filename) + if cls: + return cls + print("pyreprism: could not determine language; pass --lang", file=sys.stderr) + raise SystemExit(2) + + +def _method(cls, action: str, construct: str): + fn = getattr(cls, f'{action}_{construct}', None) + if fn is None: + raise SystemExit(f"pyreprism: '{action} {construct}' is not supported") + return fn + + +def _cmd_remove(args) -> int: + for label, text, filename in _iter_inputs(args.paths): + cls = _resolve(args.lang, filename) + result = _method(cls, 'remove', args.construct)(text) + _emit(result, filename, args.in_place, args) + return 0 + + +def _cmd_preprocess(args) -> int: + steps = [s.strip() for s in args.steps.split(',') if s.strip()] + for label, text, filename in _iter_inputs(args.paths): + cls = _resolve(args.lang, filename) + result = preprocess(text, lang=cls, steps=steps) + _emit(result, filename, args.in_place, args) + return 0 + + +def _cmd_extract(args) -> int: + multiple = len(args.paths) > 1 + for label, text, filename in _iter_inputs(args.paths): + cls = _resolve(args.lang, filename) + items = _method(cls, 'extract', args.construct)(text) + if args.json: + print(json.dumps({label: items} if multiple else items)) + else: + if multiple: + print(f"==> {label} <==") + for item in items: + print(item) + return 0 + + +def _cmd_count(args) -> int: + for label, text, filename in _iter_inputs(args.paths): + cls = _resolve(args.lang, filename) + count = _method(cls, 'count', args.construct)(text) + print(f"{count}\t{label}" if len(args.paths) > 1 else count) + return 0 + + +def _cmd_tokenize(args) -> int: + for label, text, filename in _iter_inputs(args.paths): + cls = _resolve(args.lang, filename) + tokens = cls.tokenize(text) + if args.json: + print(json.dumps([ + {'type': t.type.value, 'value': t.value, + 'start': t.start, 'end': t.end, 'line': t.line} + for t in tokens + ])) + else: + for t in tokens: + print(f"{t.line}\t{t.type.value}\t{t.value!r}") + return 0 + + +def _cmd_stats(args) -> int: + multiple = len(args.paths) > 1 + for label, text, filename in _iter_inputs(args.paths): + cls = _resolve(args.lang, filename) + data = cls.stats(text).as_dict() + if args.json: + print(json.dumps({label: data} if multiple else data)) + else: + if multiple: + print(f"==> {label} <==") + width = max(len(k) for k in data) + for key, value in data.items(): + print(f"{key.ljust(width)} {value}") + return 0 + + +def _cmd_normalize(args) -> int: + options = dict( + drop_comments=not args.keep_comments, + mask_numbers=not args.keep_numbers, + mask_strings=not args.keep_strings, + rename_identifiers=not args.keep_names, + collapse_whitespace=args.collapse_whitespace, + ) + for label, text, filename in _iter_inputs(args.paths): + cls = _resolve(args.lang, filename) + result = cls.normalize(text, **options) + _emit(result, filename, args.in_place, args) + return 0 + + +def _cmd_languages(args) -> int: + from .languages import _load_all_languages + from .languages.registry import LanguageRegistry + + _load_all_languages() + rows = [] + for name, cls in sorted(LanguageRegistry.all().items()): + try: + ext = cls.file_extension() + except Exception: + ext = '?' + rows.append((name, ext)) + width = max((len(n) for n, _ in rows), default=0) + for name, ext in rows: + print(f"{name.ljust(width)} {ext}") + print(f"\n{len(rows)} languages", file=sys.stderr) + return 0 + + +def _emit(result: str, filename: Optional[str], in_place: bool, args) -> None: + if in_place and filename: + with open(filename, 'w', encoding='utf-8') as handle: + handle.write(result) + else: + sys.stdout.write(result) + if not result.endswith('\n'): + sys.stdout.write('\n') + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog='pyreprism', + description='Preprocess source code: strip/extract/count comments, strings, ' + 'numbers, operators, keywords and identifiers across 145+ languages.', + ) + parser.add_argument('--version', action='version', version=f'pyreprism {__version__}') + sub = parser.add_subparsers(dest='command', required=True) + + def add_common(p, lang=True): + p.add_argument('paths', nargs='*', help='files or globs (default: read stdin)') + if lang: + p.add_argument('-l', '--lang', + help='language name, extension, or class (auto-detected from ' + 'filename when omitted)') + + p_remove = sub.add_parser('remove', help='remove a construct from the source') + p_remove.add_argument('construct', choices=[c for c in CONSTRUCTS if c != 'identifiers']) + add_common(p_remove) + p_remove.add_argument('-i', '--in-place', action='store_true', + help='rewrite files in place instead of printing to stdout') + p_remove.set_defaults(func=_cmd_remove) + + p_pre = sub.add_parser('preprocess', help='apply a sequence of removal steps') + p_pre.add_argument('-s', '--steps', default='comments', + help='comma-separated steps: comments,strings,numbers,operators,' + 'keywords,whitespace') + add_common(p_pre) + p_pre.add_argument('-i', '--in-place', action='store_true') + p_pre.set_defaults(func=_cmd_preprocess) + + p_extract = sub.add_parser('extract', help='extract occurrences of a construct') + p_extract.add_argument('construct', choices=CONSTRUCTS) + add_common(p_extract) + p_extract.add_argument('--json', action='store_true', help='emit JSON') + p_extract.set_defaults(func=_cmd_extract) + + p_count = sub.add_parser('count', help='count occurrences of a construct') + p_count.add_argument('construct', choices=CONSTRUCTS) + add_common(p_count) + p_count.set_defaults(func=_cmd_count) + + p_tok = sub.add_parser('tokenize', help='emit the typed token stream') + add_common(p_tok) + p_tok.add_argument('--json', action='store_true', help='emit JSON') + p_tok.set_defaults(func=_cmd_tokenize) + + p_stats = sub.add_parser('stats', help='report line/token metrics') + add_common(p_stats) + p_stats.add_argument('--json', action='store_true', help='emit JSON') + p_stats.set_defaults(func=_cmd_stats) + + p_norm = sub.add_parser('normalize', + help='canonicalize code (rename identifiers, mask literals) for ML') + add_common(p_norm) + p_norm.add_argument('--keep-comments', action='store_true') + p_norm.add_argument('--keep-numbers', action='store_true') + p_norm.add_argument('--keep-strings', action='store_true') + p_norm.add_argument('--keep-names', action='store_true', + help='do not rename identifiers') + p_norm.add_argument('--collapse-whitespace', action='store_true') + p_norm.add_argument('-i', '--in-place', action='store_true') + p_norm.set_defaults(func=_cmd_normalize) + + p_langs = sub.add_parser('languages', help='list supported languages and extensions') + p_langs.set_defaults(func=_cmd_languages) + + return parser + + +def main(argv: Optional[List[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(sys.argv[1:] if argv is None else list(argv)) + try: + return args.func(args) + except SystemExit: + raise + except ValueError as exc: + print(f"pyreprism: {exc}", file=sys.stderr) + return 2 + + +if __name__ == '__main__': # pragma: no cover + sys.exit(main()) diff --git a/src/PyReprism/languages/base.py b/src/PyReprism/languages/base.py index 2e69982..53ed67b 100644 --- a/src/PyReprism/languages/base.py +++ b/src/PyReprism/languages/base.py @@ -1,18 +1,39 @@ +import bisect import re from functools import lru_cache from typing import List, Pattern, Union +from ..metrics import CodeStats +from ..tokens import Token, TokenType + +# Shared, language-agnostic patterns used by the generic tokenizer. +_WHITESPACE_RE = re.compile(r'\s+') +_OPERATOR_RE = re.compile(r'[-+*/%=<>!&|^~?]+') + + +def _newline_offsets(source: str) -> List[int]: + return [i for i, ch in enumerate(source) if ch == '\n'] + + +def _line_at(newlines: List[int], index: int) -> int: + """Return the 1-based line number of ``index`` given precomputed newline offsets.""" + return bisect.bisect_right(newlines, index) + 1 + class BaseLanguage: - """Minimal base class for language implementations. + """Base class for language implementations. - Subclasses should provide: file_extension(), keywords(), comment_regex(), - and may override number_regex(), operator_regex(), keywords_regex(). + Subclasses must provide :meth:`file_extension` and :meth:`comment_regex`, and + typically :meth:`keywords`. They may override :meth:`number_regex`, + :meth:`operator_regex`, :meth:`string_regex`, :meth:`identifier_regex`, and + :meth:`keywords_regex`. - This base centralizes implementations for remove_comments and remove_keywords - and caches commonly used compiled regexes. + This base centralizes the ``remove``/``extract``/``count``/``match`` operations + for comments, keywords, numbers, operators, strings and identifiers, plus a + generic :meth:`tokenize`, so every language inherits the full API for free. """ + # ------------------------------------------------------------------ regexes @classmethod def file_extension(cls) -> str: raise NotImplementedError() @@ -24,15 +45,15 @@ def keywords(cls) -> List[str]: @classmethod @lru_cache(maxsize=None) def keywords_regex(cls) -> Pattern: - words = cls.keywords() or [] + words = [w for w in (cls.keywords() or []) if w] if not words: # match nothing - return re.compile(r"^$") + return re.compile(r"(?!x)x") return re.compile(r'\b(' + '|'.join(words) + r')\b', re.IGNORECASE) @classmethod def comment_regex(cls) -> Pattern: - """Return a compiled regex that contains named groups 'comment' and 'noncomment'.""" + """Return a compiled regex with named groups 'comment' and 'noncomment'.""" raise NotImplementedError() @classmethod @@ -41,7 +62,49 @@ def number_regex(cls) -> Pattern: @classmethod def operator_regex(cls) -> Pattern: - return re.compile(r'.') + return re.compile(r'[-+*/%=<>!&|^~?]+') + + @classmethod + def string_regex(cls) -> Pattern: + """Default string-literal matcher: single- or double-quoted with escapes.""" + return re.compile(r'"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'') + + @classmethod + def identifier_regex(cls) -> Pattern: + return re.compile(r'[A-Za-z_]\w*') + + # ---------------------------------------------------------------- internals + @classmethod + def _match_spans(cls, regex: Pattern, source: str, ttype: TokenType) -> List[Token]: + """Return non-empty matches of ``regex`` in ``source`` as :class:`Token`s.""" + newlines = _newline_offsets(source) + tokens = [] + for m in regex.finditer(source): + if m.end() > m.start(): # skip zero-width matches + tokens.append(Token(ttype, m.group(), m.start(), m.end(), + _line_at(newlines, m.start()))) + return tokens + + # ----------------------------------------------------------------- comments + @classmethod + def match_comments(cls, source: str) -> List[Token]: + """Return every comment in ``source`` as a :class:`Token` with position info.""" + newlines = _newline_offsets(source) + tokens = [] + for m in cls.comment_regex().finditer(source): + # A match is a comment when it did NOT bind the 'noncomment' group. + if m.groupdict().get('noncomment') is None and m.end() > m.start(): + tokens.append(Token(TokenType.COMMENT, m.group(), m.start(), m.end(), + _line_at(newlines, m.start()))) + return tokens + + @classmethod + def extract_comments(cls, source: str) -> List[str]: + return [t.value for t in cls.match_comments(source)] + + @classmethod + def count_comments(cls, source: str) -> int: + return len(cls.match_comments(source)) @classmethod def remove_comments(cls, source_code: str, isList: bool = False) -> Union[str, List[str]]: @@ -50,19 +113,255 @@ def remove_comments(cls, source_code: str, isList: bool = False) -> Union[str, L The regex is expected to provide a 'noncomment' named group for text to keep. Returns either the joined string or a list of non-comment segments when isList=True. """ - pattern = cls.comment_regex() result = [] - for match in pattern.finditer(source_code): + for match in cls.comment_regex().finditer(source_code): non = match.groupdict().get('noncomment') - # Append the 'noncomment' group even when it's an empty string. - # Use `is not None` to avoid skipping valid empty segments which - # may carry whitespace that should be preserved. if non is not None: result.append(non) if isList: return result return ''.join(result) + # ----------------------------------------------------------------- keywords + @classmethod + def match_keywords(cls, source: str) -> List[Token]: + if not [w for w in (cls.keywords() or []) if w]: + return [] + return cls._match_spans(cls.keywords_regex(), source, TokenType.KEYWORD) + + @classmethod + def extract_keywords(cls, source: str) -> List[str]: + return [t.value for t in cls.match_keywords(source)] + + @classmethod + def count_keywords(cls, source: str) -> int: + return len(cls.match_keywords(source)) + @classmethod def remove_keywords(cls, source: str) -> str: return re.sub(cls.keywords_regex(), '', source) + + # ------------------------------------------------------------------ numbers + @classmethod + def match_numbers(cls, source: str) -> List[Token]: + return cls._match_spans(cls.number_regex(), source, TokenType.NUMBER) + + @classmethod + def extract_numbers(cls, source: str) -> List[str]: + return [t.value for t in cls.match_numbers(source)] + + @classmethod + def count_numbers(cls, source: str) -> int: + return len(cls.match_numbers(source)) + + @classmethod + def remove_numbers(cls, source: str) -> str: + return re.sub(cls.number_regex(), '', source) + + # ---------------------------------------------------------------- operators + @classmethod + def match_operators(cls, source: str) -> List[Token]: + return cls._match_spans(cls.operator_regex(), source, TokenType.OPERATOR) + + @classmethod + def extract_operators(cls, source: str) -> List[str]: + return [t.value for t in cls.match_operators(source)] + + @classmethod + def count_operators(cls, source: str) -> int: + return len(cls.match_operators(source)) + + @classmethod + def remove_operators(cls, source: str) -> str: + return re.sub(cls.operator_regex(), '', source) + + # ------------------------------------------------------------------ strings + @classmethod + def match_strings(cls, source: str) -> List[Token]: + return cls._match_spans(cls.string_regex(), source, TokenType.STRING) + + @classmethod + def extract_strings(cls, source: str) -> List[str]: + return [t.value for t in cls.match_strings(source)] + + @classmethod + def count_strings(cls, source: str) -> int: + return len(cls.match_strings(source)) + + @classmethod + def remove_strings(cls, source: str) -> str: + return re.sub(cls.string_regex(), '', source) + + # -------------------------------------------------------------- identifiers + @classmethod + def match_identifiers(cls, source: str) -> List[Token]: + """Match identifiers (words that are not language keywords).""" + kws = {w.lower() for w in (cls.keywords() or []) if w} + newlines = _newline_offsets(source) + tokens = [] + for m in cls.identifier_regex().finditer(source): + if m.group().lower() in kws: + continue + tokens.append(Token(TokenType.IDENTIFIER, m.group(), m.start(), m.end(), + _line_at(newlines, m.start()))) + return tokens + + @classmethod + def extract_identifiers(cls, source: str) -> List[str]: + return [t.value for t in cls.match_identifiers(source)] + + @classmethod + def count_identifiers(cls, source: str) -> int: + return len(cls.match_identifiers(source)) + + # ----------------------------------------------------------------- tokenize + @classmethod + def tokenize(cls, source: str) -> List[Token]: + """Split ``source`` into a flat list of typed :class:`Token`s. + + Comments (and strings, where the language's ``comment_regex`` keeps them in + the non-comment group) are separated first; remaining code is classified + into strings, numbers, keywords, identifiers, operators and other + single-character tokens. This is best-effort and driven by the language's + own regexes. + """ + newlines = _newline_offsets(source) + tokens = [] + pos = 0 + for m in cls.comment_regex().finditer(source): + if m.start() > pos: + # Characters the comment regex does not classify (often newlines); + # tokenize them as code so nothing is lost. + cls._tokenize_code(source[pos:m.start()], pos, newlines, tokens) + pos = m.start() + if m.end() <= m.start(): + continue + if m.groupdict().get('noncomment') is None: + tokens.append(Token(TokenType.COMMENT, m.group(), m.start(), m.end(), + _line_at(newlines, m.start()))) + else: + cls._tokenize_code(m.group('noncomment'), m.start(), newlines, tokens) + pos = m.end() + if pos < len(source): + cls._tokenize_code(source[pos:], pos, newlines, tokens) + return tokens + + @classmethod + def _tokenize_code(cls, text: str, base: int, newlines: List[int], + tokens: List[Token]) -> None: + strx = cls.string_regex() + numx = cls.number_regex() + idx = cls.identifier_regex() + kws = {w.lower() for w in (cls.keywords() or []) if w} + stages = ( + (_WHITESPACE_RE, TokenType.WHITESPACE), + (strx, TokenType.STRING), + (numx, TokenType.NUMBER), + (idx, None), # identifier or keyword + (_OPERATOR_RE, TokenType.OPERATOR), + ) + i, n = 0, len(text) + while i < n: + for regex, ttype in stages: + m = regex.match(text, i) + if m and m.end() > i: + val = m.group() + if ttype is None: + tt = TokenType.KEYWORD if val.lower() in kws else TokenType.IDENTIFIER + else: + tt = ttype + start = base + i + tokens.append(Token(tt, val, start, start + len(val), + _line_at(newlines, start))) + i = m.end() + break + else: + start = base + i + tokens.append(Token(TokenType.OTHER, text[i], start, start + 1, + _line_at(newlines, start))) + i += 1 + + # -------------------------------------------------------------- line-preserving + @classmethod + def blank_comments(cls, source: str, replacement: str = ' ') -> str: + """Remove comment *content* while preserving line numbers. + + Each comment is replaced with ``replacement`` for every non-newline + character and its newlines are kept, so downstream line/column mapping + stays intact. Use ``replacement=''`` to blank only the content. + """ + result = [] + pos = 0 + for tok in cls.match_comments(source): + result.append(source[pos:tok.start]) + result.append(''.join('\n' if ch == '\n' else replacement for ch in tok.value)) + pos = tok.end + result.append(source[pos:]) + return ''.join(result) + + # --------------------------------------------------------------------- metrics + @classmethod + def stats(cls, source: str) -> CodeStats: + """Compute line- and token-level :class:`CodeStats` for ``source``.""" + counts = {t: 0 for t in TokenType} + code_lines = set() + comment_lines = set() + for tok in cls.tokenize(source): + counts[tok.type] += 1 + if tok.type is TokenType.WHITESPACE: + continue + span = range(tok.line, tok.line + tok.value.count('\n') + 1) + target = comment_lines if tok.type is TokenType.COMMENT else code_lines + target.update(span) + total = len(source.splitlines()) + code = len(code_lines) + comment = len(comment_lines - code_lines) + blank = max(total - code - comment, 0) + return CodeStats( + lines=total, code_lines=code, comment_lines=comment, blank_lines=blank, + characters=len(source), + comment_tokens=counts[TokenType.COMMENT], + string_tokens=counts[TokenType.STRING], + number_tokens=counts[TokenType.NUMBER], + keyword_tokens=counts[TokenType.KEYWORD], + identifier_tokens=counts[TokenType.IDENTIFIER], + operator_tokens=counts[TokenType.OPERATOR], + ) + + # ------------------------------------------------------------------ normalize + @classmethod + def normalize(cls, source: str, *, rename_identifiers: bool = True, + mask_numbers: bool = True, mask_strings: bool = True, + drop_comments: bool = True, collapse_whitespace: bool = False, + identifier_prefix: str = 'VAR', number_placeholder: str = '0', + string_placeholder: str = '"STR"') -> str: + """Return a canonicalized form of ``source`` for ML / clone detection. + + By default: comments are dropped; string and number literals are replaced + with fixed placeholders; and identifiers are consistently renamed to + ``VAR1``, ``VAR2``, ... (keywords are preserved). Toggle each behavior via + the keyword arguments; set ``collapse_whitespace`` to reduce every + whitespace run to a single space. + """ + mapping = {} + parts = [] + for tok in cls.tokenize(source): + t = tok.type + if t is TokenType.COMMENT: + if not drop_comments: + parts.append(tok.value) + elif t is TokenType.STRING and mask_strings: + parts.append(string_placeholder) + elif t is TokenType.NUMBER and mask_numbers: + parts.append(number_placeholder) + elif t is TokenType.IDENTIFIER and rename_identifiers: + name = mapping.get(tok.value) + if name is None: + name = f"{identifier_prefix}{len(mapping) + 1}" + mapping[tok.value] = name + parts.append(name) + elif t is TokenType.WHITESPACE and collapse_whitespace: + parts.append(' ') + else: + parts.append(tok.value) + return ''.join(parts) diff --git a/src/PyReprism/tokens.py b/src/PyReprism/tokens.py new file mode 100644 index 0000000..7431553 --- /dev/null +++ b/src/PyReprism/tokens.py @@ -0,0 +1,37 @@ +"""Token model shared by the tokenizer and the match/extract APIs.""" +from dataclasses import dataclass +from enum import Enum + + +class TokenType(str, Enum): + """The kind of source-code construct a :class:`Token` represents.""" + + COMMENT = "comment" + STRING = "string" + NUMBER = "number" + KEYWORD = "keyword" + OPERATOR = "operator" + IDENTIFIER = "identifier" + WHITESPACE = "whitespace" + OTHER = "other" + + def __str__(self) -> str: # pragma: no cover - cosmetic + return self.value + + +@dataclass(frozen=True) +class Token: + """A single lexical token located within a source string. + + :param type: the :class:`TokenType` of the token. + :param value: the exact substring the token spans. + :param start: 0-based start offset in the source. + :param end: 0-based end offset (exclusive) in the source. + :param line: 1-based line number of ``start``. + """ + + type: TokenType + value: str + start: int + end: int + line: int From f3df7bcd86ebf67cde0d09210d523d84f82d1806 Mon Sep 17 00:00:00 2001 From: danielogen Date: Mon, 6 Jul 2026 21:47:29 -0700 Subject: [PATCH 12/19] update: track code metrics and update readme --- README.md | 127 ++++++++++++++++++++++++------ src/PyReprism/__main__.py | 7 ++ src/PyReprism/languages/ruby.py | 4 +- src/PyReprism/metrics.py | 41 ++++++++++ src/tests/test_api.py | 133 ++++++++++++++++++++++++++++++++ src/tests/test_cli.py | 106 +++++++++++++++++++++++++ src/tests/test_metrics.py | 127 ++++++++++++++++++++++++++++++ 7 files changed, 520 insertions(+), 25 deletions(-) create mode 100644 src/PyReprism/__main__.py create mode 100644 src/PyReprism/metrics.py create mode 100644 src/tests/test_api.py create mode 100644 src/tests/test_cli.py create mode 100644 src/tests/test_metrics.py diff --git a/README.md b/README.md index d9557c3..e1a7921 100644 --- a/README.md +++ b/README.md @@ -20,53 +20,134 @@ PyReprism is a Python framework that helps researchers and developers the task o pip install PyReprism ``` ## Quick Usage -Use case 1: Removing comments -```python -from PyReprism.languages import Python -# from PyReprism.languages import Java +The simplest entry point is the top-level API. Pass `lang` as a language name +(`"python"`), a file extension (`".py"`), or a language class. + +```python +import PyReprism as pr source = """ # single line comment x = 5 + 6 -''' -multiline -comment -''' -print(x) +print(x) # inline comment """ -source = Python.remove_comments(source) +pr.remove_comments(source, lang="python") # -> code with comments stripped +pr.extract_comments(source, lang="python") # -> ['# single line comment', '# inline comment'] +pr.count_comments(source, lang="python") # -> 2 +``` -# expected output +Every construct supports the same **match / extract / count / remove** verbs: -x = 5 + 6 +```python +code = 'total = price * 42 + tax # compute' +pr.extract_numbers(code, lang="python") # -> ['42'] +pr.extract_keywords(code, lang="python") # -> [] +pr.extract_identifiers(code, lang="python") # -> ['total', 'price', 'tax', 'compute'] +pr.remove_numbers(code, lang="python") # -> 'total = price * + tax # compute' +pr.extract_strings('s = "hi"', lang="python") # -> ['"hi"'] +``` -print(x) +`match_*` returns positioned `Token`s (`type`, `value`, `start`, `end`, `line`): +```python +for tok in pr.match_comments(source, lang="python"): + print(tok.line, tok.value) ``` -Use case 2: Removing whitespaces +### Tokenize + +`tokenize()` returns a lossless, typed token stream (comment/string/number/ +keyword/identifier/operator/whitespace/other): + ```python -from PyReprism.utils.normalizer import Normalizer -source = """ +for tok in pr.tokenize("x = 5 + y // c", lang="clike"): + print(tok.type, repr(tok.value)) +``` -x = 5 + 6 +### Pipelines + +Chain removal steps in one call: +```python +pr.preprocess(source, lang="java", + steps=["comments", "strings", "whitespace"]) +``` -print(x) +Valid steps: `comments`, `strings`, `numbers`, `operators`, `keywords`, `whitespace`. -""" +### Code metrics + +```python +s = pr.stats(source, lang="python") +s.code_lines, s.comment_lines, s.blank_lines +s.comment_to_code_ratio, s.comment_density +s.as_dict() # includes per-token-type counts, ready for JSON / dataframes +``` + +### Normalization for ML / clone detection + +Canonicalize code so that only its structure remains — rename identifiers to +`VAR1, VAR2, …`, mask string/number literals, and drop comments: +```python +pr.normalize("total = price * 42 # cost", lang="python") +# -> 'VAR1 = VAR2 * 0' +``` + +Every part is toggleable (`rename_identifiers`, `mask_numbers`, `mask_strings`, +`drop_comments`, `collapse_whitespace`). To strip comments while keeping line +numbers stable (useful for tools that map back to source): + +```python +pr.blank_comments(source, lang="python") # comment content removed, newlines kept +``` -source = Normalizer.remove_whitespaces(source) +### Detect the language from a filename -# expected output -x=5+6 -print(x) +```python +cls = pr.detect_language(filename="src/main.go") # -> Go language class +``` + +### Language-independent whitespace normalization + +```python +pr.remove_whitespaces("x = 5 + 6\n\n\nprint(x)") # -> 'x=5+6\nprint(x)' ``` +You can also import a language class directly if you prefer: + +```python +from PyReprism.languages import Python +Python.remove_comments(source) +``` + +## Command line + +Installing PyReprism also provides a `pyreprism` command (works on stdin/stdout, +files, and globs; the language is auto-detected from a file's extension): + +```shell +pyreprism remove comments file.py +cat file.go | pyreprism remove comments --lang go +pyreprism extract comments "src/**/*.py" --json +pyreprism count comments --lang python file.py +pyreprism preprocess --steps comments,strings,whitespace file.java +pyreprism tokenize --json file.py +pyreprism stats --json file.py # line/token metrics +pyreprism normalize file.py # canonicalize for ML +pyreprism remove comments --in-place file.py # rewrite in place +pyreprism languages # list supported languages +``` + +Equivalently: `python -m PyReprism ...`. + +It can also be wired into [pre-commit](https://pre-commit.com) via the bundled +`.pre-commit-hooks.yaml` (`pyreprism-strip-comments`, `pyreprism-normalize-whitespace`). +These hooks rewrite files, so scope them with `files:`/`exclude:`. + Read the [docs](https://pyreprism.readthedocs.io) for more usage examples. NB: The beta versions of PyReprism is still unstable, but we are working 24/7 to ensure the tool is usable. diff --git a/src/PyReprism/__main__.py b/src/PyReprism/__main__.py new file mode 100644 index 0000000..f55979e --- /dev/null +++ b/src/PyReprism/__main__.py @@ -0,0 +1,7 @@ +"""Enable ``python -m PyReprism``.""" +import sys + +from .cli import main + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/PyReprism/languages/ruby.py b/src/PyReprism/languages/ruby.py index 2a8866b..e27829c 100644 --- a/src/PyReprism/languages/ruby.py +++ b/src/PyReprism/languages/ruby.py @@ -21,11 +21,11 @@ def comment_regex(cls): @classmethod def number_regex(cls): - return '' + return re.compile(r'\b0x[\da-fA-F]+\b|\b0b[01]+\b|(?:\b\d[\d_]*\.?\d*|\B\.\d+)(?:[eE][+-]?\d+)?') @classmethod def operator_regex(cls): - return '' + return re.compile(r'\*\*=?|<=>|===?|=~|!~|<<|>>|&&|\|\||\.\.\.?|::|[-+*/%=<>!&|^~]=?') @classmethod def remove_comments(cls, source_code: str, isList: bool = False) -> str: diff --git a/src/PyReprism/metrics.py b/src/PyReprism/metrics.py new file mode 100644 index 0000000..3e6fc13 --- /dev/null +++ b/src/PyReprism/metrics.py @@ -0,0 +1,41 @@ +"""Code metrics produced by :meth:`BaseLanguage.stats`.""" +from dataclasses import asdict, dataclass + + +@dataclass(frozen=True) +class CodeStats: + """Line- and token-level metrics for a source string. + + Line counts are mutually exclusive: ``lines == code_lines + comment_lines + + blank_lines``. A line with both code and a trailing comment counts as a code + line. + """ + + lines: int + code_lines: int + comment_lines: int + blank_lines: int + characters: int + comment_tokens: int + string_tokens: int + number_tokens: int + keyword_tokens: int + identifier_tokens: int + operator_tokens: int + + @property + def comment_to_code_ratio(self) -> float: + """Comment lines divided by code lines (0.0 when there is no code).""" + return self.comment_lines / self.code_lines if self.code_lines else 0.0 + + @property + def comment_density(self) -> float: + """Fraction of non-blank lines that are comment-only.""" + non_blank = self.code_lines + self.comment_lines + return self.comment_lines / non_blank if non_blank else 0.0 + + def as_dict(self) -> dict: + data = asdict(self) + data['comment_to_code_ratio'] = round(self.comment_to_code_ratio, 4) + data['comment_density'] = round(self.comment_density, 4) + return data diff --git a/src/tests/test_api.py b/src/tests/test_api.py new file mode 100644 index 0000000..d83fe0f --- /dev/null +++ b/src/tests/test_api.py @@ -0,0 +1,133 @@ +"""Tests for the high-level facade and the extract/count/match/tokenize API.""" +import pytest + +import PyReprism as pr +from PyReprism.languages import _load_all_languages +from PyReprism.languages.registry import LanguageRegistry +from PyReprism.tokens import Token, TokenType + +_load_all_languages() +ALL_LANGUAGES = sorted(LanguageRegistry.all().items()) + +PY = '''# header +def add(a, b): + n = 5 + 6 # inline + s = "text 42" + return a + b # tail +''' + + +# --------------------------------------------------------------- facade routing +def test_get_language_by_name(): + assert pr.get_language('python').__name__ == 'Python' + + +def test_get_language_by_extension(): + assert pr.get_language('.go').__name__ == 'Go' + + +def test_get_language_by_class(): + cls = pr.get_language('python') + assert pr.get_language(cls) is cls + + +def test_get_language_unknown_raises(): + with pytest.raises(ValueError): + pr.get_language('not-a-language') + + +def test_detect_language_from_filename(): + assert pr.detect_language(filename='src/main.go').__name__ == 'Go' + assert pr.detect_language(filename='unknown.zzz') is None + + +# ------------------------------------------------------------------- operations +def test_extract_and_count_comments(): + assert pr.extract_comments(PY, lang='python') == ['# header', '# inline', '# tail'] + assert pr.count_comments(PY, lang='python') == 3 + + +def test_match_comments_positions(): + matches = pr.match_comments(PY, lang='python') + assert all(isinstance(m, Token) and m.type is TokenType.COMMENT for m in matches) + assert matches[0].value == '# header' + assert matches[0].line == 1 + # spans round-trip back to the exact substring + assert PY[matches[0].start:matches[0].end] == '# header' + + +def test_extract_numbers_and_keywords(): + assert pr.extract_numbers(PY, lang='python') == ['5', '6', '42'] + assert pr.extract_keywords(PY, lang='python') == ['def', 'return'] + + +def test_extract_strings(): + assert pr.extract_strings(PY, lang='python') == ['"text 42"'] + + +def test_remove_numbers(): + assert pr.remove_numbers('a = 5 + 60', lang='python') == 'a = + ' + + +def test_preprocess_pipeline(): + out = pr.preprocess(PY, lang='python', steps=['comments', 'whitespace']) + assert '#' not in out + assert 'def' in out + + +def test_preprocess_unknown_step(): + with pytest.raises(ValueError): + pr.preprocess(PY, lang='python', steps=['bogus']) + + +def test_remove_whitespaces_language_independent(): + assert pr.remove_whitespaces('a =\n\n b') == 'a=\nb' + + +# --------------------------------------------------------------------- tokenize +def test_tokenize_basic_classification(): + toks = pr.tokenize('x = 5 + y // c\n', lang='clike') + kinds = {t.type for t in toks} + assert TokenType.IDENTIFIER in kinds + assert TokenType.NUMBER in kinds + assert TokenType.OPERATOR in kinds + assert TokenType.COMMENT in kinds + # every token round-trips to the original source + src = 'x = 5 + y // c\n' + assert ''.join(t.value for t in toks) == src + + +def test_tokenize_keyword_vs_identifier(): + toks = pr.tokenize('def foo(): return 1', lang='python') + by_val = {t.value: t.type for t in toks} + assert by_val['def'] is TokenType.KEYWORD + assert by_val['return'] is TokenType.KEYWORD + assert by_val['foo'] is TokenType.IDENTIFIER + + +# --------------------------------------------------------- registry-wide safety +SAMPLE = 'value = 1 + 2 // c\n# h\nname = "x"\n' + + +@pytest.mark.parametrize('name, cls', ALL_LANGUAGES) +class TestEveryLanguageAPI: + def test_extract_ops_return_lists(self, name, cls): + assert isinstance(cls.extract_comments(SAMPLE), list) + assert isinstance(cls.extract_numbers(SAMPLE), list) + assert isinstance(cls.extract_strings(SAMPLE), list) + assert isinstance(cls.extract_identifiers(SAMPLE), list) + assert isinstance(cls.extract_keywords(SAMPLE), list) + + def test_count_ops_return_ints(self, name, cls): + assert isinstance(cls.count_comments(SAMPLE), int) + assert isinstance(cls.count_numbers(SAMPLE), int) + + def test_remove_ops_return_str(self, name, cls): + assert isinstance(cls.remove_numbers(SAMPLE), str) + assert isinstance(cls.remove_strings(SAMPLE), str) + assert isinstance(cls.remove_operators(SAMPLE), str) + + def test_tokenize_covers_all_input(self, name, cls): + toks = cls.tokenize(SAMPLE) + # tokenizer must be lossless: concatenating token values rebuilds the input + assert ''.join(t.value for t in toks) == SAMPLE diff --git a/src/tests/test_cli.py b/src/tests/test_cli.py new file mode 100644 index 0000000..535d679 --- /dev/null +++ b/src/tests/test_cli.py @@ -0,0 +1,106 @@ +"""Tests for the pyreprism command-line interface.""" +import json + +import pytest + +from PyReprism.cli import main + + +def run(argv, stdin=None, monkeypatch=None, capsys=None): + """Invoke the CLI, returning (exit_code, stdout, stderr).""" + if stdin is not None: + import io + monkeypatch.setattr('sys.stdin', io.StringIO(stdin)) + code = main(argv) + out, err = capsys.readouterr() + return code, out, err + + +def test_remove_comments_stdin(monkeypatch, capsys): + code, out, err = run(['remove', 'comments', '--lang', 'python'], + stdin='x = 1 # c\n', monkeypatch=monkeypatch, capsys=capsys) + assert code == 0 + assert '# c' not in out + assert 'x = 1' in out + + +def test_remove_comments_file_autodetect(tmp_path, monkeypatch, capsys): + f = tmp_path / 'a.py' + f.write_text('y = 2 # note\n') + code, out, err = run(['remove', 'comments', str(f)], monkeypatch=monkeypatch, capsys=capsys) + assert code == 0 + assert 'note' not in out + assert 'y = 2' in out + + +def test_remove_in_place(tmp_path, monkeypatch, capsys): + f = tmp_path / 'b.py' + f.write_text('z = 3 # gone\n') + code, out, err = run(['remove', 'comments', '--in-place', str(f)], + monkeypatch=monkeypatch, capsys=capsys) + assert code == 0 + assert 'gone' not in f.read_text() + assert out == '' # nothing to stdout in in-place mode + + +def test_count_comments(monkeypatch, capsys): + code, out, err = run(['count', 'comments', '--lang', 'python'], + stdin='# a\ncode\n# b\n', monkeypatch=monkeypatch, capsys=capsys) + assert code == 0 + assert out.strip() == '2' + + +def test_extract_comments_json(monkeypatch, capsys): + code, out, err = run(['extract', 'comments', '--lang', 'python', '--json'], + stdin='# a\ncode # b\n', monkeypatch=monkeypatch, capsys=capsys) + assert code == 0 + assert json.loads(out) == ['# a', '# b'] + + +def test_extract_numbers(monkeypatch, capsys): + code, out, err = run(['extract', 'numbers', '--lang', 'python'], + stdin='a = 5 + 42\n', monkeypatch=monkeypatch, capsys=capsys) + assert code == 0 + assert out.split() == ['5', '42'] + + +def test_preprocess_steps(monkeypatch, capsys): + code, out, err = run(['preprocess', '--steps', 'comments,whitespace', '--lang', 'python'], + stdin='# c\nx = 5 + 6\n', monkeypatch=monkeypatch, capsys=capsys) + assert code == 0 + assert out.strip() == 'x=5+6' + + +def test_tokenize_json(monkeypatch, capsys): + code, out, err = run(['tokenize', '--lang', 'clike', '--json'], + stdin='x=5', monkeypatch=monkeypatch, capsys=capsys) + assert code == 0 + types = [t['type'] for t in json.loads(out)] + assert 'identifier' in types and 'number' in types + + +def test_languages_lists_many(monkeypatch, capsys): + code, out, err = run(['languages'], monkeypatch=monkeypatch, capsys=capsys) + assert code == 0 + assert 'Python' in out + assert out.count('\n') > 100 + + +def test_no_language_errors(monkeypatch, capsys): + with pytest.raises(SystemExit) as excinfo: + run(['count', 'comments'], stdin='x\n', monkeypatch=monkeypatch, capsys=capsys) + assert excinfo.value.code == 2 + + +def test_unknown_language_errors(monkeypatch, capsys): + code, out, err = run(['count', 'comments', '--lang', 'nope'], + stdin='x\n', monkeypatch=monkeypatch, capsys=capsys) + assert code == 2 + assert 'Unknown language' in err + + +def test_version(capsys): + with pytest.raises(SystemExit) as excinfo: + main(['--version']) + assert excinfo.value.code == 0 + assert 'pyreprism' in capsys.readouterr().out diff --git a/src/tests/test_metrics.py b/src/tests/test_metrics.py new file mode 100644 index 0000000..f87c111 --- /dev/null +++ b/src/tests/test_metrics.py @@ -0,0 +1,127 @@ +"""Tests for stats(), normalize() and blank_comments().""" +import json + +import pytest + +import PyReprism as pr +from PyReprism.languages import _load_all_languages +from PyReprism.metrics import CodeStats +from PyReprism.cli import main + +_load_all_languages() + +SRC = '''# header +def add(price, qty): + total = price * qty + 42 # inline + label = "cost" + return total + +''' + + +# ------------------------------------------------------------------------ stats +def test_stats_line_classification(): + s = pr.stats(SRC, lang='python') + assert isinstance(s, CodeStats) + assert s.lines == 6 # trailing blank line dropped by splitlines + assert s.code_lines == 4 + assert s.comment_lines == 1 + assert s.blank_lines == 1 + assert s.lines == s.code_lines + s.comment_lines + s.blank_lines + + +def test_stats_token_counts_and_ratios(): + s = pr.stats(SRC, lang='python') + assert s.comment_tokens == 2 + assert s.string_tokens == 1 + assert s.number_tokens == 1 + assert s.keyword_tokens == 2 # def, return + assert s.comment_to_code_ratio == pytest.approx(0.25) + assert 'comment_density' in s.as_dict() + + +def test_stats_block_comment_with_trailing_code_counts_as_code(): + # line 1 has both code and a comment -> counts as a code line, not comment + s = pr.stats('x = 1 # c\n', lang='python') + assert s.code_lines == 1 + assert s.comment_lines == 0 + + +# -------------------------------------------------------------------- normalize +def test_normalize_defaults(): + out = pr.normalize('total = price * 42 # c\n', lang='python') + assert '# c' not in out + assert '42' not in out and '0' in out # number masked + assert 'VAR1' in out and 'VAR2' in out # identifiers renamed + assert 'price' not in out + + +def test_normalize_consistent_identifier_mapping(): + out = pr.normalize('a = b + a', lang='python', mask_numbers=False) + # 'a' -> VAR1 both times, 'b' -> VAR2 + assert out == 'VAR1 = VAR2 + VAR1' + + +def test_normalize_keeps_keywords(): + out = pr.normalize('def foo(): return 1', lang='python') + assert out.startswith('def ') + assert 'return' in out + + +def test_normalize_toggles(): + src = 'x = "hi" + 3' + assert '"hi"' in pr.normalize(src, lang='python', mask_strings=False) + assert '3' in pr.normalize(src, lang='python', mask_numbers=False) + assert 'x' in pr.normalize(src, lang='python', rename_identifiers=False) + + +def test_normalize_collapse_whitespace(): + assert pr.normalize('a = b', lang='python', collapse_whitespace=True) == 'VAR1 = VAR2' + + +# ---------------------------------------------------------------- blank_comments +def test_blank_comments_preserves_line_count(): + src = 'a = 1 // secret\nb = 2\n/* x\n y */\nc = 3\n' + out = pr.blank_comments(src, lang='clike') + assert 'secret' not in out + assert out.count('\n') == src.count('\n') # line numbers stable + assert 'a = 1' in out and 'c = 3' in out + + +def test_blank_comments_empty_replacement(): + out = pr.blank_comments('x = 1 # c\n', lang='python', replacement='') + assert out == 'x = 1 \n' + + +# ---------------------------------------------------------------------- registry +def test_stats_and_normalize_work_for_all_languages(): + from PyReprism.languages.registry import LanguageRegistry + sample = 'a = 1 + 2 // c\n# h\nname = "x"\n' + for name, cls in LanguageRegistry.all().items(): + s = cls.stats(sample) + assert s.lines >= 1 + assert isinstance(cls.normalize(sample), str) + + +# --------------------------------------------------------------------------- CLI +def test_cli_stats_json(monkeypatch, capsys): + import io + monkeypatch.setattr('sys.stdin', io.StringIO('# c\nx = 1\n')) + assert main(['stats', '--lang', 'python', '--json']) == 0 + data = json.loads(capsys.readouterr().out) + assert data['comment_lines'] == 1 and data['code_lines'] == 1 + + +def test_cli_normalize(monkeypatch, capsys): + import io + monkeypatch.setattr('sys.stdin', io.StringIO('total = price * 42\n')) + assert main(['normalize', '--lang', 'python']) == 0 + out = capsys.readouterr().out + assert 'VAR1' in out and '0' in out and 'price' not in out + + +def test_cli_normalize_keep_flags(monkeypatch, capsys): + import io + monkeypatch.setattr('sys.stdin', io.StringIO('total = 42\n')) + assert main(['normalize', '--lang', 'python', '--keep-names', '--keep-numbers']) == 0 + assert capsys.readouterr().out.strip() == 'total = 42' From 9435583149ccfff6c7a294a92ab23311e9a10ae2 Mon Sep 17 00:00:00 2001 From: danielogen Date: Tue, 7 Jul 2026 05:44:07 -0700 Subject: [PATCH 13/19] refactor: extract generic token ops into _tokenops --- src/PyReprism/_tokenops.py | 78 +++++++++++++++++++++++++++++++++ src/PyReprism/languages/base.py | 64 +++++---------------------- 2 files changed, 88 insertions(+), 54 deletions(-) create mode 100644 src/PyReprism/_tokenops.py diff --git a/src/PyReprism/_tokenops.py b/src/PyReprism/_tokenops.py new file mode 100644 index 0000000..ccee1d5 --- /dev/null +++ b/src/PyReprism/_tokenops.py @@ -0,0 +1,78 @@ +"""Generic operations derived from a token stream. + +These work on any ``List[Token]`` regardless of which engine produced it, so the +regex and pygments backends share one implementation of remove/extract/count, +normalization and metrics. +""" +from typing import List + +from .metrics import CodeStats +from .tokens import Token, TokenType + + +def remove(tokens: List[Token], ttype: TokenType) -> str: + return ''.join(t.value for t in tokens if t.type is not ttype) + + +def extract(tokens: List[Token], ttype: TokenType) -> List[str]: + return [t.value for t in tokens if t.type is ttype] + + +def count(tokens: List[Token], ttype: TokenType) -> int: + return sum(1 for t in tokens if t.type is ttype) + + +def normalize(tokens: List[Token], *, rename_identifiers: bool = True, + mask_numbers: bool = True, mask_strings: bool = True, + drop_comments: bool = True, collapse_whitespace: bool = False, + identifier_prefix: str = 'VAR', number_placeholder: str = '0', + string_placeholder: str = '"STR"') -> str: + mapping = {} + parts = [] + for tok in tokens: + t = tok.type + if t is TokenType.COMMENT: + if not drop_comments: + parts.append(tok.value) + elif t is TokenType.STRING and mask_strings: + parts.append(string_placeholder) + elif t is TokenType.NUMBER and mask_numbers: + parts.append(number_placeholder) + elif t is TokenType.IDENTIFIER and rename_identifiers: + name = mapping.get(tok.value) + if name is None: + name = f"{identifier_prefix}{len(mapping) + 1}" + mapping[tok.value] = name + parts.append(name) + elif t is TokenType.WHITESPACE and collapse_whitespace: + parts.append(' ') + else: + parts.append(tok.value) + return ''.join(parts) + + +def stats(tokens: List[Token], source: str) -> CodeStats: + counts = {t: 0 for t in TokenType} + code_lines = set() + comment_lines = set() + for tok in tokens: + counts[tok.type] += 1 + if tok.type is TokenType.WHITESPACE: + continue + span = range(tok.line, tok.line + tok.value.count('\n') + 1) + target = comment_lines if tok.type is TokenType.COMMENT else code_lines + target.update(span) + total = len(source.splitlines()) + code = len(code_lines) + comment = len(comment_lines - code_lines) + blank = max(total - code - comment, 0) + return CodeStats( + lines=total, code_lines=code, comment_lines=comment, blank_lines=blank, + characters=len(source), + comment_tokens=counts[TokenType.COMMENT], + string_tokens=counts[TokenType.STRING], + number_tokens=counts[TokenType.NUMBER], + keyword_tokens=counts[TokenType.KEYWORD], + identifier_tokens=counts[TokenType.IDENTIFIER], + operator_tokens=counts[TokenType.OPERATOR], + ) diff --git a/src/PyReprism/languages/base.py b/src/PyReprism/languages/base.py index 53ed67b..3bdd046 100644 --- a/src/PyReprism/languages/base.py +++ b/src/PyReprism/languages/base.py @@ -303,65 +303,21 @@ def blank_comments(cls, source: str, replacement: str = ' ') -> str: @classmethod def stats(cls, source: str) -> CodeStats: """Compute line- and token-level :class:`CodeStats` for ``source``.""" - counts = {t: 0 for t in TokenType} - code_lines = set() - comment_lines = set() - for tok in cls.tokenize(source): - counts[tok.type] += 1 - if tok.type is TokenType.WHITESPACE: - continue - span = range(tok.line, tok.line + tok.value.count('\n') + 1) - target = comment_lines if tok.type is TokenType.COMMENT else code_lines - target.update(span) - total = len(source.splitlines()) - code = len(code_lines) - comment = len(comment_lines - code_lines) - blank = max(total - code - comment, 0) - return CodeStats( - lines=total, code_lines=code, comment_lines=comment, blank_lines=blank, - characters=len(source), - comment_tokens=counts[TokenType.COMMENT], - string_tokens=counts[TokenType.STRING], - number_tokens=counts[TokenType.NUMBER], - keyword_tokens=counts[TokenType.KEYWORD], - identifier_tokens=counts[TokenType.IDENTIFIER], - operator_tokens=counts[TokenType.OPERATOR], - ) + from .. import _tokenops + return _tokenops.stats(cls.tokenize(source), source) # ------------------------------------------------------------------ normalize @classmethod - def normalize(cls, source: str, *, rename_identifiers: bool = True, - mask_numbers: bool = True, mask_strings: bool = True, - drop_comments: bool = True, collapse_whitespace: bool = False, - identifier_prefix: str = 'VAR', number_placeholder: str = '0', - string_placeholder: str = '"STR"') -> str: + def normalize(cls, source: str, **options) -> str: """Return a canonicalized form of ``source`` for ML / clone detection. By default: comments are dropped; string and number literals are replaced with fixed placeholders; and identifiers are consistently renamed to - ``VAR1``, ``VAR2``, ... (keywords are preserved). Toggle each behavior via - the keyword arguments; set ``collapse_whitespace`` to reduce every - whitespace run to a single space. + ``VAR1``, ``VAR2``, ... (keywords are preserved). Options mirror + :func:`PyReprism._tokenops.normalize` (``rename_identifiers``, + ``mask_numbers``, ``mask_strings``, ``drop_comments``, + ``collapse_whitespace``, ``identifier_prefix``, ``number_placeholder``, + ``string_placeholder``). """ - mapping = {} - parts = [] - for tok in cls.tokenize(source): - t = tok.type - if t is TokenType.COMMENT: - if not drop_comments: - parts.append(tok.value) - elif t is TokenType.STRING and mask_strings: - parts.append(string_placeholder) - elif t is TokenType.NUMBER and mask_numbers: - parts.append(number_placeholder) - elif t is TokenType.IDENTIFIER and rename_identifiers: - name = mapping.get(tok.value) - if name is None: - name = f"{identifier_prefix}{len(mapping) + 1}" - mapping[tok.value] = name - parts.append(name) - elif t is TokenType.WHITESPACE and collapse_whitespace: - parts.append(' ') - else: - parts.append(tok.value) - return ''.join(parts) + from .. import _tokenops + return _tokenops.normalize(cls.tokenize(source), **options) From 2e2e0d8cef50f83262c36bf3f04ceec437acc419 Mon Sep 17 00:00:00 2001 From: danielogen Date: Tue, 7 Jul 2026 05:44:47 -0700 Subject: [PATCH 14/19] feat: pluggable tokenization backends (regex default, optional Pygments) --- src/PyReprism/engines/__init__.py | 34 +++++++ src/PyReprism/engines/base.py | 18 ++++ src/PyReprism/engines/pygments_engine.py | 108 +++++++++++++++++++++++ src/PyReprism/engines/regex_engine.py | 14 +++ 4 files changed, 174 insertions(+) create mode 100644 src/PyReprism/engines/__init__.py create mode 100644 src/PyReprism/engines/base.py create mode 100644 src/PyReprism/engines/pygments_engine.py create mode 100644 src/PyReprism/engines/regex_engine.py diff --git a/src/PyReprism/engines/__init__.py b/src/PyReprism/engines/__init__.py new file mode 100644 index 0000000..2132069 --- /dev/null +++ b/src/PyReprism/engines/__init__.py @@ -0,0 +1,34 @@ +"""Tokenization backends. + +``regex`` (default) is zero-dependency and uses each language's own regexes. +``pygments`` is more accurate but needs the optional Pygments dependency. +``auto`` uses pygments when it is importable, otherwise falls back to regex. +""" +from .base import Engine +from .pygments_engine import PygmentsEngine +from .regex_engine import RegexEngine + +_CACHE = {} + + +def get_engine(name: str = 'regex') -> Engine: + """Return an :class:`Engine` instance for ``name`` (``regex``/``pygments``/``auto``).""" + key = (name or 'regex').lower() + if key in _CACHE: + return _CACHE[key] + if key == 'regex': + engine = RegexEngine() + elif key == 'pygments': + engine = PygmentsEngine() + elif key == 'auto': + try: + engine = PygmentsEngine() + except ImportError: + engine = RegexEngine() + else: + raise ValueError(f"Unknown engine: {name!r} (use 'regex', 'pygments' or 'auto')") + _CACHE[key] = engine + return engine + + +__all__ = ['Engine', 'RegexEngine', 'PygmentsEngine', 'get_engine'] diff --git a/src/PyReprism/engines/base.py b/src/PyReprism/engines/base.py new file mode 100644 index 0000000..a0c17e6 --- /dev/null +++ b/src/PyReprism/engines/base.py @@ -0,0 +1,18 @@ +"""Backend abstraction: an engine turns source into a typed token stream.""" +from typing import List + +from ..tokens import Token + + +class Engine: + """Base class for tokenization backends. + + An engine only has to implement :meth:`tokenize`; every higher-level + operation (remove/extract/count/normalize/stats) is derived from the token + stream by :mod:`PyReprism._tokenops`. + """ + + name = 'base' + + def tokenize(self, source: str, language_cls) -> List[Token]: + raise NotImplementedError diff --git a/src/PyReprism/engines/pygments_engine.py b/src/PyReprism/engines/pygments_engine.py new file mode 100644 index 0000000..d55859b --- /dev/null +++ b/src/PyReprism/engines/pygments_engine.py @@ -0,0 +1,108 @@ +"""A higher-accuracy backend powered by the optional Pygments dependency. + +Pygments ships real lexers for every language PyReprism supports, so this engine +classifies comments, strings and other constructs far more reliably than the +regex fallback — at the cost of one extra dependency (``pip install +pyreprism[accurate]``). +""" +from typing import List + +from ..tokens import Token, TokenType +from .base import Engine + +# Language class name -> pygments lexer alias, only where they differ or the +# regex/name form is ambiguous. Everything else falls back to the lowercased +# class name and then to the file extension. +_ALIASES = { + 'CPP': 'cpp', + 'CSharp': 'csharp', + 'ObjectiveC': 'objective-c', + 'Clike': 'c', + 'MarkUp': 'html', + 'MarkupTemplating': 'html', + 'JavaScript': 'javascript', + 'TypeScript': 'typescript', + 'VisualBasic': 'vbnet', + 'Vbnet': 'vbnet', + 'MakeFile': 'make', + 'MarkDown': 'markdown', +} + + +class PygmentsEngine(Engine): + """Tokenize using Pygments lexers, mapped onto PyReprism's token model.""" + + name = 'pygments' + + def __init__(self): + try: + import pygments # noqa: F401 + except ImportError as exc: # pragma: no cover - exercised only without pygments + raise ImportError( + "the 'pygments' engine requires Pygments; install it with " + "`pip install pyreprism[accurate]` (or `pip install pygments`)." + ) from exc + self._lexers = {} + + def _lexer(self, language_cls): + from pygments.lexers import get_lexer_by_name, get_lexer_for_filename + from pygments.util import ClassNotFound + + name = language_cls.__name__ + if name in self._lexers: + return self._lexers[name] + alias = _ALIASES.get(name, name.lower()) + lexer = None + try: + lexer = get_lexer_by_name(alias) + except ClassNotFound: + try: + ext = language_cls.file_extension() + fname = ('file' + ext) if ext.startswith('.') else ext + lexer = get_lexer_for_filename(fname) + except ClassNotFound as exc: + raise ValueError( + f"Pygments has no lexer for language {name!r}" + ) from exc + self._lexers[name] = lexer + return lexer + + @staticmethod + def _map(ttype, value: str) -> TokenType: + from pygments.token import (Comment, Keyword, Name, Number, Operator, + String, Whitespace) + if ttype in Comment: + return TokenType.COMMENT + if ttype in String: + return TokenType.STRING + if ttype in Number: + return TokenType.NUMBER + if ttype in Keyword: + return TokenType.KEYWORD + if ttype in Name: + return TokenType.IDENTIFIER + if ttype in Operator: + return TokenType.OPERATOR + if ttype in Whitespace or value.isspace(): + return TokenType.WHITESPACE + return TokenType.OTHER + + def tokenize(self, source: str, language_cls) -> List[Token]: + lexer = self._lexer(language_cls) + tokens = [] + # get_tokens_unprocessed is lossless and yields absolute offsets. + # Pygments splits e.g. a string literal into quote/content/quote tokens; + # coalesce consecutive tokens of the same kind so a whole comment or + # string surfaces as one token (matching the regex engine's granularity). + for index, ttype, value in lexer.get_tokens_unprocessed(source): + if value == '': + continue + kind = self._map(ttype, value) + if tokens and tokens[-1].type is kind and tokens[-1].end == index: + prev = tokens[-1] + tokens[-1] = Token(kind, prev.value + value, prev.start, + index + len(value), prev.line) + else: + line = source.count('\n', 0, index) + 1 + tokens.append(Token(kind, value, index, index + len(value), line)) + return tokens diff --git a/src/PyReprism/engines/regex_engine.py b/src/PyReprism/engines/regex_engine.py new file mode 100644 index 0000000..ac7ef40 --- /dev/null +++ b/src/PyReprism/engines/regex_engine.py @@ -0,0 +1,14 @@ +"""The default, zero-dependency backend: the language's own regexes.""" +from typing import List + +from ..tokens import Token +from .base import Engine + + +class RegexEngine(Engine): + """Tokenize using the language class's regex-based :meth:`tokenize`.""" + + name = 'regex' + + def tokenize(self, source: str, language_cls) -> List[Token]: + return language_cls.tokenize(source) From 1a41fc705bf0c11c9af801954f5c6617db66a0c0 Mon Sep 17 00:00:00 2001 From: danielogen Date: Tue, 7 Jul 2026 05:46:08 -0700 Subject: [PATCH 15/19] fix: resolve ambiguous extensions to canonical language --- src/PyReprism/languages/__init__.py | 30 ++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/PyReprism/languages/__init__.py b/src/PyReprism/languages/__init__.py index 5e7dbdb..e9acfde 100644 --- a/src/PyReprism/languages/__init__.py +++ b/src/PyReprism/languages/__init__.py @@ -60,15 +60,39 @@ def _load_all_languages() -> None: continue +# For extensions shared by several languages, prefer the canonical/primary one. +_PREFERRED_LANGUAGE = { + '.py': 'Python', + '.h': 'C', + '.r': 'R', + '.pl': 'Perl', + '.ts': 'TypeScript', + '.php': 'PHP', + '.sql': 'SQL', + '.vb': 'Vbnet', + '.js': 'JavaScript', +} + + def get_language_by_extension(ext: str) -> Optional[Type[BaseLanguage]]: - """Return the first registered language class whose ``file_extension()`` matches ``ext``. + """Return a registered language class whose ``file_extension()`` matches ``ext``. All language modules are imported on first call so cold lookups succeed. Some extensions are shared by several languages (``.py`` -> Python/Django, ``.m`` -> - MatLab/ObjectiveC); in those cases the first registered match is returned. + MatLab/ObjectiveC); a small preference table picks the canonical language for the + common cases, otherwise the first registered match is returned. """ _load_all_languages() - for cls in LanguageRegistry.all().values(): + registry = LanguageRegistry.all() + preferred = _PREFERRED_LANGUAGE.get(ext) + if preferred: + cls = registry.get(preferred) + try: + if cls is not None and cls.file_extension() == ext: + return cls + except Exception: + pass + for cls in registry.values(): try: if cls.file_extension() == ext: return cls From 2080653bba33641de29711c0f94857d1ceb6f470 Mon Sep 17 00:00:00 2001 From: danielogen Date: Tue, 7 Jul 2026 05:46:46 -0700 Subject: [PATCH 16/19] feat: batch/corpus processing + CLI (scan, --engine, --output); tests & docs --- README.md | 38 +++++++ pyproject.toml | 4 + src/PyReprism/__init__.py | 157 +++++++++++++++++++--------- src/PyReprism/batch.py | 208 ++++++++++++++++++++++++++++++++++++++ src/PyReprism/cli.py | 183 ++++++++++++++++++++++++++------- src/tests/test_batch.py | 120 ++++++++++++++++++++++ src/tests/test_engines.py | 82 +++++++++++++++ 7 files changed, 712 insertions(+), 80 deletions(-) create mode 100644 src/PyReprism/batch.py create mode 100644 src/tests/test_batch.py create mode 100644 src/tests/test_engines.py diff --git a/README.md b/README.md index e1a7921..6e55086 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,42 @@ numbers stable (useful for tools that map back to source): pr.blank_comments(source, lang="python") # comment content removed, newlines kept ``` +### Accuracy: choose a backend + +By default PyReprism uses its own zero-dependency regex engine (fast, no installs). +For higher accuracy — e.g. correctly ignoring a `#` inside a string — pass +`engine="pygments"` (install the optional extra with `pip install pyreprism[accurate]`): + +```python +src = 'url = "http://x#frag" # real comment' + +pr.remove_comments(src, lang="python") # regex (default) +pr.remove_comments(src, lang="python", engine="pygments") # keeps the URL, drops the comment +``` + +`engine` accepts `"regex"` (default), `"pygments"`, or `"auto"` (use pygments if +installed, else regex). It works on every operation — `remove_*`, `extract_*`, +`count_*`, `tokenize`, `normalize`, `stats`, `preprocess` — and on the CLI via +`--engine`. + +### Batch / whole-directory processing + +Analyze or transform an entire source tree (junk folders like `.git`, +`node_modules`, `venv` are skipped automatically): + +```python +from PyReprism import batch + +report = batch.analyze("myproject/") # walk the tree, compute metrics +report.totals() # aggregate line/token counts +report.by_language() # per-language breakdown +report.to_json(); report.to_csv() # export for dataframes / notebooks + +# Bulk-transform into a mirrored output directory: +batch.transform("myproject/", lambda text, lang: lang.remove_comments(text), + output="stripped/") +``` + ### Detect the language from a filename ```python @@ -139,6 +175,8 @@ pyreprism tokenize --json file.py pyreprism stats --json file.py # line/token metrics pyreprism normalize file.py # canonicalize for ML pyreprism remove comments --in-place file.py # rewrite in place +pyreprism scan myproject/ --csv # aggregate metrics over a tree +pyreprism remove comments src/ --output out/ # bulk-transform a directory pyreprism languages # list supported languages ``` diff --git a/pyproject.toml b/pyproject.toml index 07c7bb6..6abf031 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,10 +28,14 @@ classifiers = [ ] [project.optional-dependencies] +accurate = [ + "pygments", +] dev = [ "flake8", "pytest", "pytest-cov", + "pygments", ] docs = [ "sphinx", diff --git a/src/PyReprism/__init__.py b/src/PyReprism/__init__.py index dc363f1..839a8b1 100644 --- a/src/PyReprism/__init__.py +++ b/src/PyReprism/__init__.py @@ -83,65 +83,100 @@ def _resolve(lang: LanguageLike) -> Type: return get_language(lang) -def remove_comments(source: str, lang: LanguageLike, isList: bool = False): +_CONSTRUCT_TYPE = { + 'comments': TokenType.COMMENT, 'strings': TokenType.STRING, + 'numbers': TokenType.NUMBER, 'keywords': TokenType.KEYWORD, + 'operators': TokenType.OPERATOR, 'identifiers': TokenType.IDENTIFIER, +} + + +def _engine_tokens(source: str, lang: LanguageLike, engine: str) -> List[Token]: + from .engines import get_engine + return get_engine(engine).tokenize(source, _resolve(lang)) + + +def _is_regex(engine) -> bool: + return engine in (None, 'regex') + + +def _op(action: str, construct: str, source: str, lang: LanguageLike, engine: str): + """Run remove/extract/count for a construct via the selected engine.""" + if _is_regex(engine): + return getattr(_resolve(lang), f'{action}_{construct}')(source) + from . import _tokenops + tokens = _engine_tokens(source, lang, engine) + return getattr(_tokenops, action)(tokens, _CONSTRUCT_TYPE[construct]) + + +def remove_comments(source: str, lang: LanguageLike, engine: str = 'regex', isList: bool = False): """Remove comments from ``source`` for the given language.""" - return _resolve(lang).remove_comments(source, isList=isList) + if _is_regex(engine): + return _resolve(lang).remove_comments(source, isList=isList) + tokens = _engine_tokens(source, lang, engine) + if isList: + return [t.value for t in tokens if t.type is not TokenType.COMMENT] + from . import _tokenops + return _tokenops.remove(tokens, TokenType.COMMENT) -def extract_comments(source: str, lang: LanguageLike) -> List[str]: - return _resolve(lang).extract_comments(source) +def extract_comments(source: str, lang: LanguageLike, engine: str = 'regex') -> List[str]: + return _op('extract', 'comments', source, lang, engine) -def count_comments(source: str, lang: LanguageLike) -> int: - return _resolve(lang).count_comments(source) +def count_comments(source: str, lang: LanguageLike, engine: str = 'regex') -> int: + return _op('count', 'comments', source, lang, engine) -def match_comments(source: str, lang: LanguageLike) -> List[Token]: - return _resolve(lang).match_comments(source) +def match_comments(source: str, lang: LanguageLike, engine: str = 'regex') -> List[Token]: + if _is_regex(engine): + return _resolve(lang).match_comments(source) + return [t for t in _engine_tokens(source, lang, engine) if t.type is TokenType.COMMENT] -def remove_keywords(source: str, lang: LanguageLike) -> str: - return _resolve(lang).remove_keywords(source) +def remove_keywords(source: str, lang: LanguageLike, engine: str = 'regex') -> str: + if _is_regex(engine): + return _resolve(lang).remove_keywords(source) + return _op('remove', 'keywords', source, lang, engine) -def extract_keywords(source: str, lang: LanguageLike) -> List[str]: - return _resolve(lang).extract_keywords(source) +def extract_keywords(source: str, lang: LanguageLike, engine: str = 'regex') -> List[str]: + return _op('extract', 'keywords', source, lang, engine) -def count_keywords(source: str, lang: LanguageLike) -> int: - return _resolve(lang).count_keywords(source) +def count_keywords(source: str, lang: LanguageLike, engine: str = 'regex') -> int: + return _op('count', 'keywords', source, lang, engine) -def remove_numbers(source: str, lang: LanguageLike) -> str: - return _resolve(lang).remove_numbers(source) +def remove_numbers(source: str, lang: LanguageLike, engine: str = 'regex') -> str: + return _op('remove', 'numbers', source, lang, engine) -def extract_numbers(source: str, lang: LanguageLike) -> List[str]: - return _resolve(lang).extract_numbers(source) +def extract_numbers(source: str, lang: LanguageLike, engine: str = 'regex') -> List[str]: + return _op('extract', 'numbers', source, lang, engine) -def count_numbers(source: str, lang: LanguageLike) -> int: - return _resolve(lang).count_numbers(source) +def count_numbers(source: str, lang: LanguageLike, engine: str = 'regex') -> int: + return _op('count', 'numbers', source, lang, engine) -def remove_operators(source: str, lang: LanguageLike) -> str: - return _resolve(lang).remove_operators(source) +def remove_operators(source: str, lang: LanguageLike, engine: str = 'regex') -> str: + return _op('remove', 'operators', source, lang, engine) -def extract_operators(source: str, lang: LanguageLike) -> List[str]: - return _resolve(lang).extract_operators(source) +def extract_operators(source: str, lang: LanguageLike, engine: str = 'regex') -> List[str]: + return _op('extract', 'operators', source, lang, engine) -def remove_strings(source: str, lang: LanguageLike) -> str: - return _resolve(lang).remove_strings(source) +def remove_strings(source: str, lang: LanguageLike, engine: str = 'regex') -> str: + return _op('remove', 'strings', source, lang, engine) -def extract_strings(source: str, lang: LanguageLike) -> List[str]: - return _resolve(lang).extract_strings(source) +def extract_strings(source: str, lang: LanguageLike, engine: str = 'regex') -> List[str]: + return _op('extract', 'strings', source, lang, engine) -def extract_identifiers(source: str, lang: LanguageLike) -> List[str]: - return _resolve(lang).extract_identifiers(source) +def extract_identifiers(source: str, lang: LanguageLike, engine: str = 'regex') -> List[str]: + return _op('extract', 'identifiers', source, lang, engine) def remove_whitespaces(source: str) -> str: @@ -149,27 +184,44 @@ def remove_whitespaces(source: str) -> str: return Normalizer.remove_whitespaces(source) -def tokenize(source: str, lang: LanguageLike) -> List[Token]: +def tokenize(source: str, lang: LanguageLike, engine: str = 'regex') -> List[Token]: """Return the flat list of typed tokens for ``source``.""" - return _resolve(lang).tokenize(source) + if _is_regex(engine): + return _resolve(lang).tokenize(source) + return _engine_tokens(source, lang, engine) -def blank_comments(source: str, lang: LanguageLike, replacement: str = ' ') -> str: +def blank_comments(source: str, lang: LanguageLike, replacement: str = ' ', + engine: str = 'regex') -> str: """Remove comment content while preserving line numbers.""" - return _resolve(lang).blank_comments(source, replacement=replacement) - - -def stats(source: str, lang: LanguageLike) -> CodeStats: + if _is_regex(engine): + return _resolve(lang).blank_comments(source, replacement=replacement) + parts = [] + for tok in _engine_tokens(source, lang, engine): + if tok.type is TokenType.COMMENT: + parts.append(''.join('\n' if ch == '\n' else replacement for ch in tok.value)) + else: + parts.append(tok.value) + return ''.join(parts) + + +def stats(source: str, lang: LanguageLike, engine: str = 'regex') -> CodeStats: """Return line- and token-level :class:`CodeStats` for ``source``.""" - return _resolve(lang).stats(source) + if _is_regex(engine): + return _resolve(lang).stats(source) + from . import _tokenops + return _tokenops.stats(_engine_tokens(source, lang, engine), source) -def normalize(source: str, lang: LanguageLike, **options) -> str: +def normalize(source: str, lang: LanguageLike, engine: str = 'regex', **options) -> str: """Return a canonicalized form of ``source`` for ML / clone detection. See :meth:`PyReprism.languages.base.BaseLanguage.normalize` for options. """ - return _resolve(lang).normalize(source, **options) + if _is_regex(engine): + return _resolve(lang).normalize(source, **options) + from . import _tokenops + return _tokenops.normalize(_engine_tokens(source, lang, engine), **options) _STEPS = { @@ -182,20 +234,35 @@ def normalize(source: str, lang: LanguageLike, **options) -> str: } -def preprocess(source: str, lang: LanguageLike, steps: Sequence[str] = ('comments',)) -> str: +def preprocess(source: str, lang: LanguageLike, steps: Sequence[str] = ('comments',), + engine: str = 'regex') -> str: """Apply a sequence of removal ``steps`` in order and return the result. Valid steps: ``comments``, ``strings``, ``numbers``, ``operators``, ``keywords``, ``whitespace``. """ - cls = _resolve(lang) + if steps is None: + steps = ('comments',) + if _is_regex(engine): + cls = _resolve(lang) + out = source + for step in steps: + fn = _STEPS.get(step) + if fn is None: + raise ValueError(f"Unknown preprocessing step: {step!r}. " + f"Valid steps: {', '.join(sorted(_STEPS))}") + out = fn(cls, out) + return out + from . import _tokenops out = source for step in steps: - fn = _STEPS.get(step) - if fn is None: + if step == 'whitespace': + out = Normalizer.remove_whitespaces(out) + elif step in _CONSTRUCT_TYPE: + out = _tokenops.remove(_engine_tokens(out, lang, engine), _CONSTRUCT_TYPE[step]) + else: raise ValueError(f"Unknown preprocessing step: {step!r}. " f"Valid steps: {', '.join(sorted(_STEPS))}") - out = fn(cls, out) return out diff --git a/src/PyReprism/batch.py b/src/PyReprism/batch.py new file mode 100644 index 0000000..1129ffe --- /dev/null +++ b/src/PyReprism/batch.py @@ -0,0 +1,208 @@ +"""Batch/directory processing: analyze or transform whole source trees. + +Discovery walks directories, skips common junk folders, and keeps only files +whose extension maps to a supported language. :func:`analyze` returns aggregate +metrics (per file, per language, and totals) exportable as JSON or CSV; +:func:`transform` applies a text transformation across a tree, optionally writing +a mirrored output directory. +""" +import csv +import fnmatch +import io +import json +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union + +from .metrics import CodeStats + +PathLike = Union[str, os.PathLike] + +_SKIP_DIRS = { + '.git', '.hg', '.svn', '__pycache__', 'node_modules', '.venv', 'venv', 'env', + 'dist', 'build', '.tox', '.mypy_cache', '.pytest_cache', '.idea', '.vscode', + 'htmlcov', '.eggs', +} + +_STAT_FIELDS = ( + 'lines', 'code_lines', 'comment_lines', 'blank_lines', 'characters', + 'comment_tokens', 'string_tokens', 'number_tokens', 'keyword_tokens', + 'identifier_tokens', 'operator_tokens', +) + + +def _zero_stats() -> Dict[str, int]: + data = {k: 0 for k in _STAT_FIELDS} + data['files'] = 0 + return data + + +def _accept(path: Path, include: Optional[Sequence[str]], exclude: Optional[Sequence[str]]) -> bool: + name, full = path.name, str(path) + if include and not any(fnmatch.fnmatch(name, g) or fnmatch.fnmatch(full, g) for g in include): + return False + if exclude and any(fnmatch.fnmatch(name, g) or fnmatch.fnmatch(full, g) for g in exclude): + return False + return True + + +def _walk(root: Path, recursive: bool) -> Iterator[Path]: + for current, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d not in _SKIP_DIRS and not d.startswith('.')] + for name in sorted(files): + yield Path(current) / name + if not recursive: + dirs[:] = [] + + +def iter_source_files(paths: Union[PathLike, Sequence[PathLike]], *, recursive: bool = True, + include: Optional[Sequence[str]] = None, + exclude: Optional[Sequence[str]] = None) -> Iterator[Tuple[Path, Path, type]]: + """Yield ``(base, path, language_cls)`` for every supported source file. + + ``base`` is the top-level input the file was discovered under (used to mirror + directory structure). Files with an unrecognized extension are skipped. + """ + from . import detect_language + + if isinstance(paths, (str, os.PathLike)): + paths = [paths] + for raw in paths: + base = Path(raw) + if base.is_dir(): + candidates = _walk(base, recursive) + elif base.is_file(): + candidates = [base] + else: + continue + for path in candidates: + if not _accept(path, include, exclude): + continue + cls = detect_language(filename=str(path)) + if cls is not None: + yield base, path, cls + + +@dataclass +class FileReport: + """Metrics for one processed file (``stats`` is ``None`` on error).""" + + path: str + language: str + stats: Optional[CodeStats] = None + error: Optional[str] = None + + +@dataclass +class BatchReport: + """Aggregate report over a set of files.""" + + files: List[FileReport] = field(default_factory=list) + + @property + def ok(self) -> List[FileReport]: + return [f for f in self.files if f.stats is not None] + + @property + def errors(self) -> List[FileReport]: + return [f for f in self.files if f.stats is None] + + def totals(self) -> Dict[str, int]: + total = _zero_stats() + for report in self.ok: + total['files'] += 1 + for key in _STAT_FIELDS: + total[key] += getattr(report.stats, key) + return total + + def by_language(self) -> Dict[str, Dict[str, int]]: + out: Dict[str, Dict[str, int]] = {} + for report in self.ok: + agg = out.setdefault(report.language, _zero_stats()) + agg['files'] += 1 + for key in _STAT_FIELDS: + agg[key] += getattr(report.stats, key) + return dict(sorted(out.items())) + + def to_dict(self) -> dict: + return { + 'files': [ + {'path': f.path, 'language': f.language, 'error': f.error, + **(f.stats.as_dict() if f.stats else {})} + for f in self.files + ], + 'by_language': self.by_language(), + 'totals': self.totals(), + } + + def to_json(self, indent: int = 2) -> str: + return json.dumps(self.to_dict(), indent=indent) + + def to_csv(self) -> str: + buffer = io.StringIO() + writer = csv.writer(buffer) + writer.writerow(['path', 'language', *_STAT_FIELDS]) + for report in self.files: + if report.stats is not None: + writer.writerow([report.path, report.language, + *[getattr(report.stats, k) for k in _STAT_FIELDS]]) + return buffer.getvalue() + + +def analyze(paths: Union[PathLike, Sequence[PathLike]], *, engine: str = 'regex', + recursive: bool = True, include: Optional[Sequence[str]] = None, + exclude: Optional[Sequence[str]] = None) -> BatchReport: + """Walk ``paths`` and compute :class:`CodeStats` for every supported file.""" + from . import _tokenops + from .engines import get_engine + + reports: List[FileReport] = [] + use_regex = engine in (None, 'regex') + for _base, path, cls in iter_source_files(paths, recursive=recursive, + include=include, exclude=exclude): + try: + text = path.read_text(encoding='utf-8', errors='replace') + if use_regex: + stats = cls.stats(text) + else: + stats = _tokenops.stats(get_engine(engine).tokenize(text, cls), text) + reports.append(FileReport(str(path), cls.__name__, stats)) + except Exception as exc: # pragma: no cover - defensive I/O guard + reports.append(FileReport(str(path), cls.__name__, None, str(exc))) + return BatchReport(reports) + + +def _destination(base: Path, path: Path, output: Path) -> Path: + if base.is_dir(): + return output / path.relative_to(base) + return output / path.name + + +def transform(paths: Union[PathLike, Sequence[PathLike]], + transformer: Callable[[str, type], str], *, + output: Optional[PathLike] = None, in_place: bool = False, + recursive: bool = True, include: Optional[Sequence[str]] = None, + exclude: Optional[Sequence[str]] = None) -> List[Tuple[str, str]]: + """Apply ``transformer(text, language_cls)`` to every discovered file. + + With ``output`` the tree is mirrored under that directory; with ``in_place`` + files are rewritten. Returns ``(path, status)`` pairs. + """ + out_root = Path(output) if output else None + results: List[Tuple[str, str]] = [] + for base, path, cls in iter_source_files(paths, recursive=recursive, + include=include, exclude=exclude): + text = path.read_text(encoding='utf-8', errors='replace') + transformed = transformer(text, cls) + if out_root is not None: + dest = _destination(base, path, out_root) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(transformed, encoding='utf-8') + results.append((str(path), str(dest))) + elif in_place: + path.write_text(transformed, encoding='utf-8') + results.append((str(path), 'in-place')) + else: + results.append((str(path), transformed)) + return results diff --git a/src/PyReprism/cli.py b/src/PyReprism/cli.py index 88d6a92..0f83c16 100644 --- a/src/PyReprism/cli.py +++ b/src/PyReprism/cli.py @@ -16,25 +16,50 @@ import sys from typing import List, Optional -from . import __version__, detect_language, get_language, preprocess +from . import __version__, _tokenops, detect_language, get_language, preprocess +from .engines import get_engine +from .tokens import TokenType CONSTRUCTS = ['comments', 'keywords', 'numbers', 'operators', 'strings', 'identifiers'] +_CONSTRUCT_TYPE = { + 'comments': TokenType.COMMENT, 'keywords': TokenType.KEYWORD, + 'numbers': TokenType.NUMBER, 'operators': TokenType.OPERATOR, + 'strings': TokenType.STRING, 'identifiers': TokenType.IDENTIFIER, +} def _iter_inputs(paths: List[str]): - """Yield ``(label, text, filename)`` for each input; filename is None for stdin.""" + """Yield ``(label, text, filename, base)`` for each input. + + ``filename`` and ``base`` are None for stdin. Directories are walked for + supported source files (``base`` is the directory, for output mirroring). + """ + import os + + from .batch import iter_source_files + if not paths: - yield ('', sys.stdin.read(), None) + yield ('', sys.stdin.read(), None, None) return for pattern in paths: + if os.path.isdir(pattern): + for base, path, _cls in iter_source_files(pattern): + try: + yield (str(path), path.read_text(encoding='utf-8', errors='replace'), + str(path), str(base)) + except OSError as exc: + print(f"pyreprism: {exc}", file=sys.stderr) + continue matches = glob.glob(pattern, recursive=True) if any(c in pattern for c in '*?[') else [pattern] if not matches: print(f"pyreprism: no such file: {pattern}", file=sys.stderr) continue for filepath in matches: + if os.path.isdir(filepath): + continue try: with open(filepath, 'r', encoding='utf-8') as handle: - yield (filepath, handle.read(), filepath) + yield (filepath, handle.read(), filepath, None) except OSError as exc: print(f"pyreprism: {exc}", file=sys.stderr) @@ -50,35 +75,45 @@ def _resolve(lang: Optional[str], filename: Optional[str]): raise SystemExit(2) -def _method(cls, action: str, construct: str): - fn = getattr(cls, f'{action}_{construct}', None) - if fn is None: - raise SystemExit(f"pyreprism: '{action} {construct}' is not supported") - return fn +def _multiple(paths) -> bool: + """True when the inputs are expected to expand to more than one file.""" + import os + return len(paths) > 1 or any(os.path.isdir(p) for p in paths) + + +def _op(cls, action: str, construct: str, text: str, engine: str): + """Run remove/extract/count for a construct via the selected engine.""" + if engine in (None, 'regex'): + fn = getattr(cls, f'{action}_{construct}', None) + if fn is None: + raise SystemExit(f"pyreprism: '{action} {construct}' is not supported") + return fn(text) + tokens = get_engine(engine).tokenize(text, cls) + return getattr(_tokenops, action)(tokens, _CONSTRUCT_TYPE[construct]) def _cmd_remove(args) -> int: - for label, text, filename in _iter_inputs(args.paths): + for label, text, filename, base in _iter_inputs(args.paths): cls = _resolve(args.lang, filename) - result = _method(cls, 'remove', args.construct)(text) - _emit(result, filename, args.in_place, args) + result = _op(cls, 'remove', args.construct, text, args.engine) + _emit(result, filename, base, args) return 0 def _cmd_preprocess(args) -> int: steps = [s.strip() for s in args.steps.split(',') if s.strip()] - for label, text, filename in _iter_inputs(args.paths): + for label, text, filename, base in _iter_inputs(args.paths): cls = _resolve(args.lang, filename) - result = preprocess(text, lang=cls, steps=steps) - _emit(result, filename, args.in_place, args) + result = preprocess(text, lang=cls, steps=steps, engine=args.engine) + _emit(result, filename, base, args) return 0 def _cmd_extract(args) -> int: - multiple = len(args.paths) > 1 - for label, text, filename in _iter_inputs(args.paths): + multiple = _multiple(args.paths) + for label, text, filename, base in _iter_inputs(args.paths): cls = _resolve(args.lang, filename) - items = _method(cls, 'extract', args.construct)(text) + items = _op(cls, 'extract', args.construct, text, args.engine) if args.json: print(json.dumps({label: items} if multiple else items)) else: @@ -90,17 +125,17 @@ def _cmd_extract(args) -> int: def _cmd_count(args) -> int: - for label, text, filename in _iter_inputs(args.paths): + for label, text, filename, base in _iter_inputs(args.paths): cls = _resolve(args.lang, filename) - count = _method(cls, 'count', args.construct)(text) - print(f"{count}\t{label}" if len(args.paths) > 1 else count) + count = _op(cls, 'count', args.construct, text, args.engine) + print(f"{count}\t{label}" if _multiple(args.paths) else count) return 0 def _cmd_tokenize(args) -> int: - for label, text, filename in _iter_inputs(args.paths): + for label, text, filename, base in _iter_inputs(args.paths): cls = _resolve(args.lang, filename) - tokens = cls.tokenize(text) + tokens = get_engine(args.engine).tokenize(text, cls) if args.json: print(json.dumps([ {'type': t.type.value, 'value': t.value, @@ -114,10 +149,13 @@ def _cmd_tokenize(args) -> int: def _cmd_stats(args) -> int: - multiple = len(args.paths) > 1 - for label, text, filename in _iter_inputs(args.paths): + multiple = _multiple(args.paths) + for label, text, filename, base in _iter_inputs(args.paths): cls = _resolve(args.lang, filename) - data = cls.stats(text).as_dict() + if args.engine in (None, 'regex'): + data = cls.stats(text).as_dict() + else: + data = _tokenops.stats(get_engine(args.engine).tokenize(text, cls), text).as_dict() if args.json: print(json.dumps({label: data} if multiple else data)) else: @@ -137,10 +175,46 @@ def _cmd_normalize(args) -> int: rename_identifiers=not args.keep_names, collapse_whitespace=args.collapse_whitespace, ) - for label, text, filename in _iter_inputs(args.paths): + for label, text, filename, base in _iter_inputs(args.paths): cls = _resolve(args.lang, filename) - result = cls.normalize(text, **options) - _emit(result, filename, args.in_place, args) + if args.engine in (None, 'regex'): + result = cls.normalize(text, **options) + else: + result = _tokenops.normalize(get_engine(args.engine).tokenize(text, cls), **options) + _emit(result, filename, base, args) + return 0 + + +def _cmd_scan(args) -> int: + from .batch import analyze + + report = analyze(args.paths, engine=args.engine, recursive=not args.no_recursive, + include=args.include, exclude=args.exclude) + if args.json: + print(report.to_json()) + return 0 + if args.csv: + print(report.to_csv(), end='') + return 0 + + if args.per_file: + for f in report.ok: + print(f"{f.path}\t{f.language}\t{f.stats.code_lines} code, " + f"{f.stats.comment_lines} comment") + print() + by_lang = report.by_language() + if by_lang: + width = max(len(name) for name in by_lang) + print(f"{'language'.ljust(width)} files code comment blank") + for name, agg in by_lang.items(): + print(f"{name.ljust(width)} {agg['files']:>5} {agg['code_lines']:>4} " + f"{agg['comment_lines']:>7} {agg['blank_lines']:>5}") + totals = report.totals() + print(f"\n{totals['files']} files, {totals['lines']} lines " + f"({totals['code_lines']} code, {totals['comment_lines']} comment, " + f"{totals['blank_lines']} blank)") + if report.errors: + print(f"{len(report.errors)} file(s) could not be processed", file=sys.stderr) return 0 @@ -163,8 +237,21 @@ def _cmd_languages(args) -> int: return 0 -def _emit(result: str, filename: Optional[str], in_place: bool, args) -> None: - if in_place and filename: +def _emit(result: str, filename: Optional[str], base: Optional[str], args) -> None: + import os + + output = getattr(args, 'output', None) + if output and filename: + src = os.path.abspath(filename) + if base and os.path.isdir(base): + rel = os.path.relpath(src, os.path.abspath(base)) + else: + rel = os.path.basename(src) + dest = os.path.join(output, rel) + os.makedirs(os.path.dirname(dest) or '.', exist_ok=True) + with open(dest, 'w', encoding='utf-8') as handle: + handle.write(result) + elif getattr(args, 'in_place', False) and filename: with open(filename, 'w', encoding='utf-8') as handle: handle.write(result) else: @@ -188,12 +275,21 @@ def add_common(p, lang=True): p.add_argument('-l', '--lang', help='language name, extension, or class (auto-detected from ' 'filename when omitted)') + p.add_argument('-e', '--engine', default='regex', + choices=['regex', 'pygments', 'auto'], + help="tokenization backend (default: regex; 'pygments' is more " + "accurate but needs pyreprism[accurate])") + + def add_output(p): + p.add_argument('-i', '--in-place', action='store_true', + help='rewrite files in place instead of printing to stdout') + p.add_argument('-o', '--output', metavar='DIR', + help='write results into DIR, mirroring the input tree') p_remove = sub.add_parser('remove', help='remove a construct from the source') p_remove.add_argument('construct', choices=[c for c in CONSTRUCTS if c != 'identifiers']) add_common(p_remove) - p_remove.add_argument('-i', '--in-place', action='store_true', - help='rewrite files in place instead of printing to stdout') + add_output(p_remove) p_remove.set_defaults(func=_cmd_remove) p_pre = sub.add_parser('preprocess', help='apply a sequence of removal steps') @@ -201,7 +297,7 @@ def add_common(p, lang=True): help='comma-separated steps: comments,strings,numbers,operators,' 'keywords,whitespace') add_common(p_pre) - p_pre.add_argument('-i', '--in-place', action='store_true') + add_output(p_pre) p_pre.set_defaults(func=_cmd_preprocess) p_extract = sub.add_parser('extract', help='extract occurrences of a construct') @@ -234,9 +330,26 @@ def add_common(p, lang=True): p_norm.add_argument('--keep-names', action='store_true', help='do not rename identifiers') p_norm.add_argument('--collapse-whitespace', action='store_true') - p_norm.add_argument('-i', '--in-place', action='store_true') + add_output(p_norm) p_norm.set_defaults(func=_cmd_normalize) + p_scan = sub.add_parser('scan', + help='walk directories and report aggregate metrics') + p_scan.add_argument('paths', nargs='+', help='directories or files to scan') + p_scan.add_argument('-e', '--engine', default='regex', + choices=['regex', 'pygments', 'auto']) + p_scan.add_argument('--json', action='store_true', help='emit the full JSON report') + p_scan.add_argument('--csv', action='store_true', help='emit per-file CSV') + p_scan.add_argument('--per-file', action='store_true', + help='include a per-file breakdown in the text report') + p_scan.add_argument('--include', action='append', metavar='GLOB', + help='only include files matching GLOB (repeatable)') + p_scan.add_argument('--exclude', action='append', metavar='GLOB', + help='skip files matching GLOB (repeatable)') + p_scan.add_argument('--no-recursive', action='store_true', + help='do not descend into subdirectories') + p_scan.set_defaults(func=_cmd_scan) + p_langs = sub.add_parser('languages', help='list supported languages and extensions') p_langs.set_defaults(func=_cmd_languages) diff --git a/src/tests/test_batch.py b/src/tests/test_batch.py new file mode 100644 index 0000000..33008bb --- /dev/null +++ b/src/tests/test_batch.py @@ -0,0 +1,120 @@ +"""Tests for batch/directory processing (discovery, analyze, transform, CLI scan).""" +import json + +import pytest + +from PyReprism import batch +from PyReprism.cli import main + + +@pytest.fixture +def tree(tmp_path): + (tmp_path / 'sub').mkdir() + (tmp_path / '.git').mkdir() + (tmp_path / 'node_modules').mkdir() + (tmp_path / 'a.py').write_text('def f():\n return 1 # c\n') + (tmp_path / 'sub' / 'b.py').write_text('x = 2 # c\ny = 3\n') + (tmp_path / 'sub' / 'm.c').write_text('int main(){return 0;} // c\n') + (tmp_path / '.git' / 'junk.py').write_text('secret = 1\n') # skipped dir + (tmp_path / 'node_modules' / 'dep.js').write_text('var x = 1\n') # skipped dir + (tmp_path / 'readme.zzz').write_text('not source\n') # unknown ext + return tmp_path + + +# -------------------------------------------------------------------- discovery +def test_discovery_skips_junk_and_unknown(tree): + found = sorted(p.name for _base, p, _cls in batch.iter_source_files(tree)) + assert found == ['a.py', 'b.py', 'm.c'] + + +def test_discovery_non_recursive(tree): + found = sorted(p.name for _b, p, _c in batch.iter_source_files(tree, recursive=False)) + assert found == ['a.py'] + + +def test_discovery_include_exclude(tree): + only_py = sorted(p.name for _b, p, _c in batch.iter_source_files(tree, include=['*.py'])) + assert only_py == ['a.py', 'b.py'] + no_b = sorted(p.name for _b, p, _c in batch.iter_source_files(tree, exclude=['b.py'])) + assert no_b == ['a.py', 'm.c'] + + +def test_py_detected_as_python(tree): + langs = {p.name: cls.__name__ for _b, p, cls in batch.iter_source_files(tree)} + assert langs['a.py'] == 'Python' # not Django (canonical preference) + assert langs['m.c'] == 'C' + + +# ---------------------------------------------------------------------- analyze +def test_analyze_totals_and_by_language(tree): + report = batch.analyze(tree) + totals = report.totals() + assert totals['files'] == 3 + assert totals['code_lines'] == 5 + langs = report.by_language() + assert langs['Python']['files'] == 2 + assert langs['C']['files'] == 1 + + +def test_analyze_json_and_csv(tree): + report = batch.analyze(tree) + data = json.loads(report.to_json()) + assert set(data) == {'files', 'by_language', 'totals'} + assert len(data['files']) == 3 + csv_text = report.to_csv() + assert csv_text.splitlines()[0].startswith('path,language,lines') + assert len(csv_text.splitlines()) == 4 # header + 3 files + + +# -------------------------------------------------------------------- transform +def test_transform_to_output_mirror(tree, tmp_path): + out = tmp_path.parent / 'out' + results = batch.transform(tree, lambda text, cls: cls.remove_comments(text), output=out) + assert len(results) == 3 + assert (out / 'a.py').exists() + assert (out / 'sub' / 'b.py').exists() + assert '# c' not in (out / 'a.py').read_text() + + +def test_transform_in_place(tree): + batch.transform(tree, lambda text, cls: cls.remove_comments(text), in_place=True) + assert '# c' not in (tree / 'a.py').read_text() + + +def test_transform_collect_results(tree): + results = batch.transform(tree, lambda text, cls: cls.remove_comments(text)) + # no output/in_place -> returns (path, transformed_text) + assert all(isinstance(text, str) for _path, text in results) + + +# --------------------------------------------------------------------------- CLI +def test_cli_scan_text(tree, capsys): + assert main(['scan', str(tree)]) == 0 + out = capsys.readouterr().out + assert 'Python' in out and 'C' in out + assert '3 files' in out + + +def test_cli_scan_json(tree, capsys): + assert main(['scan', str(tree), '--json']) == 0 + data = json.loads(capsys.readouterr().out) + assert data['totals']['files'] == 3 + + +def test_cli_scan_csv(tree, capsys): + assert main(['scan', str(tree), '--csv']) == 0 + assert capsys.readouterr().out.splitlines()[0].startswith('path,language') + + +def test_cli_directory_transform_output(tree, tmp_path, capsys): + out = tmp_path.parent / 'cliout' + assert main(['remove', 'comments', str(tree), '--output', str(out)]) == 0 + assert (out / 'a.py').exists() + assert '# c' not in (out / 'a.py').read_text() + + +def test_cli_count_over_directory(tree, capsys): + assert main(['count', 'comments', str(tree)]) == 0 + out = capsys.readouterr().out + # per-file labelled output because a directory expands to many files + assert out.count('\t') == 3 diff --git a/src/tests/test_engines.py b/src/tests/test_engines.py new file mode 100644 index 0000000..602d90a --- /dev/null +++ b/src/tests/test_engines.py @@ -0,0 +1,82 @@ +"""Tests for the tokenization backends (regex default, optional pygments).""" +import pytest + +import PyReprism as pr +from PyReprism.engines import RegexEngine, get_engine + +pygments = pytest.importorskip("pygments") # skip this module if pygments is absent + + +# ------------------------------------------------------------------- engine selection +def test_get_engine_regex_default(): + assert isinstance(get_engine(), RegexEngine) + assert isinstance(get_engine('regex'), RegexEngine) + + +def test_get_engine_auto_prefers_pygments_when_available(): + assert type(get_engine('auto')).__name__ == 'PygmentsEngine' + + +def test_get_engine_unknown(): + with pytest.raises(ValueError): + get_engine('nope') + + +# --------------------------------------------------------------------- correctness win +def test_pygments_does_not_treat_hash_in_string_as_comment(): + # regex engine may mis-handle this; pygments understands the string. + src = 'url = "http://x#frag" # real\nx = 1\n' + out = pr.remove_comments(src, lang='python', engine='pygments') + assert '"http://x#frag"' in out # string preserved intact + assert '# real' not in out # actual comment removed + + +def test_pygments_extract_strings_whole_literals(): + src = 's = "a#b"\n' + assert pr.extract_strings(src, lang='python', engine='pygments') == ['"a#b"'] + + +def test_pygments_extract_comments(): + src = '# top\ncode = 1 # side\n' + assert pr.extract_comments(src, lang='python', engine='pygments') == ['# top', '# side'] + + +# ------------------------------------------------------------------------- lossless +@pytest.mark.parametrize('lang', ['python', 'java', 'javascript', 'go', 'c', 'ruby']) +def test_pygments_tokenize_is_lossless(lang): + src = 'x = 1 + 2 // c\n# h\nname = "hi"\nreturn x\n' + toks = pr.tokenize(src, lang=lang, engine='pygments') + assert ''.join(t.value for t in toks) == src + + +# ------------------------------------------------------------------------- parity +def test_engine_derived_ops_agree_with_lossless_join(): + # For a token-derived engine, remove_comments == join of non-comment tokens. + src = 'def f():\n return 40 + 2 # c\n' + toks = pr.tokenize(src, lang='python', engine='pygments') + expected = ''.join(t.value for t in toks if t.type.value != 'comment') + assert pr.remove_comments(src, lang='python', engine='pygments') == expected + + +def test_normalize_and_stats_via_pygments(): + src = 'total = price * 42 # c\n' + assert pr.normalize(src, lang='python', engine='pygments') == 'VAR1 = VAR2 * 0 \n' + s = pr.stats(src, lang='python', engine='pygments') + assert s.comment_lines == 0 and s.code_lines == 1 + + +def test_preprocess_with_engine(): + src = 's = "x" + 3 # c\n' + out = pr.preprocess(src, lang='python', steps=['comments', 'strings', 'numbers'], + engine='pygments') + assert '"x"' not in out and '3' not in out and '# c' not in out + + +# ------------------------------------------------------------------------------ CLI +def test_cli_engine_flag(monkeypatch, capsys): + import io + from PyReprism.cli import main + monkeypatch.setattr('sys.stdin', io.StringIO('u = "a#b" # c\n')) + assert main(['remove', 'comments', '--lang', 'python', '--engine', 'pygments']) == 0 + out = capsys.readouterr().out + assert '"a#b"' in out and '# c' not in out From 164740e991c88ecbfadd33a2808d561856b19fb6 Mon Sep 17 00:00:00 2001 From: danielogen Date: Tue, 7 Jul 2026 12:57:36 -0700 Subject: [PATCH 17/19] ci and docs: fix PR/main triggers, add build check + caching, modernize publish to trusted publishing --- .flake8 | 2 +- .github/workflows/ci.yml | 53 +++++++++++++++++------- .github/workflows/publish.yml | 71 ++++++++++++++++++------------- .gitignore | 4 +- .pre-commit-hooks.yaml | 2 +- .readthedocs.yaml | 7 ++-- CHANGELOG.md | 78 +++++++++++++++++++++++++++++++++++ README.md | 31 ++++++++++++++ dev-requirements.txt | 9 ---- docs/api.rst | 30 ++++++++++++++ docs/batch.rst | 26 ++++++++++++ docs/cli.rst | 45 ++++++++++++++++++++ docs/diffs.rst | 31 ++++++++++++++ docs/engines.rst | 22 ++++++++++ docs/index.rst | 9 +++- docs/metrics.rst | 15 +++++++ docs/requirements.txt | 1 - docs/tokens.rst | 15 +++++++ pyproject.toml | 6 ++- requirements.txt | 6 --- 20 files changed, 396 insertions(+), 67 deletions(-) create mode 100644 CHANGELOG.md delete mode 100644 dev-requirements.txt create mode 100644 docs/api.rst create mode 100644 docs/batch.rst create mode 100644 docs/cli.rst create mode 100644 docs/diffs.rst create mode 100644 docs/engines.rst create mode 100644 docs/metrics.rst delete mode 100644 docs/requirements.txt create mode 100644 docs/tokens.rst delete mode 100644 requirements.txt diff --git a/.flake8 b/.flake8 index 9fc97bc..0967758 100644 --- a/.flake8 +++ b/.flake8 @@ -8,4 +8,4 @@ max-line-length = 120 # style debt across the legacy language modules; they are ignored so flake8 can # focus on real correctness issues (pyflakes F-codes, undefined names, etc.). extend-ignore = E501, W191, W291, W292, W293, W391, E101, E117, E128, E231, E302, E303, E265, E266 -exclude = .git,.github,venv,.venv,build,dist,docs,htmlcov,src/tests,src/PyReprism/languages/__init__.py,__test__.py +exclude = .git,.github,venv,.venv,build,dist,docs,htmlcov,tests,src/PyReprism/languages/__init__.py,__test__.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e83635..b6fd257 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,18 @@ name: CI + on: push: - branches-ignore: - - main + branches: [main] pull_request: - branches-ignore: - - main + +# Least-privilege by default. +permissions: + contents: read + +# Cancel superseded runs on the same ref. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true jobs: test: @@ -14,15 +21,17 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - name: Check out the code uses: actions/checkout@v4 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml - name: Install dependencies run: | @@ -30,17 +39,33 @@ jobs: pip install -e ".[dev]" - name: Lint with flake8 - run: | - flake8 src/PyReprism + run: flake8 src/PyReprism - name: Run tests with pytest - run: | - pytest --cov=src/PyReprism --cov-report=xml + run: pytest --cov=src/PyReprism --cov-report=xml - name: Upload coverage to Codecov if: matrix.python-version == '3.12' - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: - verbose: true - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + fail_ci_if_error: false + + build: + name: "Build & check package" + runs-on: ubuntu-latest + steps: + - name: Check out the code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build and validate the distribution + run: | + python -m pip install --upgrade pip build twine + python -m build + twine check dist/* diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f55fe5d..415b86a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,38 +1,53 @@ -name: Publish Python 🐍 distributions 📦 to PyPI +name: Publish 📦 to PyPI + on: push: tags: - - 'v*' - branches: - - main - - stable + - 'v*' -jobs: - publish: - name: "Build and upload to pypi" - runs-on: "ubuntu-latest" - # environment: - # name: pypi - # url: https://pypi.org/project/PyReprism/ - # permissions: - # id-token: write +permissions: + contents: read +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Install python - uses: actions/setup-python@v4 + - name: Check out the code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install hatchling twine build hatch-vcs - - name: Build app + python-version: '3.12' + + - name: Build and validate the distribution run: | + python -m pip install --upgrade pip build twine python -m build - - name: Publish distribution 📦 to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 + twine check dist/* + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 with: - password: ${{ secrets.PYPI_API_TOKEN }} - # run: twine upload dist/* \ No newline at end of file + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/PyReprism/ + permissions: + id-token: write # Trusted Publishing (OIDC) — no API token needed. + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index 6639b20..837e50e 100644 --- a/.gitignore +++ b/.gitignore @@ -166,4 +166,6 @@ cython_debug/ __test__.py # macOS -.DS_Store \ No newline at end of file +.DS_Store +# Claude Code agent workspace +.claude/ diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index a4e36c8..2af67a8 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -3,7 +3,7 @@ # Add to your .pre-commit-config.yaml, e.g.: # # - repo: https://github.com/unlv-evol/PyReprism -# rev: v0.0.4 +# rev: v0.1.0 # hooks: # - id: pyreprism-strip-comments # files: ^generated/ # scope it! these hooks rewrite files diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d13bca4..8df1847 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -23,9 +23,10 @@ sphinx: # formats: # - pdf -# Optionally declare the Python requirements required to build your docs +# Install the package together with its documentation extra. python: install: - - requirements: docs/requirements.txt - method: pip - path: . \ No newline at end of file + path: . + extra_requirements: + - docs \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d825142 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,78 @@ +# Changelog + +All notable changes to PyReprism are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- **Diff processing** (`PyReprism.diffs`): parse unified/`git` diffs and analyze + the changed code per file in its own language. Includes churn metrics + (`diff_stats`: added/removed split into code, comment and blank), cosmetic + (comment/whitespace-only) change detection, reconstruction of the added/removed + code, and ML-oriented `tokenize`/`normalize`/`extract_comments` on changes. + Fragment mode works from the diff alone; setting a file's `new_source` / + `old_source` enables accurate full-file classification. New CLI command + `pyreprism diff` (`--json` / `--csv` / `--per-file` / `--cosmetic`). + +## [0.1.0] - 2026-07-07 + +This release turns PyReprism from a comment-removal helper into a full +source-preprocessing toolkit with a high-level API, a CLI, code metrics, +ML-oriented normalization, an optional accurate backend, and batch processing. + +### Added +- **High-level API** (`import PyReprism as pr`): `remove_*`, `extract_*`, + `count_*`, `match_*` for comments, keywords, numbers, operators, strings and + identifiers; plus `preprocess`, `remove_whitespaces`, `get_language` and + `detect_language`. `lang=` accepts a language name, file extension, or class. +- **Tokenizer**: `tokenize()` on every language returns a lossless, typed + `Token` stream (`Token`/`TokenType` in `PyReprism.tokens`). +- **Code metrics**: `stats()` returns a `CodeStats` (line/token counts, + comment-to-code ratio, comment density; `as_dict()` for JSON/dataframes). +- **ML normalization**: `normalize()` canonicalizes code (rename identifiers to + `VAR1…`, mask string/number literals, drop comments) for clone/plagiarism + detection and code-embedding pipelines; `blank_comments()` strips comments + while preserving line numbers. +- **Pluggable backends**: `engine="regex"` (default, zero-dependency) or + `engine="pygments"` (`pip install pyreprism[accurate]`) for higher accuracy, + plus `"auto"`. Available on every operation. +- **Batch processing** (`PyReprism.batch`): `analyze()` walks a directory tree + and returns an aggregate report (`to_json`/`to_csv`, per-language breakdown); + `transform()` bulk-applies a transformation into a mirrored output directory. +- **Command-line interface** (`pyreprism`, also `python -m PyReprism`): + `remove`, `extract`, `count`, `preprocess`, `tokenize`, `stats`, `normalize`, + `scan`, and `languages`; reads stdin/files/globs/directories; supports + `--lang`, `--engine`, `--in-place`, `--output`, and `--json`/`--csv`. +- **Source-based language detection**: `detect_language(source=...)` recognizes + `#!` shebangs and (when Pygments is installed) makes a best-effort content + guess; the CLI uses this to auto-detect the language of piped input. +- Pre-commit hooks (`.pre-commit-hooks.yaml`), a `py.typed` marker (PEP 561), + and a registry-wide test suite covering all supported languages. + +### Changed +- Implemented real support for 25 previously-empty language stubs and migrated + 22 legacy languages (HTML, JSON, YAML, CSS, Markdown, …) to the + `BaseLanguage` + registry pattern, making them importable via the public API. +- Ambiguous file extensions now resolve to a canonical language + (`.py` → Python, `.js` → JavaScript, …). +- Consolidated packaging into `pyproject.toml` (single version source, pytest + and coverage config), updated CI to a Python 3.8–3.12 matrix, and refreshed + the README. + +### Fixed +- Corrected comment-stripping for ~30 languages whose regexes either failed to + remove line comments or consumed surrounding code, and fixed duplicate/broken + definitions (Perl, csp, glsl, django, apacheconf, php_extras, ruby, and the + Python triple-quoted-docstring bug). + +## [0.0.4] - 2024 + +- Early beta releases: comment removal for an initial set of languages and the + `Normalizer` whitespace helper. + +[Unreleased]: https://github.com/unlv-evol/PyReprism/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.1.0 +[0.0.4]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.0.4 diff --git a/README.md b/README.md index 6e55086..786cae6 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,37 @@ batch.transform("myproject/", lambda text, lang: lang.remove_comments(text), output="stripped/") ``` +### Process diffs / pull requests + +Parse a unified (`git`) diff and analyze the *changed code* per file, language-aware: + +```python +from PyReprism import diffs + +d = diffs.parse(diff_text) # git diff / .diff / .patch text + +report = diffs.diff_stats(d) # churn split into code vs comment vs blank +report.totals(); report.to_csv() + +diffs.cosmetic_files(d) # files whose change is comment/whitespace-only + +f = d.files[0] +f.language # detected from the file path +f.added_text(); f.removed_text() # reconstructed changed code +f.normalize("added") # canonicalize the added code for ML +f.extract_comments("added") +``` + +For accuracy where a comment/string spans a hunk boundary, set +`f.new_source` / `f.old_source` to the full file contents (e.g. from `git show`) +and the changed lines are classified against the whole file. On the CLI: + +```shell +git diff | pyreprism diff --per-file # churn report +git diff | pyreprism diff --json +git diff | pyreprism diff --cosmetic # list comment/whitespace-only changes +``` + ### Detect the language from a filename ```python diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 57a33a2..0000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -pytest -flake8 -pytest-cov -build -twine -hatchling -hatch-vcs -sphinx -sphinx_rtd_theme \ No newline at end of file diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..f60a3b1 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,30 @@ +.. _api_toplevel: + +================ +High-level API +================ + +The top-level :mod:`PyReprism` package exposes a convenience API. Import it as +``pr`` and pass ``lang`` as a language name (``"python"``), a file extension +(``".py"``), or a language class:: + + import PyReprism as pr + + pr.remove_comments(source, lang="python") + pr.extract_comments(source, lang="python") + pr.count_comments(source, lang="python") + pr.tokenize(source, lang="python") + pr.normalize(source, lang="python") + pr.stats(source, lang="python") + pr.preprocess(source, lang="java", steps=["comments", "strings", "whitespace"]) + +Every operation accepts ``engine="regex"`` (default), ``engine="pygments"`` +(requires ``pip install pyreprism[accurate]``), or ``engine="auto"``. + +The re-exported ``Token``, ``TokenType``, ``CodeStats`` and ``Normalizer`` are +documented on their own pages (:doc:`tokens`, :doc:`metrics`, :doc:`normalizer`). + +.. automodule:: PyReprism + :members: + :undoc-members: + :exclude-members: __dict__, __weakref__, Token, TokenType, CodeStats, Normalizer diff --git a/docs/batch.rst b/docs/batch.rst new file mode 100644 index 0000000..2428f86 --- /dev/null +++ b/docs/batch.rst @@ -0,0 +1,26 @@ +.. _batch_toplevel: + +============================ +Batch / directory processing +============================ + +The :mod:`PyReprism.batch` module analyzes or transforms whole source trees. +Discovery skips junk folders (``.git``, ``node_modules``, ``venv``, …) and files +with unrecognized extensions:: + + from PyReprism import batch + + report = batch.analyze("myproject/") + report.totals() + report.by_language() + print(report.to_csv()) + + batch.transform("myproject/", + lambda text, lang: lang.remove_comments(text), + output="stripped/") + +.. automodule:: PyReprism.batch + :members: + :undoc-members: + :show-inheritance: + :exclude-members: __dict__, __weakref__ diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 0000000..d8250b7 --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,45 @@ +.. _cli_toplevel: + +====================== +Command-line interface +====================== + +Installing PyReprism provides a ``pyreprism`` command (equivalently +``python -m PyReprism``). It reads from stdin, files, globs, or directories, and +auto-detects the language from a file's extension (or a ``#!`` shebang on stdin) +when ``--lang`` is omitted. + +Common usage +============ + +.. code-block:: shell + + pyreprism remove comments file.py + cat file.go | pyreprism remove comments --lang go + pyreprism extract comments "src/**/*.py" --json + pyreprism count comments --lang python file.py + pyreprism preprocess --steps comments,strings,whitespace file.java + pyreprism tokenize --json file.py + pyreprism normalize file.py + pyreprism stats --json file.py + +Directories and reports +======================= + +.. code-block:: shell + + pyreprism scan myproject/ --csv # aggregate metrics over a tree + pyreprism scan myproject/ --json --per-file + pyreprism remove comments src/ --output out/ # bulk-transform, mirroring the tree + pyreprism remove comments file.py --in-place + +Options +======= + +- ``-l, --lang`` — language name, extension, or class. +- ``-e, --engine`` — ``regex`` (default), ``pygments``, or ``auto``. +- ``-o, --output DIR`` — write results into ``DIR``, mirroring the input tree. +- ``-i, --in-place`` — rewrite files in place. +- ``--json`` / ``--csv`` — machine-readable output where supported. + +Run ``pyreprism --help`` or ``pyreprism --help`` for the full list. diff --git a/docs/diffs.rst b/docs/diffs.rst new file mode 100644 index 0000000..ba09cea --- /dev/null +++ b/docs/diffs.rst @@ -0,0 +1,31 @@ +.. _diffs_toplevel: + +=============== +Diff processing +=============== + +The :mod:`PyReprism.diffs` module parses unified/``git`` diffs and analyzes the +changed code in each file's own language. It powers churn metrics (added/removed +lines split into code, comment and blank), cosmetic-change detection, and +ML-oriented extraction of the changed code:: + + from PyReprism import diffs + + d = diffs.parse(diff_text) + report = diffs.diff_stats(d) + report.totals() + diffs.cosmetic_files(d) + + f = d.files[0] + f.added_text() + f.normalize("added") + +By default analysis is *fragment mode* (from the diff text alone). Setting a +:class:`~PyReprism.diffs.DiffFile`'s ``new_source`` / ``old_source`` to the full +file contents switches it to accurate *full-file mode* for that file. + +.. automodule:: PyReprism.diffs + :members: + :undoc-members: + :show-inheritance: + :exclude-members: __dict__, __weakref__ diff --git a/docs/engines.rst b/docs/engines.rst new file mode 100644 index 0000000..da73fe0 --- /dev/null +++ b/docs/engines.rst @@ -0,0 +1,22 @@ +.. _engines_toplevel: + +============ +Engines +============ + +PyReprism can tokenize with different backends. The default ``regex`` engine is +zero-dependency; the optional ``pygments`` engine (``pip install +pyreprism[accurate]``) is more accurate — for example it will not treat a ``#`` +inside a string as a comment. Select one per call with ``engine=``. + +.. automodule:: PyReprism.engines + :members: + :undoc-members: + :show-inheritance: + :exclude-members: __dict__, __weakref__ + +.. automodule:: PyReprism.engines.base + :members: + :undoc-members: + :show-inheritance: + :exclude-members: __dict__, __weakref__ diff --git a/docs/index.rst b/docs/index.rst index ed2c56a..caf7b99 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,8 +5,15 @@ PyReprism documentation! .. toctree:: :maxdepth: 2 - + intro + api + cli + tokens + metrics + engines + batch + diffs language extension normalizer diff --git a/docs/metrics.rst b/docs/metrics.rst new file mode 100644 index 0000000..12c039e --- /dev/null +++ b/docs/metrics.rst @@ -0,0 +1,15 @@ +.. _metrics_toplevel: + +============ +Metrics +============ + +:meth:`~PyReprism.languages.base.BaseLanguage.stats` (and +:func:`PyReprism.stats`) return a :class:`~PyReprism.metrics.CodeStats` with +line- and token-level metrics, suitable for building code datasets. + +.. automodule:: PyReprism.metrics + :members: + :undoc-members: + :show-inheritance: + :exclude-members: __dict__, __weakref__ diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 52b04f2..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -sphinx_rtd_theme \ No newline at end of file diff --git a/docs/tokens.rst b/docs/tokens.rst new file mode 100644 index 0000000..59e260a --- /dev/null +++ b/docs/tokens.rst @@ -0,0 +1,15 @@ +.. _tokens_toplevel: + +=========== +Tokens +=========== + +:meth:`~PyReprism.languages.base.BaseLanguage.tokenize` returns a flat, lossless +list of :class:`~PyReprism.tokens.Token` objects; concatenating every token's +``value`` reproduces the original source exactly. + +.. automodule:: PyReprism.tokens + :members: + :undoc-members: + :show-inheritance: + :exclude-members: __dict__, __weakref__ diff --git a/pyproject.toml b/pyproject.toml index 6abf031..09ededf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "PyReprism" -version = "0.0.4" +version = "0.1.0" authors = [ { name = "UNLV EVOL LAB", email = "ogenrwot@unlv.nevada.edu" }, ] @@ -36,6 +36,8 @@ dev = [ "pytest", "pytest-cov", "pygments", + "build", + "twine", ] docs = [ "sphinx", @@ -61,7 +63,7 @@ exclude = [ ] [tool.pytest.ini_options] -testpaths = ["src/tests"] +testpaths = ["tests"] addopts = "--cov=src/PyReprism --cov-report=term" [tool.coverage.run] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index ffea41e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# PyReprism has no runtime dependencies (Python standard library only). -# -# For development tooling (tests, linting) install the project with its -# optional "dev" extras instead: -# -# pip install -e ".[dev]" From 7e6311959b51f0b8ef9542981115e9b763708546 Mon Sep 17 00:00:00 2001 From: danielogen Date: Tue, 7 Jul 2026 12:57:57 -0700 Subject: [PATCH 18/19] fix: updates --- src/PyReprism/__init__.py | 71 +++- src/PyReprism/cli.py | 73 +++- src/PyReprism/diffs.py | 391 ++++++++++++++++++ {src/tests => tests}/__init__.py | 0 {src/tests => tests}/languages/__init__.py | 0 {src/tests => tests}/languages/test_abap.py | 0 .../languages/test_actionscript.py | 0 {src/tests => tests}/languages/test_c.py | 0 {src/tests => tests}/languages/test_cpp.py | 0 {src/tests => tests}/languages/test_dart.py | 0 {src/tests => tests}/languages/test_django.py | 0 {src/tests => tests}/languages/test_go.py | 0 {src/tests => tests}/languages/test_html.py | 0 {src/tests => tests}/languages/test_java.py | 0 .../languages/test_javascript.py | 0 {src/tests => tests}/languages/test_kotlin.py | 0 .../languages/test_new_languages.py | 0 {src/tests => tests}/languages/test_python.py | 0 {src/tests => tests}/languages/test_rust.py | 0 {src/tests => tests}/test_api.py | 0 {src/tests => tests}/test_batch.py | 0 {src/tests => tests}/test_cli.py | 0 tests/test_detect.py | 70 ++++ tests/test_diffs.py | 159 +++++++ {src/tests => tests}/test_engines.py | 0 {src/tests => tests}/test_metrics.py | 0 {src/tests => tests}/test_registry.py | 0 {src/tests => tests}/utils/__init__.py | 0 {src/tests => tests}/utils/test_extensions.py | 0 {src/tests => tests}/utils/test_normalizer.py | 0 30 files changed, 751 insertions(+), 13 deletions(-) create mode 100644 src/PyReprism/diffs.py rename {src/tests => tests}/__init__.py (100%) rename {src/tests => tests}/languages/__init__.py (100%) rename {src/tests => tests}/languages/test_abap.py (100%) rename {src/tests => tests}/languages/test_actionscript.py (100%) rename {src/tests => tests}/languages/test_c.py (100%) rename {src/tests => tests}/languages/test_cpp.py (100%) rename {src/tests => tests}/languages/test_dart.py (100%) rename {src/tests => tests}/languages/test_django.py (100%) rename {src/tests => tests}/languages/test_go.py (100%) rename {src/tests => tests}/languages/test_html.py (100%) rename {src/tests => tests}/languages/test_java.py (100%) rename {src/tests => tests}/languages/test_javascript.py (100%) rename {src/tests => tests}/languages/test_kotlin.py (100%) rename {src/tests => tests}/languages/test_new_languages.py (100%) rename {src/tests => tests}/languages/test_python.py (100%) rename {src/tests => tests}/languages/test_rust.py (100%) rename {src/tests => tests}/test_api.py (100%) rename {src/tests => tests}/test_batch.py (100%) rename {src/tests => tests}/test_cli.py (100%) create mode 100644 tests/test_detect.py create mode 100644 tests/test_diffs.py rename {src/tests => tests}/test_engines.py (100%) rename {src/tests => tests}/test_metrics.py (100%) rename {src/tests => tests}/test_registry.py (100%) rename {src/tests => tests}/utils/__init__.py (100%) rename {src/tests => tests}/utils/test_extensions.py (100%) rename {src/tests => tests}/utils/test_normalizer.py (100%) diff --git a/src/PyReprism/__init__.py b/src/PyReprism/__init__.py index 839a8b1..07c8260 100644 --- a/src/PyReprism/__init__.py +++ b/src/PyReprism/__init__.py @@ -21,7 +21,7 @@ from .tokens import Token, TokenType from .utils.normalizer import Normalizer -__version__ = "0.0.4" +__version__ = "0.1.0" LanguageLike = Union[str, "type"] @@ -63,18 +63,79 @@ def get_language(lang: LanguageLike) -> Type: raise ValueError(f"Unknown language: {lang!r}") +# Interpreter substrings found in a ``#!`` shebang -> registry class name. +_SHEBANG_LANGUAGES = [ + ('python', 'Python'), ('pypy', 'Python'), + ('nodejs', 'JavaScript'), ('node', 'JavaScript'), + ('ruby', 'Ruby'), ('perl', 'Perl'), ('php', 'PHP'), + ('rscript', 'R'), ('groovy', 'Groovy'), ('lua', 'LUA'), + ('tclsh', 'Tcl'), ('wish', 'Tcl'), + ('bash', 'Bash'), ('zsh', 'Bash'), ('ksh', 'Bash'), ('/sh', 'Bash'), +] + + +def _detect_from_shebang(source: str) -> Optional[Type]: + stripped = source.lstrip() + first = stripped.splitlines()[0] if stripped else '' + if not first.startswith('#!'): + return None + from .languages import _load_all_languages + from .languages.registry import LanguageRegistry + line = first.lower() + for token, name in _SHEBANG_LANGUAGES: + if token in line: + _load_all_languages() + cls = LanguageRegistry.get(name) + if cls is not None: + return cls + return None + + +def _guess_with_pygments(source: str) -> Optional[Type]: + try: + from pygments.lexers import guess_lexer + from pygments.util import ClassNotFound + except ImportError: + return None + from .languages import get_language_by_extension + try: + lexer = guess_lexer(source) + except ClassNotFound: + return None + for alias in getattr(lexer, 'aliases', []): + try: + return get_language(alias) + except ValueError: + continue + for pattern in getattr(lexer, 'filenames', []): + ext = os.path.splitext(pattern)[1] + if ext: + cls = get_language_by_extension(ext) + if cls is not None: + return cls + return None + + def detect_language(filename: Optional[str] = None, source: Optional[str] = None) -> Optional[Type]: - """Infer a language class from ``filename`` (by extension or basename). + """Infer a language class from a ``filename`` and/or a ``source`` string. - ``source``-based detection is not yet implemented and is accepted for - forward compatibility. Returns ``None`` when no language matches. + Resolution order: filename extension, then (if ``source`` is given) a ``#!`` + shebang line, then a best-effort Pygments content guess when Pygments is + installed. Returns ``None`` when nothing matches. """ from .languages import get_language_by_extension if filename: base = os.path.basename(filename) _, ext = os.path.splitext(base) - return get_language_by_extension(ext or base) + cls = get_language_by_extension(ext or base) + if cls is not None: + return cls + if source: + cls = _detect_from_shebang(source) + if cls is not None: + return cls + return _guess_with_pygments(source) return None diff --git a/src/PyReprism/cli.py b/src/PyReprism/cli.py index 0f83c16..6f7fbaa 100644 --- a/src/PyReprism/cli.py +++ b/src/PyReprism/cli.py @@ -64,13 +64,17 @@ def _iter_inputs(paths: List[str]): print(f"pyreprism: {exc}", file=sys.stderr) -def _resolve(lang: Optional[str], filename: Optional[str]): +def _resolve(lang: Optional[str], filename: Optional[str], source: Optional[str] = None): if lang: return get_language(lang) if filename: cls = detect_language(filename=filename) if cls: return cls + if source is not None: + cls = detect_language(source=source) + if cls: + return cls print("pyreprism: could not determine language; pass --lang", file=sys.stderr) raise SystemExit(2) @@ -94,7 +98,7 @@ def _op(cls, action: str, construct: str, text: str, engine: str): def _cmd_remove(args) -> int: for label, text, filename, base in _iter_inputs(args.paths): - cls = _resolve(args.lang, filename) + cls = _resolve(args.lang, filename, text) result = _op(cls, 'remove', args.construct, text, args.engine) _emit(result, filename, base, args) return 0 @@ -103,7 +107,7 @@ def _cmd_remove(args) -> int: def _cmd_preprocess(args) -> int: steps = [s.strip() for s in args.steps.split(',') if s.strip()] for label, text, filename, base in _iter_inputs(args.paths): - cls = _resolve(args.lang, filename) + cls = _resolve(args.lang, filename, text) result = preprocess(text, lang=cls, steps=steps, engine=args.engine) _emit(result, filename, base, args) return 0 @@ -112,7 +116,7 @@ def _cmd_preprocess(args) -> int: def _cmd_extract(args) -> int: multiple = _multiple(args.paths) for label, text, filename, base in _iter_inputs(args.paths): - cls = _resolve(args.lang, filename) + cls = _resolve(args.lang, filename, text) items = _op(cls, 'extract', args.construct, text, args.engine) if args.json: print(json.dumps({label: items} if multiple else items)) @@ -126,7 +130,7 @@ def _cmd_extract(args) -> int: def _cmd_count(args) -> int: for label, text, filename, base in _iter_inputs(args.paths): - cls = _resolve(args.lang, filename) + cls = _resolve(args.lang, filename, text) count = _op(cls, 'count', args.construct, text, args.engine) print(f"{count}\t{label}" if _multiple(args.paths) else count) return 0 @@ -134,7 +138,7 @@ def _cmd_count(args) -> int: def _cmd_tokenize(args) -> int: for label, text, filename, base in _iter_inputs(args.paths): - cls = _resolve(args.lang, filename) + cls = _resolve(args.lang, filename, text) tokens = get_engine(args.engine).tokenize(text, cls) if args.json: print(json.dumps([ @@ -151,7 +155,7 @@ def _cmd_tokenize(args) -> int: def _cmd_stats(args) -> int: multiple = _multiple(args.paths) for label, text, filename, base in _iter_inputs(args.paths): - cls = _resolve(args.lang, filename) + cls = _resolve(args.lang, filename, text) if args.engine in (None, 'regex'): data = cls.stats(text).as_dict() else: @@ -176,7 +180,7 @@ def _cmd_normalize(args) -> int: collapse_whitespace=args.collapse_whitespace, ) for label, text, filename, base in _iter_inputs(args.paths): - cls = _resolve(args.lang, filename) + cls = _resolve(args.lang, filename, text) if args.engine in (None, 'regex'): result = cls.normalize(text, **options) else: @@ -218,6 +222,48 @@ def _cmd_scan(args) -> int: return 0 +def _cmd_diff(args) -> int: + from .diffs import cosmetic_files, diff_stats, parse + + if args.paths: + texts = [] + for path in args.paths: + try: + with open(path, 'r', encoding='utf-8', errors='replace') as handle: + texts.append(handle.read()) + except OSError as exc: + print(f"pyreprism: {exc}", file=sys.stderr) + diff = parse('\n'.join(texts)) + else: + diff = parse(sys.stdin.read()) + + if args.cosmetic: + for f in cosmetic_files(diff): + print(f.path) + return 0 + + report = diff_stats(diff) + if args.json: + print(report.to_json()) + return 0 + if args.csv: + print(report.to_csv(), end='') + return 0 + + if args.per_file: + for f in report.files: + tag = ' [binary]' if f.is_binary else '' + print(f"{f.path}\t{f.language}\t+{f.added_code}/-{f.removed_code} code, " + f"+{f.added_comment}/-{f.removed_comment} comment{tag}") + print() + totals = report.totals() + print(f"{totals['files']} files: " + f"+{totals['added_code']}/-{totals['removed_code']} code, " + f"+{totals['added_comment']}/-{totals['removed_comment']} comment, " + f"+{totals['added_blank']}/-{totals['removed_blank']} blank") + return 0 + + def _cmd_languages(args) -> int: from .languages import _load_all_languages from .languages.registry import LanguageRegistry @@ -350,6 +396,17 @@ def add_output(p): help='do not descend into subdirectories') p_scan.set_defaults(func=_cmd_scan) + p_diff = sub.add_parser('diff', + help='analyze a unified/git diff (churn metrics, cosmetic detection)') + p_diff.add_argument('paths', nargs='*', help='diff files (default: read stdin)') + p_diff.add_argument('--json', action='store_true', help='emit the full JSON report') + p_diff.add_argument('--csv', action='store_true', help='emit per-file CSV') + p_diff.add_argument('--per-file', action='store_true', + help='include a per-file breakdown in the text report') + p_diff.add_argument('--cosmetic', action='store_true', + help='list only files whose change is comment/whitespace-only') + p_diff.set_defaults(func=_cmd_diff) + p_langs = sub.add_parser('languages', help='list supported languages and extensions') p_langs.set_defaults(func=_cmd_languages) diff --git a/src/PyReprism/diffs.py b/src/PyReprism/diffs.py new file mode 100644 index 0000000..ee4dc64 --- /dev/null +++ b/src/PyReprism/diffs.py @@ -0,0 +1,391 @@ +"""Process unified/``git`` diffs with PyReprism's language-aware operations. + +A diff is parsed into files and hunks; each file's path is used to detect its +language, so the added/removed code can be tokenized, normalized, or measured. + +Two accuracy modes: + +* **Fragment mode** (default) works from the diff text alone. It is self-contained + but best-effort where a block comment or string spans a hunk boundary. +* **Full-file mode** is used automatically for a :class:`DiffFile` when its + ``new_source`` / ``old_source`` are set (e.g. from ``git show``); changed lines + are then classified against the whole file for accuracy. +""" +import csv +import io +import json +import re +from dataclasses import dataclass, field +from enum import Enum +from typing import Dict, List, Optional + +from .tokens import TokenType + +_HUNK_RE = re.compile(r'^@@+ (?:-\d+(?:,\d+)? )+\+(\d+)(?:,(\d+))? @@+(.*)$') +_HUNK_OLD_RE = re.compile(r'-(\d+)(?:,(\d+))?') + + +class LineKind(str, Enum): + CONTEXT = 'context' + ADDED = 'added' + REMOVED = 'removed' + + +@dataclass(frozen=True) +class DiffLine: + kind: LineKind + text: str # content without the +/-/space prefix + old_lineno: Optional[int] + new_lineno: Optional[int] + + +@dataclass +class Hunk: + old_start: int + new_start: int + section: str = '' + lines: List[DiffLine] = field(default_factory=list) + + +@dataclass +class DiffFile: + old_path: Optional[str] = None + new_path: Optional[str] = None + hunks: List[Hunk] = field(default_factory=list) + is_binary: bool = False + is_new: bool = False + is_deleted: bool = False + is_rename: bool = False + # Optional whole-file contents for accurate (full-file) classification. + new_source: Optional[str] = None + old_source: Optional[str] = None + + @property + def path(self) -> Optional[str]: + """The most representative path (new side, falling back to old side).""" + return self.new_path or self.old_path + + @property + def language(self): + """Detected language class for this file, or ``None``.""" + from . import detect_language + if self.path: + return detect_language(filename=self.path) + return None + + # -- reconstruction ----------------------------------------------------- + def _collect(self, kinds) -> List[DiffLine]: + return [dl for h in self.hunks for dl in h.lines if dl.kind in kinds] + + def added_text(self) -> str: + """Text of the added (``+``) lines.""" + return '\n'.join(dl.text for dl in self._collect((LineKind.ADDED,))) + + def removed_text(self) -> str: + """Text of the removed (``-``) lines.""" + return '\n'.join(dl.text for dl in self._collect((LineKind.REMOVED,))) + + def new_text(self) -> str: + """The new side of the hunks (context + added lines).""" + return '\n'.join(dl.text for dl in self._collect((LineKind.CONTEXT, LineKind.ADDED))) + + def old_text(self) -> str: + """The old side of the hunks (context + removed lines).""" + return '\n'.join(dl.text for dl in self._collect((LineKind.CONTEXT, LineKind.REMOVED))) + + def changed_text(self, side: str = 'added') -> str: + return self.added_text() if side == 'added' else self.removed_text() + + # -- language-aware analyses ------------------------------------------- + def tokenize(self, side: str = 'added'): + """Tokenize the changed code on ``side`` (``'added'`` or ``'removed'``).""" + lang = self.language + if lang is None: + return [] + return lang.tokenize(self.changed_text(side)) + + def normalize(self, side: str = 'added', **options) -> str: + """Canonicalize the changed code on ``side`` for ML/clone detection.""" + lang = self.language + text = self.changed_text(side) + return lang.normalize(text, **options) if lang is not None else text + + def extract_comments(self, side: str = 'added') -> List[str]: + lang = self.language + if lang is None: + return [] + return lang.extract_comments(self.changed_text(side)) + + +@dataclass +class Diff: + files: List[DiffFile] = field(default_factory=list) + + def __iter__(self): + return iter(self.files) + + def __len__(self): + return len(self.files) + + +# --------------------------------------------------------------------- parsing +def _clean_path(raw: str) -> Optional[str]: + path = raw.split('\t', 1)[0].strip() + if path == '/dev/null': + return None + if path.startswith(('a/', 'b/')): + path = path[2:] + return path or None + + +def parse(text: str) -> Diff: + """Parse unified/``git`` diff ``text`` into a :class:`Diff`.""" + files: List[DiffFile] = [] + current: Optional[DiffFile] = None + hunk: Optional[Hunk] = None + rem_old = rem_new = 0 + old_ln = new_ln = 0 + + def start_file() -> DiffFile: + f = DiffFile() + files.append(f) + return f + + for line in text.splitlines(): + if line.startswith('diff --git'): + current = start_file() + hunk = None + parts = line.split(' ') + if len(parts) >= 4: + current.old_path = _clean_path(parts[-2]) + current.new_path = _clean_path(parts[-1]) + continue + if line.startswith('--- '): + if current is None or current.hunks or hunk is not None: + current = start_file() + current.old_path = _clean_path(line[4:]) + hunk = None + continue + if line.startswith('+++ '): + if current is None: + current = start_file() + current.new_path = _clean_path(line[4:]) + continue + if current is not None: + if line.startswith('new file'): + current.is_new = True + continue + if line.startswith('deleted file'): + current.is_deleted = True + continue + if line.startswith('rename from '): + current.is_rename = True + current.old_path = _clean_path(line[len('rename from '):]) + continue + if line.startswith('rename to '): + current.is_rename = True + current.new_path = _clean_path(line[len('rename to '):]) + continue + if line.startswith('Binary files') or line.startswith('GIT binary patch'): + current.is_binary = True + continue + if line.startswith(('index ', 'similarity ', 'copy ', 'old mode', 'new mode', + 'dissimilarity')): + continue + match = _HUNK_RE.match(line) + if match and current is not None: + old_match = _HUNK_OLD_RE.search(line) + old_start = int(old_match.group(1)) if old_match else 0 + old_count = int(old_match.group(2)) if old_match and old_match.group(2) else 1 + new_start = int(match.group(1)) + new_count = int(match.group(2)) if match.group(2) else 1 + hunk = Hunk(old_start=old_start, new_start=new_start, section=match.group(3).strip()) + current.hunks.append(hunk) + rem_old, rem_new = old_count, new_count + old_ln, new_ln = old_start, new_start + continue + if hunk is not None and (rem_old > 0 or rem_new > 0): + if line.startswith('\\'): # "\ No newline at end of file" + continue + prefix = line[:1] + content = line[1:] + if prefix == '+': + hunk.lines.append(DiffLine(LineKind.ADDED, content, None, new_ln)) + new_ln += 1 + rem_new -= 1 + elif prefix == '-': + hunk.lines.append(DiffLine(LineKind.REMOVED, content, old_ln, None)) + old_ln += 1 + rem_old -= 1 + elif prefix == ' ' or line == '': + hunk.lines.append(DiffLine(LineKind.CONTEXT, content, old_ln, new_ln)) + old_ln += 1 + new_ln += 1 + rem_old -= 1 + rem_new -= 1 + else: + hunk = None + return Diff(files) + + +# ------------------------------------------------------------------- analyses +def _line_classes(source: str, lang) -> Dict[int, str]: + """Map 1-based line number -> 'code' | 'comment' | 'blank' for ``source``.""" + classes: Dict[int, str] = {} + total = len(source.splitlines()) + if lang is None: + for i, line in enumerate(source.splitlines(), 1): + classes[i] = 'code' if line.strip() else 'blank' + return classes + code, comment = set(), set() + for tok in lang.tokenize(source): + if tok.type is TokenType.WHITESPACE: + continue + span = range(tok.line, tok.line + tok.value.count('\n') + 1) + (comment if tok.type is TokenType.COMMENT else code).update(span) + for i in range(1, total + 1): + if i in code: + classes[i] = 'code' + elif i in comment: + classes[i] = 'comment' + else: + classes[i] = 'blank' + return classes + + +def _side_counts(file: DiffFile, side: str) -> Dict[str, int]: + lang = file.language + kind = LineKind.ADDED if side == 'added' else LineKind.REMOVED + lines = [dl for h in file.hunks for dl in h.lines if dl.kind is kind] + source = file.new_source if side == 'added' else file.old_source + counts = {'code': 0, 'comment': 0, 'blank': 0} + + if source is not None: + classes = _line_classes(source, lang) + for dl in lines: + lineno = dl.new_lineno if side == 'added' else dl.old_lineno + counts[classes.get(lineno, 'code')] += 1 + return counts + + # Fragment mode: classify the reconstructed blob. + if not lines: + return counts + blob = '\n'.join(dl.text for dl in lines) + if lang is None: + nonblank = sum(1 for dl in lines if dl.text.strip()) + counts['code'] = nonblank + counts['blank'] = len(lines) - nonblank + return counts + stats = lang.stats(blob) + counts['code'] = stats.code_lines + counts['comment'] = stats.comment_lines + counts['blank'] = stats.blank_lines + return counts + + +@dataclass +class DiffFileStats: + path: Optional[str] + language: Optional[str] + added_code: int = 0 + added_comment: int = 0 + added_blank: int = 0 + removed_code: int = 0 + removed_comment: int = 0 + removed_blank: int = 0 + is_binary: bool = False + + @property + def added(self) -> int: + return self.added_code + self.added_comment + self.added_blank + + @property + def removed(self) -> int: + return self.removed_code + self.removed_comment + self.removed_blank + + def as_dict(self) -> dict: + data = {k: getattr(self, k) for k in ( + 'path', 'language', 'added_code', 'added_comment', 'added_blank', + 'removed_code', 'removed_comment', 'removed_blank', 'is_binary')} + data['added'] = self.added + data['removed'] = self.removed + return data + + +_STAT_KEYS = ('added_code', 'added_comment', 'added_blank', + 'removed_code', 'removed_comment', 'removed_blank') + + +@dataclass +class DiffReport: + files: List[DiffFileStats] = field(default_factory=list) + + def totals(self) -> Dict[str, int]: + total = {k: 0 for k in _STAT_KEYS} + total['files'] = len(self.files) + for f in self.files: + for k in _STAT_KEYS: + total[k] += getattr(f, k) + total['added'] = sum(getattr(f, 'added') for f in self.files) + total['removed'] = sum(getattr(f, 'removed') for f in self.files) + return total + + def to_dict(self) -> dict: + return {'files': [f.as_dict() for f in self.files], 'totals': self.totals()} + + def to_json(self, indent: int = 2) -> str: + return json.dumps(self.to_dict(), indent=indent) + + def to_csv(self) -> str: + buffer = io.StringIO() + writer = csv.writer(buffer) + writer.writerow(['path', 'language', *_STAT_KEYS, 'is_binary']) + for f in self.files: + writer.writerow([f.path, f.language, *[getattr(f, k) for k in _STAT_KEYS], + f.is_binary]) + return buffer.getvalue() + + +def diff_stats(diff: Diff) -> DiffReport: + """Compute per-file added/removed churn split into code / comment / blank.""" + rows = [] + for file in diff.files: + lang = file.language + lang_name = lang.__name__ if lang is not None else None + if file.is_binary: + rows.append(DiffFileStats(file.path, lang_name, is_binary=True)) + continue + added = _side_counts(file, 'added') + removed = _side_counts(file, 'removed') + rows.append(DiffFileStats( + file.path, lang_name, + added_code=added['code'], added_comment=added['comment'], added_blank=added['blank'], + removed_code=removed['code'], removed_comment=removed['comment'], + removed_blank=removed['blank'])) + return DiffReport(rows) + + +def _code_only(lang, text: str) -> str: + from . import remove_whitespaces + if lang is None: + return remove_whitespaces(text) + return remove_whitespaces(lang.remove_comments(text)) + + +def is_cosmetic_change(file: DiffFile) -> bool: + """True if the file's change is comments/whitespace only (no code change).""" + if file.is_binary: + return False + lang = file.language + if file.new_source is not None and file.old_source is not None: + old, new = file.old_source, file.new_source + else: + old, new = file.old_text(), file.new_text() + if not file.hunks: + return False + return _code_only(lang, old) == _code_only(lang, new) + + +def cosmetic_files(diff: Diff) -> List[DiffFile]: + """Return the files whose change is comment/whitespace only.""" + return [f for f in diff.files if is_cosmetic_change(f)] diff --git a/src/tests/__init__.py b/tests/__init__.py similarity index 100% rename from src/tests/__init__.py rename to tests/__init__.py diff --git a/src/tests/languages/__init__.py b/tests/languages/__init__.py similarity index 100% rename from src/tests/languages/__init__.py rename to tests/languages/__init__.py diff --git a/src/tests/languages/test_abap.py b/tests/languages/test_abap.py similarity index 100% rename from src/tests/languages/test_abap.py rename to tests/languages/test_abap.py diff --git a/src/tests/languages/test_actionscript.py b/tests/languages/test_actionscript.py similarity index 100% rename from src/tests/languages/test_actionscript.py rename to tests/languages/test_actionscript.py diff --git a/src/tests/languages/test_c.py b/tests/languages/test_c.py similarity index 100% rename from src/tests/languages/test_c.py rename to tests/languages/test_c.py diff --git a/src/tests/languages/test_cpp.py b/tests/languages/test_cpp.py similarity index 100% rename from src/tests/languages/test_cpp.py rename to tests/languages/test_cpp.py diff --git a/src/tests/languages/test_dart.py b/tests/languages/test_dart.py similarity index 100% rename from src/tests/languages/test_dart.py rename to tests/languages/test_dart.py diff --git a/src/tests/languages/test_django.py b/tests/languages/test_django.py similarity index 100% rename from src/tests/languages/test_django.py rename to tests/languages/test_django.py diff --git a/src/tests/languages/test_go.py b/tests/languages/test_go.py similarity index 100% rename from src/tests/languages/test_go.py rename to tests/languages/test_go.py diff --git a/src/tests/languages/test_html.py b/tests/languages/test_html.py similarity index 100% rename from src/tests/languages/test_html.py rename to tests/languages/test_html.py diff --git a/src/tests/languages/test_java.py b/tests/languages/test_java.py similarity index 100% rename from src/tests/languages/test_java.py rename to tests/languages/test_java.py diff --git a/src/tests/languages/test_javascript.py b/tests/languages/test_javascript.py similarity index 100% rename from src/tests/languages/test_javascript.py rename to tests/languages/test_javascript.py diff --git a/src/tests/languages/test_kotlin.py b/tests/languages/test_kotlin.py similarity index 100% rename from src/tests/languages/test_kotlin.py rename to tests/languages/test_kotlin.py diff --git a/src/tests/languages/test_new_languages.py b/tests/languages/test_new_languages.py similarity index 100% rename from src/tests/languages/test_new_languages.py rename to tests/languages/test_new_languages.py diff --git a/src/tests/languages/test_python.py b/tests/languages/test_python.py similarity index 100% rename from src/tests/languages/test_python.py rename to tests/languages/test_python.py diff --git a/src/tests/languages/test_rust.py b/tests/languages/test_rust.py similarity index 100% rename from src/tests/languages/test_rust.py rename to tests/languages/test_rust.py diff --git a/src/tests/test_api.py b/tests/test_api.py similarity index 100% rename from src/tests/test_api.py rename to tests/test_api.py diff --git a/src/tests/test_batch.py b/tests/test_batch.py similarity index 100% rename from src/tests/test_batch.py rename to tests/test_batch.py diff --git a/src/tests/test_cli.py b/tests/test_cli.py similarity index 100% rename from src/tests/test_cli.py rename to tests/test_cli.py diff --git a/tests/test_detect.py b/tests/test_detect.py new file mode 100644 index 0000000..fbb0d1d --- /dev/null +++ b/tests/test_detect.py @@ -0,0 +1,70 @@ +"""Tests for language detection (filename, shebang, content).""" +import io + +import pytest + +import PyReprism as pr +from PyReprism.cli import main + + +# ----------------------------------------------------------------- by filename +def test_detect_by_filename(): + assert pr.detect_language(filename="main.go").__name__ == 'Go' + assert pr.detect_language(filename="a/b/c.rs").__name__ == 'Rust' + + +def test_detect_filename_unknown_returns_none(): + assert pr.detect_language(filename="notes.zzz") is None + + +def test_detect_py_is_python_not_django(): + assert pr.detect_language(filename="script.py").__name__ == 'Python' + + +# ------------------------------------------------------------------- by shebang +@pytest.mark.parametrize('shebang, expected', [ + ('#!/usr/bin/env python3\n', 'Python'), + ('#!/usr/bin/python\n', 'Python'), + ('#!/usr/bin/env node\n', 'JavaScript'), + ('#!/usr/bin/ruby\n', 'Ruby'), + ('#!/usr/bin/perl\n', 'Perl'), + ('#!/bin/bash\n', 'Bash'), + ('#!/bin/sh\n', 'Bash'), +]) +def test_detect_by_shebang(shebang, expected): + assert pr.detect_language(source=shebang + 'code here\n').__name__ == expected + + +def test_detect_shebang_takes_priority_over_content(): + # Even if the body looks like something else, the shebang wins. + src = '#!/usr/bin/env python\nint main() { return 0; }\n' + assert pr.detect_language(source=src).__name__ == 'Python' + + +def test_filename_takes_priority_over_shebang(): + src = '#!/usr/bin/env python\ncode\n' + assert pr.detect_language(filename="x.rb", source=src).__name__ == 'Ruby' + + +# -------------------------------------------------------------------- no signal +def test_detect_plain_text_is_none(): + assert pr.detect_language(source="just prose, nothing code-like at all") is None + + +def test_detect_nothing_returns_none(): + assert pr.detect_language() is None + + +# -------------------------------------------------------------------------- CLI +def test_cli_stdin_autodetects_from_shebang(monkeypatch, capsys): + monkeypatch.setattr('sys.stdin', io.StringIO('#!/usr/bin/env python\nx = 1 # c\n')) + assert main(['remove', 'comments']) == 0 # no --lang + out = capsys.readouterr().out + assert '# c' not in out and 'x = 1' in out + + +def test_cli_stdin_without_signal_errors(monkeypatch, capsys): + monkeypatch.setattr('sys.stdin', io.StringIO('plain prose\n')) + with pytest.raises(SystemExit) as excinfo: + main(['remove', 'comments']) + assert excinfo.value.code == 2 diff --git a/tests/test_diffs.py b/tests/test_diffs.py new file mode 100644 index 0000000..f89b0b6 --- /dev/null +++ b/tests/test_diffs.py @@ -0,0 +1,159 @@ +"""Tests for unified/git diff processing.""" +import io +import json + +import pytest + +from PyReprism import diffs +from PyReprism.cli import main +from PyReprism.diffs import LineKind + +GIT_DIFF = '''diff --git a/src/app.py b/src/app.py +index 1234567..89abcde 100644 +--- a/src/app.py ++++ b/src/app.py +@@ -1,5 +1,6 @@ + import os +-x = 1 ++x = 2 # changed value ++# a brand new comment + + def main(): + pass +diff --git a/logo.png b/logo.png +index aaa..bbb 100644 +Binary files a/logo.png and b/logo.png differ +''' + + +# --------------------------------------------------------------------- parsing +def test_parse_basic_structure(): + d = diffs.parse(GIT_DIFF) + assert len(d) == 2 + app, logo = d.files + assert app.new_path == 'src/app.py' + assert app.language.__name__ == 'Python' + assert logo.is_binary is True + assert logo.language is None + + +def test_parse_line_kinds_and_numbers(): + app = diffs.parse(GIT_DIFF).files[0] + added = [dl for h in app.hunks for dl in h.lines if dl.kind is LineKind.ADDED] + removed = [dl for h in app.hunks for dl in h.lines if dl.kind is LineKind.REMOVED] + assert [dl.text for dl in added] == ['x = 2 # changed value', '# a brand new comment'] + assert [dl.text for dl in removed] == ['x = 1'] + assert added[0].new_lineno == 2 # tracked against the new file + assert removed[0].old_lineno == 2 + + +def test_parse_plain_unified_without_git_header(): + text = '--- a/y.py\n+++ b/y.py\n@@ -1 +1 @@\n-x=1\n+x = 1\n' + d = diffs.parse(text) + assert len(d) == 1 and d.files[0].new_path == 'y.py' + + +def test_parse_new_and_deleted_files(): + text = ('diff --git a/new.py b/new.py\nnew file mode 100644\n--- /dev/null\n' + '+++ b/new.py\n@@ -0,0 +1 @@\n+import os\n' + 'diff --git a/gone.py b/gone.py\ndeleted file mode 100644\n' + '--- a/gone.py\n+++ /dev/null\n@@ -1 +0,0 @@\n-old = 1\n') + new, gone = diffs.parse(text).files + assert new.is_new and new.old_path is None and new.new_path == 'new.py' + assert gone.is_deleted and gone.new_path is None and gone.old_path == 'gone.py' + + +# -------------------------------------------------------------- reconstruction +def test_reconstruction_text(): + app = diffs.parse(GIT_DIFF).files[0] + assert app.added_text() == 'x = 2 # changed value\n# a brand new comment' + assert app.removed_text() == 'x = 1' + assert 'import os' in app.new_text() and 'x = 2' in app.new_text() + assert 'x = 1' in app.old_text() + + +def test_language_aware_change_ops(): + app = diffs.parse(GIT_DIFF).files[0] + assert app.extract_comments('added') == ['# changed value', '# a brand new comment'] + assert 'VAR1' in app.normalize('added') # identifiers renamed + assert app.tokenize('added') # non-empty token list + + +# ---------------------------------------------------------------------- churn +def test_diff_stats_splits_code_comment_blank(): + report = diffs.diff_stats(diffs.parse(GIT_DIFF)) + app = report.files[0] + assert app.added_code == 1 and app.added_comment == 1 + assert app.removed_code == 1 + totals = report.totals() + assert totals['files'] == 2 + assert totals['added_code'] == 1 and totals['added_comment'] == 1 + + +def test_diff_report_json_and_csv(): + report = diffs.diff_stats(diffs.parse(GIT_DIFF)) + data = json.loads(report.to_json()) + assert data['totals']['files'] == 2 + assert report.to_csv().splitlines()[0].startswith('path,language,added_code') + + +# ------------------------------------------------------------------- cosmetic +def test_cosmetic_comment_only_change(): + text = '--- a/x.py\n+++ b/x.py\n@@ -1,2 +1,2 @@\n x = 1\n-# old\n+# new\n' + assert diffs.is_cosmetic_change(diffs.parse(text).files[0]) is True + + +def test_cosmetic_whitespace_only_change(): + text = '--- a/x.py\n+++ b/x.py\n@@ -1 +1 @@\n-x=1\n+x = 1\n' + assert diffs.is_cosmetic_change(diffs.parse(text).files[0]) is True + + +def test_real_change_is_not_cosmetic(): + text = '--- a/x.py\n+++ b/x.py\n@@ -1 +1 @@\n-x = 1\n+x = 99\n' + assert diffs.is_cosmetic_change(diffs.parse(text).files[0]) is False + + +def test_cosmetic_files_helper(): + two = ('--- a/a.py\n+++ b/a.py\n@@ -1,2 +1,2 @@\n x = 1\n-# old\n+# new\n' + '--- a/b.py\n+++ b/b.py\n@@ -1 +1 @@\n-y = 1\n+y = 2\n') + cosmetic = diffs.cosmetic_files(diffs.parse(two)) + assert [f.path for f in cosmetic] == ['a.py'] + + +# ------------------------------------------------------------------ full-file +def test_full_file_mode_reclassifies_block_comment(): + text = ('--- a/c.js\n+++ b/c.js\n@@ -2,2 +2,3 @@\n' + ' line inside comment\n' + '+ another inside the block comment\n' + ' more comment text\n') + f = diffs.parse(text).files[0] + # fragment mode: the added line looks like code + frag = diffs._side_counts(f, 'added') + assert frag['code'] == 1 and frag['comment'] == 0 + # full-file mode: accurate classification as comment + f.new_source = ('/*\n line inside comment\n another inside the block comment\n' + ' more comment text\n*/\ncode = 1\n') + accurate = diffs._side_counts(f, 'added') + assert accurate['comment'] == 1 and accurate['code'] == 0 + + +# ------------------------------------------------------------------------ CLI +def test_cli_diff_text_report(monkeypatch, capsys): + monkeypatch.setattr('sys.stdin', io.StringIO(GIT_DIFF)) + assert main(['diff', '--per-file']) == 0 + out = capsys.readouterr().out + assert 'src/app.py' in out and '2 files' in out + + +def test_cli_diff_json(monkeypatch, capsys): + monkeypatch.setattr('sys.stdin', io.StringIO(GIT_DIFF)) + assert main(['diff', '--json']) == 0 + data = json.loads(capsys.readouterr().out) + assert data['totals']['added_comment'] == 1 + + +def test_cli_diff_cosmetic(monkeypatch, capsys): + monkeypatch.setattr('sys.stdin', + io.StringIO('--- a/x.py\n+++ b/x.py\n@@ -1,2 +1,2 @@\n x = 1\n-# old\n+# new\n')) + assert main(['diff', '--cosmetic']) == 0 + assert capsys.readouterr().out.strip() == 'x.py' diff --git a/src/tests/test_engines.py b/tests/test_engines.py similarity index 100% rename from src/tests/test_engines.py rename to tests/test_engines.py diff --git a/src/tests/test_metrics.py b/tests/test_metrics.py similarity index 100% rename from src/tests/test_metrics.py rename to tests/test_metrics.py diff --git a/src/tests/test_registry.py b/tests/test_registry.py similarity index 100% rename from src/tests/test_registry.py rename to tests/test_registry.py diff --git a/src/tests/utils/__init__.py b/tests/utils/__init__.py similarity index 100% rename from src/tests/utils/__init__.py rename to tests/utils/__init__.py diff --git a/src/tests/utils/test_extensions.py b/tests/utils/test_extensions.py similarity index 100% rename from src/tests/utils/test_extensions.py rename to tests/utils/test_extensions.py diff --git a/src/tests/utils/test_normalizer.py b/tests/utils/test_normalizer.py similarity index 100% rename from src/tests/utils/test_normalizer.py rename to tests/utils/test_normalizer.py From c7cd13a2095210de1069ace906176df11944e328 Mon Sep 17 00:00:00 2001 From: danielogen Date: Tue, 7 Jul 2026 14:16:57 -0700 Subject: [PATCH 19/19] =?UTF-8?q?fix:=20use=20argparse-safe=20flag=20order?= =?UTF-8?q?ing=20in=20CLI=20tests/docs=20(Python=20=E2=89=A43.11=20compat)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- docs/cli.rst | 2 +- src/PyReprism/cli.py | 2 +- tests/test_cli.py | 5 ++++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 786cae6..ee861f8 100644 --- a/README.md +++ b/README.md @@ -200,12 +200,12 @@ files, and globs; the language is auto-detected from a file's extension): pyreprism remove comments file.py cat file.go | pyreprism remove comments --lang go pyreprism extract comments "src/**/*.py" --json -pyreprism count comments --lang python file.py +pyreprism count comments file.py --lang python pyreprism preprocess --steps comments,strings,whitespace file.java pyreprism tokenize --json file.py pyreprism stats --json file.py # line/token metrics pyreprism normalize file.py # canonicalize for ML -pyreprism remove comments --in-place file.py # rewrite in place +pyreprism remove comments file.py --in-place # rewrite in place pyreprism scan myproject/ --csv # aggregate metrics over a tree pyreprism remove comments src/ --output out/ # bulk-transform a directory pyreprism languages # list supported languages diff --git a/docs/cli.rst b/docs/cli.rst index d8250b7..20de5b7 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -17,7 +17,7 @@ Common usage pyreprism remove comments file.py cat file.go | pyreprism remove comments --lang go pyreprism extract comments "src/**/*.py" --json - pyreprism count comments --lang python file.py + pyreprism count comments file.py --lang python pyreprism preprocess --steps comments,strings,whitespace file.java pyreprism tokenize --json file.py pyreprism normalize file.py diff --git a/src/PyReprism/cli.py b/src/PyReprism/cli.py index 6f7fbaa..60ea14e 100644 --- a/src/PyReprism/cli.py +++ b/src/PyReprism/cli.py @@ -5,7 +5,7 @@ pyreprism remove comments file.py cat file.go | pyreprism remove comments --lang go pyreprism extract comments src/**/*.py - pyreprism count comments --lang python file.py + pyreprism count comments file.py --lang python pyreprism preprocess --steps comments,strings,whitespace file.java pyreprism tokenize --json file.py pyreprism languages diff --git a/tests/test_cli.py b/tests/test_cli.py index 535d679..b69a96d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -36,7 +36,10 @@ def test_remove_comments_file_autodetect(tmp_path, monkeypatch, capsys): def test_remove_in_place(tmp_path, monkeypatch, capsys): f = tmp_path / 'b.py' f.write_text('z = 3 # gone\n') - code, out, err = run(['remove', 'comments', '--in-place', str(f)], + # Options go after positionals: argparse on Python <= 3.11 only matches + # positionals in the first contiguous block, so a flag between the construct + # and the path would drop the path ("unrecognized arguments"). + code, out, err = run(['remove', 'comments', str(f), '--in-place'], monkeypatch=monkeypatch, capsys=capsys) assert code == 0 assert 'gone' not in f.read_text()