Major updates

.flake8

@@ -2,6 +2,10 @@
 show-source = True
 count = True
 statistics = True
-; max-line-length = 150
-ignore = E501
-exclude = .github,venv,build,dist,docs,.venv,src/tests,src/PyReprism/languages/__init__.py,__test__.py
+max-line-length = 120
+# E501: long lines are common in this project's regex/keyword tables.
+# The W1/W2/W3 and E1/E2/E3 codes below are pre-existing whitespace/indentation
+# style debt across the legacy language modules; they are ignored so flake8 can
+# focus on real correctness issues (pyflakes F-codes, undefined names, etc.).
+extend-ignore = E501, W191, W291, W292, W293, W391, E101, E117, E128, E231, E302, E303, E265, E266
+exclude = .git,.github,venv,.venv,build,dist,docs,htmlcov,tests,src/PyReprism/languages/__init__.py,__test__.py

.github/workflows/ci.yml

@@ -1,41 +1,71 @@
 name: CI
+
 on:
   push:
-    branches-ignore:
-      - main
+    branches: [main]
   pull_request:
-    branches-ignore:
-      - main
+
+# Least-privilege by default.
+permissions:
+  contents: read
+
+# Cancel superseded runs on the same ref.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  publish:
-    name: "Lint, test and upload to codecov"
+  test:
+    name: "Lint & test (Python ${{ matrix.python-version }})"
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
     steps:
       - name: Check out the code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.x'
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flake8 pytest pytest-cov
+          pip install -e ".[dev]"
 
       - name: Lint with flake8
-        run: |
-          flake8 .
+        run: flake8 src/PyReprism
 
       - name: Run tests with pytest
-        run: |
-          pytest --cov=src/ --cov-report=xml
+        run: pytest --cov=src/PyReprism --cov-report=xml
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        if: matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
         with:
-          verbose: true
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          fail_ci_if_error: false
+
+  build:
+    name: "Build & check package"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Build and validate the distribution
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          twine check dist/*

.github/workflows/publish.yml

@@ -1,38 +1,53 @@
-name: Publish Python 🐍 distributions 📦 to PyPI
+name: Publish 📦 to PyPI
+
 on:
   push:
     tags:
-      - 'v*' 
-    branches:
-      - main 
-      - stable
+      - 'v*'
 
-jobs:
-  publish:
-    name: "Build and upload to pypi"
-    runs-on: "ubuntu-latest"
-    # environment:
-    #   name: pypi
-    #   url: https://pypi.org/project/PyReprism/
-    # permissions:
-    #   id-token: write
+permissions:
+  contents: read
 
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Install python
-        uses: actions/setup-python@v4
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.11' 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install hatchling twine build hatch-vcs
-      - name: Build app
+          python-version: '3.12'
+
+      - name: Build and validate the distribution
         run: |
+          python -m pip install --upgrade pip build twine
           python -m build
-      - name: Publish distribution 📦 to PyPI
-        if: startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@release/v1
+          twine check dist/*
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
         with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
-        # run: twine upload dist/*
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/PyReprism/
+    permissions:
+      id-token: write   # Trusted Publishing (OIDC) — no API token needed.
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -163,4 +163,9 @@ cython_debug/
 #.idea/
 
 # others
-__test__.py
+__test__.py
+
+# macOS
+.DS_Store
+# Claude Code agent workspace
+.claude/

.pre-commit-hooks.yaml

@@ -0,0 +1,27 @@
+# PyReprism pre-commit hooks.
+#
+# Add to your .pre-commit-config.yaml, e.g.:
+#
+#   - repo: https://github.com/unlv-evol/PyReprism
+#     rev: v0.1.0
+#     hooks:
+#       - id: pyreprism-strip-comments
+#         files: ^generated/           # scope it! these hooks rewrite files
+#
+# NOTE: these hooks modify source in place. Scope them with `files:`/`exclude:`
+# to directories where stripping comments/whitespace is intended (e.g. generated
+# or vendored code), not your whole repository.
+
+- id: pyreprism-strip-comments
+  name: PyReprism – strip comments
+  description: Remove comments in place (language auto-detected from file extension).
+  entry: pyreprism remove comments --in-place
+  language: python
+  types: [text]
+
+- id: pyreprism-normalize-whitespace
+  name: PyReprism – normalize whitespace
+  description: Collapse insignificant whitespace in place.
+  entry: pyreprism preprocess --steps whitespace --in-place
+  language: python
+  types: [text]

.readthedocs.yaml

@@ -23,9 +23,10 @@ sphinx:
 # formats:
 #    - pdf
 
-# Optionally declare the Python requirements required to build your docs
+# Install the package together with its documentation extra.
 python:
    install:
-   - requirements: docs/requirements.txt
    - method: pip
-     path: .
+     path: .
+     extra_requirements:
+       - docs

CHANGELOG.md

@@ -0,0 +1,78 @@
+# Changelog
+
+All notable changes to PyReprism are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- **Diff processing** (`PyReprism.diffs`): parse unified/`git` diffs and analyze
+  the changed code per file in its own language. Includes churn metrics
+  (`diff_stats`: added/removed split into code, comment and blank), cosmetic
+  (comment/whitespace-only) change detection, reconstruction of the added/removed
+  code, and ML-oriented `tokenize`/`normalize`/`extract_comments` on changes.
+  Fragment mode works from the diff alone; setting a file's `new_source` /
+  `old_source` enables accurate full-file classification. New CLI command
+  `pyreprism diff` (`--json` / `--csv` / `--per-file` / `--cosmetic`).
+
+## [0.1.0] - 2026-07-07
+
+This release turns PyReprism from a comment-removal helper into a full
+source-preprocessing toolkit with a high-level API, a CLI, code metrics,
+ML-oriented normalization, an optional accurate backend, and batch processing.
+
+### Added
+- **High-level API** (`import PyReprism as pr`): `remove_*`, `extract_*`,
+  `count_*`, `match_*` for comments, keywords, numbers, operators, strings and
+  identifiers; plus `preprocess`, `remove_whitespaces`, `get_language` and
+  `detect_language`. `lang=` accepts a language name, file extension, or class.
+- **Tokenizer**: `tokenize()` on every language returns a lossless, typed
+  `Token` stream (`Token`/`TokenType` in `PyReprism.tokens`).
+- **Code metrics**: `stats()` returns a `CodeStats` (line/token counts,
+  comment-to-code ratio, comment density; `as_dict()` for JSON/dataframes).
+- **ML normalization**: `normalize()` canonicalizes code (rename identifiers to
+  `VAR1…`, mask string/number literals, drop comments) for clone/plagiarism
+  detection and code-embedding pipelines; `blank_comments()` strips comments
+  while preserving line numbers.
+- **Pluggable backends**: `engine="regex"` (default, zero-dependency) or
+  `engine="pygments"` (`pip install pyreprism[accurate]`) for higher accuracy,
+  plus `"auto"`. Available on every operation.
+- **Batch processing** (`PyReprism.batch`): `analyze()` walks a directory tree
+  and returns an aggregate report (`to_json`/`to_csv`, per-language breakdown);
+  `transform()` bulk-applies a transformation into a mirrored output directory.
+- **Command-line interface** (`pyreprism`, also `python -m PyReprism`):
+  `remove`, `extract`, `count`, `preprocess`, `tokenize`, `stats`, `normalize`,
+  `scan`, and `languages`; reads stdin/files/globs/directories; supports
+  `--lang`, `--engine`, `--in-place`, `--output`, and `--json`/`--csv`.
+- **Source-based language detection**: `detect_language(source=...)` recognizes
+  `#!` shebangs and (when Pygments is installed) makes a best-effort content
+  guess; the CLI uses this to auto-detect the language of piped input.
+- Pre-commit hooks (`.pre-commit-hooks.yaml`), a `py.typed` marker (PEP 561),
+  and a registry-wide test suite covering all supported languages.
+
+### Changed
+- Implemented real support for 25 previously-empty language stubs and migrated
+  22 legacy languages (HTML, JSON, YAML, CSS, Markdown, …) to the
+  `BaseLanguage` + registry pattern, making them importable via the public API.
+- Ambiguous file extensions now resolve to a canonical language
+  (`.py` → Python, `.js` → JavaScript, …).
+- Consolidated packaging into `pyproject.toml` (single version source, pytest
+  and coverage config), updated CI to a Python 3.8–3.12 matrix, and refreshed
+  the README.
+
+### Fixed
+- Corrected comment-stripping for ~30 languages whose regexes either failed to
+  remove line comments or consumed surrounding code, and fixed duplicate/broken
+  definitions (Perl, csp, glsl, django, apacheconf, php_extras, ruby, and the
+  Python triple-quoted-docstring bug).
+
+## [0.0.4] - 2024
+
+- Early beta releases: comment removal for an initial set of languages and the
+  `Normalizer` whitespace helper.
+
+[Unreleased]: https://github.com/unlv-evol/PyReprism/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.1.0
+[0.0.4]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.0.4