diff --git a/README.md b/README.md index 98ed695..bff0c41 100644 --- a/README.md +++ b/README.md @@ -7,28 +7,27 @@ [PyPI - Downloads](https://pypi.org/project/fluxative/) [GitHub Repo stars](https://github.com/JakePIXL/fluxative/stargazers) +Fluxative streamlines the conversion of Git repositories into standardized context files optimized for LLM consumption. The project architecture consists of three core components working together: -A tool to convert Git repositories into standardized context files for LLM consumption. Consists of three main components: - -- `converter.py`: Converts GitIngest output to llms.txt and llms-full.txt formats -- `expander.py`: Expands llms.txt files with actual file content from GitIngest -- `llmgentool.py`: Integrates both modules for an end-to-end solution +- `converter.py`: Transforms GitIngest output into structured llms.txt and llms-full.txt formats +- `expander.py`: Enhances llms.txt files by embedding actual file content from GitIngest +- `fluxative.py`: Integrates both modules for a seamless end-to-end solution ## Features - Generate LLM-friendly context files from Git repositories or GitHub URLs -- Creates five output files: +- Create a comprehensive set of output files: - `repo-raw.txt`: Complete original GitIngest output with Summary, Tree, and File Contents - `repo-llms.txt`: Basic repository summary with original structure preserved - `repo-llms-full.txt`: Comprehensive repository summary with original structure preserved - - `repo-llms-ctx.txt`: Basic summary with file contents - - `repo-llms-full-ctx.txt`: Comprehensive summary with file contents -- Preserves the full structure (Summary, Tree, and Content) from GitIngest -- Automatically organizes output files in a directory named after the repository + - `repo-llms-ctx.txt`: Basic summary with embedded file contents + - `repo-llms-full-ctx.txt`: Comprehensive summary with embedded file contents +- Preserve the full structure (Summary, Tree, and Content) from GitIngest +- Automatically organize output files in a structured directory named after the repository ## Installation -### Using uv +### Using uv (Recommended) ```bash uv install fluxative @@ -37,16 +36,16 @@ uv install fluxative ### From source ```bash -git clone https://github.com/JakePIXL/Fluxative.git -cd Fluxative +git clone https://github.com/JakePIXL/fluxative.git +cd fluxative pip install -e . ``` ### For development ```bash -git clone https://github.com/JakePIXL/Fluxative.git -cd Fluxative +git clone https://github.com/JakePIXL/fluxative.git +cd fluxative pip install -e ".[dev]" ``` @@ -61,33 +60,38 @@ fluxative /path/to/repo # Process a GitHub URL fluxative https://github.com/username/repo -# Specify an output directory +# Specify a custom output directory fluxative /path/to/repo --output-dir /custom/output/path ``` ### With uvx -If you have [uv](https://docs.astral.sh/uv) installed: +If you have [uv](https://docs.astral.sh/uv) installed, you can run Fluxative directly without installation: ```bash # Process a repository uvx fluxative /path/to/repo -# With output directory +# With custom output directory uvx fluxative /path/to/repo -o /custom/output/path ``` ## Output -The tool creates a directory named `-docs` containing: +Fluxative creates a directory named `-docs` containing different files based on the arguments used: -- `-raw.txt`: Complete original GitIngest output with Summary, Tree structure, and File Contents -- `-llms.txt`: Basic overview of the repository including original structure -- `-llms-full.txt`: Comprehensive overview with all files including original structure -- `-llms-ctx.txt`: Basic overview with embedded file contents +### Default Output (Always Generated) +- `-llms.txt`: Basic overview of the repository preserving original structure +- `-llms-ctx.txt`: Basic overview with embedded file contents for quick reference + +### With `--full-context` Flag +- `-llms-full.txt`: Comprehensive overview including all files with original structure - `-llms-full-ctx.txt`: Comprehensive overview with all embedded file contents -Each file preserves the original structure from GitIngest, ensuring you have access to: +### With `--dump-raw` Flag +- `-raw.txt`: Complete original GitIngest output with Summary, Tree structure, and File Contents + +Each output file maintains the original structure from GitIngest, providing you with: - Repository summary (name, URL, branch, commit) - Complete directory tree structure - File contents organized by category @@ -96,6 +100,7 @@ Each file preserves the original structure from GitIngest, ensuring you have acc - Python 3.10+ - GitIngest 0.1.4 or higher +- Typer 0.15.2 or higher ## License diff --git a/pyproject.toml b/pyproject.toml index 0a1e0d4..d94d833 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" requires-python = ">=3.10" authors = [{ name = 'JakePIXL' }] license = { text = "MIT" } -dependencies = ["gitingest>=0.1.4"] +dependencies = ["gitingest>=0.1.4", "typer>=0.15.2"] [project.urls] Homepage = "https://github.com/JakePIXL/fluxative" diff --git a/src/llmgentool.py b/src/fluxative.py similarity index 78% rename from src/llmgentool.py rename to src/fluxative.py index b992f8a..04e259f 100644 --- a/src/llmgentool.py +++ b/src/fluxative.py @@ -5,22 +5,24 @@ This script integrates converter.py and expander.py to generate standardized context files for LLMs from either a local repository or a GitHub URL. -It generates: -- llms.txt: Basic repository summary -- llms-full.txt: Comprehensive repository summary -- llms-ctx.txt: Basic summary with file contents -- llms-ctx-full.txt: Comprehensive summary with file contents +It can generate the following: + - llms.txt: Basic repository summary + - llms-ctx.txt: Basic summary with file contents + +with --full-context enabled: + - llms-full.txt: Comprehensive repository summary + - llms-ctx-full.txt: Comprehensive summary with file contents Usage: - python llmgentool.py [output_directory] + python fluxative.py [output_directory] """ import sys import os import shutil import tempfile -import argparse from typing import Optional +from typing_extensions import Annotated from urllib.parse import urlparse # Import from local modules @@ -28,6 +30,10 @@ from src.converter import parse_gitingest_output, generate_llms_txt from src.expander import generate_ctx_file +import typer + +app = typer.Typer(context_settings={"help_option_names": ["-h", "--help"]}) + try: from gitingest import ingest except ImportError: @@ -70,7 +76,12 @@ def get_repo_name(repo_path_or_url: str) -> str: return os.path.basename(path) -def process_repository(repo_path_or_url: str, output_dir: Optional[str] = None) -> tuple[str, Dict]: +def process_repository( + repo_path_or_url: str, + output_dir: Optional[str] = None, + full_context: bool = False, + dump_raw: bool = False, +) -> tuple[str, Dict]: """ Process a repository to generate LLM context files. @@ -134,14 +145,16 @@ def process_repository(repo_path_or_url: str, output_dir: Optional[str] = None) llms_full_txt_path = os.path.join(temp_dir, f"{repo_name}-llms-full.txt") generate_llms_txt(repo_info, llms_txt_path, full_version=False) - generate_llms_txt(repo_info, llms_full_txt_path, full_version=True) + if full_context: + generate_llms_txt(repo_info, llms_full_txt_path, full_version=True) # Generate context files ctx_output_path = os.path.join(temp_dir, f"{repo_name}-llms-ctx.txt") ctx_full_output_path = os.path.join(temp_dir, f"{repo_name}-llms-full-ctx.txt") generate_ctx_file(llms_txt_path, gitingest_file, ctx_output_path) - generate_ctx_file(llms_full_txt_path, gitingest_file, ctx_full_output_path) + if full_context: + generate_ctx_file(llms_full_txt_path, gitingest_file, ctx_full_output_path) # Copy files to output directory for file in [ @@ -164,29 +177,28 @@ def process_repository(repo_path_or_url: str, output_dir: Optional[str] = None) sys.exit(1) -def main(): - """Main function""" - parser = argparse.ArgumentParser( - description="Generate LLM context files from a repository or GitHub URL" - ) - parser.add_argument("repo_path_or_url", help="Local path or URL to a git repository") - parser.add_argument( - "--output-dir", - "-o", - help="Directory to save output files (default: current directory)", - default=None, - ) - - # Support for command-line usage within a package like 'uvx' - if len(sys.argv) > 1 and sys.argv[1] == "fluxative": - # Handle the case where this is called as 'uvx fluxative ' - args = parser.parse_args(sys.argv[2:]) - else: - # Normal command-line usage - args = parser.parse_args() +@app.command(no_args_is_help=True) +def main( + repo_path_or_url: Annotated[str, typer.Argument(help="Local path or URL to a git repository")], + output_dir: Annotated[ + Optional[str], + typer.Option( + "--output-dir", "-o", help="Directory to save output files (default: current directory)" + ), + ] = None, + dump_raw: Annotated[ + bool, typer.Option("--dump-raw", "-d", help="Dump raw GitIngest output") + ] = False, + full_context: Annotated[ + bool, typer.Option("--full-context", "-f", help="Generate full context files") + ] = False, +): + """ + Generate LLM context files from a repository. + """ # Process the repository - output_path, repo_info = process_repository(args.repo_path_or_url, args.output_dir) + output_path, repo_info = process_repository(repo_path_or_url, output_dir) repo_name = repo_info["name"] print(f"Files generated in: {output_path}") @@ -198,4 +210,4 @@ def main(): if __name__ == "__main__": - main() + app() diff --git a/tests/test_import.py b/tests/test_import.py index ac37b5e..f2de052 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -2,8 +2,8 @@ def test_import(): """Test that modules can be imported directly.""" import src.converter as converter import src.expander as expander - import src.llmgentool as llmgentool + import src.fluxative as fluxative assert converter is not None assert expander is not None - assert llmgentool is not None + assert fluxative is not None diff --git a/uv.lock b/uv.lock index 36f7a00..3504e6c 100644 --- a/uv.lock +++ b/uv.lock @@ -93,6 +93,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, ] +[[package]] +name = "exceptiongroup" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/35/2495c4ac46b980e4ca1f6ad6db102322ef3ad2410b79fdde159a4b0f3b92/exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc", size = 28883 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/cc/b7e31358aac6ed1ef2bb790a9746ac2c69bcb3c8588b41616914eb106eaf/exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b", size = 16453 }, +] + +[[package]] +name = "fluxative" +source = { editable = "." } +dependencies = [ + { name = "gitingest" }, + { name = "typer" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "gitingest", specifier = ">=0.1.4" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.10" }, + { name = "typer", specifier = ">=0.15.2" }, +] +provides-extras = ["dev"] + [[package]] name = "gitingest" version = "0.1.4" @@ -117,18 +149,77 @@ wheels = [ ] [[package]] -name = "llmsgenerator" -version = "0.1.0" -source = { virtual = "." } +name = "iniconfig" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 }, +] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "gitingest" }, - { name = "ruff" }, + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528 }, ] -[package.metadata] -requires-dist = [ - { name = "gitingest", specifier = ">=0.1.4" }, - { name = "ruff", specifier = ">=0.9.10" }, +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, +] + +[[package]] +name = "packaging" +version = "24.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f", size = 163950 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451 }, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 }, +] + +[[package]] +name = "pygments" +version = "2.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 }, +] + +[[package]] +name = "pytest" +version = "8.3.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634 }, ] [[package]] @@ -215,6 +306,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928 }, ] +[[package]] +name = "rich" +version = "13.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/3a/0316b28d0761c6734d6bc14e770d85506c986c85ffb239e688eeaab2c2bc/rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098", size = 223149 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", size = 242424 }, +] + [[package]] name = "ruff" version = "0.9.10" @@ -240,6 +345,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/85/338e603dc68e7d9994d5d84f24adbf69bae760ba5efd3e20f5ff2cec18da/ruff-0.9.10-py3-none-win_arm64.whl", hash = "sha256:5fd804c0327a5e5ea26615550e706942f348b197d5475ff34c19733aee4b2e69", size = 10436892 }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 }, +] + [[package]] name = "tiktoken" version = "0.9.0" @@ -315,6 +429,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257 }, ] +[[package]] +name = "typer" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/6f/3991f0f1c7fcb2df31aef28e0594d8d54b05393a0e4e34c65e475c2a5d41/typer-0.15.2.tar.gz", hash = "sha256:ab2fab47533a813c49fe1f16b1a370fd5819099c00b119e0633df65f22144ba5", size = 100711 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/fc/5b29fea8cee020515ca82cc68e3b8e1e34bb19a3535ad854cac9257b414c/typer-0.15.2-py3-none-any.whl", hash = "sha256:46a499c6107d645a9c13f7ee46c5d5096cae6f5fc57dd11eccbbb9ae3e44ddfc", size = 45061 }, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 }, +] + [[package]] name = "urllib3" version = "2.3.0"