diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..95e7494 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: pomponchik + +--- + +## Short description + +Replace this text with a short description of the error and the behavior that you expected to see instead. + + +## Describe the bug in detail + +Please add a test that reproduces the bug (i.e., currently fails): + +```python +def test_your_bug(): + ... +``` + +When writing the test, please ensure compatibility with the [`pytest`](https://docs.pytest.org/) framework. + +If for some reason you cannot describe the error in the test format, describe the steps to reproduce it here. + + +## Environment + - OS: ... + - Python version (the output of the `python --version` command): ... + - Version of this package: ... diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000..5f5fdc0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,26 @@ +--- +name: Documentation fix +about: Add something to the documentation, delete it, or change it +title: '' +labels: documentation +assignees: pomponchik +--- + +## It's cool that you're here! + +Documentation is an important part of the project; we strive to make it high-quality and keep it up to date. Please adjust this template by outlining your proposal. + + +## Type of action + +What do you want to do: remove something, add something, or change something? + + +## Where? + +Specify which part of the documentation you want to change. For example, the name of an existing documentation section or a line number in `README.md`. + + +## The essence + +Please describe the essence of the proposed change. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..117d79f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: pomponchik + +--- + +## Short description + +What do you propose and why do you consider it important? + + +## Some details + +If you can, provide code examples that will show how your proposal will work. Also, if you can, indicate which alternative approaches you have considered. And finally, describe how you propose to verify that your idea is implemented correctly, if at all possible. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..6f86494 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,12 @@ +--- +name: Question or consultation +about: Ask anything about this project +title: '' +labels: question +assignees: pomponchik + +--- + +## Your question + +Here you can freely describe your question about the project. Please read the documentation provided before doing this, and ask the question only if it is not answered there. In addition, please keep in mind that this is a free non-commercial project and user support is optional for its author. Response times are not guaranteed. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..e9db245 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,60 @@ +name: Lint + +on: + push + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15.0-alpha.1'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dependencies + shell: bash + run: uv pip install --system -r requirements_dev.txt + + - name: Install the library + shell: bash + run: uv pip install --system . + + - name: Run ruff + shell: bash + run: ruff check throng + + - name: Run ruff for tests + shell: bash + run: ruff check tests + + - name: Run mypy + shell: bash + run: >- + mypy + --show-error-codes + --strict + --disallow-any-decorated + --disallow-any-explicit + --disallow-any-expr + --disallow-any-generics + --disallow-any-unimported + --disallow-subclassing-any + --warn-return-any + throng + + - name: Run mypy for tests + shell: bash + run: mypy --exclude '^tests/typing/' tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..641ef68 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,39 @@ +name: Release + +on: + push: + branches: + - main + +jobs: + pypi-publish: + name: upload release to PyPI + runs-on: ubuntu-latest + # Specifying a GitHub environment is optional, but strongly encouraged + environment: release + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dependencies + shell: bash + run: uv pip install --system -r requirements_dev.txt + + - name: Build the project + shell: bash + run: python -m build . + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/tests_and_coverage.yml b/.github/workflows/tests_and_coverage.yml new file mode 100644 index 0000000..f4a7f8b --- /dev/null +++ b/.github/workflows/tests_and_coverage.yml @@ -0,0 +1,68 @@ +name: Tests + +on: + push + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t', '3.15.0-alpha.1'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install dependencies + shell: bash + run: uv pip install --system -r requirements_dev.txt + + - name: Install the library + shell: bash + run: uv pip install --system . + + - name: Print all libs + shell: bash + run: uv pip list --system + + - name: Run tests and show coverage on the command line + shell: bash + run: | + pth_file="$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/throng_coverage_process_startup.pth" + printf "import os; os.getenv('COVERAGE_PROCESS_START') and __import__('coverage').process_startup()\n" > "$pth_file" + coverage erase + COVERAGE_PROCESS_START="$PWD/pyproject.toml" coverage run -m pytest -n auto --cache-clear --assert=plain + coverage combine + coverage report -m --fail-under=100 + coverage xml + + - name: Upload coverage to Coveralls + if: runner.os == 'Linux' + env: + COVERALLS_REPO_TOKEN: ${{secrets.COVERALLS_REPO_TOKEN}} + uses: coverallsapp/github-action@v2 + with: + format: cobertura + file: coverage.xml + continue-on-error: true + + - name: Run tests and show the branch coverage on the command line + shell: bash + run: | + pth_file="$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')/throng_coverage_process_startup.pth" + printf "import os; os.getenv('COVERAGE_PROCESS_START') and __import__('coverage').process_startup()\n" > "$pth_file" + coverage erase + THRONG_COVERAGE_BRANCH=true COVERAGE_PROCESS_START="$PWD/pyproject.toml" coverage run -m pytest -n auto --cache-clear --assert=plain + coverage combine + coverage report -m --fail-under=100 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dddf337 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +.DS_Store +__pycache__ +venv +.pytest_cache +build +dist +*.egg-info +test.py +.coverage +.coverage.* +.idea +.ruff_cache +.mutmut-cache +.mypy_cache +html +CLAUDE.md +.claude +mutants +planning_features.md +coverage.xml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1a71cb6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,131 @@ +# PolyForm Noncommercial License 1.0.0 + + + +## Acceptance + +In order to get any license under these terms, you must agree +to them as both strict obligations and conditions to all +your licenses. + +## Copyright License + +The licensor grants you a copyright license for the +software to do everything you might do with the software +that would otherwise infringe the licensor's copyright +in it for any permitted purpose. However, you may +only distribute the software according to [Distribution +License](#distribution-license) and make changes or new works +based on the software according to [Changes and New Works +License](#changes-and-new-works-license). + +## Distribution License + +The licensor grants you an additional copyright license +to distribute copies of the software. Your license +to distribute covers distributing the software with +changes and new works permitted by [Changes and New Works +License](#changes-and-new-works-license). + +## Notices + +You must ensure that anyone who gets a copy of any part of +the software from you also gets a copy of these terms or the +URL for them above, as well as copies of any plain-text lines +beginning with `Required Notice:` that the licensor provided +with the software. For example: + +> Required Notice: Copyright Yoyodyne, Inc. (http://example.com) + +## Changes and New Works License + +The licensor grants you an additional copyright license to +make changes and new works based on the software for any +permitted purpose. + +## Patent License + +The licensor grants you a patent license for the software that +covers patent claims the licensor can license, or becomes able +to license, that you would infringe by using the software. + +## Noncommercial Purposes + +Any noncommercial purpose is a permitted purpose. + +## Personal Uses + +Personal use for research, experiment, and testing for +the benefit of public knowledge, personal study, private +entertainment, hobby projects, amateur pursuits, or religious +observance, without any anticipated commercial application, +is use for a permitted purpose. + +## Noncommercial Organizations + +Use by any charitable organization, educational institution, +public research organization, public safety or health +organization, environmental protection organization, +or government institution is use for a permitted purpose +regardless of the source of funding or obligations resulting +from the funding. + +## Fair Use + +You may have "fair use" rights for the software under the +law. These terms do not limit them. + +## No Other Rights + +These terms do not allow you to sublicense or transfer any of +your licenses to anyone else, or prevent the licensor from +granting licenses to anyone else. These terms do not imply +any other licenses. + +## Patent Defense + +If you make any written claim that the software infringes or +contributes to infringement of any patent, your patent license +for the software granted under these terms ends immediately. If +your company makes such a claim, your patent license ends +immediately for work on behalf of your company. + +## Violations + +The first time you are notified in writing that you have +violated any of these terms, or done anything with the software +not covered by your licenses, your licenses can nonetheless +continue if you come into full compliance with these terms, +and take practical steps to correct past violations, within +32 days of receiving notice. Otherwise, all your licenses +end immediately. + +## No Liability + +***As far as the law allows, the software comes as is, without +any warranty or condition, and the licensor will not be liable +to you for any damages arising out of these terms or the use +or nature of the software, under any kind of legal claim.*** + +## Definitions + +The **licensor** is the individual or entity offering these +terms, and the **software** is the software the licensor makes +available under these terms. + +**You** refers to the individual or entity agreeing to these +terms. + +**Your company** is any legal entity, sole proprietorship, +or other kind of organization that you work for, plus all +organizations that have control over, are under the control of, +or are under common control with that organization. **Control** +means ownership of substantially all the assets of an entity, +or the power to direct its management and policies by vote, +contract, or otherwise. Control can be direct or indirect. + +**Your licenses** are all the licenses granted to you for the +software under these terms. + +**Use** means anything you do with the software requiring one +of your licenses. diff --git a/README.md b/README.md index 63b3964..7810cc7 100644 --- a/README.md +++ b/README.md @@ -1 +1,30 @@ -# bubbies \ No newline at end of file +![logo](https://raw.githubusercontent.com/mutating/throng/develop/docs/assets/logo_1.svg) + +Sometimes our programs need to execute console commands. In some cases, a command may be executed locally, while in others it may be executed in parallel across thousands of machines in the cloud. This library serves as an abstraction layer over various command execution environments, allowing you to write code once that will run anywhere. Specific execution environments are connected here as plugins (and you can even write your own!) with a unified API, and your code doesn’t need to know the internal workings of a specific plugin to run commands within it. + +This library provides: + +- Complete isolation of the program’s core logic from where and how commands are executed. The logic remains compact and describes the essence of the problem. +- The ability to easily swap one implementation for another. For example, you can replace local command execution with execution inside a Docker container or a cloud virtual machine without changing the code. +- Parallelization is simple. Each plugin decides for itself what level of parallelism it needs, so the main code doesn’t even need to know that it’s running in parallel. + + +## Issues + +- Cancellation during `dump()` and `load()` is currently observed between files, +not while the contents of one large regular file are being copied. A future +implementation should copy large file payloads in chunks and call the +operation cancellation token between chunks. Benchmarks with a never-cancelled +`SimpleToken` indicate that **256 KiB** is the recommended default chunk size. + +- Add reprs to all basic classes. + +- Add size limits and some reactions to overflow. + +- Add more optional backends. + +- Log information about file sizes. + +- Add to pristan possibility to declare signature as a list (not only str as now). + +- Add to dirstree apply method, and sorting, and non-pass non-file mode, and fix the tree before using. And use it all here after. And add watchdog support (https://github.com/gorakhargosh/watchdog/). diff --git a/docs/assets/logo_1.svg b/docs/assets/logo_1.svg new file mode 100644 index 0000000..9f0cac1 --- /dev/null +++ b/docs/assets/logo_1.svg @@ -0,0 +1,496 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3801e4b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,89 @@ +[build-system] +requires = ["setuptools==68.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "throng" +version = "0.0.1" +authors = [ + { name="Evgeniy Blinov", email="zheni-b@yandex.ru" }, +] +description = 'Running commands in isolated environments' +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + 'suby>=0.0.13', + 'cantok>=0.0.42', + 'pristan>=0.0.23', + 'skelet>=0.0.21', + 'emptylog>=0.0.12', + 'dirstree>=0.0.12', + 'locklib>=0.0.25', + 'pathspec>=0.12.0', +] +classifiers = [ + "Operating System :: OS Independent", + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', + 'Programming Language :: Python :: 3.15', + 'Programming Language :: Python :: Free Threading', + 'Programming Language :: Python :: Free Threading :: 3 - Stable', + 'License :: OSI Approved :: MIT License', + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Libraries', + 'Typing :: Typed', +] +keywords = [ + 'subprocesses', + 'subprocesses wrapper', + 'execute commands', + 'isolated environments', + 'sandboxes', +] + +[tool.setuptools.package-data] +"throng" = ["py.typed"] + +[tool.setuptools.packages.find] +include = ["throng*"] + +[tool.mutmut] +paths_to_mutate=["throng"] + +[tool.coverage.run] +branch = "${THRONG_COVERAGE_BRANCH-false}" +omit = ["*tests*"] +parallel = true +plugins = ["coverage_pyver_pragma"] +source = ["throng"] + +[tool.pytest.ini_options] +norecursedirs = ["build", "mutants"] + +[tool.ruff] +lint.ignore = ['E501', 'E712', 'PTH123', 'PTH118', 'PLR2004', 'PTH107', 'SIM105', 'SIM102', 'RET503', 'PLR0912', 'C901', 'E731', 'F821'] +lint.select = ["ERA001", "YTT", "ASYNC", "BLE", "B", "A", "COM", "INP", "PIE", "T20", "PT", "RSE", "RET", "SIM", "SLOT", "TID252", "ARG", "PTH", "I", "C90", "N", "E", "W", "D201", "D202", "D419", "F", "PL", "PLE", "PLR", "PLW", "RUF", "TRY201", "TRY400", "TRY401"] +lint.isort.combine-as-imports = true +format.quote-style = "single" + +[project.urls] +'Source' = 'https://github.com/mutating/throng' +'Tracker' = 'https://github.com/mutating/throng/issues' + +[project.entry-points.throng] +local = "throng.extensions.plugins:create_local_throng" +temporary_directory = "throng.extensions.plugins:create_temporary_directory_throng" +gzip = "throng.extensions.plugins:create_gzip_serializer" +bz2 = "throng.extensions.plugins:create_bz2_serializer" +lzma = "throng.extensions.plugins:create_lzma_serializer" +none = "throng.extensions.plugins:create_none_serializer" diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..75164b5 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,15 @@ +pytest==8.3.5 +pytest-xdist==3.6.1; python_version < '3.9' +pytest-xdist==3.8.0; python_version >= '3.9' +coverage==7.6.1; python_version == '3.8' +coverage==7.6.10; python_version >= '3.9' +coverage-pyver-pragma==0.4.0 +build==1.2.2.post1 +mypy==1.14.1 +pytest-mypy-testing==0.1.3 +ruff==0.14.6 +mutmut==3.2.3 +cosmic-ray==8.3.15; python_version < '3.9' +cosmic-ray==8.4.6; python_version >= '3.9' +full_match==0.0.3 +locklib==0.0.22 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/plugins/__init__.py b/tests/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/__init__.py b/throng/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/abstracts/__init__.py b/throng/abstracts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/abstracts/isolate.py b/throng/abstracts/isolate.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/abstracts/result.py b/throng/abstracts/result.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/abstracts/serializer.py b/throng/abstracts/serializer.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/abstracts/throng.py b/throng/abstracts/throng.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/errors.py b/throng/errors.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/__init__.py b/throng/extensions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/isolation/__init__.py b/throng/extensions/isolation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/isolation/directory.py b/throng/extensions/isolation/directory.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/isolation/throngs/__init__.py b/throng/extensions/isolation/throngs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/isolation/throngs/configs.py b/throng/extensions/isolation/throngs/configs.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/isolation/throngs/local.py b/throng/extensions/isolation/throngs/local.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/isolation/throngs/temporary_directory.py b/throng/extensions/isolation/throngs/temporary_directory.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/plugins.py b/throng/extensions/plugins.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/serialization/__init__.py b/throng/extensions/serialization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/extensions/serialization/tar.py b/throng/extensions/serialization/tar.py new file mode 100644 index 0000000..e69de29 diff --git a/throng/py.typed b/throng/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/throng/slots.py b/throng/slots.py new file mode 100644 index 0000000..e69de29