diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 202db323..b72885af 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,7 +2,7 @@ name: main on: push: - branches: [master, develop] + branches: [main] pull_request: jobs: @@ -15,7 +15,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4.1.0 with: - python-version: 3.8 + python-version: 3.12 - name: Install flake8 run: pip --disable-pip-version-check install flake8 @@ -24,10 +24,16 @@ jobs: run: flake8 --count tests: - runs-on: ubuntu-latest + name: tests - Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: + fail-fast: true matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + os: ["ubuntu-latest"] + python-version: ["3.13"] + env: + VENV: .venv + PACKAGE: pipe_segment steps: - uses: actions/checkout@v4 @@ -39,37 +45,36 @@ jobs: # cache option make the step fail if you don´t have requirements.txt or pyproject.toml on root. # https://github.com/actions/setup-python/issues/807. - - name: Install dependencies + - name: Create virtual environment + run: python -m venv $VENV + + - name: Install package with test dependencies run: | - python -m pip install --upgrade pip - pip install -r requirements/test.txt - pip install -e . - + . $VENV/bin/activate + make install-test + pip install . + - name: Test with pytest run: | - make testintegration + . $VENV/bin/activate + pytest --cov=$PACKAGE --cov-report=xml - # Remove this step when this is done in tests-in-docker job. - name: Upload coverage reports to Codecov - if: ${{ matrix.python-version == '3.8' }} + if: github.ref == 'refs/heads/main' && matrix.python-version == '3.13' uses: codecov/codecov-action@v4.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} -# Use this job when base docker image is not pulled from GFW gcr. -# tests-in-docker: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 - -# - name: Test with pytest in docker -# run: | -# docker volume create --name=gcp -# docker compose run test - -# - name: Upload coverage reports to Codecov -# uses: codecov/codecov-action@v4.0.1 -# with: -# token: ${{ secrets.CODECOV_TOKEN }} + tests-in-docker: + name: tests - Python 3.12 on Docker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4.1.0 + + - name: Build docker image + run: make docker-build + + - name: Run tests + run: make docker-ci-test diff --git a/CHANGES.md b/CHANGES__DEPRECTAED.md similarity index 100% rename from CHANGES.md rename to CHANGES__DEPRECTAED.md diff --git a/Dockerfile b/Dockerfile index 29098a71..052edfbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,50 +1,67 @@ # --------------------------------------------------------------------------------------- -# BASE +# BUILDER # --------------------------------------------------------------------------------------- -FROM python:3.8 AS base +FROM python:3.12-slim-bookworm AS builder -# Configure the working directory -RUN mkdir -p /opt/project -WORKDIR /opt/project +VOLUME ["/root/.config"] -# Copy files from official SDK image, including script/dependencies. -COPY --from=apache/beam_python3.8_sdk:2.56.0 /opt/apache/beam /opt/apache/beam +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gcc g++ build-essential git && \ + rm -rf /var/lib/apt/lists/* -# Install SDK. (needed for Python SDK) -RUN pip install --no-cache-dir apache-beam[gcp]==2.56.0 +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv -# Install application dependencies -COPY requirements.txt /opt/requirements.txt -RUN pip install --no-cache-dir -r /opt/requirements.txt +WORKDIR /install -# Set the entrypoint to Apache Beam SDK launcher. -ENTRYPOINT ["/opt/apache/beam/boot"] +COPY requirements.txt . + +RUN uv pip install --system --upgrade pip && \ + uv pip install --system build && \ + uv pip install --system --prefix=/install -r requirements.txt + +COPY pyproject.toml README.md MANIFEST.in ./ +COPY src ./src +RUN uv pip install --system --prefix=/install . # --------------------------------------------------------------------------------------- -# PROD +# PRODUCTION IMAGE # --------------------------------------------------------------------------------------- -FROM base AS prod +FROM python:3.12-slim-bookworm AS prod + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + +# COPY PYTHON PACKAGES +COPY --from=builder /install /usr/local + +# APACHE BEAM INTEGRATION +COPY --from=apache/beam_python3.12_sdk:2.71.0 /opt/apache/beam /opt/apache/beam +ENTRYPOINT ["/opt/apache/beam/boot"] -# Install app package -COPY . /opt/project -RUN pip install . +WORKDIR /opt/project # --------------------------------------------------------------------------------------- -# DEV +# DEVELOPMENT IMAGE # --------------------------------------------------------------------------------------- -FROM base AS dev +FROM builder AS dev + +WORKDIR /opt/project -COPY ./requirements/dev.txt ./ -COPY ./requirements/test.txt ./ +COPY . . +RUN uv pip install --system -e .[lint,dev,build] && \ + uv pip install --system -r requirements-test.txt + +# --------------------------------------------------------------------------------------- +# TEST IMAGE +# --------------------------------------------------------------------------------------- +FROM prod AS test -RUN pip install --no-cache-dir -r dev.txt -RUN pip install --no-cache-dir -r test.txt +COPY ./requirements-test.txt . +RUN pip install -r requirements-test.txt -# Install app package -COPY . /opt/project -ENV PYTHONPATH /opt/project -RUN cd /usr/local/lib/python3.8/site-packages && \ - python /opt/project/setup.py develop +COPY ./tests ./tests -# Set the entrypoint to Apache Beam SDK launcher. -ENTRYPOINT ["pipe"] +# Suppress all warnings during tests +# To see/address warnings, run tests in your development environment. +ENV PYTHONWARNINGS=ignore \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index ba3115c3..e33d4e84 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,6 @@ include LICENSE -include CHANGES.md -include MANIFEST.in include README.md -include setup.py -recursive-include assets * -recursive-include pipe_segment * + +recursive-include src/**/assets * +recursive-exclude * __pycache__ +recursive-exclude * *.py[cod] diff --git a/Makefile b/Makefile index a6f1db38..5298d464 100644 --- a/Makefile +++ b/Makefile @@ -1,66 +1,167 @@ +.DEFAULT_GOAL:=help + VENV_NAME:=.venv -REQS_PROD_IN:=requirements/prod.in -REQS_PROD_TXT:=requirements.txt -REQS_DEV:=requirements/dev.txt -DOCKER_IMAGE_DEV:=dev +REQS_PROD:=requirements.txt +DOCKER_DEV_SERVICE:=dev +DOCKER_CI_TEST_SERVICE:=test GCP_PROJECT:=world-fishing-827 GCP_DOCKER_VOLUME:=gcp -## help: Prints this list of commands. -## gcp: Authenticates to google cloud and configure the project. -## build: Builds docker image. -## docker-shell: Enters to docker container shell. -## reqs: Compiles requirements file with pip-tools. -## upgrade-reqs: Upgrades requirements file based on .in constraints. -## venv: Creates a virtual environment. -## install: Installs all dependencies needed for development. -## test: Runs unit tests. -## testintegration: Runs unit and integration tests. -## testdocker: Runs unit and integration tests inside docker container. +sources = src +# --------------------- +# DOCKER +# --------------------- -help: - @echo "\nUsage: \n" - @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/-/' +.PHONY: docker-build ## Builds docker image. +docker-build: + docker compose build -gcp: +.PHONY: docker-volume ## Creates the docker volume for GCP. +docker-volume: docker volume create --name ${GCP_DOCKER_VOLUME} - docker compose run --rm gcloud auth application-default login - docker compose run --rm gcloud config set project ${GCP_PROJECT} - docker compose run --rm gcloud auth application-default set-quota-project ${GCP_PROJECT} -build: - docker compose build +.PHONY: docker-gcp ## gcp: Authenticates to google cloud and configure the project. +docker-gcp: + make docker-volume + docker compose run gcloud auth application-default login + docker compose run gcloud config set project ${GCP_PROJECT} + docker compose run gcloud auth application-default set-quota-project ${GCP_PROJECT} + +.PHONY: docker-ci-test ## Runs tests using prod image, exporting coverage.xml report. +docker-ci-test: + docker compose run --rm ${DOCKER_CI_TEST_SERVICE} +.PHONY: docker-shell ## Enters to docker container shell. docker-shell: - docker compose run --rm --entrypoint /bin/bash -it ${DOCKER_IMAGE_DEV} + docker compose run --rm -it ${DOCKER_DEV_SERVICE} +.PHONY: reqs ## Compiles requirements.txt with pip-tools. reqs: - docker compose run --rm --entrypoint /bin/bash -it ${DOCKER_IMAGE_DEV} -c \ - 'pip-compile -o ${REQS_PROD_TXT} ${REQS_PROD_IN} -v' + docker compose run --rm ${DOCKER_DEV_SERVICE} -c \ + 'pip-compile -o ${REQS_PROD} -v' + +.PHONY: reqs-upgrade ## Upgrades requirements.txt with pip-tools. +reqs-upgrade: + docker compose run --rm ${DOCKER_DEV_SERVICE} -c \ + 'pip-compile -o ${REQS_PROD} -U -v' -upgrade-reqs: - docker compose run --rm --entrypoint /bin/bash -it ${DOCKER_IMAGE_DEV} -c \ - 'pip-compile -o ${REQS_PROD_TXT} -U ${REQS_PROD_IN} -v' +# --------------------- +# VIRTUAL ENVIRONMENT +# --------------------- +.PHONY: venv ## Creates virtual environment. venv: - python3 -m venv ${VENV_NAME} + python -m venv ${VENV_NAME} -install: - pip install -r ${REQS_DEV} - pip install -e . +.PHONY: upgrade-pip ## Upgrades pip. +upgrade-pip: + python -m pip install -U pip +.PHONY: install-test ## Install and only test dependencies. +install-test: upgrade-pip + python -m pip install -r requirements-test.txt + +.PHONY: install ## Install the package in editable mode & all dependencies for local development. +install: upgrade-pip + python -m pip install -e .[lint,dev,build] + +.PHONY: test ## Run all unit tests exporting coverage.xml report. test: - pytest + python -m pytest -m "not integration" --cov-report term --cov-report=xml --cov=$(sources) +.PHONY: testintegration ## Runs all unit tests and integration tests exporting coverage.xml report. testintegration: docker volume create --name ${GCP_DOCKER_VOLUME} docker compose up bigquery --detach - INTEGRATION_TESTS=true pytest + INTEGRATION_TESTS=true pytest --cov-report term --cov-report=xml --cov=$(sources) docker compose down bigquery -testdocker: - docker compose run --rm test +# --------------------- +# QUALITY CHECKS +# --------------------- + +.PHONY: hooks ## Install and pre-commit hooks. +hooks: + python -m pre_commit install --install-hooks + python -m pre_commit install --hook-type commit-msg + +.PHONY: format ## Auto-format python source files according with PEP8. +format: + python -m black $(sources) + python -m ruff check --fix $(sources) + python -m ruff format $(sources) + +.PHONY: lint ## Lint python source files. +lint: + python -m ruff check $(sources) + python -m ruff format --check $(sources) + python -m black $(sources) --check --diff + +.PHONY: codespell ## Use Codespell to do spell checking. +codespell: + python -m codespell_lib -.PHONY: help gcp build docker-shell reqs upgrade-reqs venv install test testintegration testdocker +.PHONY: typecheck ## Perform type-checking. +typecheck: + python -m mypy + +.PHONY: audit ## Use pip-audit to scan for known vulnerabilities. +audit: + python -m pip_audit . + +.PHONY: pre-commit ## Run all pre-commit hooks. +pre-commit: + python -m pre_commit run --all-files + +.PHONY: all ## Run the standard set of checks performed in CI. +all: lint codespell typecheck audit test + +# --------------------- +# PACKAGE BUILD +# --------------------- + + +.PHONY: build ## Build a source distribution and a wheel distribution. +build: all clean + python -m build + +.PHONY: publish ## Publish the distribution to PyPI. +publish: build + python -m twine upload dist/* --verbose + +.PHONY: clean ## Clear local caches and build artifacts. +clean: + # remove Python file artifacts + rm -rf `find . -name __pycache__` + rm -f `find . -type f -name '*.py[co]'` + rm -f `find . -type f -name '*~'` + rm -f `find . -type f -name '.*~'` + rm -rf .cache + rm -rf .mypy_cache + rm -rf .ruff_cache + # remove build artifacts + rm -rf build + rm -rf dist + rm -rf `find . -name '*.egg-info'` + rm -rf `find . -name '*.egg'` + # remove test and coverage artifacts + rm -rf .tox/ + rm -f .coverage + rm -f .coverage.* + rm -rf coverage.* + rm -rf htmlcov/ + rm -rf .pytest_cache + rm -rf htmlcov + + +# --------------------- +# HELP +# --------------------- + +.PHONY: help ## Display this message +help: + @grep -E \ + '^.PHONY: .*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ".PHONY: |## "}; {printf "\033[36m%-19s\033[0m %s\n", $$2, $$3}' \ No newline at end of file diff --git a/README.md b/README.md index 17a26155..58f947f4 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@

- Coverage + Coverage - Python versions + Python versions Last release @@ -31,54 +31,52 @@ which are broadcasting using the same MMSI at the same time. [Semantic Versioning]: https://semver.org -If you are going to be contribuiting, jump directly to the [How to contribute](#how-to-contribute) section. -If you just want to run the pipeline, use the following instructions. +## Usage -# How to run +### System Dependencies + +Install Docker Engine using the [docker official instructions] (avoid snap packages) +and the [docker compose plugin]. No other system dependencies are required. + +### Installation First, make sure you have [git installed], and [configure a SSH-key for GitHub]. + Then, clone the repository: ```bash git clone git@github.com:GlobalFishingWatch/pipe-segment.git ``` -You can check the [examples](examples/) folder to see how to run the pipe - - -## Dependencies - -Install Docker Engine using the [docker official instructions] (avoid snap packages) -and the [docker compose plugin]. No other dependencies are required. - -## Google Cloud setup +Create virtual environment and activate it: +```shell +python -m venv .venv +. ./.venv/bin/activate +``` -The pipeline reads it's input from (and write its output to) BigQuery, -so you need to first authenticate with your google cloud account inside the docker images. +Install dependencies +```shell +make install +``` -1. Create external volume to share GCP authentication across containers: -```bash -docker volume create --name=gcp +Make sure you can run unit tests +```shell +make test ``` -2. Run authentication service -```bash -docker compose run gcloud auth application-default login +Make sure you can build the docker image: +```shell +make docker-build ``` -3. Configure the project: -```bash -docker compose run gcloud config set project world-fishing-827 -docker compose run gcloud auth application-default set-quota-project world-fishing-827 +In order to be able to connect to BigQuery, authenticate and configure the project: +```shell +make docker-gcp ``` -## Building docker image +You can check the [examples](examples/) folder to see how to run the pipe -To build the docker image, run: -```bash -docker compose build -``` -## CLI +### CLI The pipeline includes a CLI that can be used to start both local test runs and remote full runs. @@ -104,48 +102,15 @@ If you want to know the parameters of one of the processes, run for example: docker compose run dev segment --help ``` -# How to contribute +## How to contribute The [Makefile] should ease the development process. -## Git Workflow +### Git Workflow Please refer to our [git workflow documentation] to know how to manage branches in this repository. -## Setup the environment - -Create a virtual environment: -```shell -make venv -. .venv/bin/activate -``` - -Authenticate to google cloud and set up project (not necessary if you already did it on this machine): -```shell -make gcp -``` - -Install dependencies: -```shell -make install -``` - -Run unit tests: -```shell -make test -``` - -Run unit tests & integration tests (uses [bigquery-emulator]): -```shell -make testintegration -``` - -Run unit tests and integration inside docker container: -```shell -make testdocker -``` - -## Updating dependencies +### Updating dependencies The [requirements.txt] contains all transitive dependencies pinned to specific versions. This file is compiled automatically with [pip-tools], based on [requirements/prod.in]. @@ -161,7 +126,7 @@ make reqs If you want to upgrade all dependencies to latest available versions (compatible with restrictions declared), just run: ```shell -make upgrade-reqs +make reqs-upgrade ``` ## Schema diff --git a/compose.yaml b/docker-compose.yaml similarity index 61% rename from compose.yaml rename to docker-compose.yaml index 40121380..d8619164 100644 --- a/compose.yaml +++ b/docker-compose.yaml @@ -1,36 +1,26 @@ services: - build_config: &build_config + dev: + image: gfw/pipe-segment-dev build: context: . - dockerfile: Dockerfile target: dev - - dev: - image: gfw/pipe-segment-dev - <<: *build_config volumes: - ".:/opt/project" - "gcp:/root/.config/" - entrypoint: "pipe" + entrypoint: /bin/bash test: - image: gfw/pipe-segment-dev - <<: *build_config + image: gfw/pipe-segment-test environment: BIGQUERY_HOST: bigquery INTEGRATION_TESTS: true - volumes: - - ".:/opt/project" - entrypoint: "pytest" + build: + context: . + target: test + entrypoint: "pytest -v" depends_on: - bigquery - gcloud: - image: google/cloud-sdk:latest - entrypoint: gcloud - volumes: - - "gcp:/root/.config/" - bigquery: platform: linux/x86_64 ports: diff --git a/pipe_segment/version.py b/pipe_segment/version.py deleted file mode 100644 index 6a92b244..00000000 --- a/pipe_segment/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '5.0.2' diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..ef5fe91b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,176 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] +exclude = ["tests*"] +namespaces = false + +[project] +name = "pipe-segment" +version = "5.0.3" +description = "Divides vessel tracks into contiguous 'segments' separating out signals that come from two or more vessels which are broadcasting using the same MMSI at the same time." +readme = "README.md" +license = "Apache-2.0" +authors = [ + { name = "GFW Engineering Team", email = "engineering-team@globalfishingwatch.org" }, +] +maintainers = [] +classifiers = [ + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.12", + +] +requires-python = ">= 3.12" +dependencies = [ + "apache-beam[gcp]~=2.71", + "gpsdio-segment @ git+https://github.com/GlobalFishingWatch/gpsdio-segment.git@v3.0.0", + "jinja2~=3.1", + "jinja2-cli~=0.8", + "newlinejson~=1.0", + "pandas~=2.0", + "python-stdnum~=1.20", + "rich~=13.7", + "shipdataprocess~=0.8", + "ujson~=5.10", +] + +[project.urls] +Homepage = "https://github.com/GlobalFishingWatch/pipe-segment" +Documentation = "https://globalfishingwatch.github.io/pipe-segment/" +Changelog = "https://github.com/GlobalFishingWatch/pipe-segment/blob/main/CHANGELOG.md" +Repository = "https://github.com/GlobalFishingWatch/pipe-segment" +Issues = "https://github.com/GlobalFishingWatch/pipe-segment/issues" + +[project.scripts] +pipe-segment = "pipe_segment.cli.main:main" + +[project.optional-dependencies] + +# Linting and code quality tools +lint = [ + "black~=25.1", # Code formatting tool. + "isort~=6.0", # Python imports sorting tool. + "mypy~=1.15", # Static type checker. + "pydocstyle~=6.3", # Python docstring style checker. + "ruff~=0.11", # Linter and code analysis tool. + "codespell[toml]~=2.4", # Spell checker for code. + "flake8~=7.0", # Simple PEP8 checker. + "types-PyYAML", # MyPy stubs for pyyaml. +] + +# Development workflow and tools +dev = [ + "pre-commit~=4.2", # Framework for managing pre-commit hooks. + "pip-tools~=7.0", # Freezing dependencies for production containers. + "pip-audit~=2.8", # Audit for finding vulnerabilities in dependencies. + "matplotlib~=3.0", + "memory_profiler~=0.6", + "snakeviz~=2.0", +] + +# Build tools +build = [ + "build~=1.2", # Python PEP 517 compliant build system. + "setuptools~=78.1", # Python packaging library. + "twine~=6.1", # For uploading Python packages to PyPI. +] + +[tool.ruff] +fix = true +line-length = 99 +src = ["src", "tests"] +target-version = "py312" + +[tool.ruff.format] +docstring-code-format = true + +[tool.ruff.lint] +unfixable = [] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "RUF", # Ruff-specific rules + "ANN", # flake8-annotations + "C", # flake8-comprehensions + "B", # flake8-bugbear + "I", # isort + "D", # pydocstyle +] +ignore = [ + "E501", # line too long, handled by black + "C901", # too complex + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `**kwargs` +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] + +[tool.ruff.lint.isort] +lines-after-imports = 2 +lines-between-types = 1 +known-first-party = ["gfw", "tests"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.black] +target-version = ["py312"] +line-length = 99 + +[tool.isort] +profile = "black" +line_length = 99 +known_first_party = ["gfw"] +lines_after_imports = 2 +lines_between_sections = 1 +lines_between_types = 1 +ensure_newline_before_comments = true +force_sort_within_sections = true +src_paths = ["src", "tests"] + +[tool.pydocstyle] +convention = "google" + +[tool.mypy] +strict = true +ignore_missing_imports = true +files = "src" +disallow_untyped_calls = false + +[tool.pytest.ini_options] +minversion = "6.0" +testpaths = ["tests"] +addopts = "-v --cov=src --cov-report=term-missing" + +[tool.coverage.run] +source = ["src", "tests"] +branch = true +parallel = true +context = "${CONTEXT}" + +[tool.coverage.report] +precision = 0 +skip_empty = true +ignore_errors = false +show_missing = true +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", + # Don't complain if tests don't hit defensive assertion code: + "raise AssertionError", + "raise NotImplementedError", + "AbstractMethodError", + # Don't complain if non-runnable code isn't run: + "if 0:", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.codespell] +skip = '.git,env*,venv*,.venv*, build*,tmp*' \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index 54bec378..c42c140e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,7 @@ [pytest] -addopts = -v --cov=pipe_segment --cov-report=term-missing --ignore=sandbox +addopts = -v --cov=src --cov-report=term-missing filterwarnings = # Because bad naming of apache beam TestPipeline, pytest tries to discover it as a test. - ignore::pytest.PytestCollectionWarning \ No newline at end of file + ignore::pytest.PytestCollectionWarning + ignore::pyparsing.PyparsingDeprecationWarning: \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..c5602147 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +pytest # Core testing framework. +pytest-cov # Coverage plugin for pytest. +pytest-mock # Mocking plugin for pytest. diff --git a/requirements.txt b/requirements.txt index 3f90bf28..d5aeffab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,52 +1,69 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --output-file=requirements.txt requirements/prod.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile -o requirements.txt pyproject.toml +aiofiles==25.1.0 + # via cloud-sql-python-connector +aiohappyeyeballs==2.6.1 + # via aiohttp +aiohttp==3.13.3 + # via cloud-sql-python-connector +aiosignal==1.4.0 + # via aiohttp annotated-types==0.7.0 # via pydantic -apache-beam[gcp]==2.56.0 - # via -r requirements/prod.in -async-timeout==4.0.3 - # via redis -attrs==23.2.0 - # via - # jsonschema - # referencing -backports-zoneinfo==0.2.1 ; python_version < "3.9" - # via - # -r requirements/prod.in - # tzlocal -cachetools==5.3.3 - # via - # apache-beam - # google-auth -certifi==2024.2.2 - # via requests -charset-normalizer==3.3.2 - # via requests -cloudpickle==2.2.1 +anyio==4.12.1 + # via + # google-genai + # httpx +apache-beam==2.71.0 + # via pipe-segment (pyproject.toml) +asn1crypto==1.5.1 + # via scramp +attrs==25.4.0 + # via aiohttp +beartype==0.22.9 # via apache-beam -crcmod==1.7 +betterproto==1.2.5 + # via envoy-data-plane +cachetools==6.2.6 # via apache-beam -dill==0.3.1.1 +certifi==2026.2.25 + # via + # httpcore + # httpx + # requests +cffi==2.0.0 + # via cryptography +charset-normalizer==3.4.4 + # via requests +cloud-sql-python-connector==1.20.0 # via apache-beam -dnspython==2.6.1 - # via pymongo -docopt==0.6.2 - # via hdfs -docstring-parser==0.16 +cryptography==46.0.5 + # via + # apache-beam + # cloud-sql-python-connector + # google-auth + # secretstorage +distro==1.9.0 + # via google-genai +dnspython==2.8.0 + # via + # cloud-sql-python-connector + # pymongo +docstring-parser==0.17.0 # via google-cloud-aiplatform -exceptiongroup==1.2.1 - # via pytest -fastavro==1.9.4 +envoy-data-plane==0.1.0 # via apache-beam -fasteners==0.19 +fastavro==1.12.1 + # via apache-beam +fasteners==0.20 # via # apache-beam # google-apitools -google-api-core[grpc]==2.19.0 +frozenlist==1.8.0 + # via + # aiohttp + # aiosignal +google-api-core==2.30.0 # via # apache-beam # google-cloud-aiplatform @@ -56,47 +73,58 @@ google-api-core[grpc]==2.19.0 # google-cloud-core # google-cloud-datastore # google-cloud-dlp + # google-cloud-kms # google-cloud-language + # google-cloud-monitoring # google-cloud-pubsub # google-cloud-pubsublite # google-cloud-recommendations-ai # google-cloud-resource-manager + # google-cloud-secret-manager # google-cloud-spanner # google-cloud-storage # google-cloud-videointelligence # google-cloud-vision google-apitools==0.5.31 # via apache-beam -google-auth==2.29.0 +google-auth==2.48.0 # via # apache-beam + # cloud-sql-python-connector # google-api-core # google-auth-httplib2 # google-cloud-aiplatform # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-core + # google-cloud-datastore # google-cloud-dlp + # google-cloud-kms # google-cloud-language + # google-cloud-monitoring # google-cloud-pubsub # google-cloud-recommendations-ai # google-cloud-resource-manager + # google-cloud-secret-manager # google-cloud-storage # google-cloud-videointelligence # google-cloud-vision -google-auth-httplib2==0.2.0 + # google-genai + # keyrings-google-artifactregistry-auth +google-auth-httplib2==0.2.1 # via apache-beam -google-cloud-aiplatform==1.52.0 +google-cloud-aiplatform==1.140.0 # via apache-beam -google-cloud-bigquery==3.23.1 +google-cloud-bigquery==3.40.1 # via # apache-beam # google-cloud-aiplatform -google-cloud-bigquery-storage==2.25.0 +google-cloud-bigquery-storage==2.36.2 # via apache-beam -google-cloud-bigtable==2.23.1 +google-cloud-bigtable==2.35.0 # via apache-beam -google-cloud-core==2.4.1 +google-cloud-core==2.5.0 # via # apache-beam # google-cloud-bigquery @@ -104,132 +132,211 @@ google-cloud-core==2.4.1 # google-cloud-datastore # google-cloud-spanner # google-cloud-storage -google-cloud-datastore==2.19.0 +google-cloud-datastore==2.23.0 # via apache-beam -google-cloud-dlp==3.18.0 +google-cloud-dlp==3.34.0 # via apache-beam -google-cloud-language==2.13.3 +google-cloud-kms==3.11.0 # via apache-beam -google-cloud-pubsub==2.21.1 +google-cloud-language==2.19.0 + # via apache-beam +google-cloud-monitoring==2.29.1 + # via google-cloud-spanner +google-cloud-pubsub==2.35.0 # via # apache-beam # google-cloud-pubsublite -google-cloud-pubsublite==1.10.0 +google-cloud-pubsublite==1.13.0 # via apache-beam -google-cloud-recommendations-ai==0.10.10 +google-cloud-recommendations-ai==0.10.18 # via apache-beam -google-cloud-resource-manager==1.12.3 +google-cloud-resource-manager==1.16.0 # via google-cloud-aiplatform -google-cloud-spanner==3.46.0 +google-cloud-secret-manager==2.26.0 # via apache-beam -google-cloud-storage==2.16.0 +google-cloud-spanner==3.63.0 + # via apache-beam +google-cloud-storage==2.19.0 # via # apache-beam # google-cloud-aiplatform -google-cloud-videointelligence==2.13.3 +google-cloud-videointelligence==2.18.0 # via apache-beam -google-cloud-vision==3.7.2 +google-cloud-vision==3.12.1 # via apache-beam -google-crc32c==1.5.0 +google-crc32c==1.8.0 # via + # google-cloud-bigtable # google-cloud-storage # google-resumable-media -google-resumable-media==2.7.0 +google-genai==1.66.0 + # via google-cloud-aiplatform +google-resumable-media==2.8.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.63.0 +googleapis-common-protos==1.72.0 # via # google-api-core # grpc-google-iam-v1 # grpcio-status -gpsdio-segment @ https://codeload.github.com/GlobalFishingWatch/gpsdio-segment/tar.gz/v3.0.0 - # via -r requirements/prod.in -grpc-google-iam-v1==0.13.0 +gpsdio-segment @ git+https://github.com/GlobalFishingWatch/gpsdio-segment.git@16864b97a009544e9659f2b36349ae70b4400f7c + # via pipe-segment (pyproject.toml) +grpc-google-iam-v1==0.14.3 # via # google-cloud-bigtable + # google-cloud-kms # google-cloud-pubsub # google-cloud-resource-manager + # google-cloud-secret-manager # google-cloud-spanner grpc-interceptor==0.15.4 # via google-cloud-spanner -grpcio==1.64.0 +grpcio==1.65.5 # via # apache-beam # google-api-core + # google-cloud-bigquery-storage + # google-cloud-datastore + # google-cloud-dlp + # google-cloud-kms + # google-cloud-language + # google-cloud-monitoring # google-cloud-pubsub # google-cloud-pubsublite + # google-cloud-resource-manager + # google-cloud-secret-manager + # google-cloud-videointelligence + # google-cloud-vision # googleapis-common-protos # grpc-google-iam-v1 # grpc-interceptor # grpcio-status -grpcio-status==1.62.2 +grpcio-status==1.62.3 # via # google-api-core # google-cloud-pubsub # google-cloud-pubsublite -hdfs==2.7.3 - # via apache-beam +grpclib==0.4.9 + # via betterproto +h11==0.16.0 + # via httpcore +h2==4.3.0 + # via grpclib +hpack==4.1.0 + # via h2 +httpcore==1.0.9 + # via httpx httplib2==0.22.0 # via # apache-beam # google-apitools # google-auth-httplib2 # oauth2client -idna==3.7 - # via requests -importlib-resources==6.4.0 - # via - # jsonschema - # jsonschema-specifications -iniconfig==2.0.0 +httpx==0.28.1 + # via google-genai +hyperframe==6.1.0 + # via h2 +idna==3.11 + # via + # anyio + # httpx + # requests + # yarl +importlib-metadata==8.7.1 + # via opentelemetry-api +iniconfig==2.3.0 # via pytest -jinja2==3.1.4 - # via - # -r requirements/prod.in +jaraco-classes==3.4.0 + # via keyring +jaraco-context==6.1.0 + # via keyring +jaraco-functools==4.4.0 + # via keyring +jeepney==0.9.0 + # via + # keyring + # secretstorage +jinja2==3.1.6 + # via + # pipe-segment (pyproject.toml) # jinja2-cli jinja2-cli==0.8.2 - # via -r requirements/prod.in -js2py==0.74 - # via apache-beam -jsonpickle==3.0.4 + # via pipe-segment (pyproject.toml) +jsonpickle==3.4.2 # via apache-beam -jsonschema==4.22.0 +keyring==25.7.0 + # via keyrings-google-artifactregistry-auth +keyrings-google-artifactregistry-auth==1.1.2 # via apache-beam -jsonschema-specifications==2023.12.1 - # via jsonschema -markdown-it-py==3.0.0 +markdown-it-py==4.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.3 # via jinja2 mdurl==0.1.2 # via markdown-it-py +mmh3==5.2.1 + # via google-cloud-spanner +more-itertools==10.8.0 + # via + # jaraco-classes + # jaraco-functools +multidict==6.7.1 + # via + # aiohttp + # grpclib + # yarl newlinejson==1.0 - # via -r requirements/prod.in -numpy==1.24.4 + # via pipe-segment (pyproject.toml) +numpy==2.4.2 # via # apache-beam - # pyarrow - # shapely + # pandas oauth2client==4.1.3 # via google-apitools -objsize==0.7.0 +objsize==0.7.1 # via apache-beam -orjson==3.10.3 +opentelemetry-api==1.40.0 + # via + # google-cloud-pubsub + # google-cloud-spanner + # opentelemetry-resourcedetector-gcp + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-resourcedetector-gcp==1.11.0a0 + # via google-cloud-spanner +opentelemetry-sdk==1.40.0 + # via + # google-cloud-pubsub + # google-cloud-spanner + # opentelemetry-resourcedetector-gcp +opentelemetry-semantic-conventions==0.61b0 + # via + # google-cloud-spanner + # opentelemetry-sdk +orjson==3.11.7 # via apache-beam overrides==7.7.0 # via google-cloud-pubsublite -packaging==24.0 +packaging==26.0 # via # apache-beam # google-cloud-aiplatform # google-cloud-bigquery # pytest -pkgutil-resolve-name==1.3.10 - # via jsonschema -pluggy==1.5.0 - # via pytest -proto-plus==1.23.0 +pandas==2.3.3 + # via pipe-segment (pyproject.toml) +pg8000==1.31.5 + # via apache-beam +pluggy==1.6.0 + # via + # keyrings-google-artifactregistry-auth + # pytest +propcache==0.4.1 + # via + # aiohttp + # yarl +proto-plus==1.27.1 # via # apache-beam # google-api-core @@ -238,14 +345,18 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-dlp + # google-cloud-kms # google-cloud-language + # google-cloud-monitoring # google-cloud-pubsub + # google-cloud-pubsublite # google-cloud-recommendations-ai # google-cloud-resource-manager + # google-cloud-secret-manager # google-cloud-spanner # google-cloud-videointelligence # google-cloud-vision -protobuf==4.25.3 +protobuf==6.33.5 # via # apache-beam # google-api-core @@ -254,10 +365,13 @@ protobuf==4.25.3 # google-cloud-bigtable # google-cloud-datastore # google-cloud-dlp + # google-cloud-kms # google-cloud-language + # google-cloud-monitoring # google-cloud-pubsub # google-cloud-recommendations-ai # google-cloud-resource-manager + # google-cloud-secret-manager # google-cloud-spanner # google-cloud-videointelligence # google-cloud-vision @@ -265,106 +379,127 @@ protobuf==4.25.3 # grpc-google-iam-v1 # grpcio-status # proto-plus -pyarrow==14.0.2 +pyarrow==18.1.0 # via apache-beam -pyarrow-hotfix==0.6 +pyarrow-hotfix==0.7 # via apache-beam -pyasn1==0.6.0 +pyasn1==0.6.2 # via # oauth2client # pyasn1-modules # rsa -pyasn1-modules==0.4.0 +pyasn1-modules==0.4.2 # via # google-auth # oauth2client -pydantic==2.7.2 - # via google-cloud-aiplatform -pydantic-core==2.18.3 +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via + # google-cloud-aiplatform + # google-genai +pydantic-core==2.41.5 # via pydantic -pydot==1.4.2 +pygments==2.19.2 + # via + # pytest + # rich +pymongo==4.16.0 # via apache-beam -pygments==2.18.0 - # via rich -pyjsparser==2.7.1 - # via js2py -pymongo==4.7.2 +pymysql==1.1.2 # via apache-beam -pyparsing==3.1.2 - # via - # httplib2 - # pydot -pytest==8.2.1 +pyparsing==3.3.2 + # via httplib2 +pytest==9.0.2 # via shipdataprocess python-dateutil==2.9.0.post0 # via # apache-beam # google-cloud-bigquery + # pandas + # pg8000 python-stdnum==1.20 - # via -r requirements/prod.in -pytz==2024.1 + # via pipe-segment (pyproject.toml) +python-tds==1.17.1 + # via apache-beam +pytz==2026.1.post1 # via - # -r requirements/prod.in # apache-beam -redis==5.0.4 + # pandas +pyyaml==6.0.3 # via apache-beam -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -regex==2024.5.15 +regex==2026.2.28 # via apache-beam -requests==2.32.3 +requests==2.32.5 # via # apache-beam + # cloud-sql-python-connector # google-api-core + # google-auth # google-cloud-bigquery # google-cloud-storage - # hdfs -rich==13.7.1 - # via -r requirements/prod.in -roman==4.2 + # google-genai + # keyrings-google-artifactregistry-auth + # opentelemetry-resourcedetector-gcp +rich==13.9.4 + # via pipe-segment (pyproject.toml) +roman==5.2 # via shipdataprocess -rpds-py==0.18.1 - # via - # jsonschema - # referencing -rsa==4.9 +rsa==4.9.1 # via # google-auth # oauth2client -shapely==2.0.4 - # via google-cloud-aiplatform -shipdataprocess==0.8.5 - # via -r requirements/prod.in -six==1.16.0 +scramp==1.4.8 + # via pg8000 +secretstorage==3.5.0 + # via keyring +shipdataprocess==0.8.6 + # via pipe-segment (pyproject.toml) +six==1.17.0 # via # google-apitools - # hdfs - # js2py # newlinejson # oauth2client # python-dateutil -sqlparse==0.5.0 +sniffio==1.3.1 + # via google-genai +sortedcontainers==2.4.0 + # via apache-beam +sqlparse==0.5.5 # via google-cloud-spanner -tomli==2.0.1 - # via pytest -typing-extensions==4.12.0 - # via - # annotated-types +stringcase==1.2.0 + # via betterproto +tenacity==9.1.4 + # via google-genai +typing-extensions==4.15.0 + # via + # aiosignal + # anyio # apache-beam + # google-cloud-aiplatform + # google-genai + # opentelemetry-api + # opentelemetry-resourcedetector-gcp + # opentelemetry-sdk + # opentelemetry-semantic-conventions # pydantic # pydantic-core - # rich -tzlocal==5.2 - # via js2py -ujson==5.10.0 - # via -r requirements/prod.in -unidecode==1.3.8 + # typing-inspection +typing-inspection==0.4.2 + # via pydantic +tzdata==2025.3 + # via pandas +ujson==5.11.0 + # via pipe-segment (pyproject.toml) +unidecode==1.4.0 # via shipdataprocess -urllib3==2.2.1 +urllib3==2.6.3 # via requests -zipp==3.19.0 - # via importlib-resources -zstandard==0.22.0 +websockets==16.0 + # via google-genai +yarl==1.23.0 + # via aiohttp +zipp==3.23.0 + # via importlib-metadata +zstandard==0.25.0 # via apache-beam diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index 71930804..00000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -pip-tools~=7.4 -flake8~=7.0 - --r test.txt \ No newline at end of file diff --git a/requirements/prod.in b/requirements/prod.in deleted file mode 100644 index c3d9cdf4..00000000 --- a/requirements/prod.in +++ /dev/null @@ -1,11 +0,0 @@ -apache-beam[gcp]==2.56.0 -backports.zoneinfo~=0.2;python_version<"3.9" -gpsdio-segment @ https://codeload.github.com/GlobalFishingWatch/gpsdio-segment/tar.gz/v3.0.0 -jinja2~=3.1 -jinja2-cli~=0.8 -newlinejson~=1.0 -python-stdnum~=1.17 -rich~=13.7 -shipdataprocess~=0.7 -ujson~=5.9 -pytz~=2024.1 diff --git a/requirements/test.txt b/requirements/test.txt deleted file mode 100644 index f40bdc5f..00000000 --- a/requirements/test.txt +++ /dev/null @@ -1,3 +0,0 @@ -pytest~=8.2 -pytest-cov~=5.0 -pandas~=2.0 \ No newline at end of file diff --git a/requirements/worker.txt b/requirements/worker.txt deleted file mode 100644 index d64f7f75..00000000 --- a/requirements/worker.txt +++ /dev/null @@ -1,34 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --output-file=requirements/worker.txt requirements/worker.in -# -exceptiongroup==1.2.1 - # via pytest -gpsdio-segment @ https://codeload.github.com/GlobalFishingWatch/gpsdio-segment/tar.gz/v3.0.0 - # via -r requirements/worker.in -iniconfig==2.0.0 - # via pytest -jinja2==3.1.4 - # via -r requirements/worker.in -markupsafe==2.1.5 - # via jinja2 -packaging==24.0 - # via pytest -pluggy==1.5.0 - # via pytest -pytest==8.2.1 - # via shipdataprocess -python-stdnum==1.20 - # via -r requirements/worker.in -roman==4.2 - # via shipdataprocess -shipdataprocess==0.8.5 - # via -r requirements/worker.in -tomli==2.0.1 - # via pytest -ujson==5.10.0 - # via -r requirements/worker.in -unidecode==1.3.8 - # via shipdataprocess diff --git a/setup.py b/setup.py index 3f9eebd1..a2e7a98e 100644 --- a/setup.py +++ b/setup.py @@ -1,49 +1,5 @@ -#!/usr/bin/env python +"""This setup.py is added for compatibility with Apache Beam dataflow pipelines.""" import setuptools -from pipe_segment.version import __version__ - -setuptools.setup( - name="pipe_segment", - version=__version__, - author="Global Fishing Watch.", - # author_email="", - # maintainer="", - description=( - "Divides vessel tracks into contiguous 'segments' " - "separating out signals that come from two or more vessels " - "which are broadcasting using the same MMSI at the same time." - ), - long_description_content_type="text/markdown", - url="https://github.com/GlobalFishingWatch/pipe-segment", - packages=setuptools.find_packages(exclude=["test*.*", "tests"]), - include_package_data=True, - classifiers=[ - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Operating System :: OS Independent", - ], - install_requires=[ - 'apache-beam[gcp]<3', - 'backports.zoneinfo<1;python_version<"3.9"', - 'gpsdio-segment ' - '@ https://codeload.github.com/GlobalFishingWatch/gpsdio-segment/tar.gz/v3.0.0', - 'jinja2<4', - 'jinja2-cli<1', - 'newlinejson<2', - 'python-stdnum<2', - 'rich<14', - 'shipdataprocess<1', - 'ujson<6', - ], - entry_points={ - 'console_scripts': [ - 'pipe = pipe_segment.cli.cli:main', - ] - }, -) +setuptools.setup() diff --git a/pipe_segment/__init__.py b/src/pipe_segment/__init__.py similarity index 100% rename from pipe_segment/__init__.py rename to src/pipe_segment/__init__.py diff --git a/pipe_segment/segment_identity/__init__.py b/src/pipe_segment/assets/__init__.py similarity index 100% rename from pipe_segment/segment_identity/__init__.py rename to src/pipe_segment/assets/__init__.py diff --git a/assets/examples/small_test_set.csv b/src/pipe_segment/assets/examples/small_test_set.csv similarity index 100% rename from assets/examples/small_test_set.csv rename to src/pipe_segment/assets/examples/small_test_set.csv diff --git a/assets/examples/small_test_set.md b/src/pipe_segment/assets/examples/small_test_set.md similarity index 100% rename from assets/examples/small_test_set.md rename to src/pipe_segment/assets/examples/small_test_set.md diff --git a/pipe_segment/transform/__init__.py b/src/pipe_segment/assets/queries/__init__.py similarity index 100% rename from pipe_segment/transform/__init__.py rename to src/pipe_segment/assets/queries/__init__.py diff --git a/assets/queries/segment_info.sql.j2 b/src/pipe_segment/assets/queries/segment_info.sql.j2 similarity index 100% rename from assets/queries/segment_info.sql.j2 rename to src/pipe_segment/assets/queries/segment_info.sql.j2 diff --git a/assets/queries/segment_vessel.sql.j2 b/src/pipe_segment/assets/queries/segment_vessel.sql.j2 similarity index 100% rename from assets/queries/segment_vessel.sql.j2 rename to src/pipe_segment/assets/queries/segment_vessel.sql.j2 diff --git a/assets/queries/segment_vessel_daily.sql.j2 b/src/pipe_segment/assets/queries/segment_vessel_daily.sql.j2 similarity index 100% rename from assets/queries/segment_vessel_daily.sql.j2 rename to src/pipe_segment/assets/queries/segment_vessel_daily.sql.j2 diff --git a/assets/queries/util.sql.j2 b/src/pipe_segment/assets/queries/util.sql.j2 similarity index 100% rename from assets/queries/util.sql.j2 rename to src/pipe_segment/assets/queries/util.sql.j2 diff --git a/assets/queries/validate_vessel_identity.sql.j2 b/src/pipe_segment/assets/queries/validate_vessel_identity.sql.j2 similarity index 100% rename from assets/queries/validate_vessel_identity.sql.j2 rename to src/pipe_segment/assets/queries/validate_vessel_identity.sql.j2 diff --git a/assets/queries/vessel_info.sql.j2 b/src/pipe_segment/assets/queries/vessel_info.sql.j2 similarity index 100% rename from assets/queries/vessel_info.sql.j2 rename to src/pipe_segment/assets/queries/vessel_info.sql.j2 diff --git a/pipe_segment/transform/assets/satellite_offsets.sql.j2 b/src/pipe_segment/assets/satellite_offsets.sql.j2 similarity index 100% rename from pipe_segment/transform/assets/satellite_offsets.sql.j2 rename to src/pipe_segment/assets/satellite_offsets.sql.j2 diff --git a/pipe_segment/transform/assets/__init__.py b/src/pipe_segment/assets/schemas/__init__.py similarity index 100% rename from pipe_segment/transform/assets/__init__.py rename to src/pipe_segment/assets/schemas/__init__.py diff --git a/assets/schemas/segment_info.schema.json b/src/pipe_segment/assets/schemas/segment_info.schema.json similarity index 100% rename from assets/schemas/segment_info.schema.json rename to src/pipe_segment/assets/schemas/segment_info.schema.json diff --git a/assets/schemas/segment_vessel.schema.json b/src/pipe_segment/assets/schemas/segment_vessel.schema.json similarity index 100% rename from assets/schemas/segment_vessel.schema.json rename to src/pipe_segment/assets/schemas/segment_vessel.schema.json diff --git a/assets/schemas/segment_vessel_daily.schema.json b/src/pipe_segment/assets/schemas/segment_vessel_daily.schema.json similarity index 100% rename from assets/schemas/segment_vessel_daily.schema.json rename to src/pipe_segment/assets/schemas/segment_vessel_daily.schema.json diff --git a/assets/schemas/vessel_info.schema.json b/src/pipe_segment/assets/schemas/vessel_info.schema.json similarity index 100% rename from assets/schemas/vessel_info.schema.json rename to src/pipe_segment/assets/schemas/vessel_info.schema.json diff --git a/src/pipe_segment/cli/__init__.py b/src/pipe_segment/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipe_segment/cli/commands/__init__.py b/src/pipe_segment/cli/commands/__init__.py similarity index 100% rename from pipe_segment/cli/commands/__init__.py rename to src/pipe_segment/cli/commands/__init__.py diff --git a/pipe_segment/cli/commands/base.py b/src/pipe_segment/cli/commands/base.py similarity index 100% rename from pipe_segment/cli/commands/base.py rename to src/pipe_segment/cli/commands/base.py diff --git a/pipe_segment/cli/commands/segment.py b/src/pipe_segment/cli/commands/segment.py similarity index 100% rename from pipe_segment/cli/commands/segment.py rename to src/pipe_segment/cli/commands/segment.py diff --git a/pipe_segment/cli/commands/segment_identity.py b/src/pipe_segment/cli/commands/segment_identity.py similarity index 100% rename from pipe_segment/cli/commands/segment_identity.py rename to src/pipe_segment/cli/commands/segment_identity.py diff --git a/pipe_segment/cli/commands/segment_info.py b/src/pipe_segment/cli/commands/segment_info.py similarity index 100% rename from pipe_segment/cli/commands/segment_info.py rename to src/pipe_segment/cli/commands/segment_info.py diff --git a/pipe_segment/cli/commands/segment_vessel.py b/src/pipe_segment/cli/commands/segment_vessel.py similarity index 100% rename from pipe_segment/cli/commands/segment_vessel.py rename to src/pipe_segment/cli/commands/segment_vessel.py diff --git a/pipe_segment/cli/commands/segment_vessel_daily.py b/src/pipe_segment/cli/commands/segment_vessel_daily.py similarity index 100% rename from pipe_segment/cli/commands/segment_vessel_daily.py rename to src/pipe_segment/cli/commands/segment_vessel_daily.py diff --git a/pipe_segment/cli/commands/validator.py b/src/pipe_segment/cli/commands/validator.py similarity index 100% rename from pipe_segment/cli/commands/validator.py rename to src/pipe_segment/cli/commands/validator.py diff --git a/pipe_segment/cli/commands/vessel_info.py b/src/pipe_segment/cli/commands/vessel_info.py similarity index 100% rename from pipe_segment/cli/commands/vessel_info.py rename to src/pipe_segment/cli/commands/vessel_info.py diff --git a/pipe_segment/cli/cli.py b/src/pipe_segment/cli/main.py similarity index 100% rename from pipe_segment/cli/cli.py rename to src/pipe_segment/cli/main.py diff --git a/pipe_segment/pipeline.py b/src/pipe_segment/pipeline.py similarity index 100% rename from pipe_segment/pipeline.py rename to src/pipe_segment/pipeline.py diff --git a/src/pipe_segment/schemas/__init__.py b/src/pipe_segment/schemas/__init__.py new file mode 100644 index 00000000..5775da06 --- /dev/null +++ b/src/pipe_segment/schemas/__init__.py @@ -0,0 +1,7 @@ +from importlib import resources + +SCHEMAS_FOLDER = resources.files('pipe_segment.assets.schemas') + + +def get_schema_path(filename: str): + return str(SCHEMAS_FOLDER / filename) diff --git a/pipe_segment/schemas/message_schema.py b/src/pipe_segment/schemas/message_schema.py similarity index 100% rename from pipe_segment/schemas/message_schema.py rename to src/pipe_segment/schemas/message_schema.py diff --git a/pipe_segment/schemas/segment_schema.py b/src/pipe_segment/schemas/segment_schema.py similarity index 100% rename from pipe_segment/schemas/segment_schema.py rename to src/pipe_segment/schemas/segment_schema.py diff --git a/src/pipe_segment/segment_identity/__init__.py b/src/pipe_segment/segment_identity/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipe_segment/segment_identity/pipeline.py b/src/pipe_segment/segment_identity/pipeline.py similarity index 100% rename from pipe_segment/segment_identity/pipeline.py rename to src/pipe_segment/segment_identity/pipeline.py diff --git a/pipe_segment/segment_identity/read_source.py b/src/pipe_segment/segment_identity/read_source.py similarity index 100% rename from pipe_segment/segment_identity/read_source.py rename to src/pipe_segment/segment_identity/read_source.py diff --git a/pipe_segment/segment_identity/transforms.py b/src/pipe_segment/segment_identity/transforms.py similarity index 100% rename from pipe_segment/segment_identity/transforms.py rename to src/pipe_segment/segment_identity/transforms.py diff --git a/src/pipe_segment/segment_info/__init__.py b/src/pipe_segment/segment_info/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipe_segment/segment_info/segment_info.py b/src/pipe_segment/segment_info/segment_info.py similarity index 94% rename from pipe_segment/segment_info/segment_info.py rename to src/pipe_segment/segment_info/segment_info.py index a6ca077f..15dd38b3 100644 --- a/pipe_segment/segment_info/segment_info.py +++ b/src/pipe_segment/segment_info/segment_info.py @@ -2,10 +2,11 @@ from pipe_segment.utils.bq_tools import BigQueryHelper, Schemas, SimpleTable from pipe_segment.utils.template_tools import format_query from pipe_segment.version import __version__ +from pipe_segment.schemas import get_schema_path import logging logger = logging.getLogger(__name__) -SCHEMA_PATH = "./assets/schemas/segment_info.schema.json" +SCHEMA_PATH = get_schema_path("segment_info.schema.json") QUERY = "segment_info.sql.j2" CLUSTERING_FIELDS = "seg_id" diff --git a/pipe_segment/segment_vessel/__init__.py b/src/pipe_segment/segment_vessel/__init__.py similarity index 100% rename from pipe_segment/segment_vessel/__init__.py rename to src/pipe_segment/segment_vessel/__init__.py diff --git a/pipe_segment/segment_vessel/segment_vessel.py b/src/pipe_segment/segment_vessel/segment_vessel.py similarity index 94% rename from pipe_segment/segment_vessel/segment_vessel.py rename to src/pipe_segment/segment_vessel/segment_vessel.py index fb9f4c6d..da84aaaa 100644 --- a/pipe_segment/segment_vessel/segment_vessel.py +++ b/src/pipe_segment/segment_vessel/segment_vessel.py @@ -2,10 +2,11 @@ from pipe_segment.utils.bq_tools import BigQueryHelper, Schemas, SimpleTable from pipe_segment.utils.template_tools import format_query from pipe_segment.version import __version__ +from pipe_segment.schemas import get_schema_path import logging logger = logging.getLogger(__name__) -SCHEMA_PATH = "./assets/schemas/segment_vessel.schema.json" +SCHEMA_PATH = get_schema_path("segment_vessel.schema.json") QUERY = "segment_vessel.sql.j2" diff --git a/pipe_segment/segment_vessel/segment_vessel_daily.py b/src/pipe_segment/segment_vessel/segment_vessel_daily.py similarity index 95% rename from pipe_segment/segment_vessel/segment_vessel_daily.py rename to src/pipe_segment/segment_vessel/segment_vessel_daily.py index 475b1e11..f8f39cd7 100644 --- a/pipe_segment/segment_vessel/segment_vessel_daily.py +++ b/src/pipe_segment/segment_vessel/segment_vessel_daily.py @@ -5,10 +5,11 @@ from pipe_segment.utils.template_tools import format_query from pipe_segment.version import __version__ from datetime import timedelta +from pipe_segment.schemas import get_schema_path import logging logger = logging.getLogger(__name__) -SCHEMA_PATH = "./assets/schemas/segment_vessel_daily.schema.json" +SCHEMA_PATH = get_schema_path("segment_vessel_daily.schema.json") QUERY = "segment_vessel_daily.sql.j2" PARTITION_FIELD = "day" diff --git a/pipe_segment/tools.py b/src/pipe_segment/tools.py similarity index 100% rename from pipe_segment/tools.py rename to src/pipe_segment/tools.py diff --git a/src/pipe_segment/transform/__init__.py b/src/pipe_segment/transform/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipe_segment/transform/create_segment_map.py b/src/pipe_segment/transform/create_segment_map.py similarity index 100% rename from pipe_segment/transform/create_segment_map.py rename to src/pipe_segment/transform/create_segment_map.py diff --git a/pipe_segment/transform/create_segments.py b/src/pipe_segment/transform/create_segments.py similarity index 100% rename from pipe_segment/transform/create_segments.py rename to src/pipe_segment/transform/create_segments.py diff --git a/pipe_segment/transform/filter_bad_satellite_times.py b/src/pipe_segment/transform/filter_bad_satellite_times.py similarity index 100% rename from pipe_segment/transform/filter_bad_satellite_times.py rename to src/pipe_segment/transform/filter_bad_satellite_times.py diff --git a/pipe_segment/transform/fragment.py b/src/pipe_segment/transform/fragment.py similarity index 100% rename from pipe_segment/transform/fragment.py rename to src/pipe_segment/transform/fragment.py diff --git a/pipe_segment/transform/fragment_implementation.py b/src/pipe_segment/transform/fragment_implementation.py similarity index 100% rename from pipe_segment/transform/fragment_implementation.py rename to src/pipe_segment/transform/fragment_implementation.py diff --git a/pipe_segment/transform/invalid_values.py b/src/pipe_segment/transform/invalid_values.py similarity index 100% rename from pipe_segment/transform/invalid_values.py rename to src/pipe_segment/transform/invalid_values.py diff --git a/pipe_segment/transform/read_fragments.py b/src/pipe_segment/transform/read_fragments.py similarity index 100% rename from pipe_segment/transform/read_fragments.py rename to src/pipe_segment/transform/read_fragments.py diff --git a/pipe_segment/transform/read_messages.py b/src/pipe_segment/transform/read_messages.py similarity index 100% rename from pipe_segment/transform/read_messages.py rename to src/pipe_segment/transform/read_messages.py diff --git a/pipe_segment/transform/satellite_offsets.py b/src/pipe_segment/transform/satellite_offsets.py similarity index 98% rename from pipe_segment/transform/satellite_offsets.py rename to src/pipe_segment/transform/satellite_offsets.py index 6e1731b1..46322924 100644 --- a/pipe_segment/transform/satellite_offsets.py +++ b/src/pipe_segment/transform/satellite_offsets.py @@ -103,7 +103,7 @@ def expand(self, xs): ] | "MergeSatOffsets" >> beam.Flatten() def _sat_offset_iter(self): - with resources.path('pipe_segment.transform.assets', 'satellite_offsets.sql.j2') as t: + with resources.path('pipe_segment.assets', 'satellite_offsets.sql.j2') as t: with open(t) as f: template = Template(f.read()) diff --git a/pipe_segment/transform/tag_with_fragid_and_timebin.py b/src/pipe_segment/transform/tag_with_fragid_and_timebin.py similarity index 100% rename from pipe_segment/transform/tag_with_fragid_and_timebin.py rename to src/pipe_segment/transform/tag_with_fragid_and_timebin.py diff --git a/pipe_segment/transform/tag_with_seg_id.py b/src/pipe_segment/transform/tag_with_seg_id.py similarity index 100% rename from pipe_segment/transform/tag_with_seg_id.py rename to src/pipe_segment/transform/tag_with_seg_id.py diff --git a/pipe_segment/transform/util.py b/src/pipe_segment/transform/util.py similarity index 100% rename from pipe_segment/transform/util.py rename to src/pipe_segment/transform/util.py diff --git a/pipe_segment/transform/whitelist_messages_segmented.py b/src/pipe_segment/transform/whitelist_messages_segmented.py similarity index 100% rename from pipe_segment/transform/whitelist_messages_segmented.py rename to src/pipe_segment/transform/whitelist_messages_segmented.py diff --git a/src/pipe_segment/utils/__init__.py b/src/pipe_segment/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipe_segment/utils/bq_source.py b/src/pipe_segment/utils/bq_source.py similarity index 100% rename from pipe_segment/utils/bq_source.py rename to src/pipe_segment/utils/bq_source.py diff --git a/pipe_segment/utils/bq_tools.py b/src/pipe_segment/utils/bq_tools.py similarity index 100% rename from pipe_segment/utils/bq_tools.py rename to src/pipe_segment/utils/bq_tools.py diff --git a/pipe_segment/utils/template_tools.py b/src/pipe_segment/utils/template_tools.py similarity index 85% rename from pipe_segment/utils/template_tools.py rename to src/pipe_segment/utils/template_tools.py index fc347cb6..15a2ec12 100644 --- a/pipe_segment/utils/template_tools.py +++ b/src/pipe_segment/utils/template_tools.py @@ -1,6 +1,6 @@ from jinja2 import ( Environment, - FileSystemLoader, + PackageLoader, StrictUndefined, ) @@ -15,7 +15,7 @@ def format_query(template_file: str, **params) -> str: :param params: The dictionary of params to replace in the template. """ jinja2_env = Environment( - loader=FileSystemLoader(["./assets/queries/"]), undefined=StrictUndefined + loader=PackageLoader("pipe_segment", "assets/queries"), undefined=StrictUndefined ) sql_template = jinja2_env.get_template(template_file) formatted_template = sql_template.render(params) diff --git a/src/pipe_segment/version.py b/src/pipe_segment/version.py new file mode 100644 index 00000000..e3cd9a0a --- /dev/null +++ b/src/pipe_segment/version.py @@ -0,0 +1,5 @@ +"""Holds the current version of the program.""" +import importlib.metadata + + +__version__ = importlib.metadata.version("pipe-segment") diff --git a/src/pipe_segment/vessel_info/__init__.py b/src/pipe_segment/vessel_info/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipe_segment/vessel_info/vessel_info.py b/src/pipe_segment/vessel_info/vessel_info.py similarity index 94% rename from pipe_segment/vessel_info/vessel_info.py rename to src/pipe_segment/vessel_info/vessel_info.py index 4dcb0527..bd84c19f 100644 --- a/pipe_segment/vessel_info/vessel_info.py +++ b/src/pipe_segment/vessel_info/vessel_info.py @@ -2,10 +2,11 @@ from pipe_segment.utils.bq_tools import BigQueryHelper, Schemas, SimpleTable from pipe_segment.utils.template_tools import format_query from pipe_segment.version import __version__ +from pipe_segment.schemas import get_schema_path import logging logger = logging.getLogger(__name__) -SCHEMA_PATH = "./assets/schemas/vessel_info.schema.json" +SCHEMA_PATH = get_schema_path("vessel_info.schema.json") QUERY = "vessel_info.sql.j2" diff --git a/tests/test_cli.py b/tests/test_cli.py index 146d2f73..2cbe6e0a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,7 +1,7 @@ import os import pytest -from pipe_segment.cli import cli +from pipe_segment.cli import main from pipe_segment import pipeline from pipe_segment.segment_identity import pipeline as identity_pipeline @@ -14,7 +14,7 @@ def test_cli(monkeypatch, tmp_path): log_file = os.path.join(tmp_path, 'segment.log') dummy_table = 'dummy_proj.dummy_dataset.dummy_table' - cli.run([ + main.run([ '-v', '--log_file', log_file, 'segment', @@ -28,7 +28,7 @@ def test_cli(monkeypatch, tmp_path): # TODO: replace this monkey patch when design allows for more easy testing. monkeypatch.setattr(identity_pipeline, "run", lambda *x, **y: 0) - cli.run([ + main.run([ '-v', '--log_file', log_file, 'segment_identity', @@ -40,4 +40,4 @@ def test_cli(monkeypatch, tmp_path): assert os.path.exists(log_file) with pytest.raises(SystemExit): - cli.run([]) + main.run([])