From e75d1ed7cf38feea61b39f77188285831ae356ee Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 8 Feb 2023 16:17:26 -0500 Subject: [PATCH 1/4] Add `--check_python_reqs` option to build_image.py --- build_image.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/build_image.py b/build_image.py index 1ebe4d70d..941dcde12 100644 --- a/build_image.py +++ b/build_image.py @@ -31,6 +31,7 @@ class OfrakImageConfig: build_finish: bool # Whether to supply --no-cache to docker build commands no_cache: bool + check_python_reqs: bool extra_build_args: Optional[List[str]] install_target: InstallTarget cache_from: List[str] @@ -140,6 +141,7 @@ def parse_args() -> OfrakImageConfig: parser.add_argument("--base", action="store_true") parser.add_argument("--finish", action="store_true") parser.add_argument("--no-cache", action="store_true") + parser.add_argument("--check-python-reqs", action="store_true") parser.add_argument( "--target", choices=[InstallTarget.DEVELOP.value, InstallTarget.INSTALL.value], @@ -157,6 +159,7 @@ def parse_args() -> OfrakImageConfig: args.base, args.finish, args.no_cache, + args.check_python_reqs, config_dict.get("extra_build_args"), InstallTarget(args.target), args.cache_from, @@ -233,6 +236,7 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: def create_dockerfile_finish(config: OfrakImageConfig) -> str: full_base_image_name = "/".join((config.registry, config.base_image_name)) dockerfile_finish_parts = [ + "# syntax = docker/dockerfile:1.3\n\n", f"FROM {full_base_image_name}:{GIT_COMMIT_HASH}\n\n", f"ARG OFRAK_SRC_DIR=/\n", ] @@ -252,8 +256,19 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: "\\n", ] ) - dockerfile_finish_parts.append(f'RUN printf "{develop_makefile}" >> Makefile\n') - dockerfile_finish_parts.append("RUN make $INSTALL_TARGET\n\n") + dockerfile_finish_parts += [ + f'RUN printf "{develop_makefile}" >> Makefile\n\n', + ] + if config.check_python_reqs: + dockerfile_finish_parts += [ + '# We use --network="none" to ensure all dependencies were installed in base.Dockerfile\n', + 'RUN --network="none" make $INSTALL_TARGET\n', + "RUN python3 -m pip check\n\n", + ] + else: + dockerfile_finish_parts += [ + "RUN make $INSTALL_TARGET\n\n", + ] finish_makefile = "\\n\\\n".join( [ "test:", From 8a31e0eacb765440b8616a298aa90ddc40c94695 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 12:23:03 -0500 Subject: [PATCH 2/4] Addressing PR feedback --- build_image.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/build_image.py b/build_image.py index d073af842..93ebab3c8 100644 --- a/build_image.py +++ b/build_image.py @@ -32,7 +32,6 @@ class OfrakImageConfig: build_finish: bool # Whether to supply --no-cache to docker build commands no_cache: bool - check_python_reqs: bool extra_build_args: Optional[List[str]] install_target: InstallTarget cache_from: List[str] @@ -173,7 +172,6 @@ def parse_args() -> OfrakImageConfig: args.base, args.finish, args.no_cache, - args.check_python_reqs, config_dict.get("extra_build_args"), InstallTarget(args.target), args.cache_from, @@ -277,22 +275,18 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str: "\\n", ] ) - dockerfile_finish_parts.append(f'RUN printf "{develop_makefile}" >> Makefile\n') - if config.check_python_reqs: - dockerfile_finish_parts += [ - '# We use --network="none" to ensure all dependencies were installed in base.Dockerfile\n', - 'RUN --network="none" make $INSTALL_TARGET\n', - "RUN python3 -m pip check\n\n", - ] - else: - dockerfile_finish_parts += [ - "RUN make $INSTALL_TARGET\n\n", - ] + dockerfile_finish_parts += [ + f'RUN printf "{develop_makefile}" >> Makefile\n' + '# We use --network="none" to ensure all dependencies were installed in base.Dockerfile\n', + 'RUN --network="none" make $INSTALL_TARGET\n\n', + ] test_names = " ".join([f"test_{package_name}" for package_name in package_names]) finish_makefile = "\\n\\\n".join( [ - ".PHONY: test " + test_names, - "test: " + test_names, + ".PHONY: test inspect" + test_names, + "inspect:", + "\tpython3 -m pip check", + "test: inspect" + test_names, ] + [ f"test_{package_name}:\\n\\\n\t\\$(MAKE) -C {package_name} test" From 43d3fcaf8f2e910042ddfa2fe8696022acee963c Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 15:49:53 -0500 Subject: [PATCH 3/4] Need to use a different workaround to `pip install -e`/mypy issues --- disassemblers/ofrak_angr/Makefile | 4 +++- disassemblers/ofrak_binary_ninja/Makefile | 4 +++- disassemblers/ofrak_capstone/Makefile | 4 +++- disassemblers/ofrak_ghidra/Makefile | 4 +++- ofrak_core/Makefile | 4 +++- ofrak_io/Makefile | 4 +++- ofrak_patch_maker/Makefile | 4 +++- ofrak_tutorial/Makefile | 4 +++- ofrak_type/Makefile | 4 +++- 9 files changed, 27 insertions(+), 9 deletions(-) diff --git a/disassemblers/ofrak_angr/Makefile b/disassemblers/ofrak_angr/Makefile index fb8cc4b83..4a09b1268 100644 --- a/disassemblers/ofrak_angr/Makefile +++ b/disassemblers/ofrak_angr/Makefile @@ -7,7 +7,9 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] --config-settings editable_mode=compat + # Legacy-edinable is needed to allow mypy to find packages. + # See https://github.com/python/mypy/issues/13392 + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" $(PIP) install -e .[test] .PHONY: inspect inspect: diff --git a/disassemblers/ofrak_binary_ninja/Makefile b/disassemblers/ofrak_binary_ninja/Makefile index e9c3f30a7..5d45dbc60 100644 --- a/disassemblers/ofrak_binary_ninja/Makefile +++ b/disassemblers/ofrak_binary_ninja/Makefile @@ -7,7 +7,9 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] --config-settings editable_mode=compat + # Legacy-edinable is needed to allow mypy to find packages. + # See https://github.com/python/mypy/issues/13392 + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" $(PIP) install -e .[test] .PHONY: inspect inspect: diff --git a/disassemblers/ofrak_capstone/Makefile b/disassemblers/ofrak_capstone/Makefile index 88e4cfe32..0ca3f9de7 100644 --- a/disassemblers/ofrak_capstone/Makefile +++ b/disassemblers/ofrak_capstone/Makefile @@ -7,7 +7,9 @@ install: .PHONY: develop develop: - $(PIP) install --pre -e .[test] --config-settings editable_mode=compat + # Legacy-edinable is needed to allow mypy to find packages. + # See https://github.com/python/mypy/issues/13392 + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" $(PIP) install --pre -e .[test] .PHONY: inspect inspect: diff --git a/disassemblers/ofrak_ghidra/Makefile b/disassemblers/ofrak_ghidra/Makefile index dc78fcc47..d24276e80 100644 --- a/disassemblers/ofrak_ghidra/Makefile +++ b/disassemblers/ofrak_ghidra/Makefile @@ -5,7 +5,9 @@ install: $(PIP) install . develop: - $(PIP) install -e .[test] --config-settings editable_mode=compat + # Legacy-edinable is needed to allow mypy to find packages. + # See https://github.com/python/mypy/issues/13392 + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" $(PIP) install -e .[test] test: $(PYTHON) -m pytest --cov=ofrak_ghidra --cov-report=term-missing ofrak_ghidra_test diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index 847b3cdd3..113f0248b 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -7,7 +7,9 @@ install: ofrak/gui/public .PHONY: develop develop: ofrak/gui/public - $(PIP) install -e .[docs,test] --config-settings editable_mode=compat + # Legacy-edinable is needed to allow mypy to find packages. + # See https://github.com/python/mypy/issues/13392 + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" $(PIP) install -e .[docs,test] .PHONY: inspect inspect: diff --git a/ofrak_io/Makefile b/ofrak_io/Makefile index f49e49397..e6be10d3a 100644 --- a/ofrak_io/Makefile +++ b/ofrak_io/Makefile @@ -7,7 +7,9 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] --config-settings editable_mode=compat + # Legacy-edinable is needed to allow mypy to find packages. + # See https://github.com/python/mypy/issues/13392 + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" $(PIP) install -e .[test] .PHONY: inspect inspect: diff --git a/ofrak_patch_maker/Makefile b/ofrak_patch_maker/Makefile index 5466c18f7..eb6e14aa9 100644 --- a/ofrak_patch_maker/Makefile +++ b/ofrak_patch_maker/Makefile @@ -8,7 +8,9 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] --config-settings editable_mode=compat + # Legacy-edinable is needed to allow mypy to find packages. + # See https://github.com/python/mypy/issues/13392 + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" $(PIP) install -e .[test] .PHONY: inspect inspect: diff --git a/ofrak_tutorial/Makefile b/ofrak_tutorial/Makefile index d1e70c727..4603cbcbd 100644 --- a/ofrak_tutorial/Makefile +++ b/ofrak_tutorial/Makefile @@ -29,7 +29,9 @@ run: .PHONY: develop develop: - ${PIP} install -e .[test] --config-settings editable_mode=compat + # Legacy-edinable is needed to allow mypy to find packages. + # See https://github.com/python/mypy/issues/13392 + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" ${PIP} install -e .[test] make generate_stripped_notebooks .PHONY: install diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index f448930ce..c009cd954 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -7,7 +7,9 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] --config-settings editable_mode=compat + # Legacy-edinable is needed to allow mypy to find packages. + # See https://github.com/python/mypy/issues/13392 + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" $(PIP) install -e .[test] .PHONY: inspect inspect: From 2895cdcbcb7fa1acd95b64e0c67161d287cba310 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 17:14:34 -0800 Subject: [PATCH 4/4] Update build_image.py Co-authored-by: Wyatt <53830972+whyitfor@users.noreply.github.com> --- build_image.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build_image.py b/build_image.py index 93ebab3c8..8385ab700 100644 --- a/build_image.py +++ b/build_image.py @@ -145,7 +145,6 @@ def parse_args() -> OfrakImageConfig: parser.add_argument("--base", action="store_true") parser.add_argument("--finish", action="store_true") parser.add_argument("--no-cache", action="store_true") - parser.add_argument("--check-python-reqs", action="store_true") parser.add_argument( "--target", choices=[InstallTarget.DEVELOP.value, InstallTarget.INSTALL.value],