Add plugin example with embedded python

cmake/ArchivePythonStdlib.py

@@ -1,4 +1,6 @@
 import os
+import fnmatch
+import sys
 import stat
 import shutil
 import hashlib
@@ -7,6 +9,11 @@
 from argparse import ArgumentParser
 
 
+def uniquify(seq):
+    seen = set()
+    return [x for x in seq if x and x not in seen and not seen.add(x)]
+
+
 def file_hash(file):
     h = hashlib.md5()
 
@@ -16,12 +23,41 @@ def file_hash(file):
     return h.hexdigest()
 
 
-def make_archive(file, directory):
+def cleanup_files(directory, file_patterns = None, folder_patterns = None):
+    for dirname, folders, files in os.walk(directory):
+        if folder_patterns:
+            for foldername in folders:
+                for pattern in folder_patterns:
+                    if fnmatch.fnmatch(foldername, pattern):
+                        shutil.rmtree(os.path.join(dirname, foldername))
+                        break
+
+        if file_patterns:
+            for filename in files:
+                for pattern in file_patterns:
+                    if fnmatch.fnmatch(filename, pattern):
+                        os.remove(os.path.join(dirname, filename))
+                        break
+
+
+def append_files(*args):
     archived_files = []
-    for dirname, _, files in os.walk(directory):
-        for filename in files:
-            path = os.path.join(dirname, filename)
-            archived_files.append((path, os.path.relpath(path, directory)))
+
+    for directory_or_file in args:
+        if os.path.isdir(directory_or_file):
+            for dirname, _, files in os.walk(directory_or_file):
+                for filename in files:
+                    path = os.path.join(dirname, filename)
+                    archived_files.append((path, os.path.relpath(path, directory_or_file)))
+        else:
+            assert os.path.isabs(directory_or_file)
+            archived_files.append((directory_or_file, os.path.split(directory_or_file)[1]))
+
+    return archived_files
+
+
+def make_archive(file, directory_or_file, *args):
+    archived_files = append_files(directory_or_file, *args)
 
     with zipfile.ZipFile(file, "w") as zf:
         for path, archive_path in sorted(archived_files):
@@ -42,17 +78,22 @@ def make_archive(file, directory):
     parser.add_argument("-M", "--version-major", type=int, help="Major version number (integer).")
     parser.add_argument("-m", "--version-minor", type=int, help="Minor version number (integer).")
     parser.add_argument("-x", "--exclude-patterns", type=str, default=None, help="Excluded patterns (semicolon separated list).")
+    parser.add_argument("-p", "--pip-packages", type=str, default=None, help="Pip packages to install (semicolon separated list).")
+    parser.add_argument("-z", "--zip-python-executable", action="store_true", help="Store the python executable in an archive.")
 
     args = parser.parse_args()
 
     version = f"{args.version_major}.{args.version_minor}"
     version_nodot = f"{args.version_major}{args.version_minor}"
 
-    final_location: Path = args.output_folder / "python"
-    site_packages = final_location / "site-packages"
     base_python: Path = args.base_folder / "lib" / f"python{version}"
-    final_archive = args.output_folder / f"python{version_nodot}.zip"
-    temp_archive = args.output_folder / f"temp{version_nodot}.zip"
+    final_location: Path = args.output_folder / "python"
+    site_packages: Path = final_location / "site-packages"
+
+    final_archive: Path = args.output_folder / f"python{version_nodot}.zip"
+    temp_archive: Path = args.output_folder / f"temp{version_nodot}.zip"
+    final_python_archive: Path = args.output_folder / f"python{version_nodot}_executable.zip"
+    temp_python_archive: Path = args.output_folder / f"temp{version_nodot}_executable.zip"
 
     base_patterns = [
         "*.pyc",
@@ -65,33 +106,79 @@ def make_archive(file, directory):
         "Tk*",
         "_tk*",
         "_test*",
+        "_xxtestfuzz*",
+        "doctest*",
+        "idlelib",
+        "lib2to3",
         "libpython*",
         "pkgconfig",
-        "idlelib",
+        "pydoc*",
         "site-packages",
         "test",
-        "turtledemo",
+        "turtle*",
         "LICENSE.txt",
     ]
 
     if args.exclude_patterns:
-        custom_patterns = [x.strip() for x in args.exclude_patterns.split(";")]
-        base_patterns += custom_patterns
+        base_patterns += [x.strip() for x in args.exclude_patterns.split(";")]
+
+    ignored_files = shutil.ignore_patterns(*uniquify(base_patterns))
 
-    ignored_files = shutil.ignore_patterns(*base_patterns)
+    #==============================================================================================
 
     print("cleaning up...")
     if final_location.exists():
         shutil.rmtree(final_location)
 
+    #==============================================================================================
+
     print("copying library...")
-    shutil.copytree(base_python, final_location, ignore=ignored_files, dirs_exist_ok=True)
+    shutil.copytree(
+        base_python,
+        final_location,
+        ignore=ignored_files,
+        ignore_dangling_symlinks=True,
+        dirs_exist_ok=True)
     os.makedirs(site_packages, exist_ok=True)
 
-    print("making archive...")
+    #==============================================================================================
+
+    if args.pip_packages:
+        print("installing packages...")
+        packages = uniquify([x.strip() for x in args.pip_packages.split(";")])
+        os.system(f"{sys.executable} -m pip install --no-compile --target={site_packages} {' '.join(packages)}")
+
+        print("cleaning up packages...")
+        cleanup_files(site_packages, ["*.pyi"], ["tests", "bin"])
+
+    #==============================================================================================
+
+    print("making packages archive...")
+    #if sys.platform == "darwin":
+    #    os.system(f"xattr -cr {final_location}")
+
     if os.path.exists(final_archive):
         make_archive(temp_archive, final_location)
         if file_hash(temp_archive) != file_hash(final_archive):
             shutil.copy(temp_archive, final_archive)
     else:
         make_archive(final_archive, final_location)
+
+    #==============================================================================================
+
+    if args.zip_python_executable:
+        print("making python executable archive...")
+        python_executable = os.path.realpath(sys.executable)
+        print(python_executable)
+
+        # TODO (darwin)
+        # 1. otool -L python_executable
+        # 2. if this contains a Python shared library in the Frameworks/Python.framework whatever, copy that as well
+        # 3. fix install_name_tool on python_executable to use the local
+
+        if os.path.exists(final_python_archive):
+            make_archive(temp_python_archive, python_executable)
+            if file_hash(temp_python_archive) != file_hash(final_python_archive):
+                shutil.copy(temp_python_archive, final_python_archive)
+        else:
+            make_archive(final_python_archive, python_executable)

demos/plugin/CMakeLists.txt

@@ -41,8 +41,13 @@ add_subdirectory (${MODULES_PATH} ./modules)
 if (APPLE)
     set (Python_ROOT_DIR "/Library/Frameworks/Python.framework/Versions/Current")
 endif()
-set (Python_USE_STATIC_LIBS TRUE)
+set (Python_USE_STATIC_LIBS FALSE)
 find_package (Python REQUIRED Interpreter Development.Embed)
+message("-- Popsicle -- Python_FOUND:                   ${Python_FOUND}")
+message("-- Popsicle -- Python_VERSION:                 ${Python_VERSION}")
+message("-- Popsicle -- Python_Development.Embed_FOUND: ${Python_Development.Embed_FOUND}")
+message("-- Popsicle -- Python_INCLUDE_DIRS:            ${Python_INCLUDE_DIRS}")
+message("-- Popsicle -- Python_LIBRARIES:               ${Python_LIBRARIES}")
 
 # Setup the juce app
 juce_add_plugin ("${PROJECT_NAME}"
@@ -56,19 +61,20 @@ juce_add_plugin ("${PROJECT_NAME}"
     NEEDS_MIDI_INPUT OFF
     NEEDS_MIDI_OUTPUT OFF
     IS_MIDI_EFFECT OFF
-    APP_SANDBOX_ENABLED ON
+    #APP_SANDBOX_ENABLED ON
     COPY_PLUGIN_AFTER_BUILD ON)
 juce_generate_juce_header (${PROJECT_NAME})
 
 # Add the binary target for the python standard library
-set (ADDITIONAL_IGNORED_PYTHON_PATTERNS "lib2to3" "pydoc_data" "_xxtestfuzz*")
+set (ADDITIONAL_IGNORED_PYTHON_PATTERNS "email" "curses" "dbm" "html" "http" "unittest" "tomllib" "wsgiref" "xmlrpc")
+set (ADDITIONAL_PYTHON_PACKAGES "numpy")
 set (PYTHON_STANDARD_LIBRARY "${CMAKE_CURRENT_BINARY_DIR}/python${Python_VERSION_MAJOR}${Python_VERSION_MINOR}.zip")
 
 add_custom_target (
     ${PROJECT_NAME}_stdlib
     ${Python_EXECUTABLE} ${ROOT_PATH}/cmake/ArchivePythonStdlib.py
         -b ${Python_ROOT_DIR} -o ${CMAKE_CURRENT_BINARY_DIR} -M ${Python_VERSION_MAJOR} -m ${Python_VERSION_MINOR}
-        -x "\"${ADDITIONAL_IGNORED_PYTHON_PATTERNS}\""
+        -x "\"${ADDITIONAL_IGNORED_PYTHON_PATTERNS}\"" -p "\"${ADDITIONAL_PYTHON_PACKAGES}\""
     BYPRODUCTS ${PYTHON_STANDARD_LIBRARY})
 add_dependencies (${PROJECT_NAME} ${PROJECT_NAME}_stdlib)
 
@@ -82,16 +88,15 @@ target_sources (${PROJECT_NAME} PRIVATE
     PopsiclePluginProcessor.cpp
     PopsiclePluginProcessor.h)
 
-#set_target_properties (${PROJECT_NAME} PROPERTIES JUCE_TARGET_KIND_STRING "App")
-set_target_properties (${PROJECT_NAME} PROPERTIES CXX_STANDARD 17)
+set_target_properties (${PROJECT_NAME} PROPERTIES CXX_STANDARD 20)
 set_target_properties (${PROJECT_NAME} PROPERTIES CXX_VISIBILITY_PRESET "hidden")
 set_target_properties (${PROJECT_NAME} PROPERTIES VISIBILITY_INLINES_HIDDEN TRUE)
 set_target_properties (${PROJECT_NAME} PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
 
 if (APPLE)
     #set_target_properties (${PROJECT_NAME} PROPERTIES OSX_ARCHITECTURES "arm64;x86_64")
     #set_target_properties (BinaryData PROPERTIES OSX_ARCHITECTURES "arm64;x86_64")
-    target_link_options (${PROJECT_NAME} PRIVATE "-Wl,-weak_reference_mismatches,weak")
+    target_link_options (${PROJECT_NAME} PRIVATE "-Wl,-weak_reference_mismatches,weak -Wl,-undefined,dynamic_lookup")
     #set (LTO_CONFIGURATION "juce::juce_recommended_lto_flags")
     set (LTO_CONFIGURATION "")
 else()
@@ -111,9 +116,8 @@ elseif (UNIX)
 endif()
 
 target_compile_definitions (${PROJECT_NAME} PRIVATE
-    #JUCE_STANDALONE_APPLICATION=1
     JUCE_DISPLAY_SPLASH_SCREEN=0
-    JUCE_MODAL_LOOPS_PERMITTED=1
+    JUCE_MODAL_LOOPS_PERMITTED=0
     JUCE_CATCH_UNHANDLED_EXCEPTIONS=0
     JUCE_LOG_ASSERTIONS=1
     JUCE_ALLOW_STATIC_NULL_VARIABLES=0
@@ -123,6 +127,9 @@ target_compile_definitions (${PROJECT_NAME} PRIVATE
     JUCE_SILENCE_XCODE_15_LINKER_WARNING=1
     PYBIND11_DETAILED_ERROR_MESSAGES=1)
 
+target_include_directories(${PROJECT_NAME} PRIVATE
+    ${Python_INCLUDE_DIRS})
+
 target_link_libraries (${PROJECT_NAME} PRIVATE
     #juce::juce_analytics
     juce::juce_audio_basics
@@ -144,8 +151,9 @@ target_link_libraries (${PROJECT_NAME} PRIVATE
     #juce::juce_video
     juce::juce_recommended_config_flags
     juce::juce_recommended_warning_flags
-    Python::Python
+    #Python::Python
     popsicle::juce_python
     popsicle::juce_python_recommended_warning_flags
     BinaryData
+    ${Python_LIBRARIES}
     ${LTO_CONFIGURATION})