diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index c7b2c1418..f59247959 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -216,6 +216,9 @@ def respond_with_template( "hostname": get_hostname(), "microscope": get_microscope(), "version": murfey.__version__, + # Extra parameters to reconstruct URLs for forwarded requests + "netloc": request.url.netloc, + "proxy_path": "", } if parameters: template_parameters.update(parameters) diff --git a/src/murfey/server/api/bootstrap.py b/src/murfey/server/api/bootstrap.py index 63fd518a1..cc4fb1542 100644 --- a/src/murfey/server/api/bootstrap.py +++ b/src/murfey/server/api/bootstrap.py @@ -19,6 +19,7 @@ import logging import random import re +import zipfile from io import BytesIO from urllib.parse import quote @@ -105,8 +106,23 @@ def get_bootstrap_instructions(request: Request): machine with no internet access. """ + # Constructs the netloc (hostname + port) and proxy path depending on if the + # request was forwarded via proxy + netloc = ( + f"{request.headers['X-Forwarded-Host']}:{request.headers['X-Forwarded-Port']}" + if request.headers.get("X-Forwarded-Host") + and request.headers.get("X-Forwarded-Port") + else request.url.netloc + ) + # Additional bit in URL path after the netloc caused by the proxy reroute + proxy_path = request.url.path.removesuffix(f"{bootstrap.prefix}/") + return respond_with_template( request=request, + parameters={ + "netloc": netloc, + "proxy_path": proxy_path, + }, filename="bootstrap.html", ) @@ -314,7 +330,80 @@ def parse_cygwin_request( ) -@msys2.get("/distrib/{setup_file}", response_class=StreamingResponse) +@msys2.get("/config/pacman.d.zip", response_class=StreamingResponse) +def get_pacman_mirrors(request: Request): + """ + Dynamically generates a zip file containing mirrorlist files that have been set + up to mirror the MSYS2 package database for each environment. + + The files in this folder should be pasted into, and overwrite, the 'mirrorlist' + files present in the %MSYS64%\\etc\\pacman.d folder. The default path to this + folder is C:\\msys64\\etc\\pacman.d. + """ + + # Check if this is a forwarded request from somewhere else and construct netloc + netloc = ( + f"{request.headers['X-Forwarded-Host']}:{request.headers['X-Forwarded-Port']}" + if request.headers.get("X-Forwarded-Host") + and request.headers.get("X-Forwarded-Port") + else request.url.netloc + ) + + # Find path to Rust router using current URL Path + path_to_router = request.url.path.removesuffix("/config/pacman.d.zip") + + # Construct base URL for subsequent use + base_url = f"{request.url.scheme}://{netloc}{path_to_router}" + logger.debug(f"Base URL to MSYS2 sub-router determined to be {base_url}") + + # Construct package database mirrors + # Files are called mirrorlist.{environment} + # URL format: {scheme}://{netloc}{proxy_path}/{router_prefix}/path/to/repo + url_paths = { + "clang64": "mingw/clang64", + "mingw": "mingw/$repo", + "mingw32": "mingw/i686", + "mingw64": "mingw/x86_64", + "msys": "msys/$arch", + "ucrt64": "mingw/ucrt64", + } + # Construct file names and contents + mirror_lists = { + f"mirrorlist.{env}": "\n".join( + [ + "# See https://www.msys2.org/dev/mirrors", + "", + "## Primary", + f"Server = {base_url}/repo/{repo_path}", + "", + ] + ) + for env, repo_path in url_paths.items() + } + + # Create in-memory buffer for the ZIP file + zip_buffer = BytesIO() + + # Create a zip file in the buffer + with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file: + for file_name, content in mirror_lists.items(): + zip_file.writestr(file_name, content) + zip_buffer.seek(0) # Move object pointer back to start + + # Construct and return streaming response + headers = { + "Content-Disposition": "attachment; filename=pacman.d.zip", + "Content-Length": str(zip_buffer.getbuffer().nbytes), + } + return StreamingResponse( + zip_buffer, + status_code=200, + headers=headers, + media_type="application/zip", + ) + + +@msys2.get("/repo/distrib/{setup_file}", response_class=StreamingResponse) def get_msys2_setup( request: Request, setup_file: str, @@ -351,7 +440,7 @@ def get_msys2_setup( ) -@msys2.get("/", response_class=Response) +@msys2.get("/repo/", response_class=Response) def get_msys2_main_index( request: Request, ) -> Response: @@ -391,7 +480,7 @@ def get_msys2_main_index( ) -@msys2.get("/{system}/", response_class=Response) +@msys2.get("/repo/{system}/", response_class=Response) def get_msys2_environment_index( request: Request, system: str, @@ -435,7 +524,7 @@ def get_msys2_environment_index( ) -@msys2.get("/{system}/{environment}/", response_class=Response) +@msys2.get("/repo/{system}/{environment}/", response_class=Response) def get_msys2_package_index( request: Request, system: str, @@ -464,7 +553,7 @@ def get_msys2_package_index( ) -@msys2.get("/{system}/{environment}/{package}", response_class=StreamingResponse) +@msys2.get("/repo/{system}/{environment}/{package}", response_class=StreamingResponse) def get_msys2_package_file( request: Request, system: str, diff --git a/src/murfey/templates/base.html b/src/murfey/templates/base.html index 942518e3a..ce6fa129e 100644 --- a/src/murfey/templates/base.html +++ b/src/murfey/templates/base.html @@ -1,22 +1,24 @@ Murfey - {% block title %}{% endblock %} - +
- Home - Active Visits - Installation instructions - FastAPI PyPI + Home + Installation Instructions + FastAPI (PyPI)

diff --git a/src/murfey/templates/bootstrap.html b/src/murfey/templates/bootstrap.html index deb83f6ba..3880e4ff1 100644 --- a/src/murfey/templates/bootstrap.html +++ b/src/murfey/templates/bootstrap.html @@ -1,89 +1,230 @@ {% extends "base.html" %} {% block title %}Bootstrapping instructions{% endblock %} {% block content %}

Bootstrapping instructions

-

Installing a Linux Terminal

-

Installing Cygwin

+

1. Setting Up a POSIX Environment

+

Option 1: Installing Cygwin

- If you already have a Cygwin install, rename it so that it doesn't get - overwritten (something like "[install name]-old"). + Cygwin is a lightweight POSIX environment that provides the minimum + requirements needed for a Windows client PC to efficiently transfer files to a + Unix storage server. However, it currently does not support building and + installing packages which have been written in Rust, which many modern Python + packages, including Murfey's dependencies, now make use of. Nevertheless, + older versions of Murfey will still work with it.

- Download the Cygwin setup executable using this - mirror, and then run the following from - a terminal (both Command Prompt and Windows Powershell work) + To install, download the Cygwin setup executable using this + mirror, and then run + the following from a terminal (both Command Prompt and Windows Powershell + work):

-
-    $ setup-x86_64.exe -O -R C:\cygwin64 -s {{ request.url.scheme }}://{{ request.url.netloc }}/cygwin -P curl,python3,rsync -q
+
+    $ setup-x86_64.exe -O -R C:\cygwin64 -s {{ request.url.scheme }}://{{ netloc }}{{ proxy_path }}/cygwin -P curl,python3,rsync -q
 

- The Cygwin install sometimes hangs even when it is finished, hit Enter to - return to a command prompt. + This will install Cygwin with the minimum packages needed to run Murfey. The + Cygwin install command will occasionally hang after completion. In such an + event, just hit Enter to return to a normal command prompt window. +

+

+ If you already have a Cygwin install that you would like to preserve, rename + it so that it doesn't get overwritten (something like "cygwin64-old") before + running the command line above.

-

Installing MSYS2

+

Option 2: Installing MSYS2

- MSYS2 is a lightweight Linux environment which provides compiler support for - the more modern programming languages used in the backend of Murfey's package - dependencies. + MSYS2 is a lightweight POSIX environment which provides compiler support for + the more modern programming languages used by Murfey's package dependencies.

- The Murfey server supports the forwarding of download requests to client PCs - that cannot access the wider internet. Download the MSYS2 setup executable - using this mirror, and run the - executable using the default settings. + The Murfey server supports the forwarding of download requests to + network-restricted client PCs. To install MSYS2, download the setup executable + using this + mirror, then run it using the default settings. This will install MSYS2 to + C:\msys64.

+

A. Setting Up the Package Manager (If Network-Restricted)

By default, MSYS2 comes with preset lists of mirrors and servers that it - installs its packages from. These will need to be disabled, and replaced with - URLs of the same format that point to the Murfey server the client PC is - connected to. + installs its packages from. On a network-restricted PC, these will need to be + replaced with files that point to the Murfey server instead. They can be + downloaded via this + link.

- These lists can be found in the following folder, if the default installation - options were chosen: + Once downloaded, extract the files to + %MSYS64%\etc\pacman.d. If MSYS2 was installed at the default location, this will be:

-
-    C:\msys64\etc\pacman.d\mirrorlist.{environment}
-
-

This is an example of how the URL to the Murfey server should look like:

-
-    Server = https://repo.msys2.org/mingw/x86_64/  # Original URL
-    Server = {{ request.url.scheme }}://{{ request.url.netloc }}/msys2/mingw/x86_64  # Murfey URL
+
+    C:\msys64\etc\pacman.d
 
+

B. Installing Dependencies

MSYS2 comes with multiple environments, but UCRT64 is the most modern one. In order for the Murfey client to be able to install and run its dependencies properly, the following packages will need to be installed in the UCRT64 environment. This can be achieved using the following commands:

-
-    $ pacman -Syu --disable-download-timeout  # Downloads the package database and searches for updates
-    $ pacman -S rsync --disable-download-timeout
-    $ pacman -S mingw-w64-python-pip --disable-download-timeout
-    $ pacman -S mingw-w64-x86_64-rust --disable-download-timeout
+
+    $ pacman -Syu
+    $ pacman -S msys2-runtime-3.6
+    $ pacman -S rsync
+    $ pacman -S mingw-w64-ucrt-x86_64-python-pip
+    $ pacman -S mingw-w64-ucrt-x86_64-rust
+
+

+ Other utility packages such as + vim + can also be installed by running + pacman -S <package-name>. You can browse the other packages available on MSYS2 by searching the repo + using + pacman -Ss <package-name> +

+

C. Configuring the Rust Package Manager (If Network-Restricted)

+

+ Many newer Python packages now have dependencies written in Rust that allow + them to operate more efficiently. MSYS2 supports the compilation and + installation of such packages, and is thus our recommended POSIX environment + to use Murfey with. +

+

+ Rust packages and their associated metadata are, by default, stored in + https://crates.io. Package download and installation is in turn conducted by the package + manager + cargo. For network-restricted client PCs, Murfey also supports mirroring + https://crates.io + to facilitate the installation of Rust packages. +

+

+ To configure + cargo, simply download the pre-configured + config.toml + file via this link. This + file should then be pasted in a + .cargo + folder, which, by default, shold be located in your User Profile homespace: +

+
+    %USERPROFILE%\.cargo
+
+

For a user named Murfey, for example, this would take the form:

+
+    C:\Users\Murfey\.cargo
 
+

+ With this file configured, + cargo + will know to look for package metadata and files via the Murfey mirror + instead. +

+ +

D. Running MSYS2 Through Command Prompt

+

+ In order to run Murfey via the terminal, MSYS2 will have to be run through + Windoww's Command Prompt terminal, as there is an ongoing bug with MSYS2's + pre-packaged terminal that prevents mouse interaction with interactive apps in + the terminal. +

+

+ To do so, simply right-click on your desktop and navigate to + New > Shortcut. When prompted for the location of the item, enter + the following into the text box: +

+
+    cmd.exe /k "C:\msys64\msys2_shell.cmd -defterm -no-start -ucrt64 -shell bash"
+
+

+ After naming the shortcut, click Finish to create the shortcut. This will run + a UCRT64 instance of MSYS2 through the Command Prompt terminal that starts you + off in MSYS2's default home directory. You can proceed to customise the + shortcut icon to taste. +

-

Setting Up Python

+

2. Setting Up Python

Once Python and pip are installed in the terminal, you have the option to install Murfey in either the base environment or a virtual environment. The base environment is simpler, but uninstallation of the Python packages in the future could potentially interfere with the base environment's functionality.

-

Setting Up a Virtual Environment

+

A. (Optional) Setting Up a Virtual Environment

To set up a virtual environment, run the following commands:

-
-    $ pip install virtualenv --index-url {{ request.url.scheme }}://{{ request.url.netloc }}/pypi --trusted-host {{ request.url.hostname }}
+
+    $ pip install virtualenv --index-url {{ request.url.scheme }}://{{ netloc }}{{ proxy_path }}/pypi --trusted-host {{ netloc }}
     $ virtualenv your-env-name  # Create the virtual environment
     $ source your-env-name/bin/activate  # Activate the virtual environment
 
-

Installing Murfey

+

B. Installing Murfey

You can install Murfey in the Python environment (the base one or a virtual environment) in either the Cygwin or UCRT64 terminal using the following commands:

-
-    $ pip install murfey[client] --index-url {{ request.url.scheme }}://{{ request.url.netloc }}/pypi --trusted-host {{ request.url.hostname }}
+
+    $ pip install murfey[client] --index-url {{ request.url.scheme }}://{{ netloc }}{{ proxy_path }}/pypi --trusted-host {{ netloc }}
 
+

+ If you wish to install the client-side dependencies needed to run Murfey via + the web UI, replace + murfey[client] + with + murfey[client,instrument-server]. +

{% endblock %}