diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 000000000..101d69530 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,483 @@ +name: Build and Package + +on: + push: + branches: [ main, develop, develop-weekly ] + pull_request: + branches: [ main, develop, develop-weekly ] + workflow_dispatch: + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - name: Linux x64 + os: ubuntu-latest + triplet: x64-linux + artifact-name: ginan-linux-x64 + build-dir: linux-Release + cache-key: vcpkg-linux-v4 + install-deps: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ninja-build curl zip unzip tar pkg-config git + preset: release + + - name: macOS ARM64 + os: macos-14 + triplet: arm64-osx + artifact-name: ginan-macos-arm64 + build-dir: mac-arm64-Release + cache-key: vcpkg-macos-arm64-v4 + install-deps: | + brew install cmake ninja pkg-config libomp gcc + echo "/opt/homebrew/bin" >> $GITHUB_PATH + preset: macos-arm64-release + + - name: macOS x64 + os: macos-15-intel + triplet: x64-osx + artifact-name: ginan-macos-x64 + build-dir: mac-x64-Release + cache-key: vcpkg-macos-x64-v3 + install-deps: | + brew install cmake ninja pkg-config libomp gcc + echo "/usr/local/bin" >> $GITHUB_PATH + preset: macos-x64-release + + - name: Windows x64 (Cross) + os: ubuntu-latest + triplet: x64-mingw-static + artifact-name: ginan-windows-x64 + build-dir: windows-cross-Release + cache-key: vcpkg-windows-cross-v4 + install-deps: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ninja-build curl zip unzip tar pkg-config git mingw-w64 g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64 + preset: windows-cross-release + vcpkg-extra-flags: --allow-unsupported + + name: Build ${{ matrix.name }} + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install system dependencies + run: ${{ matrix.install-deps }} + + - name: Setup vcpkg binary cache + run: | + mkdir -p ${{ github.workspace }}/.vcpkg-cache + echo "VCPKG_BINARY_SOURCES=clear;files,${{ github.workspace }}/.vcpkg-cache,readwrite" >> $GITHUB_ENV + echo "VCPKG_BUILD_TYPE=release" >> $GITHUB_ENV + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + .vcpkg-cache + vcpkg + key: ${{ matrix.cache-key }}-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ matrix.cache-key }}- + + - name: Setup vcpkg + run: | + if [ ! -d "vcpkg" ]; then + git clone https://github.com/Microsoft/vcpkg.git + ./vcpkg/bootstrap-vcpkg.sh -disableMetrics + fi + echo "VCPKG_ROOT=${{ github.workspace }}/vcpkg" >> $GITHUB_ENV + + - name: Install vcpkg dependencies + run: | + export VCPKG_BUILD_TYPE=release + cd ${{ github.workspace }} + # Retry logic for transient network failures + for i in 1 2 3; do + ./vcpkg/vcpkg install --triplet ${{ matrix.triplet }} --x-install-root=./vcpkg_installed ${{ matrix.vcpkg-extra-flags }} && break || { + echo "vcpkg install attempt $i failed, retrying..."; + sleep 10; + } + done + + - name: Clean build directory + working-directory: src + run: rm -rf build/${{ matrix.build-dir }} + + - name: Configure CMake + working-directory: src + run: cmake --preset ${{ matrix.preset }} + + - name: Build + working-directory: src + run: | + if [ "$RUNNER_OS" = "Linux" ]; then + cmake --build build/${{ matrix.build-dir }} --parallel $(nproc) + else + cmake --build build/${{ matrix.build-dir }} --parallel $(sysctl -n hw.ncpu) + fi + + - name: Collect artifacts + run: | + mkdir -p artifacts + cp -r bin/* artifacts/ || true + find . -name "*.exe" -exec cp {} artifacts/ \; 2>/dev/null || true + + # Include required runtime libraries for macOS + if [ "$RUNNER_OS" = "macOS" ]; then + if [ -f "/opt/homebrew/lib/libomp.dylib" ]; then + cp /opt/homebrew/lib/libomp.dylib artifacts/ + elif [ -f "/usr/local/lib/libomp.dylib" ]; then + cp /usr/local/lib/libomp.dylib artifacts/ + fi + if [ -f "/opt/homebrew/lib/libgfortran.5.dylib" ]; then + cp /opt/homebrew/lib/libgfortran.5.dylib artifacts/ || true + elif [ -f "/usr/local/lib/libgfortran.5.dylib" ]; then + cp /usr/local/lib/libgfortran.5.dylib artifacts/ || true + fi + fi + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact-name }} + path: artifacts/ + retention-days: 30 + + build-gui: + needs: build + strategy: + fail-fast: false + matrix: + include: + - name: Linux x64 + os: ubuntu-latest + binary-artifact: ginan-linux-x64 + gui-artifact: ginan-gui-linux-x64 + ui-sed-cmd: sed -i '24s/.*/from scripts.GinanUI.app.resources import ginan_logo_rc/' app/views/main_window_ui.py + pyinstaller-args: --windowed + extra-steps: "" + + - name: macOS ARM64 + os: macos-15 + binary-artifact: ginan-macos-arm64 + gui-artifact: ginan-gui-macos-arm64 + ui-sed-cmd: sed -i '' '24s/.*/from scripts.GinanUI.app.resources import ginan_logo_rc/' app/views/main_window_ui.py + pyinstaller-args: --onedir --clean --target-arch arm64 --noconfirm + extra-steps: macos + + - name: macOS x64 + os: macos-15-intel + binary-artifact: ginan-macos-x64 + gui-artifact: ginan-gui-macos-x64 + ui-sed-cmd: sed -i '' '24s/.*/from scripts.GinanUI.app.resources import ginan_logo_rc/' app/views/main_window_ui.py + pyinstaller-args: --onedir --clean --noconfirm + extra-steps: macos + + - name: Windows x64 + os: windows-latest + binary-artifact: ginan-windows-x64 + gui-artifact: ginan-gui-windows-x64 + ui-sed-cmd: | + (Get-Content app/views/main_window_ui.py) | ForEach-Object { + if ($_.ReadCount -eq 24) { + "from scripts.GinanUI.app.resources import ginan_logo_rc" + } else { + $_ + } + } | Set-Content app/views/main_window_ui.py + pyinstaller-args: --windowed + pyinstaller-separator: ";" + extra-steps: "" + + name: Build GUI ${{ matrix.name }} + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Download binaries + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.binary-artifact }} + path: bin/ + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + pip install -r scripts/GinanUI/requirements.txt + pip install pyinstaller + + - name: Convert UI files to Python (Unix) + if: runner.os != 'Windows' + working-directory: scripts/GinanUI + run: | + pyside6-uic app/views/main_window.ui -o app/views/main_window_ui.py + ${{ matrix.ui-sed-cmd }} + + - name: Convert UI files to Python (Windows) + if: runner.os == 'Windows' + working-directory: scripts/GinanUI + shell: pwsh + run: | + pyside6-uic app/views/main_window.ui -o app/views/main_window_ui.py + ${{ matrix.ui-sed-cmd }} + + - name: Make binaries executable (Unix) + if: runner.os != 'Windows' + run: chmod +x bin/* + + - name: Install OpenMP (macOS) + if: matrix.extra-steps == 'macos' + run: | + echo "Installing OpenMP via Homebrew..." + brew install libomp + echo "OpenMP installed, checking location..." + ls -la /opt/homebrew/lib/libomp* 2>/dev/null || ls -la /usr/local/lib/libomp* 2>/dev/null || true + + - name: Copy required libraries (macOS) + if: matrix.extra-steps == 'macos' + run: | + echo "Looking for OpenMP library..." + + # libomp is keg-only, so check Cellar and opt paths + LIBOMP_PATH="" + if [ -f "/opt/homebrew/opt/libomp/lib/libomp.dylib" ]; then + LIBOMP_PATH="/opt/homebrew/opt/libomp/lib/libomp.dylib" + elif [ -f "/usr/local/opt/libomp/lib/libomp.dylib" ]; then + LIBOMP_PATH="/usr/local/opt/libomp/lib/libomp.dylib" + elif [ -f "/opt/homebrew/lib/libomp.dylib" ]; then + LIBOMP_PATH="/opt/homebrew/lib/libomp.dylib" + elif [ -f "/usr/local/lib/libomp.dylib" ]; then + LIBOMP_PATH="/usr/local/lib/libomp.dylib" + fi + + if [ -n "$LIBOMP_PATH" ]; then + echo "Found libomp at: $LIBOMP_PATH" + cp -v "$LIBOMP_PATH" bin/ + chmod 644 bin/libomp.dylib + else + echo "ERROR: libomp.dylib not found!" + echo "Searching in Cellar..." + find /usr/local/Cellar/libomp -name "libomp.dylib" 2>/dev/null || true + find /opt/homebrew/Cellar/libomp -name "libomp.dylib" 2>/dev/null || true + exit 1 + fi + + # Copy other potential dependencies + if [ -f "/opt/homebrew/lib/libgfortran.5.dylib" ]; then + cp /opt/homebrew/lib/libgfortran.5.dylib bin/ || true + elif [ -f "/usr/local/lib/libgfortran.5.dylib" ]; then + cp /usr/local/lib/libgfortran.5.dylib bin/ || true + fi + + echo "Contents of bin/ after library copy:" + ls -lh bin/*.dylib 2>/dev/null || echo "No .dylib files in bin/" + + - name: Build GUI with PyInstaller (Unix) + if: runner.os != 'Windows' + run: | + python -m PyInstaller --name GinanUI \ + ${{ matrix.pyinstaller-args }} \ + --add-data "scripts/GinanUI/app:app" \ + --add-data "scripts/plot_pos.py:scripts" \ + --add-binary "bin/*:bin" \ + --hidden-import PySide6 \ + --hidden-import PySide6.QtWebEngineWidgets \ + --hidden-import PySide6.QtWebEngineCore \ + --hidden-import plotly \ + --hidden-import scripts.plot_pos \ + --hidden-import scripts.GinanUI.app \ + --hidden-import scripts.GinanUI.app.models \ + --hidden-import scripts.GinanUI.app.models.execution \ + --hidden-import scripts.GinanUI.app.controllers \ + --hidden-import scripts.GinanUI.app.controllers.input_controller \ + --hidden-import scripts.GinanUI.app.controllers.visualisation_controller \ + --hidden-import scripts.GinanUI.app.utils \ + --hidden-import scripts.GinanUI.app.utils.workers \ + --hidden-import scripts.GinanUI.app.utils.cddis_credentials \ + --hidden-import scripts.GinanUI.app.utils.cddis_email \ + --hidden-import scripts.GinanUI.app.utils.common_dirs \ + --hidden-import scripts.GinanUI.app.utils.gn_functions \ + --hidden-import scripts.GinanUI.app.utils.yaml \ + --hidden-import scripts.GinanUI.app.views.main_window_ui \ + --collect-all scripts.GinanUI \ + scripts/GinanUI/main.py + + - name: Build GUI with PyInstaller (Windows) + if: runner.os == 'Windows' + run: | + pyinstaller --name GinanUI ` + ${{ matrix.pyinstaller-args }} ` + --add-data "scripts/GinanUI/app;app" ` + --add-data "scripts/plot_pos.py;scripts" ` + --add-binary "bin/*;bin" ` + --hidden-import PySide6 ` + --hidden-import PySide6.QtWebEngineWidgets ` + --hidden-import PySide6.QtWebEngineCore ` + --hidden-import plotly ` + --hidden-import scripts.plot_pos ` + --hidden-import scripts.GinanUI.app ` + --hidden-import scripts.GinanUI.app.models ` + --hidden-import scripts.GinanUI.app.models.execution ` + --hidden-import scripts.GinanUI.app.controllers ` + --hidden-import scripts.GinanUI.app.controllers.input_controller ` + --hidden-import scripts.GinanUI.app.controllers.visualisation_controller ` + --hidden-import scripts.GinanUI.app.utils ` + --hidden-import scripts.GinanUI.app.utils.workers ` + --hidden-import scripts.GinanUI.app.utils.cddis_credentials ` + --hidden-import scripts.GinanUI.app.utils.cddis_email ` + --hidden-import scripts.GinanUI.app.utils.common_dirs ` + --hidden-import scripts.GinanUI.app.utils.gn_functions ` + --hidden-import scripts.GinanUI.app.utils.yaml ` + --hidden-import scripts.GinanUI.app.views.main_window_ui ` + --collect-all scripts.GinanUI ` + scripts/GinanUI/main.py + + # Post-build cleanup + - name: Remove unnecessary files (Unix) + if: runner.os != 'Windows' + run: | + cd dist/GinanUI + # Only remove clearly unnecessary files + find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + find . -type f -name "*.pyc" -delete 2>/dev/null || true + if [ "$RUNNER_OS" = "macOS" ]; then + find . -name ".DS_Store" -delete 2>/dev/null || true + fi + + - name: Remove unnecessary files (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + cd dist/GinanUI + # Only remove clearly unnecessary files + Get-ChildItem -Recurse -Directory | Where-Object { $_.Name -eq "__pycache__" } | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue + Get-ChildItem -Recurse -File -Filter "*.pyc" | Remove-Item -Force -ErrorAction SilentlyContinue + + # macOS-specific post-build steps + - name: Copy Qt WebEngine resources (macOS) + if: matrix.extra-steps == 'macos' + run: | + set -e + PYSIDE_DIR=$(python -c 'import PySide6, os; print(os.path.dirname(PySide6.__file__))') + QT_DIR="$PYSIDE_DIR/Qt" + RES_SRC1="$QT_DIR/resources" + RES_SRC2="$QT_DIR/lib/QtWebEngineCore.framework/Resources" + RES_DEST="dist/GinanUI/_internal/PySide6/Qt/resources" + mkdir -p "$RES_DEST" + copied=false + if [ -d "$RES_SRC1" ]; then + cp -R "$RES_SRC1"/* "$RES_DEST/" 2>/dev/null || true + copied=true + fi + if [ -d "$RES_SRC2" ]; then + cp -R "$RES_SRC2"/* "$RES_DEST/" 2>/dev/null || true + copied=true + fi + if [ "$copied" != true ]; then + echo "Qt WebEngine resources not found; build may fail"; + fi + + - name: Create launcher script (macOS) + if: matrix.extra-steps == 'macos' + run: | + cat > dist/GinanUI/run.sh << 'EOF' + #!/usr/bin/env zsh + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + export QTWEBENGINEPROCESS_PATH="$SCRIPT_DIR/Helpers/QtWebEngineProcess.app/Contents/MacOS/QtWebEngineProcess" + export QTWEBENGINE_RESOURCES_PATH="$SCRIPT_DIR/_internal/PySide6/Qt/resources" + export QTWEBENGINE_LOCALES_PATH="$SCRIPT_DIR/_internal/PySide6/Qt/resources/qtwebengine_locales" + export QTWEBENGINE_DISABLE_SANDBOX=1 + export QTWEBENGINE_CHROMIUM_FLAGS="--no-sandbox --disable-gpu-sandbox --disable-seccomp-filter-sandbox --disable-logging --log-level=3" + export DYLD_FRAMEWORK_PATH="$SCRIPT_DIR/_internal/PySide6/Qt/lib" + export DYLD_LIBRARY_PATH="$SCRIPT_DIR/_internal/PySide6/Qt/lib" + if command -v xattr >/dev/null 2>&1; then + xattr -dr com.apple.quarantine "$SCRIPT_DIR" 2>/dev/null || true + fi + exec "$SCRIPT_DIR/GinanUI" + EOF + chmod +x dist/GinanUI/run.sh + + - name: Copy QtWebEngineProcess (macOS) + if: matrix.extra-steps == 'macos' + run: | + PYSIDE_PATH=$(python -c "import PySide6; import os; print(os.path.dirname(PySide6.__file__))") + mkdir -p dist/GinanUI/Helpers + if [ -d "$PYSIDE_PATH/Qt/libexec/QtWebEngineProcess.app" ]; then + cp -R "$PYSIDE_PATH/Qt/libexec/QtWebEngineProcess.app" dist/GinanUI/Helpers/ + elif [ -d "$PYSIDE_PATH/Qt/lib/QtWebEngineCore.framework/Helpers/QtWebEngineProcess.app" ]; then + cp -R "$PYSIDE_PATH/Qt/lib/QtWebEngineCore.framework/Helpers/QtWebEngineProcess.app" dist/GinanUI/Helpers/ + fi + + - name: Copy runtime libraries to package (macOS) + if: matrix.extra-steps == 'macos' + run: | + echo "Checking for libraries to copy..." + ls -lh bin/*.dylib 2>/dev/null || echo "No .dylib files in bin/" + + # Copy OpenMP and other libraries into the package + mkdir -p dist/GinanUI/_internal/lib + + if [ -f "bin/libomp.dylib" ]; then + echo "Copying libomp.dylib to package..." + cp -v bin/libomp.dylib dist/GinanUI/_internal/lib/ + chmod 644 dist/GinanUI/_internal/lib/libomp.dylib + else + echo "ERROR: bin/libomp.dylib not found!" + exit 1 + fi + + if [ -f "bin/libgfortran.5.dylib" ]; then + echo "Copying libgfortran.5.dylib to package..." + cp -v bin/libgfortran.5.dylib dist/GinanUI/_internal/lib/ + fi + + echo "Libraries copied to package:" + ls -lh dist/GinanUI/_internal/lib/ + + # Update binaries to look in @executable_path/../lib for libraries + echo "Updating binary rpaths..." + for binary in dist/GinanUI/_internal/bin/*; do + if [ -f "$binary" ] && file "$binary" | grep -q "Mach-O"; then + echo "Processing $binary" + install_name_tool -add_rpath "@executable_path/../lib" "$binary" 2>/dev/null || true + # Update libomp reference + install_name_tool -change "/opt/homebrew/lib/libomp.dylib" "@rpath/libomp.dylib" "$binary" 2>/dev/null || true + install_name_tool -change "/usr/local/lib/libomp.dylib" "@rpath/libomp.dylib" "$binary" 2>/dev/null || true + fi + done + + echo "Verifying binary rpaths after update:" + otool -L dist/GinanUI/_internal/bin/pea | grep -i omp || echo "No OpenMP reference found" + + - name: Code sign binaries (macOS) + if: matrix.extra-steps == 'macos' + working-directory: dist/GinanUI + run: | + find . -type f -perm +111 -exec codesign --force --deep -s - {} \; + find . -name "*.dylib" -exec codesign --force -s - {} \; + find . -name "*.so" -exec codesign --force -s - {} \; + codesign --force -s - _internal/Python || true + codesign --force --deep -s - Helpers/QtWebEngineProcess.app || true + find _internal/PySide6/Qt/resources -type f -name "*.pak" -exec codesign --force --sign - {} \; || true + + - name: Upload GUI artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.gui-artifact }} + path: dist/GinanUI/ + retention-days: 30 diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ce98fd8..cea31a8dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,101 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +# [4.0] 2025-12-16 + +## Added + +A Qt-based Graphical User Interface (GUI) for Ginan + +Ginan and GUI binaries for Linux, MacOS and Windows + +Major improvement in handling state errors in pre-fit and post-fit outlier screening + +New TRACE file plotting script (plot_trace_res.py) + +RNX2 --> RNX3 phase signal mapping in config + +Check on difference between BRDC and Kalman clocks before alignment + +Additional info to TRACE files for monitoring (snr, azimuth, signal data availability) + +Postfit omega test and address its numerical instability + +Incorrect error count increments (For phase rejects, receiver error counts and satellite error counts) + +Option to reset filter at specific times (periodic_reset) + +## Changed + +VCPKG as the primary package manager for Ginan + +Started refactor of code to use OpenBLAS / LAPACK instead of Eigen - including RTS and core Kalman filter
+(Improved efficiency with up to 30% faster on network runs) + +TRACE epoch from week / sec to generic datetime + +Use normal epoch instead of transmission time for satellite clock initialisation + +Improved SPP via Code bias correction + +Improved higher order ionospheric corrections in IF combination mode + +Improvements and restructure of Least squares estimation code + +EDA Updates: + +- Add ability to view differences for States and Measurements +- Check data exists before plotting +- Fix plotting of satellite / site availability +- Add cycle detection info to database + +LEO drag and SRP coefficient estimation (Properly estimate and resolve conflict with EMP estimation) + +Update variance before detecting cycle slips + +Trop SNX file written for all stations now + +Pass LLI flags through to savedSlip + +Move from better_enum to magic_enum + +Rename Ginan LLI flags to "retrack" + + +## Fixed + +Mutex compilation problem for OSX arm64 + +noTime with Sp3Write when filtered states not available + +Day-bound no date issue and finite vs mincon inconsistency + +Elevation calculation in preprocessor (PEA can run in preprocessing-only mode) + +NaN biases in SSR bias map + +Limit tropospheric model to reasonable heights + +Correctly write out Antenna delta in RNX file (H/E/N) + +IERS mean pole function to use TT time + +Ensure commit hash is generated for every compilation (help with ID version) + +Errors in writing POS and GPX files when RTS is on + +Issues with Chunking (rewrite) + +RNX file reading (Correct field width) + +Cmake file so loading software can compile + +## Deprecated + +## Removed + +## Security + # [3.1] 2024-09-02 ### Added diff --git a/Docs/announcements.md b/Docs/announcements.md index 4cb7bb385..5a53abb63 100644 --- a/Docs/announcements.md +++ b/Docs/announcements.md @@ -1,22 +1,29 @@ -> **2 Sept 2024** - the Ginan team is pleased to release v3.1.0 of the toolkit. +> **16 Dec 2025** - the Ginan team is pleased to release v4.0.0 of the toolkit. > -> The improvements delivered by this version include: +> **Main Highlights**: +> +> * Introduce Ginan binaries for Linux, MacOS and Windows +> * The binaries can be found on the GitHub website: +> * [Ginan Binaries](https://github.com/GeoscienceAustralia/ginan/releases/tag/v4.0.0) +> * Introduce a Qt-based Graphical User Interface (GUI) for Ginan: +> * Simply open RNX file and choose from drop-downs +> * Download the GUI from GitHub release page +> * Works across Linux, MacOS and Windows +> * Built using PySide 6 +> * A user manual for the GUI can be found here: +> * [Ginan GUI User Manual](https://github.com/GeoscienceAustralia/ginan/tree/main/scripts/GinanUI/docs/USER_GUIDE.md) +> +> ![GinanGUI Screenshot](images/GinanGUI-screenshot.png) +> *A screenshot of the Ginan GUI in action.* +> +> **Major Changes**: +> +> * Major refactor: use OpenBLAS / LAPACK instead of Eigen +> * Refactored RTS code +> * Refactored core Kalman filter code +> * Improved efficiency (up to 30% faster on network runs) +> * Major improvement in handling state errors in pre-fit and post-fit outlier screening +> * Move to vcpkg as the primary package manager for Ginan +> * Add RNX2 --> RNX3 phase signal mapping in config > -> * Boxwing model for the albedo -> * Sisnet (SouthPan) message support -> * SLR processing capability -> * PBO Position (.pos) format file output support -> * Apple silicon (M-chip) support -> * VMF3 file download python script (get_vmf3.py) -> * POS file visualisation python script (plot_pos.py) -> * EDA improvements -> * Improved documentation -> * Use case examples updated -> * Frequency dependent GLONASS receiver code bias estimation enabled -> * Improved missing/bad data handling -> * Bias rates from .BIA/BSX files parsed and used -> * Measurment and State error handling sigma_limit thresholds separated -> * Config file reorganisation (rec_reference_system: moved to receiver_options:) -> * Clock code handling modified -> * Many bug fixes. diff --git a/Docs/codingStandard.md b/Docs/codingStandard.md index 390593c04..05d480cbf 100644 --- a/Docs/codingStandard.md +++ b/Docs/codingStandard.md @@ -1,5 +1,18 @@ # Coding Standards for C++ +## Automatic Formatting + +This project uses clang-format (we use version 20) for automatic code formatting. The formatting rules are defined in the `.clang-format` file in the project root. + +To format your code: +```bash +# Format a single file +clang-format -i path/to/your/file.cpp + +# Format all C++ files in src/cpp except 3rdpart +find src/cpp -type f \( -name "*.cpp" -o -name "*.hpp" -o -name "*.c" -o -name "*.h" -o -name "*.cc" -o -name "*.cxx" \) -not -path "src/cpp/3rdparty/*" -exec clang-format -i {} + +``` + ## Code style Decades of experience has shown that codebases that are built with concise, clean code have fewer issues and are easier to maintain. If submitting a pull request for a patch to the software, please ensure your code meets the following standards. @@ -14,66 +27,68 @@ Overall we are aiming to ### Unconcise code - Not recommended - //check first letter of satellite type against something + //check first letter of satellite type against something + + if (obs.Sat.id().c_str()[0]) == 'G') + doSomething(); + else if (obs.Sat.id().c_str()[0]) == 'R') + doSomething(); + else if (obs.Sat.id().c_str()[0]) == 'E') + doSomething(); + else if (obs.Sat.id().c_str()[0]) == 'I') + doSomething(); - if (obs.Sat.id().c_str()[0]) == 'G') - doSomething(); - else if (obs.Sat.id().c_str()[0]) == 'R') - doSomething(); - else if (obs.Sat.id().c_str()[0]) == 'E') - doSomething(); - else if (obs.Sat.id().c_str()[0]) == 'I') - doSomething(); - ### Clear Code - Good - char& sysChar = obs.Sat.id().c_str()[0]; + char& sysChar = obs.Sat.id().c_str()[0]; - switch (sysChar) - { - case 'G': doSomething(); break; - case 'R': doSomething(); break; - case 'E': doSomething(); break; - case 'I': doSomething(); break; - } + switch (sysChar) + { + case 'G': doSomething(); break; + case 'R': doSomething(); break; + case 'E': doSomething(); break; + case 'I': doSomething(); break; + } ## Spacing, Indentation, and layout -* Use tabs, with tab spacing set to 4. -* Use space or tabs before and after any ` + - * / = < > == != % ` etc.. -* Use space, tab or new line after any `, ;` +* Use spaces (not tabs), with indentation width set to 4 spaces. +* Use space before and after any ` + - * / = < > == != % ` etc.. +* Use space or new line after any `, ;` * Use a new line after if statements. -* Use tabs to keep things tidy - If the same function is called multiple times with different parameters, the parameters should line up. +* Use spaces to keep things tidy - If the same function is called multiple times with different parameters, the parameters should line up. +* Line length should not exceed 100 characters. +* Use Allman brace style (braces on new lines). ### Scattered Parameters - Bad - trySetFromYaml(mongo_metadata,output_files,{"mongo_metadata" }); - trySetFromYaml(mongo_output_measurements,output_files,{"mongo_output_measurements" }); - trySetFromYaml(mongo_states,output_files,{"mongo_states" }); + trySetFromYaml(mongo_metadata,output_files,{"mongo_metadata" }); + trySetFromYaml(mongo_output_measurements,output_files,{"mongo_output_measurements" }); + trySetFromYaml(mongo_states,output_files,{"mongo_states" }); ### Aligned Parameters - Good - trySetFromYaml(mongo_metadata, output_files, {"mongo_metadata" }); - trySetFromYaml(mongo_output_measurements, output_files, {"mongo_output_measurements" }); - trySetFromYaml(mongo_states, output_files, {"mongo_states" }); + trySetFromYaml(mongo_metadata, output_files, {"mongo_metadata" }); + trySetFromYaml(mongo_output_measurements, output_files, {"mongo_output_measurements" }); + trySetFromYaml(mongo_states, output_files, {"mongo_states" }); ## Statements -One statement per line +One statement per line - `*`unless you have a very good reason ### Multiple Statements per Line - Bad - z[k]=ROUND(zb[k]); y=zb[k]-z[k]; step[k]=SGN(y); + z[k]=ROUND(zb[k]); y=zb[k]-z[k]; step[k]=SGN(y); ### Single Statement per Line - Good - z[k] = ROUND(zb[k]); - y = zb[k]-z[k]; - step[k] = SGN(y); + z[k] = ROUND(zb[k]); + y = zb[k]-z[k]; + step[k] = SGN(y); ### Example of a good reason: @@ -81,53 +96,61 @@ One statement per line #### Normal - switch (sysChar) - { - case ' ': - case 'G': - *sys = E_Sys::GPS; - *tsys = TSYS_GPS; - break; - case 'R': - *sys = E_Sys::GLO; - *tsys = TSYS_UTC; - break; - case 'E': - *sys = E_Sys::GAL; - *tsys = TSYS_GAL; - break; - //...continues - } + switch (sysChar) + { + case ' ': + case 'G': + *sys = E_Sys::GPS; + *tsys = TSYS_GPS; + break; + case 'R': + *sys = E_Sys::GLO; + *tsys = TSYS_UTC; + break; + case 'E': + *sys = E_Sys::GAL; + *tsys = TSYS_GAL; + break; + //...continues + } #### Ok - if (sys == SYS_GLO) fact = EFACT_GLO; - else if (sys == SYS_CMP) fact = EFACT_CMP; - else if (sys == SYS_GAL) fact = EFACT_GAL; - else if (sys == SYS_SBS) fact = EFACT_SBS; - else fact = EFACT_GPS; + if (sys == SYS_GLO) fact = EFACT_GLO; + else if (sys == SYS_CMP) fact = EFACT_CMP; + else if (sys == SYS_GAL) fact = EFACT_GAL; + else if (sys == SYS_SBS) fact = EFACT_SBS; + else fact = EFACT_GPS; #### Ok - switch (sysChar) - { - case ' ': - case 'G': *sys = E_Sys::GPS; *tsys = TSYS_GPS; break; - case 'R': *sys = E_Sys::GLO; *tsys = TSYS_UTC; break; - case 'E': *sys = E_Sys::GAL; *tsys = TSYS_GAL; break; - case 'S': *sys = E_Sys::SBS; *tsys = TSYS_GPS; break; - case 'J': *sys = E_Sys::QZS; *tsys = TSYS_QZS; break; - //...continues - } + switch (sysChar) + { + case ' ': + case 'G': *sys = E_Sys::GPS; *tsys = TSYS_GPS; break; + case 'R': *sys = E_Sys::GLO; *tsys = TSYS_UTC; break; + case 'E': *sys = E_Sys::GAL; *tsys = TSYS_GAL; break; + case 'S': *sys = E_Sys::SBS; *tsys = TSYS_GPS; break; + case 'J': *sys = E_Sys::QZS; *tsys = TSYS_QZS; break; + //...continues + } + +## Include Organization + +* Includes should be sorted automatically by clang-format. +* Include order priority: + 1. System headers (e.g., ``, ``) + 2. Project headers (e.g., `"common/common.hpp"`) + 3. Other headers ## Braces New line for braces. - if (pass) - { - doSomething(); - } + if (pass) + { + doSomething(); + } ## Comments @@ -155,17 +178,17 @@ if ( ( testA > 10) ### Bad - if (doSomeParsing(someObject)) - { - //code contingent on parsing success? failure? - } + if (doSomeParsing(someObject)) + { + //code contingent on parsing success? failure? + } ### Good - bool fail = doSomeParsing(someObject); - if (fail) - { - //This code is clearly a response to a failure - } + bool fail = doSomeParsing(someObject); + if (fail) + { + //This code is clearly a response to a failure + } ## Variable declaration @@ -192,20 +215,21 @@ for (int i = 0; i < 10; i++) if (found) { //... -} +} ``` ## Function parameters -* One per line. +* When parameters don't fit on one line, place each parameter on its own line. * Add doxygen compatible documentation after parameters in the cpp file. * Prefer references rather than pointers unless unavoidable. ``` void function( - bool runTests, ///< Run unit test while processing - MyStruct& myStruct, ///< Structure to modify - OtherStr* otherStr = nullptr) ///< Optional structure object to populate (cant use reference because its optional) + bool runTests, ///< Run unit test while processing + MyStruct& myStruct, ///< Structure to modify + OtherStr* otherStr = nullptr ///< Optional structure object to populate (cant use reference because its optional) +) { //... } @@ -237,7 +261,7 @@ struct MyStruct double offset_arr[10] = {}; OtherStruct* refStruct_ptr = nullptr; - map offsetMap; + map offsetMap; list> variationMapList; map subStructMap; }; @@ -254,7 +278,7 @@ if (acsConfig.some_parameter) ## Undesirable Code -* Do not use 'magic numbers', which require knowledge of other code fragments for comprehension. If a comment is required for explaining what a value means, the code should be rewritten with enums or defined constants. +* Do not use 'magic numbers', which require knowledge of other code fragments for comprehension. If a comment is required for explaining what a value means, the code should be rewritten with enums or defined constants. * Do not append `.0` to integer valued doubles unless they are required. * Never use `free()`, `malloc()`, or `new` unless it cannot be avoided. * Threads create synchronisation issues; they should not be used unless manual synchronisation is never required. @@ -277,11 +301,12 @@ struct MyStruct /** Function to demonstrate documentation */ void function( - bool runTests, ///< Run unit test while processing - MyStruct& myStruct, ///< Structure to modify - OtherStr* otherStr = nullptr) ///< Optional string to populate + bool runTests, ///< Run unit test while processing + MyStruct& myStruct, ///< Structure to modify + OtherStr* otherStr = nullptr ///< Optional string to populate +) { - //... + //... } ``` @@ -292,52 +317,52 @@ void function( ### Bad - double double_arr[10] = {}; + double double_arr[10] = {}; - //..(Populate array) + //..(Populate array) - for (int i = 0; i < 10; i++) //Magic number 10 - bad. - { + for (int i = 0; i < 10; i++) //Magic number 10 - bad. + { - } + } - map doubleMap; + map doubleMap; - //..(Populate Map) + //..(Populate Map) - for (auto iter = doubleMap.begin(); iter != doubleMap.end(); iter++) //long, undescriptive - bad - { - if (iter->first == someVar) //'first' is undescriptive - bad - { - //.. - } - } + for (auto iter = doubleMap.begin(); iter != doubleMap.end(); iter++) //long, undescriptive - bad + { + if (iter->first == someVar) //'first' is undescriptive - bad + { + //.. + } + } ### Good - Iterating Maps - map offsetMap; + map offsetMap; - //..(Populate Map) + //..(Populate Map) - for (auto& [siteName, offset] : doubleMap) //give readable names to map keys and values - { - if (siteName.empty() == false) - { - - } - } + for (auto& [siteName, offset] : doubleMap) //give readable names to map keys and values + { + if (siteName.empty() == false) + { + + } + } ### Good - Iterating Lists - list obsList; + list obsList; - //..(Populate list) + //..(Populate list) - for (auto& obs : obsList) //give readable names to list elements - { - doSomethingWithObs(obs); - } + for (auto& obs : obsList) //give readable names to list elements + { + doSomethingWithObs(obs); + } ### Special Case - Deleting from maps/lists @@ -346,7 +371,7 @@ Use iterators when you need to delete from STL containers: ``` for (auto it = someMap.begin(); it != someMap.end(); ) { - KFKey key = it->first; //give some alias to the key/value so they're readable + KFKey key = it->first; //give some alias to the key/value so they're readable if (measuredStates[key] == false) { @@ -363,15 +388,15 @@ for (auto it = someMap.begin(); it != someMap.end(); ) Commonly used std containers may be included with `using` - #include - #include - #include - #include + #include + #include + #include + #include - using std::string; - using std::map; - using std::list - using std::unordered_map; + using std::string; + using std::map; + using std::list + using std::unordered_map; ## Code sequencing @@ -379,11 +404,7 @@ Commonly used std containers may be included with `using` The software is to be kept largely sequential - using threads sparingly to limit the overhead of collision avoidance. Where possible tasks are completed in parallel using parallelisation libraries to take advantage of all CPU cores in multi-processor systems while still retaining a linear flow through the execution. -Sections of the software that create and modify global objects, such as while reading ephemeris data, will be executed on a single core only. +Sections of the software that create and modify global objects, such as while reading ephemeris data, will be executed on a single core only. This will ensure that collisions are avoided and the debugging of these functions is deterministic. For sections of the software that have clear delineation between objects, such as per-receiver calculations, these may be completed in parallel, provided they do not attempt to modify or create objects with more global scope. When globally accessible objects need to be created for individual receivers, they should be pre-initialised before the entry to parallel execution section. - - - - diff --git a/Docs/ginanFAQ.html b/Docs/ginanFAQ.html index 0310137e0..0c323ca90 100644 --- a/Docs/ginanFAQ.html +++ b/Docs/ginanFAQ.html @@ -14,7 +14,7 @@

Frequently asked questions about Ginan

-

Ginan is a precise point positioning software toolkit. It can calculate positions with centimetre-level accuracy using observations of global navigation satellite systems (GNSS) and correction data. It can also be used to create that correction data. More

+

Ginan is a precise point positioning software toolkit. It can calculate positions with centimetre-level accuracy using observations of global navigation satellite systems (GNSS) and correction data. It can also be used to create that correction data. More

@@ -57,7 +57,7 @@

Frequently asked questions about Ginan

The Ginan software is available from the Geoscience Australia GitHub site.

-

Geoscience Australia uses Ginan to produce precise positioning products and data correction streams. These are also free to use.

+

Geoscience Australia uses Ginan to produce precise positioning products and data correction streams. These are also free to use.

@@ -120,7 +120,7 @@

Frequently asked questions about Ginan

-

The science of calculating a position on Earth using observations of GNSS satellites is well known and has become a ubiquitous feature of our lives. Ginan uses the well-established State Space Representation (SSR) positioning model to enable users to achieve Precise Point Positioning (PPP). In the standard GNSS positioning process, small error sources limit the users calculated position accuracy at the meter level. The SSR methodology enables augmenting the users GNSS observations with corrections for those small errors, allowing Ginan to calculate PPP positions at the decimetre to millimetre level of accuracy, depending on the quality of PPP corrections used. More

+

The science of calculating a position on Earth using observations of GNSS satellites is well known and has become a ubiquitous feature of our lives. Ginan uses the well-established State Space Representation (SSR) positioning model to enable users to achieve Precise Point Positioning (PPP). In the standard GNSS positioning process, small error sources limit the users calculated position accuracy at the meter level. The SSR methodology enables augmenting the users GNSS observations with corrections for those small errors, allowing Ginan to calculate PPP positions at the decimetre to millimetre level of accuracy, depending on the quality of PPP corrections used. More

@@ -134,7 +134,7 @@

Frequently asked questions about Ginan

-

At the heart of Ginan is a Kalman filter processing engine. The Kalman filter is an algorithm that takes observations received from GNSS satellites, and other auxiliary metadata, and produces estimates of unknown variables or states, such as the user’s position. The Ginan Kalman filter uses GNSS observations in their raw un-differenced form and can estimate all GNSS observation model states using either un-combined or the ionosphere free combination of the GNSS observations. More

+

At the heart of Ginan is a Kalman filter processing engine. The Kalman filter is an algorithm that takes observations received from GNSS satellites, and other auxiliary metadata, and produces estimates of unknown variables or states, such as the user’s position. The Ginan Kalman filter uses GNSS observations in their raw un-differenced form and can estimate all GNSS observation model states using either un-combined or the ionosphere free combination of the GNSS observations. More

@@ -148,7 +148,8 @@

Frequently asked questions about Ginan

-

There are two ways to get Ginan running:

+

There are three ways to get Ginan running:

+

You can download the Ginan binaries from the GitHub release website, either with or without the graphical user interface (GUI). These binaries work on Linux, MacOS and Windows. This can be found here: Ginan Releases

You can download the Ginan source code from the GitHub site, and the other required software packages, onto a Linux machine (including Windows Subsystem for Linux) and compile them to create the Ginan executable application. The instructions are in the Ginan README.

You can use Docker Desktop. Docker Desktop is a software application you can download onto a Linux, Mac or Windows computer. It creates a container that provides all the resources that the Ginan Docker image needs to run. Using Docker makes the process of getting started with Ginan simpler. You must download and install Docker Desktop, and then get the Ginan Docker image.

@@ -165,7 +166,7 @@

Frequently asked questions about Ginan

Ginan is a sophisticated software application with hundreds of parameters that control what it does. All these parameters are contained in configuration files using the yaml syntax (yaml - which might stand for "yet another markup language").

-

Ginan is released with example files to help users achieve particular outcomes. To understand more about Ginan's yaml files and parameters read the Ginan yaml guide.

+

Ginan is released with example files to help users achieve particular outcomes. To understand more about Ginan's yaml files and parameters read the Ginan yaml guide.

diff --git a/Docs/home.md b/Docs/home.md index ca833bec6..a8c9560eb 100644 --- a/Docs/home.md +++ b/Docs/home.md @@ -7,3 +7,10 @@ Ginan is an open source toolkit for creating precise point positioning (PPP) ana The source code for the current version of Ginan is available for download from [this site](https://github.com/GeoscienceAustralia/ginan). New versions of Ginan with enhanced capabilities will be developed and released over time. Geoscience Australia is establishing operational instances of Ginan that produce PPP analysis [products and streams](page.html?c=on&p=products.md) on a continuous basis and which are available free of charge to the public. + +## How to cite + +If you use Ginan in a publication, please cite: +``` +McClusky, Simon; Hammond, Aaron; Maj, Ronald; Allgeyer, Sébastien; Harima, Ken; Yeo, Mark; Du, Eugene; Riddell, Anna, "Precise Point Positioning with Ginan: Geoscience Australia’s Open-Source GNSS Analysis Centre Software," Proceedings of the ION 2024 Pacific PNT Meeting, Honolulu, Hawaii, April 2024, pp. 248-280. https://doi.org/10.33012/2024.19598 +``` diff --git a/Docs/images/GinanGUI-screenshot.png b/Docs/images/GinanGUI-screenshot.png new file mode 100644 index 000000000..b0d18a0e6 Binary files /dev/null and b/Docs/images/GinanGUI-screenshot.png differ diff --git a/Docs/images/GinanWorkshop2025WindowsSetUp-TN2.png b/Docs/images/GinanWorkshop2025WindowsSetUp-TN2.png new file mode 100644 index 000000000..b1c8d2716 Binary files /dev/null and b/Docs/images/GinanWorkshop2025WindowsSetUp-TN2.png differ diff --git a/Docs/images/ICRF-75pc-20250121.png b/Docs/images/ICRF-75pc-20250121.png new file mode 100644 index 000000000..d8d24d8f2 Binary files /dev/null and b/Docs/images/ICRF-75pc-20250121.png differ diff --git a/Docs/images/ICRF-75pc.png b/Docs/images/ICRF-75pc.png deleted file mode 100644 index dc91848cd..000000000 Binary files a/Docs/images/ICRF-75pc.png and /dev/null differ diff --git a/Docs/images/UseCasesv04.jpg b/Docs/images/UseCasesv04.jpg new file mode 100644 index 000000000..e920d017c Binary files /dev/null and b/Docs/images/UseCasesv04.jpg differ diff --git a/Docs/overview.md b/Docs/overview.md index cc1b92b56..2c4368122 100644 --- a/Docs/overview.md +++ b/Docs/overview.md @@ -4,13 +4,6 @@ > This page provides an overview of the documentation and artefacts available to support your use of Ginan. -#### Videos: -The Ginan team have made a series of short videos to describe Ginan and help with its installation. They are: - -* [Ginan Video Tutorial - Introduction](https://www.youtube.com/watch?v=oP_vk5sci1k&list=PL0jP_ahe-BFnChGLpQmXYpHNFiRze4DZR&index=2) -* [Ginan Video Tutorial - Installation](https://www.youtube.com/watch?v=FAi2fg-7tbs&list=PL0jP_ahe-BFnChGLpQmXYpHNFiRze4DZR&index=2) -* [Ginan Video Tutorial - Docker Install](https://www.youtube.com/watch?v=uW1DcIbZk1g&list=PL0jP_ahe-BFnChGLpQmXYpHNFiRze4DZR&index=3) - #### Installation: The written notes on how to download and install supporting packages and the Ginan software are detailed [here](page.html?c=on&p=install.index) diff --git a/Docs/resources.md b/Docs/resources.md index a892979d1..51d95504a 100644 --- a/Docs/resources.md +++ b/Docs/resources.md @@ -10,11 +10,14 @@ > [![](images/GinanWorkshopS.png) Slides from the Ginan Workshop hosted at the IGNSS 2024 conference.](resources/Ginan_Workshop_Slides_-_IGNSS_2024.pdf) -> [![](images/GinanWorkshop1.png) Set Up Guide for the Ginan Workshop.](resources/Ginan_Workshop_1_Set-up_Guide_-_IGNSS_2024_-_6_Feb.pdf) +> [![](images/GinanWorkshop2025WindowsSetUp-TN2.png) Set-up Guide for Windows Users from 2025 Geodesy Workshop at ANU.](resources/Ginan-Workshop--Windows-Set-up--2025-06-23.pdf) -> [![](images/GinanWorkshop2.png) Guide outlining how to run Ginan in the Workshop Tutorial.](resources/Ginan_Workshop_2_Running_Ginan_-_IGNSS_2024_-_6_Feb.pdf) +> [![](images/GinanWorkshop1.png) Set Up Guide for the Ginan 2024 Workshop.](resources/Ginan_Workshop_1_Set-up_Guide_-_IGNSS_2024_-_6_Feb.pdf) + +> [![](images/GinanWorkshop2.png) Guide outlining how to run Ginan in the 2024 Workshop Tutorial.](resources/Ginan_Workshop_2_Running_Ginan_-_IGNSS_2024_-_6_Feb.pdf) + +> [![](images/GinanWorkshopD.png) Guide for those using Docker to run Ginan - 2024.](resources/Ginan_Workshop_Docker_Guide_-_IGNSS_2024_-_6_Feb.pdf) -> [![](images/GinanWorkshopD.png) Guide for those using Docker to run Ginan.](resources/Ginan_Workshop_Docker_Guide_-_IGNSS_2024_-_6_Feb.pdf) *** diff --git a/Docs/resources/Ginan-Workshop--Windows-Set-up--2025-06-23.pdf b/Docs/resources/Ginan-Workshop--Windows-Set-up--2025-06-23.pdf new file mode 100644 index 000000000..4054e102a Binary files /dev/null and b/Docs/resources/Ginan-Workshop--Windows-Set-up--2025-06-23.pdf differ diff --git a/Docs/scripts.index b/Docs/scripts.index index 8d961183a..6efe0afc3 100644 --- a/Docs/scripts.index +++ b/Docs/scripts.index @@ -1,4 +1,3 @@ ginanEDA.md autoDownload.md - s3_filehandler.md - flexPower.md \ No newline at end of file + s3_filehandler.md \ No newline at end of file diff --git a/Docs/theory.md b/Docs/theory.md index b7a55c597..b5d3334d4 100644 --- a/Docs/theory.md +++ b/Docs/theory.md @@ -14,7 +14,7 @@ Well fortunately for the vast majority of applications we only need to know wher ## The International Celestial Reference Frame (ICRF) -![The International Celestial Reference Frame (ICRF)](images/ICRF-75pc.png) +![The International Celestial Reference Frame (ICRF)](images/ICRF-75pc-20250121.png) *The International Celestial Reference Frame (ICRF). J2000.0 is a standard Julian equinox and epoch - January 1, 2000 at 12:00 TT.* diff --git a/README.md b/README.md index ff5818f7f..e0ebba329 100755 --- a/README.md +++ b/README.md @@ -1,274 +1,465 @@ -# ![gn_logo](https://raw.githubusercontent.com/GeoscienceAustralia/ginan/gh-pages/images/GinanLogo273-with-background.png) +# ![Ginan Logo](https://raw.githubusercontent.com/GeoscienceAustralia/ginan/gh-pages/images/GinanLogo273-with-background.png) -# Ginan: Software toolkit and service +# Ginan: GNSS Analysis Software Toolkit -#### `Ginan v3.1.0` +[![Version](https://img.shields.io/badge/version-v4.0.0-blue.svg)](https://github.com/GeoscienceAustralia/ginan/releases) +[![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.md) +[![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20macOS%20%7C%20Windows-lightgrey.svg)](#supported-platforms) +[![Docker](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/gnssanalysis/ginan) + +**Ginan** is a powerful, open-source software toolkit for processing Global Navigation Satellite System (GNSS) observations for geodetic applications. Developed by Geoscience Australia, Ginan provides state-of-the-art capabilities for precise positioning, orbit determination, and atmospheric modeling. + +## How to cite + +If you use Ginan in a publication, please cite: +``` +McClusky, Simon; Hammond, Aaron; Maj, Ronald; Allgeyer, Sébastien; Harima, Ken; Yeo, Mark; Du, Eugene; Riddell, Anna, "Precise Point Positioning with Ginan: Geoscience Australia’s Open-Source GNSS Analysis Centre Software," Proceedings of the ION 2024 Pacific PNT Meeting, Honolulu, Hawaii, April 2024, pp. 248-280. https://doi.org/10.33012/2024.19598 +``` + +## Table of Contents + +- [Quick Start](#quick-start) +- [Overview](#overview) + - [How to cite](#how-to-cite) + - [Supported GNSS Constellations](#supported-gnss-constellations) + - [Key Features and Capabilities](#key-features-and-capabilities) + - [Architecture](#architecture) +- [Installation](#installation) + - [Using Ginan with Docker](#using-ginan-with-docker) + - [Precompiled binaries](#precompiled-binaries) + - [Installation from Source](#installation-from-source) + - [Tested Platforms](#tested-platforms) + - [Prerequisites](#prerequisites) + - [Build Process using `vcpkg` + CMake presets (Recommanded)](#build-process-using-vcpkg--cmake-presets-recommanded) + - [Legacy: manual `cmake` + `make` instructions](#legacy-manual-cmake---make-instructions) + - [Python Environment Setup](#python-environment-setup) +- [Getting Started with the examples](#getting-started-with-the-examples) + - [Running Your First Example](#running-your-first-example) + - [Adding Ginan to PATH](#adding-ginan-to-path) +- [Additional Tools and Scripts](#additional-tools-and-scripts) +- [Documentation](#documentation) + - [User Documentation](#user-documentation) + - [Developer Documentation](#developer-documentation) + - [Generating Code Documentation](#generating-code-documentation) +- [Contributing](#contributing) + - [Reporting Issues](#reporting-issues) + - [Contributing Code](#contributing-code) + - [Development Setup](#development-setup) +- [Support](#support) + - [Getting Help](#getting-help) +- [License](#license) + - [Third-Party Components](#third-party-components) +- [Acknowledgements](#acknowledgements) + +## Quick Start + +The fastest way to get started with Ginan is using Docker: + +```bash +# Pull and run the latest Ginan container +docker run -it -v $(pwd):/data gnssanalysis/ginan:v4.0.0 bash + +# Verify installation +pea --help + +# Run a basic example +cd /ginan/exampleConfigs +pea --config ppp_example.yaml +``` ## Overview -Ginan is a processing package being developed to process GNSS observations for geodetic applications. +Ginan is a comprehensive processing package for GNSS observations in geodetic applications, supporting multiple satellite constellations and providing advanced analysis capabilities. + -We currently support the processing of: -* the United States' Global Positioning System (**GPS**); -* the European Union's Galileo system (**Galileo**); -* the Russian GLONASS system (**GLONASS**)\*; -* the Chinese Navigation Satellite System (**BeiDou**)\*; -* the Japanese QZSS develop system (**QZSS**)\*. +### Supported GNSS Constellations -We are actively developing Ginan to have the following capabilities and features: +We currently support processing of: -* Precise Orbit & Clock determination of GNSS satellites (GNSS POD); -* Precise Point Positioning (PPP) of GNSS stations in network and individual mode; -* Real-Time corrections for PPP users; -* Analyse full, single and multi-frequency, multi-GNSS data; -* Delivering atmospheric products such as ionosphere and troposphere models; -* Servicing a wide range of users and receiver types; -* Delivering outputs usable and accessible by non-experts; -* Providing both a real-time and off-line processing capability; -* Delivering both position and integrity information; -* Routinely produce IGS final, rapid, ultra-rapid and real-time (RT) products; -* Model Ocean Tide Loading (OTL) displacements. +- **GPS** - United States' Global Positioning System +- **Galileo** - European Union's Galileo system +- **GLONASS** - Russian GLONASS system* +- **BeiDou** - Chinese Navigation Satellite System* +- **QZSS** - Japanese Quasi-Zenith Satellite System* + +*\*Development ongoing* + +### Key Features and Capabilities + +Ginan provides the following advanced capabilities: + +- **Precise Orbit & Clock Determination** (POD) of GNSS satellites +- **Precise Point Positioning** (PPP) for stations in network and individual modes +- **Real-time corrections** generation for PPP users +- **Multi-frequency, multi-GNSS** data analysis +- **Atmospheric products** including ionosphere and troposphere models +- **Low Earth Orbiter** orbit modeling capabilities +- **Satellite Laser Ranging** processing capabilites +- Support for **wide range of users and receiver types** +- **User-friendly outputs** accessible by non-experts +- **Real-time and offline** processing capabilities +- **IGS products** generation (final, rapid, ultra-rapid, and real-time) +- **Ocean Tide Loading** (OTL) displacement modeling + +### Architecture The software consists of three main components: -* Network Parameter Estimation Algorithm (PEA), and -* Various scripts for combination and analysis of solutions -## Using Ginan with an AppImage +- **Parameter Estimation Algorithm (PEA)** - Core processing engine incorporating Precise Orbit Determination +- **Analysis Scripts** - Tools for data preparation, solution combination and analysis +- **Visualization Tools** - Python-based plotting and comparison utilities + +## Installation -You can quickly download a precompiled binary of Ginan's pea from the `develop-weekly-appimage` branch of github. -This allows you to run Ginan without the need for installing external dependencies. -It contains no python scripts or example data, but is possible to run immediately on linux and windows systems as simply as: +Choose the installation method that best fits your needs: - git clone -b develop-weekly-appimage --depth 1 --single-branch https://github.com/GeoscienceAustralia/ginan.git +### Using Ginan with Docker - ginan/Ginan-x86_64.AppImage +**Recommended for most users** - Get started quickly with a pre-configured environment: -or on windows: +```bash +# Run Ginan container with data volume mounting +docker run -it -v ${pwd}:/data gnssanalysis/ginan:v4.0.0 bash +``` + +This command: - wsl --install -d ubuntu - ginan/Ginan-x86_64.AppImage +- Mounts your current directory (`${pwd}`) to `/data` in the container +- Provides access to all Ginan tools and dependencies +- Opens an interactive bash shell -If the image fails to run, first ensure it is executable and all requires libraries are available +**Prerequisites:** [Docker](https://docs.docker.com/get-docker/) must be installed on your system. + +**Verify installation:** +```bash +pea --help +``` - chmod 777 ginan/Ginan-x86_64.AppImage - apt install fuse libfuse2 +### Precompiled binaries +Precompiled binaries for **Ginan** and **GinanUI** are available on the project's GitHub Releases page: https://github.com/GeoscienceAustralia/ginan/releases +We publish builds for the following platforms: -## Using Ginan with Docker +- Linux (x86_64) +- macOS (arm64 and x86_64) +- Windows (x86_64) -You can quickly download a ready-to-run Ginan environment using docker by running: +These artifacts are provided for convenience and have been tested on our CI runners and a subset of target systems. They may not work on every configuration — if you encounter problems please try the Docker image or build from source (see the Build Process section) and open an issue on GitHub with your OS and steps to reproduce. - docker run -it -v ${host_data_folder}:/data gnssanalysis/ginan:v3.1.0 bash +Note about Windows binaries: We have observed an output file-size limitation on Windows builds where RTS/output files appear limited at about 2.1 GB (roughly equivalent to a PPP processing of two stations over one day at 30 s resolution). If you require larger RTS outputs, run the processing on Linux/macOS (or in the Docker image) or build from source on a platform without this limitation. We plan to implement a permanent solution in a future release. -This command connects the `${host_data_folder}` directory on the host (your pc), with the `/data` directory in the container, to allow file access between the two systems, and opens a command line (`bash`) for executing commands. +### Installation from Source -You will need to have [docker](https://docs.docker.com/get-docker/) installed to use this method. +**For developers and advanced users** who need to modify the source code or require specific configurations. -To verify you have the Ginan executables available once at the Ginan command line, run: +#### Tested Platforms - pea --help +| Platform | Tested Versions | Notes | +|----------|-----------------|-------| +| **Linux** | Ubuntu 22.04, 24.04 | Primary development platform | +| **macOS** | 10.15+ (x86) | Limited testing | +| **Windows** | 10+ | Limited testing| +#### Prerequisites -## Installation from source -### Supported Platforms +##### System Dependencies -Ginan is supported and tested on the following platforms +**Compilers:** -* Linux: tested on Ubuntu 18.04 and 20.04 and 22.04 -* MacOS: tested on 10.15 (x86) -* Windows: via docker or WSL on Windows 10 and above +- GCC/G++ (recommended, tested and supported) or equivalent C/C++ compiler -### Dependencies +**Required Dependencies:** -If instead you wish to build Ginan from source, there are several software dependencies: +- **CMake** ≥ 3.0 -* C/C++ and Fortran compiler. We use and recommend [gcc, g++, and gfortran](https://gcc.gnu.org) -* BLAS and LAPACK linear algebra libraries. We use and recommend [OpenBlas](https://www.openblas.net/) as this contains both libraries required -* CMAKE > 3.0 -* YAML > 0.6 -* Boost >= 1.73 (tested on 1.73). On Ubuntu 22.04 which uses gcc-11, you need Boost >= 1.74.0 -* MongoDB -* Mongo_C >= 1.17.1 -* Mongo_cxx >= 3.6.0 -* Eigen3 > 3.4 -* netCDF4 -* Python >= 3.7 +- **YAML** ≥ 0.6 -If using gcc verion 11 or about, the minimum version of libraries are: -* Boost >= 1.74.0 -* Mongo_cxx = 3.7.0 +- **Boost** ≥ 1.75 + -Scripts to install dependencies for Ubuntu 18.04/20.04, 22.04, Fedora 38 are available on the `scripts/installation` directory. Users on other system might need to have a look at the `scripts/installation/generic.md` file, which contains the major steps. +- **Eigen3** ≥ 3.4 -### Python +- **OpenBLAS** (provides BLAS and LAPACK) -We use Python for automated process (download), postprocessing and visualisation. To use the developed tools, we recommand to use a virtual-environement (or Anaconda equivalent). A requirements file is available in the `scripts/` directory and can be run via -```python -pip3 install -r requirements.txt +**Optional Dependencies:** + +- **Mongo C Driver** ≥ 1.17.1 + +- **Mongo C++ Driver** ≥ 3.6.0 (= 3.7.0 for GCC 11+) + +- **MongoDB** (for database features) + +- **netCDF4** (for tidal loading computation) + +- **Python** ≥ 3.9 + +#### Build Process using `vcpkg` + CMake presets (Recommanded) + +We recommend using `vcpkg` for dependency management together with the repository CMake presets. + +1. Bootstrap and install `vcpkg` (from repository root): + +```bash +# Clone/bootstrap vcpkg (if not present) +./vcpkg/bootstrap-vcpkg.sh + +# Install packages for your target triplet (example: Linux x86_64) +./vcpkg/vcpkg install --triplet x64-linux --x-install-root=./vcpkg_installed +# For macOS: use `arm64-osx` or `x64-osx`. For Windows cross builds (on linux) use `x64-mingw-static`. ``` -### Build -Prepare a directory to build in - it's better practice to keep this separated from the source code. -From the Ginan git root directory: +2. Configure and build with a CMake preset (run from `src`): ```bash -mkdir -p src/build +cd src +# Choose the preset that matches your platform (examples: `release`, `macos-arm64-release`, `macos-x64-release`, `windows-cross-release`) +cmake --preset release +cmake --build --preset release -cd src/build -cmake ../ +# Or build the preset directory directly (example for Linux): +cmake --build build/linux-Release --parallel $(nproc) +``` + +Note on loading / netCDF: the ocean-tide loading components currently have known problems when built from the `vcpkg` dependency set due to issues with the `netcdf` package in some vcpkg triplets. If you rely on tidal-loading features (the `make_otl_blq` target and related tools), either: + +- Build those components from source using your system `netcdf` (install `netcdf`/`netcdf-c` via the OS package manager and use the legacy `cmake`/`make` flow), or +- Track the vcpkg `netcdf` fixes and retry when upstream provides a compatible package for your target triplet. + +If you need help reproducing or a suggested workaround for your platform, open an issue with your OS/triplet and vcpkg versions. + +Notes: +- The CI uses `--x-install-root=./vcpkg_installed` to install packages locally for reproducible builds. +- If you prefer not to use `vcpkg`, the legacy manual flow below remains supported. + +#### Legacy: manual `cmake` + `make` instructions + +##### Quick Installation Scripts (legacy) + +Pre-written installation scripts are available in `scripts/installation/` for systems where you prefer distro-specific package installation instead of `vcpkg`: + +```bash +# Ubuntu 24.04 +./scripts/installation/ubuntu24.sh + +# Ubuntu 22.04 +./scripts/installation/ubuntu22.sh + +# Ubuntu 20.04 +./scripts/installation/ubuntu20.sh + +# Fedora 38 +./scripts/installation/fedora38.sh + +# Generic instructions +cat scripts/installation/generic.md ``` -To build every package simply run `make` or `make -jX` , where X is a number of parallel threads you want to use for the compilation: +**Note:** These scripts are maintained as best-effort and may require adjustments for your environment. If you are using the `vcpkg` + CMake presets workflow, follow the `vcpkg` steps in the Build Process section instead. + +The older manual flow is still available for users who prefer it: + +1. Create build directory: ```bash -make -j2 +mkdir -p src/build +cd src/build ``` -Alternatively, to build only a specific package (`pea`, `make_otl_blq`, `interpolate_loading`), run as below: +2. Configure with CMake (legacy): ```bash -make pea -j2 +cmake ../ ``` -This should create executables in the `bin` directory of Ginan. +3. Compile (legacy): + +```bash +# Build everything (parallel compilation recommended) +make -j$(nproc) + +# Build specific components +make pea -j$(nproc) # Core PEA executable +make make_otl_blq -j$(nproc) # Ocean tide loading +make interpolate_loading -j$(nproc) # Loading interpolation +``` -Check to see if you can execute the PEA from the exampleConfigs directory +4. Verify installation: ```bash cd ../../exampleConfigs - ../bin/pea --help ``` -and you should see something similar to: +Expected output: ``` -PEA starting... (main ginan-v3.0.0 from Mon Feb 05 15:15:22 2024) - +PEA starting... (main ginan-v4.0.0 from ...) Options: - -h [ --help ] Help - -q [ --quiet ] Less output - -v [ --verbose ] More output - -V [ --very-verbose ] Much more output - . - . - . - --dump-config-only Dump the configuration and exit - --walkthrough Run demonstration code interactively with - commentary - -PEA finished + -h [ --help ] Help + -q [ --quiet ] Less output + ... ``` - -Then download all of the example data using the scripts and filelists provided. From the Ginan git root directory: +5. Download example data: ```bash -cd inputData/data +cd ../inputData/data ./getData.sh -cd ../products +cd ../products ./getProducts.sh ``` -### Directory Structure - -Upon installation, the `ginan` directory should have the following structure: - - ginan/ - ├── README.md ! General README information - ├── LICENSE.md ! Software License information - ├── ChangeLOG.md ! Release Change history - ├── aws/ ! Amazon Web Services config - ├── bin/ ! Binary executables directory* - ├── Docs/ ! Documentation directory - ├── inputData/ ! Input data for examples - │ ├── data/ ! Example dataset (rinex files)** - │ └── products/ ! Example products and aux files** - ├── exampleConfigs ! Example configuration files - │ ├── ppp_example.yaml ! Basic user-mode example - │ └── pod_example.yaml ! Basic network-mode example - ├── lib/ ! Compiled object library directory* - ├── scripts/ ! Auxiliary Python and Shell scripts and libraries - └── src/ ! Source code directory - ├── cpp/ ! Ginan source code - ├── cmake/ - ├── doc_templates/ - ├── build/ ! Cmake build directory* - └── CMakeLists.txt - -*\*created during installation process* - -*\*\* contents retrieved with getData.sh, getProducts.sh scripts* +### Python Environment Setup +Ginan uses Python for automation, post-processing, and visualization: -## Documentation +```bash +# Create virtual environment (recommended) +python3 -m venv ginan-env +source ginan-env/bin/activate + +# Install Python dependencies +pip3 install -r scripts/requirements.txt +``` + + +## Getting Started with the examples + +Congratulations! Ginan is now ready to use. The examples in `exampleConfigs/` provide a great starting point. -Ginan documentation consists of two parts: these documents, and separate Doxygen-generated documentation that shows the actual code infrastructure. -It can be found [here](https://geoscienceaustralia.github.io/ginan/codeDocs/index.html), or generated manually as below. +- **Working directory:** All examples must be run from the `exampleConfigs/` directory due to relative paths +- **MongoDB:** If MongoDB is not installed, set `mongo: enable: None` in configuration files +- **Performance tip:** For single-station PPP, limit cores to improve performance: + ```bash + OMP_NUM_THREADS=1 ../bin/pea --config ppp_example.yaml + ``` -### Doxygen -The Doxygen documentation for Ginan requires `doxygen` and `graphviz`. If not already installed, type as follows: +### Running Your First Example + +1. **Navigate to examples directory:** + ```bash + cd exampleConfigs + ``` + +2. **Run basic PPP example:** + ```bash + ../bin/pea --config ppp_example.yaml + ``` + +3. **Check outputs:** + + The processing will create an directory named `outputs/ppp_example/` or similar containing: + - `*.trace` files with station processing details + - `Network*.trace` with Kalman filter results + - Other auxiliary outputs as configured + + +### Adding Ginan to PATH + +For convenience, add Ginan binaries to your system PATH: ```bash -sudo apt -y install doxygen graphviz +# Add to ~/.bashrc or ~/.zshrc +export PATH="/path/to/ginan/bin:$PATH" + +# Then run from anywhere +pea --config /path/to/config.yaml ``` -On success, proceed to the build directory and call make with `docs` target: -```bash -cd ../src/build -cmake ../ +## Additional Tools and Scripts -make docs -``` +Beyond the core PEA executable, Ginan includes [comprehensive scripts](https://geoscienceaustralia.github.io/ginan/page.html?c=on&p=scripts.index) for: -The documentation can then be found at `Docs/codeDocs/index.html`. +- **Data downloading** and preprocessing +- **Output visualization** and analysis +- **Solution comparison** and validation +- **Performance monitoring** and reporting -Note that documentation is also generated automatically if `make` is called without arguments and `doxygen` and `graphviz` dependencies are satisfied. +## Documentation + +Ginan documentation is available in multiple formats: + +### User Documentation +- **Online Manual:** [geoscienceaustralia.github.io/ginan](https://geoscienceaustralia.github.io/ginan/) +- **Configuration Guide:** [Detailed parameter explanations and examples](https://geoscienceaustralia.github.io/ginan/page.html?c=on&p=ginanConfiguration.md) +- **FAQ:** [Ginan FAQ](https://geoscienceaustralia.github.io/ginan/page.html?p=ginanFAQ.html) -## Ready! -Congratulations! You are now ready to trial the examples from the `exampleConfigs` directory. See Ginan's manual for detailed explanation of each example. Note that examples have relative paths to files in them and rely on the presence of `products` and `data` directories inside the `inputData` directory. Make sure you've run `s3_filehandler.py` from the Build step of these instructions. +### Developer Documentation -The paths are relative to the `exampleConfigs` directory and hence all the examples must be run from the `exampleConfigs` directory. +- **Code Documentation:** [API Reference](https://geoscienceaustralia.github.io/ginan/codeDocs/index.html) -NB: Examples may be configured to use mongoDB. If you have not installed it, please set `mongo: enable` to false in the pea config files. +### Generating Code Documentation -To run the first example of the PEA: +Requirements: `doxygen` and `graphviz` ```bash -cd ../exampleConfigs +# Install dependencies (Ubuntu/Debian) +sudo apt install doxygen graphviz -../bin/pea --config ppp_example.yaml +# Generate documentation +cd src/build +cmake ../ +make docs + +# View documentation +open ../../Docs/codeDocs/index.html ``` -This should create `outputs/ppp_example` directory with various `*.trace` files, which contain details about stations processing, a `Network*.trace` file, which contains the results of Kalman filtering, and other auxiliary output files as configured in the yamls. +## Contributing -You can remove the need for path specification to the executable by using the symlink within `exampleConfigs`, or by adding Ginan's bin directory to `~/.bashrc` file: -``` -PATH="path_to_ginan_bin:$PATH" -``` +We welcome contributions from the community! Here's how to get involved: + +### Reporting Issues +- Use [GitHub Issues](https://github.com/GeoscienceAustralia/ginan/issues) for bug reports +- Provide detailed reproduction steps and system information +- Check existing issues before creating new ones + +### Contributing Code +1. Fork the repository +2. Create a feature branch: `git checkout -b feature-name` +3. Follow our [coding standards](Docs/codingStandard.md) +4. Submit a pull request with clear description -NB: For PPP positioning of a single station, we have noted that limiting the number of cores to 1 can reduce processing times. This can be achieved via setting the environment variable `OMP_NUM_THREADS`: +### Development Setup +- Follow the source installation instructions above +- Review `Docs/codingStandard.md` for guidelines +- Run tests before submitting. - OMP_NUM_THREADS=1 ginan/Ginan-x86_64.AppImage +## Support +### Getting Help +- **Documentation:** Check the [online manual](https://geoscienceaustralia.github.io/ginan/) first +- **Issues:** Report bugs and feature requests on [GitHub](https://github.com/GeoscienceAustralia/ginan/issues) +- **Discussions:** Join community discussions on [GitHub Discussions](https://github.com/GeoscienceAustralia/ginan/discussions) +## License -## Scripts -In addition to the Ginan binaries, [scripts](https://geoscienceaustralia.github.io/ginan/page.html?c=on&p=scripts.index) are available to assist with downloading input files, and viewing and comparing generated outputs. +Ginan is released under the **Apache License 2.0**. See [LICENSE.md](LICENSE.md) for details. +### Third-Party Components +This software incorporates components from several open-source projects. See [Acknowledgements](#acknowledgements) below for detailed attribution. +## Acknowledgements -### Acknowledgements: -We have used routines obtained from RTKLIB, released under a BSD-2 license, these routines have been preserved with modifications in the folder `cpp/src/rtklib`. The original source code from RTKLib can be obtained from https://github.com/tomojitakasu/RTKLIB. +Ginan incorporates code from several excellent open-source projects: -We have used routines obtained from Better Enums, released under the BSD-2 license, these routines have been preserved in the folder `cpp/src/3rdparty` The original source code from Better Enums can be obtained from http://github.com/aantron/better-enums. +| Project | License | Purpose | Original Source | +|---------|---------|---------|-----------------| +| **Magic Enum** | MIT | Enhanced enum support | [github.com/Neargye/magic_enums](https://github.com/Neargye/magic_enums) | +| **EGM96** | zlib | Earth gravitational model | [github.com/emericg/EGM96](https://github.com/emericg/EGM96) | +| **IERS2010**| Public Domain | Tidal displacement computation | [github.com/xanthospap/iers2010](https://github.com/xanthospap/iers2010) +| **JPL Ephemeris** | GPL-3 | Planetary ephemeris | [github.com/Bill-Gray/jpl_eph](https://github.com/Bill-Gray/jpl_eph) | +| **NRLMSISE** | Public Domain | Atmospheric modeling | [github.com/c0d3runn3r/nrlmsise](https://github.com/c0d3runn3r/nrlmsise/tree/master) | +| **RTKLIB** | BSD-2-Clause | GNSS processing routines | [github.com/tomojitakasu/RTKLIB](https://github.com/tomojitakasu/RTKLIB) | +| **SLR** | Public Domain | SLR input file managements | [ilrs.gsfc.nasa.gov](https://ilrs.gsfc.nasa.gov/data_and_products/formats/crd.html)* +| **SOFA** | SOFA License | Astronomical computations | [iausofa.org](https://www.iausofa.org/) | -We have used routines obtained from EGM96, released under the zlib license, these routines have been preserved in the folder `cpp/src/3rdparty/egm96` The original source code from EGM96 can be obtained from https://github.com/emericg/EGM96. +All incorporated code has been preserved with appropriate modifications in the `cpp/src/` directory structure, maintaining original licensing and attribution requirements. -We have used routines obtained from SOFA, released under the SOFA license, these routines have been preserved in the folder `cpp/src/3rdparty/sofa` The original source code from SOFA can be obtained from https://www.iausofa.org/. +--- -We have used routines obtained from project Pluto, released under the GPL-3 license, these routines have been preserved in the folder `cpp/src/3rdparty/jplephem` The original source code from jpl ephem can be obtained from https://github.com/Bill-Gray/jpl_eph. +**Developed by [Geoscience Australia](https://www.ga.gov.au/)** | **Version 4.0.0** | **[GitHub Repository](https://github.com/GeoscienceAustralia/ginan)** diff --git a/debugConfigs/pod_rt_example1.yaml b/debugConfigs/pod_rt_example1.yaml index 2553d6c9b..461457fc4 100644 --- a/debugConfigs/pod_rt_example1.yaml +++ b/debugConfigs/pod_rt_example1.yaml @@ -106,7 +106,7 @@ inputs: - "TID100AUS0" - "MSSA00JPN0" - "ONS100SWE0" -# + outputs: outputs_root: outputs/ @@ -126,6 +126,15 @@ outputs: output_predicted_states: true output_config: true + streams: + labels: + - GAA1 + GAA1: + url: https://ginan-isg-testing:p4_bL8-ctt7tn4ddZ@ntrip.test-data.gnss.ga.gov.au/SSRA00ISG8 + messages: + rtcm_1060: {udi: 10} + rtcm_1059: {udi: 10} + satellite_options: global: @@ -148,12 +157,8 @@ satellite_options: enable: true code_bias: enable: true - default_bias: 0 - undefined_sigma: 0 phase_bias: enable: false - default_bias: 0 - undefined_sigma: 0 orbit_propagation: albedo: cannonball @@ -216,12 +221,8 @@ receiver_options: # Options to c enable: true # (bool) Enable modelling of phase center variations code_bias: enable: true # (bool) Enable modelling of code biases - default_bias: 0 # (float) Bias to use when no code bias is found - undefined_sigma: 0 # (float) Uncertainty sigma to apply to default code biases phase_bias: enable: false # (bool) Enable modelling of phase biases - default_bias: 0 # (float) Bias to use when no phase bias is found - undefined_sigma: 0 # (float) Uncertainty sigma to apply to default phase biases pos: enable: true # (bool) Enable modelling of position ionospheric_components: # Ionospheric models produce frequency-dependent effects diff --git a/debugConfigs/record_ssr_stream.yaml b/debugConfigs/record_ssr_stream.yaml index be47993a8..40982949a 100644 --- a/debugConfigs/record_ssr_stream.yaml +++ b/debugConfigs/record_ssr_stream.yaml @@ -2,7 +2,7 @@ inputs: - inputs_root: ./products/RAP/ + inputs_root: ./products/latest/ atx_files: - igs20.atx @@ -10,6 +10,8 @@ inputs: snx_files: - igs_satellite_metadata.snx - tables/sat_yaw_bias_rate.snx + - tables/bds_yaw_modes.snx + - tables/qzss_yaw_modes.snx gnss_observations: gnss_observations_root: "https://:@ntrip.data.gnss.ga.gov.au/" @@ -36,6 +38,10 @@ outputs: output_rotation: period: 86400 + log: + output: true + filename: __LOG.json + rtcm_nav: output: true filename: __NAV.rtcm @@ -62,12 +68,27 @@ outputs: clock_sources: [ SSR ] output_interval: 300 +satellite_options: + + global: + models: + pos: + enable: true + sources: [ SSR ] + clock: + enable: true + sources: [ SSR ] + processing_options: + process_modes: + spp: false + epoch_control: require_obs: false - epoch_interval: 1 - wait_next_epoch: 1 + epoch_interval: 10 + wait_next_epoch: 10 + max_rec_latency: 1 sleep_milliseconds: 1 gnss_general: diff --git a/debugConfigs/sp3_ecef2eci.yaml b/debugConfigs/sp3_ecef2eci.yaml new file mode 100644 index 000000000..462f32be5 --- /dev/null +++ b/debugConfigs/sp3_ecef2eci.yaml @@ -0,0 +1,51 @@ +inputs: + + inputs_root: products/ + + erp_files: [ "podTest/2019195_07D/COD0MGXFIN*.ERP" ] + + satellite_data: + satellite_data_root: podTest/2019195_07D/ + sp3_files: [ "COD0MGXFIN*.SP3" ] + + +outputs: + + outputs_root: products/podTest/2019195_07D/SP3i/ + + metadata: + config_description: COD0MGXFIN + time_system: G # (string) Time system - e.g. "G", "UTC" + + sp3: + output: true + filename: __01D_05M_ORB.SP3 + orbit_sources: [ PRECISE ] + clock_sources: [ PRECISE ] + output_inertial: true + output_interval: 300 + + +receiver_options: # Options to configure individual stations or global configs + + global: + models: + eop: + enable: true + + +processing_options: + + epoch_control: + start_epoch: 2019-07-14 00:00:00 + end_epoch: 2019-07-20 23:55:00 + epoch_interval: 300 # seconds + require_obs: false + + gnss_general: + sys_options: + gps: { process: true } + gal: { process: true } + glo: { process: true } + bds: { process: true } + # qzs: { process: true } diff --git a/docker/Dockerfile b/docker/Dockerfile index 1a066d3b1..447cee469 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,6 @@ # ------ Docker build target: Run heavyweight build environment locally, on top of ginan-env image ------ # FROM gnssanalysis/ginan-env:latest as ginan-build -# ^ Recent build with patched (old) version of Doxygen. -# TODO switch from custom build to latest, once Doxygen build is incorporated into it. # NOTE: the doc build step later on requires a version of ginan-env with Doxygen installed in it. # This is added by (internal) Ginan PR #520, around Apr 2024 @@ -21,6 +19,11 @@ WORKDIR /ginan # Code assets for building PEA ADD src /ginan/src +# Add .git directory so build process can extract current tag / commit hash to embed in build. +# This data will not be published. The build image is not pushed, only the built artefacts (which are copied +# into the minimal image) +ADD .git /ginan/.git + # Docs, example configs, data and products. All relied on by Doxygen to generate documentation. # Add exampleConfigs and logical subdirs (ADD doesn't seem to follow symlinks). These are used by Doxygen. ADD exampleConfigs /ginan/exampleConfigs @@ -35,12 +38,14 @@ ARG TARGET=pea # Values >2 may cause memory issues on BitBucket pipelines. On development machines, set higher for speed (eg 4). ARG BUILD_THREADS=2 -# Main outputs are ginan/bin and (if TARGET = pea or ALL) ginan/Docs. Don't need to keep intermediaries in 'build'. +# Main outputs are ginan/bin and (if TARGET = pea or ALL) ginan/Docs. +# NOTE: while the cmake line appears to enable doc building, docs don't actually get written unless the +# make target on the following line is set to 'docs' or 'ALL'. RUN \ mkdir -p src/build \ && cd src/build \ - && cmake -DENABLE_OPTIMISATION=TRUE -DENABLE_PARALLELISATION=TRUE -DBUILD_DOC=TRUE .. \ - && make -j $BUILD_THREADS $TARGET \ + && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_DOC=TRUE .. \ + && make -j$BUILD_THREADS $TARGET \ && cd - \ && rm -rf src/build @@ -54,19 +59,61 @@ FROM scratch as ginan-build-cache COPY --from=ginan-build /root/.ccache /root/.ccache -# ------ Docker build target: minimal image with just PEA and its dependencies (TODO base on Ubuntu) ------ # -FROM gnssanalysis/ginan-env:latest as ginan-minimal -# TODO this should really be based on Ubuntu:24.04, with only those packages installed that are necessary to run pea -# and utilities. - -ENV PATH "/ginan/bin:${PATH}" +# ------ Docker build target: minimal image with just PEA and its dependencies ------ # +FROM ubuntu:24.04 as ginan-minimal + +# Was 1 thread, before being broken out into an argument +ARG BUILD_THREADS=1 + +# Install runtime dependencies, then clean up apt artifacts/caches to avoid saving them in the image +# Curl and gpg only required here to set up Mongo repo file. +RUN apt-get update -y \ + && apt-get upgrade -y --no-install-recommends \ + && apt-get install -y --no-install-recommends \ + curl \ + gpg \ + liblapacke-dev \ + libopenblas0 \ + libyaml-cpp0.8 \ + libncurses6 \ + libgomp1 \ + libmongoc-1.0-0 \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + + #TODO check which of these libraries we can remove, if we build PEA with the static version of them instead. + +# Install MongoDB 7, via third party repo, clean up apt caches +RUN curl -fsSL https://pgp.mongodb.com/server-7.0.asc | gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor \ + && echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list \ + && apt update -y \ + && apt-get install -y --no-install-recommends mongodb-org \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean -ADD scripts /ginan/scripts # Ensure our Python environment is installed so we can run gnssanalysis and relevant scripts. -# Somewhat redundant while image is based on ginan-env, but will still get updates to gnssanalysis applied quicker. COPY scripts/requirements.txt /tmp/requirements.txt WORKDIR /tmp/ RUN python3 -m pip install -r requirements.txt --no-cache-dir --break-system-packages -COPY --from=ginan-build /ginan/bin/ /ginan/bin +ADD scripts /ginan/scripts + +# Copy the built PEA binary from the build stage +COPY --from=ginan-build /ginan/bin/ /usr/bin/ +# TODO: we could update the build process to output to /usr/bin/ instead +# For backwards compatibility, put a link at /ginan/bin/pea pointing to /usr/bin/pea: +RUN mkdir /ginan/bin/ \ + && ln -s /usr/bin/pea /ginan/bin/pea \ + && chmod go-w /usr/bin/pea +# Permissions update probably not important (all processes have the same access level in the container) + +# Copy Mongocxx driver (including symlinks) from what is effectively the pre-build stage (ginan-env.Dockerfile) +COPY --from=gnssanalysis/ginan-env:latest /opt/mongocxx/lib/ /usr/local/lib/ +# Alternatively, if we built mongocxx at the pea build stage above: +# COPY --from=ginan-build /opt/mongocxx/lib/ /usr/local/lib/ + +# Render yaml of PEA's default parameter values, for reference. Note: -Y 3 is used in pipeline. +# This doubles as a check that pea loads successfully. +RUN pea -Y 3 > /ginan/pea-defaults.yaml \ No newline at end of file diff --git a/docker/ginan-env.Dockerfile b/docker/ginan-env.Dockerfile index 7c78ded96..8e3a9f3cc 100644 --- a/docker/ginan-env.Dockerfile +++ b/docker/ginan-env.Dockerfile @@ -1,4 +1,7 @@ -FROM ubuntu:24.04 +FROM ubuntu:24.04 as ginan-env +# These are set for Bitbucket pipelines memory ceiling. You can set higher on a development machine if you have the memory for it. +ARG BUILD_THREADS_DOXYGEN=4 +ARG BUILD_THREADS_GENERAL=2 ARG DEBIAN_FRONTEND=noninteractive ENV HOME /root @@ -6,6 +9,7 @@ ENV HOME /root RUN apt-get update -y \ && apt-get upgrade -y --no-install-recommends \ && apt-get install -y --no-install-recommends \ + ca-certificates \ git \ gcc \ g++ \ @@ -19,100 +23,97 @@ RUN apt-get update -y \ make \ gzip \ vim \ + libboost1.83-all-dev \ + libeigen3-dev \ libopenblas-dev \ liblapack-dev \ + liblapacke-dev \ libssl-dev \ + libmongoc-1.0-0 \ libnetcdf-dev \ libnetcdf-c++4-dev \ libncurses-dev \ libzstd-dev \ libssl-dev \ libgomp1 \ - python3-pip \ - python3-dev \ + libyaml-cpp-dev \ ssh \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean +# As of Dec 2024, Ubuntu 24.04 has: +# - libeigen3-dev: version 3.4.0-4 +# - boost 1.74 and 1.83 (we previously built version 1.75 from source) +# Note: ca-certificates is a dependency of python3-pip (so would commonly be installed as a result of that). +# It is needed to validate github's cert. + RUN ulimit -c unlimited RUN mkdir -p /tmp/build WORKDIR /tmp/build -COPY scripts/requirements.txt /tmp/build/requirements.txt -RUN python3 -m pip install -r requirements.txt --no-cache-dir --break-system-packages - ENV CFLAGS="-fno-omit-frame-pointer" ENV CXXFLAGS="-fno-omit-frame-pointer" ENV CMAKE_CXX_STANDARD="20" -RUN git clone --depth 1 --branch 0.8.0 https://github.com/jbeder/yaml-cpp.git \ - && mkdir -p yaml-cpp/cmake-build \ - && cd yaml-cpp/cmake-build \ - && cmake .. -DCMAKE\_INSTALL\_PREFIX=/usr/local/ -DYAML\_CPP\_BUILD\_TESTS=OFF \ - && make install yaml-cpp \ - && cd - \ - && rm -rf yaml-cpp - -RUN git clone --depth 1 --branch 3.4.0 https://gitlab.com/libeigen/eigen.git \ - && mkdir -p eigen/cmake-build \ - && cd eigen/cmake-build \ - && cmake .. \ - && make install \ - && cd - \ - && rm -rf eigen -RUN git clone --depth 1 --branch 1.26.1 https://github.com/mongodb/mongo-c-driver.git \ - && mkdir -p mongo-c-driver/cmake-build \ - && cd mongo-c-driver/cmake-build \ - && cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_EXTRA_ALIGNMENT=OFF .. \ - && cmake --build . \ - && cmake --build . --target install \ - && cd - \ - && rm -rf mongo-c-driver +# We currently avoid building this (installing via apt above). But the tradeoff is being +# slightly behind. The latest on Ubuntu 24.04 is currently from Feb (1.26.0). +# RUN git clone --depth 1 --branch 1.26.1 https://github.com/mongodb/mongo-c-driver.git \ +# && mkdir -p mongo-c-driver/cmake-build \ +# && cd mongo-c-driver/cmake-build \ +# && cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_EXTRA_ALIGNMENT=OFF .. \ +# && cmake --build . -j${BUILD_THREADS_GENERAL} \ +# && cmake --build . --target install -j${BUILD_THREADS_GENERAL} \ +# && cd - \ +# && rm -rf mongo-c-driver +# Amazingly, this is not available as a built library. Our only realistic option is to build it +# See here: https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/get-started/download-and-install/ RUN git clone --depth 1 --branch r3.10.1 https://github.com/mongodb/mongo-cxx-driver.git \ && mkdir -p mongo-cxx-driver/cmake-build \ && cd mongo-cxx-driver/cmake-build \ && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \ - && cmake --build . \ - && cmake --build . --target install \ + && cmake --build . -j${BUILD_THREADS_GENERAL} \ + && cmake --build . --target install -j${BUILD_THREADS_GENERAL} \ && cd - \ && rm -rf mongo-cxx-driver -ARG BOOST_VER=1.75.0 -RUN BOOST_NAME=$(echo boost_${BOOST_VER} | tr . _); \ - wget -c --no-verbose https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VER}/source/${BOOST_NAME}.tar.gz \ - && tar xf ${BOOST_NAME}.tar.gz \ - && rm ${BOOST_NAME}.tar.gz \ - && cd ${BOOST_NAME}/ \ - && ./bootstrap.sh \ - && ./b2 -j6 install \ - && cd - \ - && rm -rf ${BOOST_NAME}/ +# Copy the built MongoCXX driver library into a dedicated directory from which it can be picked up +# by the Ginan minimal image build. Also copy libbsoncxx, a dependency(?) of mongocxx. +# We use the following somewhat convoluted command because: +# - The Dockerfile COPY directive can't filter based on a glob +# - The default shell is sh, so we can't use shopt -s dotglob +RUN mkdir -p /opt/mongocxx/lib/ \ + && cd /usr/local/lib \ + && find -mindepth 1 -maxdepth 1 -name "libmongocxx*" -exec cp --parents -t /opt/mongocxx/lib/ {} + \ + && find -mindepth 1 -maxdepth 1 -name "libbsoncxx*" -exec cp --parents -t /opt/mongocxx/lib/ {} + + +# TODO: We're making the assumption we don't need the cmake (ie /usr/local/lib/cmake/mongocxx*) and pkgconfig +# subdirs, but that assumption may be wrong. +# Leaving out -maxdepth 1 could also allow us to grab other relevant files from their respective dirs, +# and keep dir structure. But it might not find everything... + # Note there should be no actual data left in this directory apart from requirements.txt. # This is a cosmetic rather than space saving cleanup step. - RUN rm -rf /tmp/build # Install a forked version of doxygen that has some nice features made just for ginan docs # To avoid an extra container image layer, we also copy the patch to the parent directory not the doxygen working dir - +# TODO: revist where to install this fork from, and whether we need it here. RUN apt-get update -y \ && apt-get install -y --no-install-recommends flex bison \ && git clone --depth 1 --branch customColors https://github.com/polytopes-design/doxygen.git \ && mkdir -p doxygen/build \ && cd doxygen/build \ && cmake -G "Unix Makefiles" .. \ - && make -j4 \ - && make install \ + && make -j${BUILD_THREADS_DOXYGEN} \ + && make -j${BUILD_THREADS_GENERAL} install \ && cd - \ && rm -rf doxygen \ && apt-get remove -y flex bison \ - && apt-get autoremove -y m4 \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean # For clean up, we uninstall flex and bison (apparently build-only dependencies for doxygen). -# We also remove m4 (required by bison/flex) but do this conditionally on whether anything else still needs it. -# Finally, we delete the apt package list, and cached packages. +# Finally, we delete the apt package list, and cached packages. \ No newline at end of file diff --git a/docker/tags b/docker/tags deleted file mode 100644 index 45f958d00..000000000 --- a/docker/tags +++ /dev/null @@ -1,5 +0,0 @@ - -PEA="7b1de342" -POD="a03f60a" -PEAPOD="fa2b375" -OTHER="817ccee" diff --git a/exampleConfigs/LEO_dynPOD.yaml b/exampleConfigs/LEO_dynPOD.yaml new file mode 100644 index 000000000..126cd8041 --- /dev/null +++ b/exampleConfigs/LEO_dynPOD.yaml @@ -0,0 +1,340 @@ +# Yaml config file for Reduced-dynamic POD of GRACE-FO C satellite; Date: 2023 12 01 +inputs: + inputs_root: ./products/ + atx_files: [igs20.atx] # Antenna models for receivers and satellites in ANTEX format + erp_files: [tables/finals.data.iau2000.txt] + egm_files: [tables/EGM2008.gfc] # Earth gravity model coefficients file + planetary_ephemeris_files: [tables/DE436.1950.2050] # JPL planetary and lunar ephemerides file + hfeop_files: [tables/desai_model_jgrb51665-sup-0002-ds01.txt] + tides: + atmos_tide_potential_files: [tables/atmosTide_AOD1bRL06.potential.iers.txt] + ocean_tide_potential_files: [tables/fes2014b_Cnm-Snm.dat] + ocean_pole_tide_potential_files: [tables/oceanPoleTide_desai2004.txt] + atmos_ocean_dealiasing_files: + - LEO/AOD1B_2019-02-18_X_06.asc + - LEO/AOD1B_2019-02-19_X_06.asc + - LEO/AOD1B_2019-02-20_X_06.asc + + snx_files: + # - "*.SNX" # use a wild card to include all files matching the description in the directory + - tables/igs_satellite_metadata_2203_plus.snx + #- tables/sat_yaw_bias_rate.snx + #- tables/qzss_yaw_modes.snx + #- tables/bds_yaw_modes.snx + - LEO.snx + + satellite_data: + sp3_files: [LEO/2019_02/COD0R03FIN_20190450000_01D_05M_ORB.SP3] # satellite orbit files in SP3 format + clk_files: [LEO/2019_02/COD0R03FIN_20190450000_01D_05S_CLK.CLK] # satellite clock files in RNX CLK format + bsx_files: [LEO/2019_02/COD0R03FIN_20190450000_01D_01D_OSB.BIA] # daily signal biases files + obx_files: [LEO/2019_02/SCA1B_2019-02-14_C_04.OBX] + + gnss_observations: + gnss_observations_root: LEO/2019_02/ + rnx_inputs: + - L64: + - "GPS1B_2019-02-14_C_04.rnx" + +outputs: + outputs_root: ./outputs/ + + trace: + level: 5 + output_receivers: true + output_satellites: true + output_network: true + receiver_filename: _.Reciever + satellite_filename: _.Sat + network_filename: _.TRACE + output_residuals: true + output_residual_chain: true + output_config: true + + metadata: + config_description: dynPOD_v3_20190214_LEO1 + analysis_agency: GAA + analysis_centre: Geoscience Australia -----FILE NOT FOR OPERATIONAL USE----- + analysis_software: Ginan v3 + rinex_comment: AUSNETWORK1 + #gradient_mapping_function: Chen & Herring, 1992 # (string) Name of mapping function used for mapping horizontal troposphere gradients + ocean_tide_loading_model: FES2004 # (string) Ocean tide loading model applied + reference_system: igs20 # (string) Terrestrial Reference System Code + time_system: G # (string) Time system - e.g. "G", "UTC" + + network_statistics: + output: true # (bool) Enable exporting network statistics data to file + directory: ./ # (string) Directory to export network statistics data + filename: _network_statistics.json # (string) Network statistics data filename + + sp3: + output: true # (bool) Enable exporting SP3 data to file + directory: ./ # (string) Directory to export SP3 data + # filename: _.sp3 # (string) SP3 data filename +satellite_options: + global: + antenna_boresight: [0, 0, +1] + antenna_azimuth: [0, +1, 0] + L64: + orbit_propagation: + solar_radiation_pressure: NONE + antenna_thrust: false + albedo: NONE + empirical: true + planetary_perturbations: + [ + moon, + sun, + mercury, + venus, + mars, + jupiter, + saturn, + uranus, + neptune, + pluto, + ] + + mass: 601.214 + area: 0.9551567 + srp_cr: 1.25 + +receiver_options: # Options to configure individual stations or global configs + global: + rec_reference_system: gps + models: + eop: + enable: true + L64: + antenna_type: "ANTTYPE" + receiver_type: "TRIG" + models: + eccentricity: + enable: true + offset: [0.2602, -0.0013, -0.4862] # ENU + + phase_windup: + enable: true + relativity2: + enable: true + relativity: + enable: true + sagnac: + enable: true + tides: + enable: false + attitude: + enable: true # (bool) Enables non-nominal attitude types + sources: [PRECISE, MODEL, NOMINAL] # List of sourecs to use for attitudes + troposphere: + enable: false + pco: + enable: true # (bool) Enable modelling of phase center offsets + eop: + enable: true + ionospheric_components: + enable: true + antenna_boresight: [0, 0, -1] + antenna_azimuth: [+1, 0, 0] + sat_id: "L64" + rinex2: + rnx_code_conversions: + P1: l1w + P2: l2w + rnx_phase_conversions: + L1: l1w + L2: l2w + + error_model: elevation_dependent # uniform, elevation_dependent + elevation_mask: 1 + code_sigma: 0.2 + phase_sigma: 0.002 # F0, F1, F2, F5, F6, F7, F8 + clock_codes: [l1w, l2w] + +processing_options: + orbit_propagation: + central_force: true + indirect_J2: true + egm_field: true + solid_earth_tide: true + ocean_tide: true + atm_tide: true + aod: true + general_relativity: true + pole_tide_ocean: true + pole_tide_solid: true + + egm_degree: 120 + integrator_time_step: 10 + process_modes: + preprocessor: true + spp: true + ppp: true + ionosphere: false # Compute Ionosphere models based on GNSS measurements + slr: false # Process SLR observations + + epoch_control: + epoch_interval: 10 #seconds + wait_next_epoch: 3600 # Wait up to an hour for next data point - When processing RINEX causes PEA to wait a long as need for last epoch to be processed. + max_rec_latency: 10 + #max_epochs: 50 + #start_epoch: 2019-02-14 00:00:00 + #end_epoch: 2019-02-14 00:59:59 + + gnss_general: + minimise_sat_clock_offsets: + enable: false + sys_options: + gps: + process: true + ambiguity_resolution: true + reject_eclipse: false + code_priorities: [L1W, L2W] + #network_amb_pivot: false # Constrain: set of ambiguities, to eliminate network rank deficiencies + #receiver_amb_pivot: false # Constrain: set of ambiguities, to eliminate receiver rank deficiencies + + leo: + process: true + + model_error_handling: + meas_deweighting: # Measurements that are outside the expected confidence bounds may be deweighted so that outliers do not contaminate the filtered solution + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all rejected measurement + state_deweighting: # Any "state" errors cause deweighting of all measurements that reference the state + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all referencing measurements + ambiguities: + phase_reject_limit: 2 # Reset ambiguity after 2 large fractional residuals are found (replaces phase_reject_count:) + reset_on: # Reset ambiguities when slip is detected by the following + gf: true # GF test + lli: false # LLI test + mw: true # MW test + scdia: true # SCDIA test + exclusions: + gf: true # (bool) Exclude measurements that fail GF slip test in preprocessor + lli: false # (bool) Exclude measurements that fail LLI slip test in preprocessor + mw: true # (bool) Exclude measurements that fail MW slip test in preprocessor + scdia: true # (bool) Exclude measurements that fail SCDIA test in preprocessor + + preprocessor: # Configurations for the kalman filter and its sub processes + cycle_slips: # Cycle slips may be detected by the preprocessor and measurements rejected or ambiguities reinitialised + mw_process_noise: 0 # Process noise applied to filtered Melbourne-Wubenna measurements to detect cycle slips + slip_threshold: 0.5 # Value used to determine when a slip has occurred + preprocess_all_data: true + + spp: + always_reinitialise: false # Reset SPP state to zero to avoid potential for lock-in of bad states + max_lsq_iterations: 12 # Maximum number of iterations of least squares allowed for convergence + outlier_screening: + raim: + enable: true # Enable Receiver Autonomous Integrity Monitoring + max_gdop: 30 # Maximum dilution of precision before error is flagged + + ambiguity_resolution: # Warning: lambda_bie is NOT recommended for network processing, use bootst option instead + elevation_mask: 15 + lambda_set_size: 200 + mode: off + success_rate_threshold: 0.99 + solution_ratio_threshold: 30 + fix_and_hold: true + # once_per_epoch: false + + ppp_filter: + outlier_screening: + chi_square: + enable: false # Enable Chi-square test + prefit: + max_iterations: 3 # Maximum number of measurements to exclude using prefit checks before attempting to filter + omega_test: false # Enable omega-test + sigma_check: true # Enable sigma check + state_sigma_threshold: 4 # Sigma threshold + meas_sigma_threshold: 4 # Sigma threshold + postfit: + max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter + sigma_check: true # Enable sigma check + state_sigma_threshold: 4 # Sigma threshold + meas_sigma_threshold: 4 # Sigma threshold + ionospheric_components: # Slant ionospheric components + common_ionosphere: true # Use the same ionosphere state for code and phase observations + use_gf_combo: false # Combine 'uncombined' measurements to simulate a geometry-free solution + use_if_combo: true # Combine 'uncombined' measurements to simulate an ionosphere-free solution + corr_mode: iono_free_linear_combo + + chunking: + by_receiver: false # Split large filter and measurement matrices blockwise by receiver ID to improve processing speed + by_satellite: false # Split large filter and measurement matrices blockwise by satellite ID to improve processing speed + size: 0 + + rts: # Rauch-Tung-Striebel (RTS) backwards smoothing + enable: true + lag: -1 + filename: _.rts + +estimation_parameters: + receivers: + L64: + orbit: + estimated: [true] + sigma: [1, 1, 1, 0.1, 0.1, 0.1] + process_noise: [0] + apriori_value: + [ + 5618061.478891041, + 2452804.124419954, + -3139258.743281315, + -3239.732805501764, + -1244.60090822831, + -6762.166783502897, + ] + + emp_r_0: + estimated: [true] + sigma: [50] + apriori_value: [0] + process_noise: [20] + + emp_t_0: + estimated: [true] + sigma: [50] + apriori_value: [0] + process_noise: [20] + + emp_n_0: + estimated: [true] + sigma: [50] + apriori_value: [0] + process_noise: [20] + + clock: + estimated: [true] + sigma: [500] + process_noise: [500] + + ambiguities: + estimated: [true] + sigma: [60] + process_noise: [0] + outage_limit: [50] + + ion_stec: + estimated: [true] + sigma: [1000] + process_noise: [100] + +mongo: # Mongo is a database used to store results and intermediate values for later analysis and inter-process communication + enable: primary # Enable and connect to mongo database {none,primary,secondary,both} + primary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to + primary_database: + primary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison + # secondary_database: + # secondary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison + # secondary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to + # output_config: NONE # Output config {none,primary,secondary,both} + output_components: primary # Output components of measurements {none,primary,secondary,both} + output_states: primary # Output states {none,primary,secondary,both} + output_measurements: primary # Output measurements and their residuals {none,primary,secondary,both} + output_test_stats: primary # Output test statistics {none,primary,secondary,both} + delete_history: primary # Drop the collection in the database at the beginning of the run to only show fresh data {none,primary,secondary,both} + output_trace: primary + +debug: + # output_mincon: true + # mincon_only: true diff --git a/exampleConfigs/LEO_kinPOD.yaml b/exampleConfigs/LEO_kinPOD.yaml new file mode 100644 index 000000000..6b8f01392 --- /dev/null +++ b/exampleConfigs/LEO_kinPOD.yaml @@ -0,0 +1,328 @@ +# Yaml config file for Reduced-dynamic POD of GRACE-FO C satellite; Date: 2023 12 01 +inputs: + inputs_root: ./products/ + atx_files: [igs20.atx] # Antenna models for receivers and satellites in ANTEX format + erp_files: [tables/finals.data.iau2000.txt] + egm_files: [tables/EGM2008.gfc] # Earth gravity model coefficients file + planetary_ephemeris_files: [tables/DE436.1950.2050] # JPL planetary and lunar ephemerides file + hfeop_files: [tables/desai_model_jgrb51665-sup-0002-ds01.txt] + tides: + #ocean_tide_loading_blq_files: [ OLOAD_GO.BLQ ] # required if ocean loading is applied + #atmos_tide_loading_blq_files: [ ALOAD_GO.BLQ ] # required if atmospheric tide loading is applied + #ocean_pole_tide_loading_files: [ tables/opoleloadcoefcmcor.txt ] # required if ocean pole tide loading is applied + atmos_tide_potential_files: [tables/atmosTide_AOD1bRL06.potential.iers.txt] + ocean_tide_potential_files: [tables/fes2014b_Cnm-Snm.dat] + ocean_pole_tide_potential_files: [tables/oceanPoleTide_desai2004.txt] + + snx_files: + # - "*.SNX" # use a wild card to include all files matching the description in the directory + - tables/igs_satellite_metadata_2203_plus.snx + #- tables/sat_yaw_bias_rate.snx + #- tables/qzss_yaw_modes.snx + #- tables/bds_yaw_modes.snx + - LEO.snx + + satellite_data: + sp3_files: [LEO/2019_02/COD0R03FIN_20190450000_01D_05M_ORB.SP3] # satellite orbit files in SP3 format + clk_files: [LEO/2019_02/COD0R03FIN_20190450000_01D_05S_CLK.CLK] # satellite clock files in RNX CLK format + bsx_files: [LEO/2019_02/COD0R03FIN_20190450000_01D_01D_OSB.BIA] # daily signal biases files + obx_files: [LEO/2019_02/SCA1B_2019-02-14_C_04.OBX] + + gnss_observations: + gnss_observations_root: LEO/2019_02/ + rnx_inputs: + - L64: + - "GPS1B_2019-02-14_C_04.rnx" + +outputs: + outputs_root: ./outputs/ + + trace: + level: 5 + output_receivers: true + output_satellites: true + output_network: true + receiver_filename: _.Reciever + satellite_filename: _.Sat + network_filename: _.TRACE + output_residuals: true + output_residual_chain: true + output_config: true + + metadata: + config_description: kinPOD_v3_20190214_LEO1 + analysis_agency: GAA + analysis_centre: Geoscience Australia -----FILE NOT FOR OPERATIONAL USE----- + analysis_software: Ginan + rinex_comment: AUSNETWORK1 + #gradient_mapping_function: Chen & Herring, 1992 # (string) Name of mapping function used for mapping horizontal troposphere gradients + ocean_tide_loading_model: FES2004 # (string) Ocean tide loading model applied + reference_system: igs20 # (string) Terrestrial Reference System Code + time_system: G # (string) Time system - e.g. "G", "UTC" + + network_statistics: + output: true # (bool) Enable exporting network statistics data to file + directory: ./ # (string) Directory to export network statistics data + filename: _network_statistics.json # (string) Network statistics data filename + + pos: + output: true # (bool) Enable exporting position data to file + directory: ./ # (string) Directory to export position data + # filename: _pos.csv # (string) Position data filename + +satellite_options: + global: + antenna_boresight: [0, 0, +1] + antenna_azimuth: [0, +1, 0] + # L64: + # orbit_propagation: + # solar_radiation_pressure: NONE + # drag: false + # antenna_thrust: false + # albedo: NONE + # empirical: true #true/false => false/ecom/srf + # planetary_perturbations: + # [ + # moon, + # sun, + # mercury, + # venus, + # mars, + # jupiter, + # saturn, + # uranus, + # neptune, + # pluto, + # ] + + # mass: 601.214 + # area: 0.9551567 + # srp_cr: 1.25 + # drag_cd: 2.2 + +receiver_options: # Options to configure individual stations or global configs + global: + rec_reference_system: GPS + models: + eop: + enable: true + L64: + antenna_type: "ANTTYPE" + receiver_type: "TRIG" + models: + eccentricity: + enable: true + offset: [0.2602, -0.0013, -0.4862] # ENU + + phase_windup: + enable: true + relativity2: + enable: true + relativity: + enable: true + sagnac: + enable: true + tides: + enable: false + attitude: + enable: true # (bool) Enables non-nominal attitude types + sources: [PRECISE, MODEL, NOMINAL] #[PRECISE, MODEL, NOMINAL] # List of sourecs to use for attitudes + troposphere: + enable: false + pco: + enable: true # (bool) Enable modelling of phase center offsets + eop: + enable: true + ionospheric_components: + enable: true + antenna_boresight: [0, 0, -1] + antenna_azimuth: [+1, 0, 0] + sat_id: "L64" + rinex2: + rnx_code_conversions: + P1: l1w + P2: l2w + rnx_phase_conversions: + L1: l1w + L2: l2w + + error_model: elevation_dependent # uniform, elevation_dependent + elevation_mask: 1 + code_sigma: 0.2 + phase_sigma: 0.002 # F0, F1, F2, F5, F6, F7, F8 + clock_codes: [l1w, l2w] + +processing_options: + # orbit_propagation: + # central_force: true + # indirect_J2: true + # egm_field: true + # solid_earth_tide: true + # ocean_tide: true + # atm_tide: true + # aod: false + # general_relativity: true + # pole_tide_ocean: true + # pole_tide_solid: true + + # egm_degree: 120 + # integrator_time_step: 10 + process_modes: + preprocessor: true + spp: true + ppp: true + ionosphere: false # Compute Ionosphere models based on GNSS measurements + slr: false # Process SLR observations + + epoch_control: + epoch_interval: 10 #seconds + wait_next_epoch: 3600 # Wait up to an hour for next data point - When processing RINEX causes PEA to wait a long as need for last epoch to be processed. + max_rec_latency: 10 + #max_epochs: 50 + #start_epoch: 2019-02-14 00:00:00 + #end_epoch: 2019-02-14 00:59:59 + + gnss_general: + minimise_sat_clock_offsets: + enable: false + sys_options: + gps: + process: true + ambiguity_resolution: true + reject_eclipse: false + code_priorities: [L1W, L2W] + #network_amb_pivot: false # Constrain: set of ambiguities, to eliminate network rank deficiencies + #receiver_amb_pivot: false # Constrain: set of ambiguities, to eliminate receiver rank deficiencies + + leo: + process: true + + model_error_handling: + meas_deweighting: # Measurements that are outside the expected confidence bounds may be deweighted so that outliers do not contaminate the filtered solution + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all rejected measurement + state_deweighting: # Any "state" errors cause deweighting of all measurements that reference the state + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all referencing measurements + ambiguities: + phase_reject_limit: 2 # Reset ambiguity after 2 large fractional residuals are found (replaces phase_reject_count:) + reset_on: # Reset ambiguities when slip is detected by the following + gf: true # GF test + lli: false # LLI test + mw: true # MW test + scdia: true # SCDIA test + exclusions: + gf: true # (bool) Exclude measurements that fail GF slip test in preprocessor + lli: false # (bool) Exclude measurements that fail LLI slip test in preprocessor + mw: true # (bool) Exclude measurements that fail MW slip test in preprocessor + scdia: true # (bool) Exclude measurements that fail SCDIA test in preprocessor + + preprocessor: # Configurations for the kalman filter and its sub processes + cycle_slips: # Cycle slips may be detected by the preprocessor and measurements rejected or ambiguities reinitialised + mw_process_noise: 0 # Process noise applied to filtered Melbourne-Wubenna measurements to detect cycle slips + slip_threshold: 0.5 # Value used to determine when a slip has occurred + preprocess_all_data: true + + spp: + always_reinitialise: false # Reset SPP state to zero to avoid potential for lock-in of bad states + max_lsq_iterations: 12 # Maximum number of iterations of least squares allowed for convergence + outlier_screening: + raim: + enable: true # Enable Receiver Autonomous Integrity Monitoring + max_gdop: 30 # Maximum dilution of precision before error is flagged + + ambiguity_resolution: # Warning: lambda_bie is NOT recommended for network processing, use bootst option instead + elevation_mask: 15 + lambda_set_size: 200 + mode: off + success_rate_threshold: 0.99 + solution_ratio_threshold: 30 + fix_and_hold: true + # once_per_epoch: false + + ppp_filter: + outlier_screening: + chi_square: + enable: false # Enable Chi-square test + mode: INNOVATION # Chi-square test mode {none,innovation,measurement,state} + prefit: + max_iterations: 3 # Maximum number of measurements to exclude using prefit checks before attempting to filter + omega_test: false # Enable omega-test + sigma_check: true # Enable sigma check + state_sigma_threshold: 4 # Sigma threshold + meas_sigma_threshold: 4 # Sigma threshold + + postfit: + max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter + sigma_check: true # Enable sigma check + state_sigma_threshold: 4 # Sigma threshold + meas_sigma_threshold: 4 # Sigma threshold + + ionospheric_components: # Slant ionospheric components + common_ionosphere: true # Use the same ionosphere state for code and phase observations + use_gf_combo: false # Combine 'uncombined' measurements to simulate a geometry-free solution + use_if_combo: true # Combine 'uncombined' measurements to simulate an ionosphere-free solution + corr_mode: iono_free_linear_combo + + chunking: + by_receiver: false # Split large filter and measurement matrices blockwise by receiver ID to improve processing speed + by_satellite: false # Split large filter and measurement matrices blockwise by satellite ID to improve processing speed + size: 0 + + rts: # Rauch-Tung-Striebel (RTS) backwards smoothing + enable: true + lag: -1 + filename: _.rts + +estimation_parameters: + receivers: + L64: + pos: + estimated: [true] + sigma: [30] + pos_rate: + estimated: [true] + sigma: [5000] + process_noise: [1000] + + clock: + estimated: [true] + sigma: [500] + process_noise: [500] # [100] + # apriori_values: [60] + + ambiguities: + estimated: [true] + sigma: [60] + process_noise: [0] + outage_limit: [50] + # process_noise_dt: day + + ion_stec: + estimated: [true] + sigma: [1000] + process_noise: [100] + #process_noise_dt: second + #apriori_value: [0] + #comment: [""] + #mu: [0] + #tau: [-1] + +mongo: # Mongo is a database used to store results and intermediate values for later analysis and inter-process communication + enable: primary # Enable and connect to mongo database {none,primary,secondary,both} + primary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to + primary_database: + primary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison + # secondary_database: + # secondary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison + # secondary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to + # output_config: NONE # Output config {none,primary,secondary,both} + output_components: primary # Output components of measurements {none,primary,secondary,both} + output_states: primary # Output states {none,primary,secondary,both} + output_measurements: primary # Output measurements and their residuals {none,primary,secondary,both} + output_test_stats: primary # Output test statistics {none,primary,secondary,both} + delete_history: primary # Drop the collection in the database at the beginning of the run to only show fresh data {none,primary,secondary,both} + output_trace: primary + +debug: + # output_mincon: true + # mincon_only: true diff --git a/exampleConfigs/brdc2sp3.yml b/exampleConfigs/brdc2sp3.yml index c36579f1d..c3b17458d 100644 --- a/exampleConfigs/brdc2sp3.yml +++ b/exampleConfigs/brdc2sp3.yml @@ -1,70 +1,61 @@ inputs: + inputs_root: products/ - inputs_root: products/ - - atx_files: # Antenna models for receivers and satellites in ANTEX format + atx_files: # Antenna models for receivers and satellites in ANTEX format - igs14_2045_plus.atx - snx_files: # SINEX files for meta data and initial position + snx_files: # SINEX files for meta data and initial position - igs19P2062.snx - tables/igs_satellite_metadata_2203_plus.snx - tables/sat_yaw_bias_rate.snx - tables/qzss_yaw_modes.snx - tables/bds_yaw_modes.snx - - satellite_data: - nav_files: # broadcast navigation files - - brdm1990.19p + satellite_data: + nav_files: # broadcast navigation files + - brdm1990.19p outputs: - - outputs_root: outputs// - - metadata: - config_description: brdc2sp3 - analysis_agency: GAA - analysis_centre: Geoscience Australia - analysis_program: AUSACS - rinex_comment: AUSNETWORK1 - - sp3: - output: true - filename: --.sp3 - output_interval: 900 - output_velocities: true - # output_inertial: true - + outputs_root: outputs// + + metadata: + config_description: brdc2sp3 + analysis_agency: GAA + analysis_centre: Geoscience Australia + analysis_program: AUSACS + rinex_comment: AUSNETWORK1 + + sp3: + output: true + filename: --.sp3 + output_interval: 900 + output_velocities: true + # output_inertial: true satellite_options: - - global: - models: - attitude: - enable: true - sources: [ MODEL ] - + global: + models: + attitude: + enable: true + sources: [MODEL] receiver_options: - - global: - elevation_mask: 7 # degrees - + global: + elevation_mask: 7 # degrees processing_options: - - process_modes: - preprocessor: false - - epoch_control: - start_epoch: 2019-07-18 00:00:00 - end_epoch: 2019-07-18 23:45:00 - epoch_interval: 900 # seconds - require_obs: false - - gnss_general: - sys_options: - gps: { process: true } - gal: { process: true } - glo: { process: true } - bds: { process: true } - qzs: { process: true } \ No newline at end of file + process_modes: + preprocessor: false + + epoch_control: + start_epoch: 2019-07-18 00:00:00 + end_epoch: 2019-07-18 23:45:00 + epoch_interval: 900 # seconds + require_obs: false + + gnss_general: + sys_options: + gps: { process: true } + gal: { process: true } + glo: { process: true } + bds: { process: true } + qzs: { process: true } diff --git a/exampleConfigs/compare_orbits.yaml b/exampleConfigs/compare_orbits.yaml index 46d26d110..99fbc44bc 100644 --- a/exampleConfigs/compare_orbits.yaml +++ b/exampleConfigs/compare_orbits.yaml @@ -1,53 +1,53 @@ - debug: - compare_orbits: true - + compare_orbits: true inputs: - inputs_root: ./products + inputs_root: ./products - satellite_data: - sp3_files: - - gag20624.sp3 - - igs20624.sp3 + satellite_data: + sp3_files: + - IGS2R03FIN_20191990000_01D_05M_ORB.SP3 + - TUG0R03FIN_20191990000_01D_05M_ORB.SP3 outputs: - outputs_root: ./outputs/compare_orbits - - trace: - level: 3 - output_config: true - output_network: true - output_residuals: true - network_filename: compare_orbits.TRACE + outputs_root: ./outputs/compare_orbits + trace: + level: 3 + output_config: true + output_network: true + output_residuals: true + network_filename: compare_orbits.TRACE satellite_options: - global: - mincon_scale_apriori_sigma: 0.01 - + global: + exclude: true + mincon_scale_apriori_sigma: 1 + GPS: + exclude: false processing_options: - minimum_constraints: - - enable: true - - translation: { estimated: [true] } - rotation: { estimated: [true] } - # scale: { estimated: [true] } - delay: { estimated: [true] } - - outlier_screening: - chi_square: - enable: false # (bool) Enable Chi-square test - mode: none # (enum) Chi-square test mode - innovation, measurement, state {NONE,INNOVATION,MEASUREMENT,STATE} - prefit: - max_iterations: 3 # Maximum number of measurements to exclude using prefit checks before attempting to filter - sigma_check: true # (bool) Enable prefit sigma check - state_sigma_threshold: 3.0 # (float) sigma threshold for states - meas_sigma_threshold: 3.0 # (float) sigma threshold for measurements - postfit: - max_iterations: 20 # Maximum number of measurements to exclude using postfit checks while iterating filter - sigma_check: true # (bool) Enable postfit sigma check - state_sigma_threshold: 3.0 # (float) sigma threshold for states - meas_sigma_threshold: 3.0 # (float) sigma threshold for measurements + minimum_constraints: + enable: true + # once_per_epoch: true + translation: { estimated: [true], sigma: [10] } + rotation: { estimated: [true], sigma: [10] } + # translation_rate: { estimated: [true], sigma: [100]} + # rotation_rate: { estimated: [true], sigma: [100]} + # scale: { estimated: [true], sigma: [10]} + # delay: { estimated: [true], sigma: [10]} + + outlier_screening: + chi_square: + enable: false # (bool) Enable Chi-square test + mode: none # (enum) Chi-square test mode - innovation, measurement, state {NONE,INNOVATION,MEASUREMENT,STATE} + prefit: + max_iterations: 3 # Maximum number of measurements to exclude using prefit checks before attempting to filter + sigma_check: true # (bool) Enable prefit sigma check + state_sigma_threshold: 3.0 # (float) sigma threshold for states + meas_sigma_threshold: 3.0 # (float) sigma threshold for measurements + postfit: + max_iterations: 20 # Maximum number of measurements to exclude using postfit checks while iterating filter + sigma_check: true # (bool) Enable postfit sigma check + state_sigma_threshold: 3.0 # (float) sigma threshold for states + meas_sigma_threshold: 3.0 # (float) sigma threshold for measurements diff --git a/exampleConfigs/fit_sp3_pseudoobs.yaml b/exampleConfigs/fit_sp3_pseudoobs.yaml index 3ce1b31dc..01ad598fc 100644 --- a/exampleConfigs/fit_sp3_pseudoobs.yaml +++ b/exampleConfigs/fit_sp3_pseudoobs.yaml @@ -1,191 +1,203 @@ inputs: + include_yamls: [products/boxwing.yaml] # required if using boxwing model - include_yamls: [ products/boxwing.yaml ] # required if using boxwing model + inputs_root: ./products/ - inputs_root: ./products/ + atx_files: [igs20.atx] + erp_files: [igs96p02.erp] + egm_files: [tables/EGM2008.gfc] # Earth gravity model coefficients file + planetary_ephemeris_files: [tables/DE436.1950.2050] # JPL planetary and lunar ephemerides file - atx_files: [ igs20.atx ] - erp_files: [ igs96p02.erp ] - egm_files: [ tables/EGM2008.gfc ] # Earth gravity model coefficients file - planetary_ephemeris_files: [ tables/DE436.1950.2050 ] # JPL planetary and lunar ephemerides file + tides: + # ocean_tide_loading_blq_files: [ OLOAD_GO.BLQ ] + # atmos_tide_loading_blq_files: [ ALOAD_GO.BLQ ] + # ocean_pole_tide_loading_files: [ tables/opoleloadcoefcmcor.txt ] + ocean_tide_potential_files: [tables/fes2014b_Cnm-Snm.dat] - tides: - # ocean_tide_loading_blq_files: [ OLOAD_GO.BLQ ] - # atmos_tide_loading_blq_files: [ ALOAD_GO.BLQ ] - # ocean_pole_tide_loading_files: [ tables/opoleloadcoefcmcor.txt ] - ocean_tide_potential_files: [ tables/fes2014b_Cnm-Snm.dat ] - - snx_files: + snx_files: - IGS1R03SNX_20191950000_07D_07D_CRD.SNX - tables/igs_satellite_metadata_2203_plus.snx - tables/sat_yaw_bias_rate.snx - tables/qzss_yaw_modes.snx - tables/bds_yaw_modes.snx - - satellite_data: - nav_files: - - brdm1990.19p - sp3_files: - - IGS2R03FIN_20191980000_01D_05M_ORB.SP3 - - IGS2R03FIN_20191990000_01D_05M_ORB.SP3 - - IGS2R03FIN_20192000000_01D_05M_ORB.SP3 - - pseudo_observations: - eci_pseudoobs: false - sp3_inputs: - - IGS2R03FIN_20191980000_01D_05M_ORB.SP3 - - IGS2R03FIN_20191990000_01D_05M_ORB.SP3 - - IGS2R03FIN_20192000000_01D_05M_ORB.SP3 + satellite_data: + nav_files: + - brdm1990.19p + sp3_files: + - IGS2R03FIN_20191980000_01D_05M_ORB.SP3 + - IGS2R03FIN_20191990000_01D_05M_ORB.SP3 + - IGS2R03FIN_20192000000_01D_05M_ORB.SP3 + + pseudo_observations: + eci_pseudoobs: false + sp3_inputs: + - IGS2R03FIN_20191980000_01D_05M_ORB.SP3 + - IGS2R03FIN_20191990000_01D_05M_ORB.SP3 + - IGS2R03FIN_20192000000_01D_05M_ORB.SP3 outputs: - - outputs_root: outputs// - - metadata: - config_description: fit_sp3_pseudoobs - analysis_agency: GAA - analysis_centre: Geoscience Australia - analysis_program: Ginan - rinex_comment: AUSNETWORK1 - - trace: - output_receivers: true - output_network: true - level: 3 - directory: ./ - receiver_filename: _.TRACE - network_filename: _.TRACE - output_residuals: true - output_config: true - - log: - output: true - directory: ./ - filename: log_.json - - orbit_ics: - output: true - directory: ./orbit_ics - filename: __orbits.yaml - - sp3: - output: true - directory: ./ - filename: __.sp3 - output_interval: 300 - output_velocities: true - output_inertial: true - orbit_sources: [ KALMAN ] - clock_sources: [ PRECISE ] - - output_rotation: - period: 1 - period_units: day - - -satellite_options: # Options to configure individual satellites, systems, or global configs - - global: - pseudo_sigma: 1 - orbit_propagation: - albedo: cannonball - antenna_thrust: true - empirical: true - empirical_dyb_eclipse: [true, false, false] - planetary_perturbations: [moon,sun,mercury,venus,mars,jupiter,saturn,uranus,neptune,pluto] - pseudo_pulses: - enable: true - solar_radiation_pressure: boxwing - mass: 1000 - area: 15 - srp_cr: 1.75 - power: 20 - models: - pos: - enable: true - sources: [KALMAN, PRECISE, BROADCAST] - attitude: - enable: true # (bool) Enables non-nominal attitude types - sources: [ NOMINAL ] # List of sourecs to use for attitudes - + outputs_root: outputs// + + metadata: + config_description: fit_sp3_pseudoobs_ + analysis_agency: GAA + analysis_centre: Geoscience Australia + analysis_software: Ginan + rinex_comment: AUSNETWORK1 + + trace: + output_receivers: true + output_network: true + level: 3 + directory: ./ + receiver_filename: _.TRACE + network_filename: _.TRACE + output_residuals: true + output_config: true + + log: + output: true + directory: ./ + filename: log_.json + + orbit_ics: + output: true + directory: ./orbit_ics + filename: __orbits.yaml + + sp3: + output: true + directory: ./ + filename: __.sp3 + output_interval: 300 + output_velocities: true + output_inertial: true + orbit_sources: [KALMAN] + clock_sources: [PRECISE] + + output_rotation: + period: 1 + period_units: day + +satellite_options: # Options to configure individual satellites, systems, or global configs + global: + pseudo_sigma: 1 + orbit_propagation: + albedo: cannonball + antenna_thrust: true + empirical: true + empirical_dyb_eclipse: [true, false, false] + planetary_perturbations: + [ + moon, + sun, + mercury, + venus, + mars, + jupiter, + saturn, + uranus, + neptune, + pluto, + ] + pseudo_pulses: + enable: true + solar_radiation_pressure: boxwing + mass: 1000 + area: 15 + srp_cr: 1.75 + power: 20 + models: + pos: + enable: true + sources: [KALMAN, PRECISE, BROADCAST] + attitude: + enable: true # (bool) Enables non-nominal attitude types + sources: [NOMINAL] # List of sourecs to use for attitudes receiver_options: - - global: - models: - eop: - enable: true - + global: + models: + eop: + enable: true processing_options: - - epoch_control: - epoch_interval: 300 # seconds - - - process_modes: - ppp: true - preprocessor: false - spp: false - - gnss_general: - sys_options: - gps: - process: true - - orbit_propagation: - central_force: true - indirect_J2: true - egm_field: true - solid_earth_tide: true - ocean_tide: true - general_relativity: true - pole_tide_ocean: true - pole_tide_solid: true - integrator_time_step: 900 - egm_degree: 15 - - model_error_handling: - orbit_errors: - enable: true - pos_process_noise: 100 - vel_process_noise: 1 - vel_process_noise_trail: 1 - vel_process_noise_trail_tau: 900 - + epoch_control: + epoch_interval: 300 # seconds + + process_modes: + ppp: true + preprocessor: false + spp: false + + gnss_general: + sys_options: + gps: + process: true + + orbit_propagation: + central_force: true + indirect_J2: true + egm_field: true + solid_earth_tide: true + ocean_tide: true + general_relativity: true + pole_tide_ocean: true + pole_tide_solid: true + integrator_time_step: 900 + egm_degree: 15 + + ppp_filter: + outlier_screening: + prefit: + sigma_check: false + omega_test: false + postfit: + max_iterations: 10 + sigma_check: false + omega_test: true + + model_error_handling: + satellite_errors: + enable: true + pos_process_noise: 100 + vel_process_noise: 1 + vel_process_noise_trail: 1 + vel_process_noise_trail_tau: 900 estimation_parameters: + global_models: + eop: + estimated: [true] + sigma: [10] + eop_rates: + estimated: [true] + sigma: [10] + + satellites: + global: + orbit: + estimated: [true] + sigma: + [1] + #cr: + #estimated: [true] + #sigma: [1] + emp_d_0: { estimated: [true], sigma: [1e3] } + emp_y_0: { estimated: [true], sigma: [1e3] } + emp_b_0: { estimated: [true], sigma: [1e3] } - global_models: - eop: - estimated: [ true ] - sigma: [ 10 ] - eop_rates: - estimated: [ true ] - sigma: [ 10 ] - - satellites: - global: - orbit: - estimated: [true] - sigma: [1] - - emp_d_0: { estimated: [true], sigma: [1e3]} - emp_y_0: { estimated: [true], sigma: [1e3]} - emp_b_0: { estimated: [true], sigma: [1e3]} - - emp_d_1: { estimated: [true], sigma: [1e3]} - emp_b_1: { estimated: [true], sigma: [1e3]} - - emp_d_2: { estimated: [true], sigma: [1e2]} + emp_d_1: { estimated: [true], sigma: [1e3] } + emp_b_1: { estimated: [true], sigma: [1e3] } - emp_d_4: { estimated: [true], sigma: [1e3]} + emp_d_2: { estimated: [true], sigma: [1e2] } + emp_d_4: { estimated: [true], sigma: [1e3] } mongo: - enable: primary - primary_database: - output_measurements: primary - output_states: primary - delete_history: primary \ No newline at end of file + enable: primary + primary_database: + output_measurements: primary + output_states: primary + delete_history: primary diff --git a/exampleConfigs/pod_example.yaml b/exampleConfigs/pod_example.yaml index 94a6cce8e..820af8907 100644 --- a/exampleConfigs/pod_example.yaml +++ b/exampleConfigs/pod_example.yaml @@ -1,497 +1,513 @@ inputs: - include_yamls: [products/boxwing.yaml] # required if using boxwing model - - inputs_root: ./products/ - - atx_files: [igs20.atx] # required - egm_files: [tables/EGM2008.gfc] # Earth gravity model coefficients file - igrf_files: [tables/igrf13coeffs.txt] - erp_files: [finals.data.iau2000.txt] - planetary_ephemeris_files: [tables/DE436.1950.2050] - - troposphere: - gpt2grid_files: [tables/gpt_25.grd] - - tides: - ocean_tide_loading_blq_files: [tables/OLOAD_GO.BLQ] # required if ocean loading is applied - atmos_tide_loading_blq_files: [tables/ALOAD_GO.BLQ] # required if atmospheric tide loading is applied - ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] # required if ocean pole tide loading is applied - ocean_tide_potential_files: [tables/fes2014b_Cnm-Snm.dat] - - snx_files: - # - "*.SNX" # use a wild card to include all files matching the description in the directory - - tables/igs_satellite_metadata_2203_plus.snx - - tables/sat_yaw_bias_rate.snx - - tables/qzss_yaw_modes.snx - - tables/bds_yaw_modes.snx - - IGS1R03SNX_20191950000_07D_07D_CRD.SNX - - satellite_data: - nav_files: [brdm1990.19p] - clk_files: [IGS2R03FIN_20191990000_01D_30S_CLK.CLK] - - gnss_observations: - gnss_observations_root: ../data/ - rnx_inputs: - - AREG00PER_R_20191990000_01D_30S_MO.rnx - - ASCG00SHN_R_20191990000_01D_30S_MO.rnx - - CEDU00AUS_R_20191990000_01D_30S_MO.rnx - - COCO00AUS_R_20191990000_01D_30S_MO.rnx - - CPVG00CPV_R_20191990000_01D_30S_MO.rnx - - DARW00AUS_R_20191990000_01D_30S_MO.rnx - - DGAR00GBR_R_20191990000_01D_30S_MO.rnx - - DJIG00DJI_R_20191990000_01D_30S_MO.rnx - - FAIR00USA_R_20191990000_01D_30S_MO.rnx - - HERS00GBR_R_20191990000_01D_30S_MO.rnx - - HOB200AUS_R_20191990000_01D_30S_MO.rnx - - IISC00IND_R_20191990000_01D_30S_MO.rnx - - JFNG00CHN_R_20191990000_01D_30S_MO.rnx - - KARR00AUS_R_20191990000_01D_30S_MO.rnx - - KIRI00KIR_R_20191990000_01D_30S_MO.rnx - - KOKV00USA_R_20191990000_01D_30S_MO.rnx - - LHAZ00CHN_R_20191990000_01D_30S_MO.rnx - - LMMF00MTQ_R_20191990000_01D_30S_MO.rnx - - MAW100ATA_R_20191990000_01D_30S_MO.rnx - - MBAR00UGA_R_20191990000_01D_30S_MO.rnx - - METG00FIN_R_20191990000_01D_30S_MO.rnx - - MGUE00ARG_R_20191990000_01D_30S_MO.rnx - - NICO00CYP_R_20191990000_01D_30S_MO.rnx - - NKLG00GAB_R_20191990000_01D_30S_MO.rnx - - OHI300ATA_R_20191990000_01D_30S_MO.rnx - - POAL00BRA_R_20191990000_01D_30S_MO.rnx - - QUIN00USA_R_20191990000_01D_30S_MO.rnx - - REYK00ISL_R_20191990000_01D_30S_MO.rnx - - RGDG00ARG_R_20191990000_01D_30S_MO.rnx - - SAMO00WSM_R_20191990000_01D_30S_MO.rnx - - SEY200SYC_R_20191990000_01D_30S_MO.rnx - - SOLO00SLB_R_20191990000_01D_30S_MO.rnx - - TONG00TON_R_20191990000_01D_30S_MO.rnx - - TOPL00BRA_R_20191990000_01D_30S_MO.rnx - - TOW200AUS_R_20191990000_01D_30S_MO.rnx - - USN700USA_R_20191990000_01D_30S_MO.rnx - - VACS00MUS_R_20191990000_01D_30S_MO.rnx - - ZIM200CHE_R_20191990000_01D_30S_MO.rnx - - CUSV00THA_R_20191990000_01D_30S_MO.rnx + include_yamls: [products/boxwing.yaml] # required if using boxwing model + + inputs_root: ./products/ + + atx_files: [igs20.atx] # required + egm_files: [tables/EGM2008.gfc] # Earth gravity model coefficients file + igrf_files: [tables/igrf14coeffs.txt] + erp_files: [finals.data.iau2000.txt] + planetary_ephemeris_files: [tables/DE436.1950.2050] + + troposphere: + gpt2grid_files: [tables/gpt_25.grd] + + tides: + ocean_tide_loading_blq_files: [tables/OLOAD_GO.BLQ] # required if ocean loading is applied + atmos_tide_loading_blq_files: [tables/ALOAD_GO.BLQ] # required if atmospheric tide loading is applied + ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] # required if ocean pole tide loading is applied + ocean_tide_potential_files: [tables/fes2014b_Cnm-Snm.dat] + + snx_files: + # - "*.SNX" # use a wild card to include all files matching the description in the directory + - tables/igs_satellite_metadata_2203_plus.snx + - tables/sat_yaw_bias_rate.snx + - tables/qzss_yaw_modes.snx + - tables/bds_yaw_modes.snx + - IGS1R03SNX_20191950000_07D_07D_CRD.SNX + + satellite_data: + nav_files: [brdm1990.19p] + clk_files: [IGS2R03FIN_20191990000_01D_30S_CLK.CLK] + + pseudo_observations: # Use data from pre-processed data products as observations + eci_pseudoobs: false # Pseudo observations are provided in eci frame rather than standard ECEF SP3 files + sp3_inputs: [IGS2R03FIN_20191990000_01D_05M_ORB.SP3] + + gnss_observations: + gnss_observations_root: ../data/ + rnx_inputs: + - AREG00PER_R_20191990000_01D_30S_MO.rnx + - ASCG00SHN_R_20191990000_01D_30S_MO.rnx + - CEDU00AUS_R_20191990000_01D_30S_MO.rnx + - COCO00AUS_R_20191990000_01D_30S_MO.rnx + - CPVG00CPV_R_20191990000_01D_30S_MO.rnx + - DARW00AUS_R_20191990000_01D_30S_MO.rnx + - DGAR00GBR_R_20191990000_01D_30S_MO.rnx + - DJIG00DJI_R_20191990000_01D_30S_MO.rnx + - FAIR00USA_R_20191990000_01D_30S_MO.rnx + - HERS00GBR_R_20191990000_01D_30S_MO.rnx + - HOB200AUS_R_20191990000_01D_30S_MO.rnx + - IISC00IND_R_20191990000_01D_30S_MO.rnx + - JFNG00CHN_R_20191990000_01D_30S_MO.rnx + - KARR00AUS_R_20191990000_01D_30S_MO.rnx + - KIRI00KIR_R_20191990000_01D_30S_MO.rnx + - KOKV00USA_R_20191990000_01D_30S_MO.rnx + - LHAZ00CHN_R_20191990000_01D_30S_MO.rnx + - LMMF00MTQ_R_20191990000_01D_30S_MO.rnx + - MAW100ATA_R_20191990000_01D_30S_MO.rnx + - MBAR00UGA_R_20191990000_01D_30S_MO.rnx + - METG00FIN_R_20191990000_01D_30S_MO.rnx + - MGUE00ARG_R_20191990000_01D_30S_MO.rnx + - NICO00CYP_R_20191990000_01D_30S_MO.rnx + - NKLG00GAB_R_20191990000_01D_30S_MO.rnx + - OHI300ATA_R_20191990000_01D_30S_MO.rnx + - POAL00BRA_R_20191990000_01D_30S_MO.rnx + - QUIN00USA_R_20191990000_01D_30S_MO.rnx + - REYK00ISL_R_20191990000_01D_30S_MO.rnx + - RGDG00ARG_R_20191990000_01D_30S_MO.rnx + - SAMO00WSM_R_20191990000_01D_30S_MO.rnx + - SEY200SYC_R_20191990000_01D_30S_MO.rnx + - SOLO00SLB_R_20191990000_01D_30S_MO.rnx + - TONG00TON_R_20191990000_01D_30S_MO.rnx + - TOPL00BRA_R_20191990000_01D_30S_MO.rnx + - TOW200AUS_R_20191990000_01D_30S_MO.rnx + - USN700USA_R_20191990000_01D_30S_MO.rnx + - VACS00MUS_R_20191990000_01D_30S_MO.rnx + - ZIM200CHE_R_20191990000_01D_30S_MO.rnx + - CUSV00THA_R_20191990000_01D_30S_MO.rnx outputs: - outputs_root: ./outputs/ - - metadata: - config_description: pod_example_ - analysis_agency: GAA - analysis_centre: Geoscience Australia -----FILE NOT FOR OPERATIONAL USE----- - analysis_software: Ginan v3.0 - rinex_comment: AUSNETWORK1 - gradient_mapping_function: Chen & Herring, 1992 # (string) Name of mapping function used for mapping horizontal troposphere gradients - ocean_tide_loading_model: FES2014 # (string) Ocean tide loading model applied - reference_system: igs20 # (string) Terrestrial Reference System Code - time_system: G # (string) Time system - e.g. "G", "UTC" - - trace: - level: 3 - output_receivers: true - output_network: true - receiver_filename: __.TRACE - network_filename: __.TRACE - output_residuals: true - output_residual_chain: true - output_config: true - output_json: true - - network_statistics: - output: true # (bool) Enable exporting network statistics data to file - directory: ./ # (string) Directory to export network statistics data - filename: _network_statistics.json # (string) Network statistics data filename - - gpx: - output: true - pos: - output: true - filename: __.POS - sinex: - output: true - directory: ./ - filename: _.SNX - - sp3: - output: true - filename: _.SP3 - output_interval: 300 # (int) Update interval for sp3 records - - clocks: - output: true - directory: ./ - filename: _.CLK - - # erp: - # output: true - # directory: ./ - # filename: _.ERP - - orbex: - output: true - filename: _.OBX - attitude_sources: [MODEL, NOMINAL] + outputs_root: ./outputs/ + + metadata: + config_description: pod_example_ + analysis_agency: GAA + analysis_centre: Geoscience Australia -----FILE NOT FOR OPERATIONAL USE----- + analysis_software: Ginan v3.0 + rinex_comment: AUSNETWORK1 + gradient_mapping_function: Chen & Herring, 1992 # (string) Name of mapping function used for mapping horizontal troposphere gradients + ocean_tide_loading_model: FES2014 # (string) Ocean tide loading model applied + reference_system: igs20 # (string) Terrestrial Reference System Code + time_system: G # (string) Time system - e.g. "G", "UTC" + + trace: + level: 3 + output_receivers: true + output_network: true + receiver_filename: __.TRACE + network_filename: __.TRACE + output_residuals: true + output_residual_chain: false + output_config: true + output_json: false + + network_statistics: + output: true # (bool) Enable exporting network statistics data to file + directory: ./ # (string) Directory to export network statistics data + filename: _network_statistics.json # (string) Network statistics data filename + + gpx: + output: true + pos: + output: true + filename: __.POS + sinex: + output: true + directory: ./ + filename: _.SNX + + sp3: + output: true + filename: _.SP3 + output_interval: 300 # (int) Update interval for sp3 records + + clocks: + output: true + directory: ./ + filename: _.CLK + + # erp: + # output: true + # directory: ./ + # filename: _.ERP + + orbex: + output: true + filename: _.OBX + attitude_sources: [MODEL, NOMINAL] satellite_options: - global: - #clock_codes: [AUTO, AUTO] - - models: - clock: - enable: true - sources: [KALMAN, PRECISE, BROADCAST] - pos: - enable: true - sources: [KALMAN, PRECISE, BROADCAST] - attitude: - enable: true - sources: [MODEL, PRECISE, NOMINAL] - pco: - enable: true - pcv: - enable: true - code_bias: - enable: true - default_bias: 0 - undefined_sigma: 0 - phase_bias: - enable: false - default_bias: 0 - undefined_sigma: 0 - - orbit_propagation: - albedo: cannonball - antenna_thrust: true - empirical: true - empirical_dyb_eclipse: [true, false, false] - planetary_perturbations: - [ - moon, - sun, - mercury, - venus, - mars, - jupiter, - saturn, - uranus, - neptune, - pluto, - ] - pseudo_pulses: - enable: false - solar_radiation_pressure: boxwing - mass: 1000 - area: 15 - srp_cr: 1.75 - power: 20 - - GPS: - clock_codes: [L1W, L2W] - - G04: - exclude: true - - E05: { exclude: true } - E06: { exclude: true } - E10: { exclude: true } - E16: { exclude: true } - E17: { exclude: true } - E23: { exclude: true } - E28: { exclude: true } - E29: { exclude: true } - E32: { exclude: true } - E34: { exclude: true } - E35: { exclude: true } + global: + # clock_codes: [AUTO, AUTO] + + models: + clock: + enable: true + sources: [KALMAN, PRECISE, BROADCAST] + pos: + enable: true + sources: [KALMAN, PRECISE, BROADCAST] + attitude: + enable: true + sources: [MODEL, PRECISE, NOMINAL] + pco: + enable: true + pcv: + enable: true + code_bias: + enable: true + phase_bias: + enable: false + + orbit_propagation: + albedo: cannonball + antenna_thrust: true + empirical: true + empirical_dyb_eclipse: [true, false, false] + planetary_perturbations: + [ + moon, + sun, + mercury, + venus, + mars, + jupiter, + saturn, + uranus, + neptune, + pluto, + ] + pseudo_pulses: + enable: false + solar_radiation_pressure: boxwing + mass: 1000 + area: 15 + srp_cr: 1.75 + power: 20 + + GPS: + clock_codes: [L1W, L2W] + + G04: + exclude: true + + E05: { exclude: true } + E06: { exclude: true } + E10: { exclude: true } + E16: { exclude: true } + E17: { exclude: true } + E23: { exclude: true } + E28: { exclude: true } + E29: { exclude: true } + E32: { exclude: true } + E34: { exclude: true } + E35: { exclude: true } receiver_options: # Options to configure individual stations or global configs - USN7: - aliases: [PIVOT] + USN7: + aliases: [PIVOT] + + global: + error_model: elevation_dependent # uniform, elevation_dependent + elevation_mask: 10 + code_sigma: 0.4 + phase_sigma: 0.002 + clock_codes: [AUTO, AUTO] + zero_dcb_codes: [NONE, NONE] + rec_reference_system: GPS + models: + eccentricity: + enable: true # (bool) Enable modelling of antenna eccentricities + attitude: + enable: true # (bool) Enables non-nominal attitude types + sources: [MODEL, NOMINAL] # List of sourecs to use for attitudes + clock: + enable: true # (bool) Enable modelling of clocks + pco: + enable: true # (bool) Enable modelling of phase center offsets + pcv: + enable: true # (bool) Enable modelling of phase center variations + code_bias: + enable: true # (bool) Enable modelling of code biases + phase_bias: + enable: false # (bool) Enable modelling of phase biases + pos: + enable: true # (bool) Enable modelling of position + ionospheric_components: # Ionospheric models produce frequency-dependent effects + enable: true # Enable ionospheric modelling + use_2nd_order: true + use_3rd_order: true + troposphere: + enable: true + models: [gpt2] + eop: + enable: true - global: - error_model: elevation_dependent # uniform, elevation_dependent - elevation_mask: 10 - code_sigma: 0.4 - phase_sigma: 0.002 - clock_codes: [AUTO, AUTO] - zero_dcb_codes: [NONE, NONE] - rec_reference_system: GPS - models: - eccentricity: - enable: true # (bool) Enable modelling of antenna eccentricities - attitude: - enable: true # (bool) Enables non-nominal attitude types - sources: [MODEL, NOMINAL] # List of sourecs to use for attitudes - clock: - enable: true # (bool) Enable modelling of clocks - pco: - enable: true # (bool) Enable modelling of phase center offsets - pcv: - enable: true # (bool) Enable modelling of phase center variations - code_bias: - enable: true # (bool) Enable modelling of code biases - default_bias: 0 # (float) Bias to use when no code bias is found - undefined_sigma: 0 # (float) Uncertainty sigma to apply to default code biases - phase_bias: - enable: false # (bool) Enable modelling of phase biases - default_bias: 0 # (float) Bias to use when no phase bias is found - undefined_sigma: 0 # (float) Uncertainty sigma to apply to default phase biases - pos: - enable: true # (bool) Enable modelling of position - ionospheric_components: # Ionospheric models produce frequency-dependent effects - enable: true # Enable ionospheric modelling - use_2nd_order: true - use_3rd_order: true - troposphere: - enable: true - models: [gpt2] - eop: - enable: true - - apriori_sigma_enu: [0.003, 0.003, 0.009] # Use these fixed igma'sfor sites listed below - mincon_scale_apriori_sigma: 1 # Use ALL fixed and/or SINEX file sigma's (!! first preference to the fixed sigma's !!) - mincon_scale_filter_sigma: 0 - #ABMF: {mincon_scale_apriori_sigma: 3 } - #ALBH: {mincon_scale_apriori_sigma: 3 } - #ALGO: {mincon_scale_apriori_sigma: 3 } + apriori_sigma_enu: [0.003, 0.003, 0.009] # Use these fixed igma'sfor sites listed below + mincon_scale_apriori_sigma: 1 # Use ALL fixed and/or SINEX file sigma's (!! first preference to the fixed sigma's !!) + mincon_scale_filter_sigma: 0 + # ABMF: {mincon_scale_apriori_sigma: 3 } + # ALBH: {mincon_scale_apriori_sigma: 3 } + # ALGO: {mincon_scale_apriori_sigma: 3 } processing_options: - process_modes: - preprocessor: true - spp: true - ppp: true - ionosphere: false - - epoch_control: - epoch_interval: 300 - wait_next_epoch: 3600 # Wait up to an hour for next data point - When processing RINEX causes PEA to wait a long as need for last epoch to be processed. - max_rec_latency: 1 - - gnss_general: - minimise_sat_clock_offsets: true - pivot_receiver: - sys_options: - gps: - process: true - ambiguity_resolution: false - reject_eclipse: false - code_priorities: [L1W, L1C, L2W] - # gal: - # process: true - # ambiguity_resolution: false - # reject_eclipse: false - # code_priorities: [ L1C, L5Q, L1X, L5X ] - # glo: - # process: true - # ambiguity_resolution: false - # reject_eclipse: true - # code_priorities: [ L1P, L1C, L2P, L2C ] - # qzs: - # process: true - # ambiguity_resolution: false - # reject_eclipse: true - # code_priorities: [ L1C, L2L, L2X ] - - spp: - always_reinitialise: false - max_lsq_iterations: 12 - outlier_screening: - max_gdop: 30 - postfit: - sigma_check: true - - ppp_filter: - ionospheric_components: - common_ionosphere: true - use_if_combo: false - outlier_screening: - prefit: - max_iterations: 2 - sigma_check: true - state_sigma_threshold: 5 # Sigma threshold for states - meas_sigma_threshold: 5 # Sigma threshold for measurements - omega_test: false - postfit: - max_iterations: 10 - sigma_check: true - state_sigma_threshold: 3 # Sigma threshold for states - meas_sigma_threshold: 3 # Sigma threshold for measurements - - rts: - enable: true - - model_error_handling: - meas_deweighting: - deweight_factor: 10000 - state_deweighting: - deweight_factor: 10000 - ambiguities: - outage_reset_limit: 300 - - phase_reject_limit: 2 - reset_on: - gf: true - lli: true - mw: true - scdia: true - exclusions: - gf: true - lli: true - mw: true - scdia: true - eclipse: false - ionospheric_components: - outage_reset_limit: 300 - orbit_errors: - enable: false - pos_process_noise: 10 - vel_process_noise: 1 - vel_process_noise_trail: 0 - vel_process_noise_trail_tau: 0 - - minimum_constraints: + process_modes: + preprocessor: true + spp: true + ppp: true + ionosphere: false + + epoch_control: + epoch_interval: 300 + wait_next_epoch: 3600 # Wait up to an hour for next data point - When processing RINEX causes PEA to wait a long as need for last epoch to be processed. + max_rec_latency: 1 + + gnss_general: + minimise_sat_clock_offsets: + enable: true + pivot_receiver: + sys_options: + gps: + process: true + ambiguity_resolution: false + reject_eclipse: false + code_priorities: [L1W, L1C, L2W] + # gal: + # process: true + # ambiguity_resolution: false + # reject_eclipse: false + # code_priorities: [L1C, L5Q, L1X, L5X] + # glo: + # process: true + # ambiguity_resolution: false + # reject_eclipse: true + # code_priorities: [L1P, L1C, L2P, L2C] + # qzs: + # process: true + # ambiguity_resolution: false + # reject_eclipse: true + # code_priorities: [L1C, L2L, L2X] + + spp: + outlier_screening: + chi_square: enable: true - rotation: - estimated: [true] - scale: - estimated: [true] - translation: - estimated: [true] - application_mode: weight_matrix - # once_per_epoch: true - constrain_orbits: false - outlier_screening: # Statistical checks allow for detection of outliers that exceed their confidence intervals - postfit: - max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter - sigma_check: true # Enable sigma check - state_sigma_threshold: 3 # Sigma threshold for states - meas_sigma_threshold: 3 # Sigma threshold for measurements - prefit: - max_iterations: 2 # Maximum number of measurements to exclude using prefit checks before attempting to filter - omega_test: false # Enable omega-test - sigma_check: true # Enable sigma check - state_sigma_threshold: 5 # Sigma threshold for states - meas_sigma_threshold: 5 # Sigma threshold for measurements - - orbit_propagation: - integrator_time_step: 60 # Timestep for the integrator, must be smaller than the processing time step, might be adjusted if the processing time step isn't a integer number of time steps - central_force: true - egm_field: true # Acceleration due to the high degree model of the Earth gravity model (exclude degree 0, made by central_force) - egm_degree: 15 # J2 acceleration perturbation due to the Sun and Moon - solid_earth_tide: true # Model accelerations due to solid earth tides - ocean_tide: true # Model accelerations due to ocean tides model - pole_tide_solid: true # Model accelerations due to solid pole tide (degree 2 only) - pole_tide_ocean: true - general_relativity: true - indirect_J2: true + sigma_threshold: 5 + least_square: + max_iterations: 1 + sigma_check: false + omega_test: true + meas_sigma_threshold: 5 + raim: + enable: true + max_iterations: 2 + ppp_filter: + ionospheric_components: + common_ionosphere: true + use_if_combo: false + outlier_screening: + prefit: + max_iterations: 2 + sigma_check: false + omega_test: false + state_sigma_threshold: 5 # Sigma threshold for states + meas_sigma_threshold: 5 # Sigma threshold for measurements + postfit: + max_iterations: 20 + sigma_check: false + omega_test: true + state_sigma_threshold: 8 # Sigma threshold for states + meas_sigma_threshold: 6 # Sigma threshold for measurements + + rts: + enable: true + filename: Filter---.rts + + model_error_handling: + error_accumulation: # Any receivers or satellites that are consistently getting many measurement rejections may be reinitialiased + enable: true # Enable reinitialisation of receivers upon many rejections + receiver_error_count_threshold: 0 # Number of errors for a receiver to be considered in error for a single epoch + receiver_error_epochs_threshold: 0 # Number of consecutive epochs with receiver in error before it is removed and reinitialised + satellite_error_count_threshold: 0 # Number of errors for a satellite to be considered in error for a single epoch + satellite_error_epochs_threshold: 0 # Number of consecutive epochs with satellite in error before it is reinitialised using the orbit_errors configs + state_error_count_threshold: 3 # Number of consecutive epochs with satellite in error before it is reinitialised using the orbit_errors configs + meas_deweighting: + deweight_factor: 10000 + state_deweighting: + deweight_factor: 10000 + ambiguities: + phase_reject_limit: 2 + reset_on: + gf: true + lli: true + mw: true + scdia: true + retrack: true + single_freq: true + exclusions: + gf: true + lli: false + mw: true + scdia: true + eclipse: false + retrack: false + single_freq: true + ionospheric_components: + outage_reset_limit: 300 + satellite_errors: + enable: false + pos_process_noise: 10 + vel_process_noise: 1 + vel_process_noise_trail: 0 + vel_process_noise_trail_tau: 0 + + minimum_constraints: + enable: true + rotation: + sigma: [400] + estimated: [true] + scale: + sigma: [400] + estimated: [true] + translation: + sigma: [400] + estimated: [true] + application_mode: weight_matrix + once_per_epoch: false + constrain_orbits: false + outlier_screening: # Statistical checks allow for detection of outliers that exceed their confidence intervals + postfit: + max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter + sigma_check: true # Enable sigma check + state_sigma_threshold: 3 # Sigma threshold for states + meas_sigma_threshold: 3 # Sigma threshold for measurements + prefit: + max_iterations: 2 # Maximum number of measurements to exclude using prefit checks before attempting to filter + omega_test: false # Enable omega-test + sigma_check: true # Enable sigma check + state_sigma_threshold: 5 # Sigma threshold for states + meas_sigma_threshold: 5 # Sigma threshold for measurements + + orbit_propagation: + integrator_time_step: 60 # Timestep for the integrator, must be smaller than the processing time step, might be adjusted if the processing time step isn't a integer number of time steps + central_force: true + egm_field: true # Acceleration due to the high degree model of the Earth gravity model (exclude degree 0, made by central_force) + egm_degree: 15 # J2 acceleration perturbation due to the Sun and Moon + solid_earth_tide: true # Model accelerations due to solid earth tides + ocean_tide: true # Model accelerations due to ocean tides model + pole_tide_solid: true # Model accelerations due to solid pole tide (degree 2 only) + pole_tide_ocean: true + general_relativity: true + indirect_J2: true estimation_parameters: - global_models: - eop: - estimated: [true] - sigma: [10, 10, 1e-9] - eop_rates: - estimated: [true] - sigma: [10] - - receivers: - PIVOT: - #clock: - # estimated: [true] - # process_noise: [0] - # sigma: [1e-9] - code_bias: - estimated: [false] - - global: - pos: - estimated: [true] - sigma: [1] - process_noise: [0.0] - # process_noise_dt: second - clock: - estimated: [true] - sigma: [1000] - process_noise: [10] # [100] - ambiguities: - estimated: [true] - sigma: [1000] - process_noise: [0] - # process_noise_dt: day - trop: - estimated: [true] - sigma: [0.3] - process_noise: [0.0001] - # process_noise_dt: second - trop_grads: - estimated: [true] - sigma: [0.03] - process_noise: [1.0E-6] - # process_noise_dt: second - ion_stec: - estimated: [true] - sigma: [500] - process_noise: [10] - code_bias: - estimated: [true] - sigma: [20] - process_noise: [0] - # USN7: - # clk: - # estimated: [false] # Set reference (pivot) station clock - # code_bias: - # estimated: [false] - - satellites: - global: - clock: - estimated: [true] - sigma: [1000] - process_noise: [1] - tau: [100] - #mu: [10000] - code_bias: - estimated: [true] - sigma: [10] - process_noise: [0] - orbit: # Orbital state - estimated: [true] # [bools] Estimate state in kalman filter - sigma: [10, 10, 10, 0.01] - process_noise: [0] - - emp_d_0: { estimated: [true], sigma: [10] } - emp_y_0: { estimated: [true], sigma: [1] } - emp_b_0: { estimated: [true], sigma: [1] } - - # emp_d_1: { estimated: [true], sigma: [1]} - # emp_y_1: { estimated: [true], sigma: [1]} - emp_b_1: { estimated: [true], sigma: [1] } - - emp_d_2: { estimated: [true], sigma: [1] } - # emp_y_2: { estimated: [true], sigma: [1]} - # emp_b_2: { estimated: [true], sigma: [1]} - - # emp_d_3: { estimated: [true], sigma: [1]} - # emp_y_3: { estimated: [true], sigma: [1]} - # emp_b_3: { estimated: [true], sigma: [1]} - - # emp_d_4: { estimated: [true], sigma: [1]} - # emp_y_4: { estimated: [true], sigma: [1]} - # emp_b_4: { estimated: [true], sigma: [1]} + global_models: + eop: + estimated: [true] + sigma: [10, 10, 1e-9] + eop_rates: + estimated: [true] + sigma: [10] + + receivers: + PIVOT: + # clock: + # estimated: [true] + # process_noise: [0] + # sigma: [1e-9] + code_bias: + estimated: [false] + + global: + pos: + estimated: [true] + sigma: [1] + process_noise: [0.0] + # process_noise_dt: second + clock: + estimated: [true] + sigma: [1000] + process_noise: [10] # [100] + ambiguities: + estimated: [true] + sigma: [1000] + process_noise: [0] + outage_limit: [900] + # process_noise_dt: day + trop: + estimated: [true] + sigma: [0.3] + process_noise: [0.0001] + # process_noise_dt: second + trop_grads: + estimated: [true] + sigma: [0.03] + process_noise: [1.0E-6] + # process_noise_dt: second + ion_stec: + estimated: [true] + sigma: [500] + process_noise: [10] + outage_limit: [900] + code_bias: + estimated: [true] + sigma: [20] + process_noise: [0] + # USN7: + # clk: + # estimated: [false] # Set reference (pivot) station clock + # code_bias: + # estimated: [false] + + satellites: + global: + clock: + estimated: [true] + sigma: [1000] + process_noise: [1] + tau: [100] + # mu: [10000] + code_bias: + estimated: [true] + sigma: [10] + process_noise: [0] + orbit: # Orbital state + estimated: [true] # [bools] Estimate state in kalman filter + sigma: [10, 10, 10, 0.01] + process_noise: [0] + + emp_d_0: { estimated: [true], sigma: [10] } + emp_y_0: { estimated: [true], sigma: [1] } + emp_b_0: { estimated: [true], sigma: [1] } + + # emp_d_1: { estimated: [true], sigma: [1]} + # emp_y_1: { estimated: [true], sigma: [1]} + emp_b_1: { estimated: [true], sigma: [1] } + + emp_d_2: { estimated: [true], sigma: [1] } + # emp_y_2: { estimated: [true], sigma: [1]} + # emp_b_2: { estimated: [true], sigma: [1]} + + # emp_d_3: { estimated: [true], sigma: [1]} + # emp_y_3: { estimated: [true], sigma: [1]} + # emp_b_3: { estimated: [true], sigma: [1]} + + # emp_d_4: { estimated: [true], sigma: [1]} + # emp_y_4: { estimated: [true], sigma: [1]} + # emp_b_4: { estimated: [true], sigma: [1]} mongo: - enable: primary - #enable: none - output_components: primary - output_states: primary - output_measurements: primary - output_test_stats: none - output_trace: primary - delete_history: primary - -debug: - # instrument: true - #output_mincon: true - #mincon_filename: preMinconState.bin - #mincon_only: true - # mincon_only: true + enable: primary + # enable: none + output_components: primary + output_states: primary + output_measurements: primary + output_test_stats: none + output_trace: primary + delete_history: primary +# debug: +# instrument: true +# output_mincon: true +# mincon_filename: preMinconState.bin +# mincon_only: true diff --git a/exampleConfigs/ppp_example.yaml b/exampleConfigs/ppp_example.yaml index 7a5ad0e81..f800a0d7b 100644 --- a/exampleConfigs/ppp_example.yaml +++ b/exampleConfigs/ppp_example.yaml @@ -1,299 +1,310 @@ inputs: - inputs_root: ./products/ + inputs_root: ./products/ - atx_files: [igs20.atx] # required - igrf_files: [tables/igrf13coeffs.txt] - erp_files: [finals.data.iau2000.txt] - planetary_ephemeris_files: [tables/DE436.1950.2050] + atx_files: [igs20.atx] # required + igrf_files: [tables/igrf14coeffs.txt] + erp_files: [finals.data.iau2000.txt] + planetary_ephemeris_files: [tables/DE436.1950.2050] - troposphere: - gpt2grid_files: [tables/gpt_25.grd] + troposphere: + gpt2grid_files: [tables/gpt_25.grd] - tides: - ocean_tide_loading_blq_files: [tables/OLOAD_GO.BLQ] # required if ocean loading is applied - atmos_tide_loading_blq_files: [tables/ALOAD_GO.BLQ] # required if atmospheric tide loading is applied - ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] # required if ocean pole tide loading is applied + tides: + ocean_tide_loading_blq_files: [tables/OLOAD_GO.BLQ] # required if ocean loading is applied + atmos_tide_loading_blq_files: [tables/ALOAD_GO.BLQ] # required if atmospheric tide loading is applied + ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] # required if ocean pole tide loading is applied - snx_files: - # - "*.SNX" # use a wild card to include all files matching the description in the directory - - igs_satellite_metadata.snx - - tables/sat_yaw_bias_rate.snx - - tables/qzss_yaw_modes.snx - - tables/bds_yaw_modes.snx - - IGS1R03SNX_20191950000_07D_07D_CRD.SNX + snx_files: + # - "*.SNX" # use a wild card to include all files matching the description in the directory + - igs_satellite_metadata.snx + - tables/sat_yaw_bias_rate.snx + - tables/qzss_yaw_modes.snx + - tables/bds_yaw_modes.snx + - IGS1R03SNX_20191950000_07D_07D_CRD.SNX - satellite_data: - # nav_files: - # - "*.rnx" - clk_files: - # - "*.CLK" - - IGS2R03FIN_20191990000_01D_30S_CLK.CLK - bsx_files: - # - "*.BIA" - - IGS2R03FIN_20191990000_01D_01D_OSB.BIA - sp3_files: - # - "*.SP3" - - IGS2R03FIN_20191990000_01D_05M_ORB.SP3 + satellite_data: + nav_files: + - brdm1990.19p + clk_files: + # - "*.CLK" + - IGS2R03FIN_20191990000_01D_30S_CLK.CLK + bsx_files: + # - "*.BIA" + - IGS2R03FIN_20191990000_01D_01D_OSB.BIA + # - TUG0R03FIN_20191990000_01D_01D_OSB.BIA + sp3_files: + # - "*.SP3" + - IGS2R03FIN_20191990000_01D_05M_ORB.SP3 - gnss_observations: - gnss_observations_root: ../data/ - rnx_inputs: - # - "*.rnx" - - ALIC00AUS_R_20191990000_01D_30S_MO.rnx - # - DARW00AUS_R_20191990000_01D_30S_MO.rnx - # - HOB200AUS_R_20191990000_01D_30S_MO.rnx - # - "M*.rnx" + gnss_observations: + gnss_observations_root: ../data/ + rnx_inputs: + # - "*.rnx" + - ALIC00AUS_R_20191990000_01D_30S_MO.rnx + # - ALIC2.rnx + # - DARW00AUS_R_20191990000_01D_30S_MO.rnx + # - HOB200AUS_R_20191990000_01D_30S_MO.rnx + # - "M*.rnx" outputs: - metadata: - config_description: ppp_example_ + metadata: + config_description: ppp_example_ - outputs_root: ./outputs/ + outputs_root: ./outputs/ - gpx: - output: true - filename: __.GPX - pos: - output: true - filename: __.POS - trace: - level: 2 - output_receivers: true - output_network: true - receiver_filename: __.TRACE - network_filename: __.TRACE - output_residuals: true - output_residual_chain: true - output_config: true - output_initialised_states: true - output_predicted_states: true + gpx: + output: true + filename: __.GPX + pos: + output: true + filename: .POS + trace: + level: 2 + output_receivers: true + output_network: true + receiver_filename: __.TRACE + network_filename: __.TRACE + output_residuals: true + output_residual_chain: true + output_config: true + output_initialised_states: false + output_predicted_states: false satellite_options: - global: - error_model: ELEVATION_DEPENDENT # {uniform,elevation_dependent} - code_sigma: 0 # Standard deviation of code measurements - phase_sigma: 0 # Standard deviation of phase measurmeents - models: - phase_bias: - enable: false + global: + error_model: ELEVATION_DEPENDENT # {uniform,elevation_dependent} + code_sigma: 0 # Standard deviation of code measurements + phase_sigma: 0 # Standard deviation of phase measurmeents + models: + phase_bias: + enable: true - # E05: - # exclude: true # Exclude satellites - # E06: - # exclude: true + # E05: + # exclude: true # Exclude satellites + # E06: + # exclude: true receiver_options: - global: - elevation_mask: 15 # degrees - error_model: ELEVATION_DEPENDENT # {uniform,elevation_dependent} - code_sigma: 0.3 # Standard deviation of code measurements, m - phase_sigma: 0.003 # Standard deviation of phase measurmeents, m - clock_codes: [AUTO, AUTO] - zero_dcb_codes: [AUTO, AUTO] - rec_reference_system: GPS - models: - phase_bias: - enable: false - troposphere: # Tropospheric modelling accounts for delays due to refraction of light in water vapour - enable: true - models: [gpt2] # List of models to use for troposphere [standard,sbas,vmf3,gpt2,cssr] - tides: - atl: true # Enable atmospheric tide loading - enable: true # Enable modelling of tidal disaplacements - opole: true # Enable ocean pole tides - otl: true # Enable ocean tide loading - solid: true # Enable solid Earth tides - spole: true # Enable solid Earth pole tides - # ALIC: - # receiver_type: "LEICA GR25" # (string) - # antenna_type: "LEIAR25.R3 NONE" # (string) - # apriori_position: [-4052052.7254, 4212835.9872,-2545104.6139] # [floats] - # aliases: [PIVOT] # set as pivot station - # models: - # eccentricity: - # enable: true - # offset: [0.0000, 0.0000, 0.0015] # [floats] + global: + elevation_mask: 15 # (degrees) + error_model: ELEVATION_DEPENDENT # {uniform,elevation_dependent} + code_sigma: 0.3 # Standard deviation of code measurements (m) + phase_sigma: 0.003 # Standard deviation of phase measurmeents (m) + clock_codes: [AUTO, AUTO] + zero_dcb_codes: [AUTO, AUTO] + rec_reference_system: GPS + models: + phase_bias: + enable: false + troposphere: # Tropospheric modelling accounts for delays due to refraction of light in water vapour + enable: true + models: [gpt2] # List of models to use for troposphere [standard,sbas,vmf3,gpt2,cssr] + tides: + enable: true # Enable modelling of tidal disaplacements + solid: true # Enable solid Earth tides + otl: true # Enable ocean tide loading + atl: true # Enable atmospheric tide loading + spole: true # Enable solid Earth pole tides + opole: true # Enable ocean pole tides + ionospheric_components: + use_2nd_order: true + use_3rd_order: true + + # ALIC: + # receiver_type: "LEICA GR25" # (string) + # antenna_type: "LEIAR25.R3 NONE" # (string) + # apriori_position: [-4052052.7254, 4212835.9872, -2545104.6139] # [floats] + # aliases: [PIVOT] # set as pivot station + # models: + # eccentricity: + # enable: true + # offset: [0.0000, 0.0000, 0.0015] # [floats] processing_options: - process_modes: - preprocessor: true # Preprocessing and quality checks - spp: true # Perform SPP on receiver data - ppp: true # Perform PPP network or end user mode - ionosphere: false # Compute Ionosphere models based on GNSS measurements - slr: false # Process SLR observations + process_modes: + preprocessor: true # Preprocessing and quality checks + spp: true # Perform SPP on receiver data + ppp: true # Perform PPP network or end user mode + ionosphere: false # Compute Ionosphere models based on GNSS measurements + slr: false # Process SLR observations - epoch_control: - # start_epoch: 2019-07-18 00:00:00 - # end_epoch: 2019-07-18 23:59:30 - # max_epochs: 2880 - epoch_interval: 30 # seconds - wait_next_epoch: 3600 # seconds (make large for post-processing) + epoch_control: + # start_epoch: 2019-07-18 00:00:00 + # end_epoch: 2019-07-18 23:59:30 + #max_epochs: 30 + epoch_interval: 30 # seconds + wait_next_epoch: 3600 # seconds (make large for post-processing) - gnss_general: - add_eop_component: true - sys_options: - gps: - process: true - reject_eclipse: false - # clock_codes: [ L1W, L2W ] - code_priorities: [L1W, L1C, L2W] + gnss_general: + add_eop_component: true + use_primary_signals: true + sys_options: + gps: + process: true + reject_eclipse: false + code_priorities: [L1W, L1C, L2W, L2S] + # code_priorities: [L1W, L1C, L2W, L2S, L5Q, L5X] - gal: - # process: true - reject_eclipse: false - code_priorities: [L1C, L5Q, L1X, L5X] + gal: + process: true + reject_eclipse: false + code_priorities: [L1C, L1X, L5Q, L5X] - preprocessor: # Configurations for the kalman filter and its sub processes - cycle_slips: # Cycle slips may be detected by the preprocessor and measurements rejected or ambiguities reinitialised - mw_process_noise: 0 # Process noise applied to filtered Melbourne-Wubenna measurements to detect cycle slips - slip_threshold: 0.05 # Value used to determine when a slip has occurred - preprocess_all_data: true + glo: + process: true + reject_eclipse: false + code_priorities: [L1C, L1P, L2C, L2P] - spp: - # always_reinitialise: false # Reset SPP state to zero to avoid potential for lock-in of bad states - max_lsq_iterations: 12 # Maximum number of iterations of least squares allowed for convergence - outlier_screening: - raim: true # Enable Receiver Autonomous Integrity Monitoring - max_gdop: 30 # Maximum dilution of precision before error is flagged + # phase_measurements: + # process: false - ppp_filter: - outlier_screening: - chi_square: - enable: false # Enable Chi-square test - mode: innovation # Chi-square test mode {none,innovation,measurement,state} - prefit: - max_iterations: 3 # Maximum number of measurements to exclude using prefit checks before attempting to filter - omega_test: false # Enable omega-test - sigma_check: true # Enable sigma check - state_sigma_threshold: 4 # Sigma threshold for states - meas_sigma_threshold: 4 # Sigma threshold for measurements - postfit: - max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter - sigma_check: true # Enable sigma check - state_sigma_threshold: 4 # Sigma threshold for states - meas_sigma_threshold: 4 # Sigma threshold for measurements + preprocessor: # Configurations for the kalman filter and its sub processes + cycle_slips: # Cycle slips may be detected by the preprocessor and measurements rejected or ambiguities reinitialised + mw_process_noise: 0 # Process noise applied to filtered Melbourne-Wubenna measurements to detect cycle slips + slip_threshold: 0.05 # Value used to determine when a slip has occurred + preprocess_all_data: true - ionospheric_components: # Slant ionospheric components - common_ionosphere: true # Use the same ionosphere state for code and phase observations - use_gf_combo: false # Combine 'uncombined' measurements to simulate a geometry-free solution - use_if_combo: false # Combine 'uncombined' measurements to simulate an ionosphere-free solution + spp: + outlier_screening: + chi_square: + enable: true + sigma_threshold: 5 + least_square: + max_iterations: 1 + sigma_check: false + omega_test: true + meas_sigma_threshold: 5 + raim: + enable: true + max_iterations: 2 + ppp_filter: + outlier_screening: + chi_square: + enable: false # Enable Chi-square test + mode: innovation # Chi-square test mode {none,innovation,measurement,state} + prefit: + max_iterations: 3 # Maximum number of measurements to exclude using prefit checks before attempting to filter + omega_test: false # Enable omega-test + sigma_check: false # Enable sigma check + state_sigma_threshold: 4 # Sigma threshold for states + meas_sigma_threshold: 4 # Sigma threshold for measurements + postfit: + max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter + omega_test: true # Enable omega-test + sigma_check: false # Enable sigma check + state_sigma_threshold: 8 # Sigma threshold for states + meas_sigma_threshold: 6 # Sigma threshold for measurements - chunking: - by_receiver: true # Split large filter and measurement matrices blockwise by receiver ID to improve processing speed - by_satellite: false # Split large filter and measurement matrices blockwise by satellite ID to improve processing speed - size: 0 + ionospheric_components: # Slant ionospheric components + common_ionosphere: true # Use the same ionosphere state for code and phase observations + use_gf_combo: false # Combine 'uncombined' measurements to simulate a geometry-free solution + use_if_combo: false # Combine 'uncombined' measurements to simulate an ionosphere-free solution - rts: # Rauch-Tung-Striebel (RTS) backwards smoothing - enable: true - lag: -1 - # interval: 86400 - inverter: LDLT # Inverter to be used within the rts processor, which may provide different performance outcomes in terms of processing time and accuracy and stability - filename: _.rts + chunking: + by_receiver: true # Split large filter and measurement matrices blockwise by receiver ID to improve processing speed + size: 0 - periodic_reset: - # enable: true - # interval: 86400 - # states: [REC_POS] + rts: # Rauch-Tung-Striebel (RTS) backwards smoothing + enable: true + lag: -1 + # interval: 86400 + filename: _.rts - model_error_handling: - meas_deweighting: # Measurements that are outside the expected confidence bounds may be deweighted so that outliers do not contaminate the filtered solution - deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors - enable: true # Enable deweighting of all rejected measurement - state_deweighting: # Any "state" errors cause deweighting of all measurements that reference the state - deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors - enable: true # Enable deweighting of all referencing measurements - error_accumulation: - enable: true - receiver_error_count_threshold: 4 - receiver_error_epochs_threshold: 4 - ambiguities: - outage_reset_limit: 300 - phase_reject_limit: 2 # Reset ambiguity after 2 large fractional residuals are found (replaces phase_reject_count:) - reset_on: # Reset ambiguities when slip is detected by the following - gf: true # GF test - lli: true # LLI test - mw: true # MW test - scdia: true # SCDIA test + # periodic_reset: + # enable: true + # interval: 86400 + # states: [REC_POS] -estimation_parameters: - global_models: - eop: - # estimated: [true] # Estimate state in kalman filter - sigma: [1000] # Apriori sigma values - process_noise: [0] # Process noise sigmas - eop_rates: - # estimated: [true] # Estimate state in kalman filter - sigma: [1000] # Apriori sigma values - process_noise: [0] # Process noise sigmas - ion: - estimated: [false] # Estimate state in kalman filter - sigma: [-1] # Apriori sigma values - process_noise: [0] # Process noise sigmas + model_error_handling: + meas_deweighting: # Measurements that are outside the expected confidence bounds may be deweighted so that outliers do not contaminate the filtered solution + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all rejected measurement + state_deweighting: # Any "state" errors cause deweighting of all measurements that reference the state + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all referencing measurements + error_accumulation: + enable: true + receiver_error_count_threshold: 0 + receiver_error_epochs_threshold: 0 + state_error_count_threshold: 3 + ambiguities: + phase_reject_limit: 2 # Reset ambiguity after 2 large fractional residuals are found (replaces phase_reject_count:) + reset_on: # Reset ambiguities when slip is detected by the following + gf: true # GF test + lli: true # LLI test + mw: true # MW test + scdia: true # SCDIA test + retrack: true - receivers: - global: - pos: - estimated: [true] - sigma: [100] - # process_noise: [30] - pos_rate: # Velocity - estimated: [false] # [bools] Estimate state in kalman filter - sigma: [0] # [floats] Apriori sigma values - process_noise: [0] # [floats] Process noise sigmas - # process_noise_dt: SECOND # (enum) Time unit for process noise - sqrt_sec, sqrt_day etc - # apriori_val: [0] # [floats] Apriori state values - # mu: [0] # [floats] Desired mean value for gauss markov states - # tau: [-1] # [floats] Correlation times for gauss markov noise, defaults to -1 -> inf (Random Walk) - clock: - estimated: [true] - sigma: [1000] - process_noise: [100] - ant_delta: - estimated: [true] - sigma: [10] - process_noise: [1] - tau: [100] - clock_rate: - estimated: [false] - sigma: [0.005] - process_noise: [1e-4] - ambiguities: - estimated: [true] - sigma: [1000] - process_noise: [0] - ion_stec: # Ionospheric slant delay - estimated: [true] # Estimate state in kalman filter - sigma: [200] # Apriori sigma values - process_noise: [10] # Process noise sigmas - trop: - estimated: [true] - sigma: [0.3] - process_noise: [0.0001] - trop_grads: - estimated: [true] - sigma: [0.03] - process_noise: [1.0E-6] - code_bias: - estimated: [true] # false - sigma: [20] - process_noise: [0] - phase_bias: - estimated: [false] - sigma: [10] - process_noise: [0] +estimation_parameters: + receivers: + global: + pos: + estimated: [true] + sigma: [100] + # process_noise: [30] + pos_rate: # Velocity + estimated: [false] # [bools] Estimate state in kalman filter + sigma: [0] # [floats] Apriori sigma values + process_noise: [0] # [floats] Process noise sigmas + # process_noise_dt: SECOND # (enum) Time unit for process noise - sqrt_sec, sqrt_day etc + # apriori_val: [0] # [floats] Apriori state values + # mu: [0] # [floats] Desired mean value for gauss markov states + # tau: [-1] # [floats] Correlation times for gauss markov noise, defaults to -1 -> inf (Random Walk) + clock: + estimated: [true] + sigma: [1000] + process_noise: [100] + clock_rate: + estimated: [false] + sigma: [0.005] + process_noise: [1e-4] + ambiguities: + estimated: [true] + sigma: [1000] + process_noise: [0] + outage_limit: [120] + ion_stec: # Ionospheric slant delay + estimated: [true] # Estimate state in kalman filter + sigma: [200] # Apriori sigma values + process_noise: [10] # Process noise sigmas + outage_limit: [120] + sigma_limit: [1000] + trop: + estimated: [true] + sigma: [0.3] + process_noise: [0.0001] + trop_grads: + estimated: [true] + sigma: [0.03] + process_noise: [1.0E-6] + code_bias: + estimated: [true] + sigma: [20] + process_noise: [0] + phase_bias: + estimated: [false] + sigma: [10] + process_noise: [0] + gps: + l5q: + phase_bias: + estimated: [true] + sigma: [10] + process_noise: [0.001] mongo: # Mongo is a database used to store results and intermediate values for later analysis and inter-process communication - enable: primary # Enable and connect to mongo database {none,primary,secondary,both} - primary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to - primary_database: - primary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison - # secondary_database: - # secondary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison - # secondary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to - # output_config: primary # Output config {none,primary,secondary,both} - output_components: primary # Output components of measurements {none,primary,secondary,both} - output_states: primary # Output states {none,primary,secondary,both} - output_measurements: primary # Output measurements and their residuals {none,primary,secondary,both} - output_test_stats: primary # Output test statistics {none,primary,secondary,both} - delete_history: primary # Drop the collection in the database at the beginning of the run to only show fresh data {none,primary,secondary,both} - output_trace: primary + enable: none # Enable and connect to mongo database {none,primary,secondary,both} + primary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to + primary_database: + primary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison + # secondary_database: + # secondary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison + # secondary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to + # output_config: primary # Output config {none,primary,secondary,both} + output_components: primary # Output components of measurements {none,primary,secondary,both} + output_states: primary # Output states {none,primary,secondary,both} + output_measurements: primary # Output measurements and their residuals {none,primary,secondary,both} + output_test_stats: primary # Output test statistics {none,primary,secondary,both} + delete_history: primary # Drop the collection in the database at the beginning of the run to only show fresh data {none,primary,secondary,both} + output_trace: primary diff --git a/exampleConfigs/record_snr.yaml b/exampleConfigs/record_snr.yaml index 7af7755a0..3c30b4ac1 100644 --- a/exampleConfigs/record_snr.yaml +++ b/exampleConfigs/record_snr.yaml @@ -2,11 +2,11 @@ inputs: gnss_observations: gnss_observations_root: "https://:@ntrip.data.gnss.ga.gov.au/" rtcm_inputs: - - ALIC00AUS0 - - DARW00AUS0 - - MOBS00AUS0 - - TKBG00JPN0 - - TLSE00FRA0 + - ALIC00AUS0 + - DARW00AUS0 + - MOBS00AUS0 + - TKBG00JPN0 + - TLSE00FRA0 outputs: metadata: @@ -50,5 +50,3 @@ mongo: primary_database: "" primary_suffix: "" delete_history: primary - - diff --git a/exampleConfigs/record_streams.yaml b/exampleConfigs/record_streams.yaml index f8cc94c87..7a66c3035 100644 --- a/exampleConfigs/record_streams.yaml +++ b/exampleConfigs/record_streams.yaml @@ -1,78 +1,124 @@ # Record and decode streams inputs: + inputs_root: ./products/ - inputs_root: ./products/ - - atx_files: + atx_files: - igs20.atx - snx_files: - # - tables/igs_satellite_metadata.snx - - tables/igs_satellite_metadata_2203_plus.snx - - gnss_observations: - gnss_observations_root: "https://:@ntrip.data.gnss.ga.gov.au/" - rtcm_inputs: - - ALIC00AUS0 - - satellite_data: - satellite_data_root: "https://:@ntrip.data.gnss.ga.gov.au/" - rtcm_inputs: - ssr_antenna_offset: APC - rtcm_inputs: - - BCEP00BKG0 - - SSRA00CNE0 + planetary_ephemeris_files: + - tables/DE436.1950.2050 + + snx_files: + - igs_satellite_metadata.snx + - tables/sat_yaw_bias_rate.snx + - tables/bds_yaw_modes.snx + - tables/qzss_yaw_modes.snx + # - "*_CRD.SNX" + + gnss_observations: + gnss_observations_root: "https://:@ntrip.data.gnss.ga.gov.au/" + rtcm_inputs: + - ALIC00AUS0 + + satellite_data: + satellite_data_root: "https://:@ntrip.data.gnss.ga.gov.au/" + rtcm_inputs: + ssr_antenna_offset: APC + rtcm_inputs: + - BCEP00BKG0 + - SSRA00CNE0 outputs: - - metadata: - config_description: record_streams - - outputs_root: ./outputs// - - rtcm_nav: - output: true - - rtcm_obs: - output: true - - decoded_rtcm: - output: true - - rinex_nav: - output: true - - rinex_obs: - output: true - - sp3: - output: true - filename: --.sp3 - orbit_sources: [ SSR ] - output_interval: 300 + metadata: + config_description: record_streams + user: your_NTRIP_username + pass: your_NTRIP_password + reference_system: IGS20 + + outputs_root: ./outputs// + + log: + output: true + filename: __LOG.json + + rtcm_obs: + output: true + filename: __OBS.rtcm + + rtcm_nav: + output: true + filename: __NAV.rtcm + + decoded_rtcm: + output: true + filename: __DEC.json + + rinex_obs: + output: true + filename: __OBS.rnx + + rinex_nav: + output: true + filename: __NAV.rnx + + clocks: + output: true + filename: __CLK.clk + receiver_sources: [NONE] + satellite_sources: [SSR] + output_interval: 30 + + sp3: + output: true + filename: __ORB.sp3 + orbit_sources: [SSR] + clock_sources: [SSR] + output_interval: 300 + +satellite_options: # Required if write out SP3 files with SSRA streams + global: + models: + pos: + enable: true + sources: [SSR] + clock: + enable: true + sources: [SSR] + +receiver_options: # Receiver and antenna information to write to Rinex file headers (can use valid Sinex files instead) + ALIC: + receiver_type: "SEPT POLARX5" + antenna_type: "TWIVC6050 NONE" + apriori_position: [-4052052.7352, 4212835.9833, -2545104.5853] + models: + eccentricity: + enable: true + offset: [0.0000, 0.0000, 0.0250] processing_options: - - epoch_control: - # max_epochs: 60 - epoch_interval: 1 - wait_next_epoch: 1 - max_rec_latency: 0 - require_obs: false - sleep_milliseconds: 1 - - gnss_general: - common_sat_pco: true - delete_old_ephemerides: true - sys_options: - gps: - process: true - gal: - process: true - glo: - process: false - bds: - process: false - qzs: - process: false + process_modes: + spp: false + + epoch_control: + require_obs: false + epoch_interval: 10 + wait_next_epoch: 10 + max_rec_latency: 1 + + gnss_general: + common_sat_pco: true + delete_old_ephemerides: true + gpst_utc_leap_seconds: 18 + sys_options: + gps: + process: true + gal: + process: true + glo: + process: true + bds: + process: true + code_priorities: [L1P, L5P, L2I, L6I, L7I, L7D] + qzs: + process: true diff --git a/exampleConfigs/rt_ppp_example.yaml b/exampleConfigs/rt_ppp_example.yaml index 04ea5a9ec..29b81cab3 100644 --- a/exampleConfigs/rt_ppp_example.yaml +++ b/exampleConfigs/rt_ppp_example.yaml @@ -1,204 +1,333 @@ inputs: - inputs_root: ./products/ - - atx_files: [igs20.atx] - egm_files: [tables/EGM2008.gfc] - igrf_files: [tables/igrf13coeffs.txt] - erp_files: [finals.data.iau2000.txt] - planetary_ephemeris_files: [tables/DE436.1950.2050] - - troposphere: - gpt2grid_files: [tables/gpt_25.grd] - - tides: - ocean_tide_loading_blq_files: [tables/OLOAD_GO.BLQ] # required if ocean loading is applied - atmos_tide_loading_blq_files: [tables/ALOAD_GO.BLQ] # required if atmospheric tide loading is applied - ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] # required if ocean pole tide loading is applied - ocean_tide_potential_files: [tables/fes2014b_Cnm-Snm.dat] - - snx_files: [tables/igs_satellite_metadata_2203_plus.snx] - - gnss_observations: - gnss_observations_root: "https://:@ntrip.data.gnss.ga.gov.au/" - rtcm_inputs: - - ALIC00AUS0 - - MAW100ATA0 - - DARW00AUS0 - - STR200AUS0 - - satellite_data: - satellite_data_root: "https://:@ntrip.data.gnss.ga.gov.au/" - rtcm_inputs: - ssr_antenna_offset: APC - rtcm_inputs: - - BCEP00BKG0 - - SSRA00BKG0 + inputs_root: ./products/ + + atx_files: [igs20.atx] # required + igrf_files: [tables/igrf14coeffs.txt] + # erp_files: [finals.data.iau2000.txt] + # planetary_ephemeris_files: [tables/DE436.1950.2050] + + troposphere: + gpt2grid_files: [tables/gpt_25.grd] + + tides: + ocean_tide_loading_blq_files: [tables/OLOAD_GO.BLQ] # required if ocean loading is applied + atmos_tide_loading_blq_files: [tables/ALOAD_GO.BLQ] # required if atmospheric tide loading is applied + ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] # required if ocean pole tide loading is applied + + snx_files: + - igs_satellite_metadata.snx + - tables/sat_yaw_bias_rate.snx + - tables/bds_yaw_modes.snx + - tables/qzss_yaw_modes.snx + # - "*_CRD.SNX" + + satellite_data: + satellite_data_root: "https://:@ntrip.data.gnss.ga.gov.au/" + rtcm_inputs: + ssr_antenna_offset: APC + rtcm_inputs: + - BCEP00BKG0 + - SSRA00CNE0 + + gnss_observations: + gnss_observations_root: "https://:@ntrip.data.gnss.ga.gov.au/" + rtcm_inputs: + - ALIC00AUS0 + - MAW100ATA0 + - DARW00AUS0 + - STR200AUS0 outputs: - metadata: - config_description: rt_ppp_example + metadata: + config_description: rt_ppp_example_ + user: your_ntrip_user + pass: your_ntrip_pass + + outputs_root: ./outputs/ - outputs_root: ./outputs/ + trace: + level: 2 + output_receivers: true + output_network: true + receiver_filename: __.TRACE + network_filename: __.TRACE + output_residuals: true + output_residual_chain: true + output_config: true + output_initialised_states: false + output_predicted_states: false - trace: - level: 4 - output_receivers: true - output_network: true - receiver_filename: __.TRACE - network_filename: __.TRACE - output_residuals: true - output_residual_chain: true - output_config: true + gpx: + output: true + filename: __.GPX - gpx: - output: true - filename: __.GPX + pos: + output: true + filename: __.POS satellite_options: - global: - models: - pos: - enable: true - sources: [SSR] - clock: - enable: true - sources: [SSR] - code_bias: - enable: true - undefined_sigma: 3 - phase_bias: - enable: true - undefined_sigma: 3 + global: + error_model: ELEVATION_DEPENDENT # {uniform,elevation_dependent} + code_sigma: 0 # Standard deviation of code measurements + phase_sigma: 0 # Standard deviation of phase measurmeents + models: + pos: + enable: true + sources: [SSR] + clock: + enable: true + sources: [SSR] + phase_bias: + enable: true receiver_options: - global: - elevation_mask: 15 # degrees - error_model: ELEVATION_DEPENDENT # {uniform,elevation_dependent} - code_sigma: 0.3 # Standard deviation of code measurements, m - phase_sigma: 0.003 # Standard deviation of phase measurmeents, m - rec_reference_system: GPS - models: - phase_bias: - enable: true - - ALIC: - receiver_type: "SEPT POLARX5" # (string) - antenna_type: "LEIAR25.R3 NONE" # (string) - apriori_position: [-4052052.8638, 4212835.9618, -2545104.4038] # [floats] - models: - eccentricity: - enable: true - offset: [0.0000, 0.0000, 0.0015] # [floats] - - MAW1: - receiver_type: "SEPT POLARX5" # (string) - antenna_type: "AOAD/M_T AUST" # (string) - apriori_position: [1111287.2209, 2168911.1847, -5874493.6128] # [floats] - models: - eccentricity: - enable: true - offset: [0.0000, 0.0000, 0.0035] # [floats] - - DARW: - receiver_type: "SEPT POLARX5" # (string) - antenna_type: "JAVRINGANT_DM NONE" # (string) - apriori_position: [-4091359.7273, 4684606.3705, -1408578.9291] # [floats] - models: - eccentricity: - enable: true - offset: [0.0000, 0.0000, 0.0000] # [floats] - - STR2: - receiver_type: "TRIMBLE ALLOY" # (string) - antenna_type: "LEIAR25.R3 NONE" # (string) - apriori_position: [-4467075.3642, 2683011.8533, -3667006.8945] # [floats] - models: - eccentricity: - enable: true - offset: [0.0000, 0.0000, 0.0000] # [floats] + global: + elevation_mask: 15 # (degrees) + error_model: ELEVATION_DEPENDENT # {uniform,elevation_dependent} + code_sigma: 0.3 # Standard deviation of code measurements (m) + phase_sigma: 0.003 # Standard deviation of phase measurmeents (m) + clock_codes: [AUTO, AUTO] + zero_dcb_codes: [AUTO, AUTO] + rec_reference_system: GPS + models: + phase_bias: + enable: false + troposphere: # Tropospheric modelling accounts for delays due to refraction of light in water vapour + enable: true + models: [gpt2] # List of models to use for troposphere [standard,sbas,vmf3,gpt2,cssr] + tides: + enable: true # Enable modelling of tidal disaplacements + solid: true # Enable solid Earth tides + otl: true # Enable ocean tide loading + atl: true # Enable atmospheric tide loading + spole: true # Enable solid Earth pole tides + opole: true # Enable ocean pole tides + ionospheric_components: + use_2nd_order: true + use_3rd_order: true + + ALIC: + receiver_type: "SEPT POLARX5" + antenna_type: "TWIVC6050 NONE" + apriori_position: [-4052052.7352, 4212835.9833, -2545104.5853] + models: + eccentricity: + enable: true + offset: [0.0000, 0.0000, 0.0250] + + MAW1: + receiver_type: "SEPT POLARX5" + antenna_type: "AOAD/M_T AUST" + apriori_position: [1111287.1380, 2168911.2970, -5874493.6440] + models: + eccentricity: + enable: true + offset: [0.0000, 0.0000, 0.0035] + + DARW: + receiver_type: "SEPT POLARX5" + antenna_type: "JAVRINGANT_DM NONE" + apriori_position: [-4091359.6055, 4684606.4197, -1408579.1195] + models: + eccentricity: + enable: true + offset: [0.0000, 0.0000, 0.0000] + + STR2: + receiver_type: "TRIMBLE ALLOY" + antenna_type: "LEIAR25.R3 NONE" + apriori_position: [-4467075.2351, 2683011.8470, -3667007.0408] + models: + eccentricity: + enable: true + offset: [0.0000, 0.0000, 0.0000] processing_options: - process_modes: - ppp: true - - epoch_control: - epoch_interval: 20 - max_rec_latency: 1 - - gnss_general: - # use_rtk_combo: true - # common_atmosphere: true - sys_options: - gps: - process: true - reject_eclipse: false - # clock_codes: [ L1W, L2W ] - code_priorities: [L1W, L1C, L2W] - ambiguity_resolution: false + process_modes: + preprocessor: true # Preprocessing and quality checks + spp: true # Perform SPP on receiver data + ppp: true # Perform PPP network or end user mode + ionosphere: false # Compute Ionosphere models based on GNSS measurements + slr: false # Process SLR observations + + epoch_control: + epoch_interval: 10 + max_rec_latency: 1 + + gnss_general: + add_eop_component: true + use_primary_signals: true + sys_options: + gps: + process: true + reject_eclipse: false + code_priorities: [L1W, L1C, L2W, L2S] + # code_priorities: [L1W, L1C, L2W, L2S, L5Q, L5X] + + gal: + process: true + reject_eclipse: false + code_priorities: [L1C, L1X, L5Q, L5X] + + glo: + process: true + reject_eclipse: false + code_priorities: [L1C, L1P, L2C, L2P] + + # phase_measurements: + # process: false + + preprocessor: # Configurations for the kalman filter and its sub processes + cycle_slips: # Cycle slips may be detected by the preprocessor and measurements rejected or ambiguities reinitialised + mw_process_noise: 0 # Process noise applied to filtered Melbourne-Wubenna measurements to detect cycle slips + slip_threshold: 0.05 # Value used to determine when a slip has occurred + preprocess_all_data: true + + spp: + # always_reinitialise: false # Reset SPP state to zero to avoid potential for lock-in of bad states + max_lsq_iterations: 12 # Maximum number of iterations of least squares allowed for convergence + outlier_screening: + chi_square: + enable: true + sigma_threshold: 5 + least_square: + max_iterations: 1 + sigma_check: false + omega_test: true + meas_sigma_threshold: 5 + raim: + enable: true + max_iterations: 2 + max_gdop: 30 # Maximum dilution of precision before error is flagged + + ppp_filter: + outlier_screening: + chi_square: + enable: false # Enable Chi-square test + mode: innovation # Chi-square test mode {none,innovation,measurement,state} + prefit: + max_iterations: 3 # Maximum number of measurements to exclude using prefit checks before attempting to filter + omega_test: false # Enable omega-test + sigma_check: false # Enable sigma check + state_sigma_threshold: 4 # Sigma threshold for states + meas_sigma_threshold: 4 # Sigma threshold for measurements + postfit: + max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter + omega_test: true # Enable omega-test + sigma_check: false # Enable sigma check + state_sigma_threshold: 8 # Sigma threshold for states + meas_sigma_threshold: 6 # Sigma threshold for measurements + + ionospheric_components: # Slant ionospheric components + common_ionosphere: true # Use the same ionosphere state for code and phase observations + use_gf_combo: false # Combine 'uncombined' measurements to simulate a geometry-free solution + use_if_combo: false # Combine 'uncombined' measurements to simulate an ionosphere-free solution + + chunking: + by_receiver: true # Split large filter and measurement matrices blockwise by receiver ID to improve processing speed + size: 0 + + rts: # Rauch-Tung-Striebel (RTS) backwards smoothing + enable: false + lag: -1 + # interval: 86400 + filename: _.rts + + # periodic_reset: + # enable: true + # interval: 86400 + # states: [REC_POS] + + model_error_handling: + meas_deweighting: # Measurements that are outside the expected confidence bounds may be deweighted so that outliers do not contaminate the filtered solution + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all rejected measurement + state_deweighting: # Any "state" errors cause deweighting of all measurements that reference the state + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all referencing measurements + error_accumulation: + enable: true + receiver_error_count_threshold: 0 + receiver_error_epochs_threshold: 0 + state_error_count_threshold: 3 + ambiguities: + phase_reject_limit: 2 # Reset ambiguity after 2 large fractional residuals are found (replaces phase_reject_count:) + reset_on: # Reset ambiguities when slip is detected by the following + gf: true # GF test + lli: true # LLI test + mw: true # MW test + scdia: true # SCDIA test + retrack: true estimation_parameters: - satellites: - global: - clock: - estimated: [true] - sigma: [1000] - process_noise: [10] - phase_bias: - estimated: [true] - sigma: [10] - # process_noise: [-1] - - receivers: - BASE: - pos: - estimated: [false] - clock: - # estimated: [false] - - global: - pos: - estimated: [true] - sigma: [100] - process_noise: [0] - clock: - estimated: [true] - sigma: [1000] - process_noise: [100] - ambiguities: - estimated: [true] - sigma: [1000] - process_noise: [0] - ion_stec: # Ionospheric slant delay - estimated: [true] # Estimate state in kalman filter - sigma: [200] # Apriori sigma values - if zero, will be initialised using least squares - process_noise: [10] # Process noise sigmas - trop: - estimated: [true] - sigma: [0.3] - process_noise: [0.0001] - trop_grads: - estimated: [true] - sigma: [0.03] - process_noise: [1.0E-6] - code_bias: - estimated: [true] # false - sigma: [30] - process_noise: [0] - phase_bias: - # estimated: [true] - sigma: [10] - process_noise: [0] + receivers: + global: + pos: + estimated: [true] + sigma: [100] + # process_noise: [30] + pos_rate: # Velocity + estimated: [false] # [bools] Estimate state in kalman filter + sigma: [0] # [floats] Apriori sigma values + process_noise: [0] # [floats] Process noise sigmas + # process_noise_dt: SECOND # (enum) Time unit for process noise - sqrt_sec, sqrt_day etc + # apriori_val: [0] # [floats] Apriori state values + # mu: [0] # [floats] Desired mean value for gauss markov states + # tau: [-1] # [floats] Correlation times for gauss markov noise, defaults to -1 -> inf (Random Walk) + clock: + estimated: [true] + sigma: [1000] + process_noise: [100] + clock_rate: + estimated: [false] + sigma: [0.005] + process_noise: [1e-4] + ambiguities: + estimated: [true] + sigma: [1000] + process_noise: [0] + outage_limit: [120] + ion_stec: # Ionospheric slant delay + estimated: [true] # Estimate state in kalman filter + sigma: [200] # Apriori sigma values + process_noise: [10] # Process noise sigmas + outage_limit: [120] + sigma_limit: [1000] + trop: + estimated: [true] + sigma: [0.3] + process_noise: [0.0001] + trop_grads: + estimated: [true] + sigma: [0.03] + process_noise: [1.0E-6] + code_bias: + estimated: [true] + sigma: [20] + process_noise: [0] + phase_bias: + estimated: [false] + sigma: [10] + process_noise: [0] + gps: + l5q: + phase_bias: + estimated: [true] + sigma: [10] + process_noise: [0.001] mongo: # Mongo is a database used to store results and intermediate values for later analysis and inter-process communication - enable: primary # Enable and connect to mongo database {none,primary,secondary,both} - primary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to - primary_database: - output_components: primary # Output components of measurements {none,primary,secondary,both} - output_states: primary # Output states {none,primary,secondary,both} - output_measurements: primary # Output measurements and their residuals {none,primary,secondary,both} - output_test_stats: primary # Output test statistics {none,primary,secondary,both} - delete_history: primary # Drop the collection in the database at the beginning of the run to only show fresh data {none,primary,secondary,both} - -debug: - # explain_measurements: true - # instrument: true + enable: primary # Enable and connect to mongo database {none,primary,secondary,both} + primary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to + primary_database: + primary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison + # secondary_database: + # secondary_suffix: "" # Suffix to append to database elements to make distinctions between runs for comparison + # secondary_uri: mongodb://localhost:27017 # Location and port of the mongo database to connect to + # output_config: primary # Output config {none,primary,secondary,both} + output_components: primary # Output components of measurements {none,primary,secondary,both} + output_states: primary # Output states {none,primary,secondary,both} + output_measurements: primary # Output measurements and their residuals {none,primary,secondary,both} + output_test_stats: primary # Output test statistics {none,primary,secondary,both} + delete_history: primary # Drop the collection in the database at the beginning of the run to only show fresh data {none,primary,secondary,both} + output_trace: primary diff --git a/exampleConfigs/rt_rtk_example.yaml b/exampleConfigs/rt_rtk_example.yaml index fcc41f00b..39a2c3b8f 100644 --- a/exampleConfigs/rt_rtk_example.yaml +++ b/exampleConfigs/rt_rtk_example.yaml @@ -1,204 +1,191 @@ inputs: + inputs_root: ./products/ - inputs_root: ./products/ + atx_files: [igs20.atx] + egm_files: [tables/EGM2008.gfc] + igrf_files: [tables/igrf14coeffs.txt] + erp_files: [finals.data.iau2000.txt] + planetary_ephemeris_files: [tables/DE436.1950.2050] - atx_files: [ igs20.atx ] - egm_files: [ tables/EGM2008.gfc ] - igrf_files: [ tables/igrf13coeffs.txt ] - erp_files: [ finals.data.iau2000.txt ] - planetary_ephemeris_files: [ tables/DE436.1950.2050 ] + troposphere: + gpt2grid_files: [tables/gpt_25.grd] - troposphere: - gpt2grid_files: [ tables/gpt_25.grd ] + tides: + ocean_tide_loading_blq_files: [tables/OLOAD_GO.BLQ] + atmos_tide_loading_blq_files: [tables/ALOAD_GO.BLQ] + ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] + ocean_tide_potential_files: [tables/fes2014b_Cnm-Snm.dat] - tides: - ocean_tide_loading_blq_files: [ tables/OLOAD_GO.BLQ ] - atmos_tide_loading_blq_files: [ tables/ALOAD_GO.BLQ ] - ocean_pole_tide_loading_files: [ tables/opoleloadcoefcmcor.txt ] - ocean_tide_potential_files: [ tables/fes2014b_Cnm-Snm.dat ] - - snx_files: + snx_files: - tables/igs_satellite_metadata_2203_plus.snx - IGS1R03SNX_20191950000_07D_07D_CRD.SNX - meta_gather_20210721.snx - gnss_observations: - gnss_observations_root: "https://:@ntrip.data.gnss.ga.gov.au/" - rtcm_inputs: - - NWCS00AUS0 - - NEWE00AUS0 - - - satellite_data: - satellite_data_root: "https://:@ntrip.data.gnss.ga.gov.au/" - rtcm_inputs: - ssr_antenna_offset: APC - rtcm_inputs: - - BCEP00BKG0 + gnss_observations: + gnss_observations_root: "https://:@ntrip.data.gnss.ga.gov.au/" + rtcm_inputs: + - NWCS00AUS0 + - NEWE00AUS0 + satellite_data: + satellite_data_root: "https://:@ntrip.data.gnss.ga.gov.au/" + rtcm_inputs: + ssr_antenna_offset: APC + rtcm_inputs: + - BCEP00BKG0 outputs: - metadata: - config_description: rt_rtk_example + metadata: + config_description: rt_rtk_example - outputs_root: ./outputs/ + outputs_root: ./outputs/ - trace: - level: 6 - output_receivers: true - output_network: true - receiver_filename: __.TRACE - network_filename: __.TRACE - output_residuals: true - output_residual_chain: true - output_config: true + trace: + level: 6 + output_receivers: true + output_network: true + receiver_filename: __.TRACE + network_filename: __.TRACE + output_residuals: true + output_residual_chain: true + output_config: true - gpx: - output: true - filename: __.GPX + gpx: + output: true + filename: __.GPX satellite_options: - - global: - models: - pos: - enable: true - clock: - enable: true - code_bias: - enable: true - undefined_sigma: 3 - phase_bias: - enable: true - undefined_sigma: 3 - + global: + models: + pos: + enable: true + clock: + enable: true + code_bias: + enable: true + undefined_sigma: 3 + phase_bias: + enable: true + undefined_sigma: 3 receiver_options: - - global: - elevation_mask: 15 # degrees - error_model: elevation_dependent # {uniform,elevation_dependent} - code_sigma: 0.3 # Standard deviation of code measurements, m - phase_sigma: 0.003 # Standard deviation of phase measurmeents, m - rec_reference_system: GPS - models: - phase_bias: - enable: true - - NEWE: - apriori_position: [-4721191.0941328, 2535299.6155289,-3447499.6521773] - aliases: [BASE] - + global: + elevation_mask: 15 # degrees + error_model: elevation_dependent # {uniform,elevation_dependent} + code_sigma: 0.3 # Standard deviation of code measurements, m + phase_sigma: 0.003 # Standard deviation of phase measurmeents, m + rec_reference_system: GPS + models: + phase_bias: + enable: true + + NEWE: + apriori_position: [-4721191.0941328, 2535299.6155289, -3447499.6521773] + aliases: [BASE] processing_options: - - process_modes: - ppp: true - - epoch_control: - epoch_interval: 1 - max_rec_latency: 1 - - gnss_general: - # use_rtk_combo: true - equate_tropospheres: true - equate_ionospheres: true - sys_options: - gps: - process: true - reject_eclipse: false - # clock_codes: [ L1W, L2W ] - code_priorities: [ L1C, L2W ] - ambiguity_resolution: true - # network_amb_pivot: true - # receiver_amb_pivot: true - # gal: - # process: true - # code_priorities: [ L1C, L5Q, L1X, L5X ] - # - ambiguity_resolution: - # mode: LAMBDA_bie - success_rate_threshold: 0.999999 - once_per_epoch: true - - - ppp_filter: - ionospheric_components: - common_ionosphere: true + process_modes: + ppp: true + + epoch_control: + epoch_interval: 1 + max_rec_latency: 1 + + gnss_general: + # use_rtk_combo: true + equate_tropospheres: true + equate_ionospheres: true + sys_options: + gps: + process: true + reject_eclipse: false + # clock_codes: [ L1W, L2W ] + code_priorities: [L1C, L2W] + ambiguity_resolution: true + # network_amb_pivot: true + # receiver_amb_pivot: true + # gal: + # process: true + # code_priorities: [ L1C, L5Q, L1X, L5X ] + # + ambiguity_resolution: + # mode: LAMBDA_bie + success_rate_threshold: 0.999999 + once_per_epoch: true + + ppp_filter: + ionospheric_components: + common_ionosphere: true estimation_parameters: + satellites: + global: + clock: + estimated: [true] + sigma: [1000] + process_noise: [-1] + phase_bias: + estimated: [true] + sigma: [1] + # process_noise: [-1] + code_bias: + estimated: [true] + sigma: [100] + # process_noise: [-1] + + receivers: + BASE: + pos: + estimated: [false] + clock: + estimated: [false] + phase_bias: + estimated: [false] - satellites: - global: - clock: - estimated: [true] - sigma: [1000] - process_noise: [-1] - phase_bias: - estimated: [true] - sigma: [1] - # process_noise: [-1] - code_bias: - estimated: [true] - sigma: [100] - # process_noise: [-1] - - - receivers: - BASE: - pos: - estimated: [false] - clock: - estimated: [false] - phase_bias: - estimated: [false] - - global: - pos: - estimated: [true] - sigma: [1000] - process_noise: [0.001] - # process_noise_dt: MINUTE - clock: - estimated: [true] - sigma: [1000] - process_noise: [100] - ambiguities: - estimated: [true] - sigma: [1000] - process_noise: [0] - ion_stec: # Ionospheric slant delay - estimated: [true] # Estimate state in kalman filter - sigma: [200] # Apriori sigma values - if zero, will be initialised using least squares - process_noise: [10] # Process noise sigmas - trop: - estimated: [true] - sigma: [0.3] - process_noise: [0.0001] - trop_grads: - estimated: [true] - sigma: [0.03] - process_noise: [1.0E-6] - code_bias: - estimated: [true] # false - sigma: [30] - process_noise: [0] - phase_bias: - estimated: [true] - sigma: [1] - process_noise: [0] - - -mongo: # Mongo is a database used to store results and intermediate values for later analysis and inter-process communication - - enable: primary # Enable and connect to mongo database {none,primary,secondary,both} - output_components: primary # Output components of measurements {none,primary,secondary,both} - output_states: primary # Output states {none,primary,secondary,both} - output_measurements: primary # Output measurements and their residuals {none,primary,secondary,both} - output_test_stats: primary # Output test statistics {none,primary,secondary,both} - delete_history: primary # Drop the collection in the database at the beginning of the run to only show fresh data {none,primary,secondary,both} - output_trace: primary + global: + pos: + estimated: [true] + sigma: [1000] + process_noise: [0.001] + # process_noise_dt: MINUTE + clock: + estimated: [true] + sigma: [1000] + process_noise: [100] + ambiguities: + estimated: [true] + sigma: [1000] + process_noise: [0] + ion_stec: # Ionospheric slant delay + estimated: [true] # Estimate state in kalman filter + sigma: [200] # Apriori sigma values - if zero, will be initialised using least squares + process_noise: [10] # Process noise sigmas + trop: + estimated: [true] + sigma: [0.3] + process_noise: [0.0001] + trop_grads: + estimated: [true] + sigma: [0.03] + process_noise: [1.0E-6] + code_bias: + estimated: [true] # false + sigma: [30] + process_noise: [0] + phase_bias: + estimated: [true] + sigma: [1] + process_noise: [0] + +mongo: # Mongo is a database used to store results and intermediate values for later analysis and inter-process communication + enable: primary # Enable and connect to mongo database {none,primary,secondary,both} + output_components: primary # Output components of measurements {none,primary,secondary,both} + output_states: primary # Output states {none,primary,secondary,both} + output_measurements: primary # Output measurements and their residuals {none,primary,secondary,both} + output_test_stats: primary # Output test statistics {none,primary,secondary,both} + delete_history: primary # Drop the collection in the database at the beginning of the run to only show fresh data {none,primary,secondary,both} + output_trace: primary debug: - # explain_measurements: true - # instrument: true + # explain_measurements: true + # instrument: true diff --git a/exampleConfigs/rtk_example.yaml b/exampleConfigs/rtk_example.yaml index 00164781b..fd531e74a 100644 --- a/exampleConfigs/rtk_example.yaml +++ b/exampleConfigs/rtk_example.yaml @@ -1,201 +1,197 @@ inputs: + inputs_root: ./products/ - inputs_root: ./products/ + troposphere: + gpt2grid_files: [tables/gpt_25.grd] - troposphere: - gpt2grid_files: [ tables/gpt_25.grd ] - - snx_files: + snx_files: - tables/igs_satellite_metadata_2203_plus.snx - IGS1R03SNX_20191950000_07D_07D_CRD.SNX - meta_gather_20210721.snx - gnss_observations: - rnx_inputs: - - ALEX: - - "/rtk/A*.rnx" - - BRAD: - - "/rtk/B*.rnx" - - CHAZ: - - "/rtk/C*.rnx" - - DAVE: - - "/rtk/D*.rnx" - # - FRAN: - # - "/rtk/F*.rnx" - # - GARY: - # - "/rtk/G*.rnx" - - satellite_data: - sp3_files: - - "/rtk/*.SP3" + gnss_observations: + rnx_inputs: + - ALEX: + - "/rtk/A*.rnx" + - BRAD: + - "/rtk/B*.rnx" + - CHAZ: + - "/rtk/C*.rnx" + - DAVE: + - "/rtk/D*.rnx" + # - FRAN: + # - "/rtk/F*.rnx" + # - GARY: + # - "/rtk/G*.rnx" + + satellite_data: + sp3_files: + - "/rtk/*.SP3" outputs: - metadata: - config_description: rtk_A + metadata: + config_description: rtk_A - outputs_root: ./outputs/ + outputs_root: ./outputs/ - trace: - level: 5 - output_receivers: true - output_network: true - output_residuals: true - output_residual_chain: true - output_initialised_states: true + trace: + level: 5 + output_receivers: true + output_network: true + output_residuals: true + output_residual_chain: true + output_initialised_states: true satellite_options: - - global: - models: - code_bias: - enable: true - pco: - enable: false - pcv: - enable: false + global: + models: + code_bias: + enable: true + pco: + enable: false + pcv: + enable: false receiver_options: - - global: - elevation_mask: 10 - code_sigma: 0.3 - # code_sigma: 3000 - phase_sigma: 0.003 # Standard deviation of phase measurmeents, m - models: - phase_bias: - enable: true - code_bias: - enable: true - tides: - enable: false - pco: - enable: false - pcv: - enable: false - eccentricity: - enable: false - ionospheric_components: - iono_sigma_limit: 10000000 - apriori_position: [-4454625.6083166, 2669830.4257644,-3691275.8084798] - - ALEX: - models: - code_bias: - enable: true - - BRAD: - aliases: [BASE] - FRAN: - # aliases: [BASE] - apriori_position: [ -4454695.3171, 2669791.4467, -3691215.6759 ] - - GARY: - # aliases: [BASE] - apriori_position: [ -4454622.6297, 2669912.5450, -3691219.1597 ] - - BASE: - code_sigma: 0.3 + global: + elevation_mask: 10 + code_sigma: 0.3 + # code_sigma: 3000 + phase_sigma: 0.003 # Standard deviation of phase measurmeents, m + models: + phase_bias: + enable: true + code_bias: + enable: true + tides: + enable: false + pco: + enable: false + pcv: + enable: false + eccentricity: + enable: false + ionospheric_components: + iono_sigma_limit: 10000000 + apriori_position: [-4454625.6083166, 2669830.4257644, -3691275.8084798] + + ALEX: + models: + code_bias: + enable: true + + BRAD: + aliases: [BASE] + FRAN: + # aliases: [BASE] + apriori_position: [-4454695.3171, 2669791.4467, -3691215.6759] + + GARY: + # aliases: [BASE] + apriori_position: [-4454622.6297, 2669912.5450, -3691219.1597] + + BASE: + code_sigma: 0.3 processing_options: + process_modes: + ppp: true - process_modes: - ppp: true - - epoch_control: - epoch_interval: 1 + epoch_control: + epoch_interval: 1 - gnss_general: - phase_measurements: - process: false + gnss_general: + phase_measurements: + process: false - # equate_tropospheres: true - # equate_ionospheres: true - # use_rtk_combo: true - sys_options: - gps: - process: true - code_priorities: [ L1C, L2W ] + # equate_tropospheres: true + # equate_ionospheres: true + # use_rtk_combo: true + sys_options: + gps: + process: true + code_priorities: [L1C, L2W] - ppp_filter: - ionospheric_components: - common_ionosphere: false - # use_if_combo: true + ppp_filter: + ionospheric_components: + common_ionosphere: false + # use_if_combo: true - advanced_postfits: true + advanced_postfits: true estimation_parameters: - satellites: - global: - clock: - estimated: [true] - sigma: [10000] - process_noise: [10000] - # process_noise: [-1] - phase_bias: - estimated: [true] - sigma: [1] - # process_noise: [-1] - code_bias: - estimated: [true] - sigma: [10000] - process_noise: [1] - - receivers: - BASE: - phase_bias: - # estimated: [false] - pos: - estimated: [false] - code_bias: - estimated: [false] - ALEX: - code_bias: - apriori_value: [20] - CHAZ: - code_bias: - # apriori_value: [-0.9] - DAVE: - code_bias: - # apriori_value: [0] - global: - pos: - estimated: [true] - sigma: [1000] - process_noise: [10] - clock: - estimated: [true] - sigma: [1000] - process_noise: [10000] - ion_stec: - estimated: [true] - sigma: [2000] - process_noise: [-1] - trop: - estimated: [true] - sigma: [4] - process_noise: [0.001] - phase_bias: - estimated: [true] - sigma: [100] - process_noise: [0] - ambiguities: - estimated: [true] - sigma: [100000] - process_noise: [0] - code_bias: - # estimated: [false, false, true] - estimated: [true] - sigma: [10000] - process_noise: [0] - # apriori_value: [1.5, 1.5, -1.5] + satellites: + global: + clock: + estimated: [true] + sigma: [10000] + process_noise: [10000] + # process_noise: [-1] + phase_bias: + estimated: [true] + sigma: [1] + # process_noise: [-1] + code_bias: + estimated: [true] + sigma: [10000] + process_noise: [1] + + receivers: + BASE: + phase_bias: + # estimated: [false] + pos: + estimated: [false] + code_bias: + estimated: [false] + ALEX: + code_bias: + apriori_value: [20] + CHAZ: + code_bias: + # apriori_value: [-0.9] + DAVE: + code_bias: + # apriori_value: [0] + global: + pos: + estimated: [true] + sigma: [1000] + process_noise: [10] + clock: + estimated: [true] + sigma: [1000] + process_noise: [10000] + ion_stec: + estimated: [true] + sigma: [2000] + process_noise: [-1] + trop: + estimated: [true] + sigma: [4] + process_noise: [0.001] + phase_bias: + estimated: [true] + sigma: [100] + process_noise: [0] + ambiguities: + estimated: [true] + sigma: [100000] + process_noise: [0] + code_bias: + # estimated: [false, false, true] + estimated: [true] + sigma: [10000] + process_noise: [0] + # apriori_value: [1.5, 1.5, -1.5] mongo: - enable: primary - output_components: primary - output_cumulative: primary - output_states: primary - output_measurements: primary - delete_history: primary + enable: primary + output_components: primary + output_cumulative: primary + output_states: primary + output_measurements: primary + delete_history: primary debug: - explain_measurements: true + explain_measurements: true diff --git a/exampleConfigs/slr_pod_with_pseudoobs_gal.yaml b/exampleConfigs/slr_pod_with_pseudoobs_gal.yaml index d7d45ecce..5ec3a79ea 100644 --- a/exampleConfigs/slr_pod_with_pseudoobs_gal.yaml +++ b/exampleConfigs/slr_pod_with_pseudoobs_gal.yaml @@ -1,239 +1,240 @@ inputs: - - include_yamls: [ products/boxwing.yaml ] # required if using boxwing model - - inputs_root: products/ - - snx_files: [ slr/meta/ecc_une.snx, # SLR station eccentricities - slr/ILRS_Data_Handling_File_2024.02.13.snx, # SLR station biases - slr/meta/ITRF2014-ILRS-TRF-SSC.SNX, # SLR station positions + drifts - tables/igs_satellite_metadata_2203_plus.snx, - tables/sat_yaw_bias_rate.snx ] - erp_files: [ tables/EOP_14_C04_IAU2000A_one_file_1962-now.txt ] - egm_files: [ tables/goco05s.gfc ] # Earth gravity model coefficients file - planetary_ephemeris_files: [ tables/DE436.1950.2050 ] # JPL planetary and lunar ephemerides file - - satellite_data: - sp3_files: [ IGS2R03FIN_20191950000_01D_05M_ORB.SP3, - IGS2R03FIN_20191960000_01D_05M_ORB.SP3, - IGS2R03FIN_20191970000_01D_05M_ORB.SP3, - IGS2R03FIN_20191980000_01D_05M_ORB.SP3, - IGS2R03FIN_20191990000_01D_05M_ORB.SP3, - IGS2R03FIN_20192000000_01D_05M_ORB.SP3, - IGS2R03FIN_20192010000_01D_05M_ORB.SP3 ] - sid_files: [ slr/meta/sp3c-satlist.txt ] - com_files: [ slr/com/com_lageos.txt ] - crd_files: [ slr/obs/galileo/galileo101_201907.npt ] - - tides: - ocean_tide_loading_blq_files: [ slr/meta/OLOAD_SLR.BLQ ] # required if ocean loading is applied - atmos_tide_loading_blq_files: [ slr/meta/ALOAD_SLR.BLQ ] - ocean_pole_tide_loading_files: [ tables/opoleloadcoefcmcor.txt ] - ocean_tide_potential_files: [ tables/fes2014b_Cnm-Snm.dat ] - - pseudo_observations: - sp3_inputs: [ IGS2R03FIN_20191950000_01D_05M_ORB.SP3, - IGS2R03FIN_20191960000_01D_05M_ORB.SP3, - IGS2R03FIN_20191970000_01D_05M_ORB.SP3, - IGS2R03FIN_20191980000_01D_05M_ORB.SP3, - IGS2R03FIN_20191990000_01D_05M_ORB.SP3, - IGS2R03FIN_20192000000_01D_05M_ORB.SP3, - IGS2R03FIN_20192010000_01D_05M_ORB.SP3 ] - eci_pseudoobs: false + include_yamls: [products/boxwing.yaml] # required if using boxwing model + + inputs_root: products/ + + snx_files: [ + slr/meta/ecc_une.snx, # SLR station eccentricities + slr/ILRS_Data_Handling_File_2024.02.13.snx, # SLR station biases + slr/meta/ITRF2014-ILRS-TRF-SSC.SNX, # SLR station positions + drifts + tables/igs_satellite_metadata_2203_plus.snx, + tables/sat_yaw_bias_rate.snx, + ] + erp_files: [tables/EOP_14_C04_IAU2000A_one_file_1962-now.txt] + egm_files: [tables/goco05s.gfc] # Earth gravity model coefficients file + planetary_ephemeris_files: [tables/DE436.1950.2050] # JPL planetary and lunar ephemerides file + + satellite_data: + sp3_files: + [ + IGS2R03FIN_20191950000_01D_05M_ORB.SP3, + IGS2R03FIN_20191960000_01D_05M_ORB.SP3, + IGS2R03FIN_20191970000_01D_05M_ORB.SP3, + IGS2R03FIN_20191980000_01D_05M_ORB.SP3, + IGS2R03FIN_20191990000_01D_05M_ORB.SP3, + IGS2R03FIN_20192000000_01D_05M_ORB.SP3, + IGS2R03FIN_20192010000_01D_05M_ORB.SP3, + ] + sid_files: [slr/meta/sp3c-satlist.txt] + com_files: [slr/com/com_lageos.txt] + crd_files: [slr/obs/galileo/galileo101_201907.npt] + + tides: + ocean_tide_loading_blq_files: [slr/meta/OLOAD_SLR.BLQ] # required if ocean loading is applied + atmos_tide_loading_blq_files: [slr/meta/ALOAD_SLR.BLQ] + ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] + ocean_tide_potential_files: [tables/fes2014b_Cnm-Snm.dat] + + pseudo_observations: + sp3_inputs: + [ + IGS2R03FIN_20191950000_01D_05M_ORB.SP3, + IGS2R03FIN_20191960000_01D_05M_ORB.SP3, + IGS2R03FIN_20191970000_01D_05M_ORB.SP3, + IGS2R03FIN_20191980000_01D_05M_ORB.SP3, + IGS2R03FIN_20191990000_01D_05M_ORB.SP3, + IGS2R03FIN_20192000000_01D_05M_ORB.SP3, + IGS2R03FIN_20192010000_01D_05M_ORB.SP3, + ] + eci_pseudoobs: false outputs: - - metadata: - config_description: slr_pod_with_pseudoobs_gal - - outputs_root: outputs// - colourise_terminal: true - - trace: - output_receivers: true - output_network: true - level: 2 - receiver_filename: --.TRACE - network_filename: --.TRACE - output_residuals: true - output_residual_chain: true - output_config: true - - log: - output: true - directory: ./ - filename: log_.json - - output_rotation: - period: 1 - period_units: day - - sinex: - output: false - - erp: - output: false - - orbit_ics: - output: true - directory: ./orbit_ics/ - filename: __orbits.yaml - - sp3: - output: true - output_interval: 1 - output_inertial: false - output_velocities: true - orbit_sources: [KALMAN] - clock_sources: [PRECISE] - - slr_obs: - output: true - directory: ./slr_obs/ - filename: .slr_obs + metadata: + config_description: slr_pod_with_pseudoobs_gal + + outputs_root: outputs// + colourise_terminal: true + + trace: + output_receivers: true + output_network: true + level: 2 + receiver_filename: --.TRACE + network_filename: --.TRACE + output_residuals: true + output_residual_chain: true + output_config: true + + log: + output: true + directory: ./ + filename: log_.json + + output_rotation: + period: 1 + period_units: day + + sinex: + output: false + + erp: + output: false + + orbit_ics: + output: true + directory: ./orbit_ics/ + filename: __orbits.yaml + + sp3: + output: true + output_interval: 1 + output_inertial: false + output_velocities: true + orbit_sources: [KALMAN] + clock_sources: [PRECISE] + + slr_obs: + output: true + directory: ./slr_obs/ + filename: .slr_obs mongo: - - enable: primary - primary_database: - output_config: primary - output_measurements: primary - output_states: primary - output_test_stats: primary - delete_history: primary - primary_uri: mongodb://127.0.0.1:27017 - primary_suffix: "" + enable: primary + primary_database: + output_config: primary + output_measurements: primary + output_states: primary + output_test_stats: primary + delete_history: primary + primary_uri: mongodb://127.0.0.1:27017 + primary_suffix: "" satellite_options: + global: + pseudo_sigma: 1 - global: - pseudo_sigma: 1 - - orbit_propagation: - mass: 1000 - area: 15 - srp_cr: 1.75 - power: 20 - planetary_perturbations: [sun, moon, jupiter] - solar_radiation_pressure: boxwing - antenna_thrust: true - albedo: cannonball - empirical: true - empirical_dyb_eclipse: [true, false, false] - pseudo_pulses: - enable: true - - models: - pos: - enable: true - sources: [KALMAN, PRECISE, BROADCAST] - attitude: - enable: true - sources: [MODEL, PRECISE, NOMINAL] + orbit_propagation: + mass: 1000 + area: 15 + srp_cr: 1.75 + power: 20 + planetary_perturbations: [sun, moon, jupiter] + solar_radiation_pressure: boxwing + antenna_thrust: true + albedo: cannonball + empirical: true + empirical_dyb_eclipse: [true, false, false] + pseudo_pulses: + enable: true + + models: + pos: + enable: true + sources: [KALMAN, PRECISE, BROADCAST] + attitude: + enable: true + sources: [MODEL, PRECISE, NOMINAL] receiver_options: + global: + elevation_mask: 10 # degrees + error_model: elevation_dependent + laser_sigma: 0.10 - global: - elevation_mask: 10 # degrees - error_model: elevation_dependent - laser_sigma: 0.10 - - models: - eop: - enable: true + models: + eop: + enable: true processing_options: - - epoch_control: - start_epoch: 2019-07-17 00:00:00 - end_epoch: 2019-07-19 23:55:00 - epoch_interval: 60 # seconds - require_obs: true - assign_closest_epoch: true - - process_modes: - ppp: true - slr: true # Process SLR observations - preprocessor: true - spp: false - - gnss_general: - require_apriori_positions: true - require_site_eccentricity: true - require_reflector_com: true - - sys_options: - gal: - process: true - - ppp_filter: - inverter: ldlt # LLT LDLT INV - - outlier_screening: - prefit: - max_iterations: 10 # Maximum number of measurements to exclude using prefit checks before attempting to filter - - postfit: - max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter - - rts: - enable: true - - orbit_propagation: - integrator_time_step: 900 - central_force: true - egm_field: true - egm_degree: 15 - indirect_J2: true - general_relativity: true - solid_earth_tide: true - ocean_tide: true - atm_tide: true - pole_tide_ocean: true - pole_tide_solid: true - - model_error_handling: - meas_deweighting: - deweight_factor: 1000 + epoch_control: + start_epoch: 2019-07-17 00:00:00 + end_epoch: 2019-07-19 23:55:00 + epoch_interval: 60 # seconds + require_obs: true + assign_closest_epoch: true + + process_modes: + ppp: true + slr: true # Process SLR observations + preprocessor: true + spp: false + + gnss_general: + require_apriori_positions: true + require_site_eccentricity: true + require_reflector_com: true + + sys_options: + gal: + process: true + + ppp_filter: + inverter: ldlt # LLT LDLT INV + + outlier_screening: + prefit: + max_iterations: 10 # Maximum number of measurements to exclude using prefit checks before attempting to filter + + postfit: + max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter + + rts: + enable: true + + orbit_propagation: + integrator_time_step: 900 + central_force: true + egm_field: true + egm_degree: 15 + indirect_J2: true + general_relativity: true + solid_earth_tide: true + ocean_tide: true + atm_tide: true + pole_tide_ocean: true + pole_tide_solid: true + + model_error_handling: + meas_deweighting: + deweight_factor: 1000 estimation_parameters: + global_models: + eop: + estimated: [true] + sigma: [10] - global_models: - eop: - estimated: [true] - sigma: [10] - - eop_rates: - estimated: [true] - sigma: [10] + eop_rates: + estimated: [true] + sigma: [10] - receivers: - global: - pos: - estimated: [false] - sigma: [1.0] + receivers: + global: + pos: + estimated: [false] + sigma: [1.0] - slr_range_bias: - estimated: [false] - sigma: [0.01] + slr_range_bias: + estimated: [false] + sigma: [0.01] - slr_time_bias: - estimated: [false] - sigma: [0.00001] + slr_time_bias: + estimated: [false] + sigma: [0.00001] - satellites: - global: - orbit: - estimated: [true] - sigma: [1] # posX/Y/Z, velX/Y/Z (final element repeated as necessary) + satellites: + global: + orbit: + estimated: [true] + sigma: [1] # posX/Y/Z, velX/Y/Z (final element repeated as necessary) - emp_d_0: { estimated: [true], sigma: [1e3] } - emp_y_0: { estimated: [true], sigma: [1e3] } - emp_b_0: { estimated: [true], sigma: [1e3] } + emp_d_0: { estimated: [true], sigma: [1e3] } + emp_y_0: { estimated: [true], sigma: [1e3] } + emp_b_0: { estimated: [true], sigma: [1e3] } - emp_d_1: { estimated: [true], sigma: [1e3] } - emp_b_1: { estimated: [true], sigma: [1e3] } + emp_d_1: { estimated: [true], sigma: [1e3] } + emp_b_1: { estimated: [true], sigma: [1e3] } - emp_d_2: { estimated: [true], sigma: [1e3] } + emp_d_2: { estimated: [true], sigma: [1e3] } - emp_d_4: { estimated: [true], sigma: [1e3] } + emp_d_4: { estimated: [true], sigma: [1e3] } diff --git a/exampleConfigs/slr_pod_with_pseudoobs_lag.yaml b/exampleConfigs/slr_pod_with_pseudoobs_lag.yaml index 1a3e35244..72dd502d6 100644 --- a/exampleConfigs/slr_pod_with_pseudoobs_lag.yaml +++ b/exampleConfigs/slr_pod_with_pseudoobs_lag.yaml @@ -1,211 +1,206 @@ inputs: - - inputs_root: products/ - - snx_files: [ slr/meta/ecc_une.snx, # SLR station eccentricities - slr/ILRS_Data_Handling_File_2024.02.13.snx, # SLR station biases - slr/meta/ITRF2014-ILRS-TRF-SSC.SNX ] # SLR station positions + drifts - erp_files: [ tables/EOP_14_C04_IAU2000A_one_file_1962-now.txt ] - egm_files: [ tables/goco05s.gfc ] # Earth gravity model coefficients file - planetary_ephemeris_files: [ tables/DE436.1950.2050 ] # JPL planetary and lunar ephemerides file - - satellite_data: - sp3_files: [ slr/orbits/lageos1/ilrsa.orb.lageos1.190720.v71.sp3 ] - sid_files: [ slr/meta/sp3c-satlist.txt ] - com_files: [ slr/com/com_lageos.txt ] - crd_files: [ slr/obs/lageos1/lageos1_201907.npt ] - - tides: - ocean_tide_loading_blq_files: [ slr/meta/OLOAD_SLR.BLQ ] # required if ocean loading is applied - atmos_tide_loading_blq_files: [ slr/meta/ALOAD_SLR.BLQ ] - ocean_pole_tide_loading_files: [ tables/opoleloadcoefcmcor.txt ] - ocean_tide_potential_files: [ tables/fes2014b_Cnm-Snm.dat ] - - pseudo_observations: - sp3_inputs: [ slr/orbits/lageos1/ilrsa.orb.lageos1.190720.v71.sp3 ] - eci_pseudoobs: false + inputs_root: products/ + + snx_files: [ + slr/meta/ecc_une.snx, # SLR station eccentricities + slr/ILRS_Data_Handling_File_2024.02.13.snx, # SLR station biases + slr/meta/ITRF2014-ILRS-TRF-SSC.SNX, + ] # SLR station positions + drifts + erp_files: [tables/EOP_14_C04_IAU2000A_one_file_1962-now.txt] + egm_files: [tables/goco05s.gfc] # Earth gravity model coefficients file + planetary_ephemeris_files: [tables/DE436.1950.2050] # JPL planetary and lunar ephemerides file + + satellite_data: + sp3_files: [slr/orbits/lageos1/ilrsa.orb.lageos1.190720.v71.sp3] + sid_files: [slr/meta/sp3c-satlist.txt] + com_files: [slr/com/com_lageos.txt] + crd_files: [slr/obs/lageos1/lageos1_201907.npt] + + tides: + ocean_tide_loading_blq_files: [slr/meta/OLOAD_SLR.BLQ] # required if ocean loading is applied + atmos_tide_loading_blq_files: [slr/meta/ALOAD_SLR.BLQ] + ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] + ocean_tide_potential_files: [tables/fes2014b_Cnm-Snm.dat] + + pseudo_observations: + sp3_inputs: [slr/orbits/lageos1/ilrsa.orb.lageos1.190720.v71.sp3] + eci_pseudoobs: false outputs: - - metadata: - config_description: slr_pod_with_pseudoobs_lag - - outputs_root: outputs// - colourise_terminal: true - - trace: - output_receivers: true - output_network: true - level: 2 - receiver_filename: --.TRACE - network_filename: --.TRACE - output_residuals: true - output_residual_chain: true - output_config: true - - log: - output: true - directory: ./ - filename: log_.json - - output_rotation: - period: 1 - period_units: day - - sinex: - output: false - - erp: - output: false - - orbit_ics: - output: true - directory: ./orbit_ics/ - filename: __orbits.yaml - - sp3: - output: true - output_interval: 1 - output_inertial: false - output_velocities: true - orbit_sources: [KALMAN] - clock_sources: [PRECISE] - - slr_obs: - output: true - directory: ./slr_obs/ - filename: .slr_obs + metadata: + config_description: slr_pod_with_pseudoobs_lag + + outputs_root: outputs// + colourise_terminal: true + + trace: + output_receivers: true + output_network: true + level: 2 + receiver_filename: --.TRACE + network_filename: --.TRACE + output_residuals: true + output_residual_chain: true + output_config: true + + log: + output: true + directory: ./ + filename: log_.json + + output_rotation: + period: 1 + period_units: day + + sinex: + output: false + + erp: + output: false + + orbit_ics: + output: true + directory: ./orbit_ics/ + filename: __orbits.yaml + + sp3: + output: true + output_interval: 1 + output_inertial: false + output_velocities: true + orbit_sources: [KALMAN] + clock_sources: [PRECISE] + + slr_obs: + output: true + directory: ./slr_obs/ + filename: .slr_obs mongo: - - enable: primary - primary_database: - output_config: primary - output_measurements: primary - output_states: primary - output_test_stats: primary - delete_history: primary - primary_uri: mongodb://127.0.0.1:27017 - primary_suffix: "" + enable: primary + primary_database: + output_config: primary + output_measurements: primary + output_states: primary + output_test_stats: primary + delete_history: primary + primary_uri: mongodb://127.0.0.1:27017 + primary_suffix: "" satellite_options: + global: + pseudo_sigma: 1 - global: - pseudo_sigma: 1 - - orbit_propagation: - mass: 400 - area: 0.28 - srp_cr: 1.75 - planetary_perturbations: [sun, moon, jupiter] - solar_radiation_pressure: cannonball - antenna_thrust: false - albedo: cannonball - empirical: true - empirical_rtn_eclipse: [false, false, false] - - models: - pos: - enable: true - sources: [KALMAN, PRECISE, BROADCAST] + orbit_propagation: + mass: 400 + area: 0.28 + srp_cr: 1.75 + planetary_perturbations: [sun, moon, jupiter] + solar_radiation_pressure: cannonball + antenna_thrust: false + albedo: cannonball + empirical: true + empirical_rtn_eclipse: [false, false, false] + + models: + pos: + enable: true + sources: [KALMAN, PRECISE, BROADCAST] receiver_options: + global: + elevation_mask: 10 # degrees + error_model: elevation_dependent + laser_sigma: 0.10 - global: - elevation_mask: 10 # degrees - error_model: elevation_dependent - laser_sigma: 0.10 - - models: - eop: - enable: true + models: + eop: + enable: true processing_options: - - epoch_control: - start_epoch: 2019-07-14 00:00:18 - end_epoch: 2019-07-20 23:58:18 - epoch_interval: 60 # seconds - require_obs: true - assign_closest_epoch: true - - process_modes: - ppp: true - slr: true # Process SLR observations - preprocessor: true - spp: false - - gnss_general: - require_apriori_positions: true - require_site_eccentricity: true - require_reflector_com: true - - sys_options: - leo: - process: true # includes Lageos1 - - ppp_filter: - inverter: ldlt # LLT LDLT INV - - outlier_screening: - prefit: - max_iterations: 10 # Maximum number of measurements to exclude using prefit checks before attempting to filter - - postfit: - max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter - - rts: - enable: true - - orbit_propagation: - integrator_time_step: 60 - central_force: true - egm_field: true - egm_degree: 60 - indirect_J2: true - general_relativity: true - solid_earth_tide: true - ocean_tide: true - atm_tide: true - pole_tide_ocean: true - pole_tide_solid: true - - model_error_handling: - meas_deweighting: - deweight_factor: 1000 + epoch_control: + start_epoch: 2019-07-14 00:00:18 + end_epoch: 2019-07-20 23:58:18 + epoch_interval: 60 # seconds + require_obs: true + assign_closest_epoch: true + + process_modes: + ppp: true + slr: true # Process SLR observations + preprocessor: true + spp: false + + gnss_general: + require_apriori_positions: true + require_site_eccentricity: true + require_reflector_com: true + + sys_options: + leo: + process: true # includes Lageos1 + + ppp_filter: + inverter: ldlt # LLT LDLT INV + + outlier_screening: + prefit: + max_iterations: 10 # Maximum number of measurements to exclude using prefit checks before attempting to filter + + postfit: + max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter + + rts: + enable: true + + orbit_propagation: + integrator_time_step: 60 + central_force: true + egm_field: true + egm_degree: 60 + indirect_J2: true + general_relativity: true + solid_earth_tide: true + ocean_tide: true + atm_tide: true + pole_tide_ocean: true + pole_tide_solid: true + + model_error_handling: + meas_deweighting: + deweight_factor: 1000 estimation_parameters: + global_models: + eop: + estimated: [false] + sigma: [10] - global_models: - eop: - estimated: [false] - sigma: [10] + eop_rates: + estimated: [false] + sigma: [10] - eop_rates: - estimated: [false] - sigma: [10] - - receivers: - global: - pos: - estimated: [false] - sigma: [1.0] + receivers: + global: + pos: + estimated: [false] + sigma: [1.0] - slr_range_bias: - estimated: [false] - sigma: [0.01] + slr_range_bias: + estimated: [false] + sigma: [0.01] - slr_time_bias: - estimated: [false] - sigma: [0.00001] + slr_time_bias: + estimated: [false] + sigma: [0.00001] - satellites: - global: - orbit: - estimated: [true] - sigma: [1] # posX/Y/Z, velX/Y/Z (final element repeated as necessary) + satellites: + global: + orbit: + estimated: [true] + sigma: [1] # posX/Y/Z, velX/Y/Z (final element repeated as necessary) - emp_t_0: { estimated: [true], sigma: [1e3] } + emp_t_0: { estimated: [true], sigma: [1e3] } - emp_t_1: { estimated: [true], sigma: [1e3] } - emp_n_1: { estimated: [true], sigma: [1e3] } + emp_t_1: { estimated: [true], sigma: [1e3] } + emp_n_1: { estimated: [true], sigma: [1e3] } diff --git a/ginan.code-workspace b/ginan.code-workspace new file mode 100644 index 000000000..876a1499c --- /dev/null +++ b/ginan.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/inputData/products/products.list b/inputData/products/products.list index b5faca813..5864fe0de 100644 --- a/inputData/products/products.list +++ b/inputData/products/products.list @@ -52,13 +52,18 @@ ./IGS_Orbit_Test_sp3/tug_eci_not_nst_esocSettings.sp3 ./IGS_Orbit_Test_sp3/tug_eci_wot.sp3 ./IGS_Orbit_Test_sp3/tug_eci_wot_esocSettings.sp3 -./LEO/SCA1B_2022-01-01_C_04.OBXa -./LEO/cut00010.22na -./LEO/igs21906.clk_30sa -./LEO/igs21906.sp3a -./LEO/igs21907.erpa -./LEO/leoAtt_2023.001.001.01.OBXa -./LEO/leoAtt_2023.001.124.11.OBXa +./LEO/2019_02/GPS1B_2019-02-14_C_04.rnx.gz +./LEO/2019_02/COD0R03FIN_20190450000_01D_01D_OSB.BIA +./LEO/2019_02/COD0R03FIN_20190450000_01D_05M_ORB.SP3 +./LEO/2019_02/GNV1B_2019-02-14_C_04.txt +./LEO/2019_02/GNI1B_2019-02-14_C_04.sp3 +./LEO/2019_02/SCA1B_2019-02-14_C_04.OBX +./LEO/2019_02/GPS1B_2019-02-14_C_04.rnx +./LEO/2019_02/COD0R03FIN_20190450000_01D_05S_CLK.CLK +./LEO/AOD1B_2019-02-19_X_06.asc +./LEO/AOD1B_2019-02-20_X_06.asc +./LEO/AOD1B_2019-02-18_X_06.asc +./LEO.snx ./M20.ATX ./NGS0R03FIN_20191990000_01D_01D_ERP.ERP ./NGS0R03FIN_20191990000_01D_15M_ORB.SP3 @@ -286,7 +291,6 @@ ./tables/boxwing.yaml ./tables/DE405.1950.2050 ./tables/DE436.1950.2050 -./tables/EGM2008.gfc ./tables/EOP_14_C04_IAU2000A_one_file_1962-now.txt ./tables/OLOAD_GO.BLQ ./tables/ascp1950.430 @@ -296,13 +300,18 @@ ./tables/goco05s.gfc ./tables/gpt_25.grd ./tables/header.430_229 -./tables/igrf13coeffs.txt +./tables/igrf14coeffs.txt ./tables/igs_metadata_2063.snx ./tables/igs_metadata_2081.snx ./tables/igs_metadata_2081_GA.snx ./tables/igs_satellite_metadata_2203_plus.snx ./tables/igs_satellite_metadata_2219.snx ./tables/leap.second +./tables/EGM2008.gfc +./tables/atmosTide_AOD1bRL06.potential.iers.txt +./tables/desai_model_jgrb51665-sup-0002-ds01.txt +./tables/finals.data.iau2000.txt +./tables/oceanPoleTide_desai2004.txt ./tables/opoleloadcoefcmcor.txt ./tables/qzss_ohi/ohi-qzs1.txt ./tables/qzss_ohi/ohi-qzs1r.txt @@ -311,5 +320,4 @@ ./tables/qzss_ohi/ohi-qzs4.txt ./tables/qzss_yaw_modes.snx ./tables/sat_yaw_bias_rate.snx -./tables/satinfo.dat -./test.sh \ No newline at end of file +./tables/satinfo.dat \ No newline at end of file diff --git a/scripts/GinanEDA/backend/data/measurements.py b/scripts/GinanEDA/backend/data/measurements.py index 937b7cbae..d96a60f3c 100644 --- a/scripts/GinanEDA/backend/data/measurements.py +++ b/scripts/GinanEDA/backend/data/measurements.py @@ -25,6 +25,8 @@ ValueError: Raised when the input dictionary to the `Measurements` class constructor does not contain any data. """ + +import copy import logging import concurrent.futures @@ -85,6 +87,7 @@ def __init__( self.info = {} self.subset = slice(None, None, None) self.gaps = [] + self._yaxis = [] @classmethod def from_dictionary( @@ -205,21 +208,26 @@ def __sub__(self, other): ] ) raise ValueError(f"differencing Apples with oranges: {diffs}") - self_keys = set(self.data.keys()) other_keys = set(other.data.keys()) - common_keys = self_keys & other_keys - missing_keys = (self_keys | other_keys) - common_keys + common_keys = set(self._yaxis) if self._yaxis else self_keys.intersection(other_keys) + missing_keys_self = self_keys - common_keys + missing_keys_other = other_keys - common_keys + print(f"Common keys: {common_keys}") + print(f"Missing keys: {missing_keys_self}") + print(f"Missing keys: {missing_keys_other}") if len(common_keys) == 0: raise ValueError("Warning: no common keys found between dictionaries") - results = self + results = copy.deepcopy(self) _common, in_self, in_t = np.intersect1d(self.epoch, other.epoch, return_indices=True) results.epoch = self.epoch[in_self] results.data = {key: self.data[key][in_self] - other.data[key][in_t] for key in common_keys} + results.data.update({key: self.data[key][in_self] for key in missing_keys_self}) + # results.data.update({key: other.data[key][in_t] for key in missing_keys_other}) - if len(missing_keys) > 0: + if len(missing_keys_self) + len(missing_keys_other) > 0: logger.warning("Warning: keys not present in both dictionaries:") logger.warning(f"Present in self.data only: {sorted(self_keys - other_keys)}") logger.warning(f"Present in other.data only: {sorted(other_keys - self_keys)}") @@ -338,14 +346,13 @@ def select_range(self, tmin: int = None, tmax: int = None) -> None: if tmin is None: first_index = 0 else: - first_index = np.searchsorted(self.epoch, tmin, side='left') + first_index = np.searchsorted(self.epoch, tmin, side="left") if tmax is None: last_index = len(self.epoch) else: - last_index = np.searchsorted(self.epoch, tmax, side='right') + last_index = np.searchsorted(self.epoch, tmax, side="right") self.subset = slice(first_index, last_index) - def trim(self) -> None: """ trim the data to remove the nan at the extremities @@ -378,6 +385,7 @@ def __init__(self) -> None: self.arr = [] self.tmin = None self.tmax = None + self.yaxis = [] self.difference_check = False def __iter__(self): @@ -433,6 +441,16 @@ def from_mongolist(cls, data_lst: list) -> "MeasurementArray": logger.info("skipping this one") return temporary_loader + @property + def yaxis(self) -> list: + return self._yaxis + + @yaxis.setter + def yaxis(self, yaxis: list) -> None: + self._yaxis = yaxis + for data in self.arr: + data._yaxis = yaxis + def find_minmax(self): """ find_minmax determine the minimum and maximum time of all series in the array @@ -502,7 +520,7 @@ def merge(self, other) -> None: if _data.id["sat"] == _other.id["sat"] and _data.id["site"] == _other.id["site"]: common_time = np.union1d(_data.epoch, _other.epoch) data = {} - for key in set(_data.data) | set(_other.data): + for key in set(_data.data).union(set(_other.data)): data[key] = np.full_like(common_time, np.nan, dtype="float64") for name, val in _data.data.items(): mask = ~np.isnan(val) diff --git a/scripts/GinanEDA/backend/dbconnector/mongo.py b/scripts/GinanEDA/backend/dbconnector/mongo.py index 48a903c33..91b989b35 100644 --- a/scripts/GinanEDA/backend/dbconnector/mongo.py +++ b/scripts/GinanEDA/backend/dbconnector/mongo.py @@ -126,7 +126,7 @@ def get_data( "$push": {"$cond": [{"$eq": [{"$ifNull": [f"${key}", None]}, None]}, float("nan"), f"${key}"]} } logger.info(agg_pipeline) - cursor = self.mongo_client[self.mongo_db][collection].aggregate(agg_pipeline) + cursor = self.mongo_client[self.mongo_db][collection].aggregate(agg_pipeline, allowDiskUse=True) # check if cursor is empty if not cursor.alive: raise ValueError("No data found") @@ -205,7 +205,8 @@ def get_arbitrary(self, collection, match_thing, group_thing, yvalue): "_id": groupObj, "Epoch": {"$first": "$Epoch" }, "y": {"$addToSet": "$" + yvalue }, - "fields": {"$mergeObjects": "$id" } + "fields": {"$mergeObjects": "$id" }, + "other": {"$mergeObjects": "$val" } } }) pipeline.append({"$sort": sortObj}) diff --git a/scripts/GinanEDA/eda/routes/dbConnection.py b/scripts/GinanEDA/eda/routes/dbConnection.py index 6d6ae72d4..7afb73697 100644 --- a/scripts/GinanEDA/eda/routes/dbConnection.py +++ b/scripts/GinanEDA/eda/routes/dbConnection.py @@ -132,8 +132,11 @@ def handle_load_request(form_data): states_series += [f"{database}\{series}" for series in client.mongo_content["Series"]] measurements_series += [f"{database}\{series}" for series in client.mongo_content["Series"]] else: - states_series += [f"{database}\{series}" for series in client.mongo_content["StateSeries"]] - measurements_series += [f"{database}\{series}" for series in client.mongo_content["MeasurementsSeries"]] + try: + states_series += [f"{database}\{series}" for series in client.mongo_content["StateSeries"]] + measurements_series += [f"{database}\{series}" for series in client.mongo_content["MeasurementsSeries"]] + except Exception as e: + current_app.logger.warning(f"Error getting series {database}: {e}") if client.mongo_content["Has_measurements"]: mesurements += client.mongo_content["Measurements"] geometry += client.mongo_content["Geometry"] diff --git a/scripts/GinanEDA/eda/routes/measurements.py b/scripts/GinanEDA/eda/routes/measurements.py index 78c884247..f2f479e3b 100644 --- a/scripts/GinanEDA/eda/routes/measurements.py +++ b/scripts/GinanEDA/eda/routes/measurements.py @@ -10,39 +10,33 @@ from . import eda_bp -@eda_bp.route("/measurements", methods=["GET", "POST"]) -def measurements(): +@eda_bp.route("/measurements_diff", methods=["GET", "POST"]) +def measurements_diff(): if request.method == "POST": return handle_post_request() else: - return init_page(template="measurements.jinja") + template = "measurements_diff.jinja" + return init_page(template=template) -def handle_post_request(): - form_data = request.form - form = {} - form["plot"] = form_data.get("type") - form["series"] = form_data.getlist("series") - form["sat"] = form_data.getlist("sat") - form["site"] = form_data.getlist("site") - form["xaxis"] = form_data.get("xaxis") - form["yaxis"] = form_data.getlist("yaxis") - form["exclude"] = form_data.get("exclude") - if form["exclude"] == "": - form["exclude"] = 0 - else: - form["exclude"] = int(form["exclude"]) - form["exclude_tail"] = form_data.get("exclude_tail") - if form["exclude_tail"] == "": - form["exclude_tail"] = 0 +@eda_bp.route("/measurements", methods=["GET", "POST"]) +def measurements(): + if request.method == "POST": + return handle_post_request() else: - form["exclude_tail"] = int(form["exclude_tail"]) + template = "measurements.jinja" + return init_page(template=template) + +def log_and_set_session(form, session_key): current_app.logger.info( - f"GET {form['plot']}, {form['series']}, {form['sat']}, {form['site']}, {form['xaxis']}, {form['yaxis']}, {form['yaxis']+[form['xaxis']]}, exclude {form['exclude']} mintues" + f"GET {form['plot']}, {form['series']}, {form['sat']}, {form['site']}, {form['xaxis']}, {form['yaxis']}, {form['yaxis']+[form['xaxis']]}, exclude {form['exclude']} minutes" ) - session["measurements"] = form + session[session_key] = form current_app.logger.info("Getting Connection") + + +def retrieve_data(form): data = MeasurementArray() data2 = MeasurementArray() for series in form["series"]: @@ -50,26 +44,17 @@ def handle_post_request(): get_data(db_, "Measurements", None, form["site"], form["sat"], [series_], form["yaxis"] + [form["xaxis"]], data) if any([yaxis in session["list_geometry"] for yaxis in form["yaxis"] + [form["xaxis"]]]): get_data(db_, "Geometry", None, form["site"], form["sat"], [""], form["yaxis"] + [form["xaxis"]], data2) + return data, data2 + +def process_data(data, data2, form): if len(data.arr) + len(data2.arr) == 0: - return render_template( - "measurements.jinja", - # content=client.mongo_content, - selection=session["measurements"], - extra=extra, - message="Error getting data: No data", - ) + return None, "Error getting data: No data" - # try: - if len(data.arr) == 0 : + if len(data.arr) == 0: data = data2 else: data.merge(data2) - # except Exception as e: - # current_app.logger.warning(f"Merging error data {e}") - # pass - - # exit(0) data.sort() data.find_minmax() @@ -77,13 +62,17 @@ def handle_post_request(): for data_ in data: data_.find_gaps() data.get_stats() + return data, None + + +def generate_plots(data, form): mode = "markers" if form["plot"] == "Scatter" else "lines" if form["plot"] == "QQ": data.compute_qq() mode = "markers" trace = [] table = {} - current_app.logger.warning("starting plots") + current_app.logger.debug("starting plots") for _data in data: for _yaxis in form["yaxis"]: try: @@ -100,6 +89,8 @@ def handle_post_request(): x_hover_template = "%{x}
" else: _y = _data.data[_yaxis][_data.subset] + if isinstance(_y[0], float) and np.sum(~np.isnan(_y)) == 0: + continue legend = _data.id legend["yaxis"] = _yaxis smallLegend = [legend[a] for a in legend] @@ -118,18 +109,62 @@ def handle_post_request(): pass except Exception as e: current_app.logger.warning(f"Error plotting {_data.id} {form['xaxis']}{_yaxis} {e}") - current_app.logger.warning("end plots") + return trace, table + + +def handle_post_request(): + form_data = request.form + form = { + "plot": form_data.get("type"), + "series": form_data.getlist("series"), + "sat": form_data.getlist("sat"), + "site": form_data.getlist("site"), + "xaxis": form_data.get("xaxis"), + "yaxis": form_data.getlist("yaxis"), + "exclude": form_data.get("exclude", "0"), + "exclude_tail": form_data.get("exclude_tail", "0"), + } + for label in ["exclude", "exclude_tail"]: + form[label] = int(form[label]) if form[label] else 0 + session_key = "measurements_diff" if "series_base" in form_data else "measurements" + template_name = "measurements_diff.jinja" if session_key == "measurements_diff" else "measurements.jinja" + log_and_set_session(form, session_key) + data, data2 = retrieve_data(form) + data, error_message = process_data(data, data2, form) + if error_message: + return render_template( + template_name, + selection=session[session_key], + extra=extra, + message=error_message, + ) + + if session_key == "measurements_diff": + form["series_base"] = [form_data.get("series_base")] + req = form.copy() + req["series"] = [form_data.get("series_base")] + data_base, data2_base = retrieve_data(req) + data_base, error_message = process_data(data_base, data2_base, form) + print("getting base data", form_data.get("series_base")) + print({"series": [form_data.get("series_base")], **form}) + data.yaxis = form["yaxis"] + data = data - data_base + session[session_key] = form + data.get_stats() + else: + session[session_key] = form + print(session[session_key]) + trace, table = generate_plots(data, form) table_agg = aggregate_stats(data) return render_template( - "measurements.jinja", - # content=client.mongo_content, + template_name, extra=extra, graphJSON=generate_fig(trace), mode="plotly", - selection=session["measurements"], + selection=session[session_key], table_data=table, table_headers=["RMS", "mean"], tableagg_data=table_agg, diff --git a/scripts/GinanEDA/eda/routes/states.py b/scripts/GinanEDA/eda/routes/states.py index 9a4cf09fa..f5c242111 100644 --- a/scripts/GinanEDA/eda/routes/states.py +++ b/scripts/GinanEDA/eda/routes/states.py @@ -9,51 +9,34 @@ from . import eda_bp -@eda_bp.route("/states", methods=["GET", "POST"]) -def states(): +@eda_bp.route("/states_diff", methods=["GET", "POST"]) +def states_diff(): if request.method == "POST": return handle_post_request() else: - return init_page(template="states.jinja") - - -def handle_post_request() -> str: - """ - handle_post_request Code to process the POST request and generate the HTML code + template = "states_diff.jinja" + return init_page(template=template) - :return str: webpage code - """ - current_app.logger.info("Entering request") - form_data = request.form - form = { - "type": form_data.get("type"), - "series": form_data.getlist("series"), - "sat": form_data.getlist("sat"), - "site": form_data.getlist("site"), - "state": form_data.getlist("state"), - "xaxis": form_data.get("xaxis"), - "yaxis": form_data.getlist("yaxis"), - "exclude": form_data.get("exclude"), - "exclude_tail": form_data.get("exclude_tail"), - "process": form_data.get("process"), - "degree": form_data.get("degree"), - } - if form["exclude"] == "": - form["exclude"] = 0 +@eda_bp.route("/states", methods=["GET", "POST"]) +def states(): + if request.method == "POST": + return handle_post_request() else: - form["exclude"] = int(form["exclude"]) + template = "states.jinja" + return init_page(template=template) - if form["exclude_tail"] == "": - form["exclude_tail"] = 0 - else: - form["exclude_tail"] = int(form["exclude_tail"]) +def log_and_set_session(form, session_key): current_app.logger.info( f"GET {form['type']}, {form['series']}, {form['sat']}, {form['site']}, {form['state']}, {form['xaxis']}, {form['yaxis']}, " f"{form['yaxis']+[form['xaxis']]}, exclude {form['exclude']} minutes" ) - session["states"] = form + session[session_key] = form + current_app.logger.info("Getting Connection") + + +def retrieve_data(form): data = MeasurementArray() data2 = MeasurementArray() for series in form["series"]: @@ -72,20 +55,24 @@ def handle_post_request() -> str: ) if any([yaxis in session["list_geometry"] for yaxis in form["yaxis"] + [form["xaxis"]]]): get_data(db_, "Geometry", None, form["site"], form["sat"], [""], [form["xaxis"]], data2) + return data, data2 + +def process_data(data, data2, form): if len(data.arr) == 0: - return render_template( - "states.jinja", - # content=client.mongo_content, - selection=session["states"], - extra=extra, - message="Error getting data: No data", - ) + return None, "Error getting data: No data" data.merge(data2) data.sort() data.find_minmax() data.adjust_slice(minutes_min=form["exclude"], minutes_max=form["exclude_tail"]) + for data_ in data: + data_.find_gaps() + data.get_stats() + return data, None + + +def generate_plots(data, form): trace = [] mode = "markers" if form["type"] == "Scatter" else "lines" table = {} @@ -96,7 +83,6 @@ def handle_post_request() -> str: for _data in data: _data.polyfit(degree=int(form["degree"])) - data.get_stats() for _data in data: for _yaxis in _data.data: if _yaxis != form["xaxis"]: @@ -125,16 +111,63 @@ def handle_post_request() -> str: table[f"{_data.id}"]["Fit"] = np.array2string( _data.info["Fit"][_yaxis][::-1], precision=2, separator=", " ) + return trace, table + + +def handle_post_request(): + form_data = request.form + form = { + "type": form_data.get("type"), + "series": form_data.getlist("series"), + "sat": form_data.getlist("sat"), + "site": form_data.getlist("site"), + "state": form_data.getlist("state"), + "xaxis": form_data.get("xaxis"), + "yaxis": form_data.getlist("yaxis"), + "exclude": form_data.get("exclude", "0"), + "exclude_tail": form_data.get("exclude_tail", "0"), + "process": form_data.get("process"), + "degree": form_data.get("degree"), + } + for label in ["exclude", "exclude_tail"]: + form[label] = int(form[label]) if form[label] else 0 + + session_key = "states_diff" if "series_base" in form_data else "states" + template_name = "states_diff.jinja" if session_key == "states_diff" else "states.jinja" + log_and_set_session(form, session_key) + data, data2 = retrieve_data(form) + data, error_message = process_data(data, data2, form) + if error_message: + return render_template( + template_name, + selection=session[session_key], + extra=extra, + message=error_message, + ) + + if session_key == "states_diff": + form["series_base"] = [form_data.get("series_base")] + req = form.copy() + req["series"] = [form_data.get("series_base")] + data_base, data2_base = retrieve_data(req) + data_base, error_message = process_data(data_base, data2_base, form) + list_keys = list(set(key for data_base_ in data_base.arr for key in data_base_.data.keys())) + data.yaxis = list_keys + data = data - data_base + session[session_key] = form + data.get_stats() + else: + session[session_key] = form + trace, table = generate_plots(data, form) table_agg = aggregate_stats(data) return render_template( - "states.jinja", - # content=client.mongo_content, + template_name, extra=extra, graphJSON=generate_fig(trace), mode="plotly", - selection=session["states"], + selection=session[session_key], table_data=table, table_headers=["RMS", "mean", "Fit"], tableagg_data=table_agg, diff --git a/scripts/GinanEDA/eda/routes/trace.py b/scripts/GinanEDA/eda/routes/trace.py index fc6cdf18f..86430d1c5 100644 --- a/scripts/GinanEDA/eda/routes/trace.py +++ b/scripts/GinanEDA/eda/routes/trace.py @@ -139,12 +139,16 @@ def handle_post_request(): table = {} current_app.logger.warning("starting plots") for datax in datas: + # print("Printing datax") + # print(datax) tracesX = [] for label, traceData in datax.items(): traceX = [] traceY = [] for element in traceData: + # print("Printing element") + # print(element) element["y"] = element["y"][0] if xaxis[0] == "_": x = element["_id"][xaxis[1:]] @@ -163,8 +167,9 @@ def handle_post_request(): ybak = y if type(y) == int or type(y) == float: - lpf = lpf + (y - lpf) * float(form["fCoeff"]) - hpf = y - lpf + if form["filter"] == "HPF" or form["filter"] == "LPF": + lpf = lpf + (y - lpf) * float(form["fCoeff"]) + hpf = y - lpf if form["filter"] == "HPF": y = hpf if form["filter"] == "LPF": @@ -180,13 +185,14 @@ def handle_post_request(): traceX.append(x) traceY.append(y) x_hover_template = "%{x}
" + metadata = [a + ": " + str(element["other"][a]) for a in element["other"]] tracesX.append( go.Scatter( x=traceX, y=traceY, mode=mode, name=f"{label}", - hovertemplate=x_hover_template + "%{y:.4e%}
" + str(element["y"]) + "
" + f"{label}", + hovertemplate=x_hover_template + "%{y:.4e%}
" + str(element["y"]) + "
" + f"{label}" + "
" + f"{metadata}", legendgroup="group1", ) ) diff --git a/scripts/GinanEDA/eda/utilities.py b/scripts/GinanEDA/eda/utilities.py index 29e8d4b08..0296efb18 100644 --- a/scripts/GinanEDA/eda/utilities.py +++ b/scripts/GinanEDA/eda/utilities.py @@ -31,15 +31,20 @@ def init_page(template: str) -> str: content = [] return render_template(template, content=content, extra=extra, exlcude=0, selection=session[template.split(".")[0]]) + def initialize_session(): - if not session.get('session_initialized'): - session['measurements'] = { + if not session.get("session_initialized"): + session["measurements"] = {"plot": "Line", "series": [], "site": [], "sat": [], "xaxis": "Epoch"} + + session["measurements_diff"] = { "plot": "Line", "series": [], + "series_base": "", "site": [], "sat": [], - "xaxis": "Epoch" + "xaxis": "Epoch", } + session["states"] = { "type": "Line", "series": [], @@ -50,24 +55,30 @@ def initialize_session(): "degree": "0", "process": "None", } - session["position"] = { + session["states_diff"] = { "type": "Line", "series": [], "series_base": "", + "site": [], + "sat": [], + "xaxis": "Epoch", + "yaxis": "x", + "degree": "0", + "process": "None", } - session["clocks"] = { - "series": "", + session["position"] = { + "type": "Line", + "series": [], "series_base": "", - "subset": [], - "modes": [], - "clockType": "" } + session["clocks"] = {"series": "", "series_base": "", "subset": [], "modes": [], "clockType": ""} session["orbits"] = { "orbitType": "", "series": [], "sat": [], } - session['session_initialized'] = True + session["session_initialized"] = True + def generate_fig(trace): fig = go.Figure(data=trace) @@ -82,17 +93,14 @@ def generate_fig(trace): def generate_figs(traces): - fig = make_subplots(rows=max(1,len(traces)), cols=1, - shared_xaxes=True, - vertical_spacing=0.2 - ) - for i in range( len(traces)): + fig = make_subplots(rows=max(1, len(traces)), cols=1, shared_xaxes=True, vertical_spacing=0.2) + for i in range(len(traces)): for trace in traces[i]: fig.add_trace(trace, row=i + 1, col=1) fig.update_layout( - xaxis={"rangeslider":{"visible":True}, "showgrid":current_app.config["EDA_GRID"]}, - yaxis={"fixedrange":False, "tickformat":".3e", "showgrid":current_app.config["EDA_GRID"]}, + xaxis={"rangeslider": {"visible": True}, "showgrid": current_app.config["EDA_GRID"]}, + yaxis={"fixedrange": False, "tickformat": ".3e", "showgrid": current_app.config["EDA_GRID"]}, height=1200, # template=current_app.config["EDA_THEME"], ) @@ -189,6 +197,7 @@ def get_distinct_vals(ip, port, db, coll, element, reshape_on=None): current_app.logger.warning(err) pass + def extract_database_series(series): db_, series_ = series.split("\\") - return db_,series_ \ No newline at end of file + return db_, series_ diff --git a/scripts/GinanEDA/templates/measurements_diff.jinja b/scripts/GinanEDA/templates/measurements_diff.jinja new file mode 100644 index 000000000..c4e39c22b --- /dev/null +++ b/scripts/GinanEDA/templates/measurements_diff.jinja @@ -0,0 +1,211 @@ +{% extends 'base.jinja'%} +{# {% block header %} +

{% block title %} {{ plotingpage }} {% endblock %}

+{% endblock %} #} + +{% block title%} +Measurements +{% endblock %} +{% block menuselection %} + +
+
+ +
+ +
+ + +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+
+ + +
+ +
+ + +
+ +
+{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file diff --git a/scripts/GinanEDA/templates/sidebar.jinja b/scripts/GinanEDA/templates/sidebar.jinja index a93ca7be5..c3c7de8a0 100644 --- a/scripts/GinanEDA/templates/sidebar.jinja +++ b/scripts/GinanEDA/templates/sidebar.jinja @@ -10,6 +10,8 @@ {% endif %} {# #} +
+
  • Connect
    • @@ -19,8 +21,9 @@
  • States
    • +
  • - Clock + Clock analysis
  • Position analysis @@ -28,12 +31,19 @@
  • Orbits analysis
    • +
    + +
  • + Measurements Diff +
    • +
  • + States Diff +
    • +
    +
  • Config
    • - \ No newline at end of file diff --git a/scripts/GinanEDA/templates/states_diff.jinja b/scripts/GinanEDA/templates/states_diff.jinja new file mode 100644 index 000000000..9cce8cd7d --- /dev/null +++ b/scripts/GinanEDA/templates/states_diff.jinja @@ -0,0 +1,238 @@ +{% extends 'base.jinja'%} + +{% block title%} +States +{% endblock %} + +{% block menuselection %} +
    +
    + +
    + + +
    + + + +
    + +
    + +
    + +
    + +
    + +
    + +
    + +
    + +
    +
    + +
    + +
    +
    + +
    + +
    +
    + +
    + +
    + + +
    + +
    + +
    +
    + +
    + +
    +
    +
    + + +
    + +
    + + +
    +
    + + +
    + +
    +{% endblock %} + + +{% block scripts %} + + + +{% endblock %} diff --git a/scripts/GinanUI/README.md b/scripts/GinanUI/README.md new file mode 100644 index 000000000..d7b3e3701 --- /dev/null +++ b/scripts/GinanUI/README.md @@ -0,0 +1,9 @@ +# Ginan-UI + +An intelligent and user-friendly interface for using the Geoscience Australia GNSS processing tool Ginan. Made using PySide6 by students of the 2025 ANU TechLauncher program. + +[User manual available here](./docs/USER_GUIDE.md) + +## Installation + +Please read the user manual above for installation instructions. \ No newline at end of file diff --git a/scripts/GinanUI/__init__.py b/scripts/GinanUI/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/GinanUI/app/__init__.py b/scripts/GinanUI/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/GinanUI/app/controllers/__init__.py b/scripts/GinanUI/app/controllers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/GinanUI/app/controllers/input_controller.py b/scripts/GinanUI/app/controllers/input_controller.py new file mode 100644 index 000000000..aa109eb26 --- /dev/null +++ b/scripts/GinanUI/app/controllers/input_controller.py @@ -0,0 +1,1516 @@ +# app/controllers/input_controller.py +""" +UI input flow controller for the Ginan-UI. +""" +from __future__ import annotations + +import os +import re +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Callable, List +from decimal import Decimal, InvalidOperation + +import pandas as pd + +from scripts.GinanUI.app.utils.logger import Logger + +from scripts.GinanUI.app.models.dl_products import ( + get_valid_analysis_centers, + get_valid_series_for_provider, + get_valid_providers_with_series, + str_to_datetime +) +from PySide6.QtCore import QObject, Signal, Qt, QDateTime, QThread +from PySide6.QtGui import QStandardItemModel, QStandardItem +from PySide6.QtWidgets import ( + QFileDialog, + QDialog, + QFormLayout, + QDoubleSpinBox, + QHBoxLayout, + QVBoxLayout, + QDateTimeEdit, + QInputDialog, + QMessageBox, + QComboBox, + QLineEdit, + QPushButton, + QLabel +) + +from scripts.GinanUI.app.models.execution import Execution, GENERATED_YAML, INPUT_PRODUCTS_PATH +from scripts.GinanUI.app.models.rinex_extractor import RinexExtractor +from scripts.GinanUI.app.utils.cddis_credentials import save_earthdata_credentials +from scripts.GinanUI.app.models.archive_manager import (archive_products_if_rinex_changed) +from scripts.GinanUI.app.models.archive_manager import archive_old_outputs +from scripts.GinanUI.app.utils.workers import DownloadWorker +from scripts.GinanUI.app.utils.toast import show_toast + + +class InputController(QObject): + """ + UI controller class InputController. + """ + + ready = Signal(str, str) # rnx_path, output_path + pea_ready = Signal() # emitted when PEA processing should start + + def __init__(self, ui, parent_window, execution: Execution): + """ + UI handler: init. + + Arguments: + ui (Any): Main window UI instance (generated from Qt .ui). + parent_window (Any): Parent widget/window to anchor dialogs. + execution (Execution): Backend execution bridge used to read/apply UI config. + """ + super().__init__() + self.ui = ui + self.parent = parent_window + self.execution = execution + + self.rnx_file: Path = None + self.output_dir: Path = None + self.products_df: pd.DataFrame = pd.DataFrame() # CDDIS replaces with a populated dataframe + + # Config file path + self.config_path = GENERATED_YAML + + ### Wire: file selection buttons ### + self.ui.observationsButton.clicked.connect(self.load_rnx_file) + self.ui.outputButton.clicked.connect(self.load_output_dir) + + # Initial states + self.ui.outputButton.setEnabled(False) + self.ui.showConfigButton.setEnabled(False) + self.ui.processButton.setEnabled(False) + + ### Bind: configuration drop-downs / UIs ### + + self._bind_combo(self.ui.Mode, self._get_mode_items) + + # PPP_provider, project and series + self.ui.PPP_provider.currentTextChanged.connect(self._on_ppp_provider_changed) + self.ui.PPP_project.currentTextChanged.connect(self._on_ppp_project_changed) + self.ui.PPP_series.currentTextChanged.connect(self._on_ppp_series_changed) + + # Constellations + self._bind_multiselect_combo( + self.ui.Constellations_2, + self._get_constellations_items, + self.ui.constellationsValue, + placeholder="Select one or more", + ) + + # Receiver/Antenna types: free-text input + self._enable_free_text_for_receiver_and_antenna() + + # Antenna offset + self.ui.antennaOffsetButton.clicked.connect(self._open_antenna_offset_dialog) + self.ui.antennaOffsetButton.setCursor(Qt.CursorShape.PointingHandCursor) + self.ui.antennaOffsetValue.setText("0.0, 0.0, 0.0") + + # Time window and data interval + self.ui.timeWindowButton.clicked.connect(self._open_time_window_dialog) + self.ui.timeWindowButton.setCursor(Qt.CursorShape.PointingHandCursor) + self.ui.dataIntervalButton.clicked.connect(self._open_data_interval_dialog) + self.ui.dataIntervalButton.setCursor(Qt.CursorShape.PointingHandCursor) + + # Run buttons + self.ui.showConfigButton.clicked.connect(self.on_show_config) + self.ui.showConfigButton.setCursor(Qt.CursorShape.PointingHandCursor) + self.ui.processButton.clicked.connect(self.on_run_pea) + + # CDDIS credentials dialog + self.ui.cddisCredentialsButton.clicked.connect(self._open_cddis_credentials_dialog) + + self.setup_tooltips() + + def setup_tooltips(self): + """ + UI handler: setup tooltips and visual style for key controls. + """ + + # Consistent tooltip style for all elements + tooltip_style = """ + QToolTip { + background-color: #2c5d7c; + color: #ffffff; + border: 1px solid #999999; + padding: 4px; + border-radius: 3px; + font:13pt "Segoe UI"; + } + """ + + # Apply to parent window + self.parent.setStyleSheet(self.parent.styleSheet() + tooltip_style) + + # Add tooltip styling to buttons without changing their appearance + # Just append the tooltip style to their existing styles + + # Get current styles and append tooltip styling + obs_style = self.ui.observationsButton.styleSheet() + tooltip_style + out_style = self.ui.outputButton.styleSheet() + tooltip_style + proc_style = self.ui.processButton.styleSheet() + tooltip_style + cddis_style = self.ui.cddisCredentialsButton.styleSheet() + tooltip_style + + self.ui.observationsButton.setStyleSheet(obs_style) + self.ui.outputButton.setStyleSheet(out_style) + self.ui.processButton.setStyleSheet(proc_style) + self.ui.cddisCredentialsButton.setStyleSheet(cddis_style) + + # File selection buttons + self.ui.observationsButton.setToolTip( + "Select a RINEX observation file (.rnx or .rnx.gz).\n" + "This will automatically extract metadata and populate the UI fields." + ) + + self.ui.outputButton.setToolTip( + "Choose the directory where processing results will be saved.\n" + "Existing .POS or .GPX output in this directory will be saved in the archived subdirectory." + ) + + self.ui.processButton.setToolTip( + "Start the Ginan (pea) PPP processing using the configured parameters.\n" + "Ensure all required fields are filled before processing." + ) + + # Configuration buttons + self.ui.showConfigButton.setToolTip( + "Generate and open the YAML configuration file.\n" + "You can review and modify advanced settings before processing.\n" + "Note: UI defined parameters will ALWAYS override manual config edits." + ) + + self.ui.cddisCredentialsButton.setToolTip( + "Set your NASA Earthdata credentials for downloading PPP products\n" + "Required for accessing the CDDIS archive data" + ) + + # Input fields and combos + self.ui.Mode.setToolTip( + "Processing mode:\n" + "• Static: For stationary receivers\n" + "• Kinematic: For moving receivers\n" + "• Dynamic: For high-dynamic applications" + ) + + self.ui.Constellations_2.setToolTip( + "Select which GNSS constellations to use:\n" + "GPS, Galileo (GAL), GLONASS (GLO), BeiDou (BDS), QZSS (QZS)\n" + "More constellations generally improve accuracy" + ) + + self.ui.PPP_provider.setToolTip( + "Analysis centre that provides PPP products\n" + "Options populated based on your observation time window" + ) + + self.ui.PPP_project.setToolTip( + "PPP product project type.\n" + "Different projects types offer varying GNSS constellation PPP products." + ) + + self.ui.PPP_series.setToolTip( + "PPP product series:\n" + "• ULT: Ultra-rapid (lower latency)\n" + "• RAP: Rapid \n" + "• FIN: Final (highest accuracy)" + ) + + # Receiver/Antenna fields + self.ui.Receiver_type.setToolTip( + "Receiver model extracted from RINEX header\n" + "Click to manually edit if needed" + ) + + self.ui.Antenna_type.setToolTip( + "Antenna model extracted from RINEX header\n" + "Must match entries in the ANTEX (.atx) calibration file\n" + "Click to manually edit if needed" + ) + + # Time and offset buttons + self.ui.timeWindowButton.setToolTip( + "Observation time window extracted from RINEX file\n" + "Click to adjust start and end times for processing" + ) + + self.ui.dataIntervalButton.setToolTip( + "Data sampling interval in seconds\n" + "Click to change the processing interval" + ) + + self.ui.antennaOffsetButton.setToolTip( + "Antenna reference point offset in metres (East, North, Up)\n" + "Typically extracted from RINEX header\n" + "Click to modify if needed" + ) + + # Value display labels + self.ui.receiverTypeValue.setToolTip("Receiver type from RINEX header") + self.ui.antennaTypeValue.setToolTip("Antenna type from RINEX header") + self.ui.constellationsValue.setToolTip("Available constellations in RINEX data") + self.ui.timeWindowValue.setToolTip("Observation time span") + self.ui.dataIntervalValue.setToolTip("Data sampling interval") + self.ui.antennaOffsetValue.setToolTip("Antenna offset: East, North, Up (metres)") + + def _open_cddis_credentials_dialog(self): + """ + UI handler: open the CDDIS credentials dialog for Earthdata login. + """ + dialog = CredentialsDialog(self.parent) + dialog.exec() + + # region File Selection + Metadata Extraction + PPP product selection + def load_rnx_file(self) -> ExtractedInputs | None: + """ + UI handler: choose a RINEX file, extract metadata, update UI, and start PPP products query. + """ + path = self._select_rnx_file(self.parent) + if not path: + return None + + current_rinex_path = Path(path).resolve() + archive_products_if_rinex_changed( + current_rinex=current_rinex_path, + last_rinex=getattr(self, "last_rinex_path", None), + products_dir=INPUT_PRODUCTS_PATH + ) + # Disable until new providers found + if current_rinex_path != getattr(self, "last_rinex_path", None): + self.ui.processButton.setEnabled(False) + self._on_cddis_ready(pd.DataFrame(), False) # Clears providers until worker completes + + self.last_rinex_path = current_rinex_path + self.rnx_file = str(current_rinex_path) + + Logger.terminal(f"📄 RINEX file selected: {self.rnx_file}") + + try: + extractor = RinexExtractor(self.rnx_file) + result = extractor.extract_rinex_data(self.rnx_file) + + # Verify antenna_type against .atx file + if not self.parent.atx_required_for_rnx_extraction: + Logger.terminal( + "⚠️ ANTEX (.atx) file not installed yet. Antenna type verification will be skipped.") + else: + self.verify_antenna_type(result) + + Logger.terminal("🔍 Scanning CDDIS archive for PPP products. Please wait...") + + # Show toast notification + show_toast(self.parent, "🔍 Scanning CDDIS archive for PPP products...", duration=15000) + + # Show waiting cursor during CDDIS scan + self.parent.setCursor(Qt.CursorShape.WaitCursor) + + # Retrieve valid analysis centers + start_epoch = str_to_datetime(result['start_epoch']) + end_epoch = str_to_datetime(result['end_epoch']) + self.worker = DownloadWorker(start_epoch=start_epoch, end_epoch=end_epoch, analysis_centers=True) + self.metadata_thread = QThread() + self.worker.moveToThread(self.metadata_thread) + + self.worker.finished.connect(self._on_cddis_ready) + self.worker.finished.connect(self._restore_cursor) # Restore cursor when done + self.worker.finished.connect(self.worker.deleteLater) + self.worker.finished.connect(self.metadata_thread.quit) + self.metadata_thread.finished.connect(self.metadata_thread.deleteLater) + self.metadata_thread.started.connect(self.worker.run) + self.metadata_thread.start() + + # Populate extracted metadata immediately + self.ui.constellationsValue.setText(result["constellations"]) + self.ui.timeWindowValue.setText(f"{result['start_epoch']} to {result['end_epoch']}") + self.ui.timeWindowButton.setText(f"{result['start_epoch']} to {result['end_epoch']}") + self.ui.dataIntervalButton.setText(f"{result['epoch_interval']} s") + self.ui.receiverTypeValue.setText(result["receiver_type"]) + self.ui.antennaTypeValue.setText(result["antenna_type"]) + self.ui.antennaOffsetValue.setText(", ".join(map(str, result["antenna_offset"]))) + self.ui.antennaOffsetButton.setText(", ".join(map(str, result["antenna_offset"]))) + + self.ui.Receiver_type.clear() + self.ui.Receiver_type.addItem(result["receiver_type"]) + self.ui.Receiver_type.setCurrentIndex(0) + self.ui.Receiver_type.lineEdit().setText(result["receiver_type"]) + + self.ui.Antenna_type.clear() + self.ui.Antenna_type.addItem(result["antenna_type"]) + self.ui.Antenna_type.setCurrentIndex(0) + self.ui.Antenna_type.lineEdit().setText(result["antenna_type"]) + + self._update_constellations_multiselect(result["constellations"]) + + self.ui.outputButton.setEnabled(True) + self.ui.showConfigButton.setEnabled(True) + + Logger.terminal("⚒️ RINEX file metadata extracted and applied to UI fields") + self.ui.outputButton.setEnabled(True) + self.ui.showConfigButton.setEnabled(True) + + except Exception as e: + Logger.terminal(f"Error extracting RNX metadata: {e}") + return None + + # Always update MainWindow's state + self.parent.rnx_file = self.rnx_file + + if self.output_dir: + self.ready.emit(str(self.rnx_file), str(self.output_dir)) + + return result + + def verify_antenna_type(self, result: List[str]): + """ + UI handler: verify that the RINEX antenna_type exists in the selected ANTEX (.atx) file. + """ + # Verify antenna_type is present within the .atx file + # Return warning if not + atx_path = self.get_best_atx_path() + + with open(atx_path, "r") as file: + for line in file: + label = line[60:].strip() + + # Read and find antenna_type tag + if label == "TYPE / SERIAL NO" and line[20:24].strip() == "": + valid_antenna_type = line[0:20] + + if len(valid_antenna_type.strip()) < 16 or not valid_antenna_type[16:].strip(): + # Just the antenna part is included, need to add radome (cover) + antenna_part = valid_antenna_type[:15].strip() + valid_antenna_type = f"{antenna_part:<15} NONE" + + # Do same normalisation for result["antenna_type"] + result_antenna = result["antenna_type"] + + if len(result_antenna.strip()) < 16 or ( + len(result_antenna) > 16 and not result_antenna[16:].strip()): + antenna_part = result_antenna[:15].strip() + result_antenna = f"{antenna_part:<15} NONE" + + # Compare strings + if result_antenna.strip() == valid_antenna_type.strip(): + Logger.terminal("✅ Antenna type verified from .atx file") + return + + # Not found! Return warning to user + QMessageBox.warning( + None, + "Provided Antenna Type Invalid", + f'Provided antenna type in .rnx file: "{result["antenna_type"]}"\n' + f'not found in .atx file: "{atx_path}"' + ) + Logger.terminal(f"⚠️ Antenna type failed to verify from .atx file: {atx_path}") + return + + def get_best_atx_path(self): + """ + Select the best available ANTEX (.atx) file with a priority order. + """ + # Find all .atx files present and prioritise the newest ones + # Return filepath string to best .atx file + atx_files = list(INPUT_PRODUCTS_PATH.glob("*.atx")) + if len(atx_files) == 0: + raise FileNotFoundError("No .atx file found") + elif len(atx_files) > 1: + # Priority order: igs20 > igs14 > igs13 > igs08 > igs05 > any other .atx file + priority_order = ['igs20.atx', 'igs14.atx', 'igs13.atx', 'igs08.atx', 'igs05.atx'] + atx_path = None + for best_atx in priority_order: + matching_files = [f for f in atx_files if f.name == best_atx] + if matching_files: + atx_path = matching_files[0] + Logger.terminal(f"📁 Selected .atx file: {atx_path.name} based on priority") + break + + # If none of the preferred files found, use the first available + if atx_path is None: + atx_path = atx_files[0] + Logger.terminal(f"📁 Selected .atx file: {atx_path.name} based on fallback") + else: + atx_path = atx_files[0] + return atx_path + + def _update_constellations_multiselect(self, constellation_str: str): + """ + Populate and mirror a multi-select constellation combo with checkboxes. + + Arguments: + constellation_str (str): Comma-separated constellations (e.g., "GPS, GAL, GLO"). + + """ + from PySide6.QtGui import QStandardItemModel, QStandardItem + + constellations = [c.strip() for c in constellation_str.split(",") if c.strip()] + combo = self.ui.Constellations_2 + + # Remove previous bindings + if hasattr(combo, '_old_showPopup'): + delattr(combo, '_old_showPopup') + + combo.clear() + combo.setEditable(True) + combo.lineEdit().setReadOnly(True) + combo.setInsertPolicy(QComboBox.NoInsert) + + # Build the item model + model = QStandardItemModel(combo) + for txt in constellations: + item = QStandardItem(txt) + item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsUserCheckable) + item.setCheckState(Qt.Checked) + model.appendRow(item) + + def on_item_changed(_item): + selected = [ + model.item(i).text() + for i in range(model.rowCount()) + if model.item(i).checkState() == Qt.Checked + ] + label = ", ".join(selected) if selected else "Select one or more" + combo.lineEdit().setText(label) + self.ui.constellationsValue.setText(label) + + model.itemChanged.connect(on_item_changed) + combo.setModel(model) + combo.setCurrentIndex(-1) + + # Custom showPopup function to keep things reset + def show_popup_constellation(): + if combo.model() != model: + combo.setModel(model) + combo.setCurrentIndex(-1) + QComboBox.showPopup(combo) + + combo.showPopup = show_popup_constellation + + # Store for access and event consistency + combo._constellation_model = model + combo._constellation_on_item_changed = on_item_changed + + # Set initial label text + combo.lineEdit().setText(", ".join(constellations)) + self.ui.constellationsValue.setText(", ".join(constellations)) + + def _on_cddis_ready(self, data: pd.DataFrame, log_messages: bool = True): + """ + UI handler: receive PPP products DataFrame from worker and populate provider/project/series combos. + """ + self.products_df = data + + if data.empty: + self.valid_analysis_centers = [] + self.ui.PPP_provider.clear() + self.ui.PPP_provider.addItem("None") + self.ui.PPP_series.clear() + self.ui.PPP_series.addItem("None") + return + + self.valid_analysis_centers = list(get_valid_analysis_centers(self.products_df)) + + if len(self.valid_analysis_centers) == 0: + if log_messages: + Logger.terminal("⚠️ No valid PPP providers found.") + self.ui.PPP_provider.clear() + self.ui.PPP_provider.addItem("None") + self.ui.PPP_series.clear() + self.ui.PPP_series.addItem("None") + return + + self.ui.PPP_provider.blockSignals(True) + self.ui.PPP_provider.clear() + self.ui.PPP_provider.addItems(self.valid_analysis_centers) + self.ui.PPP_provider.setCurrentIndex(0) + + # Update PPP_series based on default PPP_provider + self.ui.PPP_provider.blockSignals(False) + self.try_enable_process_button() + self._on_ppp_provider_changed(self.valid_analysis_centers[0]) + if log_messages: + Logger.terminal( + f"✅ CDDIS archive scan complete. Found PPP product providers: {', '.join(self.valid_analysis_centers)}") + # Show success toast + show_toast(self.parent, f"✅ Found {len(self.valid_analysis_centers)} PPP provider(s)", duration=3000) + + def _on_cddis_error(self, msg): + """ + UI handler: report CDDIS worker error to the UI. + """ + Logger.terminal(f"Error loading CDDIS data: {msg}") + self.ui.PPP_provider.clear() + self.ui.PPP_provider.addItem("None") + # Restore cursor in case of error + self.parent.setCursor(Qt.CursorShape.ArrowCursor) + # Show error toast + show_toast(self.parent, "⚠️ Failed to scan CDDIS archive", duration=4000) + + def _restore_cursor(self): + """ + Restore the cursor to normal arrow after background operation completes. + """ + self.parent.setCursor(Qt.CursorShape.ArrowCursor) + + def _on_ppp_provider_changed(self, provider_name: str): + """ + UI handler: when PPP provider changes, refresh project and series options. + Only shows series that have all required files (SP3, BIA, CLK). + """ + if not provider_name or provider_name.strip() == "": + return + try: + # Get valid series for this provider (only those with all required files) + valid_series = get_valid_series_for_provider(self.products_df, provider_name) + + if not valid_series: + raise ValueError(f"No valid series (with all required files) for provider: {provider_name}") + + # Get DataFrame of valid (project, series) pairs - filter for valid series only + df = self.products_df.loc[ + (self.products_df["analysis_center"] == provider_name) & + (self.products_df["solution_type"].isin(valid_series)), + ["project", "solution_type"]] + + if df.empty: + raise ValueError(f"No valid project–series combinations for provider: {provider_name}") + + # Store for future filtering if needed + self._valid_project_series_df = df + self._valid_series_for_provider = valid_series # Cache valid series + + project_options = sorted(df['project'].unique()) + series_options = sorted(df['solution_type'].unique()) + + # Block signals before clearing and populating to prevent any duplicates in dropdown + self.ui.PPP_project.blockSignals(True) + self.ui.PPP_series.blockSignals(True) + + self.ui.PPP_project.clear() + self.ui.PPP_series.clear() + + self.ui.PPP_project.addItems(project_options) + self.ui.PPP_series.addItems(series_options) + + self.ui.PPP_project.setCurrentIndex(0) + self.ui.PPP_series.setCurrentIndex(0) + + # Unblock signals now that the population is complete + self.ui.PPP_project.blockSignals(False) + self.ui.PPP_series.blockSignals(False) + + except Exception as e: + self.ui.PPP_series.clear() + self.ui.PPP_series.addItem("None") + self.ui.PPP_project.clear() + self.ui.PPP_project.addItem("None") + + def _on_ppp_series_changed(self, selected_series: str): + """ + UI handler: when PPP series changes, filter valid projects. + + Arguments: + selected_series (str): Series code, e.g., 'ULT', 'RAP', 'FIN'. + """ + if not hasattr(self, "_valid_project_series_df"): + return + + df = self._valid_project_series_df + filtered_df = df[df["solution_type"] == selected_series] + valid_projects = sorted(filtered_df["project"].unique()) + + self.ui.PPP_project.blockSignals(True) + self.ui.PPP_project.clear() + self.ui.PPP_project.addItems(valid_projects) + self.ui.PPP_project.setCurrentIndex(0) + self.ui.PPP_project.blockSignals(False) + + def _on_ppp_project_changed(self, selected_project: str): + """ + UI handler: when PPP project changes, filter valid series. + Only displays series that have all required files (SP3, BIA, CLK). + """ + if not hasattr(self, "_valid_project_series_df"): + return + + df = self._valid_project_series_df + filtered_df = df[df["project"] == selected_project] + valid_series = sorted(filtered_df["solution_type"].unique()) + + # Ensure only series with all required files are displayed + if hasattr(self, "_valid_series_for_provider"): + valid_series = [s for s in valid_series if s in self._valid_series_for_provider] + + self.ui.PPP_series.blockSignals(True) + self.ui.PPP_series.clear() + self.ui.PPP_series.addItems(valid_series) + self.ui.PPP_series.setCurrentIndex(0) + self.ui.PPP_series.blockSignals(False) + + Logger.terminal(f"[UI] Filtered PPP_series for project '{selected_project}': {valid_series}") + + def load_output_dir(self): + """ + UI handler: choose the output directory and (if RNX is set) emit ready. + """ + """Pick an output directory; if RNX is also set, emit ready.""" + path = self._select_output_dir(self.parent) + if not path: + return + + # Ensure output_dir is a Path object + self.output_dir = Path(path).resolve() + Logger.terminal(f"📂 Output directory selected: {self.output_dir}") + + # Archive existing/old outputs + visual_dir = self.output_dir / "visual" + archive_old_outputs(self.output_dir, visual_dir) + + # Enable process button + # MainWindow owns when to enable processButton. This controller exposes a helper if needed. + self.try_enable_process_button() + + # Always update MainWindow's state + self.parent.output_dir = self.output_dir + + if self.rnx_file: + self.ready.emit(str(self.rnx_file), str(self.output_dir)) + + def try_enable_process_button(self): + """ + UI handler: enable the Process button when RNX, output path, and metadata are ready. + """ + if not self.parent.metadata_downloaded: + return + if not self.output_dir: + return + if not self.rnx_file: + return + if len(self._get_ppp_provider_items()) < 1: + return + self.ui.processButton.setEnabled(True) + + # endregion + + # region Multi-Selectors Assigning (A.K.A. Combo Plumbing) + + def _on_select(self, combo: QComboBox, label, title: str, index: int): + """ + UI handler: mirror a single-select combo choice to a label and reset placeholder. + + Arguments: + combo (QComboBox): Source combo box. + label (QLabel): Target label to mirror text. + title (str): Placeholder title to reset in the combo. + index (int): Selected index. + """ + value = combo.itemText(index) + label.setText(value) + + combo.clear() + combo.addItem(title) + + def _bind_combo(self, combo: QComboBox, items_func: Callable[[], List[str]]): + """ + Bind a single-choice combo to dynamically populate items on open and keep the UI clean. + + Arguments: + combo (QComboBox): Target combo box to bind. + items_func (Callable[[], list[str]]): Function returning the items list. + """ + combo._old_showPopup = combo.showPopup + + def new_showPopup(): + combo.clear() + combo.setEditable(True) + combo.lineEdit().setAlignment(Qt.AlignCenter) + for item in items_func(): + combo.addItem(item) + combo.setEditable(False) + combo._old_showPopup() + + combo.showPopup = new_showPopup + + def _bind_multiselect_combo( + self, + combo: QComboBox, + items_func: Callable[[], List[str]], + mirror_label, + placeholder: str, + ): + """ + Bind a multi-select combo using checkable items and mirror checked labels as comma-separated text. + + Arguments: + combo (QComboBox): Target combo box. + items_func (Callable[[], list[str]]): Function returning the items list. + mirror_label (QLabel): Label where checked values are mirrored. + placeholder (str): Placeholder text when no item is checked. + + """ + combo.setEditable(True) + combo.lineEdit().setReadOnly(True) + combo.lineEdit().setPlaceholderText(placeholder) + combo.setInsertPolicy(QComboBox.NoInsert) + + combo._old_showPopup = combo.showPopup + + def show_popup(): + model = QStandardItemModel(combo) + for txt in items_func(): + it = QStandardItem(txt) + it.setFlags(Qt.ItemIsEnabled | Qt.ItemIsUserCheckable) + it.setData(Qt.Unchecked, Qt.CheckStateRole) + model.appendRow(it) + + def on_item_changed(_item: QStandardItem): + # Collect all checked items + selected = [ + model.item(r).text() + for r in range(model.rowCount()) + if model.item(r).checkState() == Qt.Checked + ] + text = ", ".join(selected) if selected else placeholder + combo.lineEdit().setText(text) + mirror_label.setText(text) + + model.itemChanged.connect(on_item_changed) + combo.setModel(model) + combo._old_showPopup() + + combo.showPopup = show_popup + combo.clear() + combo.lineEdit().clear() + combo.lineEdit().setPlaceholderText(placeholder) + + # ========================================================== + + # Receiver / Antenna free text popups + # ========================================================== + def _enable_free_text_for_receiver_and_antenna(self): + """ + Allow users to enter custom receiver/antenna types via popup, mirroring to UI. + """ + self.ui.Receiver_type.setEditable(True) + self.ui.Receiver_type.lineEdit().setReadOnly(True) + self.ui.Antenna_type.setEditable(True) + self.ui.Antenna_type.lineEdit().setReadOnly(True) + + # Receiver type free text + def _ask_receiver_type(): + current_text = self.ui.Receiver_type.currentText().strip() + text, ok = QInputDialog.getText( + self.ui.Receiver_type, + "Receiver Type", + "Enter receiver type:", + text=current_text # prefill with current + ) + if ok and text: + self.ui.Receiver_type.clear() + self.ui.Receiver_type.addItem(text) + self.ui.Receiver_type.lineEdit().setText(text) + self.ui.receiverTypeValue.setText(text) + + self.ui.Receiver_type.showPopup = _ask_receiver_type + + # Antenna type free text + def _ask_antenna_type(): + current_text = self.ui.Antenna_type.currentText().strip() + text, ok = QInputDialog.getText( + self.ui.Antenna_type, + "Antenna Type", + "Enter antenna type:", + text=current_text # prefill with current + ) + if ok and text: + self.ui.Antenna_type.clear() + self.ui.Antenna_type.addItem(text) + self.ui.Antenna_type.lineEdit().setText(text) + self.ui.antennaTypeValue.setText(text) + + self.ui.Antenna_type.showPopup = _ask_antenna_type + + # ========================================================== + # Antenna offset popup + # ========================================================== + def _open_antenna_offset_dialog(self): + """ + UI handler: open antenna offset dialog (E, N, U) with high-precision spin boxes. + """ + dlg = QDialog(self.ui.antennaOffsetButton) + dlg.setWindowTitle("Antenna Offset") + + # Parse existing "E, N, U" + try: + e0, n0, u0 = [float(x.strip()) for x in self.ui.antennaOffsetValue.text().split(",")] + except Exception: + e0 = n0 = u0 = 0.0 + + form = QFormLayout(dlg) + + class DecimalSpinBox(QDoubleSpinBox): + def __init__(self, parent=None, top=10000, bottom=-10000, precision=15, step_size=0.1): + super().__init__(parent) + self.setRange(bottom, top) + + self.setDecimals(precision) # fallback precision + # up down arrow Step size + # note there is some float point inaccuracy when useing steps + self.setSingleStep(step_size) + + def textFromValue(self, value: float) -> str: + """Format value dynamically with Decimal for more precision""" + # Convert through Decimal to avoid scientific notation + d = Decimal(str(value)) + return str(d.normalize()) # trims trailing zeros + + def valueFromText(self, text: str) -> float: + """Parse text back into a float""" + try: + return float(Decimal(text)) + except InvalidOperation: + raise ValueError(f"Failed to convert Antenna offset to float: {text}") + + sb_e = DecimalSpinBox(dlg) + sb_e.setValue(e0) + + sb_n = DecimalSpinBox(dlg) + sb_n.setValue(n0) + + sb_u = DecimalSpinBox(dlg) + sb_u.setValue(u0) + + form.addRow("E:", sb_e) + form.addRow("N:", sb_n) + form.addRow("U:", sb_u) + + btn_row = QHBoxLayout() + ok_btn = QPushButton("OK", dlg) + cancel_btn = QPushButton("Cancel", dlg) + btn_row.addWidget(ok_btn) + btn_row.addWidget(cancel_btn) + form.addRow(btn_row) + + ok_btn.clicked.connect(lambda: self._set_antenna_offset(sb_e, sb_n, sb_u, dlg)) + cancel_btn.clicked.connect(dlg.reject) + + dlg.exec() + + def _set_antenna_offset(self, sb_e, sb_n, sb_u, dlg: QDialog): + """ + UI handler: apply antenna offset values back to UI. + + Arguments: + sb_e (QDoubleSpinBox): East (E) spin box. + sb_n (QDoubleSpinBox): North (N) spin box. + sb_u (QDoubleSpinBox): Up (U) spin box. + dlg (QDialog): Dialog to accept/close. + """ + e, n, u = sb_e.value(), sb_n.value(), sb_u.value() + text = f"{e}, {n}, {u}" + self.ui.antennaOffsetButton.setText(text) + self.ui.antennaOffsetValue.setText(text) + dlg.accept() + + # ========================================================== + # Time window popup + # ========================================================== + def _open_time_window_dialog(self): + """ + UI handler: open dialog to adjust observation start/end times. + """ + dlg = QDialog(self.ui.timeWindowValue) + dlg.setWindowTitle("Select start / end time") + + # Parse existing "yyyy-MM-dd_HH:mm:ss to yyyy-MM-dd_HH:mm:ss" + current_text = self.ui.timeWindowButton.text() + try: + s_text, e_text = current_text.split(" to ") + s_dt = QDateTime.fromString(s_text, "yyyy-MM-dd_HH:mm:ss") + e_dt = QDateTime.fromString(e_text, "yyyy-MM-dd_HH:mm:ss") + if not s_dt.isValid(): + s_dt = QDateTime.fromString(s_text, "yyyy-MM-dd HH:mm:ss") + if not e_dt.isValid(): + e_dt = QDateTime.fromString(e_text, "yyyy-MM-dd HH:mm:ss") + except Exception: + s_dt = e_dt = QDateTime.currentDateTime() + + vbox = QVBoxLayout(dlg) + start_edit = QDateTimeEdit(s_dt, dlg) + end_edit = QDateTimeEdit(e_dt, dlg) + + start_edit.setCalendarPopup(True) + end_edit.setCalendarPopup(True) + start_edit.setDisplayFormat("yyyy-MM-dd_HH:mm:ss") + end_edit.setDisplayFormat("yyyy-MM-dd_HH:mm:ss") + + vbox.addWidget(start_edit) + vbox.addWidget(end_edit) + + btn_row = QHBoxLayout() + ok_btn = QPushButton("OK", dlg) + cancel_btn = QPushButton("Cancel", dlg) + btn_row.addWidget(ok_btn) + btn_row.addWidget(cancel_btn) + vbox.addLayout(btn_row) + + ok_btn.clicked.connect(lambda: self._set_time_window(start_edit, end_edit, dlg)) + cancel_btn.clicked.connect(dlg.reject) + + dlg.exec() + + def _set_time_window(self, start_edit, end_edit, dlg: QDialog): + """ + UI handler: validate and set selected time window into UI. + + Arguments: + start_edit (QDateTimeEdit): Start time widget. + end_edit (QDateTimeEdit): End time widget. + dlg (QDialog): Dialog to accept/close. + """ + if end_edit.dateTime() < start_edit.dateTime(): + QMessageBox.warning(dlg, "Time error", + "End time cannot be earlier than start time.\nPlease select again.") + return + + s = start_edit.dateTime().toString("yyyy-MM-dd_HH:mm:ss") + e = end_edit.dateTime().toString("yyyy-MM-dd_HH:mm:ss") + self.ui.timeWindowButton.setText(f"{s} to {e}") + self.ui.timeWindowValue.setText(f"{s} to {e}") + dlg.accept() + + # ========================================================== + # Data interval popup + # ========================================================== + def _open_data_interval_dialog(self): + """ + UI handler: prompt for data interval (seconds) and update UI. + """ + # Extract current value from button text ("30 s" → 30) + current_text = self.ui.dataIntervalButton.text().replace(" s", "").strip() + try: + current_val = int(current_text) + except ValueError: + current_val = 1 # fallback if parsing fails + + val, ok = QInputDialog.getInt( + self.ui.dataIntervalButton, + "Data interval", + "Input interval (seconds):", + current_val, # prefill with current value + 1, + 999_999, + ) + if ok: + text = f"{val} s" + self.ui.dataIntervalButton.setText(text) + self.ui.dataIntervalValue.setText(text) + + # endregion + + # region Config and PEA Processing + + def extract_ui_values(self, rnx_path): + """ + Extract current UI values, parse/normalize them, and return as dataclass. + + Arguments: + rnx_path (str): Selected RINEX observation file path. + + Returns: + ExtractedInputs: Dataclass containing parsed fields and raw strings. + + """ + # Extract user input from the UI and assign it to class variables. + mode_raw = self.ui.Mode.currentText() if self.ui.Mode.currentText() != "Select one" else "Static" + + # Get constellations from the actual dropdown selections, not the label + constellations_raw = "" + combo = self.ui.Constellations_2 + if hasattr(combo, '_constellation_model') and combo._constellation_model: + model = combo._constellation_model + selected = [model.item(i).text() for i in range(model.rowCount()) if + model.item(i).checkState() == Qt.Checked] + constellations_raw = ", ".join(selected) + else: + # Fallback to the label text if no custom model exists + constellations_raw = self.ui.constellationsValue.text() + time_window_raw = self.ui.timeWindowValue.text() # Get from button, not value label + epoch_interval_raw = self.ui.dataIntervalButton.text() # Get from button, not value label + receiver_type = self.ui.receiverTypeValue.text() + antenna_type = self.ui.antennaTypeValue.text() + antenna_offset_raw = self.ui.antennaOffsetButton.text() # Get from button, not value label + ppp_provider = self.ui.PPP_provider.currentText() if self.ui.PPP_provider.currentText() != "Select one" else "" + ppp_series = self.ui.PPP_series.currentText() if self.ui.PPP_series.currentText() != "Select one" else "" + ppp_project = self.ui.PPP_project.currentText() if self.ui.PPP_project.currentText() != "Select one" else "" + + # Parsed values + start_epoch, end_epoch = self.parse_time_window(time_window_raw) + antenna_offset = self.parse_antenna_offset(antenna_offset_raw) + epoch_interval = int(epoch_interval_raw.replace("s", "").strip()) + marker_name = self.extract_marker_name(rnx_path) + mode = self.determine_mode_value(mode_raw) + + # Returned the values found as a dataclass for easier access + return self.ExtractedInputs( + marker_name=marker_name, + start_epoch=start_epoch, + end_epoch=end_epoch, + epoch_interval=epoch_interval, + antenna_offset=antenna_offset, + mode=mode, + constellations_raw=constellations_raw, + receiver_type=receiver_type, + antenna_type=antenna_type, + ppp_provider=ppp_provider, + ppp_series=ppp_series, + ppp_project=ppp_project, + rnx_path=rnx_path, + output_path=str(self.output_dir), + ) + + def on_show_config(self): + """ + UI handler: reload config, apply UI values, write changes, then open the YAML. + """ + Logger.terminal("📄 Opening YAML configuration file...") + # Reload disk version before overwriting with GUI changes + self.execution.reload_config() + inputs = self.extract_ui_values(self.rnx_file) + self.execution.apply_ui_config(inputs) + self.execution.write_cached_changes() + + # Execution class will throw error when instantiated if the file doesn't exist and it can't create it + # This code is run after Execution class is instantiated within this file, thus never will occur + if not os.path.exists(GENERATED_YAML): + QMessageBox.warning( + None, + "File not found", + f"The file {GENERATED_YAML} does not exist." + ) + return + + self.on_open_config_in_editor(self.config_path) + + def on_open_config_in_editor(self, file_path): + """ + Open the config YAML file in the OS default editor/viewer. + + Arguments: + file_path (str): Absolute or relative path to the YAML file. + """ + import subprocess + import platform + + try: + abs_path = os.path.abspath(file_path) + + # Open the file with the appropriate method for the operating system + if platform.system() == "Windows": + os.startfile(abs_path) + return + + if platform.system() == "Darwin": # macOS + subprocess.run(["open", abs_path]) + + else: # Linux and other Unix-like systems + # When compiled with pyinstaller, LD_LIBRARY_PATH is modified which prevents external app opening + env = os.environ.copy() + original = env.get("LD_LIBRARY_PATH_ORIG") + if original: + env["LD_LIBRARY_PATH"] = original # Restore original value + else: + env.pop("LD_LIBRARY_PATH", None) # Clear the value to use sys defaults + subprocess.run(["xdg-open", abs_path], env=env) + + except Exception as e: + error_message = f"Cannot open config file:\n{file_path}\n\nError: {str(e)}" + Logger.terminal(f"Error: {error_message}") + QMessageBox.critical( + None, + "Error Opening File", + error_message + ) + + def on_run_pea(self): + """ + UI handler: validate time window and config, apply UI, then emit pea_ready. + """ + raw = self.ui.timeWindowValue.text() + + # --- Parse time window --- + try: + start_str, end_str = raw.split("to") + start_time = datetime.strptime(start_str.strip(), "%Y-%m-%d_%H:%M:%S") + end_time = datetime.strptime(end_str.strip(), "%Y-%m-%d_%H:%M:%S") + except ValueError: + QMessageBox.warning( + None, + "Format error", + "Time window must be in the format:\n" + "YYYY-MM-DD_HH:MM:SS to YYYY-MM-DD_HH:MM:SS" + ) + return + + if start_time > end_time: + QMessageBox.warning(None, "Time error", "Start time cannot be later than end time.") + return + + if not getattr(self, "config_path", None): + QMessageBox.warning( + None, + "No config file", + "Please click Show config and select a YAML file first." + ) + return + + # Store time window so MainWindow can use it later + self.start_time = start_time + self.end_time = end_time + + # --- Write updated config --- + try: + self.execution.reload_config() + inputs = self.extract_ui_values(self.rnx_file) + self.execution.apply_ui_config(inputs) # config only, no product archiving here + self.execution.write_cached_changes() + except Exception as e: + Logger.terminal(f"⚠️ Failed to apply config: {e}") + return + + # --- Emit signal for MainWindow --- + self.pea_ready.emit() + + # endregion + + # region Utility Functions + + @staticmethod + def _set_combobox_by_value(combo: QComboBox, value: str): + """ + Helper: find a value in a combo and set current index if present. + + Arguments: + combo (QComboBox): Target combo box. + value (str): Text to search. + """ + if value is None: + return + idx = combo.findText(value) + if idx != -1: + combo.setCurrentIndex(idx) + + @staticmethod + def _select_rnx_file(parent) -> str: + """ + Open a file dialog to select a RINEX observation file. + + Arguments: + parent (Any): Parent widget. + + Returns: + str: Selected file path or empty string. + + """ + path, _ = QFileDialog.getOpenFileName( + parent, + "Select RINEX Observation File", + "", + "RINEX Observation Files (*.rnx *.rnx.gz *.[0-9][0-9]o *.[0-9][0-9]o.gz *.obs *.obs.gz);;All Files (*.*)" + ) + return path or "" + + @staticmethod + def _select_output_dir(parent) -> str: + """ + Open a directory dialog to select the output folder. + + Arguments: + parent (Any): Parent widget. + + Returns: + str: Selected directory path or empty string. + + """ + path = QFileDialog.getExistingDirectory(parent, "Select Output Directory") + return path or "" + + @staticmethod + def determine_mode_value(mode_raw: str) -> int: + """ + Map a mode label to its numeric value used by backend. + + Arguments: + mode_raw (str): One of 'Static', 'Kinematic', 'Dynamic'. + + Returns: + int: 0 for Static, 30 for Kinematic, 100 for Dynamic. + + Example: + >>> determine_mode_value("Static") + 0 + """ + if mode_raw == "Static": + return 0 + elif mode_raw == "Kinematic": + return 30 + elif mode_raw == "Dynamic": + return 100 + else: + raise ValueError(f"Unknown mode: {mode_raw!r}") + + @staticmethod + def extract_marker_name(rnx_path: str) -> str: + """ + Extract a 4-char site code (marker) from a RINEX filename. + + Arguments: + rnx_path (str): RNX file path. If empty/invalid, returns 'TEST'. + + Returns: + str: Upper-cased 4-char marker or 'TEST' when not found. + + Example: + >>> extract_marker_name("ALIC00AUS_R_20250190000_01D_30S_MO.rnx.gz") + 'ALIC' + """ + if not rnx_path: + return "TEST" + stem = Path(rnx_path).stem # drops .gz/.rnx + m = re.match(r"([A-Za-z]{4})", stem) + return m.group(1).upper() if m else "TEST" + + @staticmethod + def parse_time_window(time_window_raw: str): + """ + Convert 'start_time to end_time' into (start_epoch, end_epoch) strings. + + Arguments: + time_window_raw (str): e.g., 'YYYY-MM-DD_HH:MM:SS to YYYY-MM-DD_HH:MM:SS'. + + Returns: + tuple[str, str]: (start_epoch, end_epoch) with underscores preserved for UI. + + Example: + >>> parse_time_window("2025-01-01_00:00:00 to 2025-01-02_00:00:00") + ('2025-01-01 00:00:00', '2025-01-02 00:00:00') + """ + try: + start, end = map(str.strip, time_window_raw.split("to")) + + # Replace underscores with spaces in datetime strings + start = start.replace("_", " ") + end = end.replace("_", " ") + return start, end + except ValueError: + raise ValueError("Invalid time_window format. Expected: 'start_time to end_time'") + + @staticmethod + def parse_antenna_offset(antenna_offset_raw: str): + """ + Convert 'e, n, u' string into [e, n, u] floats. + + Arguments: + antenna_offset_raw (str): e.g., '0.0, 0.0, 1.234'. + + Returns: + list[float]: [e, n, u] in metres. + + Example: + >>> parse_antenna_offset("0.1, -0.2, 1.0") + [0.1, -0.2, 1.0] + """ + try: + e, n, u = map(str.strip, antenna_offset_raw.split(",")) + return [float(e), float(n), float(u)] + except ValueError: + raise ValueError("Invalid antenna offset format. Expected: 'e, n, u'") + + @dataclass + class ExtractedInputs: + """ + Dataclass container for parsed UI values and raw strings. + """ + # Parsed / derived values + marker_name: str + start_epoch: str + end_epoch: str + epoch_interval: int + antenna_offset: list[float] + mode: int + + # Raw strings / controls that are needed downstream + constellations_raw: str + receiver_type: str + antenna_type: str + ppp_provider: str + ppp_series: str + ppp_project: str + + # File paths associated to this run + rnx_path: str + output_path: str + + # endregion + + # region Statics + + @staticmethod + def _get_mode_items() -> List[str]: + """ + Provide available processing modes for the UI combo. + + Returns: + list[str]: ['Static', 'Kinematic', 'Dynamic'] + + Example: + >>> InputController._get_mode_items() + ['Static', 'Kinematic', 'Dynamic'] + """ + return ["Static", "Kinematic", "Dynamic"] + + @staticmethod + def _get_constellations_items() -> List[str]: + """ + Provide available GNSS constellations for the UI combo. + + Arguments: + None + + Returns: + list[str]: ['GPS', 'GAL', 'GLO', 'BDS', 'QZS'] + + Example: + >>> InputController._get_constellations_items() + ['GPS', 'GAL', 'GLO', 'BDS', 'QZS'] + """ + return ["GPS", "GAL", "GLO", "BDS", "QZS"] + + def _get_ppp_provider_items(self) -> List[str]: + """ + Provide available PPP providers from the cached products DataFrame. + + Returns: + list[str]: Provider names; empty when products list is not yet available. + + Example: + >>> ctrl._get_ppp_provider_items() + """ + if hasattr(self, "valid_analysis_centers") and self.valid_analysis_centers: + return self.valid_analysis_centers + return [] + + @staticmethod + def _get_ppp_series_items() -> List[str]: + """ + Provide available PPP series codes for the UI combo. + + Returns: + list[str]: ['ULT', 'RAP', 'FIN'] + + Example: + >>> InputController._get_ppp_series_items() + ['ULT', 'RAP', 'FIN'] + """ + return ["ULT", "RAP", "FIN"] + + # endregion + + +class CredentialsDialog(QDialog): + """ + UI controller class CredentialsDialog. + """ + + def __init__(self, parent=None): + """ + UI handler: initialize credential input widgets and layout. + + Arguments: + parent (Any): Optional parent widget. + """ + super().__init__(parent) + self.setWindowTitle("CDDIS Credentials") + + layout = QVBoxLayout() + + # Username + layout.addWidget(QLabel("Username:")) + self.username_input = QLineEdit() + layout.addWidget(self.username_input) + + # Password + layout.addWidget(QLabel("Password:")) + self.password_input = QLineEdit() + self.password_input.setEchoMode(QLineEdit.Password) + layout.addWidget(self.password_input) + + # Confirm button + self.confirm_button = QPushButton("Save") + self.confirm_button.clicked.connect(self.save_credentials) + layout.addWidget(self.confirm_button) + + self.setLayout(layout) + + def save_credentials(self): + """ + UI handler: validate username/password, save to netrc, and close dialog. + """ + username = self.username_input.text().strip() + password = self.password_input.text().strip() + + if not username or not password: + QMessageBox.warning(self, "Error", "Username and password cannot be empty") + return + + # ✅ Save correctly in one go (Windows will write both %USERPROFILE%\\.netrc and %USERPROFILE%\\_netrc; + # macOS/Linux will write ~/.netrc and automatically chmod 600; both URS and CDDIS entries are written) + try: + paths = save_earthdata_credentials(username, password) + except Exception as e: + QMessageBox.critical(self, "Save failed", f"❌ Failed to save credentials:\n{e}") + return + + QMessageBox.information(self, "Success", + "✅ Credentials saved to:\n" + "\n".join(str(p) for p in paths)) + self.accept() + + +# Minimal unified stop entry for InputController background worker +def _safe_call_stop(obj): + """ + Safely call .stop() on an object if present, ignoring exceptions. + + Arguments: + obj (Any): Object that may implement stop(). + """ + try: + if obj is not None and hasattr(obj, "stop"): + obj.stop() + except Exception: + pass + + +def stop_all(self): + """ + Best-effort stop for the metadata PPPWorker started by the controller. + + Arguments: + self (InputController): Controller instance owning the worker/thread. + """ + try: + if hasattr(self, "worker"): + _safe_call_stop(self.worker) + # Restore cursor when stopping + if hasattr(self, "parent"): + self.parent.setCursor(Qt.CursorShape.ArrowCursor) + except Exception: + pass + + +# Bind without touching existing class body +setattr(InputController, "stop_all", stop_all) \ No newline at end of file diff --git a/scripts/GinanUI/app/controllers/visualisation_controller.py b/scripts/GinanUI/app/controllers/visualisation_controller.py new file mode 100644 index 000000000..0519207d2 --- /dev/null +++ b/scripts/GinanUI/app/controllers/visualisation_controller.py @@ -0,0 +1,320 @@ +# app/controllers/visualisation_controller.py +"""Controller responsible for everything inside the visualisation panel. + +Responsibilities +---------------- +1. Embed one of the generated HTML files into the QTextEdit area. +2. Maintain a list (indexed) of available HTML visualisations. +3. Provide a double-click handler and an explicit *Open* action that open the + current html in the user's default browser. + +NOTE: UI widgets for selecting visualisation (e.g. a ComboBox or QListWidget) + and an *Open* button are **not** yet present in the .ui file. This + controller exposes stub `bind_open_button()` / `bind_selector()` helpers + which can be called once those widgets are added. +""" +from __future__ import annotations +import os +import platform +import subprocess +import sys +from pathlib import Path +from typing import List, Sequence, Optional +from PySide6.QtCore import QRect, QUrl, QObject, QEvent +from PySide6.QtGui import QDesktopServices +from PySide6.QtWidgets import QTextEdit, QPushButton, QComboBox, QApplication +from PySide6.QtWebEngineWidgets import QWebEngineView +from scripts.GinanUI.app.utils.logger import Logger + +HERE = Path(__file__).resolve() +ROOT = HERE.parents[2] +DEFAULT_OUT_DIR = ROOT / "tests" / "resources" / "outputData" / "visual" + + +class VisualisationController(QObject): + """ + Manage interactions and rendering inside the visualisation panel. + + Arguments: + ui (object): The main window UI object that exposes the visualisation widgets (e.g., `visualisationTextEdit`). + parent_window (QObject): The parent window/controller used as the QObject parent. + + Returns: + None: Constructor returns nothing. + + Example: + Function itself returns None; example shows how to instantiate and inspect state. + >>> controller = VisualisationController(ui, parent_window) + >>> controller.html_files + [] + """ + + def __init__(self, ui, parent_window): + """ + Initialize controller state and install required event filters. + + Arguments: + ui: The main window UI instance. + parent_window: The parent QMainWindow or controller. + + Example: + Function itself returns None; example shows initial empty html_files. + >>> ctrl = VisualisationController(ui, parent_window) + >>> ctrl.html_files + [] + """ + super().__init__(parent_window) + self.ui = ui # Ui_MainWindow instance + self.parent = parent_window + self.html_files: List[str] = [] # paths of available visualisations + self.current_index: Optional[int] = None + self.external_base_url: Optional[str] = None + self._selector: Optional[QComboBox] = None + self._open_button: Optional[QPushButton] = None + + # --------------------------------------------------------------------- + # Public API (to be called from MainWindow / other controllers) + # --------------------------------------------------------------------- + def set_html_files(self, paths: Sequence[str]): + """ + Register available HTML visualisation files and display the first one. + + Arguments: + paths (Sequence[str]): List of file paths to HTML visualisations. + + Example: + Function itself returns None; example shows state update after call. + >>> controller.set_html_files(["plot1.html", "plot2.html"]) + >>> controller.current_index + 0 + """ + self.html_files = list(paths) + # Refresh selector if bound + if self._selector: + self._refresh_selector() + if self.html_files: + self.display_html(0) + # Enable widgets once we have plots + if self._selector: + self._selector.setEnabled(True) + if self._open_button: + self._open_button.setEnabled(True) + else: + # Disable widgets if no plots available + if self._selector: + self._selector.setEnabled(False) + if self._open_button: + self._open_button.setEnabled(False) + + def display_html(self, index: int): + """ + Embed the HTML file at the given index into the visualisation panel. + + Arguments: + index (int): Zero-based index into `self.html_files`. + + Example: + Function itself returns None; example shows updated index. + >>> controller.display_html(0) + >>> controller.current_index + 0 + """ + if not isinstance(index, int) or not (0 <= index < len(self.html_files)): + return + file_path = self.html_files[index] + self.current_index = index + self._embed_html(file_path) + + def open_current_external(self): + """ + Open the currently displayed HTML in the system’s default web browser. + + Example: + Function itself returns None; example shows that return value is None. + >>> controller.open_current_external() is None + True + """ + if self.current_index is None: + return + path = self.html_files[self.current_index] + try: + url = QUrl.fromLocalFile(Path(path).resolve()) + + # Open the file with the appropriate method for the operating system + if platform.system() == "Windows": + # sys._MEIPASS and some dll file need to be changed + QDesktopServices.openUrl(url) + + elif platform.system() == "Darwin": + # sys._MEIPASS but might also work without any changes + QDesktopServices.openUrl(url) + + else: + # When compiled with pyinstaller, LD_LIBRARY_PATH is modified which prevents external app opening + env = os.environ.copy() + original = env.get("LD_LIBRARY_PATH_ORIG") + if original: + env["LD_LIBRARY_PATH"] = original # Restore original value + else: + env.pop("LD_LIBRARY_PATH", None) # Clear the value to use sys defaults + subprocess.run(["xdg-open", url.url()], env=env) + except Exception as e: + Logger.console(f"Error occurred trying to open in browser: {e}") + + # ------------------------------------------------------------------ + # Helpers for wiring additional UI elements + # ------------------------------------------------------------------ + def bind_open_button(self, button: QPushButton): + """ + Connect an *Open* button to open the current visualisation externally. + + Arguments: + button (QPushButton): The push button to connect to the handler. + + Example: + Function itself returns None; example shows valid binding. + >>> controller.bind_open_button(ui.openButton) is None + True + """ + self._open_button = button + button.clicked.connect(self.open_current_external) + button.setEnabled(False) + + def bind_selector(self, combo: QComboBox): + """ + Bind a QComboBox selector to manage and display HTML visualisations. + + Arguments: + combo (QComboBox): The combo box used as selector. + + Example: + Function itself returns None; example shows valid selector binding. + >>> controller.bind_selector(ui.comboBox) is None + True + """ + self._selector = combo + + def safe_display(): + data = combo.currentData() + if isinstance(data, int): # Only proceed if it's a valid index + self.display_html(data) + + combo.currentIndexChanged.connect(lambda _: safe_display()) + combo.setEnabled(False) + self._refresh_selector() + + def _refresh_selector(self): + """ + Populate the selector combo box with available HTML files. + + Example: + # Function itself returns None; example shows refresh success. + >>> controller._refresh_selector() is None + True + """ + if not self._selector: + return + self._selector.clear() + for idx, path in enumerate(self.html_files): + self._selector.addItem(f"#{idx} – {os.path.basename(path)}", userData=idx) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _embed_html(self, file_path: str): + """ + Embed an HTML file inside the dedicated QWebEngineView in the UI. + + Arguments: + file_path (str): Absolute or relative path to a local HTML file to display. + """ + # Use the QWebEngineView that is defined in main_window.ui + webview: QWebEngineView = self.ui.webEngineView + + # Resolve to an absolute path so QWebEngineView can load it reliably + url = QUrl.fromLocalFile(str(Path(file_path).resolve())) + webview.setUrl(url) + + # Optional zoom factor + webview.setZoomFactor(0.8) + + # Install event filter if you still want to intercept events later + webview.installEventFilter(self) + + # Keep a reference to avoid GC (and for later access) + self._webview = webview + + # ------------------------------------------------------------------ + # Optional configuration + # ------------------------------------------------------------------ + def set_external_base_url(self, url: str): + """ + Set a base HTTP URL to prefer when opening visualisations externally. + + Arguments: + url (str): Base URL (a trailing slash is appended if missing). + + Example: + Function itself returns None; example shows URL assignment. + >>> controller.set_external_base_url("http://localhost:8000/") + >>> controller.external_base_url + 'http://localhost:8000/' + """ + if not url.endswith('/'): + url += '/' + self.external_base_url = url + + def build_from_execution(self): + """ + Generate visualisation HTML files from the execution model and load them. + + Example: + Function itself returns None; example checks that call succeeds. + >>> controller.build_from_execution() is None + True + """ + try: + exec_obj = getattr(self.parent, "execution", None) + if exec_obj is None: + from PySide6.QtWidgets import QMessageBox + QMessageBox.warning(self.ui, "Plot", "execution object is not set") + return + + new_html_paths = exec_obj.build_pos_plots() # default output to tests/resources/outputData/visual + + existing_html_paths = self._find_existing_html_files() + + all_html_paths = list(set(new_html_paths + existing_html_paths)) + + all_html_paths.sort(key=lambda x: os.path.basename(x)) + + self.set_html_files(all_html_paths) + + except Exception as e: + from PySide6.QtWidgets import QMessageBox + QMessageBox.critical(self.ui, "Plot Error", str(e)) + + def _find_existing_html_files(self): + """ + Locate and return paths of existing visualisation HTML files. + + Returns: + list[str]: A list of absolute paths to discovered HTML files.git + + Example: + Function returns a list; example checks returned type. + >>> isinstance(controller._find_existing_html_files(), list) + True + """ + existing_files = [] + + default_visual_dir = DEFAULT_OUT_DIR + if default_visual_dir.exists(): + for html_file in default_visual_dir.glob("*.html"): + existing_files.append(str(html_file)) + + if self.external_base_url: + pass + + return existing_files \ No newline at end of file diff --git a/scripts/GinanUI/app/main_window.py b/scripts/GinanUI/app/main_window.py new file mode 100644 index 000000000..430b9d47a --- /dev/null +++ b/scripts/GinanUI/app/main_window.py @@ -0,0 +1,497 @@ +from pathlib import Path +from typing import Optional + +from PySide6.QtCore import QUrl, Signal, QThread, Slot, Qt, QRegularExpression +from scripts.GinanUI.app.utils.logger import Logger +from PySide6.QtWidgets import QMainWindow, QDialog, QVBoxLayout, QPushButton, QComboBox +from PySide6.QtWebEngineWidgets import QWebEngineView +from PySide6.QtGui import QTextCursor, QTextDocument + +from scripts.GinanUI.app.utils.cddis_credentials import validate_netrc as gui_validate_netrc +from scripts.GinanUI.app.models.execution import Execution +from scripts.GinanUI.app.utils.toast import show_toast +from scripts.GinanUI.app.utils.ui_compilation import compile_ui +from scripts.GinanUI.app.controllers.input_controller import InputController +from scripts.GinanUI.app.controllers.visualisation_controller import VisualisationController +from scripts.GinanUI.app.utils.cddis_email import get_username_from_netrc, write_email, test_cddis_connection +from scripts.GinanUI.app.utils.workers import PeaExecutionWorker, DownloadWorker +from scripts.GinanUI.app.models.archive_manager import archive_products_if_selection_changed, archive_products, archive_old_outputs +from scripts.GinanUI.app.models.execution import INPUT_PRODUCTS_PATH +from scripts.GinanUI.app.utils.logger import Logger + +# Optional toggle for development visualization testing +test_visualisation = False + + +def setup_main_window(): + import sys + IS_FROZEN = getattr(sys, 'frozen', False) + + if not IS_FROZEN: + # Only compile UI during development + compile_ui() + + from scripts.GinanUI.app.views.main_window_ui import Ui_MainWindow + return Ui_MainWindow() + +class MainWindow(QMainWindow): + log_signal = Signal(str) + + def __init__(self): + super().__init__() + + # Setup UI + self.ui = setup_main_window() + self.ui.setupUi(self) + + # Add rounded corners to UI elements + self.setStyleSheet(""" + QPushButton { + border-radius: 4px; + } + QPushButton:disabled { + border-radius: 4px; + } + QTextEdit { + border-radius: 4px; + } + QComboBox { + border-radius: 4px; + } + """) + + # Fix macOS tab widget styling + self._fix_macos_tab_styling() + + # Initialize the Logger system for easy logging throughout the app + Logger.initialise(self) + + # Controllers + self.execution = Execution() + self.inputCtrl = InputController(self.ui, self, self.execution) + self.visCtrl = VisualisationController(self.ui, self) + + # Connect ready signals + self.inputCtrl.ready.connect(self.on_files_ready) + self.inputCtrl.pea_ready.connect(self._on_process_clicked) + + # State + self.rnx_file: Optional[str] = None + self.output_dir: Optional[str] = None + self.download_progress: dict[str, int] = {} # track per-file progress + self.is_processing = False + self.atx_required_for_rnx_extraction = False # File required to extract info from RINEX + self.metadata_downloaded = False + self.offline_mode = False # Track if running without internet + + # Visualisation widgets + + self.visCtrl.bind_open_button(self.ui.openInBrowserBtn) + + self.visCtrl.bind_selector(self.ui.visSelector) + + archive_products(INPUT_PRODUCTS_PATH, "startup_archival", True) + + # Validate connection then start metadata download in a separate thread + self._validate_cddis_credentials_once() + + # Only start metadata download if we have internet connection + if not self.offline_mode: + self.metadata_thread = QThread() + self.metadata_worker = DownloadWorker() + self.metadata_worker.moveToThread(self.metadata_thread) + + # Signals + self.metadata_thread.started.connect(self.metadata_worker.run) + self.metadata_worker.progress.connect(self._on_download_progress) + self.metadata_worker.finished.connect(self._on_metadata_download_finished) + self.metadata_worker.atx_downloaded.connect(self._on_atx_downloaded) + + # Cleanup + self.metadata_worker.finished.connect(self.metadata_thread.quit) + self.metadata_worker.finished.connect(self.metadata_worker.deleteLater) + self.metadata_thread.finished.connect(self.metadata_thread.deleteLater) + self.metadata_thread.start() + else: + Logger.terminal("⚠️ Skipping metadata download - running in offline mode") + + # Added: wire an optional stop-all button if present in the UI + if hasattr(self.ui, "stopAllButton") and self.ui.stopAllButton: + self.ui.stopAllButton.clicked.connect(self.on_stopAllClicked) + elif hasattr(self.ui, "btnStopAll") and self.ui.btnStopAll: + self.ui.btnStopAll.clicked.connect(self.on_stopAllClicked) + + def log_message(self, msg: str, channel = "terminal"): + """Append a log line to the specified text channel""" + if channel == "terminal": + self.ui.terminalTextEdit.append(msg) + elif channel == "console": + self.ui.consoleTextEdit.append(msg) + else: + raise ValueError("[MainWindow] Invalid channel for log_message") + + def _set_processing_state(self, processing: bool): + """Enable/disable UI elements during processing""" + self.is_processing = processing + + # Disable/enable the process button + self.ui.processButton.setEnabled(not processing) + + # Optionally disable other critical UI elements during processing + self.ui.observationsButton.setEnabled(not processing) + self.ui.outputButton.setEnabled(not processing) + self.ui.showConfigButton.setEnabled(not processing) + + # Update button text to show processing state + if processing: + self.ui.processButton.setText("Processing...") + # Set cursor to waiting cursor for visual feedback + self.setCursor(Qt.CursorShape.WaitCursor) + else: + self.ui.processButton.setText("Process") + self.setCursor(Qt.CursorShape.ArrowCursor) + + def on_files_ready(self, rnx_path: str, out_path: str): + self.rnx_file = rnx_path + self.output_dir = out_path + + def _on_process_clicked(self): + if not self.rnx_file or not self.output_dir: + Logger.terminal("⚠️ Please select RINEX and output directory first.") + return + + # Check if in offline mode + if self.offline_mode: + Logger.terminal("⚠️ Cannot process: Ginan-UI is running in offline mode (no internet connection)") + from scripts.GinanUI.app.utils.toast import show_toast + show_toast(self, "⚠️ Processing requires internet connection", 4000) + + from PySide6.QtWidgets import QMessageBox + msg = QMessageBox(self) + msg.setIcon(QMessageBox.Icon.Warning) + msg.setWindowTitle("Offline Mode") + msg.setText("Processing requires an internet connection to download PPP products from CDDIS.") + msg.setInformativeText("Please check your internet connection and restart Ginan-UI.") + msg.setStandardButtons(QMessageBox.StandardButton.Ok) + msg.exec() + return + + # Prevent multiple simultaneous processing + if self.is_processing: + Logger.terminal("⚠️ Processing already in progress. Please wait...") + return + + # Lock the "Process" button and set processing state + self._set_processing_state(True) + + # Get PPP params from UI + ac = self.ui.PPP_provider.currentText() + project = self.ui.PPP_project.currentText() + series = self.ui.PPP_series.currentText() + + # Archive old products if needed + current_selection = {"ppp_provider": ac, "ppp_project": project, "ppp_series": series} + archive_dir = archive_products_if_selection_changed( + current_selection, getattr(self, "last_ppp_selection", None), INPUT_PRODUCTS_PATH + ) + self.last_ppp_selection = current_selection + if archive_dir: + Logger.terminal(f"📦 Archived old PPP products → {archive_dir}") + + output_archive = archive_old_outputs(Path(self.output_dir), archive_dir) + if output_archive: + Logger.terminal(f" Archived old outputs → {output_archive}") + + # List products to be downloaded + x = self.inputCtrl.products_df + products = x.loc[(x["analysis_center"] == ac) & (x["project"] == project) & (x["solution_type"] == series)].drop_duplicates() + + # Reset progress + self.download_progress.clear() + + # Start download in background + self.download_thread = QThread() + self.download_worker = DownloadWorker(products=products, start_epoch=self.inputCtrl.start_time, end_epoch=self.inputCtrl.end_time) + self.download_worker.moveToThread(self.download_thread) + + # Signals + self.download_thread.started.connect(self.download_worker.run) + self.download_worker.progress.connect(self._on_download_progress) + self.download_worker.finished.connect(self._on_download_finished) + + # Cleanup + self.download_worker.finished.connect(self.download_thread.quit) + self.download_worker.finished.connect(self.download_worker.deleteLater) + self.download_thread.finished.connect(self.download_thread.deleteLater) + + Logger.terminal("📡 Starting PPP product downloads...") + self.download_thread.start() + + @Slot(str, int) + def _on_download_progress(self, filename: str, percent: int): + """Update progress display in-place at the bottom of the UI terminal.""" + self.download_progress[filename] = percent + + total_length = 20 + filled_length = int(percent/100 * total_length) + bar = '[' + "█" * filled_length + "░" * (total_length - filled_length) + ']' + output = f"{filename[:30]} {bar} {percent:3d}%" + search_pattern = QRegularExpression(f"^{filename[:30]}.+%$") + + # Work with cursor & doc + cursor = self.ui.terminalTextEdit.textCursor() + cursor.movePosition(QTextCursor.End) + flags = QTextDocument.FindFlag.FindBackward + found_cursor = self.ui.terminalTextEdit.document().find(search_pattern, cursor, flags) + + on_latest_5_lines = self.ui.terminalTextEdit.document().blockCount() - found_cursor.blockNumber() <= 5 + if found_cursor.hasSelection() and on_latest_5_lines: + found_cursor.movePosition(QTextCursor.EndOfLine) # Replaces final percent symbol too + found_cursor.movePosition(QTextCursor.StartOfLine, QTextCursor.KeepAnchor) + found_cursor.removeSelectedText() + found_cursor.insertText(output) + else: # Make new progress bar + self.ui.terminalTextEdit.setTextCursor(cursor) + cursor.insertText("\n" + output) + cursor.movePosition(QTextCursor.End) + self.ui.terminalTextEdit.setTextCursor(cursor) + + def _on_atx_downloaded(self, filename: str): + self.atx_required_for_rnx_extraction = True + Logger.terminal(f"✅ ATX file {filename} installed - ready for RINEX parsing.") + + def _on_metadata_download_finished(self, message): + Logger.terminal(message) + self.metadata_downloaded = True + self.inputCtrl.try_enable_process_button() + + def _on_download_finished(self, message): + Logger.terminal(message) + self._start_pea_execution() + + def _on_download_error(self, msg): + Logger.terminal(f"⚠️ PPP download error: {msg}") + self._set_processing_state(False) + + def _start_pea_execution(self): + Logger.terminal("⚙️ Starting PEA execution in background...") + + self.thread = QThread() + self.worker = PeaExecutionWorker(self.execution) + self.worker.moveToThread(self.thread) + + self.thread.started.connect(self.worker.run) + self.worker.finished.connect(self._on_pea_finished) + + self.worker.finished.connect(self.thread.quit) + self.worker.finished.connect(self.worker.deleteLater) + self.thread.finished.connect(self.thread.deleteLater) + + self.thread.start() + + def _on_pea_finished(self): + Logger.terminal("✅ PEA processing completed.") + show_toast(self, "✅ PEA Processing complete!", 3000) + self._run_visualisation() + self._set_processing_state(False) + + def _on_pea_error(self, msg: str): + Logger.terminal(f"⚠️ PEA execution failed: {msg}") + self._set_processing_state(False) + + def _run_visualisation(self): + try: + Logger.terminal("📊 Generating plots from PEA output...") + html_files = self.execution.build_pos_plots() + if html_files: + self.visCtrl.set_html_files(html_files) + else: + Logger.terminal("⚠️ No plots found.") + except Exception as err: + Logger.terminal(f"⚠️ Plot generation failed: {err}") + + if test_visualisation: + try: + Logger.terminal("[Dev] Testing static visualisation...") + test_output_dir = Path(__file__).resolve().parents[1] / "tests" / "resources" / "outputData" + test_visual_dir = test_output_dir / "visual" + test_visual_dir.mkdir(parents=True, exist_ok=True) + self.visCtrl.build_from_execution() + Logger.terminal("[Dev] Static plot generation complete.") + except Exception as err: + Logger.terminal(f"[Dev] Test plot generation failed: {err}") + + def _validate_cddis_credentials_once(self): + """ + Validate CDDIS credentials and connectivity. + If no internet, app continues in offline mode with warning. + """ + ok, where = gui_validate_netrc() + if not ok and hasattr(self.ui, "cddisCredentialsButton"): + Logger.terminal("⚠️ No Earthdata credentials. Opening CDDIS Credentials dialog…") + self.ui.cddisCredentialsButton.click() + ok, where = gui_validate_netrc() + if not ok: + Logger.terminal(f"❌ Credentials invalid: {where}") + return + Logger.terminal(f"✅ Earthdata Credentials found: {where}") + + ok_user, email_candidate = get_username_from_netrc() + if not ok_user: + Logger.terminal(f"❌ Cannot read username from .netrc: {email_candidate}") + return + + # Wrap connection test in try-except to handle network errors gracefully + try: + ok_conn, why = test_cddis_connection() + if not ok_conn: + Logger.terminal( + f"❌ CDDIS connectivity check failed: {why}. Please verify Earthdata credentials via the CDDIS Credentials dialog." + ) + self._show_offline_warning("Connection test failed", why) + return + Logger.terminal(f"✅ CDDIS connectivity check passed in {why.split(' ')[-2]} seconds.") + + # Connection successful - set email + write_email(email_candidate) + Logger.terminal(f"✉️ EMAIL set to: {email_candidate}") + + except Exception as e: + # Network error (no internet, DNS failure, timeout, etc.) + error_msg = str(e) + Logger.terminal(f"⚠️ No internet connection detected: {error_msg}") + self._show_offline_warning("No internet connection", error_msg) + return + + def _show_offline_warning(self, title: str, details: str): + """ + Show a warning dialog when Ginan-UI starts without internet. + The app can continue to run, but very limited (some features are unavailable) + """ + from PySide6.QtWidgets import QMessageBox + + # Mark as offline mode + self.offline_mode = True + + msg = QMessageBox(self) + msg.setIcon(QMessageBox.Icon.Warning) + msg.setWindowTitle("Ginan-UI - No Internet Connection") + msg.setText( + "Ginan-UI requires internet access to function properly

    " + "The following features will be unavailable:" + ) + msg.setInformativeText( + "- Downloading PPP products from CDDIS
    " + "- Scanning for available analysis centers
    " + "- Retrieving GNSS data products

    " + "The application will continue to run in offline mode.
    " + "You can still view configurations and access local files." + ) + msg.setDetailedText(f"Error details:\n{title}: {details}") + msg.setStandardButtons(QMessageBox.StandardButton.Ok) + msg.setDefaultButton(QMessageBox.StandardButton.Ok) + + # Show the dialog + msg.exec() + + # Also show a toast notification + from scripts.GinanUI.app.utils.toast import show_toast + show_toast(self, "⚠️ Running in offline mode - limited functionality", 8000) + + # Added: unified stop entry, wired to an optional UI button + @Slot() + def on_stopAllClicked(self): + Logger.terminal("🛑 Stop requested — stopping all running tasks...") + + # Stop the metadata worker in InputController, if present + try: + if hasattr(self, "inputCtrl") and hasattr(self.inputCtrl, "stop_all"): + self.inputCtrl.stop_all() + except Exception: + pass + + # Stop PPP downloads, if running + try: + if hasattr(self, "download_worker") and self.download_worker is not None and hasattr(self.download_worker, "stop"): + self.download_worker.stop() + except Exception: + pass + + # Stop PEA execution, if running + try: + if hasattr(self, "worker") and self.worker is not None and hasattr(self.worker, "stop"): + self.worker.stop() + except Exception: + pass + + # Best-effort: ask Execution to stop any external process if supported + try: + if hasattr(self, "execution") and self.execution is not None and hasattr(self.execution, "stop_all"): + self.execution.stop_all() + except Exception: + pass + + # Restore UI state immediately + try: + self._set_processing_state(False) + except Exception: + pass + + def _fix_macos_tab_styling(self): + """ + Fix tab widget styling on macOS where native styling overrides custom stylesheets. + This method applies a comprehensive stylesheet directly to the QTabBar to ensure + consistent appearance across all platforms. + """ + import platform + + # On macOS, we need to be more aggressive with styling to override native appearance + if platform.system() == "Darwin": + # Import QStyleFactory to optionally force Fusion style + from PySide6.QtWidgets import QStyleFactory + + # Force Fusion style on the tab widget to disable native macOS rendering + fusion_style = QStyleFactory.create("Fusion") + if fusion_style: + self.ui.tabWidget.setStyle(fusion_style) + + # Apply comprehensive stylesheet to ensure consistent appearance + tab_bar_stylesheet = """ + QTabWidget::pane { + border: none; + background-color: #2c5d7c; + } + + QTabBar { + background-color: transparent; + alignment: left; + } + + QTabBar::tab { + background-color: #1a3a4d; + color: white; + padding: 8px 16px; + margin-right: 2px; + border: none; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + min-width: 60px; + } + + QTabBar::tab:selected { + background-color: #2c5d7c; + color: white; + font-weight: bold; + } + + QTabBar::tab:hover:!selected { + background-color: #234a5f; + } + + QTabBar::tab:!selected { + margin-top: 2px; + } + """ + + # Apply the stylesheet to the tab widget + self.ui.tabWidget.setStyleSheet(tab_bar_stylesheet) \ No newline at end of file diff --git a/scripts/GinanUI/app/models/__init__.py b/scripts/GinanUI/app/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/GinanUI/app/models/archive_manager.py b/scripts/GinanUI/app/models/archive_manager.py new file mode 100644 index 000000000..3fe4e6893 --- /dev/null +++ b/scripts/GinanUI/app/models/archive_manager.py @@ -0,0 +1,161 @@ +# app/utils/archive_manager.py + +from pathlib import Path +import shutil +from datetime import datetime +from typing import Optional, Dict, Any + +from scripts.GinanUI.app.utils.common_dirs import INPUT_PRODUCTS_PATH + +from scripts.GinanUI.app.utils.logger import Logger + + +def archive_old_outputs(output_dir: Path, visual_dir: Path = None): + """ + Moves existing output files to an archive directory to keep the workspace clean. + + THIS FUNCTION LOOKS FOR ALL TXT, LOG, JSON, POS files. + DON'T USE THE INPUT PRODUCTS DIRECTORY. + + :param output_dir: Path to the user-selected output directory. + :param visual_dir: Optional path to associated visualisation directory. + """ + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + archive_dir = output_dir / "archive" / timestamp + archive_dir.mkdir(parents=True, exist_ok=True) + + # Move .pos, .log, .txt, etc. from output_dir + moved_files = 0 + for ext in [".pos", ".POS", ".log", ".txt", ".json"]: + for file in output_dir.glob(f"*{ext}"): + shutil.move(str(file), archive_dir / file.name) + moved_files += 1 + + # Move HTML visual files (optional) + if visual_dir and visual_dir.exists(): + visual_archive = archive_dir / "visual" + visual_archive.mkdir(parents=True, exist_ok=True) + for html_file in visual_dir.glob("*.html"): + shutil.move(str(html_file), visual_archive / html_file.name) + moved_files += 1 + + if moved_files > 0: + Logger.console(f"📦 Archived {moved_files} old output file(s) to: {archive_dir}") + else: + Logger.console("📂 No previous outputs found to archive.") + +def archive_products(products_dir: Path = INPUT_PRODUCTS_PATH, reason: str = "manual", startup_archival: bool = False, + include_patterns: Optional[list[str]] = None) -> Optional[Path]: + """ + Archive GNSS product files from products_dir into a timestamped subfolder + under products_dir/archived/. + + :param products_dir: Directory containing GNSS product files + :param reason: String describing why the archive is happening (e.g., "rinex_change", "ppp_selection_change") + :param startup_archival: If True, archives files which are meant to be archived only once during app startup + :param include_patterns: Optional list of glob patterns to include when archiving + :return: Path to the archive folder if files were archived, else None + """ + if not products_dir.exists(): + Logger.console(f"Products dir {products_dir} does not exist.") + return None + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + archive_dir = products_dir / "archived" / f"{reason}_{timestamp}" + archive_dir.mkdir(parents=True, exist_ok=True) + + product_patterns = [ + "*.SP3", # precise orbit + "*.CLK", # clock files + "*.BIA", # biases + "*.ION", # ionosphere products (if used) + "*.TRO", # troposphere products (if used) + ] + + if startup_archival: + startup_patterns = [ + "finals.data.iau2000.txt", + "BRDC*.rnx", + "igs_satellite_metadata*.snx", + "igs20*.atx", + "tables/ALOAD*", + "tables/OLOAD*", + "tables/gpt_*.grd", + "tables/qzss_*" + "tables/igrf*coeffs.txt", + "tables/DE436*", + "tables/fes2014*.dat", + "tables/opoleloadcoefcmcor.txt", + "tables/sat_yaw*.snx", + "tables/bds_yaw*.snx", + "tables/qzss_yaw*.snx", + "tables/bds_yaw*.snx" + ] + # Ensure directories exist + products_dir.mkdir(parents=True, exist_ok=True) + (products_dir / "tables").mkdir(parents=True, exist_ok=True) + + # Scans every file and checks created within 7 days + for pattern in startup_patterns: + for file in products_dir.glob(pattern): + creation_time = datetime.fromtimestamp(file.stat().st_ctime) + if (datetime.now() - creation_time).days > 7: + Logger.console(f"Startup archival: {file.name} is older than 7 days, archiving.") + product_patterns.append(pattern) + + # Include explicit patterns if provided + if include_patterns: + product_patterns.extend(include_patterns) + + archived_files = [] + for pattern in product_patterns: + for file in products_dir.glob(pattern): + try: + target = archive_dir / file.name + shutil.move(str(file), str(target)) + archived_files.append(file.name) + except Exception as e: + Logger.console(f"Failed to archive {file.name}: {e}") + + if archived_files: + Logger.console(f"Archived {', '.join(archived_files)} → {archive_dir}") + return archive_dir + else: + Logger.console("No matching product files found to archive.") + return None + + +def archive_products_if_rinex_changed(current_rinex: Path, + last_rinex: Optional[Path], + products_dir: Path) -> Optional[Path]: + """ + If the RINEX file has changed since last load, archive the cached products. + """ + if last_rinex and current_rinex.resolve() == last_rinex.resolve(): + Logger.console("RINEX file unchanged — skipping product cleanup.") + return None + + Logger.console("RINEX file changed — archiving old products.") + # Shouldn't remove BRDC if date isn't changed but would require extracting current and last rnx + return archive_products(products_dir, reason="rinex_change", include_patterns=["BRDC*.rnx*"]) + + +def archive_products_if_selection_changed(current_selection: Dict[str, Any], + last_selection: Optional[Dict[str, Any]], + products_dir: Path) -> Optional[Path]: + """ + If the PPP product selection (AC/project/solution) has changed, archive the cached products. + Excludes BRDC and finals.data.iau2000.txt since they are reusable. + """ + if last_selection and current_selection == last_selection: + Logger.console("[Archiver] PPP product selection unchanged — skipping product cleanup.") + return None + + if last_selection: + diffs = {k: (last_selection.get(k), current_selection.get(k)) + for k in set(last_selection) | set(current_selection) + if last_selection.get(k) != current_selection.get(k)} + Logger.console(f"[Archiver] PPP selection changed → differences: {diffs}") + + return archive_products(products_dir,reason="ppp_selection_change") + diff --git a/scripts/GinanUI/app/models/dl_products.py b/scripts/GinanUI/app/models/dl_products.py new file mode 100644 index 000000000..fa726cf87 --- /dev/null +++ b/scripts/GinanUI/app/models/dl_products.py @@ -0,0 +1,489 @@ +import gzip, os, shutil, unlzw3, requests +import pandas as pd +import numpy as np +from bs4 import BeautifulSoup, SoupStrainer +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional, Callable, Generator, List + +from scripts.GinanUI.app.utils.cddis_email import get_netrc_auth +from scripts.GinanUI.app.utils.common_dirs import INPUT_PRODUCTS_PATH +from scripts.GinanUI.app.utils.gn_functions import GPSDate +from scripts.GinanUI.app.utils.logger import Logger + +BASE_URL = "https://cddis.nasa.gov/archive" +GPS_ORIGIN = np.datetime64("1980-01-06 00:00:00") # Magic date from gn_functions +MAX_RETRIES = 3 # download attempts +CHUNK_SIZE = 8192 # 8 KiB +COMPRESSED_FILETYPE = (".gz", ".gzip", ".Z") # ignore any others (maybe add crx2rnx using hatanaka package) + +METADATA = [ + "https://files.igs.org/pub/station/general/igs_satellite_metadata.snx", + "https://files.igs.org/pub/station/general/igs20.atx", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/OLOAD_GO.BLQ.gz", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/ALOAD_GO.BLQ.gz", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/igrf14coeffs.txt.gz", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/opoleloadcoefcmcor.txt.gz", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/fes2014b_Cnm-Snm.dat.gz", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/DE436.1950.2050.gz", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/gpt_25.grd.gz", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/bds_yaw_modes.snx.gz", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/qzss_yaw_modes.snx.gz", + "https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/sat_yaw_bias_rate.snx.gz", + "https://datacenter.iers.org/data/latestVersion/finals.data.iau2000.txt" +] + + +def date_to_gpswk(date: datetime) -> int: + return int(GPSDate(np.datetime64(date)).gpswk) + + +def gpswk_to_date(gps_week: int, gps_day: int = 0) -> datetime: + return GPSDate(GPS_ORIGIN + np.timedelta64(gps_week, "W") + np.timedelta64(gps_day, "D")).as_datetime + + +def str_to_datetime(date_time_str): + """ + :param date_time_str: YYYY-MM-DD_HH:mm:ss + :returns datetime: datetime.strptime() + """ + try: + # YYYY-dddHHmm format through datetime.strptime(date_time,"%Y%j%H%M") + return datetime.strptime(date_time_str, "%Y-%m-%d_%H:%M:%S") + except ValueError: + raise ValueError("Invalid datetime format. Use YYYY-MM-DDTHH:MM (e.g. 2025-05-01_00:00:00)") + + +def get_product_dataframe(start_time: datetime, end_time: datetime, target_files: List[str] = None) -> pd.DataFrame: + """ + Retrieves a DataFrame of available products for given time window and target files from CDDIS archive. + Filter the DataFrame (i.e. for a specific ppp provider) then use download_products() to download the files. + + :param start_time: the start of the time window (start_epoch) + :param end_time: the start of the time window (end_epoch) + :param target_files: list of target files to filter for, defaulted to ["CLK","BIA","SP3"] + :returns: dataframe of products, columns: "analysis_center", "project", "date", "solution_type", "period", + "resolution", "content", "format" + """ + if target_files is None: + target_files = ["CLK", "BIA", "SP3"] + else: + target_files = [file.upper() for file in target_files] + + products = pd.DataFrame( + columns=["analysis_center", "project", "date", "solution_type", "period", "resolution", "content", "format"]) + + # 1. Retrieve available options + gps_weeks = range(date_to_gpswk(start_time), date_to_gpswk(end_time) + 1) + for gps_week in gps_weeks: + url = f"https://cddis.nasa.gov/archive/gnss/products/{gps_week}/" + try: + week_files = requests.get(url, timeout=10) + week_files.raise_for_status() + except requests.RequestException as e: + raise requests.RequestException(f"Failed to fetch files for GPS week {gps_week}: {e}") + + # 2. Extract data from available options + soup = BeautifulSoup(week_files.content, "html.parser", + parse_only=SoupStrainer("div", class_="archiveItemTextContainer")) + # Above SoupStrainer makes it that only relevant items are stored in memory + for div in soup: + filename = div.get_text().split(" ")[0] + try: + if gps_week < 2237: + # Format convention changed in week 2237 + # AAAWWWWD.TYP.Z + center = filename[0:3].upper() # e.g. "COD" + _type = "FIN" # pre-2237 were probably always final solutions :shrug: + day = int(filename[7]) # e.g. "0", 0-indexed, 7 indicates weekly + _format = filename[9:12].upper() # e.g. "snx", "ssc", "sum", "erp" + project = "OPS" + sampling_resolution = None + content = None + date = gpswk_to_date(gps_week) + if 0 < day < 7: + date += timedelta(days=day) + period = timedelta(days=1) + else: + period = timedelta(days=7) + + else: + # e.g. GRG0OPSFIN_20232620000_01D_01D_SOL.SNX.gz + # AAA0OPSSNX_YYYYDDDHHMM_LEN_SMP_CNT.FMT.gz + center = filename[0:3] # e.g. "COD" + project = filename[4:7] # e.g. "OPS" or "RNN" unused + _type = filename[7:10] # e.g. "FIN" + year = int(filename[11:15]) # e.g. "2023" + day_of_year = int(filename[15:18]) # e.g. "262" + hour = int(filename[18:20]) # e.g. "00" + minute = int(filename[20:22]) # e.g. "00" + intended_period = filename[23:26] # eg "01D" + sampling_resolution = filename[27:30] # eg "01D" + content = filename[31:34] # e.g. "SOL" + _format = filename[35:38] # e.g. "SNX" + + date = datetime(year, 1, 1, hour, minute) + timedelta(day_of_year - 1) + period = timedelta(days=int(intended_period[:-1])) # Assuming all periods are in days :shrug: + + if _format in target_files and start_time <= date <= end_time: + products.loc[len(products)] = { + "analysis_center": center, + "project": project, + "date": date, + "solution_type": _type, + "period": period, + "resolution": sampling_resolution, + "content": content, + "format": _format + } + except (ValueError, IndexError): + # Skips md5 sums and other non-conforming files + continue + products = products.drop_duplicates(inplace=False) # resets indexes too + return products + + +def get_valid_analysis_centers(data: pd.DataFrame) -> set[str]: + """ + Analyzes dataframe for valid analysis centers (those that provide contiguous coverage) + AND have all required file types (SP3, BIA, CLK) for at least one series. + + :param data: products dataframe, see: get_product_dataframe(), requires columns: "analysis_center", "project", + "date", "solution_type", "period", "resolution", "content", "format" + :returns: set of valid analysis centers that have all required files for at least one series + """ + # Required file types for PPP processing + REQUIRED_FILES = {"SP3", "BIA", "CLK"} + + # 1. Check for any gaps in coverage and remove + for (center, _type, _format), group in data.groupby(["analysis_center", "solution_type", "format"]): + # Time window is filtered for in get_product_dataframe; only need to check they're contiguous + group = group.sort_values("date").reset_index(drop=True) + for i in range(len(group) - 1): + if group.loc[i]["date"] + group.loc[i]["period"] < group.loc[i + 1]["date"]: + Logger.console( + f"Gap detected for {center} {_type} {_format} between {group.loc[i, 'date']} and {group.loc[i + 1, 'date']}") + data = data[ + ~((data["analysis_center"] == center) and + (data["solution_type"] == _type) and + (data["format"] == _format))] + + # 2. Filter for centers that have all required files for at least one series + valid_centers = set() + for analysis_center in data["analysis_center"].unique(): + center_data = data[data["analysis_center"] == analysis_center] + + # Check each series to see if it has all required files + for series in center_data["solution_type"].unique(): + series_data = center_data[center_data["solution_type"] == series] + available_files = set(series_data["format"].unique()) + + # If this series has all required files, the center is valid + if REQUIRED_FILES.issubset(available_files): + valid_centers.add(analysis_center) + break # No need to check other series for this center + + # 3. Only show series with all required files + centers = set() + for analysis_center in sorted(valid_centers): + centers.add(analysis_center) + center_products = data.loc[data["analysis_center"] == analysis_center] + + # Build a dict of file types to their available series + file_series_map = {} + for _format in center_products["format"].unique(): + series_list = center_products.loc[center_products["format"] == _format, "solution_type"].unique() + file_series_map[_format] = set(series_list) + + # Find series that have all required files + valid_series_for_center = set() + for series in center_products["solution_type"].unique(): + series_data = center_products[center_products["solution_type"] == series] + available_files = set(series_data["format"].unique()) + if REQUIRED_FILES.issubset(available_files): + valid_series_for_center.add(series) + + # Build output string showing only complete series + offerings = "" + for _format in sorted(REQUIRED_FILES): + if _format in file_series_map: + # Only show series that have all three file types + complete_series = sorted(file_series_map[_format].intersection(valid_series_for_center)) + if complete_series: + offerings += f"{_format}:({'/'.join(complete_series)}) " + + Logger.console(f"Analysis centre {analysis_center} offers: {offerings.strip()}") + + return centers + + +def get_valid_series_for_provider(data: pd.DataFrame, provider: str) -> List[str]: + """ + Get list of valid series (with all required files) for a specific provider. + + :param data: products dataframe from get_product_dataframe() + :param provider: analysis center name (e.g., "COD", "GRG") + :returns: sorted list of valid series codes (e.g., ["FIN", "RAP"]) + """ + REQUIRED_FILES = {"SP3", "BIA", "CLK"} + + # Filter for this provider + provider_data = data[data["analysis_center"] == provider] + + # Find series that have all required files + valid_series = [] + for series in provider_data["solution_type"].unique(): + series_data = provider_data[provider_data["solution_type"] == series] + available_files = set(series_data["format"].unique()) + + if REQUIRED_FILES.issubset(available_files): + valid_series.append(series) + + return sorted(valid_series) + +def get_valid_providers_with_series(data: pd.DataFrame) -> dict: + """ + Get a mapping of providers to their valid series (with all required files). + + :param data: products dataframe from get_product_dataframe() + :returns: dict mapping provider names to lists of valid series + """ + provider_series_map = {} + + for provider in data["analysis_center"].unique(): + valid_series = get_valid_series_for_provider(data, provider) + if valid_series: # Only include providers with at least one valid series + provider_series_map[provider] = valid_series + + return provider_series_map + +def extract_file(filepath: Path) -> Path: + """ + Extracts [".gz", ".gzip", ".Z"] files with gzip and unlzw3 respectively. + Deletes compressed file after extraction. + + :param filepath: compressed file path + :return: path to extracted file + """ + finalpath = ".".join(str(filepath).split(".")[:-1]) + if str(filepath.name).endswith((".gz", ".gzip")): + with gzip.open(filepath, "rb") as f_in, open(finalpath, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) + elif str(filepath.name).endswith(".Z"): + decompressed_data = unlzw3.unlzw(filepath) + with open(finalpath, "wb") as f_out: + f_out.write(decompressed_data) + filepath.unlink() + return Path(finalpath) + + +def download_file(url: str, session: requests.Session, download_dir: Path = INPUT_PRODUCTS_PATH, + progress_callback: Optional[Callable] = None, + stop_requested: Callable = None) -> Path: + """ + Checks if file already exists (additionally in compressed or .part forms). + Uses provided session for CDDIS files (session made during startup). + Downloads in chunks to a file with the same file path suffixed by .part. + Deletes .part file after download. + Automatically calls extract_file() on compressed files. + + :param url: download url + :param session: requests Session preloaded with users CDDIS credentials + :param download_dir: dir to download to + :param progress_callback: reports, on every chunk, an int percentage of total download + :param stop_requested: bool callback. Raises a RuntimeError if occurred during download + :raises RuntimeError: Stop requested during download + :raises Exception: Max retries reached + :return: + """ + + filepath = Path(download_dir / url.split("/")[-1]) # Download dir + filename + # 1. When file already exists, extract if possible, then return + if filepath.exists(): + if filepath.suffix in COMPRESSED_FILETYPE: + return extract_file(filepath) + else: + return filepath + + # 2. Check if an extracted version of this file already exists + if filepath.suffix in COMPRESSED_FILETYPE: + potential_decompressed = filepath.with_suffix('') # Remove one suffix + if potential_decompressed.exists(): + return potential_decompressed + + # 3. Download the file in chunks (.part) + for i in range(MAX_RETRIES): + _partial = filepath.with_suffix(filepath.suffix + ".part") + + if _partial.exists(): + # Resume partial downloads + headers = {"Range": f"bytes={_partial.stat().st_size}-"} + Logger.terminal(f"Resuming download of {filepath.name} from byte {_partial.stat().st_size}") + else: + # Download whole file + headers = {"Range": "bytes=0-"} + Logger.terminal(f"Starting new download of {filepath.name}") + os.makedirs(_partial.parent, exist_ok=True) + + # Hack?! for windows error when open(_partial, "wb") not creating new files + ensure_file_exists = open(_partial, "w") + ensure_file_exists.close() + + try: + if url.startswith(BASE_URL): + # Download files from CDDIS with authorized session + resp = session.get(url, headers=headers, stream=True, timeout=30) + else: + resp = requests.get(url, headers=headers, stream=True, timeout=30) + resp.raise_for_status() + + if resp.status_code == 206: + # Received partial content as expected + mode = 'ab' if _partial.exists() else 'wb' + total_size = int(resp.headers.get("content-length")) + _partial.stat().st_size + else: + # likely 200 OK, server is sending the entire file again + mode = 'wb' + total_size = int(resp.headers.get("content-length")) + + with open(_partial, mode) as partial_out: + downloaded = _partial.stat().st_size + for _chunk in resp.iter_content(chunk_size=CHUNK_SIZE): + if stop_requested and stop_requested(): + raise RuntimeError("Stop requested during download.") + + if _chunk: # Filters keep-alives + partial_out.write(_chunk) + downloaded += len(_chunk) + + if progress_callback: + percent = int(downloaded / total_size * 100) + progress_callback(filepath.name, percent) + + os.rename(_partial, filepath) + + if filepath.suffix in COMPRESSED_FILETYPE: + return extract_file(filepath) + else: + return filepath + except requests.RequestException as e: + Logger.terminal(f"Failed attempt {i} to download {filepath.name}: {e}") + + raise (Exception(f"Failed to download {filepath.name} after {MAX_RETRIES} attempts")) + + +def get_brdc_urls(start_time: datetime, end_time: datetime) -> list[str]: + """ + Generates a list of BRDC file URLs for the specified date range. + + :param start_time: Start of the date range + :param end_time: End of the date range + :returns: List URLs to download BRDC files + """ + urls = [] + reference_dt = start_time + while int((end_time - reference_dt).total_seconds()) > 0: + day = reference_dt.strftime("%j") + filename = f"BRDC00IGS_R_{reference_dt.year}{day}0000_01D_MN.rnx.gz" + url = f"{BASE_URL}/gnss/data/daily/{reference_dt.year}/brdc/{filename}" + urls.append(url) + reference_dt += timedelta(days=1) + return urls + + +def download_metadata(download_dir: Path = INPUT_PRODUCTS_PATH, + progress_callback: Optional[Callable] = None, atx_callback: Optional[Callable] = None): + """ + Calls download_products() with args to download standard metadata files. Calls atx_callback("igs20.atx") + once "igs20.atx" is downloaded. Won't install duplicate files. + + :param download_dir: dir to download to + :param progress_callback: reports, on every chunk, an int percentage of total download + :param atx_callback: Optional callback function when igs20.atx is downloaded (downloaded_file) + :raises Exception: Max retries reached + """ + for download in download_products(products=pd.DataFrame(), download_dir=download_dir, + progress_callback=progress_callback, dl_urls=METADATA): + if atx_callback and download.name == "igs20.atx": + atx_callback(download.name) + + +def download_products(products: pd.DataFrame, download_dir: Path = INPUT_PRODUCTS_PATH, + dl_urls: list = None, progress_callback: Optional[Callable] = None, + stop_requested: Optional[Callable] = None) -> Generator[Path, None, None]: + """ + Creates download URLs for products and subsequently calls download_file() on them. Won't install duplicate files. + + :param pd.DataFrame products: (from get_product_dataframe) of all products to download + :param download_dir: dir to download to + :param dl_urls: Optional list of additional URLs to download (e.g. BRDC files) + :param progress_callback: reports, on every chunk, an int percentage of total download + :param stop_requested: bool callback. Raises a RuntimeError if occurred during download + :returns: Generator with paths to downloaded files + :raises RuntimeError: Stop requested during download + :raises Exception: Max retries reached + """ + + # 1. Generate filenames from the DataFrame + downloads = [] + for _, row in products.iterrows(): + gps_week = date_to_gpswk(row.date) + if gps_week < 2237: + # AAAWWWWD.TYP.Z + # e.g. COD22360.FIN.SNX.gz + if row.period == timedelta(days=7): + day = 7 + else: + day = int((row.date - gpswk_to_date(gps_week)).days) + filename = f"{row.analysis_center.lower()}{gps_week}{day}.{row.format.lower()}.Z" + else: + # e.g. GRG0OPSFIN_20232620000_01D_01D_SOL.SNX.gz + # AAA0OPSSNX_YYYYDDDHHMM_LEN_SMP_CNT.FMT.gz + filename = f"{row.analysis_center}0{row.project}{row.solution_type}_{row.date.strftime('%Y%j%H%M')}_{row.period.days:02d}D_{row.resolution}_{row.content}.{row.format}.gz" + + url = f"{BASE_URL}/gnss/products/{gps_week}/{filename}" + downloads.append(url) + + if dl_urls: + downloads.extend(dl_urls) + + Logger.terminal(f"📦 {len(downloads)} files to check or download") + download_dir.mkdir(parents=True, exist_ok=True) + (download_dir / "tables").mkdir(parents=True, exist_ok=True) + _sesh = requests.Session() + _sesh.auth = get_netrc_auth() + for url in downloads: + _x = url.split("/") + if len(_x) < 2: + fin_dir = download_dir + else: + fin_dir = download_dir / "tables" if _x[-2] == "tables" else download_dir + yield download_file(url, _sesh, fin_dir, progress_callback, stop_requested) + + +if __name__ == "__main__": + # Test whole file download + sesh = requests.Session() + sesh.auth = get_netrc_auth() + x = Path(f"{INPUT_PRODUCTS_PATH}/COD0MGXFIN_20191950000_01D_01D_OSB.BIA.gz") + if x.exists(): + x.unlink() + if x.with_suffix('').exists(): + x.with_suffix('').unlink() + if x.with_suffix(x.suffix + ".part").exists(): + x.with_suffix(x.suffix + ".part").unlink() + + download_file(f"{BASE_URL}/gnss/products/2062/{x.name}", sesh, INPUT_PRODUCTS_PATH) + + # Test resuming a partial download + os.remove(x.with_suffix('')) # should extract file + y = x.with_suffix(x.suffix + ".part") + req = sesh.get(f"{BASE_URL}/gnss/products/2062/{x.name}", headers={"Range": f"bytes=0-{CHUNK_SIZE}"}, stream=True) + with open(y, "wb") as z: + for chunk in req.iter_content(chunk_size=CHUNK_SIZE): + if chunk: # Filters keep-alives + z.write(chunk) + Logger.console(f"Downloaded {y.stat().st_size} bytes to {y}.\nAttempting to resume full download...") + download_file(f"{BASE_URL}/gnss/products/2062/{x.name}", sesh, INPUT_PRODUCTS_PATH) + Logger.console(f"Success!") + x.unlink() \ No newline at end of file diff --git a/scripts/GinanUI/app/models/execution.py b/scripts/GinanUI/app/models/execution.py new file mode 100644 index 000000000..da46da21b --- /dev/null +++ b/scripts/GinanUI/app/models/execution.py @@ -0,0 +1,456 @@ +import os +import platform +import shutil +import subprocess +import signal +import threading +import time +from importlib.resources import files + +from ruamel.yaml.scalarstring import PlainScalarString +from ruamel.yaml.comments import CommentedSeq, CommentedMap +from pathlib import Path +from scripts.GinanUI.app.utils.yaml import load_yaml, write_yaml, normalise_yaml_value +from scripts.plot_pos import plot_pos_files +from scripts.GinanUI.app.utils.common_dirs import GENERATED_YAML, TEMPLATE_PATH, INPUT_PRODUCTS_PATH + +# Import the new logger +try: + from scripts.GinanUI.app.utils.logger import Logger +except ImportError: + # Fallback if logger not yet in the correct location + class Logger: + @staticmethod + def terminal(msg): + print(f"[TERMINAL] {msg}") + + @staticmethod + def console(msg): + print(f"[CONSOLE] {msg}") + + @staticmethod + def both(msg): + print(f"[BOTH] {msg}") + + +def get_pea_exec(): + """ + Checks system platform and returns a Path to the respective executable. Also searches for "pea" on PATH. + + :return: Path to executable or str of PATH callable + :raises RuntimeError: If PEA binary cannot be found + """ + import sys + + # 1. Check if running in PyInstaller bundle + if getattr(sys, 'frozen', False): + # Running in bundled mode + base_path = Path(sys._MEIPASS) + + # On macOS .app bundles, binaries are in Resources/bin/ + if platform.system().lower() == "darwin": + # Try Resources/bin first (macOS .app structure) + pea_path = base_path.parent / "Resources" / "bin" / "pea" + if pea_path.exists(): + print(f"[Execution] Found bundled PEA binary at: {pea_path}") + return pea_path + # Fallback to _internal/bin + pea_path = base_path / "bin" / "pea" + if pea_path.exists(): + print(f"[Execution] Found bundled PEA binary at: {pea_path}") + return pea_path + + # Linux/Windows: binaries in _internal/bin + else: + # Windows uses .exe extension + exe_name = "pea.exe" if platform.system().lower() == "windows" else "pea" + pea_path = base_path / "bin" / exe_name + if pea_path.exists(): + return pea_path + + print(f"[Execution] Bundled binary not found in expected locations") + # Fall through to try other methods + + # 2. Check if 'pea' is on PATH (most reliable if user has configured their environment) + if shutil.which("pea"): + executable = "pea" + Logger.console(f"✅ Found PEA on PATH: {shutil.which('pea')}") + return executable + + # 3. Try to find PEA relative to this script's location + # Current file: ginan/scripts/GinanUI/app/models/execution.py + # Target file: ginan/bin/pea + try: + current_file = Path(__file__).resolve() + # Navigate from: "ginan/scripts/GinanUI/app/models/execution.py" to "ginan/" + ginan_root = current_file.parents[4] # Go up: models -> app -> GinanUI -> scripts -> ginan + + # Check for the binary in ginan/bin/pea + pea_binary = ginan_root / "bin" / "pea" + + if pea_binary.exists() and pea_binary.is_file(): + # Make sure it's executable (permissions are set up right) + if not os.access(pea_binary, os.X_OK): + Logger.console(f"✅ Found PEA at {pea_binary} but it's not executable. Attempting to fix...") + try: + pea_binary.chmod(pea_binary.stat().st_mode | 0o111) # Add "execute" permissions + Logger.console(f"✅ Made PEA executable") + except Exception as e: + Logger.console(f"⚠️ Could not make PEA executable: {e}") + raise RuntimeError(f"⚠️ PEA binary found at {pea_binary} but is not executable and cannot be fixed") + + Logger.console(f"✅ Found PEA binary at: {pea_binary}") + return pea_binary + else: + Logger.console(f"⚠️ Expected PEA binary at {pea_binary} but not found") + + except Exception as e: + Logger.console(f"⚠️ Error while searching for PEA relative to script location: {e}") + + # 4. Platform-specific fallbacks (optional - can be removed if not needed) + system = platform.system().lower() + + if system == "windows": + # Windows may have pea.exe set up + if shutil.which("pea.exe"): + executable = "pea.exe" + Logger.console(f"✅ Found pea.exe on PATH: {shutil.which('pea.exe')}") + return executable + raise RuntimeError( + "PEA executable not found. Please:\n" + "1. Build the PEA binary (see ginan build instructions)\n" + "2. Add ginan/bin to your PATH, or\n" + "3. Run from within the ginan directory structure" + ) + + # 5. If nothing found, provide a helpful error message + raise RuntimeError( + f"PEA executable not found. Please ensure:\n" + f"1. You have built the PEA binary (should be at ginan/bin/pea)\n" + f"2. You are running GinanUI from within the ginan directory structure, or\n" + f"3. The 'pea' executable is available on your system PATH\n" + f"\nSearched locations:\n" + f" - System PATH\n" + f" - {ginan_root / 'bin' / 'pea' if 'ginan_root' in locals() else 'Could not determine ginan root'}" + ) + + +class Execution: + def __init__(self, config_path: Path = GENERATED_YAML): + """ + Caches config changes, interacts with config file, and finally can call pea executable. + + :param config_path: Path to a config file, defaulted to GENERATED_YAML + """ + self.config_path = config_path + self.executable = get_pea_exec() # the PEA executable + self.changes = False # Flag to track if config has been changed + self._procs = [] + self._stop_event = threading.Event() + + template_file = Path(TEMPLATE_PATH) + + if config_path.exists(): + Logger.console(f"Using existing config file: {config_path}") + else: + Logger.console( + f"Existing config not found, copying default template: {template_file} → {config_path}") + try: + config_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(template_file, config_path) + except Exception as e: + raise RuntimeError(f"❌ Failed to copy default config: {e}") + + self.config = load_yaml(config_path) + + def reload_config(self): + """ + Force reload of the YAML config from disk into memory. + This allows any manual edits to be picked up before GUI changes are applied. + + :raises RuntimeError: Any error occurred during load_yaml(config_path) + """ + try: + self.config = load_yaml(self.config_path) + except Exception as e: + raise RuntimeError(f"❌ Failed to reload config from {self.config_path}: {e}") + + def edit_config(self, key_path: str, value, add_field=False): + """ + Edits the cached config while preserving YAML formatting and comments. + + :param key_path: Dot-separated YAML key path (e.g., "inputs.gnss_observations.rnx_inputs") + :param value: New value to assign (will be converted to ruamel-safe types) + :param add_field: Whether to add the field if it doesn't exist + :raises KeyError if path doesn't exist and add_field is False + """ + self.changes = True # Mark config as changed + keys = key_path.split(".") + node = self.config + + for key in keys[:-1]: + if key not in node: + if add_field: + node[key] = CommentedMap() + else: + raise KeyError(f"Key '{key}' not found in {node}") + node = node[key] + + final_key = keys[-1] + value = normalise_yaml_value(value) + + # Preserve any existing comment on the final_key + if final_key in node: + old_value = node[final_key] + if hasattr(old_value, 'ca') and not hasattr(value, 'ca'): + value.ca = old_value.ca + + if not add_field and final_key not in node: + raise KeyError(f"Key '{final_key}' not found in {key_path}") + + node[final_key] = value + + def apply_ui_config(self, inputs): + """ + Applies UI settings to **cached** config. **Call write_cached_changes()** to write them. + + :param inputs: + """ + self.changes = True + + # 1. Set core inputs / outputs + self.edit_config("inputs.inputs_root", str(INPUT_PRODUCTS_PATH) + "/", False) + + # Extract directory and filename from RINEX path + rnx_path = Path(inputs.rnx_path) + rnx_directory = str(rnx_path.parent) + rnx_filename = rnx_path.name + + # Set gnss_observations_root to the directory containing the RINEX file + self.edit_config("inputs.gnss_observations.gnss_observations_root", rnx_directory, False) + + # Use only the filename (relative path) for rnx_inputs + rnx_val = normalise_yaml_value(rnx_filename) + + # 1a. Set rnx_inputs safely, preserving formatting + try: + existing = self.config["inputs"]["gnss_observations"].get("rnx_inputs") + if isinstance(existing, CommentedSeq): + existing.clear() + existing.append(rnx_val) + existing.fa.set_block_style() + else: + new_seq = CommentedSeq([rnx_val]) + new_seq.fa.set_block_style() + self.config["inputs"]["gnss_observations"]["rnx_inputs"] = new_seq + except Exception as e: + Logger.console(f"[apply_ui_config] Error setting rnx_inputs: {e}") + + # Normalise outputs_root + out_val = normalise_yaml_value(inputs.output_path) + self.edit_config("outputs.outputs_root", out_val, False) + + # 2. Replace 'TEST' receiver block with real marker name + if "TEST" in self.config.get("receiver_options", {}): + self.config["receiver_options"][inputs.marker_name] = self.config["receiver_options"].pop("TEST") + + # 3. Include UI-extracted values + self.edit_config("processing_options.epoch_control.start_epoch", PlainScalarString(inputs.start_epoch), False) + self.edit_config("processing_options.epoch_control.end_epoch", PlainScalarString(inputs.end_epoch), False) + self.edit_config("processing_options.epoch_control.epoch_interval", inputs.epoch_interval, False) + self.edit_config(f"receiver_options.{inputs.marker_name}.receiver_type", inputs.receiver_type, True) + self.edit_config(f"receiver_options.{inputs.marker_name}.antenna_type", inputs.antenna_type, True) + self.edit_config(f"receiver_options.{inputs.marker_name}.models.eccentricity.offset", inputs.antenna_offset, + True) + + # Always format process_noise as a list + self.edit_config("estimation_parameters.receivers.global.pos.process_noise", [inputs.mode], False) + + # 4. GNSS constellation toggles + all_constellations = ["gps", "gal", "glo", "bds", "qzs"] + for const in all_constellations: + self.edit_config(f"processing_options.gnss_general.sys_options.{const}.process", False, False) + + # Then enable only the selected constellations + if inputs.constellations_raw: + selected = [c.strip().lower() for c in inputs.constellations_raw.split(",") if c.strip()] + for const in selected: + if const in all_constellations: + self.edit_config(f"processing_options.gnss_general.sys_options.{const}.process", True, False) + + def write_cached_changes(self): + write_yaml(self.config_path, self.config) + self.changes = False + + def execute_config(self): + """ + If changes were made since last write, writes config, then executes pea with config. + All PEA output is logged to the console widget. + """ + # Check if executable is available + if self.executable is None: + raise RuntimeError("❌ PEA executable not configured yet. Cannot run processing.") + + # clear stop flag before each run + self.reset_stop_flag() + + if self.changes: + self.write_cached_changes() + self.changes = False + + command = [self.executable, "--config", str(self.config_path)] + workdir = str(Path(self.config_path).parent) + + Logger.console(f"🚀 Starting PEA: {' '.join(str(c) for c in command)}") + Logger.console(f"📂 Working directory: {workdir}") + Logger.console("=" * 60) + + try: + # spawn process with process group + p = self.spawn_process(command, cwd=workdir) + + # forward stdout/stderr line by line to console, can be stopped at any time + assert p.stdout is not None and p.stderr is not None + + # Use a separate thread to read stderr so we don't miss any output + stderr_lines = [] + + def read_stderr(): + for line in p.stderr: + if line: + stderr_lines.append(line.rstrip()) + + stderr_thread = threading.Thread(target=read_stderr, daemon=True) + stderr_thread.start() + + while True: + if self._stop_event.is_set(): + # UI clicked "stop", exit loop, cleanup handled by stop_all() + Logger.console("🛑 PEA execution stopped by user") + break + + line = p.stdout.readline() + if line: + # Log each line of PEA output to console + Logger.console(line.rstrip()) + else: + # no new output, check if process has ended + if p.poll() is not None: + # Process finished, log any remaining stderr + stderr_thread.join(timeout=1.0) + for err_line in stderr_lines: + if err_line: + Logger.console(f"⚠️ {err_line}") + + if p.returncode != 0: + Logger.console(f"❌ PEA exited with code {p.returncode}") + e = subprocess.CalledProcessError(p.returncode, command) + e.add_note("Error executing PEA command") + raise e + else: + Logger.console("=" * 60) + Logger.console("✅ PEA execution completed successfully") + break + + # slight sleep to avoid busy polling + time.sleep(0.01) + + finally: + # after execution, clean up finished processes + self._procs = [proc for proc in self._procs if proc.poll() is None] + + def spawn_process(self, args, cwd=None, env=None) -> subprocess.Popen: + """ + Unified process spawning: use independent process groups for easy kill (macOS/Linux) + """ + p = subprocess.Popen( + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, # critical: new session = new process group + ) + self._procs.append(p) + return p + + def stop_all(self): + """ + One-click stop: set stop flag + terminate all child process groups + """ + self._stop_event.set() + + # try graceful termination first + for p in list(self._procs): + try: + if p.poll() is None: + os.killpg(p.pid, signal.SIGTERM) + except Exception: + pass + + time.sleep(0.5) # give it a little time + + # if still not exited, force kill + for p in list(self._procs): + try: + if p.poll() is None: + os.killpg(p.pid, signal.SIGKILL) + except Exception: + pass + + def reset_stop_flag(self): + self._stop_event.clear() + + def build_pos_plots(self, out_dir=None): + """ + Search for .pos and .POS files directly under outputs_root (not in archive/visual), + and generate one .html per file in outputs_root/visual. + Return a list of generated html paths (str). + """ + try: + outputs_root = self.config["outputs"]["outputs_root"] + root = Path(outputs_root).expanduser().resolve() + except Exception: + # Fallback to default + root = Path(__file__).resolve().parents[2] / "tests" / "resources" / "outputData" + root = root.resolve() + + # Set output dir for HTML plots + if out_dir is None: + out_dir = root / "visual" + else: + out_dir = Path(out_dir).expanduser().resolve() + out_dir.mkdir(parents=True, exist_ok=True) + + # Only look in the top-level of outputs_root + pos_files = list(root.glob("*.pos")) + list(root.glob("*.POS")) + + if pos_files: + Logger.terminal(f"📂 Found {len(pos_files)} .pos files in {root}:") + for f in pos_files: + Logger.terminal(f" • {f.name}") + else: + Logger.terminal(f"⚠️ No .pos files found in {root}") + + htmls = [] + for pos_path in pos_files: + try: + base_name = pos_path.stem + save_prefix = out_dir / f"plot_{base_name}" + + html_files = plot_pos_files( + input_files=[str(pos_path)], + save_prefix=str(save_prefix) + ) + htmls.extend(html_files) + except Exception as e: + Logger.terminal(f"[plot_pos] ❌ Failed for {pos_path.name}: {e}") + + # Final summary + if htmls: + Logger.terminal(f"✅ Generated {len(htmls)} plot(s) → saved in {out_dir}") + else: + Logger.terminal("⚠️ No plots were generated.") + + return htmls \ No newline at end of file diff --git a/scripts/GinanUI/app/models/rinex_extractor.py b/scripts/GinanUI/app/models/rinex_extractor.py new file mode 100644 index 000000000..b9918affa --- /dev/null +++ b/scripts/GinanUI/app/models/rinex_extractor.py @@ -0,0 +1,323 @@ +import re +from datetime import datetime + + +class RinexExtractor: + def __init__(self, rinex_path: str): + self.rinex_path = rinex_path + + def load_rinex_file(self, rinex_path: str): + self.rinex_path = rinex_path + + def extract_rinex_data(self, rinex_path: str): + """ + Opens a .RNX file and extracts the corresponding YAML config information + + Supports RINEX v2, v3, and v4 formats + + :param rinex_path: File path for .RNX file to extract from e.g. "resources/input/ALIC.rnx" + :raises FileNotFoundError if .RNX file is not found + :raises ValueError if required metadata cannot be extracted + """ + + system_mapping = { + "G": "GPS", + "E": "GAL", + "R": "GLO", + "C": "BDS", + "J": "QZS", + } + found_constellations = set() + + def format_time(year, month, day, hour, minute, second): + """ + Helper function to format the parameters into a usable time string for RNX extraction + + :param year: The year + :param month: The month + :param day: The day + :param hour: The hour + :param minute: The minute + :param second: The second + :returns: Formatting string in format: "[YEAR]-[MONTH]-[DAY]_[HOUR]:[MIN}:[SEC]" + i.e. "2000-10-27_16:57:49" + """ + return f"{year:04d}-{month:02d}-{day:02d}_{hour:02d}:{minute:02d}:{(int(second)):02d}" + + def normalize_year_v2(y: int) -> int: + """ + RINEX v2 sometimes uses 2-digit years in the header/body, sometimes 4-digit. + - If y >= 1000 (already 4-digit), return as is. + - Else map YY < 80 -> 2000+YY; YY >= 80 -> 1900+YY. + """ + if y >= 1000: + return y + return 2000 + y if y < 80 else 1900 + y + + def chunk_sat_ids(s: str): + """ + Given a string like 'G01G02G10R07R08' (no spaces), + return ['G01','G02','G10','R07','R08']. + Used in RINEX v2 body parsing. + """ + s = s.strip() + out = [] + for i in range(0, len(s), 3): + chunk = s[i:i+3] + if len(chunk) == 3 and chunk[0].isalpha(): + out.append(chunk) + return out + + rinex_version = None + previous_observation_dt = None + epoch_interval = None + in_header = True + start_epoch = None + end_epoch = None + marker_name = None + receiver_type = None + antenna_type = None + antenna_offset = None + + with open(rinex_path, "r", errors="replace") as f: + lines = f.readlines() + + i = 0 + n = len(lines) + + # ---------- Header ---------- + while i < n: + line = lines[i] + i += 1 + label = line[60:].strip() if len(line) >= 61 else "" + + if rinex_version is None and "RINEX VERSION / TYPE" in label: + try: + rinex_version = float(line[0:9].strip()) + except Exception: + pass + + if in_header: + # ----- RINEX v2 header ----- + if rinex_version and rinex_version < 3.0: + if label == "# / TYPES OF OBSERV": + pass + elif label == "TIME OF FIRST OBS": + parts = line.split() + if len(parts) >= 6: + y_raw, m, d, hh, mm = map(int, parts[:5]) + ss = float(parts[5]) + y = normalize_year_v2(y_raw) + start_epoch = format_time(y, m, d, hh, mm, ss) + elif label == "TIME OF LAST OBS": + parts = line.split() + if len(parts) >= 6: + y_raw, m, d, hh, mm = map(int, parts[:5]) + ss = float(parts[5]) + y = normalize_year_v2(y_raw) + end_epoch = format_time(y, m, d, hh, mm, ss) + elif label == "INTERVAL": + try: + epoch_interval = int(float(line[0:10].strip())) + except Exception: + pass + elif label == "MARKER NAME": + raw_marker = line[0:60].strip() + # v2: first 4 chars are the station ID + marker_name = raw_marker[:4] if len(raw_marker) >= 4 else raw_marker + elif label == "REC # / TYPE / VERS": + receiver_type = line[20:40].strip() + elif label == "ANT # / TYPE": + antenna_type = line[20:40].strip() + second_half = line[40:60].strip() + if second_half: + antenna_type += f" {second_half}" + elif label == "ANTENNA: DELTA H/E/N": + try: + h = float(line[0:14].strip()) + e = float(line[14:28].strip()) + nnn = float(line[28:42].strip()) + antenna_offset = [e, nnn, h] + except Exception: + pass + elif label == "END OF HEADER": + in_header = False + break + # ----- RINEX v3/v4 header ----- + else: + if label == "SYS / # / OBS TYPES": + system_id = line[0] if line else "" + if system_id in system_mapping: + found_constellations.add(system_mapping[system_id]) + elif label == "TIME OF FIRST OBS": + try: + y = int(line[0:6]); m = int(line[6:12]); d = int(line[12:18]) + hh = int(line[18:24]); mm = int(line[24:30]); ss = float(line[30:43]) + start_epoch = format_time(y, m, d, hh, mm, ss) + except Exception: + pass + elif label == "TIME OF LAST OBS": + try: + y = int(line[0:6]); m = int(line[6:12]); d = int(line[12:18]) + hh = int(line[18:24]); mm = int(line[24:30]); ss = float(line[30:43]) + end_epoch = format_time(y, m, d, hh, mm, ss) + except Exception: + pass + elif label == "INTERVAL": + try: + epoch_interval = int(float(line[0:10])) + except Exception: + pass + elif label == "MARKER NAME": + marker_name = line[0:60].strip() + elif label == "REC # / TYPE / VERS": + receiver_type = line[20:40].strip() + elif label == "ANT # / TYPE": + antenna_type = line[20:40].strip() + second_half = line[40:60].strip() + if second_half: + antenna_type += f" {second_half}" + elif label == "ANTENNA: DELTA H/E/N": + try: + h = float(line[0:14].strip()) + e = float(line[14:28].strip()) + nnn = float(line[28:42].strip()) + antenna_offset = [e, nnn, h] + except Exception: + pass + elif label == "END OF HEADER": + in_header = False + break + else: + break # safety + + if rinex_version is None: + raise ValueError("Could not determine RINEX version.") + + # Detector for a v2 epoch start (now supports 2 or 4 digit years) + epoch_v2_head_re = re.compile( + r'^\s*\d{2,4}\s+\d{1,2}\s+\d{1,2}\s+\d{1,2}\s+\d{1,2}\s+[0-9.]' + ) + + if rinex_version < 3.0: + # ---------- RINEX v2 body ---------- + # YY or YYYY MM DD hh mm ss.sssssss FLAG NSAT [SATLIST...] + epoch_re = re.compile( + r'^\s*(\d{2,4})\s+(\d{1,2})\s+(\d{1,2})\s+(\d{1,2})\s+(\d{1,2})\s+([0-9.]+)\s+(\d)\s*(\d+)?(.*)$' + ) + + while i < n: + line = lines[i] + m = epoch_re.match(line.rstrip("\n")) + if not m: + i += 1 + continue + + y_raw = int(m.group(1)) + mo = int(m.group(2)); dd = int(m.group(3)) + hh = int(m.group(4)); mmn = int(m.group(5)); ssf = float(m.group(6)) + flag = int(m.group(7)) # currently unused + nsat = m.group(8) + nsat = int(nsat) if nsat is not None else 0 + rest = m.group(9) or "" + + year = normalize_year_v2(y_raw) + + # Epoch interval from first two epochs + if previous_observation_dt and epoch_interval is None: + t1 = datetime(*previous_observation_dt) + t2 = datetime(year, mo, dd, hh, mmn, int(ssf)) + epoch_interval = int((t2 - t1).total_seconds()) + + end_epoch = format_time(year, mo, dd, hh, mmn, ssf) + if start_epoch is None: + # Header didn't contain TIME OF FIRST OBS + start_epoch = end_epoch + + previous_observation_dt = (year, mo, dd, hh, mmn, int(ssf)) + + # Satellites from this line + continuation lines + sats = chunk_sat_ids(rest) + + # Continuations: until we have nsat satellites or hit next epoch + j = i + 1 + while len(sats) < nsat and j < n: + nxt = lines[j].rstrip("\n") + if epoch_v2_head_re.match(nxt): + break # next epoch encountered + sats.extend(chunk_sat_ids(nxt)) + j += 1 + + for sid in sats[:nsat]: + sys = sid[0] + if sys in system_mapping: + found_constellations.add(system_mapping[sys]) + + i = j + + else: + # ---------- RINEX v3/v4 body ---------- + while i < n: + line = lines[i] + i += 1 + if not line.startswith(">"): + continue + + parts = line[1:].split() + if len(parts) < 6: + continue + + y, mo, dd, hh, mmn = map(int, parts[:5]) + ssf = float(parts[5]) + + if previous_observation_dt and epoch_interval is None: + t1 = datetime(*previous_observation_dt) + t2 = datetime(y, mo, dd, hh, mmn, int(ssf)) + epoch_interval = int((t2 - t1).total_seconds()) + + end_epoch = format_time(y, mo, dd, hh, mmn, ssf) + if start_epoch is None: + start_epoch = end_epoch + + previous_observation_dt = (y, mo, dd, hh, mmn, int(ssf)) + + sats = [] + if len(parts) > 8: + sats.extend(parts[8:]) + + j = i + while j < n and not lines[j].startswith(">"): + extra = lines[j].strip().split() + if extra and not extra[0][0].isalpha(): + break + for token in extra: + if token and token[0] in system_mapping and len(token) >= 2: + sats.append(token) + j += 1 + + for sid in sats: + sys = sid[0] + if sys in system_mapping: + found_constellations.add(system_mapping[sys]) + + i = j + + # ---------- Safety checks ---------- + if not start_epoch: + raise ValueError("TIME OF FIRST OBS not found (header or body)") + if not end_epoch: + raise ValueError("TIME OF LAST OBS or last observation not found") + if epoch_interval is None: + raise ValueError("Epoch interval could not be determined") + + return { + "rinex_version": rinex_version, + "start_epoch": start_epoch, + "end_epoch": end_epoch, + "epoch_interval": epoch_interval, + "marker_name": marker_name, + "receiver_type": receiver_type, + "antenna_type": antenna_type, + "antenna_offset": antenna_offset, + "constellations": ", ".join(sorted(found_constellations)) if found_constellations else "Unknown", + } \ No newline at end of file diff --git a/scripts/GinanUI/app/resources/Yaml/default_config.yaml b/scripts/GinanUI/app/resources/Yaml/default_config.yaml new file mode 100644 index 000000000..15c085890 --- /dev/null +++ b/scripts/GinanUI/app/resources/Yaml/default_config.yaml @@ -0,0 +1,303 @@ +inputs: + inputs_root: #USER_SET + + atx_files: [igs20.atx] # Required + igrf_files: [tables/igrf14coeffs.txt] + erp_files: [finals.data.iau2000.txt] + planetary_ephemeris_files: [tables/DE436.1950.2050] + + troposphere: + gpt2grid_files: [tables/gpt_25.grd] + + tides: + ocean_tide_loading_blq_files: [tables/OLOAD_GO.BLQ] # Required if ocean loading is applied + atmos_tide_loading_blq_files: [tables/ALOAD_GO.BLQ] # Required if atmospheric tide loading is applied + ocean_pole_tide_loading_files: [tables/opoleloadcoefcmcor.txt] # Required if ocean pole tide loading is applied + + snx_files: + # Use a wild card (*) to include all files matching the description in the directory + - igs_satellite_metadata.snx + - tables/sat_yaw_bias_rate.snx + - tables/qzss_yaw_modes.snx + - tables/bds_yaw_modes.snx + #- "*.SNX" + + satellite_data: + nav_files: # Not required + - "BRDC*" + + clk_files: + - "*.CLK" + + bsx_files: + - "*.BIA" + + sp3_files: + - "*.SP3" + + + gnss_observations: + gnss_observations_root: #USER_SET + rnx_inputs: + # - "*.rnx" + - #USER_SET + + +outputs: + metadata: + config_description: default_config_ #USER_SET [default_config]_ + outputs_root: #USER_SET + + gpx: + output: true + filename: __.GPX + pos: + output: true + filename: __.POS + + + + + +satellite_options: + global: + error_model: ELEVATION_DEPENDENT # {uniform,elevation_dependent} + code_sigma: 0 # Standard deviation of code measurements + phase_sigma: 0 # Standard deviation of phase measurmeents + models: + phase_bias: + enable: true + +receiver_options: + global: + elevation_mask: 15 # degrees + error_model: ELEVATION_DEPENDENT # {uniform,elevation_dependent} + code_sigma: 0.3 # Standard deviation of code measurements, m + phase_sigma: 0.003 # Standard deviation of phase measurmeents, m + clock_codes: [AUTO, AUTO] + zero_dcb_codes: [AUTO, AUTO] + + rec_reference_system: GPS + models: + phase_bias: + enable: false + troposphere: # Tropospheric modelling accounts for delays due to refraction of light in water vapour + enable: true + models: [gpt2] # List of models to use for troposphere [standard,sbas,vmf3,gpt2,cssr] + tides: + atl: true # Enable atmospheric tide loading + enable: true # Enable modelling of tidal disaplacements + opole: true # Enable ocean pole tides + otl: true # Enable ocean tide loading + solid: true # Enable solid Earth tides + spole: true # Enable solid Earth pole tides + ionospheric_components: + use_2nd_order: true + use_3rd_order: true + gps: + rinex2: + rnx_code_conversions: + P1: L1W + P2: L2W + C1: L1C + C2: L2C + rnx_phase_conversions: + L1: [ L1W, L1C ] + L2: [ L2W, L2C ] + glo: + rinex2: + rnx_code_conversions: + P1: L1P + P2: L2P + C1: L1C + C2: L2C + rnx_phase_conversions: + L1: [ L1P, L1C ] + L2: [ L2P, L2C ] + + TEST: #change to header name of RNX + receiver_type: # #USER_SET (string) + antenna_type: # #USER_SET (string) + models: + eccentricity: + enable: true + offset: [ 0.0000, 0.0000, 0.0000 ] # [floats] #USER_SET + +processing_options: + process_modes: + preprocessor: true # Preprocessing and quality checks + spp: true # Perform SPP on receiver data + ppp: true # Perform PPP network or end user mode + ionosphere: false # Compute Ionosphere models based on GNSS measurements + slr: false # Process SLR observations + + epoch_control: + start_epoch: #RNX + end_epoch: #RNX + #max_epochs: 2880 # Future user set. + epoch_interval: #USER SET + wait_next_epoch: 3600 #USER_SET seconds (make large for post-processing) + + gnss_general: + #eop_comp + add_eop_component: true + use_primary_signals: true + sys_options: + gps: + process: false + reject_eclipse: false + code_priorities: [L1W, L1C, L1X, L2W, L2C, L2X, L2S, L2L, L5Q, L5X] + gal: + process: false + reject_eclipse: false + # clock_codes: [] + code_priorities: [L1C, L1X, L5Q, L5X, L6C, L6X, L7Q, L7X] + + bds: + process: false + reject_eclipse: false + # clock_codes: [] + code_priorities: [L2I, L7I, L6I, L5P] + + + glo: + process: false + reject_eclipse: false + # clock_codes: [] + code_priorities: [L1P, L1C, L2P, L2C] + + qzs: + process: false + reject_eclipse: false + # clock_codes: [] + code_priorities: [L1C, L1X, L2L, L2X, L5Q, L5X] + + + + preprocessor: # Configurations for the kalman filter and its sub processes + cycle_slips: # Cycle slips may be detected by the preprocessor and measurements rejected or ambiguities reinitialised + mw_process_noise: 0 # Process noise applied to filtered Melbourne-Wubenna measurements to detect cycle slips + slip_threshold: 0.05 # Value used to determine when a slip has occurred + preprocess_all_data: true + + spp: + # always_reinitialise: false # Reset SPP state to zero to avoid potential for lock-in of bad states + max_lsq_iterations: 12 # Maximum number of iterations of least squares allowed for convergence + outlier_screening: + raim: + enable: true # Enable Receiver Autonomous Integrity Monitoring + max_gdop: 30 # Maximum dilution of precision before error is flagged + + ppp_filter: + outlier_screening: + chi_square: + enable: false # Enable Chi-square test + mode: innovation # Chi-square test mode {none,innovation,measurement,state} + prefit: + max_iterations: 3 # Maximum number of measurements to exclude using prefit checks before attempting to filter + omega_test: false # Enable omega-test + sigma_check: true # Enable sigma check + state_sigma_threshold: 4 # Sigma threshold for states + meas_sigma_threshold: 4 # Sigma threshold for measurements + postfit: + max_iterations: 10 # Maximum number of measurements to exclude using postfit checks while iterating filter + sigma_check: true # Enable sigma check + state_sigma_threshold: 4 # Sigma threshold for states + meas_sigma_threshold: 4 # Sigma threshold for measurements + + ionospheric_components: # Slant ionospheric components + common_ionosphere: true # Use the same ionosphere state for code and phase observations + use_gf_combo: false # Combine 'uncombined' measurements to simulate a geometry-free solution + use_if_combo: false # Combine 'uncombined' measurements to simulate an ionosphere-free solution + + chunking: + by_receiver: false # Split large filter and measurement matrices blockwise by receiver ID to improve processing speed + size: 0 + + rts: # Rauch-Tung-Striebel (RTS) backwards smoothing + enable: true + lag: -1 + # interval: 86400 + inverter: LDLT # Inverter to be used within the rts processor, which may provide different performance outcomes in terms of processing time and accuracy and stability + filename: _.rts + periodic_reset: + # enable: true + # interval: 86400 + # states: [REC_POS] + + model_error_handling: + meas_deweighting: # Measurements that are outside the expected confidence bounds may be deweighted so that outliers do not contaminate the filtered solution + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all rejected measurement + state_deweighting: # Any "state" errors cause deweighting of all measurements that reference the state + deweight_factor: 1000 # Factor to downweight the variance of measurements with statistically detected errors + enable: true # Enable deweighting of all referencing measurements + error_accumulation: + enable: true + receiver_error_count_threshold: 10 + receiver_error_epochs_threshold: 4 + ambiguities: + phase_reject_limit: 2 # Reset ambiguity after 2 large fractional residuals are found (replaces phase_reject_count:) + reset_on: # Reset ambiguities when slip is detected by the following + gf: true # GF test + lli: true # LLI test + mw: true # MW test + scdia: true # SCDIA test + +estimation_parameters: + + receivers: + global: + pos: + estimated: [true] + sigma: [100] + process_noise: [] #USER_SET [int] + pos_rate: # Velocity + estimated: [false] # [bools] Estimate state in kalman filter + sigma: [0] # [floats] Apriori sigma values + process_noise: [0] # [floats] Process noise sigmas + # process_noise_dt: SECOND # (enum) Time unit for process noise - sqrt_sec, sqrt_day etc + # apriori_val: [0] # [floats] Apriori state values + # mu: [0] # [floats] Desired mean value for gauss markov states + # tau: [-1] # [floats] Correlation times for gauss markov noise, defaults to -1 -> inf (Random Walk) + + clock: + estimated: [true] + sigma: [1000] + process_noise: [100] + clock_rate: + estimated: [false] + sigma: [0.005] + process_noise: [1e-4] + ambiguities: + estimated: [true] + sigma: [1000] + process_noise: [0] + outage_limit: [300] + ion_stec: # Ionospheric slant delay + estimated: [true] # Estimate state in kalman filter + sigma: [200] # Apriori sigma values + process_noise: [10] # Process noise sigmas + trop: + estimated: [true] + sigma: [0.3] + process_noise: [0.0001] + trop_grads: + estimated: [true] + sigma: [0.03] + process_noise: [1.0E-6] + code_bias: + estimated: [true] # false + sigma: [20] + process_noise: [0] + phase_bias: + estimated: [false] + sigma: [10] + process_noise: [0] + gps: + l5q: + phase_bias: + estimated: [true] + sigma: [10] + process_noise: [0.001] + diff --git a/scripts/GinanUI/app/resources/__init__.py b/scripts/GinanUI/app/resources/__init__.py new file mode 100644 index 000000000..1b82a7c45 --- /dev/null +++ b/scripts/GinanUI/app/resources/__init__.py @@ -0,0 +1,3 @@ +import sys +from . import ginan_logo_rc as _rc +sys.modules.setdefault("ginan_logo_rc", _rc) \ No newline at end of file diff --git a/scripts/GinanUI/app/resources/ginan_logo.qrc b/scripts/GinanUI/app/resources/ginan_logo.qrc new file mode 100644 index 000000000..7b6f1e180 --- /dev/null +++ b/scripts/GinanUI/app/resources/ginan_logo.qrc @@ -0,0 +1,5 @@ + + + ginan-logo.png + + diff --git a/scripts/GinanUI/app/resources/ginan_logo_rc.py b/scripts/GinanUI/app/resources/ginan_logo_rc.py new file mode 100644 index 000000000..91f3002b3 --- /dev/null +++ b/scripts/GinanUI/app/resources/ginan_logo_rc.py @@ -0,0 +1,249 @@ +# Resource object code (Python 3) +# Created by: object code +# Created by: The Resource Compiler for Qt version 6.9.1 +# WARNING! All changes made in this file will be lost! + +from PySide6 import QtCore + +qt_resource_data = b"\ +\x00\x00\x0d\x1b\ +\x89\ +PNG\x0d\x0a\x1a\x0a\x00\x00\x00\x0dIHDR\x00\ +\x00\x00<\x00\x00\x00<\x08\x06\x00\x00\x00:\xfc\xd9r\ +\x00\x00\x00\x06bKGD\x00\xff\x00\xff\x00\xff\xa0\xbd\ +\xa7\x93\x00\x00\x00\x09pHYs\x00\x00.#\x00\x00\ +.#\x01x\xa5?v\x00\x00\x00\x07tIME\x07\ +\xe8\x05\x02\x04\x00\x17\xd0\xf0Y\xaf\x00\x00\x0c\xa8ID\ +ATh\xde\xedZ{t\x95\xd5\x95\xff\xed}\xbe\xc7\ +\xbd77/\x92\xf0\x06\x8bB\x15\x92\x12Pq\x90.\ +\xacv\xecT\x88\x04\xa9uam\x1d\x1f\x9d\xcej\xc7\ +G\xb5\xb6\xd6\x99\xaeYV\xdbN\xed\xa2\xed\xcc\xb4\xab\ +\xd3Q;C\x95Jg:-JHP\xabuX\xea\ +R[\xb0IH\x02\xc8C@y\xe5\x01\x97<\xee\xf3\ +\xfb\xce\xd9\xf3\xc7\xbd\x97\xdc\x5c\x12y%\x81ay\xfe\ +\xb9Y9\xfb\xdb\xdf\xf9}g\x9f\xdf\xde\xfb\xf7}\xc0\ +\x87\xe3\xdc\x1cw\xde\xfd\x0d\x1a\x0e?t\xae\x03\x0dV\ +]?M\x8b\xc03\xb2\xc6a\xfa2\x80\x82d[\xc3\ ++\xa7\xeb\x8f\xcfu\xc06\xd3\xb5\x00v\x05\x15w\xfb\ +\x82z\x01V\x9c\x89\xbfs\x1ap\xa8\xaafJ\xd47\ +\xd7\xbbL+c\xad\xf5W\x17\xd9|]\xcaH\x11f\ +.\xbe\xe5\xbc\x04\x1c\xf3M\xdcanM\x1a\x99\xf8\x17\ +\x9f\xbe\x83\x12Z&:L\x1d\x10y\xef\xbc=\xc3\x98\ +\xb9\xf8kA\xc5\xf7\x12P\xec\x8btxF\x94l]\ +?=;\xfd\x91+\x96\x95\x00D{\xfe\xb4&r^\ +\x00\xb6+k\x0a\x82\x0a\xd2\xe3\xc9\xcf!\xf2\xfd\x8a`\ + \xd2\xd9\xf8l{YumA\xaf\xaf\x97\x84-k\ +A\x5c\xebRE\xb4\xae4`\xfd\xfe\xfd\x8d\xcf\x1e\xfd\ +\xff\xbd\xc3C\x8c\x89\x97-\x9d\x12I\xeaf\x87\xa9\xc9\ +3b|\x91\xf9\x04T%\xdb\x1a\xf6|\xd0u\xd6H\ +.\xaa\xea\xaa\xe5\xc1\xd6W\xff;~\xb2\xf6c\xe7.\ +-Ij3+e\xe4\xb6\x90\xb2\x1e 2^WS\ +]r0\xdb\xc3\x09\xff_\x00\x1c\xee\xde\x5c\xffI\x00\ +\xe0Y\x8b\xb7+\xa2\x9f\x01\xa8\x19\x11\xd2\x9a\xf7\xa9\xdb\ +\x86|X\xe5sn\x98\x1a\xa8\xac\xa9\xdb\xda\xd5\xd7:\ +u\xde\x8d\xa1\x93\xf1W}\xcd\xe7\x0a\x8e$\xfdmI\ +#\xff\x994\xe6\xe6\x1e\xdfk\x8e\xfb\xe6\xf6\xa1\xec]\ +\xc5O\x1b\x81\x15\xac\xaa\x99\x81\x99\x8b\xa7\xd9\xc4\xc1\x00\ +\xf3/\x86\x9d\xa5/\x9a\xbf\xbc\xc0\xae\xacyu\xe3\xbe\ +N/Tu\xfd\xaa\xdc\xb9+\xaf\xbb\x8dK\xabk\xbf\ +\x1eIy[\x13F\x96h\x91\x0b;b\x89GN\xc6\ +\xef\xe6C\xdd\x0b\xb5\xc8\xc1B\xcb\xba;\xa4x\x82\x11\ +D|\xc1\x98\x8a9\xb5E\x83\xd9\xfb\x06\xdb5\xc4\x8d\ +ky\x9e\x09\x1bR\xc6\x88\x11yw\xd8\x01\xef\xef\x8b\ +.\xf5\x8c,\x84\x001m\xbe\x10\xaaZ\xf2\xf9LE\ +t\xf9\x9b{;\xdf\x8e\xa4\xfc\x15Z$\x04I\xdb'\ +\x8c\xdc\x8f\x99\x8b\xcbN\xc8\x9e\x84*\x22L\x22\x92\xe6\ +\xbe\x96\xfa\x98\x00\xef&\x8d\xb9\xf2p\xca\x1f3h\xca\ +j]\xb7\xf5\xe21\xe1\xe9e\x8e\xf5pX\xf1\xf7g\ +\x96\x17_\x12m\xado\x1ev\x96.\xad^rI\x8f\ +gZ\x05\xa2$\x0d\xaa}\x8ck]s4\xa5\xdb\xb4\ +\x08e\x9d\x12\x01\x16\xd1N\x06\xfe6\xd1\xd6\xf0\xbf'\ +\xf2[>gi\xe8H\xca\xdf\x08\x88\x0b`\x0b\x80%\ +e\xae\xbd\xb4\xb3qm]\xae\xdd\xf8K\x97M\x04\xcc\ +\xc7D\xe8\x22\xdf\xc8\x8b\x87\x9b\xd7\xee:\x95\xf5\xabS\ +\x05\x9ch\xdf\xde\xe5\x8e\xfdhX@\x1f\x07\x01\x02\x84\ +m\xe62\x11XZ\xa4\x82\x00(\xa2dP\xa9\xc7\xa6\ +\x97\x85o9\xb0\xf1\xd9\x1d'Ud\x1cz\xc7+\x9f\ +\x5c\x19a\xc8\xcbDXTb;\x9f\xe9Jz;\xd1\ +\xb5\xa3;\xd7n\xcc\xe4\x99\x93\x0e\xc4Ro\xc5\xb5\xae\ +Q\xc4\x1b\x92\x1d\xefl;\x95\xf5\x9f\x16KO)*\ +~\xb43\xda\xbbs(\x16o\xec\x8aE\xbf2\x22\xed\ +\xe1\x1b\xcf?\xa5\x89\xf0V\xee\xee\x84\x14\xbfY\xe2\xa8\ +\xb9\xdd\x9b\xd7}\xeb@O\xfc\x87\xbe1\x95\xe5\xae\xb3\ +\x1c\x82\xd7\x0a,~(\xa1\xcdO\xf6t\x1d]\x1eI\ +\xf9{\x03\x8a\xb6\xda\x94\x09#9\x96o\xe10\xc1\x22\ + \xa0\x186\xd3\x803M\x00\x98\x08.\x13\x98do\ +\xdb\xfe\x9e\xab\x00\xd9\xc7\x84\x07\xab\xc7\x8d\x1bK\x84\xce\ +>\xdf\x0f\x8d\x9b\xbbt\xcc\x88\xf4\xc3\x05\x8a\xefcB\ +\x0f\x01=%\x8e\xba\xbb\xb7e\xdd\x82\x8e\xc6\xba\x96\xa2\ +\xd9K\x1eH\x19sWJ\xe4\xcbA\x9b\xb68\x8a\xf6\ +'\x8d,*v\x9cgz9j\x80Kf\xd7^q\xd4\xf3\xffxVuj\ +\xc5\xd5\xf1\xd6\xfa\xcd\xa7{\xfd)\xd5\xd2Q\xadk\x03\ +L0\x00\x5c&\xf8\x02$\x8d\x81\x91\xfe21S+\ +C\x11\x904\x02B\xfa7'\xfc\xe12\xc3\x88\xc0a\ +\x82'\x82\xa4\xce\x9caI\x13V\xd6\x07S\xff\xb5\xa9\ +c\xbff\x19\x80\xd3\x06|J\xfd\xb0\x08\xaeN\x89\xc0\ +\x17A\x5c\x0b\x8c \x16R\xfc#\x02\xbed1=\xc1\ +\x84\x14\x80v_\xe4\xe1\xb8\x96M\xbeH\xbd\x01\xbeM\ +@<\x1d\x96\xfc]#\xa8K\x19\x03O\x04\xbe\xe0?\ +\x14\xe8\xa7\xd9\xc7\xc1\x8c\xf5\x00\x1e&P$a\x04\x9e\ +\xe0\xdf\x5c\xa6\xdf\xfa\x03S\xd7U\xa3)\xc4_.\x02\ +\x88\x00\x06\x82B[-\x0aY\xfc=\x87\xc9w\x18\xf7\ +;\xcc_ePG\xaa\xad\xe1Q_\xe4m#h\xf0\ +\xda\x1a\x1ea\xa2\x7f\x06\x80\xcb\xa7L}\x0c\x22\xf5\xe9\ +`\xa0\xc8\xacq\xc5wE[\xeb\xef\x0d0\xef\x01\x00\ +\x9b\xe8\x05\xd9\xba\xfe\xd1\x02K}[ \xe0\xf1\x92\ +\x80\xfd\xbb\x5c\xbcL4oT\x00O\xbfry\xb1/\ +\xe2f\xa3W\x84\x9a\xbb\x9a\xd6\xbe\xda\xe7\x99\x1f'\x8d\ +\xacL\x19\xfcLDJ$'\x1c\xb2C\x8b\xfc\x13\x13\ +\x1d\x22J_l\x04p\x99~\xb3\xa3\xb3\xc7\xb6+\x17\ +O\x02\xe1\x97\xd9*\x0c\x00.\x99P\xf2s\x02\xed\xc8\ +m\xaa\xb2\xc33Rx\xc5_}10\xe2\x80{\x13\ +\xc9\xe9\x03\x08\xcc\xe1C\x00\x10\xd3\xa6\x0a\xc03\xbe\xc8\ +\xeb\x06\xe8\x19DH/+s\xec\xcb-\xc2?\x0cx\ +\x8db\xa9_i\x91O\xb9\xac>[\xe6Z\x03\xd2\xdc\ +\x96\x03\x91\xdaB\x8b\x1f\x1cj-\xbb::g\x8c8\ +\xe0\xf6xj\x80:\xd2\xe7\x99\xa2L\xc3p\x10\xc04\ +\x00;\xa7\x15\x87V\xe5'\x00\x02\xa4\xcf\xd7?\xf9\xcb\ +\xcai\xbfLy\xf1\x04\xd2\x84\xb7\xa7\xabi\xed\xeb1\ +\xdft\xc4\xb4i\xde\xff\xf6s\xbb\x1d\xa6\xd7\xb2[\xd9\ +\xe7\xeb\xc7\xc2\x8e\xbb~Zi\xe9\xce\xc1\xd6r$\xe9\ +\xab\x11\x07<\xa3\xb4\xe0\xe0\xb1\x0a(]\x05]z\xc1\ +\x157}\xa4\xd4Q\x7f\x0fAk\xb1m\xf5)\xa6c\ +\x0e\xad\x9c\xa6 i\xcc\xec7\xb6\xed\xbdc\xd3\xcb+\ +5\x88`1=u\xd9\xb5w*\x97\xe9&E\xb8\xa1\ +|Nmy@\xf1\xd39\xa1;\xfdp\x22q\xcf\x96\ +\xd7V\xc52\x0fu`\xd0\x94\x17\xed\x1b\xf1<|\xcd\ +\xd2/\xd1\x86\xed\xfb\x8d\xab\xd2up\x80\x09\x00\xed\x22\ +\xc27\xbb=\xbd\xc7\x22\x9a\xa8\x88\xee6\x90\x8bJ\x1c\ +ukR\xcb\x83qmv\x04\x98\xfe\xd0\xeb\x9b\x17\x14\ +\xd1A-r#\x01\xb5c\x03N[R\xeb\xa0\x16y\ +\xc2\x13\x81o\xf0\x90\xc5\xd4\xec\x19\xb9V \xbf%\xe0\ +\xcd\xa0\xe2\x9e\xb0\xad\x96\x0a\xb00\x92\xf4\x1fM\x19\xc9\ +\xa6>\x8d\xad\xeb\xad\x11\x07\x0c\x00NeM\x9b\x11\xcc\ +BZ\x00\x80\x06\x06\x14\xfe\xb9\xcd\x01\x03\xf02\xed^\ +~\xf3`1A$\xe3C\xd2\xf3y\x02\x00,\xea\xf7\ +\x91+\x22\xd8L\xcd^[\xc3\x9c\xd1JKo)\xca\ +H\xb0L\x03%\xcf\x9c~\x973\xf3\x9c\xf9;O\x7f\ +\x06g@\xa9\x8cM~?\x9c\x0d\xe3\xc1|\x18\xc1\xeb\ +\xa3\x96\x87-\xa2:O\x04Z\x04\x09-\xf0rw\x86\ +\xfa\xb3\x91g\xd2\xf3:S\xa4\xe4\x15/\x99\xa2#]\ +a\xe5\xfb\x90\x8cM*\xc7Gn\xb7\x15R\xf4\xfc\xa8\ +\x01\x9e5a\xec\x0b.sDe\x14\x09\x8b\xa8\xffL\ +\xe4\xec\xb0\xcd\x04We\xc8\x8d\x8e?5v\xe6\xff.\ +\x13\xec<\x1f\x94\xa3\x8a\xb8\x8a\xc0\xe8\xf7\xc1\x84\xce\xf2\ +\x90\xf3\xe2\x99\x00>%z?\xf0n\xa3\xe6\x8a\x19\xc5\ +\xbe\x91\x85>\x04&\xb7(\xa0~\xdc&snM\xa6\ +\xc8\x18L\x9d5\x02\xf8\x19e#-\x0a\xd2\x0e\x22\xda\ +\xcaDeZ\xe0\x18\xc8q>\x14\xd1\x0f\x8e4\xd5m\ +\x18\xcd\xd2\x12\xa5\x8e\xbbB\x80\xa3C\x81\xc9\x9c3d\ +IU\x86\x98\xcf\x82\xce\xbc\x92\xb9\xb3\xcc\xb5>aD\ +\xee\x9bZTp\x01\x80\xd7\xf2}\x10\xd0Y\xe2X\xff\ +z\xa6\xdd\xd6)\x03no\x5c\x13\x09[\xfc\xcd\x02\xc5\ +p\xf9\xf8\x90f\x02\x02\x8aP\x90\x11\xe8\xec<\xd6\xa2\ +L\xa7es\xda&\xac\xd4\xaf\x89\xd0\xd4\x9e\xf0v\x0a\ +\xb0i_ot\xe5\x05\x85\xc1;\x82\x8aQ\xa0\x18\x16\ +\xa5m-\xa6ot5\xad\xed\x19u\xc0\x00\xd0\xd7R\ +\xff\x84/X7\x18i\x99,\xe1\x98\xb4V\xe5\xe7\x85\ +\x81 MZZ$\xe6\x89\xbc\x1e\xd3f\xb5\xc3t;\ +\x80\x00\x00\xe3\x19\xa9)\x0e\xb9\x9d\x0c\xfcF\x03\xdb\xb5\ +\x08\x5c\xe6g\xbd\xb6\x86\xa7FE\x88\x1fjT\xcc]\ +\x1a\xeeI\xf9o%\x8dT\x0e\xe5T>\xe0\xa6%\x8e\ +\xf5\xb9>_7{F\x86|a>>h\xef\xec\xf5\ +\xf4\x0a\x06\xee\xefm\xa9\x8f\x9eU\xc0\x00P>\xa7v\ +B$\xa5_\xd2\x92\x07ZN\xca\xfb<\x00\x8f\x03\xb8\ +t\xa8\xf7NS\x0a\xdd[\xc6\x15\x07\xffk\xd3\xcb\xab\ +e\xb8\x14\x933\xfaN\xab\xab\xa9\xee\xe0\xf8\xa0\xbd\xc0\ +az\xe9\xb8\xc7H\xc3\xb2\x152\x9c`\x81a\xfa\x8a\ +\xe7\x9a\xda/\xd2\xc6\xdd\x1d_Oh\xf3\x88/\x12\xa4\ +A\xd2P\xfeM\x1d\xe6\xe5\x01\xa5\xb6*\xa2!\x15\xce\ +\xa0\xcd-\xfb7\xad\xe9>\xe7\x00g\xc7\xd4y7N\ +>\x10M|\xcb\xce\x90\x90\x97!\xac\xc1\x84x\x02\x92\ +aK\xfd\xb94\x14\xb8\xe1\xbd\x9e\xe8+\xa9\x0c\x17\xa4\ +\x1f\x06\xdd\x9e\x1c&\x92\x1a\xd6\x90\xce\x1f\xefm\xfc\xdd\ +>\x7fK\xc3W\x00L\x0e\xdb\xea.&\xbc`3E\ +\xf3\xdf-e*'\xd73\xa62\xe5{\xbdL46\ +7\x14\x92Z\x9c\x91R=G\xe5\xc3\xb4qsk\xa7\ +\xc7}\xa9\xf0D\xc6\x19\x11q\x98\xdb\x83\x16uv4\ +\xd6\xed\xca\xa8\x22\x03\xa2\xbe\xc0\xe2\x87\xa2-\xf5?\x18\ +\x89\xb5X\xa3\x01\xb8\xbd\xb1n'\x80c\xeaE\x12@\ +\xef\x07\xd8\xa7\x8c\x8c\x1f\xa9\xb5\x9c\xf5\xafi/\x9c\xff\ +\xd9\xb1\xf9\xff\xf3\x8c\x14\x9c\xb7\x80\xa3I\xbf\x9c\x8e\x17\ +\x10\xf8\xbc\x05|$\xe5\x85\xf2S\x98\xcbt\xe1y\x0b\ +\xb8\xd4\xb18_\xf1Hh\x09\x9f\xb7\x80)-\xf1\xe6\ +\xe7\x0c\xeb\xbc\x05\xdc\x95\xf4\x0b%/\xa4\x89P4R\ +\xf7\xb3\xce6`\x8b\xe8\x1dm\xe4\xf1\xbc]H\xe2\xc3\ +\xf1\xe18\xad\xf1\x7f\x05C\xfd\x0d\x0b\x86\x1c\x9c\x00\x00\ +\x00\x00IEND\xaeB`\x82\ +" + +qt_resource_name = b"\ +\x00\x03\ +\x00\x00p7\ +\x00i\ +\x00m\x00g\ +\x00\x0e\ +\x05r\x14\xa7\ +\x00g\ +\x00i\x00n\x00a\x00n\x00-\x00l\x00o\x00g\x00o\x00.\x00p\x00n\x00g\ +" + +qt_resource_struct = b"\ +\x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x01\ +\x00\x00\x00\x00\x00\x00\x00\x00\ +\x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x02\ +\x00\x00\x00\x00\x00\x00\x00\x00\ +\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\ +\x00\x00\x01\x99\xa2O\xd1\xe2\ +" + +def qInitResources(): + QtCore.qRegisterResourceData(0x03, qt_resource_struct, qt_resource_name, qt_resource_data) + +def qCleanupResources(): + QtCore.qUnregisterResourceData(0x03, qt_resource_struct, qt_resource_name, qt_resource_data) + +qInitResources() diff --git a/scripts/GinanUI/app/utils/__init__.py b/scripts/GinanUI/app/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/GinanUI/app/utils/cddis_credentials.py b/scripts/GinanUI/app/utils/cddis_credentials.py new file mode 100644 index 000000000..056050472 --- /dev/null +++ b/scripts/GinanUI/app/utils/cddis_credentials.py @@ -0,0 +1,119 @@ +# app/utils/cddis_credentials.py +from __future__ import annotations +import os, platform, stat, shutil +from pathlib import Path +import netrc + +URS = "urs.earthdata.nasa.gov" +CDDIS = "cddis.nasa.gov" + +def _win_user_home() -> Path: + """ + Return the Windows user home path. + + Returns: + Path: Path to the current user's home directory on Windows; falls back to Path.home() if env var is missing. + + Example: + >>> isinstance(_win_user_home(), Path) + True + """ + return Path(os.environ.get("USERPROFILE", str(Path.home()))) + +def netrc_candidates() -> tuple[Path, ...]: + """ + Return possible credential file paths on this OS. + + Returns: + tuple[Path, ...]: Candidate paths to search/write `.netrc`-style credentials. On Windows: (%USERPROFILE%\\.netrc, %USERPROFILE%\\_netrc); on macOS/Linux: (~/.netrc,). + + Example: + >>> tuple(map(lambda p: p.name, netrc_candidates())) # doctest: +ELLIPSIS + ('...netrc',) or ('...netrc', '_netrc') + """ + if platform.system().lower().startswith("win"): + return (_win_user_home() / ".netrc", _win_user_home() / "_netrc") + return (Path.home() / ".netrc",) + +def _write_text_secure(p: Path, content: str) -> None: + """ + Write text to a file, applying secure permissions on non-Windows. + + Arguments: + p (Path): Target file path. + content (str): File content to write (UTF-8). + """ + p.write_text(content, encoding="utf-8") + if not platform.system().lower().startswith("win"): + os.chmod(p, stat.S_IRUSR | stat.S_IWUSR) # 0600 + +def save_earthdata_credentials(username: str, password: str) -> tuple[Path, ...]: + """ + Save Earthdata credentials for both URS and CDDIS hosts. + + Arguments: + username (str): Earthdata (URS/CDDIS) account username. + password (str): Earthdata (URS/CDDIS) account password. + + Returns: + tuple[Path, ...]: The list of credential files written. Also sets environment variable NETRC to the preferred file. + + Example: + >>> paths = save_earthdata_credentials("user", "pass") # doctest: +SKIP + >>> len(paths) >= 1 + True + """ + content = ( + f"machine {URS} login {username} password {password}\n" + f"machine {CDDIS} login {username} password {password}\n" + ) + written: list[Path] = [] + for p in netrc_candidates(): + _write_text_secure(p, content) + written.append(p) + os.environ["NETRC"] = str(written[0]) + return tuple(written) + +def _ensure_windows_mirror() -> None: + """ + Ensure .netrc exists by mirroring _netrc on Windows if necessary. + """ + if not platform.system().lower().startswith("win"): + return + dot, under = _win_user_home() / ".netrc", _win_user_home() / "_netrc" + if under.exists() and not dot.exists(): + try: + shutil.copyfile(under, dot) + except Exception: + pass + +def validate_netrc(required=(URS, CDDIS)) -> tuple[bool, str]: + """ + Validate presence and completeness of Earthdata credentials. + + Arguments: + required (tuple[str, ...]): Hostnames that must have valid entries. + + Returns: + tuple[bool, str]: If valid, (True, path-to-netrc). If invalid, (False, reason). + + Example: + >>> ok, info = validate_netrc() # doctest: +SKIP + >>> ok in (True, False) + True + """ + _ensure_windows_mirror() + candidates = netrc_candidates() + p = next((c for c in candidates if c.exists()), candidates[0]) + if not p.exists(): + return False, f"not found: {p}" + try: + n = netrc.netrc(p) + for host in required: + auth = n.authenticators(host) + if not auth or not auth[0] or not auth[2]: + return False, f"missing credentials for {host} in {p}" + os.environ["NETRC"] = str(p) + return True, str(p) + except Exception as e: + return False, f"invalid netrc {p}: {e}" \ No newline at end of file diff --git a/scripts/GinanUI/app/utils/cddis_email.py b/scripts/GinanUI/app/utils/cddis_email.py new file mode 100644 index 000000000..3408e8ac4 --- /dev/null +++ b/scripts/GinanUI/app/utils/cddis_email.py @@ -0,0 +1,220 @@ +# app/utils/cddis_email.py +""" +Utilities for managing the EMAIL used by the CDDIS flow and for quick connectivity/auth checks. + +This module is used by the UI credential flow to: + • Read/Write the EMAIL value (env var first, then a local CDDIS.env file). + • Derive the email/username from `.netrc/_netrc` when the user only saved Earthdata credentials. + • Test connectivity to cddis.nasa.gov and verify Earthdata authentication via requests. + +Notes: + - This module does not present UI; it is called by UI dialogs/controllers. + - File locations are platform-aware and compatible with Windows/macOS/Linux. +""" + +from __future__ import annotations +import os +import platform +import time +from pathlib import Path +from typing import Tuple +import netrc +import requests + +ENV_FILE = Path(__file__).resolve().parent / "CDDIS.env" +EMAIL_KEY = "EMAIL" + +# ------------------------------ +# Select the .netrc/_netrc path (for compatibility with different implementations) +# ------------------------------ +def _pick_netrc() -> Path: + """ + Select a `.netrc`-style credential file path to use. + + Returns: + Path: Resolved path to the preferred credential file. + + Example: + >>> isinstance(_pick_netrc(), Path) + True + """ + try: + from app.utils.cddis_credentials import netrc_path as _netrc_path # type: ignore + except Exception: + _netrc_path = None + if _netrc_path: + try: + return _netrc_path() + except Exception: + pass + try: + from app.utils.cddis_credentials import netrc_candidates as _netrc_candidates # type: ignore + cands = _netrc_candidates() + for p in cands: + if p.exists(): + return p + return cands[0] + except Exception: + if platform.system().lower().startswith("win"): + return Path(os.environ.get("USERPROFILE", str(Path.home()))) / ".netrc" + return Path.home() / ".netrc" + +def read_email() -> str | None: + """ + Read the EMAIL used by CDDIS utilities. + + Returns: + str | None: EMAIL value if found. Lookup order: env var EMAIL → CDDIS.env → None. + + Example: + >>> os.environ.pop("EMAIL", None) + >>> _ = ENV_FILE.write_text('EMAIL="user@example.com"\\n', encoding="utf-8") + >>> read_email() + 'user@example.com' + """ + v = os.environ.get(EMAIL_KEY, "").strip() + if v: + return v + if ENV_FILE.exists(): + for line in ENV_FILE.read_text(encoding="utf-8").splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + k, _, val = s.partition("=") + if k.strip() == EMAIL_KEY: + return val.strip().strip('"').strip("'") + return None + +def write_email(email: str) -> Path: + """ + Persist the EMAIL value to `CDDIS.env` and update the process env. + + Arguments: + email (str): Email address to store. + + Returns: + Path: Path to the written CDDIS.env file. + + Example: + >>> p = write_email("user@example.com") + >>> p.exists() + True + >>> read_email() + 'user@example.com' + """ + ENV_FILE.parent.mkdir(parents=True, exist_ok=True) + ENV_FILE.write_text(f'{EMAIL_KEY}="{email}"\n', encoding="utf-8") + os.environ[EMAIL_KEY] = email + return ENV_FILE + +def get_username_from_netrc(prefer_host: str = "urs.earthdata.nasa.gov") -> Tuple[bool, str]: + """ + Read the username from `.netrc/_netrc`, assuming username equals EMAIL. + + Arguments: + prefer_host (str): Primary host to query in netrc (fallback to cddis.nasa.gov). + + Returns: + tuple[bool, str]: (True, username) if found; otherwise (False, reason). + + Example: + >>> ok, val = get_username_from_netrc() # doctest: +SKIP + >>> ok in (True, False) + True + """ + p = _pick_netrc() + if not p.exists(): + return False, f"no netrc at {p}" + try: + n = netrc.netrc(p) + auth = n.authenticators(prefer_host) or n.authenticators("cddis.nasa.gov") + if not auth or not auth[0]: + return False, f"no authenticators for {prefer_host} or cddis.nasa.gov in {p}" + return True, auth[0] + except Exception as e: + return False, f"parse netrc failed: {e}" + +def ensure_email_from_netrc(prefer_host: str = "urs.earthdata.nasa.gov") -> Tuple[bool, str]: + """ + Ensure that EMAIL is available, deriving it from netrc if necessary. + + Arguments: + prefer_host (str): Primary host to read username from in netrc. + + Returns: + tuple[bool, str]: (True, email) if resolved; otherwise (False, reason). + + Example: + >>> ok, email = ensure_email_from_netrc() # doctest: +SKIP + >>> ok in (True, False) + True + """ + existing = read_email() + if existing: + os.environ[EMAIL_KEY] = existing + return True, existing + ok, user = get_username_from_netrc(prefer_host=prefer_host) + if not ok: + return False, user + write_email(user) + return True, user + +def get_netrc_auth() -> tuple[str, str] | None: + """ + Retrieve (username, password) from `.netrc/_netrc` for Earthdata auth. + + Returns: + tuple[str, str] | None: (username, password) if found; otherwise None. + + Example: + >>> creds = get_netrc_auth() # doctest: +SKIP + >>> creds is None or isinstance(creds, tuple) + True + """ + p = _pick_netrc() + if not p.exists(): + return None + n = netrc.netrc(p) + for host in ("cddis.nasa.gov", "urs.earthdata.nasa.gov"): + auth = n.authenticators(host) + if auth and auth[0] and auth[2]: + return (auth[0], auth[2]) + return None + +def test_cddis_connection(timeout: int = 15) -> tuple[bool, str]: + """ + Test CDDIS connectivity and Earthdata authentication in two phases. + + Arguments: + timeout (int): Overall timeout in seconds for the restricted request phase. + + Returns: + tuple[bool, str]: (True, 'AUTH OK, took X.XXX seconds') on success; otherwise (False, reason). + + Example: + >>> ok, msg = test_cddis_connection() # doctest: +SKIP + >>> ok in (True, False) + True + """ + print("Testing connectivity to cddis.nasa.gov...") + start_time = time.perf_counter() + r = requests.get("https://cddis.nasa.gov/robots.txt", timeout=(5, timeout)) + if r.status_code != 200: + return False, f"HTTP {r.status_code} on robots.txt" + print(f"Connectivity OK. Took {time.perf_counter() - start_time:.3f} seconds\nTesting authentication using .netrc...") + + start_time = time.perf_counter() + creds = get_netrc_auth() + if not creds: + return False, "no usable credentials in .netrc" + session = requests.Session() + session.auth = creds + url = "https://cddis.nasa.gov/archive/00readme" + resp = session.get(url, timeout=(5, timeout), allow_redirects=True) + head = resp.text[:1200] + if resp.status_code == 200 and "Earthdata Login" not in head: + return True, f"AUTH OK, took {time.perf_counter() - start_time:.3f} seconds" + return False, f"HTTP {resp.status_code} or login page returned" + +if __name__ == "__main__": + print(test_cddis_connection()) \ No newline at end of file diff --git a/scripts/GinanUI/app/utils/common_dirs.py b/scripts/GinanUI/app/utils/common_dirs.py new file mode 100644 index 000000000..3c0c9515a --- /dev/null +++ b/scripts/GinanUI/app/utils/common_dirs.py @@ -0,0 +1,17 @@ +import sys +from pathlib import Path + +def get_base_path(): + """Get the base path for resources, handling both development and PyInstaller bundled modes.""" + if getattr(sys, 'frozen', False): + # Running in PyInstaller bundle - sys._MEIPASS is _internal/ + # and app folder is at _internal/app/ + return Path(sys._MEIPASS) / "app" + else: + # Running in development mode - __file__ is in app/utils/ + return Path(__file__).parent.parent + +BASE_PATH = get_base_path() +TEMPLATE_PATH = BASE_PATH / "resources" / "Yaml" / "default_config.yaml" +GENERATED_YAML = BASE_PATH / "resources" / "ppp_generated.yaml" +INPUT_PRODUCTS_PATH = BASE_PATH / "resources" / "inputData" / "products" diff --git a/scripts/gn_functions.py b/scripts/GinanUI/app/utils/gn_functions.py similarity index 99% rename from scripts/gn_functions.py rename to scripts/GinanUI/app/utils/gn_functions.py index 690a35f93..49e7595f9 100644 --- a/scripts/gn_functions.py +++ b/scripts/GinanUI/app/utils/gn_functions.py @@ -1,7 +1,6 @@ """Base time conversion functions""" from datetime import datetime as _datetime -from pathlib import Path as _Path import logging import shutil import os as _os diff --git a/scripts/GinanUI/app/utils/logger.py b/scripts/GinanUI/app/utils/logger.py new file mode 100644 index 000000000..e15afb268 --- /dev/null +++ b/scripts/GinanUI/app/utils/logger.py @@ -0,0 +1,98 @@ +""" +Unified logging system for Ginan-UI + +This module provides a thread-safe logging interface that can passes +messages to different UI channels ("terminal" or "console" at the moment) via Qt signals. + +Usage: + # In main_window.py initialisation: + Logger.initialise(main_window_instance) + + # Anywhere in your code: + Logger.terminal("Message for terminal") + Logger.console("Message for console") + Logger.both("Message for both channels") +""" + +from PySide6.QtCore import QObject, Signal +from typing import Optional + + +class LoggerSignals(QObject): + """Signal container for thread-safe logging""" + terminal_signal = Signal(str) + console_signal = Signal(str) + + +class Logger: + """ + Static logger class for easy logging throughout the application. + + All methods are thread-safe and can be called from worker threads. + """ + _signals: Optional[LoggerSignals] = None + _main_window = None + + @classmethod + def initialise(cls, main_window): + """ + Initialise the logger with the main window instance. + + :param main_window: MainWindow instance with log_message method + """ + cls._main_window = main_window + cls._signals = LoggerSignals() + + # Connect signals to main window's log_message method + cls._signals.terminal_signal.connect( + lambda msg: main_window.log_message(msg, channel = "terminal") + ) + cls._signals.console_signal.connect( + lambda msg: main_window.log_message(msg, channel = "console") + ) + + @classmethod + def terminal(cls, message: str): + """ + Log a message to the terminal widget. + Thread-safe. + + :param message: Message to log + """ + if cls._signals is None: + print(f"[Logger not initialised - terminal] {message}") + return + + # Simply emit the signal - Qt handles thread safety automatically + cls._signals.terminal_signal.emit(message) + + @classmethod + def console(cls, message: str): + """ + Log a message to the console widget. + Thread-safe. + + :param message: Message to log + """ + if cls._signals is None: + print(f"[Logger not initialised - console] {message}") + return + + # Simply emit the signal - Qt handles thread safety automatically + cls._signals.console_signal.emit(message) + + @classmethod + def both(cls, message: str): + """ + Log a message to both terminal and console widgets. + Thread-safe. + + :param message: Message to log + """ + cls.terminal(message) + cls.console(message) + + @classmethod + def is_initialised(cls) -> bool: + """Check if the logger has been initialised""" + return cls._signals is not None \ No newline at end of file diff --git a/scripts/GinanUI/app/utils/toast.py b/scripts/GinanUI/app/utils/toast.py new file mode 100644 index 000000000..421c1d993 --- /dev/null +++ b/scripts/GinanUI/app/utils/toast.py @@ -0,0 +1,198 @@ +from PySide6.QtWidgets import QLabel, QGraphicsOpacityEffect, QPushButton +from PySide6.QtCore import QTimer, QPropertyAnimation, QEasingCurve, Qt, QEvent +from PySide6.QtGui import QFont + + +class Toast(QLabel): + """ + A non-blocking toast notification that appears at the bottom of the window, + fades in, stays visible, then fades out automatically. + """ + + def __init__(self, parent=None): + super().__init__(parent) + + # Makes it a child widget so it moves with the program + self.setWindowFlags(Qt.WindowType.Widget) + self.setAttribute(Qt.WidgetAttribute.WA_ShowWithoutActivating) + + # Styling + self.setStyleSheet(""" + QLabel { + background-color: #2c5d7c; + color: #ffffff; + padding: 14px 40px 14px 24px; + border-radius: 8px; + font-size: 13px; + font-weight: 500; + border: 1px solid rgba(255, 255, 255, 0.2); + } + """) + self.setAlignment(Qt.AlignmentFlag.AlignCenter) + + # Font + font = QFont() + font.setPointSize(12) + font.setWeight(QFont.Weight.Medium) + self.setFont(font) + + # Close button + self.close_button = QPushButton("×", self) + self.close_button.setStyleSheet(""" + QPushButton { + background-color: transparent; + color: rgba(255, 255, 255, 0.7); + border: none; + font-size: 20px; + font-weight: bold; + padding: 0px; + margin: 0px; + } + QPushButton:hover { + color: rgba(255, 255, 255, 1.0); + background-color: rgba(255, 255, 255, 0.1); + } + """) + self.close_button.setFixedSize(24, 24) + self.close_button.setCursor(Qt.CursorShape.PointingHandCursor) + self.close_button.clicked.connect(self._on_close_clicked) + self.close_button.hide() + + # Opacity effect for fade animations + self.opacity_effect = QGraphicsOpacityEffect(self) + self.setGraphicsEffect(self.opacity_effect) + self.opacity_effect.setOpacity(0.0) + + # Animation for fade in and out + self.fade_animation = QPropertyAnimation(self.opacity_effect, b"opacity") + self.fade_animation.setDuration(300) # 300ms fade + self.fade_animation.setEasingCurve(QEasingCurve.Type.InOutQuad) + + # Timer for auto-hide + self.hide_timer = QTimer(self) + self.hide_timer.setSingleShot(True) + self.hide_timer.timeout.connect(self._fade_out) + + # Track if fade_out signal is connected + self._fade_connected = False + + # Install event filter on parent to reposition on resize + if parent: + parent.installEventFilter(self) + + def eventFilter(self, obj, event): + """Reposition toast when parent window is resized or moved.""" + if obj == self.parent() and event.type() in (QEvent.Type.Resize, QEvent.Type.Move): + if self.isVisible(): + self._update_position() + return super().eventFilter(obj, event) + + def _update_position(self): + """Update the toast position to stay centered at bottom of the program.""" + if self.parent(): + parent_rect = self.parent().rect() + x = (parent_rect.width() - self.width()) // 2 + y = parent_rect.height() - self.height() - 50 # 50px from bottom + self.move(x, y) + + # Position close button at top-right corner of toast + self.close_button.move( + self.width() - self.close_button.width() - 8, # 8px from right edge + 8 # 8px from top edge + ) + + def show_message(self, message: str, duration: int = 3000): + """ + Show a toast message for a specified duration. + + Arguments: + message (str): Text to display + duration (int): How long to show the message in milliseconds (default 3000ms = 3s) + """ + # Stop any ongoing animation and timer to prevent flashing + self.fade_animation.stop() + self.hide_timer.stop() + + # Disconnect any previous finished signal connections (only if connected) + if self._fade_connected: + try: + self.fade_animation.finished.disconnect() + self._fade_connected = False + except: + pass + + # Reset width constraints so the toast can resize for new message + self.setMinimumWidth(0) + self.setMaximumWidth(16777215) # Qt's default maximum - just really big + + self.setText(message) + self.adjustSize() + + # Ensure a minimum width + if self.width() < 250: + self.setFixedWidth(250) + + # Position at bottom-center of parent window + self._update_position() + + # Show close button + self.close_button.show() + self.close_button.raise_() + + # Fade in + self.show() + self.raise_() # Bring to front + self.fade_animation.setStartValue(self.opacity_effect.opacity()) # Start from current opacity + self.fade_animation.setEndValue(1.0) + self.fade_animation.start() + + # Schedule fade out + self.hide_timer.start(duration) + + def _on_close_clicked(self): + """Handle close button click - fade out immediately.""" + self._fade_out() + + def _fade_out(self): + """Fade out the toast and hide it.""" + self.fade_animation.stop() + + # Hide close button + self.close_button.hide() + + # Disconnect previous connections to avoid multiple hide() calls + if self._fade_connected: + try: + self.fade_animation.finished.disconnect() + self._fade_connected = False + except: + pass + + self.fade_animation.setStartValue(self.opacity_effect.opacity()) + self.fade_animation.setEndValue(0.0) + self.fade_animation.finished.connect(self.hide) + self._fade_connected = True + self.fade_animation.start() + + +def show_toast(parent, message: str, duration: int = 3000): + """ + User feedback function to show a small toast notification at the bottom of the program. + + Arguments: + parent: Parent widget (typically "main_window.py) + message (str): Message to display + duration (int): Display duration in milliseconds (default 3000ms) + + Returns: + Toast: The toast instance (not required, but this does allow handling of the toast instance) + + Example: + show_toast(self.main_window, "Scanning CDDIS archive...", 5000) + """ + # Reuse existing toast if available + if not hasattr(parent, '_toast_widget'): + parent._toast_widget = Toast(parent) + + parent._toast_widget.show_message(message, duration) + return parent._toast_widget \ No newline at end of file diff --git a/scripts/GinanUI/app/utils/ui_compilation.py b/scripts/GinanUI/app/utils/ui_compilation.py new file mode 100644 index 000000000..31160d264 --- /dev/null +++ b/scripts/GinanUI/app/utils/ui_compilation.py @@ -0,0 +1,48 @@ +import subprocess, shutil +from pathlib import Path + + +def compile_ui(): + """ + Compile the Qt `.ui` file into a Python module and fix its resource import. + + Converts `main_window.ui` into `main_window_ui.py` using `pyside6-uic`, + then updates the logo import line for correct resource loading. + + Raises: + ImportError: If `pyside6-uic` is not found. + + Example: + >>> compile_ui() + UI compiled successfully. + """ + + # File paths + ui_file = Path(__file__).parent.parent / "views" / "main_window.ui" + output_file = Path(__file__).parent.parent / "views" / "main_window_ui.py" + + # Ensure compiler exists + if shutil.which("pyside6-uic"): + with open(output_file, 'w') as f: + f.write("# This file is auto-generated. Do not edit.\n") + result = subprocess.run(["pyside6-uic", ui_file, "-o", output_file], capture_output=True) + if result.returncode != 0: + print(f"Error compiling UI: {result.stderr.decode()}") + else: + print("UI compiled successfully.") + print(result.stdout.decode()) + else: + raise ImportError("Ensure pyside6-uic is installed and available on PATH.") + + # Manually fix the file path to the logo resource + with open(output_file, 'r') as f: + lines = f.readlines() + for i, line in enumerate(lines): + if line == "import ginan_logo_rc\n": + lines[i] = "from scripts.GinanUI.app.resources import ginan_logo_rc" + break + with open(output_file, 'w') as f: + f.writelines(lines) + +if __name__ == "__main__": + compile_ui() \ No newline at end of file diff --git a/scripts/GinanUI/app/utils/workers.py b/scripts/GinanUI/app/utils/workers.py new file mode 100644 index 000000000..22bcbfbd5 --- /dev/null +++ b/scripts/GinanUI/app/utils/workers.py @@ -0,0 +1,128 @@ +# app/utils/workers.py +import traceback +from datetime import datetime +from pathlib import Path +from typing import Optional + +import pandas as pd +from PySide6.QtCore import QObject, Signal, Slot + +from scripts.GinanUI.app.models.dl_products import get_product_dataframe, download_products, get_brdc_urls, METADATA, download_metadata +from scripts.GinanUI.app.utils.common_dirs import INPUT_PRODUCTS_PATH + +from scripts.GinanUI.app.utils.logger import Logger + +class PeaExecutionWorker(QObject): + """ + Executes execute_config() method of a given PEAExecution instance. + The 'execution' object is expected to implement: + - execute_config() + - stop_all() (optional but recommended: terminate underlying process) + """ + finished = Signal(object) + + def __init__(self, execution): + super().__init__() + self.execution = execution + + @Slot() + def stop(self): + try: + Logger.terminal("🛑 Stop requested — terminating PEA...") + # recommended to implement stop_all() in Execution to terminate child processes + if hasattr(self.execution, "stop_all"): + self.execution.stop_all() + Logger.terminal("🛑 Stopped") + except Exception: + tb = traceback.format_exc() + Logger.terminal(f"⚠️ Exception during stop:\n{tb}") + + @Slot() + def run(self): + try: + self.execution.execute_config() + self.finished.emit("✅ Execution finished successfully.") + except Exception: + tb = traceback.format_exc() + Logger.terminal(f"⚠️ Error launching Execution! Exception:\n{tb}") + + +class DownloadWorker(QObject): + """ + Downloads PPP and BRDC products for a specified date range or retrieves valid analysis centers. + + :param products: DataFrame of products to download. (See get_product_dataframe()) + :param download_dir: Directory to save downloaded products. + :param start_epoch: Start datetime for BRDC files. + :param end_epoch: End datetime for BRDC files. + :param analysis_centers: Set to true to retrieve valid analysis centers, ensure start and end date specified + """ + finished = Signal(object) + progress = Signal(str, int) + atx_downloaded = Signal(str) + + def __init__(self, start_epoch: Optional[datetime]=None, end_epoch: Optional[datetime]=None, + download_dir: Path=INPUT_PRODUCTS_PATH, products: pd.DataFrame=pd.DataFrame(), analysis_centers=False): + super().__init__() + self.products = products + self.download_dir = download_dir + self.start_epoch = start_epoch + self.end_epoch = end_epoch + self.analysis_centers = analysis_centers + self._stop = False + + @Slot() + def stop(self): + self._stop = True + + @Slot() + def run(self): + + # 1. Get valid products + if self.analysis_centers: + if not self.start_epoch and not self.end_epoch: + Logger.terminal(f"📦 No start and/or end date, can't check valid analysis centers") + return + Logger.terminal(f"📦 Retrieving valid products") + try: + valid_products = get_product_dataframe(self.start_epoch, self.end_epoch) + self.finished.emit(valid_products) + except Exception as e: + tb = traceback.format_exc() + Logger.terminal(f"⚠️ Error whilst retrieving valid products:\n{tb}") + Logger.terminal(f"⚠️ {e}") + return + + # 2. Install metadata + elif self.products.empty: + try: + download_metadata(self.download_dir, self.progress.emit, self.atx_downloaded.emit) + except Exception as e: + tb = traceback.format_exc() + Logger.terminal(f"⚠️ Error whilst downloading metadata:\n{tb}") + Logger.terminal(f"⚠️ {e}") + return + + self.finished.emit("📦 Downloaded metadata successfully.") + + + # 3. Install products + else: + try: + def check_stop(): + return self._stop + # Disregard generator output + for _ in download_products(self.products, download_dir=self.download_dir, + dl_urls=get_brdc_urls(self.start_epoch, self.end_epoch), + progress_callback=self.progress.emit, stop_requested=check_stop): + pass + except RuntimeError as e: + Logger.terminal(f"⚠️ {e}") + return + except Exception as e: + tb = traceback.format_exc() + Logger.terminal(f"⚠️ Error whilst downloading products:\n{tb}") + Logger.terminal(f"⚠️ {e}") + return + + self.finished.emit("📦 Downloaded all products successfully.") \ No newline at end of file diff --git a/scripts/GinanUI/app/utils/yaml.py b/scripts/GinanUI/app/utils/yaml.py new file mode 100644 index 000000000..52b94bf19 --- /dev/null +++ b/scripts/GinanUI/app/utils/yaml.py @@ -0,0 +1,144 @@ +from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedSeq, CommentedMap +from ruamel.yaml.scalarstring import PlainScalarString +from pathlib import Path +import tempfile +import os + +from scripts.GinanUI.app.utils.logger import Logger + +""" +YAML utilities for the Ginan-UI application. + +This module provides safe wrappers around ruamel.yaml to ensure that Python +objects (e.g., pathlib.Path, lists, strings) are always serialised and +deserialised in a consistent way. + +Key functions: +- load_yaml(file_path): Load YAML into memory, converting path-like strings + into pathlib.Path objects where appropriate. +- write_yaml(file_path): Write YAML safely. Falls back to normalising values + if ruamel.yaml raises a RepresenterError. +- update_yaml_values(): Update values in-place, preserving comments/formatting. +- normalise_yaml_value(): Normalise a single value (Path → PlainScalarString, + list → CommentedSeq, str → PlainScalarString). +- _normalise_inplace(): Internal helper to recursively normalise an entire + config tree in-place. Used as a safety net in write_yaml(). + +Conventions: +- Leading underscore (_) marks helpers intended for internal use only. +- Public functions (no underscore) are part of the module’s stable API and + should be used by other parts of the application. +""" + +# Configure YAML parser +yaml = YAML() +yaml.preserve_quotes = True +yaml.indent(mapping=4, sequence=4, offset=4) +yaml.width = 4096 # Avoid line wrapping +yaml.default_flow_style = False # Use block-style lists + + +def _convert_paths(obj): + """Recursively convert plain strings that look like filesystem paths into Path objects.""" + if isinstance(obj, dict): + return {k: _convert_paths(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [_convert_paths(v) for v in obj] + elif isinstance(obj, (PlainScalarString, str)): + s = str(obj) + # heuristic: treat as path if it looks like one + if "/" in s or s.startswith(".") or s.startswith("~") or os.path.isabs(s): + return Path(s).expanduser() + return s + else: + return obj + +def load_yaml(file_path: Path) -> CommentedMap: + """ + Load a YAML file and return its contents, preserving structure and comments. + Paths are left as plain strings for consistency. + """ + with file_path.open('r', encoding='utf-8') as f: + data = yaml.load(f) + if data is None: + raise ValueError(f"Failed to parse or empty YAML file: {file_path}") + return _normalise_inplace(data) # ✅ ensure values are normalised immediately + +def write_yaml(file_path: Path, config, debug: bool = False): + """ + Write a YAML config dictionary to file with clean formatting. + All Path objects are normalised to plain strings before dumping. + """ + # Proactively normalise everything + _normalise_inplace(config) + + with file_path.open('w', encoding='utf-8') as f: + yaml.dump(config, f) + + if debug: + with tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix=".yaml") as tmp_file: + yaml.dump(config, tmp_file) + tmp_file.seek(0) + Logger.console("[DEBUG] YAML OUTPUT (from temp file):\n" + tmp_file.read()) + +def update_yaml_values(file_path: Path, updates: list[tuple[str, str]]): + """ + Update several YAML keys in-place without destroying comments or formatting. + Values are passed through normalise_yaml_value() for safety. + """ + with file_path.open('r', encoding='utf-8') as f: + data = yaml.load(f) + if data is None: + raise ValueError(f"Failed to parse YAML from {file_path}") + + for key_path, new_value in updates: + keys = key_path.split(".") + node = data + for k in keys[:-1]: + if k not in node: + raise KeyError(f"Path segment '{k}' not found in {key_path}") + node = node[k] + + final_key = keys[-1] + if final_key not in node: + raise KeyError(f"Final key '{final_key}' not found in {key_path}") + + # Normalise + node[final_key] = normalise_yaml_value(new_value) + + with file_path.open("w", encoding='utf-8') as f: + yaml.dump(data, f) + + +def normalise_yaml_value(val): + """ + Ensure values are safe for ruamel.yaml dumping: + - Path → PlainScalarString + - str (no newlines) → PlainScalarString + - list → CommentedSeq with block style + """ + if isinstance(val, Path): + return PlainScalarString(str(val)) + elif isinstance(val, str) and "\n" not in val: + return PlainScalarString(val) + elif isinstance(val, list) and not isinstance(val, CommentedSeq): + seq = CommentedSeq(val) + seq.fa.set_block_style() + return seq + return val + +def _normalise_inplace(obj): + """ + Recursively normalise values in-place using normalise_yaml_value(). + Intended for internal use as a safety net in write_yaml() and load_yaml(). + """ + if isinstance(obj, dict): + for k, v in list(obj.items()): + obj[k] = normalise_yaml_value(v) + _normalise_inplace(obj[k]) + elif isinstance(obj, list): + for i, v in enumerate(list(obj)): + obj[i] = normalise_yaml_value(v) + _normalise_inplace(obj[i]) + return obj diff --git a/scripts/GinanUI/app/views/__init__.py b/scripts/GinanUI/app/views/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/GinanUI/app/views/main_window.ui b/scripts/GinanUI/app/views/main_window.ui new file mode 100644 index 000000000..1c54b0ce3 --- /dev/null +++ b/scripts/GinanUI/app/views/main_window.ui @@ -0,0 +1,1171 @@ + + + MainWindow + + + + 0 + 0 + 1200 + 800 + + + + GINAN GNSS Processing + + + + + + + + + + 60 + 60 + + + + :/img/ginan-logo.png + + + true + + + + + + + + 16 + true + + + + GINAN GNSS PROCESSING GUI + + + + + + + + 0 + 0 + + + + + Segoe UI + 11 + false + false + + + + Qt::LayoutDirection::LeftToRight + + + + color: gray; + padding: 0px 8px; + font: 11pt "Segoe UI"; + text-align: right; + + + Ginan-UI v4.0.0 + + + Qt::AlignmentFlag::AlignRight|Qt::AlignmentFlag::AlignTop|Qt::AlignmentFlag::AlignTrailing + + + + + + + + + + + QLayout::SizeConstraint::SetDefaultConstraint + + + + + + + + 0 + 0 + + + + + 0 + 40 + + + + QPushButton { + background-color: rgb(24, 24, 24); + selection-background-color: rgb(12, 17, 109); + color: rgb(255, 255, 255); + border-color: rgb(189, 189, 189); +} +QPushButton:hover { + background-color: rgb(40, 40, 40); +} +QPushButton:pressed { + background-color: rgb(12, 12, 12); +} +QPushButton:disabled { + background-color: rgb(89, 89, 89); +} + + + Observations + + + + + + + + 0 + 0 + + + + + 0 + 40 + + + + + 367 + 16777215 + + + + QPushButton { + background-color: rgb(24, 24, 24); + selection-background-color: rgb(12, 17, 109); + color: rgb(255, 255, 255); + border-color: rgb(189, 189, 189); +} +QPushButton:hover { + background-color: rgb(40, 40, 40); +} +QPushButton:pressed { + background-color: rgb(12, 12, 12); +} +QPushButton:disabled { + background-color: rgb(89, 89, 89); +} + + + Output + + + + + + + + + + 0 + 0 + + + + background-color: rgb(24, 24, 24); +color: rgb(255, 255, 255); + + + QFrame::Shape::NoFrame + + + + QLayout::SizeConstraint::SetDefaultConstraint + + + 5 + + + 5 + + + 5 + + + 5 + + + 8 + + + 10 + + + + + + 0 + 0 + + + + false + + + Receiver Type + + + + + + + + 0 + 0 + + + + false + + + background:transparent;border:none; + + + Antenna Offset + + + true + + + + + + + Receiver Type + + + + + + + Data Interval + + + + + + + Antenna Offset + + + + + + + + 0 + 0 + + + + false + + + Data interval + + + + + + + PPP Series + + + + + + + + 0 + 0 + + + + false + + + PPP Series + + + + + + + + 0 + 0 + + + + false + + + Open Yaml config in editor + + + + + + + + 0 + 0 + + + + QPushButton { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QPushButton:hover { + background-color: #214861; +} +QPushButton:pressed { + background-color: #1a3649; +} +QPushButton:disabled { + background-color: rgb(120, 120, 120); +} + + + 0.0, 0.0, 0.0 + + + + + + + + 0 + 0 + + + + false + + + Time Window + + + + + + + Time Window + + + + + + + + 0 + 0 + + + + false + + + + + + + Antenna Type + + + + + + + + 0 + 0 + + + + false + + + PPP Provider + + + + + + + + 0 + 0 + + + + QComboBox { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QComboBox:hover { + background-color: #214861; +} +QComboBox:disabled { + background-color: rgb(120, 120, 120); +} + + + + Select one + + + + + + + + PPP Provider + + + + + + + + 0 + 0 + + + + QPushButton { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QPushButton:hover { + background-color: #214861; +} +QPushButton:pressed { + background-color: #1a3649; +} +QPushButton:disabled { + background-color: rgb(120, 120, 120); +} + + + Interval (Seconds) + + + + + + + false + + + Static + + + + + + + + 0 + 0 + + + + QComboBox { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QComboBox:hover { + background-color: #214861; +} +QComboBox:disabled { + background-color: rgb(120, 120, 120); +} + + + + Import text + + + + + + + + + 0 + 0 + + + + QComboBox { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QComboBox:hover { + background-color: #214861; +} +QComboBox:disabled { + background-color: rgb(120, 120, 120); +} + + + + Import text + + + + + + + + + 0 + 0 + + + + QPushButton { + background-color: #2c5d7c; + color: white; + padding: 2px 8px; + font: 13pt "Segoe UI"; + text-align: center; +} +QPushButton:hover { + background-color: #214861; +} +QPushButton:pressed { + background-color: #1a3649; +} +QPushButton:disabled { + background-color: rgb(120, 120, 120); +} + + + Show Config + + + + + + + Constellations + + + + + + + + 0 + 0 + + + + QComboBox { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QComboBox:hover { + background-color: #214861; +} +QComboBox:disabled { + background-color: rgb(120, 120, 120); +} + + + + Select one or more + + + + + + + + + 0 + 0 + + + + QPushButton { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QPushButton:hover { + background-color: #214861; +} +QPushButton:pressed { + background-color: #1a3649; +} +QPushButton:disabled { + background-color: rgb(120, 120, 120); +} + + + Start / End + + + + + + + false + + + Constellations + + + + + + + + 0 + 0 + + + + Mode + + + + + + + + 0 + 0 + + + + + 16777215 + 16777215 + + + + PPP Project + + + + + + + + 0 + 0 + + + + QComboBox { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QComboBox:hover { + background-color: #214861; +} +QComboBox:disabled { + background-color: rgb(120, 120, 120); +} + + + + Select one + + + + + + + + + 0 + 0 + + + + QComboBox { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QComboBox:hover { + background-color: #214861; +} +QComboBox:disabled { + background-color: rgb(120, 120, 120); +} + + + + Select one + + + + + + + + + 0 + 0 + + + + QComboBox { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 13pt "Segoe UI"; + text-align: left; +} +QComboBox:hover { + background-color: #214861; +} +QComboBox:disabled { + background-color: rgb(120, 120, 120); +} + + + + Select one + + + + + + + + + + + + 0 + 0 + + + + + 0 + 40 + + + + QPushButton { + color: black; + font-weight: bold; + background-color: rgb(21, 134, 19); + border-color: rgb(189, 189, 189); +} +QPushButton:hover { + background-color: rgb(17, 107, 15); +} +QPushButton:pressed { + background-color: rgb(13, 80, 12); +} +QPushButton:disabled { + background-color: rgb(89, 89, 89); +} + + + Process + + + + + + + Qt::LayoutDirection::LeftToRight + + + +QPushButton { background-color: #d32f2f; color: white; font-weight: bold; } +QPushButton:hover { background-color: #b71c1c; } +QPushButton:pressed { background-color: #9a0007; } +QPushButton:disabled { background-color: rgb(120,120,120); } + + + + Stop + + + + + + + + + + + + + Qt::Orientation::Horizontal + + + + 40 + 20 + + + + + + + + + 150 + 30 + + + + QPushButton { + background-color: #2c5d7c; + color: white; + padding: 2px 8px; + font: 13pt "Segoe UI"; + text-align: center; +} +QPushButton:hover { + background-color: #214861; +} +QPushButton:pressed { + background-color: #1a3649; +} +QPushButton:disabled { + background-color: rgb(120, 120, 120); +} + + + CDDIS Credentials + + + + + + + + + + 0 + 0 + + + + + 870 + 0 + + + + + 12 + false + false + false + PreferDefault + true + Medium + + + + false + + + background-color:#2c5d7c;color:white; + + + QTabWidget::TabPosition::North + + + QTabWidget::TabShape::Rounded + + + 0 + + + + 16 + 16 + + + + Qt::TextElideMode::ElideNone + + + false + + + false + + + false + + + + background-color:#2c5d7c;color:white; + + + Workflow + + + + 0 + + + 0 + + + 0 + + + 0 + + + 0 + + + + + background-color:#2c5d7c;color:white; + + + true + + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> +<html><head><meta name="qrichtext" content="1" /><meta charset="utf-8" /><style type="text/css"> +p, li { white-space: pre-wrap; } +hr { height: 1px; border-width: 0; } +li.unchecked::marker { content: "\2610"; } +li.checked::marker { content: "\2612"; } +</style></head><body style=" font-family:'Ubuntu'; font-size:10pt; font-weight:400; font-style:normal;"> +<p style=" margin-top:12px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-family:'.AppleSystemUIFont'; font-size:13pt;">Workflow Terminal</span></p></body></html> + + + + + + + + background-color: rgb(24, 24, 24); +color:white; + + + Console + + + + 0 + + + 0 + + + 0 + + + 0 + + + 0 + + + + + + + + background-color: rgb(24, 24, 24); +color:white; + + + true + + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> +<html><head><meta name="qrichtext" content="1" /><meta charset="utf-8" /><style type="text/css"> +p, li { white-space: pre-wrap; } +hr { height: 1px; border-width: 0; } +li.unchecked::marker { content: "\2610"; } +li.checked::marker { content: "\2612"; } +</style></head><body style=" font-family:'Ubuntu'; font-size:10pt; font-weight:400; font-style:normal;"> +<p style=" margin-top:12px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-family:'.AppleSystemUIFont'; font-size:13pt;">PEA Console Log</span></p></body></html> + + + + + + + + + + + + 14 + true + + + + Visualisation + + + + + + + + 0 + 0 + + + + + about:blank + + + + + + + + QPushButton { + background-color: #2c5d7c; + color: white; + padding: 2px 8px; + font: 11pt "Segoe UI"; + text-align: center; +} +QPushButton:hover { + background-color: #214861; +} +QPushButton:pressed { + background-color: #1a3649; +} +QPushButton:disabled { + background-color: rgb(120, 120, 120); +} + + + Open in Browser + + + + + + + QComboBox { + background-color: #2c5d7c; + color: white; + padding: 0px 8px; + font: 11pt "Segoe UI"; + text-align: left; +} +QComboBox:hover { + background-color: #214861; +} +QComboBox:on { + background-color: #214861; + color: white; +} +QComboBox:disabled { + background-color: rgb(120, 120, 120); +} +QComboBox QAbstractItemView { + background-color: #2c5d7c; + color: white; + selection-background-color: #214861; + selection-color: white; +} + + + + + + + + + + + + + + + QWebEngineView + QWidget +
    QtWebEngineWidgets/QWebEngineView
    +     +     + + cddisCredentialsButton + observationsButton + outputButton + terminalTextEdit + consoleTextEdit + Mode + Constellations_2 + timeWindowButton + dataIntervalButton + Receiver_type + Antenna_type + antennaOffsetButton + PPP_provider + PPP_series + PPP_project + showConfigButton + processButton + stopAllButton + tabWidget + + + + + +     diff --git a/scripts/GinanUI/docs/USER_GUIDE.md b/scripts/GinanUI/docs/USER_GUIDE.md new file mode 100644 index 000000000..9e5469e89 --- /dev/null +++ b/scripts/GinanUI/docs/USER_GUIDE.md @@ -0,0 +1,566 @@ +# Ginan-UI +## User Manual +### This guide is written to aid those using the Ginan-UI extension software. +### Version: Release 1.0 +### Last Updated: 12th December 2025 + +## 1. Introduction + +Ginan-UI is a graphical user interface for the Ginan software developed by Geoscience Australia. It aims to lower the barrier of entry for users trying to use the Ginan software by simplifying the users interaction with the software away from a command-line interface. On top of this, it automatically fills the .YAML configuration based on a user-provided .RNX file, automatically downloads all static and dynamic products required for execution, and also executes Ginan and visualises its plot output visualisation in an HTML format. + +This tool is designed for both new users to the Ginan software who are not comfortable using Ginan in its command-line interface form, and experienced Ginan users who want to streamline their use process. + +## 2. System Requirements & Installation + +### 2.1 Minimum System Requirements + +- OS: Mac, Linux, Windows +- CPU: 1 core, 2 threads +- Storage: 4GB +- Memory: 2GB +- Internet connection + +### 2.2 Installation Guide + +**It is required** to have registered credentials to access the CDDIS Archives. This is necessary to automatically download the auxiliary products and data. Once registered, enter the credentials into the CDDIS Credentials pop-up that opens on first-time launch. + +If this does not open for you and the program instead opens to the main screen, check the top-right for a button that reads: "CDDIS Credentials". + +#### Installation From an Executable + +##### Windows + +1. Download the latest Windows release from [GitHub Releases](https://github.com/GeoscienceAustralia/ginan/releases) + +2. Extract the ZIP archive to your desired location + +3. Run `Ginan-UI.exe` + +**Windows Security Warning:** On first-time launch, Windows Defender SmartScreen may display a warning because the executable is not code-signed. This is expected behaviour for unsigned open-source software. To proceed: +1. Click **"More info"** +2. Click **"Run anyway"** + +Ginan-UI is safe to run - the complete source code is available in this repository for verification. + +##### MacOS + +1. Download the latest macOS release from [GitHub Releases](https://github.com/GeoscienceAustralia/ginan/releases) + +2. Extract the archive to your desired location + +3. Remove the file from macOS quarantine: + +``` +bash +xattr -dr com.apple.quarantine /path/to/ginan-ui +``` + +4. Run the startup script, which configures environment variables and launches Ginan-UI: + +``` +bash +./run.sh +``` + +##### Linux + +1. Download the latest Linux release from [GitHub Releases](https://github.com/GeoscienceAustralia/ginan/releases) + +2. Extract the archive: + +``` +bash +tar -xf ginan-ui-linux-x64.tar.gz +cd ginan-ui +``` + +3. Make the executable runnable (if needed): + +``` +bash +chmod +x ginan-ui +``` + +4. Run Ginan-UI: + +``` +bash +./ginan-ui +``` + +**Note:** On some Linux distributions, you may need to install additional Qt dependencies. If you encounter missing library errors, refer to Section 7.1 (Troubleshooting). + +#### Installation from Source +Follow the below commands, tested with python 3.9+ + +``` +Install and navigate to the root of the Ginan repository: +cd /ginan +pip install -r scripts/GinanUI/requirements.txt +python -m scripts.GinanUI.main +``` + +## 3. Getting Started (Quick Start) + +When you open Ginan-UI for the first time, you will be taken to the main dashboard interface. The workflow is straightforward and only requires a few inputs from the user before Ginan can begin processing. + +

    Dashboard of Ginan-UI

    + +![Dashboard of Ginan-UI](./images/ginan_ui_dashboard.jpg) + + +To use Ginan-UI, you will require an account with NASA's CDDIS EarthData archives. Once you have created an account here, you can log in by clicking the “CDDIS Credentials” button in the top-right of Ginan-UI: + +

    CDDIS Credentials button in the top-right

    + +!["CDDIS Credentials" button highlighted](./images/cddis_credentials_button.jpg) + +Then, enter your CDDIS credentials and click “Save”. + +

    Type in your CDDIS login credentials here

    + +![CDDIS Credentials screen displaying username and password fields](./images/cddis_credentials_screen.jpg) + +Next, click the “Observations” button in the top-left and select the RINEX observation data file you want to process. Afterward, click the adjacent “Output” button and choose an output location for where Ginan will store its results after processing. + +

    Select your RINEX observation file and your output location

    + +!["Observations" and "Output" buttons highlighted](./images/observations_output_buttons.jpg) + +Once your RINEX observation file is set, most of the UI fields should autofill based on extracted data from the RINEX file, however the user still needs to set the “Mode” parameter. This defines how much noise should be expected in the data (i.e. “Static” = stationary GNSS receiver, “Kinematic” = a moving car, “Dynamic” = a moving plane). Set this field now. + +More experienced users may recognise this parameter as the `estimation_parameters.receivers.global.pos.process_noise` config value. By default, “Static” = 0, “Kinematic” = 30, and “Dynamic” = 100. + +

    Select a "Mode" to set the `process_noise`

    + +!["Mode" dropdown showing the options: "Static", "Kinematic", and "Dynamic"](./images/mode_dropdown.jpg) + +Once everything is configured and all fields have been autofilled by Ginan-UI, simply click “Process” to start Ginanʼs PEA processing. Ginan-UI will begin downloading the required products from the CDDIS servers for Ginan to process, and then will execute Ginan automatically. Ginanʼs processing progress will be displayed in the accompanying "Console" tab next to the default "Workflow" tab. + +

    Click "Process" when ready!

    + +!["Process" button highlighted](./images/process_button.jpg) + +

    Ginan-UI will automatically start downloading the necessary products

    + +![Automatically began downloading dynamic products for PEA processing](./images/product_downloading.jpg) + +

    Ginan's PEA tool will then begin processing using the selected configuration

    + +![PEA processing within the "Console" tab](./images/pea_processing.jpg) + +When Ginan finishes processing, you can view the generated position plot within Ginan-UI, or alternatively open the generated HTML output to review the results by clicking “Open in Browser”. + +

    Once PEA finishes processing, Ginan-UI will plot the results

    + +![Plot visualisation within Ginan-UI](./images/plot_visualisation.jpg) + +

    Visualisation plot enlarged in the web-browser

    + +![Plot visualisation opened in web-browser](./images/plot_visualisation_web.jpg) + +And that is it! Check out Section 6 for more in-depth tooling. + +## 4. User Interface Reference + +### 4.1 Input Configuration (Left Panel) + +The left-hand panel contains all the configuration options required to set up Ginan to commence processing. + +#### "Observations" Button + +- Opens a file picker to select your RINEX v2/v3/v4 observation file (optionally can be compressed). + +- Ginan-UI will automatically extract metadata from your provided RINEX file, including the time window, available constellations, and receiver and antenna type information. This metadata is then used to autopopulate the several fields below. + +#### "Output" Button + +- Opens a file picker to select where PEA will save its processing results (`.pos`, `.log`, HTML plots). + +- Remains disabled until a valid "Observations" file has been selected. + +#### Mode + +- **Critical parameter** that must be set by the user. + +- Defines expected receiver motion and sets the process noise parameter for position estimation: + - **Static** (0): Stationary GNSS receiver (e.g. reference station) + - **Kinematic** (30): Moving ground vehicle (e.g. a car) + - **Dynamic** (100): Fast-moving vehicle (e.g. an airplane) + +- Corresponds to the `estimation_parameters.receivers.global.pos.process_noise` YAML field + +#### Constellations + +- Drop-down showing GNSS systems detected from your RINEX file. Displays which constellations are available: GPS, GAL (Galileo), GLO (GLONASS), BDS (BeiDou), QZS (QZSS) + +#### Time Window + +- Displays the detected start and end epochs. + +- Useful is you only want to process a subset of your observation period + +#### Data Interval + +- Set the interval to downsample your observation data (i.e. process every 120 seconds, instead of 30 seconds). + +#### Receiver Type / Antenna Type + +- Used internally for antenna phase centre corrections + +#### Antenna Offset + +- View / edit ENU (East-North-Up) offset values. This allows manual adjustment if your antenna has a known offset position from its reference point. + +#### PPP Provider / Project / Series + +- Three drop-downs that filter available products based on the provided time window + +- **Provider:** Analysis centre or organisation (e.g. IGS, COD, GFZ, JPL) + +- **Series:** Solution type (e.g. Ultra-rapid, Rapid, Final) + +- **Project:** Product line within the provider (e.g. IGS_MGEX, CODE_MGEX) + +- Changing the provider will filter the available series and projects. + +- These fields are populated after a valid observation file has been loaded. + +#### "Show Config" Button + +- A button that opens the generated `ppp_generated.yaml` file in your system's default text editor. + +- Allows advanced users to manually edit PEA configuration parameters + +- See Section 6.1 for more details on manual config editing. + +### 4.2 Monitoring & Output (Right Panel) + +The right-hand panel contains all the monitoring tools for Ginan-UI's functionality and Ginan's processing, as well as managing your CDDIS credentials. + +#### "CDDIS Credentials" Button + +- Opens a dialog to enter your NASA EarthData username and password. + +- This is required for downloading product from CDDIS archives. + +- Credentials are validated against CDDIS servers before being saved. On success credentials are stored in a `.netrc` or `_netrc` file in your home directory (depending on the platform) + +#### "Workflow" Tab + +- Logs Ginan-UI workflow and automation messages as well as any warnings or errors with usage. Product download progress is displayed within progress bars. + +- Text is read-only but can be selected and copied for reporting any issues. + +#### "Console" Tab + +- Streams the complete `stdout` / `stderr` from the Ginan PEA executable as it processes, as well as the relevant log messages. + +- Text is read-only but can be selected and copied for reporting any issues. + +#### "Visualisation" Section + +- The visualisation panel displays an interactive HTML plot that is generated using the `plot_pos.py` script after PEA completes its processing. It allows the user to view, pan, zoom, hover over tooltips and toggle legends. + +- Below the visualisation panel, the user can choose to open the plot in their system's default web-browser, or switch between the other generated plots. + +### 4.3 Process Control + +#### "Process" Button + +- The green button in the bottom-left. Initiates Ginan's processing. + +- Will remain disabled until all required inputs are configured: Valid Observation file, Output directory, Mode parameter, and PPP products available. + +- Will disable when processing commences. + +#### "Stop" Button + +- The red button in the bottom-left. Requests a graceful termination of product downloads and PEA's execution. + +- After the stop completes, the "Process" button will re-enable again. + +## 5. Understanding the Ginan-UI Workflow + +### 5.1 What Happens When You Click "Process" + +Once all required parameters within the UI are filled and the "Process" button is clicked, Ginan-UI will begin downloading the required dynamic products from the CDDIS EarthData servers. These primarily include the `.bia`, `.clk`, `.nav (BRDC)` and `.sp3` files. + +Once these have successfully downloaded, Ginan's PEA tool will be automatically executed with the generated `.yaml` configuration file. This processing can be observed within the "Console" log tab which should look similar to PEA's command-line interface output. + +Once it finishes processing, the `plot_pos.py` script will be called automatically to plot the resulting `.pos` and `_smoothed.pos` files generated during processing, and the plots will appear within the UI under the "Visualisation" heading. + +### 5.2 Product Downloading (Static vs. Dynamic) + +Ginan-UI automatically downloads all required products for GNSS processing from NASA's CDDIS (Crustal Dynamics Data Information System) archives. These products are split into two categories: **static** and **dynamic**. + +#### Static Products (Metadata) + +Static products are reference files that rarely change and are downloaded once when Ginan-UI is launchd for the first time. These include: + +- **ATX** (Antenna exchange format) - Antenna phase centre corrections + +- **ALOAD** (Atmospheric loading) - Atmospheric pressure loading models + +- **IGRF** (International Geomagnetic Reference Field) - Geomagnetic field models + +- **OLOAD** (Ocean loading) - Ocean tide loading models + +- **OPOLE** (Ocean pole tide) - Ocean pole tide models + +- **PLANET** (Planetary ephemeris) - Solar system body positions + +- **SAT-META** (Satellite metadata) - Satellite characteristics and properties + +- **YAW** (Yaw attitude) - Satellite attitude models + +- **GPT2** (Global Pressure and Temperature 2) - Tropospheric models + +These fies are stored in `scripts/GinanUI/app/resources/inputData/products/` and are automatically archived when they become outdated (typically after one week). Fresh copies are then downloaded on the next program launch. + +#### Dynamic Products (Observation-Specific) + +Dynamic products are files specific to the provided RINEX observations and change based on the observation's time window and chosen PPP provider. These are downloaded each time you click "Process" and include: + +- **CLK** (Clock products) - Precise satellite and station clock corrections + +- **SP3** (Precise ephemeris) - Precise satellite orbit positions + +- **BIA** (Bias products) - Code and phase biases for multi-GNSS processing + +- **NAV** (Navigation/broadcast) - Broadcast navigation messages (BRDC files) + +Ginan-UI will automatically determine which dynamic products you need based on: + +1. The time window provided (either manually set or extracted from your RINEX observation file) + +2. The PPP provider / series / project selected in the UI + +3. The constellations present in your data (GPS, GLONASS, Galileo, BeiDou, QZSS) + +#### Download Process + +When you click "Process", Ginan-UI will: + +1. Check your CDDIS credentials are valid + +2. Query for the available products for the provided time window from the CDDIS servers + +3. Download any missing dynamic products with progress indicators shown in the "Workflow" log tab + +4. Verify all required products are present before launching PEA + +If a product cannot be found (which is common for either very old or very new RINEX observation files), Ginan-UI will inform you that the selected provider does not have the products available for your time window yet. Different PPP providers publish their products with varying latencies. Ultra-rapid (ULT) are available within hours, Rapid (RAP) are available within about one day, and Final (FIN) may take one or two weeks. + +All downloaded products are stored in `scripts/GinanUI/app/resources/inputData/products/` alongside the archived products from previous processing iterations in timestamped archive folders. + +### 5.3 Product Archival + +Ginan-UI will automatically archive both products and output files to prevent conflicts between processing runs and to keep your directories clean and organised. + +#### Product Archival + +Product files are automatically archived in the follow situations: + +- **On Application Startup:** Static products older than seven days are moved to timestamped archive folders within `scripts/GinanUI/app/resources/inputData/products/archived/`. Fresh versions are then downloaded to replace them. + +- **When Loading a New RINEX File:** If you select a different RINEX observation file, all dynamic products from the previous processing iteration are archived with the tag `rinex_change_[timestamp]`. This prevents incompatible products from different time windows being mixed up. + +- **When Changing PPP Selections:** If you change your PPP provider, series, or project selection the relevant dynamic products will be archived with the tag `PPP_selection_change_[timestamp]`. However, reusable files like broadcast navigation messages will be preserved. + +#### Output Archival + +When you start a new processing run, existing output files in your selected output directory are automatically moved to `output/archive/[timestamp]/` before PEA processing commences. This includes: + +- `.pos` files (position solutions) +- `.log` files (PEA execution logs) +- `.txt` and `.json` files (configuration artifacts) +- `.html` visualisation files (if a visualisation directory was used) + +This makes sure that every processing iteration produces a clean output and does not overwrite results from previous iterations. + +### 5.4 Where Files are Stored + +Ginan-UI has several important directories for its operation. All paths are relative to the Ginan installation directory unless explicitly specified by the user (observation and output directories). + +#### Product Storage + +- **Location:** `scripts/GinanUI/app/resources/inputData/products/` + +- **Contents:** All static and dynamic products downloaded from NASA's CDDIS Earthdata archives. + +- **Subdirectories:** + - `tables/` - Static metadata files (ALOAD, OLOAD, GPT2) + - `archived/` - Timestamped folders containing archived products + +#### Configuration Files + +- **Template:** `scripts/GinanUI/app/resources/Yaml/default_config.yaml` + +- **Generated Config:** `scripts/GinanUI/app/resources/ppp_generated.yaml` + +- **CDDIS Credentials:** Platform-specific (See Section 4.2) + - Windows: `%USERPROFILE%\.netrc` or `%USERPROFILE%\_netrc` + - MacOS / Linux: `~/.netrc` + +#### Output Files + +- **Location:** User-selected via the "Output" button +- **Contents:** PEA-generated `.pos` files, `.log files`, and processing artifacts +- **Visualisations:** HTML plot files generated by `plot_pos.py` +- **Subdirectories:** `archive/` - Timestamped folders containing previous run outputs + +#### Observation Data + +- **Location:** Selected by the user via the "Observations" button +- Ginan-UI will read but does not modify your RINEX files + +### 5.5 How the YAML Config is Generated + +The `.yaml` configuration file that is generated for Ginan's PEA processing originates from the template config file located within `scripts/GinanUI/app/resources/Yaml/default_config.yaml`. This template file is copied if no config file exists within `scripts/GinanUI/app/resources/ppp_generated.yaml`, or if one does exist already, the `ppp_generated.yaml` file is instead overwritten. Keep in mind however that this may maintain some artifacts from previous config generations, which can be useful in some use cases. + +If you would like to instead generate a fresh `ppp_generated.yaml` file, simply delete `ppp_generated.yaml` and on the next processing run, a new config file will be generated from the `default_config.yaml` template file. + +## 6. Advanced Usage + +### 6.1 Manual YAML Editing + +For experienced users of Ginan who need fine-grained control over Ginan's processing, the `.yaml` configuration file can be manually edited via clicking the "Show Config" button.This will open `ppp_generated.yaml` in your system's default text editor. + +#### Persistence of Manual Changes + +Manual user edits are preserved across most operations as Ginan-UI will only update specific fields when necessary: + +- RINEX metadata (time windows, constellations, receiver / antenna information) + +- Product file paths for downloaded PPP products + +- Output directory paths + +All other parameters like processing strategies, filter settings, quality control thresholds, satellite-specific options will all remain untouched. + +**Note:** YAML artifacts may persist between sessions. For example, marker names within `receiver_options` may remain if not explicitly overwritten, though this rarely causes issues. + +#### Resetting to Default + +If you experience any configuration errors and want to start fresh: + +1. Delete `scripts/GinanUI/app/resources/ppp_generated.yaml` + +2. On the next processing run, a clean configuration file will be generated from the template at `scripts/GinanUI/app/resources/Yaml/default_config.yaml` + +**For executable releases of Ginan-UI**, the config is located at `_internal/app/resources/ppp_generated.yaml` + +**Warning:** Invalid YAML syntax (like incorrect indentation, mismatched quotes, and malformed lists) will cause PEA to fail. Please verify your formatting if you encounter configuration-related errors in the logs. + +## 7. Troubleshooting + +### 7.1 Common Issues + +| Issue | Cause | Fix | +|-------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Ginan-UI launched to a black screen, then crashed | Ginan-UI's usage of the Qt framework can rarely cause race conditions causing a segmentation fault. | Try launching Ginan-UI again, it almost always fixes itself after first-time launch. | +| Missing library errors on Linux (e.g., `libxcb`, `libGL`, Qt libraries) | Required Qt dependencies not installed on your distribution. | Install Qt dependencies for your distribution:
    **Ubuntu / Debian:** `sudo apt install libxcb-xinerama0 libxcb-cursor0 libgl1`
    **Fedora:** `sudo dnf install qt6-qtbase qt6-qtwebengine`
    **Arch:** `sudo pacman -S qt6-base qt6-webengine` | +| Process button greyed out | One of the following:
    - Observations not selected (`.rnx`)
    - Output directory not selected
    - Analysis centers not processed yet | Ensure you have selected an observation file and output directory. After the observation file has been selected, the analysis centers will automatically begin processing. Once the "PPP Provider" field has been populated, the button will unlock. | +| Missing products or downloading duplicate products when process clicked | Output directory same as product directory or log message suggesting a file is downloaded when its just being existence checked. | Ensure output directory is separate from the product directory. If there's no product directory selection made, the product directory path is `ginan/scripts/GinanUI/app/resources/inputData/products`. | +| Program crash when clicking "Process" | "`Core dump whilst thread ''`" occurs when the user uses the "Stop" button before the first download has started and subsequently clicked the process button again before the thread has a chance to exit. | The thread cannot exit whilst raising a request for status. Wait a few seconds for the "stopped thread" message in the Console before clicking Process again. | +| Connection reset errors | CDDIS server timeouts and / or network problems. | Wait 30 seconds and then try again. If the issue persists, check your network connection. Note: CDDIS servers experienced reliability issues during the 2025 US government shutdown. | +| CDDIS authentication failed | Invalid or expired Earthdata credentials, or credentials not properly saved to `.netrc` / `_netrc` file. | Re-enter your credentials via the "CDDIS Credentials" button. Verify your account is active at [Earthdata Login](https://urs.earthdata.nasa.gov). Check that `.netrc` or `_netrc` exists in your home directory with correct permissions (0600 on Linux/macOS). | +| PEA configuration error / YAML syntax error | Manual edits to the YAML config contain syntax errors (incorrect indentation, mismatched quotes, malformed lists). | Verify your YAML formatting in the config editor. If errors persist, delete `ppp_generated.yaml` to reset to default template (see Section 6.1). | +| Plots not appearing in Visualisation panel | PEA processing failed before generating `.pos` files, or `plot_pos.py` script encountered errors. | Check the Console for PEA errors. Verify that `.pos` files exist in your output directory. If files exist but plots don't render, check for Qt WebEngine issues in the Console. | +| Disk space errors during processing | Insufficient disk space for downloading products or writing PEA outputs. | Free up disk space. Products can consume several GB depending on time window and number of constellations. Check available space in both the products directory and your selected output directory. | + +### 7.2 Log Message Interpretation + +The "Workflow" / "Console" log in the right panel displays real-time output from Ginan-UI's processing. These logs redirect what would normally appear in the terminal. + +#### What You Will See + +The logs stream messages from them: + +- **Product downloading:** URLs being fetched, file names, and download progress. + +- **Ginan PEA execution:** The complete `stdout / stderr` from the PEA executable as it processes your data. + +- **Toast notifications:** User feedback messages about the status of operations. + +#### When Things Go Wrong + +Common issues you may see in the logs: + +- **Network / Connection errors:** CDDIS server timeouts or network problems. Wait 30 seconds and retry. + +- **Missing products:** The selected PPP Provider may not have products available for your time window, or they haven't been published yet. + +- **YAML configuration errors:** Syntax errors may cause PEA to fail on startup if you have manually edited the `.yaml` config file. + +- **Disk space issues:** Ginan-UI has encountered problems when disk space is very limited. Please ensure you have at least a 2 - 3 Gb of disk space free. + +#### Tips + +- The logs are read-only, but you can select and copy text for reporting issues. + +- It auto-scrolls to the newest output. + +- Messages will persist until you start a new processing run. + +- The raw PEA output can be verbose (very verbose), this is normal for GNSS processing tools. + +If you encounter persistent errors, please copy the relevant log outputs when reporting issues (See Section 7.3) + +### 7.3 Where To Get Help + +If you encounter issues not covered in this troubleshooting guide, or need assistance with Ginan-UI: + +#### Primary Contact + +Sam Greenwood (Ginan-UI Engineer) - samuel.greenwood@ga.gov.au + +#### Additional Resources + +GitHub Issues: Report bugs or request features at the [Ginan-UI](https://github.com/GeoscienceAustralia/ginan) repository issue tracker + +Ginan Documentation: For questions about Ginan itself (not the UI), consult the main [Ginan documentation](https://geoscienceaustralia.github.io/ginan/) + +CDDIS Support: For issues with NASA Earthdata credentials or archive access, visit the [CDDIS help page](https://www.earthdata.nasa.gov/centers/cddis-daac/contact) + +When reporting issues, please include: +- Your operating system and version +- The steps you took before encountering the problem +- Any error messages from the Workflow / Console logs +- Screenshots if relevant + +**Note:** Ginan-UI was developed as part of the ANU TechLauncher program in collaboration with Geoscience Australia. For general enquiries about Geoscience Australia's GNSS analysis capabilities, visit [www.ga.gov.au](www.ga.gov.au) + +## 8. FAQ + +Here are some answers to the frequently asked questions: + +**Q:** *"Where are products downloaded to?"* + +**A:** Products are downloaded to: `ginan/scripts/GinanUI/app/resources/inputData/products`. Current static products are stored here. Dynamic products are downloaded to the same folder but are moved to an archive folder on app-startup and when the `.rnx` file changes. + +**Q:** *"Where is the `.yaml` config file stored?"* + +**A:** : The `.yaml` config file used by PEA is in `ginan/scripts/GinanUI/app/resources/ppp_generated.yaml` which can be edited with the "Show Config" button. The template file in `ginan/scripts/GinanUI/app/resources/Yaml/default_config.yaml` is copied and used when no `ppp_generated.yaml` can be found. + +**Q:** *"Why is pea giving me a configuration error?"* + +**A:** This could be due to a product file being deleted erroneously, which would resolve on the next click of the "Process" button, or due to manual changes to the `.yaml` config file. The app **does not overwrite** the `ppp_generated.yaml` file when the `.rnx` file is changed or when the app is restarted. If you wish to reset to the default config, delete the file in `ginan/scripts/GinanUI/app/resources/ppp_generated.yaml` and then run the app again. + +**Q:** *"Where can I learn more about Ginan itself?"* + +**A:** Visit Ginan's GitHub [here](https://github.com/GeoscienceAustralia/ginan) to learn more about the tool! + +## 9. Acknowledgements +This project was designed during the Australian National University's TechLauncher program in 2025. Ginan-UI was created for Geoscience Australia by: + +- Sam Greenwood +- Ryan Foote +- Harry Baard +- Kenita Tan +- Yuliang Yang +- Fan Jin +- Songxuan He + +Special thanks to Simon McClusky at Geoscience Australia for their continuous support and guidance throughout the project's development. \ No newline at end of file diff --git a/scripts/GinanUI/docs/images/cddis_credentials_button.jpg b/scripts/GinanUI/docs/images/cddis_credentials_button.jpg new file mode 100644 index 000000000..263fb1370 Binary files /dev/null and b/scripts/GinanUI/docs/images/cddis_credentials_button.jpg differ diff --git a/scripts/GinanUI/docs/images/cddis_credentials_screen.jpg b/scripts/GinanUI/docs/images/cddis_credentials_screen.jpg new file mode 100644 index 000000000..18e2893a0 Binary files /dev/null and b/scripts/GinanUI/docs/images/cddis_credentials_screen.jpg differ diff --git a/scripts/GinanUI/docs/images/ginan_ui_dashboard.jpg b/scripts/GinanUI/docs/images/ginan_ui_dashboard.jpg new file mode 100644 index 000000000..d1b746801 Binary files /dev/null and b/scripts/GinanUI/docs/images/ginan_ui_dashboard.jpg differ diff --git a/scripts/GinanUI/docs/images/mode_dropdown.jpg b/scripts/GinanUI/docs/images/mode_dropdown.jpg new file mode 100644 index 000000000..32000fc24 Binary files /dev/null and b/scripts/GinanUI/docs/images/mode_dropdown.jpg differ diff --git a/scripts/GinanUI/docs/images/observations_output_buttons.jpg b/scripts/GinanUI/docs/images/observations_output_buttons.jpg new file mode 100644 index 000000000..5705fa989 Binary files /dev/null and b/scripts/GinanUI/docs/images/observations_output_buttons.jpg differ diff --git a/scripts/GinanUI/docs/images/pea_processing.jpg b/scripts/GinanUI/docs/images/pea_processing.jpg new file mode 100644 index 000000000..151665cd1 Binary files /dev/null and b/scripts/GinanUI/docs/images/pea_processing.jpg differ diff --git a/scripts/GinanUI/docs/images/plot_visualisation.jpg b/scripts/GinanUI/docs/images/plot_visualisation.jpg new file mode 100644 index 000000000..83c230199 Binary files /dev/null and b/scripts/GinanUI/docs/images/plot_visualisation.jpg differ diff --git a/scripts/GinanUI/docs/images/plot_visualisation_web.jpg b/scripts/GinanUI/docs/images/plot_visualisation_web.jpg new file mode 100644 index 000000000..291bb5691 Binary files /dev/null and b/scripts/GinanUI/docs/images/plot_visualisation_web.jpg differ diff --git a/scripts/GinanUI/docs/images/process_button.jpg b/scripts/GinanUI/docs/images/process_button.jpg new file mode 100644 index 000000000..686ac3319 Binary files /dev/null and b/scripts/GinanUI/docs/images/process_button.jpg differ diff --git a/scripts/GinanUI/docs/images/product_downloading.jpg b/scripts/GinanUI/docs/images/product_downloading.jpg new file mode 100644 index 000000000..d4622ae6f Binary files /dev/null and b/scripts/GinanUI/docs/images/product_downloading.jpg differ diff --git a/scripts/GinanUI/main.py b/scripts/GinanUI/main.py new file mode 100644 index 000000000..74543cb4a --- /dev/null +++ b/scripts/GinanUI/main.py @@ -0,0 +1,23 @@ +import os +import sys +import logging + +from PySide6.QtGui import QIcon +from PySide6.QtWidgets import QApplication +from scripts.GinanUI.app.main_window import MainWindow + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) + +if __name__ == "__main__": + # Disable GPU acceleration (can cause segmentation faults on launch if enabled) + os.environ['QTWEBENGINE_CHROMIUM_FLAGS'] = '--disable-gpu --disable-software-rasterizer --no-sandbox' + + app = QApplication(sys.argv) + app.setWindowIcon(QIcon("app/resources/ginan-logo.png")) + window = MainWindow() + window.setWindowIcon(QIcon("app/resources/ginan-logo.png")) + window.show() + sys.exit(app.exec()) diff --git a/scripts/GinanUI/requirements.txt b/scripts/GinanUI/requirements.txt new file mode 100644 index 000000000..c3bacb8bd --- /dev/null +++ b/scripts/GinanUI/requirements.txt @@ -0,0 +1,10 @@ +ruamel.yaml~=0.18.15 +pandas~=2.3.3 +PySide6~=6.10.0 +plotly~=6.3.1 +numpy~=2.3.3 +statsmodels~=0.14.5 +requests~=2.32.5 +hatanaka~=2.8.1 +unlzw3~=0.2.3 +beautifulsoup4~=4.14.2 \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..7492ff621 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,232 @@ +# Ginan Scripts + +This directory contains a number of useful scripts that facilitate: + + - running Ginan via: + - a graphical user interface (under `scripts/GinanUI`) + - shell scripts for installing Ginan natively (under `scripts/installation`) + - scripts that handle downloading necessary input files (`auto_download_PPP.py`) + - plotting Ginan output files, including + - POS files (`plot_pos.py`) + - ZTD files (`plotting/ztd_plot.py`) + - exploring and debugging Ginan and it's Kalman filter via: + - The Ginan Exploratory Data Analysis (EDA) tool (`scripts/GinanEDA`) + +Each sub-directory listed above contains it's own README, which provides further details on running the various functionalities. +The rest of this README will cover the files located on the `scripts` directory, namely: +1. `auto_download_PPP.py` +2. `plot_pos.py` +3. `plot_trace_res.py ` +4. `s3_filehandler.py` + +## _**Recommended:**_ +Before continuing, it is highly recommended that you create a python virtual environment if you have not already done so as suggested on the root README file: +```bash +# Create virtual environment +python3 -m venv ginan-env +source ginan-env/bin/activate +``` +The above line will the virtual environment in your current working directory. Once the above is complete, you will have the virtual environment in your current working directory. + +You can then install all python dependencies via a `pip` command: +```bash +# Install Python dependencies +pip3 install -r requirements.txt +``` +## 1. auto_download_PPP +The `auto_download_PPP.py` script makes it easier to download the necessary high precision products and model files necessary for processing RINEX data in Ginan to produce PPP results. + +Based on a few details provided by the user via arguments in the command-line interface (CLI), the script fetches the appropriate files for a given date or date range. These files includes products such as: + + - precise orbits (`.SP3`) + - broadcast orbits (`BRDC.RNX`) + - precise clocks (`.CLK`) + - Earth rotation parameters (`.ERP` or IERS IAU2000 file) + - CORS station positions and metadata (`.SNX`), + - satellite metadata (`.SNX`) + - code biases (`.BIA`) + +The product files are mostly obtained from the NASA archive known as the Crustal Dynamics Data Information System (CDDIS). This is one of NASA's Distributed Active Archive Centers (DAACs). + +To use and download from this archive, you will need to create an Earthdata Login account and provide your username and password in a `.netrc` file. This outlined below in Section 1.2. + +It also includes the various model files needed: + + - planetary ephemerides (JPL Development Ephemeris `DE436.1950.2050`), + - atmospheric loading, + - geopotential (Earth Gravitational Model `EGM2008`), + - geomagnetic reference field (International Geomagnetic Reference Field `IGRF14`) + - ocean loading, + - ocean pole tide coefficients, + - ocean tide potential (Finite Element Solution 2014b `FES2014b`), + - troposphere (Global Pressure and Temperature model `GPT2.5`) + +These are needed for running PPP. + +### 1.1 Earthdata Login Credentials - CDDIS Downloads +To download product files from the Crustal Dynamics Data Information System (CDDIS) web archive you will need an Earthdata Login account credentials saved to your machine. + +#### 1.1.1 Create New Earthdata Login Account (if you don't have one): +You can create a new Earthdata account at the following website: +https://urs.earthdata.nasa.gov/users/new + +#### 1.1.2 Save Credentials to Your Machine + +Once you have your username and password, these must be saved in a `.netrc` file on your home directory. Depending on your operating system, this can be achieved in different ways: + +##### Unix / Linux / MacOS: +This can be done in a terminal window via: +```bash +echo "machine urs.earthdata.nasa.gov login your_username password your_password" >> ~/.netrc +``` +Make sure to set appropriately restrictive file permissions as well (read / write by the current user only): +```bash +chmod 0600 ~/.netrc +``` + +##### Windows: +1. Open Notepad or any plain-text editor. + +2. Enter the `.netrc` format shown above, replace the placeholders with your actual login and password. + +3. Save the file as `_netrc` (with an underscore instead of a period) in your home directory. + +4. Set file permissions: + - Right-click _netrc file and choose Properties + - Go to the Security tab → Click Edit + - Remove access for all other users except your own account + - Click Apply to save the changes. + +The above Earthdata credential instructions are adapted from the following website:
    +https://nsidc.org/data/user-resources/help-center/creating-netrc-file-earthdata-login + +### 1.2 Test "auto_download_PPP" in Virtual Enviroment +Once you have your credentials set up, you are ready to automatically download all necessary product and model files via python. Before we do this though, we will test that the script is working as expected. + +First, make sure you have your virtual environment activated in your current terminal. Following the way we recommended to create the environment above, this would look like: +```bash +# Activate virtual environment - ginan-env +source ginan-env/bin/activate +``` + +Next, test that the `auto_download_PPP` script functions correctly: +```bash +# Test auto_download_PPP script: +python auto_download_PPP.py --help +``` +This will display the help page with detailed information on all possible arugments into the function itself. + +### 1.3 Example Run of "auto_download_PPP" +With your virtual environment active, you can now download the product files needed for a PPP run in Ginan. + +We will use the `igs-station` preset to download RINEX files for two IGS stations for two days in 2024 together with all the product and model files needed to run this in Ginan. + +```bash +# Example run of auto_download_PPP: +python auto_download_PPP.py --target-dir /data/temp/products --rinex-data-dir /data/temp/data --station-list ALIC,HOB2 --start-datetime 2024-01-06_00:00:00 --end-datetime 2024-01-07_23:59:30 --preset=igs-station --dont-replace +``` +Each of the arguments used above are described below: + +- `--target-dir`: sets the directory where product files are downloaded into (with model files going into `target-dir/tables`). In this case `target-dir=/data/temp/products` and model files end up in `/data/temp/products/tables` +- `--rinex-data-dir`: sets the directory where observational RINEX files are downloaded into (for `ALIC` and `HOB2` in this case) +- `--station-list`: this list of stations to download RINEX files for +- `--start-datetime`: the start date and time to download files from +- `--end-datetime`: the end date and time to download files to +- `--preset=igs-station`: tells the script to download all necessary products to run this in PPP mode in Ginan: `SP3`,`CLK`, `BRDC`, `SNX`, `BIA`, satellite metadata `SNX`, IERS Earth orientation data, plus all necessary model files in `target-dir/tables` + +If you are aware of which files you need to download, you can use the default `preset` mode of `manual` and just choose the various files needed based on their flags. For detailed info on all possible flags, run: +```bash +python auto_download_PPP.py --help +``` + +## 2. plot_pos + +The `plot_pos.py` script is used to visualise the contents of a Ginan `.POS` format file. + +Output plots are plotly `.html` files that can be displayed in a web browser + +```bash +Usage: +plot_pos.py [-h] --input-files INPUT_FILES [INPUT_FILES ...] [--start-datetime START_DATETIME] [--end-datetime END_DATETIME] [--horz-smoothing HORZ_SMOOTHING] [--vert-smoothing VERT_SMOOTHING] [--colour-sigma] [--max-sigma MAX_SIGMA] [--elevation] [--demean] [--map] [--heatmap] [--sigma-threshold SIGMA_THRESHOLD SIGMA_THRESHOLD SIGMA_THRESHOLD] [--down-sample DOWN_SAMPLE] [--save-prefix [SAVE_PREFIX]] +``` +Plots positional data and uncertainties with optional smoothing and color coding. + +### Optional arguments: + + - `-h`, `--help` show this help message and exit + - `--input-files` INPUT_FILES ...: One or more input .POS files for plotting (**required**) + - `--start-datetime` START_DATETIME: Start datetime in the format YYYY-MM-DDTHH:MM:SS, optional timezone + - `--end-datetime` END_DATETIME: End datetime in the format YYYY-MM-DDTHH:MM:SS, optional timezone + - `--horz-smoothing` HORZ_SMOOTHING: Fraction of the data used for horizontal LOWESS smoothing (optional). + - `--vert-smoothing` VERT_SMOOTHING: Fraction of the data used for vertical (Up) LOWESS smoothing (optional). + - `--colour-sigma`: Colourize the timeseries using the standard deviation (sigma) values (optional). + - `--max-sigma` MAX_SIGMA: Set a maximum sigma threshold for the sigma colour scale (optional). + - `--elevation`: Plot Elevation values inplace of dU wrt the reference coord (optional). + - `--demean`: Remove the mean values from all time series before plotting (optional). + - `--map`: Create a geographic map view from the Longitude & Latitude estiamtes (optional). + - `--heatmap`: Create a 2D heatmap view of E/N coodrinates wrt the reference position (optional). + - `--sigma-threshold` THRESHOLDS: Thresholds for sE, sN, and sU to filter data. + - `--down-sample` DOWN_SAMPLE: Interval in seconds for down-sampling data. + - `--save-prefix` [SAVE_PREFIX]: Prefix for saving HTML figures, e.g., ./output/fig + +### Examples + +- Plot a Ginan output .POS file: + +```bash +python plot_pos.py --input-files ALIC00AUS_R_20191990000_01D_30S_MO.rnx.POS +``` + +- Plot a Ginan output .POS file, using colours to represent uncertainties and a heatmep of horizontal positions: + +```bash +python plot_pos.py --input-files ALIC00AUS_R_20191990000_01D_30S_MO.rnx.POS --colour-sigma --heatmap --elevation +``` + +## 3. plot_trace_res + +The `plot_trace_res.py` script is used to visualise the contents of a Ginan Network `.TRACE` format file. + +Extracts and plots GNSS code and phase residuals by receiver and/or satellite with optional markers for large-errors, state errors. + +Output plots are plotly `.html` files that can be displayed in a web browser + +```bash +Usage: +plot_trace_res.py [-h] --files FILES [FILES ...] [--residual {prefit,postfit}] [--receivers RECEIVERS [--sat SAT] [--label-regex LABEL_REGEX] [--max-abs MAX_ABS] [--start START] [--end END] [--decimate DECIMATE] [--split-per-sat | --split-per-recv] [--out-dir OUT_DIR] [--basename BASENAME] [--webgl] [--log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL}] [--out-prefix OUT_PREFIX] [--mark-large-errors] [--hover-unified] [--plot-normalised-res] [--show-stats-table] [--stats-matrix] [--stats-matrix-weighted] [--annotate-stats-matrix] [--mark-amb-resets] [--ambiguity-counts] [--ambiguity-totals] [--amb-totals-orient {h,v}] [--amb-totals-topn AMB_TOTALS_TOPN] [--use-forward-residuals] +``` + +Optional arguments: + +- `-h`, `--help` show this help message and exit +- `--files` FILES [FILES ...]: One or more TRACE files (space or , sep ), e.g. 'A.trace B.trace,C.trace' (wildcards allowed eg. *.TRACE) +- `--residual` {prefit,postfit}: Plot prefit or postfit residuals (default: postfit) +- `--receivers` RECEIVERS: One or more receiver names (space or , separated), e.g. 'ABMF,CHUR ALGO' +- `--sat` SAT, -s SAT: Filter by satellite ID +- `--label-regex` LABEL_REGEX: Regex to filter labels +- `--max-abs` MAX_ABS: Max residual to plot +- `--start` START: Start datetime or time-only +- `--end` END: End datetime (exclusive) +- `--decimate` DECIMATE: +- `--split-per-sat` : +- `--split-per-recv` : +- `--out-dir` OUT_DIR: Output directory for HTML files; defaults to CWD. +- `--basename` BASENAME: Base filename prefix for outputs (no extension). +- `--webgl`: Use webgl graphic acceleration +- `--log-level` {DEBUG,INFO,WARNING,ERROR,CRITICAL}: Logging verbosity +- `--out-prefix` OUT_PREFIX: unique prefix to add to output filenames +- `--mark-large-errors`: Mark LARGE STATE/MEAS ERROR events on plots. +- `--hover-unified`: Use unified hover tooltips across all traces (default: closest point hover). +- `--plot-normalised-res`: Also generate plots of normalised residuals (residual / sigma). +- `--show-stats-table`: Add a Mean / Std / RMS table per (sat × signal) at the bottom of each plot. +- `--stats-matrix`: Generate receiver×satellite heatmaps (Mean/Std/RMS) aggregated across signals. +- `--stats-matrix-weighted`: Use sigma-weighted statistics in the heatmaps (weights 1/σ²). +- `--annotate-stats-matrix`: Write the numeric value (mean/std/rms) into each stats heatmap cell. Hover still shows full details. +- `--mark-amb-resets`: Overlay PHASE ambiguity reset events (PREPROC=green, REJECT=blue) on PHASE per-receiver plots. +- `--ambiguity-counts`: Plot cumulative counts of ambiguity reset reasons and unique satellite resets over time. +- `--ambiguity-totals`: Bar chart of total ambiguity reset reasons (diagnostic view of detection methods). +- `--amb-totals-orient`: {h,v} Orientation for totals bar charts: 'h' (horizontal, default) or 'v' (vertical). +- `--amb-totals-topn AMB_TOTALS_TOPN`: Show only the top N receivers/satellites by total resets (to avoid clutter). +- `--use-forward-residuals`: Use residuals from forward (non-smoothed) files instead of smoothed files (default: use smoothed for more accurate residuals). + +## 4. s3_filehandler diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/auto_download_PPP.py b/scripts/auto_download_PPP.py index aafa3a910..42c928d4c 100644 --- a/scripts/auto_download_PPP.py +++ b/scripts/auto_download_PPP.py @@ -5,23 +5,28 @@ import random import ftplib import logging -import tarfile import requests import numpy as np from time import sleep from pathlib import Path -from typing import Tuple, Union -from copy import deepcopy +from typing import Tuple from urllib.parse import urlparse from contextlib import contextmanager -from datetime import datetime, timedelta, date +from datetime import datetime, timedelta -from gn_functions import ( - GPSDate, - gpswkD2dt, +from gnssanalysis.gn_datetime import GPSDate +from gnssanalysis.gn_download import ( + download_product_from_cddis, decompress_file, - download_url, + generate_content_type, + generate_sampling_rate, + generate_product_filename, + download_file_from_cddis, + check_whether_to_download, + attempt_url_download, + long_filename_cddis_cutoff, ) +from gnssanalysis.gn_utils import configure_logging, ensure_folders API_URL = "https://data.gnss.ga.gov.au/api" @@ -36,429 +41,6 @@ def ftp_tls(url: str, **kwargs) -> None: ftps.quit() -def configure_logging(verbose: bool) -> None: - if verbose: - logging_level = logging.DEBUG - else: - logging_level = logging.INFO - logging.basicConfig(format="%(asctime)s [%(funcName)s] %(levelname)s: %(message)s") - logging.getLogger().setLevel(logging_level) - - -def ensure_folders(paths: list) -> None: - """ - Ensures the list of folders exist in the file system. - """ - for path in paths: - if not isinstance(path, Path): - path = Path(path) - if not path.is_dir(): - path.mkdir(parents=True) - - -def generate_nominal_span(start_epoch: datetime, end_epoch: datetime) -> str: - """ - Generate the 3 character LEN for IGS filename based on the start and end epochs passed in - """ - span = (end_epoch - start_epoch).total_seconds() - if span % 86400 == 0.0: - unit = "D" - span = int(span // 86400) - elif span % 3600 == 0.0: - unit = "H" - span = int(span // 3600) - elif span % 60 == 0.0: - unit = "M" - span = int(span // 60) - else: - raise NotImplementedError - - return f"{span:02}{unit}" - - -def generate_long_filename( - analysis_center: str, # AAA - content_type: str, # CNT - format_type: str, # FMT - start_epoch: datetime, - end_epoch: datetime = None, - timespan: timedelta = None, - solution_type: str = "", # TTT - sampling_rate: str = "15M", # SMP - version: str = "0", # V - project: str = "EXP", # PPP, e.g. EXP, OPS -) -> str: - """ - Function to generate filename with IGS Long Product Filename convention (v1.0) as outlined in - http://acc.igs.org/repro3/Long_Product_Filenames_v1.0.pdf - - AAAVPPPTTT_YYYYDDDHHMM_LEN_SMP_CNT.FMT[.gz] - """ - initial_epoch = start_epoch.strftime("%Y%j%H%M") - if end_epoch == None: - end_epoch = start_epoch + timespan - timespan_str = generate_nominal_span(start_epoch, end_epoch) - - result = ( - f"{analysis_center}{version}{project}" - f"{solution_type}_" - f"{initial_epoch}_{timespan_str}_{sampling_rate}_" - f"{content_type}.{format_type}" - ) - return result - - -def generate_content_type(file_ext: str, analysis_center: str) -> str: - """ - IGS files following the long filename convention require a content specifier - Given the file extension, generate the content specifier - """ - file_ext = file_ext.upper() - file_ext_dict = { - "ERP": "ERP", - "SP3": "ORB", - "CLK": "CLK", - "OBX": "ATT", - "TRO": "TRO", - "SNX": "CRD", - "BIA": {"ESA": "BIA", None: "OSB"}, - } - content_type = file_ext_dict.get(file_ext) - # If result is still dictionary, use analysis_center to determine content_type - if isinstance(content_type, dict): - content_type = content_type.get(analysis_center, content_type.get(None)) - return content_type - - -def generate_sampling_rate(file_ext: str, analysis_center: str, solution_type: str) -> str: - """ - IGS files following the long filename convention require a content specifier - Given the file extension, generate the content specifier - """ - file_ext = file_ext.upper() - sampling_rates = { - "ERP": { - ("COD"): {"FIN": "12H", "RAP": "01D", "ERP": "01D"}, - (): "01D", - }, - "BIA": "01D", - "SP3": { - ("COD", "GFZ", "GRG", "IAC", "JAX", "MIT", "WUM"): "05M", - ("ESA"): {"FIN": "05M", "RAP": "15M", None: "15M"}, - (): "15M", - }, - "CLK": { - ("EMR", "MIT", "SHA", "USN"): "05M", - ("ESA", "GFZ", "GRG", "IGS"): {"FIN": "30S", "RAP": "05M", None: "30S"}, # DZ: IGS FIN has 30S CLK - (): "30S", - }, - "OBX": {"GRG": "05M", None: "30S"}, - "TRO": {"JPL": "30S", None: "01H"}, - "SNX": "01D", - } - if file_ext in sampling_rates: - file_rates = sampling_rates[file_ext] - if isinstance(file_rates, dict): - center_rates_found = False - for key in file_rates: - if analysis_center in key: - center_rates = file_rates.get(key, file_rates.get(())) - center_rates_found = True - break - # else: - # return file_rates.get(()) - if not center_rates_found: # DZ: bug fix - return file_rates.get(()) - if isinstance(center_rates, dict): - return center_rates.get(solution_type, center_rates.get(None)) - else: - return center_rates - else: - return file_rates - else: - return "01D" - - -def generate_product_filename( - reference_start: datetime, - file_ext: str, - shift: int = 0, - long_filename: bool = False, - AC: str = "IGS", - timespan: timedelta = timedelta(days=1), - solution_type: str = "ULT", - sampling_rate: str = "15M", - version: str = "0", - project: str = "OPS", - content_type: str = None, -) -> Tuple[str, GPSDate, datetime]: - """ - Generate filename, GPSDate obj from datetime - Optionally, move reference_start forward by "shift" hours - """ - reference_start += timedelta(hours=shift) - if type(reference_start == date): - gps_date = GPSDate(str(reference_start)) - else: - gps_date = GPSDate(str(reference_start.date())) - - if long_filename: - if content_type == None: - content_type = generate_content_type(file_ext, analysis_center=AC) - product_filename = ( - generate_long_filename( - analysis_center=AC, - content_type=content_type, - format_type=file_ext, - start_epoch=reference_start, - timespan=timespan, - solution_type=solution_type, - sampling_rate=sampling_rate, - version=version, - project=project, - ) - + ".gz" - ) - else: - if file_ext.lower() == "snx": - product_filename = f"igs{gps_date.yr[2:]}P{gps_date.gpswk}.snx.Z" - else: - hour = f"{reference_start.hour:02}" - product_filename = f"igu{gps_date.gpswkD}_{hour}.{file_ext}.Z" - return product_filename, gps_date, reference_start - - -def download_file_from_cddis( - filename: str, - ftp_folder: str, - output_folder: Path, - max_retries: int = 3, - decompress: bool = True, - if_file_present: str = "prompt_user", - note_filetype: str = None, -) -> None: - with ftp_tls("gdc.cddis.eosdis.nasa.gov") as ftps: - ftps.cwd(ftp_folder) - retries = 0 - download_done = False - while not download_done and retries <= max_retries: - try: - download_filepath = attempt_ftps_download( - download_dir=output_folder, - ftps=ftps, - filename=filename, - type_of_file=note_filetype, - if_file_present=if_file_present, - ) - if decompress and download_filepath: - download_filepath = decompress_file( - input_filepath=download_filepath, delete_after_decompression=True - ) - download_done = True - if download_filepath: - logging.info(f"Downloaded {download_filepath.name}") - except ftplib.all_errors as e: - retries += 1 - if retries > max_retries: - logging.info(f"Failed to download {filename} and reached maximum retry count ({max_retries}).") - if (output_folder / filename).is_file(): - (output_folder / filename).unlink() - raise e - - logging.debug(f"Received an error ({e}) while try to download {filename}, retrying({retries}).") - # Add some backoff time (exponential random as it appears to be contention based?) - sleep(random.uniform(0.0, 2.0**retries)) - return download_filepath - - -def download_product_from_cddis( - download_dir: Path, - start_epoch: datetime, - end_epoch: datetime, - file_ext: str, - limit: int = None, - long_filename: bool = False, - analysis_center: str = "IGS", - solution_type: str = "ULT", - sampling_rate: str = "15M", - project_type: str = "OPS", - timespan: timedelta = timedelta(days=2), - if_file_present: str = "prompt_user", -) -> None: - """ - Download the file/s from CDDIS based on start and end epoch, to the - provided the download directory (download_dir) - """ - # DZ: Download the correct IGS FIN ERP files - if file_ext == "ERP" and analysis_center == "IGS" and solution_type == "FIN": # get the correct start_epoch - start_epoch = GPSDate(str(start_epoch)) - start_epoch = gpswkD2dt(f"{start_epoch.gpswk}0") - timespan = timedelta(days=7) - # Details for debugging purposes: - logging.debug("Attempting CDDIS Product download/s") - logging.debug(f"Start Epoch - {start_epoch}") - logging.debug(f"End Epoch - {end_epoch}") - - reference_start = deepcopy(start_epoch) - product_filename, gps_date, reference_start = generate_product_filename( - reference_start, - file_ext, - long_filename=long_filename, - AC=analysis_center, - timespan=timespan, - solution_type=solution_type, - sampling_rate=sampling_rate, - project=project_type, - ) - logging.debug( - f"Generated filename: {product_filename}, with GPS Date: {gps_date.gpswkD} and reference: {reference_start}" - ) - with ftp_tls("gdc.cddis.eosdis.nasa.gov") as ftps: - try: - ftps.cwd(f"gnss/products/{gps_date.gpswk}") - except ftplib.all_errors as e: - logging.info(f"{reference_start} too recent") - logging.info(f"ftp_lib error: {e}") - product_filename, gps_date, reference_start = generate_product_filename( - reference_start, - file_ext, - shift=-6, - long_filename=long_filename, - AC=analysis_center, - timespan=timespan, - solution_type=solution_type, - sampling_rate=sampling_rate, - project=project_type, - ) - ftps.cwd(f"gnss/products/{gps_date.gpswk}") - - all_files = ftps.nlst() - if not (product_filename in all_files): - logging.info(f"{product_filename} not in gnss/products/{gps_date.gpswk} - too recent") - raise FileNotFoundError - - # reference_start will be changed in the first run through while loop below - reference_start -= timedelta(hours=24) - count = 0 - remain = end_epoch - reference_start - while remain.total_seconds() > timespan.total_seconds(): - if count == limit: - remain = timedelta(days=0) - else: - product_filename, gps_date, reference_start = generate_product_filename( - reference_start, - file_ext, - shift=24, # Shift at the start of the loop - speeds up total download time - long_filename=long_filename, - AC=analysis_center, - timespan=timespan, - solution_type=solution_type, - sampling_rate=sampling_rate, - project=project_type, - ) - download_filepath = check_whether_to_download( - filename=product_filename, download_dir=download_dir, if_file_present=if_file_present - ) - if download_filepath: - download_file_from_cddis( - filename=product_filename, - ftp_folder=f"gnss/products/{gps_date.gpswk}", - output_folder=download_dir, - if_file_present=if_file_present, - note_filetype=file_ext, - ) - count += 1 - remain = end_epoch - reference_start - - -def check_whether_to_download( - filename: str, download_dir: Path, if_file_present: str = "prompt_user" -) -> Union[Path, None]: - """ - Determine whether to download given file (filename) to the desired location (download_dir) based on whether it is - already present and what action to take if it is (if_file_present). Output is the Path obj to the file to be - downloaded (output_path) or None if file is not to be downloaded (already present and skipped) - """ - # Flag to determine whether to download: - download = None - # Create Path obj to where file will be - if original file is compressed, check for decompressed file - uncompressed_filename = generate_uncompressed_filename(filename) # Returns original filename if not compressed - output_path = download_dir / uncompressed_filename - # Check if file already exists - if so, then re-download or not based on if_file_present value - if output_path.is_file(): - if if_file_present == "prompt_user": - replace = click.confirm( - f"File {output_path} already present; download and replace? - Answer:", default=None - ) - if replace: - logging.info(f"Option chosen: Replace. Re-downloading {output_path.name} to {download_dir}") - download = True - else: - logging.info(f"Option chosen: Don't Replace. Leaving {output_path.name} as is in {download_dir}") - download = False - elif if_file_present == "dont_replace": - logging.info(f"File {output_path} already present; Flag --dont-replace is on ... skipping download ...") - download = False - elif if_file_present == "replace": - logging.info( - f"File {output_path} already present; Flag --replace is on ... re-downloading to {download_dir}" - ) - download = True - else: - download = True - - if download == False: - return None - elif download == True: - if uncompressed_filename == filename: # Existing Path obj is already the one we need to download - return output_path - else: - return download_dir / filename # Path to compressed file to download - elif download == None: - logging.error(f"Invalid internal flag value for if_file_present: '{if_file_present}'") - - -def attempt_ftps_download( - download_dir: Path, - ftps: ftplib.FTP_TLS, - filename: str, - type_of_file: str = None, - if_file_present: str = "prompt_user", -) -> Path: - """ - Attempt download of file (filename) given the ftps client object (ftps) to chosen location (download_dir) - """ - logging.info(f"Attempting FTPS Download of {type_of_file} file - {filename} to {download_dir}") - download_filepath = check_whether_to_download( - filename=filename, download_dir=download_dir, if_file_present=if_file_present - ) - if download_filepath: - logging.debug(f"Downloading {filename}") - with open(download_filepath, "wb") as local_file: - ftps.retrbinary(f"RETR {filename}", local_file.write) - - return download_filepath - - -def attempt_url_download( - download_dir: Path, url: str, filename: str = None, type_of_file: str = None, if_file_present: str = "prompt_user" -) -> Path: - """ - Attempt download of file given URL (url) to chosen location (download_dir) - """ - # If the filename is not provided, use the filename from the URL - if not filename: - filename = url[url.rfind("/") + 1 :] - logging.info(f"Attempting URL Download of {type_of_file} file - {filename} to {download_dir}") - # Use the check_whether_to_download function to determine whether to download the file - download_filepath = check_whether_to_download( - filename=filename, download_dir=download_dir, if_file_present=if_file_present - ) - if download_filepath: - download_url(url, download_filepath) - return download_filepath - - def download_atx(download_dir: Path, long_filename: bool = False, if_file_present: str = "prompt_user") -> None: """ Download the ATX file necessary for running the PEA provided the download directory (download_dir) @@ -475,28 +57,6 @@ def download_atx(download_dir: Path, long_filename: bool = False, if_file_presen ) -def generate_uncompressed_filename(filename: str) -> str: - """ - Name of uncompressed filename given the [assumed compressed] filename (filename). - If not one of the recognized compression format types, return original filename - - """ - if filename.endswith(".tar.gz") or filename.endswith(".tar"): - with tarfile.open(filename, "r") as tar: - # Get name of file inside tar.gz file (assuming only one file) - return tar.getmembers()[0].name - elif filename.endswith(".crx.gz"): - return filename[:-6] + "rnx" - elif filename.endswith(".gz"): - return filename[:-3] - elif filename.endswith(".Z"): - return filename[:-2] - elif filename.endswith(".bz2"): - return filename[:-4] - else: - logging.debug(f"{filename} not compressed - extension not a recognized compression format") - return filename - - def download_atmosphere_loading_model(download_dir: Path, if_file_present: str = "prompt_user") -> Path: """ Download the Atmospheric loading BLQ file necessary for running the PPP example @@ -561,19 +121,19 @@ def download_brdc( reference_dt += timedelta(days=1) -def download_geomagnetic_model(download_dir: Path, model: str = "igrf13", if_file_present: str = "prompt_user") -> Path: +def download_geomagnetic_model(download_dir: Path, model: str = "igrf14", if_file_present: str = "prompt_user") -> Path: """ Download the International Geomagnetic Reference Field model file necessary for running the PPP example provided the download directory (download_dir) - Default: IGRF13 coefficients + Default: IGRF14 coefficients """ - if model == "igrf13": + if model == "igrf14": ensure_folders([download_dir]) download_filepath = attempt_url_download( download_dir=download_dir, - url="https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/igrf13coeffs.txt.gz", - filename="igrf13coeffs.txt.gz", - type_of_file="Geomagnetic Field coefficients - IGRF13", + url="https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/products/tables/igrf14coeffs.txt.gz", + filename="igrf14coeffs.txt.gz", + type_of_file="Geomagnetic Field coefficients - IGRF14", if_file_present=if_file_present, ) else: @@ -880,7 +440,7 @@ def search_for_most_recent_file( reference_start=pointer_date.as_datetime, file_ext=file_type, long_filename=long_filename, - AC=analysis_center, + analysis_center=analysis_center, timespan=timespan, solution_type=solution_type, sampling_rate=sampling_rate, @@ -900,7 +460,7 @@ def search_for_most_recent_file( reference_start=pointer_date.as_datetime, file_ext=file_type, long_filename=long_filename, - AC=analysis_center, + analysis_center=analysis_center, timespan=timespan, solution_type=solution_type, sampling_rate=sampling_rate, @@ -996,18 +556,6 @@ def download_files_from_gnss_data( logging.info(f"Not downloaded / missing: {list(missing_stations)}") -def long_filename_cddis_cutoff(epoch: datetime) -> bool: - """ - Simple function that determines whether long filenames should be expected on the CDDIS server - """ - # Long filename cut-off: - long_filename_cutoff = datetime(2022, 11, 27) - if epoch >= long_filename_cutoff: - return True - else: - return False - - def most_recent_6_hour(): """ Returns a datetime object set to the most recent hour divisible by 6: (0000, 0600, 1200 or 1800) @@ -1195,11 +743,15 @@ def auto_download( timespan=timedelta(days=1), if_file_present=if_file_present, ) - except FileNotFoundError: - logging.info(f"Received an error ({FileNotFoundError}) while try to download - date too recent.") + except ftplib.all_errors as e: + logging.info(f"Received an error ({e}) while try to download - date too recent.") logging.info(f"Downloading most recent SNX file available.") download_most_recent_cddis_file( - download_dir=target_dir, pointer_date=start_gpsdate, file_type="SNX", long_filename=long_filename + download_dir=target_dir, + pointer_date=start_gpsdate, + file_type="SNX", + long_filename=long_filename, + if_file_present=if_file_present, ) if sp3: download_product_from_cddis( @@ -1303,6 +855,11 @@ def auto_download( help="Provide comma-separated list of IGS stations to download - daily observation RNX files", type=str, ) +@click.option( + "--station-list-file", + help="Read file of newline separated list of IGS stations to download - takes precedence over option --station-list", + type=click.File("r"), +) @click.option("--start-datetime", help="Start of date-time period to download files for", type=str) @click.option("--end-datetime", help="End of date-time period to download files for", type=str) @click.option("--replace", help=" Re-download all files already present in target-dir", default=False, is_flag=True) @@ -1328,10 +885,10 @@ def auto_download( @click.option("--gpt2", help="Flag to Download GPT 2.5 file", default=False, is_flag=True) @click.option("--rinex-data-dir", help="Directory to Download RINEX data file/s. Default: target-dir", type=Path) @click.option("--trop-dir", help="Directory to Download troposphere model file/s. Default: target-dir", type=Path) -@click.option("--model-dir", help="Directory to Download static model files. Default: product-dir / tables", type=Path) +@click.option("--model-dir", help="Directory to Download static model files. Default: target-dir / tables", type=Path) @click.option( "--solution-type", - help="The solution type of products to download from CDDIS. 'RAP': rapid, or 'ULT': ultra-rapid. Default: RAP", + help="The solution type of products to download from CDDIS. 'FIN': final, or 'RAP': rapid, or 'ULT': ultra-rapid. Default: RAP", default="RAP", type=str, ) @@ -1372,6 +929,7 @@ def auto_download_main( target_dir, preset, station_list, + station_list_file, start_datetime, end_datetime, replace, @@ -1413,6 +971,9 @@ def auto_download_main( station_list = None if not station_list == None: station_list = station_list.split(",") + if station_list_file: + content = station_list_file.read() + station_list = content.split("\n") auto_download( target_dir, preset, diff --git a/scripts/auto_generate_yaml.py b/scripts/deprecated_scripts/auto_generate_yaml.py similarity index 100% rename from scripts/auto_generate_yaml.py rename to scripts/deprecated_scripts/auto_generate_yaml.py diff --git a/scripts/auto_run_PPP.py b/scripts/deprecated_scripts/auto_run_PPP.py similarity index 100% rename from scripts/auto_run_PPP.py rename to scripts/deprecated_scripts/auto_run_PPP.py diff --git a/scripts/compareGinanJson.py b/scripts/deprecated_scripts/compareGinanJson.py similarity index 100% rename from scripts/compareGinanJson.py rename to scripts/deprecated_scripts/compareGinanJson.py diff --git a/scripts/createAppimage.sh b/scripts/deprecated_scripts/createAppimage.sh similarity index 60% rename from scripts/createAppimage.sh rename to scripts/deprecated_scripts/createAppimage.sh index 3ed3bc970..6c623bbbf 100755 --- a/scripts/createAppimage.sh +++ b/scripts/deprecated_scripts/createAppimage.sh @@ -1,9 +1,9 @@ #!/bin/bash -mkdir linuxDeploy +mkdir -p linuxDeploy cd linuxDeploy -wget https://github.com/linuxdeploy/linuxdeploy/releases/download/1-alpha-20230713-1/linuxdeploy-x86_64.AppImage +curl -L -o linuxdeploy-x86_64.AppImage https://github.com/linuxdeploy/linuxdeploy/releases/download/1-alpha-20230713-1/linuxdeploy-x86_64.AppImage apt update -y apt install -y file diff --git a/scripts/download_slr_data.py b/scripts/deprecated_scripts/download_slr_data.py similarity index 100% rename from scripts/download_slr_data.py rename to scripts/deprecated_scripts/download_slr_data.py diff --git a/scripts/qzss_ohi_merge.py b/scripts/deprecated_scripts/qzss_ohi_merge.py similarity index 100% rename from scripts/qzss_ohi_merge.py rename to scripts/deprecated_scripts/qzss_ohi_merge.py diff --git a/scripts/download_archives.py b/scripts/download_archives.py deleted file mode 100644 index 0f350339f..000000000 --- a/scripts/download_archives.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Downloads auxiliary example_input_data files to/from example_input_data dir""" -import logging -import hashlib -import concurrent.futures -import base64 -import os - -from pathlib import Path -from typing import Union as _Union -import tarfile - -import boto3 -from botocore import UNSIGNED -from botocore.config import Config - -import click as _click - -logging.basicConfig(level=logging.INFO, format="%(message)s") -logger = logging.getLogger(__name__) - - -def compute_checksum(path2file: str) -> str: - """Computes checksum of a file given its path - - :param str path2file: path to the file - :return str: checksum value - """ - logger.debug(f'Computing checksum of "{path2file}"') - with open(path2file, "rb") as file: - filehash = hashlib.md5() - for data in iter(lambda: file.read(8 * 1024 * 1024), b""): - filehash.update(data) - checksum = base64.b64encode(filehash.digest()).decode() - logger.debug(f'Got "{checksum}"') - return checksum - - -def create_s3_client(profile_name: str = None, access_key: str = None, secret_key: str = None) -> boto3.client: - """ - create_s3_client creates a boto3 client for s3 - if profile_name is provided, it will use the credentials from the profile - if access_key and secret_key are provided, it will use those credentials - otherwise it will create an anonymous client - - :param str profile_name: _description_, defaults to None - :param str access_key: _description_, defaults to None - :param str secret_key: _description_, defaults to None - :return boto3.client: _description_ - """ - if profile_name: - session = boto3.Session(profile_name=profile_name) - logger.debug("setting up s3 client with profile %s", profile_name) - elif access_key and secret_key: - session = boto3.Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key) - logger.debug("setting up s3 client with access key and secret key") - else: - session = boto3.Session() - logger.debug("setting up s3 client with no credentials") - s3 = session.client("s3", config=Config(signature_version=UNSIGNED, max_pool_connections=50)) - return s3 - - -def read_tags_from_file(file_path): - """ - read_tags_from_file _summary_ - - :param _type_ file_path: _description_ - :return _type_: _description_ - """ - dictionary = {} - with open(file_path, "r", encoding="utf-8") as file: - for line in file: - if "=" in line: - key, value = line.strip().split("=") - value = value.strip('"') # Remove quotes from the value - dictionary[key] = value - return dictionary - - -def get_list_from_tag(s3client, bucket, dictdata, target, dirs, list_to_download): - """ - get_list_from_tag - Function to get the list of files to download from the tag, - for each element in the dictionary dictdata, it will need to download the file at the following address - https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/solutions//index.json - concatenate them all in a dictionary. - :param _type_ dictdata: _description_ - """ - ex_type_dict = {"PEA": [0, 1, 4], "POD": [2], "PEAPOD": [3], "OTHER": [5]} - ex_type_dict["ALL"] = list(set([item for sublist in ex_type_dict.values() for item in sublist])) - - logger.info(f" to download {dirs}") - for key, tag in dictdata.items(): - logger.debug(f"Looking for data in {key} tagged {tag}") - request = s3client.list_objects(Bucket=bucket, Prefix=f"{target}/solutions/{tag}/") - try: - for data in request["Contents"]: - name = data["Key"].split("/")[-1].split(".")[0] - if (len(dirs) == 0 or name in dirs) and int(name[2]) in ex_type_dict[key]: - logger.debug(f"Found {data['Key']} in {key}") - list_to_download.append(data["Key"]) - except KeyError: - logger.warning(f"{request['Prefix']} on bucket {request['Name']} not found") - - -def check_checksum(s3client, bucket, file, download_file): - logger.info(f"Checking checksum of {file}") - response = s3client.head_object(Bucket=bucket, Key=file) - if compute_checksum(download_file) != response["Metadata"]["md5checksum"]: - raise Exception(f"Checksum failed for {file}") - logger.info(" -> Checksum OK") - - -def download_file(s3client, bucket, file, checksum_check=False): - logger.info(f"Downloading {file} from {bucket}") - filename = file.split("/")[-1] - s3client.download_file(bucket, file, filename) - if checksum_check: - try: - check_checksum(s3client, bucket, file, filename) - except Exception as excep: - os.remove(filename) - raise excep - return filename - - -def extract(filename, path): - logger.info(f" -> Extracting {filename} to {path}") - with tarfile.open(filename, "r:bz2") as tar: - for member in tar.getmembers(): - if not member.name.startswith("/") and ".." not in member.name: - tar.extract(member, path) - else: - logging.warning(f"Skipping dangerous member: {member.name}") - os.remove(filename) - logger.info(f" -> Extracted {filename} to {path}") - - -def process_dwl_file(s3client: boto3.client, bucket, file, path, skip_extract): - """ - download_file - Function to download the file from the bucket and extract it to the path - :param _type_ file: _description_ - """ - try: - filename = download_file(s3client, bucket, file, checksum_check=True) - if not skip_extract: - extract(filename, path) - else: - logger.info(f" -> Skipping extraction of {file} to {path}") - except Exception as e: - raise e - - -def process_dwl_files_concurrently( - s3client: boto3.client, bucket: str, files: list, path: Path, skip_extract: bool -) -> None: - """ - Download files concurrently using a ThreadPoolExecutor. - """ - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: - futures = [executor.submit(process_dwl_file, s3client, bucket, file, path, skip_extract) for file in files] - - for future in concurrent.futures.as_completed(futures): - try: - future.result() # Ensure any exceptions are raised - except Exception as e: - logger.warning(f"Failed to download file: {str(e)}") - - -def generate_list_of_files(dirs, bucket, target, solutions, data, products, loading, s3, tag_dict) -> list: - list_to_download = [] - if products: - list_to_download.append(f"{target}/products.tar.bz2") - if data: - list_to_download.append(f"{target}/data.tar.bz2") - if loading: - list_to_download.append(f"{target}/loading.tar.bz2") - if solutions: - get_list_from_tag(s3, bucket, tag_dict, target, dirs, list_to_download) - return list_to_download - - -def generate_tag_dict(tag: str, tags_file_path: str) -> dict: - """ - generate_tag_dict _summary_ - - :param _type_ tag: _description_ - :param _type_ tags_file_path: _description_ - :return _type_: _description_ - """ - tag_dict = {} - if not tag: - logger.info(f"reading tags from {tags_file_path}") - tag_dict = read_tags_from_file(tags_file_path) - else: - tag_dict["ALL"] = tag - logger.info(f"using the provided {tag_dict} tag") - return tag_dict - - -# function to download_example_data, command line argument viw click p for product, l for loading, s for solution, followed by uknown number of arguments -@_click.command() -@_click.argument("dirs", nargs=-1, type=str, default=None) -@_click.option("--bucket", default="peanpod", help="s3 bucket name to push and pull from") -@_click.option("--target", default="aux", help="s3 target name (dir) within the selected bucket") -@_click.option( - "--path", - type=Path, - default="inputData", - help="custom path to inputData dir, a dir that stores products/data etc, default is inputData", -) -@_click.option("--tagpath", type=Path, default="docker/tags") -@_click.option("--tag", type=str, default=None) -@_click.option( - "--skip_extract", - is_flag=True, - show_default=True, - default=False, - help=( - """Skips extraction of the on-disk tar file if checksum test is OK and the - destination dir exists. This is done to save time in the pipeline as - there is no need to overwrite the files.""" - ), -) -@_click.option("-s", "--solutions", is_flag=True, help="download solutions") -@_click.option("-d", "--data", is_flag=True, help="download data") -@_click.option("-p", "--products", is_flag=True, help="download products") -@_click.option("-l", "--loading", is_flag=True, help="download loadings") -@_click.option("--profile", default=None, help="aws profile name") -@_click.option("-v", "--verbose", is_flag=True, help="verbose output") -def download_example_data( - dirs: str, - bucket: str, - target: str, - path: Path, - tagpath: Path, - tag: str, - skip_extract: bool, - solutions: bool, - data: bool, - products: bool, - loading: bool, - profile: str, - verbose: bool, -): - """Downloads auxiliary example_input_data files to/from example_input_data dir""" - logger.setLevel(logging.DEBUG if verbose else logging.INFO) - # todo later plug the possible for the keys. (not needed for download) - s3 = create_s3_client(profile_name=profile, access_key=None, secret_key=None) - tag_dict = generate_tag_dict(tag, tagpath.resolve()) - logger.info(f"list of tags {tag_dict}") - list_to_download = generate_list_of_files(dirs, bucket, target, solutions, data, products, loading, s3, tag_dict) - logger.info("list of files to download") - for file in list_to_download: - logger.info(f" - {file}") - process_dwl_files_concurrently(s3, bucket, list_to_download, path.resolve(), skip_extract) - - -if __name__ == "__main__": - download_example_data() diff --git a/scripts/download_example_input_data.py b/scripts/download_example_input_data.py deleted file mode 100755 index 30674f242..000000000 --- a/scripts/download_example_input_data.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python3 - -"""Downloads/Uploads auxiliary example_input_data files to/from example_input_data dir""" -import logging as _logging -from pathlib import Path as _Path -from shutil import copy as _copy -from shutil import rmtree as _rmtree -from typing import Union as _Union - -import click as _click -import gnssanalysis as ga - -EX_GLOB_DICT = { - "ex02": ["*.TRACE"], - "ex11": ["*.TRACE", "*.snx", "*.*_smoothed"], - "ex12": ["*.TRACE", "*.snx"], - "ex13": ["*.TRACE", "*.snx"], - "ex14": ["*.TRACE", "*.snx"], - "ex16": ["*Network*.TRACE", "*.*I", "*.stec", "*.snx", "*.BIA", "*.INX*"], - "ex17": ["*Network*.TRACE", "*.snx", "*.clk*"], - "ex21": ["pod*.out", "*.sp3"], - "ex22g": ["pod*.out", "*.sp3"], - "ex22r": ["pod*.out", "*.sp3"], - "ex22e": ["pod*.out", "*.sp3"], - "ex22c": ["pod*.out", "*.sp3"], - "ex22j": ["pod*.out", "*.sp3"], - "ex23": ["pod*.out", "*.sp3"], - "ex24": ["pod*.out", "*.sp3"], - "ex25": ["pod*.out", "*.sp3"], - "ex26": ["pod*.out", "*.sp3"], - "ex31": [ - "pod_fit/pod*.out", - "pod_fit/*.sp3", - "pea/*etwork*.TRACE*", # Network starts with lower case for some reason - "pea/*.snx", - "pea/*.erp", - "pea/*clk*", - "pod_ic/pod*.out", - "pod_ic/*.sp3", - ], - "ex41": ["*.TRACE", "*.snx", "*.*_smoothed"], - "ex42": ["*.TRACE", "*.snx"], - "ex43": ["*.TRACE", "*.snx"], - "ex43a": ["*.TRACE", "*.snx"], - "ex44": ["*.TRACE", "*.snx"], - "ex48": ["*.TRACE", "*.snx"], - "ex51": [ - "*blq", - ], - "ex52": [ - "*blq", - ], -} - - -def insert_tag(name: str, tag: str) -> str: - """inserts tag name right before the filename: - insert_tag('ex11','some_tag') -> 'some_tag/ex11' - insert_tag('solutions/ex11','some_tag') -> 'solutions/some_tag/ex11' - """ - name_split = name.split("/") - name_split.insert(-1, tag) - return "/".join(name_split) - - -def get_example_type(name: str) -> _Union[str, bool]: - """ - Checks if input string is a type of example dir, e.g. ex22g, - returns string type of the example test. - ex13 (name[2]==1) is 'PEA' and ex21 (name[2]==2) is 'POD' - """ - ex_type_dict = {"0": "PEA", "1": "PEA", "2": "POD", "3": "PEAPOD", "4": "PEA", "5": "OTHER"} - if name.startswith("ex"): - try: - idx = name[2] - return ex_type_dict[idx] - except KeyError: - raise ValueError(f"Example code '{idx}' could not be matched to a type") - return False - - -def update_solutions_dict(example_input_data_dir: _Path, directory: str, ex_glob_dict: dict, tag: str = ""): - """ """ - if get_example_type(directory): # room for five-symbol name - ex22g - example_dir = example_input_data_dir / directory - ref_sol_dir = example_input_data_dir / "solutions" / tag / directory - if example_dir.exists() and ref_sol_dir.exists(): - _rmtree(ref_sol_dir) - _logging.info( - f"removing {ref_sol_dir} and its content" - ) # if actual solution exists -> clean respective reference solution before copying - l = len(example_dir.as_posix()) - - if directory not in ex_glob_dict.keys(): - raise ValueError(f"{directory} not in EX_GLOB_DICT dictionary") - - paths_list = [] - for expr in ex_glob_dict[directory]: - paths = list(example_dir.glob(expr)) - if paths == []: - _logging.warning(msg=f"no files were found using the {expr} rule") - paths_list.append(paths) - for path in paths: - dst = ref_sol_dir / (path.as_posix()[l + 1 :]) - dst.parent.mkdir(parents=True, exist_ok=True) - _logging.info(f"Copying {path} -> {_copy(src=path,dst=dst)}") - if not any(paths_list): - raise ValueError( - f"No files found in '{example_dir}' according to {directory} directory rules from EX_GLOB_DICT: {ex_glob_dict[directory]}" - ) - - -def upload_example_input_data_tar( - example_output_data_path: _Union[_Path, str], - bucket: str, - target: str, - dirs: _Union[list, tuple] = ("products", "data", "loading", "solutions"), - compression: str = "bz2", - tag: str = "", - show_progress: bool = False, - push_no_tar: bool = False, -): - __doc__ = ( - "tars selected aux dirs from ginan/inputData and compares their checksums with" - " the ones from s3 bucket. If checksums different - upload, else log info" - " message and do nothing. Default paths are [bucket] s3://peanpod/aux/ ->" - " [html] https://peanpod.s3.ap-southeast-2.amazonaws.com/aux/" - ) - base_url = f"https://{bucket}.s3.ap-southeast-2.amazonaws.com/{target}" - for directory in dirs: - # update tarname with tag - example_type = get_example_type(directory) - if example_type: - directory = "solutions/" + (f"{tag}/" if tag != "" else "") + directory - tarname = directory + ".tar." + compression - destpath_targz = example_output_data_path / tarname - dest_url = base_url + "/" + tarname - if not push_no_tar: - ga.gn_io.common.tar_compress( - srcpath=example_output_data_path / directory, - destpath=destpath_targz, - reset_info=example_type, # reset timestamp etc for examples only - compression=compression, - ) # overwrite only if push_no_tar is False (default) - md5_checksum_aws = ga.gn_download.request_metadata(dest_url) - md5_checksum = ga.gn_io.common.compute_checksum(destpath_targz) - if md5_checksum_aws != md5_checksum: - _logging.info(f'checksums different -> uploading "{tarname}"') - ga.gn_download.upload_with_chunksize_and_meta( - local_file_path=destpath_targz, - metadata={"md5checksum": md5_checksum}, - public_read=True, - bucket_name=bucket, - object_key=target + "/" + tarname, - verbose=show_progress, - ) - else: - _logging.info(f"checksums the same -> skipping upload") - _logging.info(f"------------------------------") - - -def download_example_input_data_tar( - example_input_data_path: _Union[_Path, str], - bucket: str, - target: str, - dirs: _Union[list, tuple] = ("products", "data", "solutions", "loading"), - compression: str = "bz2", - tag: str = "", - skip_extract: bool = False, - tags_file_path: _Union[str, None] = None, # fall back on tags file if no tag was provided -): - __doc__ = ( - "Downloads compressed selected tarballs from ap-southeast-2 s3 bucket, untars" - " them to ginan/inputData dir. If tarball is available locally then checksums" - " are compared at first. If the same - nothing is downloaded, local tarball" - " gets uncompressed" - ) - - base_url = f"https://{bucket}.s3.ap-southeast-2.amazonaws.com/{target}" - if tag == "": - if tags_file_path is None: - raise ValueError("tag not provided and tags_file_path not provided") - _logging.info(f"reading tags from {tags_file_path}") - tag = ga.gn_download.get_vars_from_file(tags_file_path) - else: - _logging.info(f"using the provided {tag} tag") - - for directory in dirs: - example_type = get_example_type(directory) - - if example_type: - dir_url = f"solutions/{(f'{tag[example_type]}/{directory}' if isinstance(tag,dict) else f'{tag}/{directory}')}.tar.{compression}" - directory = f"solutions/{directory}.tar.{compression}" - else: - directory = dir_url = f"{directory}.tar.{compression}" - - destpath_targz = example_input_data_path / directory - dest_url = base_url + "/" + dir_url - md5_checksum_aws = ga.gn_download.request_metadata(dest_url) - destpath = example_input_data_path / directory - if not destpath_targz.exists(): - _logging.info(msg=f"{directory} not found on disk ['{md5_checksum_aws}'].") - destpath_targz.parent.mkdir(parents=True, exist_ok=True) - ga.gn_download.download_url(url=dest_url, destfile=destpath_targz) - else: - _logging.info(msg=f"{directory} found on disk. Validating...") - md5_checksum = ga.gn_io.common.compute_checksum(destpath_targz) - if md5_checksum_aws != md5_checksum: - _logging.info(f'checksums different -> downloading "{dir_url}"') - ga.gn_download.download_url(url=dest_url, destfile=destpath_targz) - else: - _logging.info(f"checksums the same -> skipping download") - - if skip_extract and destpath.exists(): - _logging.info( - "skipping extraction step as '--skip_extract' provided, checksums the same and destination directory exists" - ) - else: - try: - ga.gn_io.common.tar_extract(srcpath=destpath_targz, destpath=destpath) - except Exception as e: - _logging.error(f"could not extract {destpath_targz} to {destpath} due to {e}") - - -@_click.command() -@_click.argument("dirs", nargs=-1, type=str) -@_click.option("--bucket", default="peanpod", help="s3 bucket name to push and pull from") -@_click.option("--target", default="aux", help="s3 target name (dir) within the selected bucket") -@_click.option( - "--path", - default=None, - help="custom path to inputData dir, a dir that stores products/data etc, default is ginan/inputData", -) -@_click.option("--tag", type=str, default="") -@_click.option( - "--skip_extract", - is_flag=True, - show_default=True, - default=False, - help=( - """Skips extraction of the on-disk tar file if checksum test is OK and the - destination dir exists. This is done to save time in the pipeline as - there is no need to overwrite the files.""" - ), -) -@_click.option("-s", "--solutions", is_flag=True, help="download/upload solutions") -@_click.option("-d", "--data", is_flag=True, help="download/upload data") -@_click.option("-p", "--products", is_flag=True, help="download/upload products") -@_click.option("-l", "--loading", is_flag=True, help="download/upload loadings") -@_click.option("--push", is_flag=True, help="tar dirs and push them to aws with checksum metadata and public read") -@_click.option("--push_no_tar", is_flag=True, help="push tar archive which is present on disk") -def download_example_input_data(dirs, bucket, target, path, tag, skip_extract, solutions, data, products, loading, push, push_no_tar): - """Downloads 'products', 'data', and 'solutions' tarballs from s3 bucket and - extracts the content into inputData dir. The list of tarballs can be - changed with the combination of [-p/-d/-s] options. Similar tarballs - upload functionality is available - can be activated with '--push' key - To configure the utility for --push functionality it is enough to create - ~/.aws/credentials file containing - [default] / aws_access_key_id=ACCESS_KEY / - aws_secret_access_key=SECRET_KEY""" - _logging.getLogger().setLevel(_logging.INFO) - script_path = _Path(__file__).resolve().parent - if path is None: - example_input_data_path = (script_path.parent / "inputData").resolve() - _logging.info(f"default input path relative to script location selected: {example_input_data_path}") - else: - example_input_data_path = _Path(path) - _logging.info(f"custom input path selected: {example_input_data_path}") - - if path is None: - example_output_data_path = _Path("./") - _logging.info(f"default output path relative to script location selected: {example_output_data_path}") - else: - example_output_data_path = _Path(path) - _logging.info(f"custom output path selected: {example_output_data_path}") - - if not dirs: - if products: - dirs += ("products",) - if loading: - dirs += ("loading",) - if data: - dirs += ("data",) - if solutions: - dirs += tuple(EX_GLOB_DICT.keys()) - if not dirs: # if nothing has been selected - dirs = ("products", "data") + tuple(EX_GLOB_DICT.keys()) - - _logging.info(f"{dirs} selected") - - if push or push_no_tar: - # copy over the required files if exist - if solutions/blah -> rm blah, copy from ../blah to solutions/blah - _logging.info(msg="updating solutions") - [ - update_solutions_dict(example_input_data_dir=example_output_data_path, directory=directory, ex_glob_dict=EX_GLOB_DICT, tag=tag) - for directory in dirs - ] - upload_example_input_data_tar( - dirs=dirs, - compression="bz2", - tag=tag, - example_output_data_path=example_output_data_path, - bucket=bucket, - target=target, - show_progress=False, - push_no_tar=push_no_tar, - ) - else: - download_example_input_data_tar( - dirs=dirs, - compression="bz2", - tag=tag, - example_input_data_path=example_input_data_path, - skip_extract=skip_extract, - bucket=bucket, - target=target, - tags_file_path=(script_path.parent / "docker" / "tags").as_posix(), - ) - - -if __name__ == "__main__": - download_example_input_data() diff --git a/scripts/formatting/fix_doxygen.py b/scripts/formatting/fix_doxygen.py new file mode 100644 index 000000000..45fa606bd --- /dev/null +++ b/scripts/formatting/fix_doxygen.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import re +import sys +import os + +def fix_doxygen_comments(content): + """ + Fix Doxygen comments that appear after function parameter closing parenthesis + by moving them to the end of the last parameter line. + """ + + # Pattern to match function declarations with trailing Doxygen comments + # This matches the last parameter line followed by closing paren and comment + pattern = r'(\n\s*)([^)\n]+)(\s*\n\s*\)\s*)(///< [^\n\r]*)' + + def replace_func(match): + indent = match.group(1) + last_param = match.group(2) + closing_section = match.group(3) + comment = match.group(4).strip() + + # Add comment to end of last parameter line + return f'{indent}{last_param} {comment}{closing_section}' + + # Apply the replacement + result = re.sub(pattern, replace_func, content) + + return result + +def process_file(filepath): + """Process a single file to fix Doxygen comments.""" + try: + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + + original_content = content + fixed_content = fix_doxygen_comments(content) + + if fixed_content != original_content: + with open(filepath, 'w', encoding='utf-8') as f: + f.write(fixed_content) + print(f"Fixed: {filepath}") + return True + else: + print(f"No changes needed: {filepath}") + return False + + except Exception as e: + print(f"Error processing {filepath}: {e}") + return False + +def main(): + if len(sys.argv) < 2: + print("Usage: python fix_doxygen.py ") + sys.exit(1) + + target = sys.argv[1] + + if os.path.isfile(target): + # Process single file + process_file(target) + elif os.path.isdir(target): + # Process all .cpp and .hpp files in directory recursively + files_processed = 0 + files_changed = 0 + + for root, dirs, files in os.walk(target): + for file in files: + if file.endswith(('.cpp', '.hpp', '.h', '.cc', '.cxx')): + filepath = os.path.join(root, file) + files_processed += 1 + if process_file(filepath): + files_changed += 1 + + print(f"\nSummary: {files_changed} files changed out of {files_processed} processed") + else: + print(f"Error: {target} is not a valid file or directory") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/scripts/formatting/reorganise_include.py b/scripts/formatting/reorganise_include.py new file mode 100644 index 000000000..e96f27ae6 --- /dev/null +++ b/scripts/formatting/reorganise_include.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 + +import re +import sys +import os + +def reorganize_includes_and_using(content): + """ + Reorganize C++ file to have: includes → namespace aliases → using statements. + Handles template aliases correctly. + """ + + # Split content into lines for easier processing + lines = content.split('\n') + + # Lists to collect different types of lines + header_comments = [] + includes = [] + namespace_aliases = [] + using_statements = [] + other_lines = [] + + # Track what section we're in + found_first_include = False + found_first_namespace = False + found_first_using = False + collecting_includes_and_using = True + + i = 0 + while i < len(lines): + line = lines[i].strip() + + # Skip empty lines when collecting + if not line and collecting_includes_and_using: + i += 1 + continue + + # Check for include statements + if line.startswith('#include'): + if not found_first_include: + found_first_include = True + includes.append(lines[i]) # Keep original formatting + i += 1 + continue + + # Check for namespace aliases (namespace alias = existing::namespace) + if line.startswith('namespace ') and '=' in line: + if not found_first_namespace: + found_first_namespace = True + namespace_aliases.append(lines[i]) # Keep original formatting + i += 1 + continue + + # Check for using statements (but not template aliases) + if line.startswith('using '): + # Check if this is a template alias by looking for 'template' on same or previous lines + is_template_alias = False + + # Check current line for template + if 'template' in line: + is_template_alias = True + + # Check if previous non-empty line contains template + j = i - 1 + while j >= 0 and not lines[j].strip(): + j -= 1 + if j >= 0 and 'template' in lines[j]: + is_template_alias = True + + if not is_template_alias: + if not found_first_using: + found_first_using = True + using_statements.append(lines[i]) # Keep original formatting + i += 1 + continue + + # If we haven't found includes/namespace/using yet, it's probably header comments + if not found_first_include and not found_first_namespace and not found_first_using: + header_comments.append(lines[i]) + i += 1 + continue + + # If we've found includes/namespace/using but this line is neither, we're done collecting + if found_first_include or found_first_namespace or found_first_using: + collecting_includes_and_using = False + + # Everything else goes to other_lines + other_lines.append(lines[i]) + i += 1 + + # Reconstruct the file + result_lines = [] + + # Add header comments + result_lines.extend(header_comments) + + # Add a blank line if we have header comments and includes + if header_comments and includes: + result_lines.append('') + + # Add all includes + result_lines.extend(includes) + + # Add a blank line between includes and namespace aliases + if includes and namespace_aliases: + result_lines.append('') + + # Add all namespace aliases + result_lines.extend(namespace_aliases) + + # Add a blank line between namespace aliases and using statements + if namespace_aliases and using_statements: + result_lines.append('') + # Add a blank line between includes and using statements when no namespace aliases + elif includes and using_statements and not namespace_aliases: + result_lines.append('') + # Add all using statements + result_lines.extend(using_statements) + + # Add a blank line between using statements and other code + if using_statements and other_lines: + result_lines.append('') + + # Add the rest of the code + result_lines.extend(other_lines) + + return '\n'.join(result_lines) + +def process_file(filepath): + """Process a single file to reorganize includes and using statements.""" + try: + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + + original_content = content + reorganized_content = reorganize_includes_and_using(content) + + if reorganized_content != original_content: + with open(filepath, 'w', encoding='utf-8') as f: + f.write(reorganized_content) + print(f"Reorganized: {filepath}") + return True + else: + print(f"No changes needed: {filepath}") + return False + + except Exception as e: + print(f"Error processing {filepath}: {e}") + return False + +def main(): + if len(sys.argv) < 2: + print("Usage: python reorganize_includes.py ") + sys.exit(1) + + target = sys.argv[1] + + if os.path.isfile(target): + # Process single file + process_file(target) + elif os.path.isdir(target): + # Process all .cpp and .hpp files in directory recursively + files_processed = 0 + files_changed = 0 + + for root, dirs, files in os.walk(target): + for file in files: + if file.endswith(('.cpp', '.hpp', '.h', '.cc', '.cxx')): + filepath = os.path.join(root, file) + files_processed += 1 + if process_file(filepath): + files_changed += 1 + + print(f"\nSummary: {files_changed} files changed out of {files_processed} processed") + else: + print(f"Error: {target} is not a valid file or directory") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/gn/templates/auto_template.yaml b/scripts/gn/templates/auto_template.yaml index 30d8adae0..ebc9f906a 100644 --- a/scripts/gn/templates/auto_template.yaml +++ b/scripts/gn/templates/auto_template.yaml @@ -50,7 +50,7 @@ outputs: output_config: true sinex: {filename: .SNX, output: true} - + # log: # output: false # directory: ./logs/ @@ -88,14 +88,14 @@ receiver_options: troposphere: # Tropospheric modelling accounts for delays due to refraction of light in water vapour enable: true models: [ gpt2 ] # List of models to use for troposphere [standard,sbas,vmf3,gpt2,cssr] - tides: + tides: atl: true # Enable atmospheric tide loading enable: true # Enable modelling of tidal disaplacements opole: true # Enable ocean pole tides otl: true # Enable ocean tide loading solid: true # Enable solid Earth tides spole: true # Enable solid Earth pole tides - + # ALIC: # receiver_type: "LEICA GR25" # (string) @@ -168,12 +168,13 @@ processing_options: mw_process_noise: 0 # Process noise applied to filtered Melbourne-Wubenna measurements to detect cycle slips slip_threshold: 0.05 # Value used to determine when a slip has occurred preprocess_all_data: true - + spp: # always_reinitialise: false # Reset SPP state to zero to avoid potential for lock-in of bad states max_lsq_iterations: 12 # Maximum number of iterations of least squares allowed for convergence outlier_screening: - raim: true # Enable Receiver Autonomous Integrity Monitoring + raim: + enable: true # Enable Receiver Autonomous Integrity Monitoring max_gdop: 30 # Maximum dilution of precision before error is flagged ppp_filter: @@ -199,7 +200,7 @@ processing_options: use_gf_combo: false # Combine 'uncombined' measurements to simulate a geometry-free solution use_if_combo: false # Combine 'uncombined' measurements to simulate an ionosphere-free solution - chunking: + chunking: by_receiver: false # Split large filter and measurement matrices blockwise by receiver ID to improve processing speed by_satellite: false # Split large filter and measurement matrices blockwise by satellite ID to improve processing speed size: 0 @@ -207,7 +208,6 @@ processing_options: rts: # Rauch-Tung-Striebel (RTS) backwards smoothing enable: true lag: -1 - inverter: LDLT # Inverter to be used within the rts processor, which may provide different performance outcomes in terms of processing time and accuracy and stability filename: _.rts model_error_handling: diff --git a/scripts/installation/apple.md b/scripts/installation/apple.md index 5aa7da0ed..b9e035fce 100644 --- a/scripts/installation/apple.md +++ b/scripts/installation/apple.md @@ -1,9 +1,52 @@ # Installation procedure on Apple -Tested on Macbook Pro (Intel) with Somona OSX. +Tested on Macbook Pro (Intel) with Somona OSX and Macbook Pro (ARM64) with Sonoma OSX -After installation of brew, install the following packages using brew +## Install Ginan dependencies + +After installation of homebrew, install the following packages using brew ```bash brew install boost cmake eigen netcdf-cxx netcdf mongo-c-driver mongo-cxx-driver openblas openssl@3 yaml-cpp libomp +``` +*** + +Follow the instructions here to install the MongoDB application: +https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-os-x/` + +## Install gnssanalysis python module + +``` +pip3 install gnssanalysis +``` + +## Download Ginan from Github + +You can download Ginan source from github using git clone: + +``` +#git clone https://github.com/GeoscienceAustralia/ginan.git +git clone -b develop-weekly --depth 1 --single-branch https://github.com/GeoscienceAustralia/ginan.git + +cd ginan +cd src +mkdir build +cd build +cmake -DCMAKE_TOOLCHAIN_FILE=compile_mac_arm64.cmake .. +make -j4 pea + +cd ../.. +./bin/pea --help +``` + +## Download Demo data and products + +Then download all of the example data using the python script provided (requires `gnssanalysis`): + +``` +cd inputData +cd products +getProducts.sh +cd ../data +getData.sh ``` \ No newline at end of file diff --git a/scripts/installation/generic.md b/scripts/installation/generic.md index 480537c04..60baeafdc 100644 --- a/scripts/installation/generic.md +++ b/scripts/installation/generic.md @@ -8,17 +8,14 @@ If instead you wish to build Ginan from source, there are several software depen * BLAS and LAPACK linear algebra libraries. We use and recommend [OpenBlas](https://www.openblas.net/) as this contains both libraries required * CMAKE > 3.0 * YAML > 0.6 -* Boost >= 1.73 (tested on 1.73). On Ubuntu 22.04 which uses gcc-11, you need Boost >= 1.74.0 +* Boost >= 1.74 * MongoDB -* Mongo_C >= 1.71.1 -* Mongo_cxx >= 3.6.0 +* Mongo_C >= 1.71.1 (automatically installed with mongo-cxx-driver) +* Mongo_cxx >= 3.9.0 * Eigen3 > 3.4 * netCDF4 * Python >= 3.7 -If using gcc verion 11 or about, the minimum version of libraries are: -* Boost >= 1.74.0 -* Mongo_cxx = 3.7.0 *** ## Installing dependencies (Example with Ubuntu) @@ -37,25 +34,11 @@ sudo apt install -y git gobjc gobjc++ gfortran libopenblas-dev openssl curl net- sudo -H pip3 install wheel pandas boto3 unlzw tdqm scipy gnssanalysis ``` -Ginan requires at least version 9 of both gcc and g++, so make sure to update the gcc/g++ alternatives prior to compilation: -(this is not required on Ubuntu 22.04) - -``` -sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - -sudo apt update - -sudo apt install -y gcc-9 g++-9 - -sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 51 - -sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 51 -``` *** ## Building additional dependencies -Depending on the user's installation choice: install PEA-only, POD-only or all software packages, a set of additional dependencies that need to be built may change. Below, we explain building all the additional dependencies: +Depending on the user's installation choice: install PEA-only, or all software packages, a set of additional dependencies that need to be built may change. Below, we explain building all the additional dependencies: Note that many `make` commands here have the option `-j 2` applied, this will enable parallel compilation and may speed up installation time. The number of threads can be increased by changing the number, such as `-j 8`, but be aware that each new thread may require up to 2GB of memory. @@ -90,16 +73,16 @@ rm -rf yaml-cpp ### Boost (PEA) PEA relies on a number of the utilities provided by [boost](https://www.boost.org/), such as their time and logging libraries. -NB for compilation using gcc-11, you need to change this to boost_1_74_0 + ``` cd $dir/tmp -wget -c https://boostorg.jfrog.io/artifactory/main/release/1.73.0/source/boost_1_73_0.tar.gz +wget -c https://archives.boost.io/release/1.83.0/source/boost_1_83_0.tar.gz -tar -xf boost_1_73_0.tar.gz +tar -xf boost_1_83_0.tar.gz -cd boost_1_73_0/ +cd boost_1_83_0/ ./bootstrap.sh @@ -107,7 +90,7 @@ sudo ./b2 -j2 install cd $dir/tmp -sudo rm -rf boost_1_73_0 boost_1_73_0.tar.gz +sudo rm -rf boost_1_83_0 boost_1_83_0.tar.gz ``` ### Eigen3 (PEA) @@ -137,49 +120,28 @@ rm -rf eigen ### Mongo_cxx_driver (PEA) -Needed for json formatting and other self-descriptive markup. - -``` -cd $dir/tmp -# NB for compilation using gcc-11, you need to change this to 1.21.2 +Needed for connection to the MongoDB database, which is used for realtime plotting and statistics in `GinanEDA`. -wget https://github.com/mongodb/mongo-c-driver/releases/download/1.17.1/mongo-c-driver-1.17.1.tar.gz - -tar -xf mongo-c-driver-1.17.1.tar.gz - -cd mongo-c-driver-1.17.1/ - -mkdir cmake-build - -cd cmake-build/ - -cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_EXAMPLES=OFF ../ - -cmake --build . -- -j 2 - -sudo cmake --build . --target install -- -j 2 +Note: for later version of mongo-cxx-driver install (3.9.0 and above) it also the mongo-c driver, so it is not needed to install it separately. +``` cd $dir/tmp -NB for compilation using gcc-11, you need to change this to 3.7.0 - -curl -OL https://github.com/mongodb/mongo-cxx-driver/releases/download/r3.6.0/mongo-cxx-driver-r3.6.0.tar.gz - -tar -xf mongo-cxx-driver-r3.6.0.tar.gz +curl -OL https://github.com/mongodb/mongo-cxx-driver/releases/download/r3.11.0/mongo-cxx-driver-r3.11.0.tar.gz -cd mongo-cxx-driver-r3.6.0/build +tar -xf mongo-cxx-driver-r3.11.0.tar.gz -cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_EXAMPLES=OFF ../ +cd mongo-cxx-driver-r3.11.0/build -sudo cmake --build . --target EP_mnmlstc_core -- -j 2 +cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -cmake --build . -- -j 2 +make -j 1 -sudo cmake --build . --target install +sudo make install cd $dir/tmp -sudo rm -rf mongo-c-driver-1.17.1 mongo-c-driver-1.17.1.tar.gz mongo-cxx-driver-r3.6.0 mongo-cxx-driver-r3.6.0.tar.gz +sudo rm -rf mongo-cxx-driver-r3.11.0 mongo-cxx-driver-r3.11.0.tar.gz ``` ### MongoDB (PEA, optional) diff --git a/scripts/installation/ubuntu20.sh b/scripts/installation/ubuntu20.sh index b423d9f89..ad7b67a66 100755 --- a/scripts/installation/ubuntu20.sh +++ b/scripts/installation/ubuntu20.sh @@ -26,7 +26,7 @@ if [[ "$ubuntu_version" != "20.04" && "$ubuntu_version" != "18.04" ]]; then fi # MongoDB library version numbers -mongo_cxx_driver_version="r3.6.7" +mongo_cxx_driver_version="r3.11.0" echo "Updating package repositories..." $sudo_cmd apt update -y @@ -79,25 +79,17 @@ $sudo_cmd make -j2 install cd /tmp -echo "Downloading, extracting, building, and installing Mongo drivers..." -cd /tmp -wget https://github.com/mongodb/mongo-c-driver/releases/download/1.17.1/mongo-c-driver-1.17.1.tar.gz -tar -xf mongo-c-driver-1.17.1.tar.gz -cd mongo-c-driver-1.17.1/ -mkdir cmake-build -cd cmake-build/ -cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_EXAMPLES=OFF ../ -cmake --build . -- -j 2 -$sudo_cmd cmake --build . --target install -- -j 2 -cd /tmp -curl -OL https://github.com/mongodb/mongo-cxx-driver/releases/download/r3.6.0/mongo-cxx-driver-r3.6.0.tar.gz -tar -xf mongo-cxx-driver-r3.6.0.tar.gz -cd mongo-cxx-driver-r3.6.0/build -cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_EXAMPLES=OFF ../ -sudo cmake --build . --target EP_mnmlstc_core -- -j 2 -$sudo_cmd cmake --build . -- -j 2 -$sudo_cmd cmake --build . --target install -cd /tmp +echo "Downloading and extracting mongo-cxx-driver version $mongo_cxx_driver_version..." +wget --no-check-certificate https://github.com/mongodb/mongo-cxx-driver/releases/download/$mongo_cxx_driver_version/mongo-cxx-driver-$mongo_cxx_driver_version.tar.gz +tar -xzf mongo-cxx-driver-$mongo_cxx_driver_version.tar.gz +rm mongo-cxx-driver-$mongo_cxx_driver_version.tar.gz + +echo "Building and installing mongo-cxx-driver..." +mkdir -p mongo-cxx-driver-$mongo_cxx_driver_version/build +cd mongo-cxx-driver-$mongo_cxx_driver_version/build +cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local +$sudo_cmd make -j 1 +$sudo_cmd make install echo "Downloading, extracting, building, and installing MongoDB ..." cd /tmp diff --git a/scripts/installation/ubuntu22.sh b/scripts/installation/ubuntu22.sh index dd4d176c5..302170099 100755 --- a/scripts/installation/ubuntu22.sh +++ b/scripts/installation/ubuntu22.sh @@ -26,7 +26,7 @@ if [[ "$ubuntu_version" != "22.04" ]]; then fi # MongoDB library version numbers -mongo_cxx_driver_version="r3.6.7" +mongo_cxx_driver_version="r3.11.0" echo "Updating package repositories..." $sudo_cmd apt update -y diff --git a/scripts/installation/ubuntu24.sh b/scripts/installation/ubuntu24.sh new file mode 100755 index 000000000..c5753669e --- /dev/null +++ b/scripts/installation/ubuntu24.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +set -e # Exit immediately if any command fails + +# Check if sudo is available +sudo_cmd="sudo" +if ! command -v sudo >/dev/null 2>&1; then + sudo_cmd="" +fi + +# Check if the system is Ubuntu +if [[ ! -f /etc/os-release ]] || ! grep -iq "ubuntu" /etc/os-release; then + echo "This script is designed for Ubuntu. Please run it on an Ubuntu system." + exit 1 +fi + +# Check if the system is Ubuntu 22.04 +ubuntu_version=$(grep -oP '(?<=^VERSION_ID=")\d{2}\.\d{2}(?="$)' /etc/os-release) +if [[ "$ubuntu_version" != "24.04" ]]; then + echo "This script is designed for Ubuntu 24.04. Do you want to continue? (y/n)" + read -r response + if [[ ! $response =~ ^[Yy]$ ]]; then + echo "Script execution aborted." + exit 0 + fi +fi + +# MongoDB library version numbers +mongo_cxx_driver_version="r3.11.0" + +echo "Updating package repositories..." +$sudo_cmd apt update -y + +echo "Installing dependencies..." +$sudo_cmd apt-get install --no-install-recommends --yes \ + libgomp1 \ + gcc \ + g++ \ + gdb \ + gfortran \ + openssl \ + curl \ + net-tools \ + wget \ + openssh-server \ + apt-transport-https \ + ca-certificates \ + libopenblas0 \ + libnetcdf-c++4-1 \ + gzip \ + gnupg2 \ + git \ + cmake \ + make \ + libssl-dev \ + libboost-all-dev \ + libeigen3-dev \ + libyaml-cpp-dev \ + libnetcdf-dev \ + libnetcdf-c++4-dev \ + libzstd-dev \ + libssl-dev \ + libncurses5-dev \ + libopenblas-dev \ + python3-pip + + +echo "Creating build directory..." +mkdir -p /tmp/build +cd /tmp/build + +echo "Downloading and extracting mongo-cxx-driver version $mongo_cxx_driver_version..." +wget --no-check-certificate https://github.com/mongodb/mongo-cxx-driver/releases/download/$mongo_cxx_driver_version/mongo-cxx-driver-$mongo_cxx_driver_version.tar.gz +tar -xzf mongo-cxx-driver-$mongo_cxx_driver_version.tar.gz +rm mongo-cxx-driver-$mongo_cxx_driver_version.tar.gz + +echo "Building and installing mongo-cxx-driver..." +mkdir -p mongo-cxx-driver-$mongo_cxx_driver_version/build +cd mongo-cxx-driver-$mongo_cxx_driver_version/build +cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local +$sudo_cmd make -j 1 +$sudo_cmd make install + +echo "Installation of Mongodb" +curl -fsSL https://pgp.mongodb.com/server-7.0.asc | $sudo_cmd gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor +echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | $sudo_cmd tee /etc/apt/sources.list.d/mongodb-org-7.0.list +$sudo_cmd apt update +$sudo_cmd apt-get install -y mongodb-org + +$sudo_cmd echo "/usr/local/lib" > /etc/ld.so.conf.d/usr-local-lib.conf +$sudo_cmd chown root:root /etc/ld.so.conf.d/usr-local-lib.conf +$sudo_cmd chmod 644 /etc/ld.so.conf.d/usr-local-lib.conf +$sudo_cmd ldconfig + +echo "Installation completed" diff --git a/scripts/plot_pos.py b/scripts/plot_pos.py index 4ae77b862..bd28f1b47 100644 --- a/scripts/plot_pos.py +++ b/scripts/plot_pos.py @@ -1,3 +1,5 @@ +from pathlib import Path +import os import pandas as pd from datetime import datetime import plotly.graph_objects as go @@ -6,6 +8,16 @@ import argparse def parse_pos_format(file_path): + """ + Parse a .POS file into a pandas DataFrame. + + Arguments: + file_path (str): Path to a .POS file with recordings following the expected format. + + Returns: + pandas.DataFrame: Table with columns such as 'Time', 'Latitude', 'Longitude', + 'Elevation', 'dN', 'dE', 'dU', 'sN', 'sE', 'sU', 'sElevation', 'Rne', 'Rnu', 'Reu', 'soln'. + """ data = [] try: with open(file_path, 'r') as file: @@ -36,11 +48,23 @@ def parse_pos_format(file_path): } data.append(record) except Exception as e: - print(f"Error parsing file {file_path}: {e}") + print(f"Error parsing file {file_path}: {e}") return pd.DataFrame(data) # Function to parse the datetime with optional timezone def parse_datetime(datetime_str): + """ + Parse a datetime string with or without timezone into a naive datetime. + + Arguments: + datetime_str (str): Datetime string, e.g., 'YYYY-MM-DDTHH:MM:SS' or 'YYYY-MM-DDTHH:MM:SS±HHMM'. + + Returns: + datetime: Timezone-naive datetime object. If timezone was present, it is stripped. + + Raises: + ValueError: If datetime string doesn't match expected formats. + """ # Attempt to parse datetime with and without timezone for fmt in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S"): try: @@ -52,9 +76,18 @@ def parse_datetime(datetime_str): except ValueError as e: print(f"ValueError: {e}") continue - raise ValueError(f"datetime {datetime_str} does not match expected formats.") + raise ValueError(f"datetime {datetime_str} does not match expected formats.") def remove_weighted_mean(data): + """ + Remove weighted mean from each component series in-place and return the DataFrame. + + Arguments: + data (pandas.DataFrame): Data with components ('dN','dE','dU','Elevation') and their sigmas ('sN','sE','sU','sElevation'). + + Returns: + pandas.DataFrame: The same DataFrame with each component demeaned by its weighted mean. + """ sigma_keys = {'dN': 'sN', 'dE': 'sE', 'dU': 'sU', 'Elevation': 'sElevation'} # Assume sElevation exists for component in ['dN', 'dE', 'dU', 'Elevation']: sigma_key = sigma_keys[component] @@ -63,12 +96,23 @@ def remove_weighted_mean(data): data[component] -= weighted_mean # Demean the series return data -def apply_smoothing(data): +def apply_smoothing(data, horz_smoothing=None, vert_smoothing=None): + """ + Apply LOWESS smoothing to horizontal and / or vertical components. + + Arguments: + data (pandas.DataFrame): Input data with time column 'Time' and components. + horz_smoothing (float or None): Fraction for LOWESS on dN/dE (0..1), or None to skip. + vert_smoothing (float or None): Fraction for LOWESS on dU/Elevation (0..1), or None to skip. + + Returns: + pandas.DataFrame: DataFrame with additional 'Smoothed_*' columns when smoothing is applied. + """ for component in ['dN', 'dE', 'dU', 'Elevation']: - if args.horz_smoothing and (component == 'dN' or component == 'dE'): - data[f'Smoothed_{component}'] = lowess(data[component], data['Time'], frac=args.horz_smoothing, return_sorted=False) - if args.vert_smoothing and (component == 'dU' or component == 'Elevation'): - data[f'Smoothed_{component}'] = lowess(data[component], data['Time'], frac=args.vert_smoothing, return_sorted=False) + if horz_smoothing and (component == 'dN' or component == 'dE'): + data[f'Smoothed_{component}'] = lowess(data[component], data['Time'], frac=horz_smoothing, return_sorted=False) + if vert_smoothing and (component == 'dU' or component == 'Elevation'): + data[f'Smoothed_{component}'] = lowess(data[component], data['Time'], frac=vert_smoothing, return_sorted=False) return data def compute_statistics(data): @@ -95,395 +139,521 @@ def compute_statistics(data): return data, stats +def create_plots(all_data, input_files, component_stats, args, show_plots=True): + """ + Create interactive HTML plots for POS analysis. + + Arguments: + all_data (pandas.DataFrame): Measurement table with columns such as 'Time', 'dN', 'dE', 'dU' or 'Elevation', and their sigmas. + input_files (list): One or more .POS filepaths + component_stats (dict): Statistics returned by compute_statistics(). + args (object): Arguments used to call plot_pos like colour_sigma, max_sigma, elevation, map, heatmap, save_prefix. + show_plots (bool): If True, display plots in browser (CLI mode), if False, only save to files (program mode). + + Returns: + None: Writes HTML files when args.save_prefix is provided. + """ + input_root = Path(input_files[0]).stem + + # Start plotting + ## Fig1 + # Determine max sigma and color scale settings for Fig1 + title_text = f"Time Series Analysis: {', '.join(input_files)}
    " + color_scale = 'Jet' if args.colour_sigma else None # Only set color scale if --colour_sigma is active + max_sigma_data = np.max([all_data['sN'].max(), all_data['sE'].max(), all_data['sU'].max()]) + min_sigma_data = np.min([all_data['sN'].min(), all_data['sE'].min(), all_data['sU'].min()]) + cmax = min(args.max_sigma, max_sigma_data) if args.max_sigma is not None else max_sigma_data + # cmin = min_sigma_data + cmin = 0.0 + + # Setting up the plot + fig1 = go.Figure() + components = ['dN', 'dE', 'Elevation'] if args.elevation else ['dN', 'dE', 'dU'] + component_colors = { + 'dN': 'red', + 'dE': 'green', + 'dU': 'blue', + 'Elevation': 'orange' + } + + for component in components: + # Correctly map the component to its sigma key + if component == 'Elevation': + sigma_key = 'sU' # Assuming sigma for Elevation is stored in 'sU' + else: + sigma_key = f's{component[-1].upper()}' + + print('Plotting: ', sigma_key) # To check if the correct sigma key is being used + + # Add the primary and smoothed series data + if args.colour_sigma: + # When using --colour_sigma, use the sigma value for coloring + fig1.add_trace(go.Scatter( + x=all_data['Time'], y=all_data[component], + mode='lines+markers', + marker=dict(size=5, color=all_data[sigma_key], coloraxis="coloraxis"), + name=component, + hoverinfo='text+x+y', + text=f'{component} Sigma: ' + all_data[sigma_key].astype(str) + )) -# Setup and parse arguments -parser = argparse.ArgumentParser(description="Plot positional data with optional smoothing and color coding.") -parser.add_argument('--start-datetime', type=str, - help="Start datetime in the format YYYY-MM-DDTHH:MM:SS, optional timezone") -parser.add_argument('--end-datetime', type=str, - help="End datetime in the format YYYY-MM-DDTHH:MM:SS, optional timezone") -parser.add_argument('--horz_smoothing', type=float, default=None, - help='Fraction of the data used for horizontal (East and North) LOWESS smoothing (optional).') -parser.add_argument('--vert_smoothing', type=float, default=None, - help='Fraction of the data used for vertical (Up) LOWESS smoothing (optional).') -parser.add_argument('--colour_sigma', action='store_true', - help='Colourize the timeseries using the standard deviation (sigma) values (optional).') -parser.add_argument('--max_sigma', type=float, default=None, - help='Set a maximum sigma threshold for the sigma colour scale (optional).') -parser.add_argument('--elevation', action='store_true', - help='Plot Elevation values inplace of dU wrt the reference coord (optional).') -parser.add_argument('--demean', action='store_true', - help='Remove the mean values from all time series before plotting (optional).') -parser.add_argument('--map', action='store_true', - help='Create a geographic map view from the Longitude & Latitude estiamtes (optional).') -parser.add_argument('--heatmap', action='store_true', - help='Create a 2D heatmap view of E & N coodrinates wrt the reference position (optional).') -parser.add_argument('--sigma_threshold', nargs=3, type=float, - help="Thresholds for sE, sN, and sU to filter data.") -parser.add_argument('--down_sample', type=int, - help="Interval in seconds for down-sampling data.") -parser.add_argument('--save', action='store_true', - help='Save requested plots as .html format files (optional).') -parser.add_argument('files', nargs='+') -args = parser.parse_args() - -# Parse the start and end datetime if provided -start_datetime = parse_datetime(args.start_datetime) if args.start_datetime else None -end_datetime = parse_datetime(args.end_datetime) if args.end_datetime else None - -# Load and process data -all_data = pd.DataFrame() -for file_path in args.files: - file_data = parse_pos_format(file_path) - all_data = pd.concat([all_data, file_data], ignore_index=True) - -all_data['Time'] = pd.to_datetime(all_data['Time'], format="%Y-%m-%dT%H:%M:%S.%f") - -# Apply time windowing -if start_datetime: - all_data = all_data[all_data['Time'] >= start_datetime] -if end_datetime: - all_data = all_data[all_data['Time'] <= end_datetime] - -# Apply threshold filtering if sigma_threshold is provided -if args.sigma_threshold: - se_threshold, sn_threshold, su_threshold = args.sigma_threshold - mask = (all_data['sE'] <= se_threshold) & (all_data['sN'] <= sn_threshold) & (all_data['sU'] <= su_threshold) & (all_data['sElevation'] <= su_threshold) - all_data = all_data[mask] - -# Down-sample the data if requested -if args.down_sample: - # Ensure the 'Time' column is datetime for proper indexing - all_data['Time'] = pd.to_datetime(all_data['Time']) - all_data.set_index('Time', inplace=True) - # Resample and take the first available data point in each bin - all_data = all_data.resample(f'{args.down_sample}s').first().dropna().reset_index() - -# Demean, smooth, and compute statistics -if args.demean: - all_data = remove_weighted_mean(all_data) -all_data = apply_smoothing(all_data) -all_data, component_stats = compute_statistics(all_data) - -# Start plotting -# Determine max sigma and color scale settings for Fig1 -title_text = f"Time Series Analysis: {', '.join(args.files)}
    " -color_scale = 'Jet' if args.colour_sigma else None # Only set color scale if --colour_sigma is active -max_sigma_data = np.max([all_data['sN'].max(), all_data['sE'].max(), all_data['sU'].max()]) -min_sigma_data = np.min([all_data['sN'].min(), all_data['sE'].min(), all_data['sU'].min()]) -cmax = min(args.max_sigma, max_sigma_data) if args.max_sigma is not None else max_sigma_data -#cmin = min_sigma_data -cmin = 0.0 - -# Setting up the plot -fig1 = go.Figure() -components = ['dN', 'dE', 'Elevation'] if args.elevation else ['dN', 'dE', 'dU'] - -for component in components: - # Correctly map the component to its sigma key - if component == 'Elevation': - sigma_key = 'sU' # Assuming sigma for Elevation is stored in 'sU' - else: - sigma_key = f's{component[-1].upper()}' - - print('Plotting: ', sigma_key) # To check if the correct sigma key is being used - - # Add the primary and smoothed series data - if args.colour_sigma: - # When using --colour_sigma, use the sigma value for coloring - fig1.add_trace(go.Scatter( - x=all_data['Time'], y=all_data[component], - mode='lines+markers', - marker=dict(size=5, color=all_data[sigma_key], coloraxis="coloraxis"), - name=component, - hoverinfo='text+x+y', - text=f'{component} Sigma: ' + all_data[sigma_key].astype(str) - )) - - else: - # When not using --colour_sigma, add error bars using the sigma values + else: + # When not using --colour_sigma, add error bars using the sigma values + fig1.add_trace(go.Scatter( + x=all_data['Time'], y=all_data[component], + mode='markers', + name=component, + error_y=dict( + type='data', # Represent error in data coordinates + array=all_data[sigma_key], # Positive error + arrayminus=all_data[sigma_key], # Negative error + visible=True, # Make error bars visible + color='gray' # Color of error bars + ), + marker=dict(size=5, color=component_colors[component]), + line=dict(color=component_colors[component]), + hoverinfo='text+x+y', + text=f'{component} Sigma: ' + all_data[sigma_key].astype(str) + )) + + if f'Smoothed_{component}' in all_data: + fig1.add_trace(go.Scatter( + x=all_data['Time'], y=all_data[f'Smoothed_{component}'], + mode='lines', + name=f'Smoothed {component}', + line=dict(color='rgba(0,0,255,0.5)') + )) + + # Add statistical lines and shaded areas for standard deviation fig1.add_trace(go.Scatter( - x=all_data['Time'], y=all_data[component], - mode='markers', - name=component, - error_y=dict( - type='data', # Represent error in data coordinates - array=all_data[sigma_key], # Positive error - arrayminus=all_data[sigma_key], # Negative error - visible=True, # Make error bars visible - color='gray' # Color of error bars - ), - marker=dict(size=5, color='blue'), - line=dict(color='blue'), - hoverinfo='text+x+y', - text=f'{component} Sigma: ' + all_data[sigma_key].astype(str) + x=all_data['Time'], y=all_data[f'{component}_weighted_mean'], + mode='lines', + name=f'{component} Weighted Mean', + line=dict(color=component_colors[component]) )) - if f'Smoothed_{component}' in all_data: fig1.add_trace(go.Scatter( - x=all_data['Time'], y=all_data[f'Smoothed_{component}'], - mode='lines', - name=f'Smoothed {component}', - line=dict(color='rgba(0,0,255,0.5)') + x=all_data['Time'].tolist() + all_data['Time'].tolist()[::-1], + y=all_data[f'{component}_std_dev_upper'].tolist() + all_data[f'{component}_std_dev_lower'].tolist()[::-1], + fill='toself', + fillcolor='rgba(68, 68, 255, 0.2)', + line=dict(color='rgba(255,255,255,0)'), + name=f'{component} CI: 2 Sigma (95%)' )) - # Add statistical lines and shaded areas for standard deviation - fig1.add_trace(go.Scatter( - x=all_data['Time'], y=all_data[f'{component}_weighted_mean'], - mode='lines', - name=f'{component} Weighted Mean', - line=dict(color='red') - )) + stats = component_stats[component] + title_text += f"{component}: Weighted Mean = {stats['weighted_mean']:.3f}, Std Dev = {stats['std_dev']:.3f}, RMS = {stats['rms']:.3f}
    " + + fig1.update_layout( + title=title_text, + xaxis_title='Time', + yaxis_title='Measurement Value', + xaxis=dict( + rangeslider=dict(visible=True), + fixedrange=False, + type='date' + ), + yaxis=dict( + fixedrange=False + ), + coloraxis=dict( + colorscale=color_scale, + cmin=cmin, + cmax=cmax, + colorbar=dict( + title='Sigma Value', + x=0.5, # Center the color bar on the x-axis + y=-0.5, # Position the color bar below the x-axis + xanchor='center', # Anchor the color bar at its center for x positioning + yanchor='bottom', # Anchor the color bar from its bottom edge for y positioning + len=0.5, # Length of the color bar (75% of the width of the plot area) + thickness=10, # Thickness of the color bar + orientation='h' # Horizontal orientation + ), + ) if args.colour_sigma else {}, + showlegend=True, + margin=dict(t=150) + ) - fig1.add_trace(go.Scatter( - x=all_data['Time'].tolist() + all_data['Time'].tolist()[::-1], - y=all_data[f'{component}_std_dev_upper'].tolist() + all_data[f'{component}_std_dev_lower'].tolist()[::-1], - fill='toself', - fillcolor='rgba(68, 68, 255, 0.2)', - line=dict(color='rgba(255,255,255,0)'), - name=f'{component} CI: 2 Sigma (95%)' - )) + if show_plots: + fig1.show() + + if args.save_prefix is not None: + output_path = os.path.join(os.path.dirname(args.save_prefix), f"{input_root}_fig1.html") + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + fig1.write_html(output_path) + + ## Fig2 + # Build the title with file names and statistics for Fig2 + title_text = f"dN vs dE Analysis: {', '.join(input_files)}
    " + for component in ['dN', 'dE']: + stats = component_stats[component] + title_text += f"{component}: Weighted Mean = {stats['weighted_mean']:.3f}, Std Dev = {stats['std_dev']:.3f}, RMS = {stats['rms']:.3f}
    " + + # Conditional sigma calculations and setup + composite_uncertainty = np.sqrt(all_data['sN'] ** 2 + all_data['sE'] ** 2) + all_data['composite_uncertainty'] = composite_uncertainty + max_sigma_data = composite_uncertainty.max() + if args.colour_sigma: + cmax = min(args.max_sigma, max_sigma_data) if args.max_sigma is not None else max_sigma_data + cmin = composite_uncertainty.min() + # cmin = 0.0 + color_scale = 'Jet' # Define the color scale here within the condition + else: + cmin = None # No cmax needed for static colors + cmax = None # No cmax needed for static colors + color_scale = None # No color scale needed for static colors - stats = component_stats[component] - title_text += f"{component}: Weighted Mean = {stats['weighted_mean']:.3f}, Std Dev = {stats['std_dev']:.3f}, RMS = {stats['rms']:.3f}
    " - -fig1.update_layout( - title=title_text, - xaxis_title='Time', - yaxis_title='Measurement Value', - xaxis=dict( - rangeslider=dict(visible=True), - fixedrange=False, - type='date' - ), - yaxis=dict( - fixedrange=False - ), - coloraxis=dict( - colorscale=color_scale, - cmin=cmin, - cmax=cmax, - colorbar=dict( - title='Sigma Value', - x=0.5, # Center the color bar on the x-axis - y=-0.5, # Position the color bar below the x-axis - xanchor='center', # Anchor the color bar at its center for x positioning - yanchor='bottom', # Anchor the color bar from its bottom edge for y positioning - len=0.5, # Length of the color bar (75% of the width of the plot area) - thickness=10, # Thickness of the color bar - orientation='h' # Horizontal orientation - ), - ) if args.colour_sigma else {}, - showlegend=True, - margin=dict(t=150) -) -fig1.show() -if args.save: - fig1.write_html("fig1.html") - -# Build the title with file names and statistics for Fig2 -title_text = f"dN vs dE Analysis: {', '.join(args.files)}
    " -for component in ['dN', 'dE']: - stats = component_stats[component] - title_text += f"{component}: Weighted Mean = {stats['weighted_mean']:.3f}, Std Dev = {stats['std_dev']:.3f}, RMS = {stats['rms']:.3f}
    " - -# Conditional sigma calculations and setup -composite_uncertainty = np.sqrt(all_data['sN']**2 + all_data['sE']**2) -all_data['composite_uncertainty'] = composite_uncertainty -max_sigma_data = composite_uncertainty.max() -if args.colour_sigma: - cmax = min(args.max_sigma, max_sigma_data) if args.max_sigma is not None else max_sigma_data - cmin = composite_uncertainty.min() - #cmin = 0.0 - color_scale = 'Jet' # Define the color scale here within the condition -else: - cmin = None # No cmax needed for static colors - cmax = None # No cmax needed for static colors - color_scale = None # No color scale needed for static colors - -# Plot configuration -fig2 = go.Figure() -fig2.add_trace(go.Scatter( - x=all_data['dE'], y=all_data['dN'], - mode='markers', - marker=dict( - size=5, - color=all_data['composite_uncertainty'] if args.colour_sigma else 'blue', # Conditional coloring - coloraxis="coloraxis" if args.colour_sigma else None # Use color axis only if color sigma is set - ), - name='dE vs dN', - text=[f"{time} Sigma dNdE: {unc:.4f}" for time, unc in zip(all_data['Time'], all_data['composite_uncertainty'])], - hoverinfo='text+x+y' -)) - -# Add smoothed data if available -if 'Smoothed_dN' in all_data.columns and 'Smoothed_dE' in all_data.columns: + # Plot configuration + fig2 = go.Figure() fig2.add_trace(go.Scatter( - x=all_data['Smoothed_dE'], y=all_data['Smoothed_dN'], + x=all_data['dE'], y=all_data['dN'], mode='markers', marker=dict( size=5, - color='red' + color=all_data['composite_uncertainty'] if args.colour_sigma else 'blue', # Conditional coloring + coloraxis="coloraxis" if args.colour_sigma else None # Use color axis only if color sigma is set ), - name='Smoothed' + name='dE vs dN', + text=[f"{time} Sigma dNdE: {unc:.4f}" for time, unc in + zip(all_data['Time'], all_data['composite_uncertainty'])], + hoverinfo='text+x+y' )) -# Layout update with conditional color axis settings -fig2.update_layout( - title=title_text, - xaxis_title='dE (meters)', - yaxis_title='dN (meters)', - xaxis=dict(scaleanchor="y", scaleratio=1), - yaxis=dict(scaleanchor="x", scaleratio=1), - coloraxis=dict( - colorscale=color_scale, - cmin=cmin, - cmax=cmax, - colorbar=dict( - title='Sigma Value', - x=0.5, y=-0.15, # Adjusted for visibility - xanchor='center', yanchor='bottom', - len=0.75, thickness=20, orientation='h' - ) - ) if args.colour_sigma else None, # Apply color axis settings only if needed - showlegend=True -) -fig2.show() -if args.save: - fig2.write_html("fig2.html") - -if args.map: - # Plotly plotting using mapbox open-street-map - - # Adjust the zoom level dynamically based on the spread of the latitude and longitude - def adjust_zoom(latitudes, longitudes): - lat_range = np.ptp(latitudes) # Peak to peak (range) of latitudes - lon_range = np.ptp(longitudes) # Peak to peak (range) of longitudes - if max(lat_range, lon_range) < 0.02: - return 13 # City level zoom - elif max(lat_range, lon_range) < 0.1: - return 10 # Regional level zoom - elif max(lat_range, lon_range) < 1: - return 7 # Country level zoom - else: - return 5 # Continental level zoom - - zoom_level = adjust_zoom(all_data['Latitude'], all_data['Longitude']) - - fig3 = go.Figure(go.Scattermapbox( - lat=all_data['Latitude'], - lon=all_data['Longitude'], - mode='markers+lines', - marker=dict(size=5, color='blue') - )) - - fig3.update_layout( - mapbox=dict( - style="open-street-map", - center=go.layout.mapbox.Center( - lat=all_data['Latitude'].mean(), - lon=all_data['Longitude'].mean() + # Add smoothed data if available + if 'Smoothed_dN' in all_data.columns and 'Smoothed_dE' in all_data.columns: + fig2.add_trace(go.Scatter( + x=all_data['Smoothed_dE'], y=all_data['Smoothed_dN'], + mode='markers', + marker=dict( + size=5, + color='red' ), - zoom=zoom_level - ), - title='Geographic Plot of Latitude and Longitude', - showlegend=False - ) - - fig3.show() - if args.save: - fig2.write_html("fig3.html") - -if args.heatmap: - # Plotly plotting dN vs dE heatmap - fig4 = go.Figure() - fig4.add_trace(go.Histogram2dContour( - x = all_data['dE'], - y = all_data['dN'], - colorscale = 'Jet', - reversescale = False, - xaxis = 'x', - yaxis = 'y' - )) - fig4.add_trace(go.Scatter( - x = all_data['dE'], - y = all_data['dN'], - xaxis = 'x', - yaxis = 'y', - mode = 'markers', - marker = dict( - color = 'rgba(0,0,0,0.3)', - size = 3 - ) - )) - fig4.add_trace(go.Scatter( - x = all_data['dE_weighted_mean'], - y = all_data['dN_weighted_mean'], - xaxis = 'x', - yaxis = 'y', - mode = 'markers', - marker = dict( - color="white", - size = 15, - line_color='black', - symbol='x-dot', - line_width=2 - ), - hoverinfo='text+x+y', - text='Weighted Mean (dE, dN)' - )) - fig4.add_trace(go.Histogram( - y = all_data['dN'], - xaxis = 'x2', - marker = dict( - color = 'rgba(0,0,0,1)' - ) - )) - fig4.add_trace(go.Histogram( - x = all_data['dE'], - yaxis = 'y2', - marker = dict( - color = 'rgba(0,0,0,1)' - ) - )) + name='Smoothed' + )) - fig4.update_layout( - autosize = False, - xaxis = dict( - zeroline = False, - domain = [0,0.85], - showgrid = False - ), - yaxis = dict( - zeroline = False, - domain = [0,0.85], - showgrid = False - ), - xaxis2 = dict( - zeroline = False, - domain = [0.85,1], - showgrid = False - ), - yaxis2 = dict( - zeroline = False, - domain = [0.85,1], - showgrid = False - ), - title=title_text, - xaxis_title='dE (meters)', - yaxis_title='dN (meters)', - height = 800, - width = 800, - bargap = 0, - hovermode = 'closest', - showlegend = False + # Layout update with conditional color axis settings + fig2.update_layout( + title=title_text, + xaxis_title='dE (meters)', + yaxis_title='dN (meters)', + xaxis=dict(scaleanchor="y", scaleratio=1), + yaxis=dict(scaleanchor="x", scaleratio=1), + coloraxis=dict( + colorscale=color_scale, + cmin=cmin, + cmax=cmax, + colorbar=dict( + title='Sigma Value', + x=0.5, y=-0.15, # Adjusted for visibility + xanchor='center', yanchor='bottom', + len=0.75, thickness=20, orientation='h' + ) + ) if args.colour_sigma else None, # Apply color axis settings only if needed + showlegend=True ) - fig4.show() - if args.save: - fig4.write_html("fig4.html") - + if show_plots: + fig2.show() + + if args.save_prefix is not None: + output_path = os.path.join(os.path.dirname(args.save_prefix), f"{input_root}_fig2.html") + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + fig2.write_html(output_path) + + ## Fig3 + if getattr(args, 'map', False) or getattr(args, 'map_view', False): + # Plotly plotting using mapbox open-street-map + + # Adjust the zoom level dynamically based on the spread of the latitude and longitude + def adjust_zoom(latitudes, longitudes): + lat_range = np.ptp(latitudes) # Peak to peak (range) of latitudes + lon_range = np.ptp(longitudes) # Peak to peak (range) of longitudes + if max(lat_range, lon_range) < 0.02: + return 13 # City level zoom + elif max(lat_range, lon_range) < 0.1: + return 10 # Regional level zoom + elif max(lat_range, lon_range) < 1: + return 7 # Country level zoom + else: + return 5 # Continental level zoom + + zoom_level = adjust_zoom(all_data['Latitude'], all_data['Longitude']) + + fig3 = go.Figure(go.Scattermapbox( + lat=all_data['Latitude'], + lon=all_data['Longitude'], + mode='markers+lines', + marker=dict(size=5, color='blue') + )) + fig3.update_layout( + mapbox=dict( + style="open-street-map", + center=go.layout.mapbox.Center( + lat=all_data['Latitude'].mean(), + lon=all_data['Longitude'].mean() + ), + zoom=zoom_level + ), + title='Geographic Plot of Latitude and Longitude', + showlegend=False + ) + if show_plots: + fig3.show() + + if args.save_prefix is not None: + output_path = os.path.join(os.path.dirname(args.save_prefix), f"{input_root}_fig3.html") + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + fig3.write_html(output_path) + + ## Fig4 + if args.heatmap: + # Plotly plotting dN vs dE heatmap + fig4 = go.Figure() + fig4.add_trace(go.Histogram2dContour( + x=all_data['dE'], + y=all_data['dN'], + colorscale='Jet', + reversescale=False, + xaxis='x', + yaxis='y' + )) + fig4.add_trace(go.Scatter( + x=all_data['dE'], + y=all_data['dN'], + xaxis='x', + yaxis='y', + mode='markers', + marker=dict( + color='rgba(0,0,0,0.3)', + size=3 + ) + )) + fig4.add_trace(go.Scatter( + x=all_data['dE_weighted_mean'], + y=all_data['dN_weighted_mean'], + xaxis='x', + yaxis='y', + mode='markers', + marker=dict( + color="white", + size=15, + line_color='black', + symbol='x-dot', + line_width=2 + ), + hoverinfo='text+x+y', + text='Weighted Mean (dE, dN)' + )) + fig4.add_trace(go.Histogram( + y=all_data['dN'], + xaxis='x2', + marker=dict( + color='rgba(0,0,0,1)' + ) + )) + fig4.add_trace(go.Histogram( + x=all_data['dE'], + yaxis='y2', + marker=dict( + color='rgba(0,0,0,1)' + ) + )) + fig4.update_layout( + autosize=False, + xaxis=dict( + zeroline=False, + domain=[0, 0.85], + showgrid=False + ), + yaxis=dict( + zeroline=False, + domain=[0, 0.85], + showgrid=False + ), + xaxis2=dict( + zeroline=False, + domain=[0.85, 1], + showgrid=False + ), + yaxis2=dict( + zeroline=False, + domain=[0.85, 1], + showgrid=False + ), + title=title_text, + xaxis_title='dE (meters)', + yaxis_title='dN (meters)', + height=800, + width=800, + bargap=0, + hovermode='closest', + showlegend=False + ) + if show_plots: + fig4.show() + + if args.save_prefix is not None: + output_path = os.path.join(os.path.dirname(args.save_prefix), f"{input_root}_fig4.html") + os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) + fig4.write_html(output_path) + +def _process_and_plot(input_files, args, show_plots=False): + """ + Internal helper to process POS data and generates plots. Shared function between the CLI and program UI call modes. + + Arguments: + input_files (list): One or more input .POS file paths. + args (object): Arguments for plot_pos call. + show_plots (bool): If True, open plots in browser, or if False, only save files for UI to access. + + Returns: + list: Paths to generated HTML files when args.save_prefix is provided, empty list otherwise. + """ + # Parse the start and end datetime if provided + start_datetime = parse_datetime(args.start_datetime) if args.start_datetime else None + end_datetime = parse_datetime(args.end_datetime) if args.end_datetime else None + + # Load and process data + all_data = pd.DataFrame() + for file_path in input_files: + file_data = parse_pos_format(file_path) + all_data = pd.concat([all_data, file_data], ignore_index=True) + + all_data['Time'] = pd.to_datetime(all_data['Time'], format="%Y-%m-%dT%H:%M:%S.%f") + + # Apply time windowing + if start_datetime: + all_data = all_data[all_data['Time'] >= start_datetime] + if end_datetime: + all_data = all_data[all_data['Time'] <= end_datetime] + + # Apply threshold filtering if sigma_threshold is provided + if args.sigma_threshold: + se_threshold, sn_threshold, su_threshold = args.sigma_threshold + mask = (all_data['sE'] <= se_threshold) & (all_data['sN'] <= sn_threshold) & ( + all_data['sU'] <= su_threshold) & (all_data['sElevation'] <= su_threshold) + all_data = all_data[mask] + + # Down-sample the data if requested + if args.down_sample: + # Ensure the 'Time' column is datetime for proper indexing + all_data['Time'] = pd.to_datetime(all_data['Time']) + all_data.set_index('Time', inplace=True) + # Resample and take the first available data point in each bin + all_data = all_data.resample(f'{args.down_sample}s').first().dropna().reset_index() + + # Demean, smooth, and compute statistics + if args.demean: + all_data = remove_weighted_mean(all_data) + all_data = apply_smoothing(all_data, args.horz_smoothing, args.vert_smoothing) + all_data, component_stats = compute_statistics(all_data) + + # Generate plots + create_plots(all_data, input_files, component_stats, args, show_plots = show_plots) + + # Return list of generated files + if args.save_prefix: + input_root = Path(input_files[0]).stem + generated_files = [] + generated_files.append(os.path.join(os.path.dirname(args.save_prefix), f"{input_root}_fig1.html")) + generated_files.append(os.path.join(os.path.dirname(args.save_prefix), f"{input_root}_fig2.html")) + # Access map and / or heatmap + if getattr(args, 'map', False) or getattr(args, 'map_view', False): + generated_files.append(os.path.join(os.path.dirname(args.save_prefix), f"{input_root}_fig3.html")) + if args.heatmap: + generated_files.append(os.path.join(os.path.dirname(args.save_prefix), f"{input_root}_fig4.html")) + return generated_files + + return [] + +def plot_pos_files(input_files, start_datetime=None, end_datetime=None, + horz_smoothing=None, vert_smoothing=None, colour_sigma=False, + max_sigma=None, elevation=False, demean=False, map_view=False, + heatmap=False, sigma_threshold=None, down_sample=None, + save_prefix=None): + """ + Generate the interactive figures from one or more POS files (the programmatic call for Ginan-UI to use). + + This function provides a programmatic way of generating visualisations + + Arguments: + input_files (list): One or more input .POS file paths. + start_datetime (str, optional): Start time (e.g., 'YYYY-MM-DDTHH:MM:SS'). + end_datetime (str, optional): End time (e.g., 'YYYY-MM-DDTHH:MM:SS'). + horz_smoothing (float, optional): LOWESS fraction for horizontal components (dN / dE). + vert_smoothing (float, optional): LOWESS fraction for vertical component (dU or Elevation). + colour_sigma (bool): If True, colour markers by sigma; otherwise show error bars. + max_sigma (float, optional): Upper cap for sigma colour scale when colour_sigma is True. + elevation (bool): If True, use 'Elevation' instead of 'dU' for vertical plotting. + demean (bool): If True, remove weighted mean from each series before plotting. + map_view (bool): If True, generate a geographic map (fig3). + heatmap (bool): If True, generate a 2D dE–dN density view (fig4). + sigma_threshold (tuple, optional): (sE, sN, sU) for filtering rows. + down_sample (int, optional): Resampling interval in seconds. + save_prefix (str, optional): If provided, write HTML files next to this prefix. + + Returns: + list: Paths to generated HTML files when save_prefix is provided; empty list otherwise. + """ + class Args: + def __init__(self): + self.input_files = input_files + self.start_datetime = start_datetime + self.end_datetime = end_datetime + self.horz_smoothing = horz_smoothing + self.vert_smoothing = vert_smoothing + self.colour_sigma = colour_sigma + self.max_sigma = max_sigma + self.elevation = elevation + self.demean = demean + self.map = map_view + self.heatmap = heatmap + self.sigma_threshold = sigma_threshold + self.down_sample = down_sample + self.save_prefix = save_prefix + + args = Args() + + # "show_plots = False" flags to remain in UI (don't open web browser) + return _process_and_plot(input_files, args, show_plots = False) + +# CLI Entry +if __name__ == "__main__": + # Setup and parse arguments + parser = argparse.ArgumentParser(description="Plot positional data with optional smoothing and color coding.") + parser.add_argument('--input-files', nargs='+', required=True, help='One or more input .POS files') + parser.add_argument('--start-datetime', type=str, + help="Start datetime in the format YYYY-MM-DDTHH:MM:SS, optional timezone") + parser.add_argument('--end-datetime', type=str, + help="End datetime in the format YYYY-MM-DDTHH:MM:SS, optional timezone") + parser.add_argument('--horz-smoothing', type=float, default=None, + help='Fraction of the data used for horizontal (East and North) LOWESS smoothing (optional).') + parser.add_argument('--vert-smoothing', type=float, default=None, + help='Fraction of the data used for vertical (Up) LOWESS smoothing (optional).') + parser.add_argument('--colour-sigma', action='store_true', + help='Colourize the timeseries using the standard deviation (sigma) values (optional).') + parser.add_argument('--max-sigma', type=float, default=None, + help='Set a maximum sigma threshold for the sigma colour scale (optional).') + parser.add_argument('--elevation', action='store_true', + help='Plot Elevation values inplace of dU wrt the reference coord (optional).') + parser.add_argument('--demean', action='store_true', + help='Remove the mean values from all time series before plotting (optional).') + parser.add_argument('--map', action='store_true', + help='Create a geographic map view from the Longitude & Latitude estiamtes (optional).') + parser.add_argument('--heatmap', action='store_true', + help='Create a 2D heatmap view of E & N coodrinates wrt the reference position (optional).') + parser.add_argument('--sigma-threshold', nargs=3, type=float, + help="Thresholds for sE, sN, and sU to filter data.") + parser.add_argument('--down-sample', type=int, + help="Interval in seconds for down-sampling data.") + parser.add_argument('--save-prefix', nargs='?', const='plot', default=None, + help='Prefix for saving HTML figures, e.g., ./output/fig') + args = parser.parse_args() + + # "show_plots = True" flags to open the HTML file in web browser + _process_and_plot(args.input_files, args, show_plots = True) diff --git a/scripts/plot_trace_res.py b/scripts/plot_trace_res.py new file mode 100644 index 000000000..c6648c45e --- /dev/null +++ b/scripts/plot_trace_res.py @@ -0,0 +1,2811 @@ +from __future__ import annotations +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +GNSS TRACE residual plotter with optional large-error and ambiguity reset markers. + +Features +--------- +- Parse residual lines beginning with '%' +- Parse 'LARGE STATE ERROR' and 'LARGE MEAS ERROR' lines (if --mark-large-errors) +- Keep only the highest iteration per observation +- Create separate CODE / PHASE HTML plots, optionally split per satellite or receiver +- Overlay LARGE MEAS (black ▲) and LARGE STATE (orange dashed) markers with concise tooltips +- Overlay ambiguity resets from the preprocessor (green), from the filter (blue) +- Create CODE & PHASE weighted and unweighted residual heatmaps of mean/std_dev/rms. +- Create cumulative and total ambiguity reset plots +""" + +import os, glob +import re +import argparse +from pathlib import Path +from typing import Iterable, Optional, List, Dict, Tuple +from datetime import datetime + + +def ensure_parent(p) -> None: + """Create parent directory for a path if it doesn't exist.""" + from pathlib import Path as _P + try: + _P(p).parent.mkdir(parents=True, exist_ok=True) + except Exception: + pass + + +def _sanitize_filename_piece(s: str) -> str: + """Sanitize string for filenames using underscores.""" + return re.sub(r"[^A-Za-z0-9._-]+", "_", str(s)) + + +def build_out_path( + base: str, + variant_suffix: str, + short: str, + *, + split: str | None = None, # "recv" | "sat" | None + key: str | None = None, # station or satellite ID + tag: str | None = None, # e.g., "residual", "h"/"v" for totals + ext: str = "html", +) -> str: + """ + Build consistent output names like: + _[_][_recv|_sat]_[]. + """ + parts = [f"{base}{variant_suffix}", short] + if tag: + parts.append(tag) + if split in ("recv", "sat"): + parts.append(split) + if key: + parts.append(_sanitize_filename_piece(key)) + return "_".join(parts) + f".{ext}" +def slugify(text: str) -> str: + """Return a safe slug for filenames: lowercase, alnum-plus-dashes.""" + import re + t = re.sub(r"[^A-Za-z0-9]+", "-", text).strip("-").lower() + return re.sub(r"-{2,}", "-", t) or "out" + +import logging +logger = logging.getLogger("plot_trace_res") + + +def _setup_logging(level: str = "INFO") -> None: + """Configure root logger for the script.""" + lvl = getattr(logging, level.upper(), logging.INFO) + logging.basicConfig( + level=lvl, + format="%(asctime)s | %(levelname)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + logger.setLevel(lvl) + +from typing import Any, Dict, Iterable, Iterator, List, Literal, Optional, Sequence, Tuple + +import pandas as pd +import plotly.graph_objects as go +from plotly.subplots import make_subplots +import plotly.io as pio +pio.templates.default = None + +import numpy as np +from collections import defaultdict + +# -------- Parsing -------- + +FLOAT = r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?" + +# AJC - added -? to iter group to allow parsing -1 from smoothed TRACE file +LINE_RE = re.compile( + rf""" + ^%\s+ + (?P-?\d+)\s+ + (?P\d{{4}}-\d{{2}}-\d{{2}})\s+ # e.g. 2025-10-05 + (?P