diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh index 7ed7eb1..549c451 100644 --- a/.github/scripts/build.sh +++ b/.github/scripts/build.sh @@ -1,32 +1,30 @@ if [ "$GOOS" == "linux" ] then set -e - go build -o cdis-data-client + go build -o calypr-cli ./cmd/calypr-cli ls -al if [ "$GITHUB_PULL_REQUEST" == "false" ]; then - mv gen3-client files && mv cdis-data-client gen3-client - zip dataclient_linux.zip gen3-client && mv dataclient_linux.zip ~/shared/. + zip calypr-cli_linux.zip calypr-cli && mv calypr-cli_linux.zip ~/shared/. aws s3 sync ~/shared s3://cdis-dc-builds/$GITHUB_BRANCH fi set +e elif [ "$GOOS" == "darwin" ] then set -e - go build -o cdis-data-client + go build -o calypr-cli ./cmd/calypr-cli ls -al if [ "$GITHUB_PULL_REQUEST" == "false" ]; then - mv gen3-client files && mv cdis-data-client gen3-client - zip dataclient_osx.zip gen3-client && mv dataclient_osx.zip ~/shared/. + zip calypr-cli_osx.zip calypr-cli && mv calypr-cli_osx.zip ~/shared/. aws s3 sync ~/shared s3://cdis-dc-builds/$GITHUB_BRANCH fi set +e elif [ "$GOOS" == "windows" ] then set -e - go build -o gen3-client.exe + go build -o calypr-cli.exe ./cmd/calypr-cli ls -al if [ "$GITHUB_PULL_REQUEST" == "false" ]; then - zip dataclient_win64.zip gen3-client.exe && mv dataclient_win64.zip ~/shared/. + zip calypr-cli_win64.zip calypr-cli.exe && mv calypr-cli_win64.zip ~/shared/. aws s3 sync ~/shared s3://cdis-dc-builds/$GITHUB_BRANCH fi set +e diff --git a/.github/workflows/build_and_test_pull.yaml b/.github/workflows/build_and_test_pull.yaml deleted file mode 100644 index df9a68e..0000000 --- a/.github/workflows/build_and_test_pull.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: "Test and build binary for pull request" -on: - pull_request: - branches: - - master -jobs: - test: - name: test - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Go 1.17 - uses: actions/setup-go@v4 - with: - go-version: '1.17' - - - name: Run Setup Script - run: | - bash ./.github/scripts/before_install.sh - env: - GITHUB_BRANCH: ${{ github.ref_name }} - ACCESS_KEY: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} - - - name: Run Tests - run: go test -v github.com/uc-cdis/gen3-client/tests - - build: - env: - goarch: amd64 - needs: test - runs-on: ubuntu-latest - strategy: - matrix: - include: - - goos: linux - goarch: amd64 - zipfile: dataclient_linux.zip - - goos: darwin - goarch: amd64 - zipfile: dataclient_osx.zip - - goos: windows - goarch: amd64 - zipfile: dataclient_win64.zip - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Go 1.17 - uses: actions/setup-go@v4 - with: - go-version: '1.17' - - - name: Run Setup Script - run: | - bash .github/scripts/before_install.sh - env: - GITHUB_BRANCH: ${{ github.ref_name }} - ACCESS_KEY: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} - - - - name: Run Build Script - run: | - bash .github/scripts/build.sh - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ env.goarch }} - GITHUB_BRANCH: ${{ github.ref_name }} - GITHUB_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/build_and_test_push.yaml b/.github/workflows/build_and_test_push.yaml deleted file mode 100644 index 0b81f2a..0000000 --- a/.github/workflows/build_and_test_push.yaml +++ /dev/null @@ -1,244 +0,0 @@ -name: "Test, build, sync to AWS, and create release (on tagged push)" -on: push - - -jobs: - - test: - name: test - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Go 1.17 - uses: actions/setup-go@v4 - with: - go-version: '1.17' - - - name: Run Setup Script - run: | - bash ./.github/scripts/before_install.sh - env: - GITHUB_BRANCH: ${{ github.ref_name }} - ACCESS_KEY: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} - - - name: Run Tests - run: go test -v github.com/uc-cdis/gen3-client/tests - - build: - env: - goarch: amd64 - needs: test - runs-on: ubuntu-latest - strategy: - matrix: - include: - - goos: linux - goarch: amd64 - zipfile: dataclient_linux.zip - - goos: darwin - goarch: amd64 - zipfile: dataclient_osx.zip - - goos: windows - goarch: amd64 - zipfile: dataclient_win64.zip - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Go 1.17 - uses: actions/setup-go@v4 - with: - go-version: '1.17' - - - name: Run Setup Script - run: | - bash .github/scripts/before_install.sh - env: - GITHUB_BRANCH: ${{ github.ref_name }} - ACCESS_KEY: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} - - - - name: Run Build Script - run: | - bash .github/scripts/build.sh - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ env.goarch }} - GITHUB_BRANCH: ${{ github.ref_name }} - GITHUB_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} - - - name: Upload Artifacts - uses: actions/upload-artifact@v4 - with: - name: build-artifact-${{ matrix.goos }} - path: ~/shared/${{ matrix.zipfile }} - retention-days: 3 - - - sign: - needs: build - runs-on: macos-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Download OSX Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-darwin - path: ./dist - - name: Unzip OSX Artifact and remove zip file - run: | - cd ./dist - ls - unzip dataclient_osx.zip - rm dataclient_osx.zip - - - - - name: Build executable - shell: bash - env: - APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }} - APPLE_NOTARY_UUID: ${{ secrets.APPLE_NOTARY_UUID }} - APPLE_NOTARY_KEY: ${{ secrets.APPLE_NOTARY_KEY}} - APPLE_NOTARY_DATA: ${{ secrets.APPLE_NOTARY_DATA }} - APPLE_CERT_DATA: ${{ secrets.APPLE_CERT_DATA }} - APPLICATION_CERT_PASSWORD: ${{ secrets.APPLICATION_CERT_PASSWORD }} - APPLICATION_CERT_DATA: ${{ secrets.APPLICATION_CERT_DATA }} - APPLE_TEAM_ID: WYQ7U7YUC9 - - run: | - # Setup - SIGNFILE="$(pwd)/dist/gen3-client" - - # Export certs - echo "$APPLE_CERT_DATA" | base64 --decode > /tmp/certs.p12 - echo "$APPLE_NOTARY_DATA" | base64 --decode > /tmp/notary.p8 - echo "$APPLICATION_CERT_DATA" | base64 --decode > /tmp/app_certs.p12 - - # Create keychain - security create-keychain -p actions macos-build.keychain - security default-keychain -s macos-build.keychain - security unlock-keychain -p actions macos-build.keychain - security set-keychain-settings -t 3600 -u macos-build.keychain - - # Import certs to keychain - security import /tmp/certs.p12 -k ~/Library/Keychains/macos-build.keychain -P "$APPLE_CERT_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign - security import /tmp/app_certs.p12 -k ~/Library/Keychains/macos-build.keychain -P "$APPLICATION_CERT_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign - - # Key signing - security set-key-partition-list -S apple-tool:,apple: -s -k actions macos-build.keychain - - # Verify keychain things - security find-identity -v macos-build.keychain | grep "$APPLE_TEAM_ID" | grep "Developer ID Application" - security find-identity -v macos-build.keychain | grep "$APPLE_TEAM_ID" | grep "Developer ID Installer" - - # Force the codesignature - codesign --force --options=runtime --keychain "/Users/runner/Library/Keychains/macos-build.keychain-db" -s "$APPLE_TEAM_ID" "$SIGNFILE" - - # Verify the code signature - codesign -v "$SIGNFILE" --verbose - - mkdir -p ./dist/pkg - cp ./dist/gen3-client ./dist/pkg/gen3-client - pkgbuild --identifier "org.uc-cdis.gen3-client.pkg" --timestamp --install-location /Applications --root ./dist/pkg installer.pkg - pwd - ls - productbuild --resources ./resources --distribution ./distribution.xml gen3-client.pkg - productsign --sign "$APPLE_TEAM_ID" --timestamp gen3-client.pkg gen3-client_signed.pkg - - xcrun notarytool store-credentials "notarytool-profile" --issuer $APPLE_NOTARY_UUID --key-id $APPLE_NOTARY_KEY --key /tmp/notary.p8 - xcrun notarytool submit gen3-client_signed.pkg --keychain-profile "notarytool-profile" --wait - xcrun stapler staple gen3-client_signed.pkg - mv gen3-client_signed.pkg dataclient_osx.pkg - - - name: Upload signed artifact - uses: actions/upload-artifact@v4 - with: - name: build-artifact-darwin-signed - path: dataclient_osx.pkg - - sync_signed_to_aws: - runs-on: ubuntu-latest - needs: sign - - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Run Setup Script - run: | - bash ./.github/scripts/before_install.sh - env: - GITHUB_BRANCH: ${{ github.ref_name }} - ACCESS_KEY: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} - - - name: Download OSX Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-darwin-signed - - - name: Sync to AWS - env: - GITHUB_BRANCH: ${{ github.ref_name }} - run: | - rm ~/shared/dataclient_osx.zip - zip dataclient_osx_signed.zip dataclient_osx.pkg - mv dataclient_osx_signed.zip ~/shared/ - aws s3 sync ~/shared s3://cdis-dc-builds/$GITHUB_BRANCH - - - get_tagged_branch: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-latest - needs: [build,sign] - outputs: - branch: ${{ steps.check_step.outputs.branch }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Get current branch - id: check_step - # 1. Get the list of branches ref where this tag exists - # 2. Remove 'origin/' from that result - # 3. Put that string in output - # => We can now use function 'contains(list, item)'' - run: | - raw=$(git branch -r --contains ${{ github.ref }}) - branch="$(echo ${raw//origin\//} | tr -d '\n')" - echo "{name}=branch" >> $GITHUB_OUTPUT - echo "Branches where this tag exists : $branch." - - - deploy: - needs: get_tagged_branch - if: startsWith(github.ref, 'refs/tags/') && contains(${{ needs.get_tagged_branch.outputs.branch }}, 'master') - runs-on: ubuntu-latest - steps: - - name: Download Linux Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-linux - - - name: Download OSX Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-darwin-signed - - - name: Download Windows Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-windows - - - name: Create Release gh cli - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_TAG: ${{ github.ref_name }} - run: gh release create "$GH_TAG" dataclient_linux.zip dataclient_osx.pkg dataclient_win64.zip --repo="$GITHUB_REPOSITORY" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..2dd47fb --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,36 @@ +name: Calypr CLI CI + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +jobs: + lint-and-test: + name: Lint and Unit Tests + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Download dependencies + run: GOTOOLCHAIN=auto go mod download + + - name: Run go vet + run: GOTOOLCHAIN=auto go vet ./... + + - name: Run unit tests + run: GOTOOLCHAIN=auto go test -v ./... diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml new file mode 100644 index 0000000..b9ba9fa --- /dev/null +++ b/.github/workflows/coverage.yaml @@ -0,0 +1,121 @@ +name: "Test Coverage Check" + +on: + pull_request: + push: + workflow_dispatch: + +jobs: + coverage: + name: Test Coverage + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run Tests with Coverage + run: | + CORE_PACKAGES=$(go list ./... | grep -Ev '/(cmd|mocks|tests)$') + go test -coverprofile=coverage.out -covermode=atomic $CORE_PACKAGES + continue-on-error: true + + - name: Generate Coverage Report + id: coverage + run: | + CORE_PACKAGES=$(go list ./... | grep -Ev '/(cmd|mocks|tests)$') + + # Get overall coverage + OVERALL=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//') + echo "overall=$OVERALL" >> $GITHUB_OUTPUT + + # Generate detailed report + echo "## Test Coverage Report" > coverage-report.md + echo "" >> coverage-report.md + echo "**Overall Coverage:** ${OVERALL}%" >> coverage-report.md + echo "" >> coverage-report.md + echo "### Package Coverage" >> coverage-report.md + echo "" >> coverage-report.md + echo "| Package | Coverage |" >> coverage-report.md + echo "|---------|----------|" >> coverage-report.md + + # Extract package coverage + go test -coverprofile=/dev/null -covermode=atomic $CORE_PACKAGES 2>&1 | \ + awk '/^ok[[:space:]]/ { + pkg=$2; + cov=$5; + gsub(/github.com\/calypr\/calypr-cli\//, "", pkg); + print "| " pkg " | " cov " |" + }' >> coverage-report.md + + cat coverage-report.md + + - name: Check Coverage Thresholds + run: | + set -euo pipefail + + CORE_PACKAGES=$(go list ./... | grep -Ev '/(cmd|mocks|tests)$') + + OVERALL=$(go tool cover -func=coverage.out | awk '/^total:/ {gsub(/%/, "", $3); print $3}') + if ! awk -v val="$OVERALL" -v min=30 'BEGIN { if (val + 0 < min) exit 1; exit 0 }'; then + echo "Overall coverage ${OVERALL}% is below the required minimum of 30%" + exit 1 + fi + + while read -r pkg cov; do + cov=${cov%%%} + if ! awk -v val="$cov" -v min=20 'BEGIN { if (val + 0 < min) exit 1; exit 0 }'; then + echo "Package ${pkg} coverage ${cov}% is below the required minimum of 20%" + exit 1 + fi + done < <( + go test -coverprofile=/dev/null -covermode=atomic $CORE_PACKAGES 2>&1 | \ + awk '/^ok[[:space:]]/ { + pkg=$2; + cov=$5; + gsub(/github.com\/calypr\/calypr-cli\//, "", pkg); + print pkg, cov + }' + ) + + - name: Upload Coverage to Codecov (Optional) + uses: codecov/codecov-action@v4 + if: always() + with: + files: ./coverage.out + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + - name: Comment PR with Coverage + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const coverage = fs.readFileSync('coverage-report.md', 'utf8'); + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: coverage + }); + + - name: Upload Coverage Artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage.out + coverage-report.md + retention-days: 30 diff --git a/.github/workflows/golang-ci-workflow.yaml b/.github/workflows/golang-ci-workflow.yaml deleted file mode 100644 index e94f859..0000000 --- a/.github/workflows/golang-ci-workflow.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Golang CI Workflow - -on: push - -jobs: - ci: - runs-on: ubuntu-latest - env: - COVERAGE_PROFILE_OUTPUT_LOCATION: "./profile.cov" - steps: - - name: Checkout code / lint code / install dependencies for goveralls / run tests - uses: uc-cdis/.github/.github/actions/golang-ci@master - with: - COVERAGE_PROFILE_OUTPUT_LOCATION: ${{ env.COVERAGE_PROFILE_OUTPUT_LOCATION }} - - name: Send coverage to coveralls using goveralls - env: - COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: goveralls -coverprofile=${{ env.COVERAGE_PROFILE_OUTPUT_LOCATION }} -service=github diff --git a/.github/workflows/image_build_push.yaml b/.github/workflows/image_build_push.yaml index 4c50ae5..3301de1 100644 --- a/.github/workflows/image_build_push.yaml +++ b/.github/workflows/image_build_push.yaml @@ -7,7 +7,7 @@ jobs: name: Build Image and Push uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: - OVERRIDE_REPO_NAME: "gen3-client" + OVERRIDE_REPO_NAME: "calypr-cli" secrets: ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index baabb42..fc01885 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -11,9 +11,9 @@ jobs: name: Integration tests uses: uc-cdis/.github/.github/workflows/integration_tests.yaml@master with: - QUAY_REPO: gen3-client + QUAY_REPO: calypr-cli # Tag used in tests that depend on this repo (https://github.com/uc-cdis/gen3-code-vigil/blob/master/gen3-integration-tests/pyproject.toml) - SERVICE_TO_TEST: gen3_client + SERVICE_TO_TEST: calypr_cli secrets: CI_AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} CI_AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..9c18781 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,135 @@ +name: Release + +on: + workflow_run: + workflows: + - Calypr CLI CI + types: + - completed + workflow_dispatch: + +concurrency: + group: release-main + cancel-in-progress: false + +permissions: + contents: write + +jobs: + tag-release: + if: > + github.event_name == 'workflow_dispatch' || + ( + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main' + ) + runs-on: ubuntu-latest + outputs: + release_tag: ${{ steps.next-tag.outputs.release_tag }} + release_sha: ${{ steps.resolve-sha.outputs.release_sha }} + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event_name == 'workflow_dispatch' && github.ref || github.event.workflow_run.head_sha }} + + - name: Resolve release SHA + id: resolve-sha + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "release_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + else + echo "release_sha=${{ github.event.workflow_run.head_sha }}" >> "$GITHUB_OUTPUT" + fi + + - name: Determine next release tag + id: next-tag + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const parseVersion = (tag) => { + const match = /^v?(\d+)\.(\d+)\.(\d+)$/.exec(tag || ""); + if (!match) { + throw new Error(`Latest release tag "${tag}" is not semver (expected vMAJOR.MINOR.PATCH)`); + } + return { + major: Number(match[1]), + minor: Number(match[2]), + patch: Number(match[3]), + }; + }; + + let latestTag = "v0.0.0"; + + try { + const latestRelease = await github.rest.repos.getLatestRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + latestTag = latestRelease.data.tag_name || latestTag; + core.info(`Latest GitHub release tag: ${latestTag}`); + } catch (error) { + if (error.status !== 404) { + throw error; + } + core.info("No GitHub release found yet; falling back to repository tags."); + const tags = await github.paginate(github.rest.repos.listTags, { + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100, + }); + const semverTags = tags + .map((tag) => tag.name) + .filter((tag) => /^v?\d+\.\d+\.\d+$/.test(tag)) + .sort((a, b) => { + const pa = parseVersion(a); + const pb = parseVersion(b); + if (pa.major !== pb.major) return pb.major - pa.major; + if (pa.minor !== pb.minor) return pb.minor - pa.minor; + return pb.patch - pa.patch; + }); + if (semverTags.length > 0) { + latestTag = semverTags[0]; + core.info(`Latest repository tag fallback: ${latestTag}`); + } + } + + const { major, minor, patch } = parseVersion(latestTag); + const nextTag = `v${major}.${minor}.${patch + 1}`; + core.info(`Next release tag: ${nextTag}`); + core.setOutput("release_tag", nextTag); + + - name: Create annotated tag + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "${{ steps.next-tag.outputs.release_tag }}" "${{ steps.resolve-sha.outputs.release_sha }}" -m "Release ${{ steps.next-tag.outputs.release_tag }}" + git push origin "${{ steps.next-tag.outputs.release_tag }}" + + goreleaser: + needs: tag-release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + ref: ${{ needs.tag-release.outputs.release_tag }} + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: 'latest' + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 849f232..453ca8f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,9 @@ # Build artifacts /build/ -checksums.txt \ No newline at end of file +/bin/ +checksums.txt +# Local caches and binaries +/.gocache/ +/.tmp/ +/data-client diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..177c5c6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "ga4gh/data-repository-service-schemas"] + path = ga4gh/data-repository-service-schemas + url = https://github.com/kellrott/data-repository-service-schemas.git + branch = feature/get-by-checksum diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..d778196 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,41 @@ +version: 2 + +project_name: calypr-cli + +before: + hooks: + - go mod download + +builds: + - id: calypr-cli + main: . + binary: calypr-cli + env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w -X github.com/calypr/calypr-cli/cmd.gitversion={{ .Version }} -X github.com/calypr/calypr-cli/cmd.gitcommit={{ .Commit }} + +archives: + - id: default + builds: + - calypr-cli + formats: + - tar.gz + format_overrides: + - goos: windows + formats: + - zip + name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}" + +checksum: + name_template: checksums.txt + +changelog: + disable: true diff --git a/Dockerfile b/Dockerfile index b191aec..753d86e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ENV CGO_ENABLED=0 ENV GOOS=linux ENV GOARCH=amd64 -WORKDIR $GOPATH/src/github.com/calypr/data-client/ +WORKDIR $GOPATH/src/github.com/calypr/calypr-cli/ COPY go.mod . COPY go.sum . @@ -15,15 +15,15 @@ COPY . . RUN COMMIT=$(git rev-parse HEAD); \ VERSION=$(git describe --always --tags); \ - printf '%s\n' 'package g3cmd'\ + printf '%s\n' 'package cmd'\ ''\ 'const ('\ ' gitcommit="'"${COMMIT}"'"'\ ' gitversion="'"${VERSION}"'"'\ - ')' > data-client/g3cmd/gitversion.go \ - && go build -o /data-client + ')' > cmd/gitversion.go \ + && go build -o /calypr-cli . FROM scratch COPY --from=build-deps /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=build-deps /data-client /data-client -CMD ["/data-client"] +COPY --from=build-deps /calypr-cli /calypr-cli +CMD ["/calypr-cli"] diff --git a/Makefile b/Makefile index e524d95..f09edb9 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,25 @@ # --- Variables --- -# The name of the resulting binary (e.g., 'data-client' if your module is called data-client) +# The name of the resulting binary. Defaults to the repo directory name. # Update this if your main package is not in the root directory. TARGET_NAME := $(shell basename $(shell pwd)) -# The default path to build the main package. Use '.' if your main package is in the root. -# Change this if your main package is in a subdirectory (e.g., ./cmd/myapp) +# The default build path. This repo keeps a root main package for `go build .`. MAIN_PACKAGE := . # The directory where the final binary will be placed BIN_DIR := ./bin +# Coverage thresholds +COVERAGE_THRESHOLD := 30 +PACKAGE_COVERAGE_THRESHOLD := 20 +CORE_PACKAGES := $(shell go list ./... | grep -Ev '/(cmd|mocks|tests)$$') + +# OpenAPI generation now lives in syfon. +SYFON_DIR ?= ../syfon + # --- Targets --- -.PHONY: all build test generate tidy clean help +.PHONY: all build test test-coverage coverage-html coverage-check generate tidy clean help # The default target run when you type 'make' all: build @@ -28,19 +35,80 @@ test: @echo "--> Running all tests..." @go test -v ./... +## test-coverage: Runs tests with coverage profiling +test-coverage: + @echo "--> Running tests with coverage..." + @go test -coverprofile=coverage.out -covermode=atomic $(CORE_PACKAGES) + @echo "--> Coverage report generated: coverage.out" + @go tool cover -func=coverage.out | tail -1 + +## coverage-html: Generates HTML coverage report +coverage-html: test-coverage + @echo "--> Generating HTML coverage report..." + @go tool cover -html=coverage.out -o coverage.html + @echo "--> HTML coverage report generated: coverage.html" + +## coverage-check: Verifies coverage meets minimum thresholds +coverage-check: test-coverage + @echo "--> Checking coverage thresholds..." + @set -euo pipefail; \ + OVERALL=$$(go tool cover -func=coverage.out | awk '/^total:/ {gsub(/%/, "", $$3); print $$3}'); \ + if ! awk -v val="$$OVERALL" -v min=$(COVERAGE_THRESHOLD) 'BEGIN { if (val + 0 < min) exit 1; exit 0 }'; then \ + echo "Overall coverage $$OVERALL% is below the required minimum of $(COVERAGE_THRESHOLD)%"; \ + exit 1; \ + fi; \ + go test -coverprofile=/dev/null -covermode=atomic $(CORE_PACKAGES) 2>&1 | \ + awk '/^ok[[:space:]]/ { \ + pkg=$$2; \ + cov=$$5; \ + gsub(/github.com\\/calypr\\/calypr-cli\\//, "", pkg); \ + print pkg, cov; \ + }' | \ + while read -r pkg cov; do \ + case "$$pkg" in \ + cmd|mocks|tests) continue ;; \ + esac; \ + cov=$${cov%%%}; \ + if ! awk -v val="$$cov" -v min=$(PACKAGE_COVERAGE_THRESHOLD) 'BEGIN { if (val + 0 < min) exit 1; exit 0 }'; then \ + echo "Package $$pkg coverage $$cov% is below the required minimum of $(PACKAGE_COVERAGE_THRESHOLD)%"; \ + exit 1; \ + fi; \ + done + ## generate: Runs go generate commands to create mocks, embedded assets, etc. generate: @echo "--> Running code generation (go generate)..." @go generate ./... +## gen: Generates Go models from OpenAPI specs +gen: + @set -euo pipefail; \ + if [[ ! -d "$(SYFON_DIR)" ]]; then \ + echo "ERROR: syfon repo not found at $(SYFON_DIR)"; \ + exit 1; \ + fi; \ + echo "--> OpenAPI generation is centralized in syfon"; \ + $(MAKE) -C "$(SYFON_DIR)" gen + +.PHONY: gen-internal +gen-internal: + @set -euo pipefail; \ + if [[ ! -d "$(SYFON_DIR)" ]]; then \ + echo "ERROR: syfon repo not found at $(SYFON_DIR)"; \ + exit 1; \ + fi; \ + echo "--> OpenAPI generation is centralized in syfon make gen"; \ + $(MAKE) -C "$(SYFON_DIR)" gen + ## tidy: Cleans up module dependencies and formats go files tidy: @echo "--> Tidying go.mod and formatting files..." @go mod tidy @go fmt ./... -## clean: Removes the compiled binary +## clean: Removes the compiled binary and coverage files clean: @echo "--> Cleaning up..." @rm -f $(BIN_DIR)/$(TARGET_NAME) - + @rm -f coverage.out coverage.html + @rm -rf .tmp diff --git a/README.md b/README.md index d8cfe6d..ace5397 100644 --- a/README.md +++ b/README.md @@ -1,92 +1,124 @@ -# data-client +# calypr-cli -[![Build Status](https://travis-ci.org/uc-cdis/cdis-data-client.svg?branch=master)](https://travis-ci.org/uc-cdis/cdis-data-client) -[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/uc-cdis/cdis-data-client?sort=semver)](https://github.com/uc-cdis/cdis-data-client/releases) +[![CI](https://github.com/calypr/calypr-cli/actions/workflows/ci.yaml/badge.svg)](https://github.com/calypr/calypr-cli/actions/workflows/ci.yaml) +[![Coverage](https://github.com/calypr/calypr-cli/actions/workflows/coverage.yaml/badge.svg)](https://github.com/calypr/calypr-cli/actions/workflows/coverage.yaml) +[![Release](https://img.shields.io/github/v/release/calypr/calypr-cli?sort=semver)](https://github.com/calypr/calypr-cli/releases) -`data-client` is a command-line tool for downloading, uploading, and submitting data files to and from a Gen3 data commons. +`calypr-cli` is Calypr's command-line client for working with Gen3-style data +commons from a terminal. It combines the standard data transfer flows with +operator-oriented surfaces for permissions, collaborators, and portal +configuration. -Read more about what it does and how to use it in the `data-client` [user guide](https://gen3.org/resources/user/data-client/). +## What It Does -`data-client` is built on Cobra, a library providing a simple interface to create powerful modern CLI interfaces similar to git & go tools. Read more about Cobra [here](https://github.com/spf13/cobra). +The CLI currently groups into four main areas: -## Installation +| Command group | Purpose | +| --- | --- | +| `configure`, `auth` | Store credentials, create named profiles, and check connectivity | +| `upload*`, `download*`, `retry-upload` | Transfer files to and from supported backends | +| `collaborators` | Manage collaboration requests and approvals | +| `permissions`, `portal` | Operator workflows for Arborist-backed permissions and Gecko-backed portal config | -(The following instruction is for compiling and installing the `data-client` from source code. There are also binary executables can be found at [here](https://github.com/uc-cdis/cdis-data-client/releases)) +Most commands require a configured `--profile`, and the default backend is +`gen3`. -First, [install Go and the Go tools](https://golang.org/doc/install) if you have not already done so. [Set up your workspace and your GOPATH.](https://golang.org/doc/code.html) +## Operator Surfaces -Then: +Two command groups are specific to the Calypr fork rather than the legacy data +transfer client: -``` -go get -d github.com/calypr/data-client -go install -``` +| Command group | Supported surface | Deliberate non-goals | +| --- | --- | --- | +| `permissions` | Public Gen3 `/authz` routes exposed through revproxy | Raw Arborist catalog/admin CRUD and arbitrary policy mutation | +| `portal` | Public `/gecko` routes exposed through revproxy | Backend-only Gecko paths and undocumented admin-only flows | + +Detailed operator docs live here: + +- [permissions CLI guide](docs/permissions-cli.md) +- [portal CLI guide](docs/portal-cli.md) +- [developer docs](docs/DEVELOPER_DOCS.md) -_TODO: Remove after GitHub repo is renamed_ -**_For now, the above actually won't work because the GitHub repo needs to be renamed. Do this instead:_** +## Install And Build +The repo keeps a root `main` package, so the primary local build path is the +repo root: + +```bash +go build . ``` -mkdir -p $GOPATH/src/github.com/uc-cdis -cd $GOPATH/src/github.com/uc-cdis -git clone git@github.com:uc-cdis/cdis-data-client.git -mv cdis-data-client data-client -cd data-client -go get -d ./... + +To install the current checkout: + +```bash go install . ``` -Now you should have `data-client` successfully installed. For a comprehensive instruction on how to configure and use `data-client` for uploading / downloading object files, please refer to the `data-client` [user guide](https://gen3.org/resources/user/data-client/). - -## Enabling New Gen3 Object Management API +There is also a compatibility entrypoint under `./cmd/calypr-cli` if you need a +subdirectory main package explicitly: -Some Gen3 data commons support uploading files through the new Gen3 Object Management API. +```bash +go build ./cmd/calypr-cli +``` -> NOTE: The service powering this API is sometimes referred to as our object "Shepherd" +## Common Setup -To enable data-client to upload using the Gen3 Object Management API, pass the `use-shepherd=true` to `data-client configure`, e.g.: +Configure a profile with credentials and an API endpoint: -``` -$ data-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true +```bash +calypr-cli configure \ + --profile=myprofile \ + --cred=/path/to/credentials.json \ + --apiendpoint=https://example.com ``` -If this flag is set, the data-client will attempt to use the Gen3 Object Management API to upload files, falling back to Fence/Indexd in case of failure. +Check the resulting profile or auth state: -> You may also need to configure the version of the Gen3 Object Management API that the client will interact with. This is set to a default of Gen3 Object Management API `v2.0.0`, but can -> be raised or lowered by passing the `min-shepherd-version` flag to `data-client configure`, e.g.: -> -> ``` -> $ data-client configure --profile=myprofile --cred=/path/to/cred --apiendpoint=https://example.com --use-shepherd=true --min-shepherd-version=1.3.0 -> ``` - -### Uploading Additional File Object Metadata to Gen3 Object Management API +```bash +calypr-cli auth --profile=myprofile +``` -The Gen3 Object Management API supports uploading additional _public access_ file object metadata when uploading data files. +## Upload Behavior -> WARNING: Additional File Object Metadata is exposed publically and thus should not be controlled/sensitive data +`calypr-cli` supports the newer Gen3 object management upload flow via the +`--use-shepherd=true` configure flag. When enabled, uploads attempt that API +first and fall back to the older Fence/Indexd path if needed. -You can upload file metadata using the `data-client upload` command with the `--metadata` flag. E.g.: +Example: -``` -data-client upload --profile=my-profile --upload-path=/path/to/myfile.bam --metadata +```bash +calypr-cli configure \ + --profile=myprofile \ + --cred=/path/to/credentials.json \ + --apiendpoint=https://example.com \ + --use-shepherd=true ``` -This will tell `data-client` to look for a metadata file `myfile_metadata.json` in the same folder as `myfile.bam`. -A metadata file should be located in the same folder as the file to be uploaded, and should be named `[filename]_metadata.json`. +Uploads can also attach public file object metadata with `--metadata`. The CLI +looks for a sibling file named `[filename]_metadata.json`. -The metadata file should be a JSON file in the format: +Example: +```bash +calypr-cli upload \ + --profile=myprofile \ + --upload-path=/path/to/myfile.bam \ + --metadata ``` + +Metadata file shape: + +```json { - "authz": ["/example/authz/resource"], - "aliases": ["example_alias"], - "metadata": { - "any": { - "arbitrary": ["json", "metadata"] - } + "authz": ["/example/authz/resource"], + "aliases": ["example_alias"], + "metadata": { + "any": { + "arbitrary": ["json", "metadata"] } + } } ``` -The `aliases` and `metadata` properties are optional. Some Gen3 data commons require the `authz` property to be specified in order to upload a data file. - -If you do not know what `authz` to use, you can look at your `Profile` tab or `/identity` page of the Gen3 data commons you are uploading to. You will see a list of _authz resources_ in the format `/example/authz/resource`: these are the authz resources you have access to. +The `authz` list may be required by the target commons. The metadata in this +file is public-facing and should not contain sensitive content. diff --git a/arborist/client.go b/arborist/client.go new file mode 100644 index 0000000..f8e38eb --- /dev/null +++ b/arborist/client.go @@ -0,0 +1,169 @@ +package arborist + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/request" +) + +const DefaultOrgMemberRole = "org-member" + +type Client struct { + request.RequestInterface + Endpoint string +} + +type ClientInterface interface { + AuthMapping(ctx context.Context, out any) error + CreateOwnedDescendant(ctx context.Context, req CreateOwnedDescendantRequest, out any) error + AddOwner(ctx context.Context, req OwnerMutationRequest, out any) error + RemoveOwner(ctx context.Context, req OwnerMutationRequest) error + GetOwnershipResource(ctx context.Context, resourcePath string, includeChildren bool, includeAdmins bool, out any) error + GrantAccessUser(ctx context.Context, req AccessUserRequest, out any) error + RevokeAccessUser(ctx context.Context, req AccessUserRequest) error + GrantOrgMembership(ctx context.Context, organization string, username string, roleID string) error + RevokeOrgMembership(ctx context.Context, organization string, username string, roleID string) error +} + +type CreateOwnedDescendantRequest struct { + ParentPath string `json:"parent_path"` + Name string `json:"name"` + Template string `json:"template,omitempty"` + Description string `json:"description,omitempty"` +} + +type OwnerMutationRequest struct { + ResourcePath string `json:"resource_path"` + Username string `json:"username"` +} + +type AccessUserRequest struct { + ResourcePath string `json:"resource_path"` + Username string `json:"username"` + RoleID string `json:"role_id"` +} + +type errorResponse struct { + Error string `json:"error,omitempty"` + Message string `json:"message,omitempty"` +} + +func (e *errorResponse) ErrorMessage() string { + if strings.TrimSpace(e.Message) != "" { + return e.Message + } + return e.Error +} + +func NewClient(req request.RequestInterface, creds *conf.Credential) ClientInterface { + return &Client{ + RequestInterface: req, + Endpoint: creds.APIEndpoint, + } +} + +func (c *Client) AuthMapping(ctx context.Context, out any) error { + return c.do(ctx, http.MethodGet, "authz/mapping", nil, out, "get auth mapping") +} + +func (c *Client) CreateOwnedDescendant(ctx context.Context, req CreateOwnedDescendantRequest, out any) error { + return c.do(ctx, http.MethodPost, "authz/ownership/descendant", req, out, "create owned descendant") +} + +func (c *Client) AddOwner(ctx context.Context, req OwnerMutationRequest, out any) error { + return c.do(ctx, http.MethodPost, "authz/ownership/owner", req, out, "add owner") +} + +func (c *Client) RemoveOwner(ctx context.Context, req OwnerMutationRequest) error { + return c.do(ctx, http.MethodDelete, "authz/ownership/owner", req, nil, "remove owner") +} + +func (c *Client) GetOwnershipResource(ctx context.Context, resourcePath string, includeChildren bool, includeAdmins bool, out any) error { + query := url.Values{} + query.Set("resource_path", strings.TrimSpace(resourcePath)) + if includeChildren { + query.Set("include_children", "true") + } + if includeAdmins { + query.Set("include_admins", "true") + } + return c.do(ctx, http.MethodGet, "authz/ownership/resource?"+query.Encode(), nil, out, "get ownership resource") +} + +func (c *Client) GrantAccessUser(ctx context.Context, req AccessUserRequest, out any) error { + return c.do(ctx, http.MethodPost, "authz/access/user", normalizeAccessUserRequest(req), out, "grant direct user access") +} + +func (c *Client) RevokeAccessUser(ctx context.Context, req AccessUserRequest) error { + return c.do(ctx, http.MethodDelete, "authz/access/user", normalizeAccessUserRequest(req), nil, "revoke direct user access") +} + +func (c *Client) GrantOrgMembership(ctx context.Context, organization string, username string, roleID string) error { + payload, err := NewOrgMembershipRequest(organization, username, roleID) + if err != nil { + return err + } + return c.GrantAccessUser(ctx, payload, nil) +} + +func (c *Client) RevokeOrgMembership(ctx context.Context, organization string, username string, roleID string) error { + payload, err := NewOrgMembershipRequest(organization, username, roleID) + if err != nil { + return err + } + return c.RevokeAccessUser(ctx, payload) +} + +func NewOrgMembershipRequest(organization string, username string, roleID string) (AccessUserRequest, error) { + organization = strings.Trim(strings.TrimSpace(organization), "/") + username = strings.ToLower(strings.TrimSpace(username)) + roleID = strings.TrimSpace(roleID) + if roleID == "" { + roleID = DefaultOrgMemberRole + } + if organization == "" { + return AccessUserRequest{}, fmt.Errorf("organization is required") + } + if strings.Contains(organization, "/") { + return AccessUserRequest{}, fmt.Errorf("organization must be a single Calypr organization name, not a resource path") + } + if username == "" { + return AccessUserRequest{}, fmt.Errorf("username is required") + } + return AccessUserRequest{ + ResourcePath: "/programs/" + organization + "/projects", + Username: username, + RoleID: roleID, + }, nil +} + +func normalizeAccessUserRequest(req AccessUserRequest) AccessUserRequest { + req.ResourcePath = strings.TrimSpace(req.ResourcePath) + req.Username = strings.ToLower(strings.TrimSpace(req.Username)) + req.RoleID = strings.TrimSpace(req.RoleID) + return req +} + +func (c *Client) do(ctx context.Context, method string, apiPath string, body any, out any, action string) error { + url := request.JoinURL(c.Endpoint, apiPath) + if path, query, found := strings.Cut(apiPath, "?"); found { + url = request.JoinURL(c.Endpoint, path) + "?" + query + } + rb, err := request.NewJSON(c.RequestInterface, method, url, body) + if err != nil { + return err + } + return request.DoJSON( + ctx, + c.RequestInterface, + rb, + out, + request.WithAction(action), + request.WithErrorEnvelope(&errorResponse{}), + ) +} diff --git a/arborist/client_test.go b/arborist/client_test.go new file mode 100644 index 0000000..39e98d0 --- /dev/null +++ b/arborist/client_test.go @@ -0,0 +1,195 @@ +package arborist + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "testing" + + "github.com/calypr/calypr-cli/request" +) + +type fakeRequest struct { + lastBody map[string]any + lastMethod string + lastPath string + lastQuery string + statusCode int +} + +func (f *fakeRequest) New(method, rawURL string) *request.RequestBuilder { + return (&request.Request{}).New(method, rawURL) +} + +func (f *fakeRequest) Do(ctx context.Context, rb *request.RequestBuilder) (*http.Response, error) { + f.lastMethod = rb.Method + parsed, _ := url.Parse(rb.Url) + f.lastPath = parsed.Path + f.lastQuery = parsed.RawQuery + if rb.Body != nil { + body, _ := io.ReadAll(rb.Body) + _ = json.Unmarshal(body, &f.lastBody) + } + status := f.statusCode + if status == 0 { + status = http.StatusOK + } + return &http.Response{ + StatusCode: status, + Body: io.NopCloser(bytes.NewBufferString(`{"ok":true}`)), + }, nil +} + +func TestGrantOrgMembership(t *testing.T) { + req := &fakeRequest{} + client := &Client{RequestInterface: req, Endpoint: "https://example.org"} + + if err := client.GrantOrgMembership(context.Background(), "Ellrott_Lab", "USER@OHSU.EDU", ""); err != nil { + t.Fatalf("grant membership: %v", err) + } + if req.lastMethod != http.MethodPost { + t.Fatalf("expected POST, got %s", req.lastMethod) + } + if req.lastPath != "/authz/access/user" { + t.Fatalf("expected access user path, got %s", req.lastPath) + } + if req.lastBody["resource_path"] != "/programs/Ellrott_Lab/projects" { + t.Fatalf("expected org projects resource path, got %#v", req.lastBody["resource_path"]) + } + if req.lastBody["username"] != "user@ohsu.edu" { + t.Fatalf("expected normalized username, got %#v", req.lastBody["username"]) + } + if req.lastBody["role_id"] != "org-member" { + t.Fatalf("expected default org-member role, got %#v", req.lastBody["role_id"]) + } +} + +func TestRevokeOrgMembership(t *testing.T) { + req := &fakeRequest{} + client := &Client{RequestInterface: req, Endpoint: "https://example.org"} + + if err := client.RevokeOrgMembership(context.Background(), "cbds", "user@ohsu.edu", "writer"); err != nil { + t.Fatalf("revoke membership: %v", err) + } + if req.lastMethod != http.MethodDelete { + t.Fatalf("expected DELETE, got %s", req.lastMethod) + } + if req.lastPath != "/authz/access/user" { + t.Fatalf("expected access user path, got %s", req.lastPath) + } + if req.lastBody["resource_path"] != "/programs/cbds/projects" { + t.Fatalf("expected org projects resource path, got %#v", req.lastBody["resource_path"]) + } + if req.lastBody["role_id"] != "writer" { + t.Fatalf("expected explicit role, got %#v", req.lastBody["role_id"]) + } +} + +func TestOrgMembershipRejectsResourcePath(t *testing.T) { + if _, err := NewOrgMembershipRequest("/programs/cbds", "user@ohsu.edu", "owner"); err == nil { + t.Fatal("expected resource path organization to fail") + } +} + +func TestRouteFamilies(t *testing.T) { + tests := []struct { + name string + call func(*Client) error + wantMethod string + wantPath string + wantQuery string + }{ + { + name: "auth mapping", + call: func(c *Client) error { return c.AuthMapping(context.Background(), nil) }, + wantMethod: http.MethodGet, + wantPath: "/authz/mapping", + }, + { + name: "create owned descendant", + call: func(c *Client) error { + return c.CreateOwnedDescendant(context.Background(), CreateOwnedDescendantRequest{Name: "proj", ParentPath: "/programs/cbds/projects"}, nil) + }, + wantMethod: http.MethodPost, + wantPath: "/authz/ownership/descendant", + }, + { + name: "get ownership resource", + call: func(c *Client) error { + return c.GetOwnershipResource(context.Background(), "/programs/cbds/projects/demo", true, true, nil) + }, + wantMethod: http.MethodGet, + wantPath: "/authz/ownership/resource", + wantQuery: "include_admins=true&include_children=true&resource_path=%2Fprograms%2Fcbds%2Fprojects%2Fdemo", + }, + { + name: "grant access user", + call: func(c *Client) error { + return c.GrantAccessUser(context.Background(), AccessUserRequest{ + ResourcePath: "/programs/cbds/projects/demo", + Username: "USER@OHSU.EDU", + RoleID: "writer", + }, nil) + }, + wantMethod: http.MethodPost, + wantPath: "/authz/access/user", + }, + { + name: "revoke access user", + call: func(c *Client) error { + return c.RevokeAccessUser(context.Background(), AccessUserRequest{ + ResourcePath: "/programs/cbds/projects/demo", + Username: "USER@OHSU.EDU", + RoleID: "writer", + }) + }, + wantMethod: http.MethodDelete, + wantPath: "/authz/access/user", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := &fakeRequest{} + client := &Client{RequestInterface: req, Endpoint: "https://example.org"} + if err := tt.call(client); err != nil { + t.Fatalf("call failed: %v", err) + } + if req.lastMethod != tt.wantMethod { + t.Fatalf("expected method %s, got %s", tt.wantMethod, req.lastMethod) + } + if req.lastPath != tt.wantPath { + t.Fatalf("expected path %s, got %s", tt.wantPath, req.lastPath) + } + if req.lastQuery != tt.wantQuery { + t.Fatalf("expected query %s, got %s", tt.wantQuery, req.lastQuery) + } + }) + } +} + +func TestAccessUserNormalizesRequest(t *testing.T) { + req := &fakeRequest{} + client := &Client{RequestInterface: req, Endpoint: "https://example.org"} + + if err := client.GrantAccessUser(context.Background(), AccessUserRequest{ + ResourcePath: " /programs/cbds/projects/demo ", + Username: " USER@OHSU.EDU ", + RoleID: " writer ", + }, nil); err != nil { + t.Fatalf("grant access user: %v", err) + } + + if req.lastBody["resource_path"] != "/programs/cbds/projects/demo" { + t.Fatalf("expected normalized resource path, got %#v", req.lastBody["resource_path"]) + } + if req.lastBody["username"] != "user@ohsu.edu" { + t.Fatalf("expected normalized username, got %#v", req.lastBody["username"]) + } + if req.lastBody["role_id"] != "writer" { + t.Fatalf("expected normalized role id, got %#v", req.lastBody["role_id"]) + } +} diff --git a/build.sh b/build.sh deleted file mode 100755 index 213b40a..0000000 --- a/build.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# Adapted from 'How To Build Go Executables for Multiple Platforms on Ubuntu 16.04' -# By Marko Mudrinić -# -# Usage: -# ./build.sh - -if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then - echo "usage: $0" - echo "output: zipped executables to ./build directory" -fi - -package=$1 - -if [[ -z "$package" ]]; then - package='gen3-client' -fi - -package_split=(${package//\// }) -package_name=${package_split[-1]} - -platforms=( - "darwin/arm64" - "darwin/amd64" - "linux/amd64" - "windows/amd64" -) - -mkdir -p ./build -> checksums.txt -for platform in "${platforms[@]}" -do - platform_split=(${platform//\// }) - GOOS=${platform_split[0]} - GOARCH=${platform_split[1]} - output_name=$package_name'-'$GOOS'-'$GOARCH - exe_name=$package_name - - if [ $GOOS = "windows" ]; then - exe_name+='.exe' - - elif [ $GOOS = "darwin" ]; then - if [ $GOARCH = "arm64" ]; then - output_name=$package_name'-macos' - - elif [ $GOARCH = "amd64" ]; then - output_name=$package_name'-macos-intel' - fi - fi - - printf 'Building %s...' "$output_name" - env GOOS=$GOOS GOARCH=$GOARCH go build -o ./build/$exe_name . - cd build - zip -r -q $output_name $exe_name - sha256sum $output_name.zip >> checksums.txt - cd .. - - if [ $? -ne 0 ]; then - echo 'An error has occurred! Aborting the script execution...' - exit 1 - fi - echo 'OK' -done - -# Clean up build artifacts -rm build/{$package_name,$package_name.exe} - diff --git a/bump-tag.sh b/bump-tag.sh new file mode 100644 index 0000000..b38169e --- /dev/null +++ b/bump-tag.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# File: `bump-patch.sh` +set -euo pipefail + +# Find latest tag excluding major v0 +LATEST_TAG=$(git tag --list --sort=-v:refname | grep -v '^v0' | head -n1 || true) +if [ -z "$LATEST_TAG" ]; then + echo "No suitable tag found (excluding v0). Aborting." >&2 + exit 1 +fi + +# check that the working directory is clean +if [ -n "$(git status --porcelain)" ]; then + echo "Working directory is not clean. Please commit or stash changes before running this script." >&2 + exit 1 +fi + +usage() { + cat <<-EOF +Usage: $0 [--major | --minor | --patch] + +LATEST_TAG: $LATEST_TAG + +Options: + --major Bump major (MAJOR+1, MINOR=0, PATCH=0) + --minor Bump minor (MINOR+1, PATCH=0) + --patch Bump patch (PATCH+1) [default] +EOF + exit 1 +} + +# Parse options +opt_major=false +opt_minor=false +opt_patch=false +count=0 + +while [ $# -gt 0 ]; do + case "$1" in + --major) + opt_major=true + count=$((count + 1)) + shift + ;; + --minor) + opt_minor=true + count=$((count + 1)) + shift + ;; + --patch) + opt_patch=true + count=$((count + 1)) + shift + ;; + --help|-h) + usage + ;; + *) + echo "Unknown option: $1" >&2 + usage + ;; + esac +done + +# Default to patch if no option provided +if [ "$count" -eq 0 ]; then + opt_patch=true +fi + +# Disallow specifying more than one +if [ "$count" -gt 1 ]; then + echo "Specify only one of --major, --minor, or --patch" >&2 + exit 1 +fi + + +# Parse semver vMAJOR.MINOR.PATCH +if [[ "$LATEST_TAG" =~ ^v?([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" +else + echo "Latest tag '$LATEST_TAG' is not in semver format. Aborting." >&2 + exit 1 +fi + +# Compute new version +if [ "$opt_major" = true ]; then + NEW_MAJOR=$((MAJOR + 1)) + NEW_MINOR=0 + NEW_PATCH=0 + NEW_TAG="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" + NEW_FILE_VER="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" +elif [ "$opt_minor" = true ]; then + NEW_MAJOR=$MAJOR + NEW_MINOR=$((MINOR + 1)) + NEW_PATCH=0 + NEW_TAG="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" + NEW_FILE_VER="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" +else + # patch + NEW_MAJOR=$MAJOR + NEW_MINOR=$MINOR + NEW_PATCH=$((PATCH + 1)) + NEW_TAG="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" + NEW_FILE_VER="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" +fi + +BRANCH="$(git rev-parse --abbrev-ref HEAD)" + +echo "Latest branch: $BRANCH" +echo "Latest tag: $LATEST_TAG" +echo "New tag: $NEW_TAG (files will use ${NEW_FILE_VER})" + +# Update internal version file +if [ -f cmd/gitversion.go ]; then + # sed on mac is -i '' + sed -E -i '' -e "s/(gitversion *= *\")[^\"]+(\")/\1${NEW_FILE_VER}\2/" cmd/gitversion.go + git add cmd/gitversion.go +fi + +# Commit, tag and push +git commit -m "chore(release): bump to ${NEW_TAG}" || echo "No changes to commit" +git tag -a "${NEW_TAG}" -m "Release ${NEW_TAG}" +echo "Created tag. Please push tag ${NEW_TAG} on branch ${BRANCH}." + +echo git push origin "${BRANCH}" +echo git push origin "${NEW_TAG}" diff --git a/client/common/common.go b/client/common/common.go deleted file mode 100644 index 8eea7bf..0000000 --- a/client/common/common.go +++ /dev/null @@ -1,205 +0,0 @@ -package common - -import ( - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/hashicorp/go-multierror" - "github.com/vbauerster/mpb/v8" -) - -// DefaultUseShepherd sets whether gen3client will attempt to use the Shepherd / Object Management API -// endpoints if available. -// The user can override this default using the `data-client configure` command. -const DefaultUseShepherd = false - -// DefaultMinShepherdVersion is the minimum version of Shepherd that the gen3client will use. -// Before attempting to use Shepherd, the client will check for Shepherd's version, and if the version is -// below this number the gen3client will instead warn the user and fall back to fence/indexd. -// The user can override this default using the `data-client configure` command. -const DefaultMinShepherdVersion = "2.0.0" - -// ShepherdEndpoint is the endpoint postfix for SHEPHERD / the Object Management API -const ShepherdEndpoint = "/mds" - -// ShepherdVersionEndpoint is the endpoint used to check what version of Shepherd a commons has deployed -const ShepherdVersionEndpoint = "/mds/version" - -// IndexdIndexEndpoint is the endpoint postfix for INDEXD index -const IndexdIndexEndpoint = "/index/index" - -// FenceUserEndpoint is the endpoint postfix for FENCE user -const FenceUserEndpoint = "/user/user" - -// FenceDataEndpoint is the endpoint postfix for FENCE data -const FenceDataEndpoint = "/user/data" - -// FenceAccessTokenEndpoint is the endpoint postfix for FENCE access token -const FenceAccessTokenEndpoint = "/user/credentials/api/access_token" - -// FenceDataUploadEndpoint is the endpoint postfix for FENCE data upload -const FenceDataUploadEndpoint = FenceDataEndpoint + "/upload" - -// FenceDataDownloadEndpoint is the endpoint postfix for FENCE data download -const FenceDataDownloadEndpoint = FenceDataEndpoint + "/download" - -// FenceDataMultipartInitEndpoint is the endpoint postfix for FENCE multipart init -const FenceDataMultipartInitEndpoint = FenceDataEndpoint + "/multipart/init" - -// FenceDataMultipartUploadEndpoint is the endpoint postfix for FENCE multipart upload -const FenceDataMultipartUploadEndpoint = FenceDataEndpoint + "/multipart/upload" - -// FenceDataMultipartCompleteEndpoint is the endpoint postfix for FENCE multipart complete -const FenceDataMultipartCompleteEndpoint = FenceDataEndpoint + "/multipart/complete" - -// PathSeparator is os dependent path separator char -const PathSeparator = string(os.PathSeparator) - -// DefaultTimeout is used to set timeout value for http client -const DefaultTimeout = 120 * time.Second - -// FileUploadRequestObject defines a object for file upload -type FileUploadRequestObject struct { - FilePath string - Filename string - FileMetadata FileMetadata - GUID string - PresignedURL string - Request *http.Request - Progress *mpb.Progress - Bar *mpb.Bar - Bucket string `json:"bucket,omitempty"` -} - -// FileDownloadResponseObject defines a object for file download -type FileDownloadResponseObject struct { - DownloadPath string - Filename string - GUID string - URL string - Range int64 - Overwrite bool - Skip bool - Response *http.Response - Writer io.Writer -} - -// FileMetadata defines the metadata accepted by the new object management API, Shepherd -type FileMetadata struct { - Authz []string `json:"authz"` - Aliases []string `json:"aliases"` - // Metadata is an encoded JSON string of any arbitrary metadata the user wishes to upload. - Metadata map[string]any `json:"metadata"` -} - -// RetryObject defines a object for retry upload -type RetryObject struct { - FilePath string - Filename string - FileMetadata FileMetadata - GUID string - RetryCount int - Multipart bool - Bucket string -} - -// ParseRootPath parses dirname that has "~" in the beginning -func ParseRootPath(filePath string) (string, error) { - if filePath != "" && filePath[0] == '~' { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", err - } - return homeDir + filePath[1:], nil - } - return filePath, nil -} - -// GetAbsolutePath parses input file path to its absolute path and removes the "~" in the beginning -func GetAbsolutePath(filePath string) (string, error) { - fullFilePath, err := ParseRootPath(filePath) - if err != nil { - return "", err - } - fullFilePath, err = filepath.Abs(fullFilePath) - return fullFilePath, err -} - -// ParseFilePaths generates all possible file paths -func ParseFilePaths(filePath string, metadataEnabled bool) ([]string, error) { - fullFilePath, err := GetAbsolutePath(filePath) - if err != nil { - return []string{}, err - } - initialPaths, err := filepath.Glob(fullFilePath) - if err != nil { - return []string{}, err - } - - var multiErr *multierror.Error - var finalFilePaths []string - for _, p := range cleanupHiddenFiles(initialPaths) { - file, err := os.Open(p) - if err != nil { - multiErr = multierror.Append(multiErr, fmt.Errorf("file open error for %s: %w", p, err)) - continue - } - - func(filePath string, file *os.File) { - defer file.Close() - - fi, _ := file.Stat() - if fi.IsDir() { - err = filepath.Walk(filePath, func(path string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - isHidden, err := IsHidden(path) - if err != nil { - return err - } - isMetadata := false - if metadataEnabled { - isMetadata = strings.HasSuffix(path, "_metadata.json") - } - if !fileInfo.IsDir() && !isHidden && !isMetadata { - finalFilePaths = append(finalFilePaths, path) - } - return nil - }) - if err != nil { - multiErr = multierror.Append(multiErr, fmt.Errorf("directory walk error for %s: %w", filePath, err)) - } - } else { - finalFilePaths = append(finalFilePaths, filePath) - } - }(p, file) - } - - return finalFilePaths, multiErr.ErrorOrNil() -} - -func cleanupHiddenFiles(filePaths []string) []string { - i := 0 - for _, filePath := range filePaths { - isHidden, err := IsHidden(filePath) - if err != nil { - log.Println("Error occurred when checking hidden files: " + err.Error()) - continue - } - - if isHidden { - log.Printf("File %s is a hidden file and will be skipped\n", filePath) - continue - } - filePaths[i] = filePath - i++ - } - return filePaths[:i] -} diff --git a/client/common/isHidden_notwindows.go b/client/common/isHidden_notwindows.go deleted file mode 100644 index 321601c..0000000 --- a/client/common/isHidden_notwindows.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build !windows -// +build !windows - -package common - -import ( - "path/filepath" -) - -func IsHidden(filePath string) (bool, error) { - filename := filepath.Base(filePath) - if filename[0:1] == "." || filename[0:1] == "~" { // also takes care of temp files - return true, nil - } - return false, nil -} diff --git a/client/common/isHidden_windows.go b/client/common/isHidden_windows.go deleted file mode 100644 index b719ae2..0000000 --- a/client/common/isHidden_windows.go +++ /dev/null @@ -1,30 +0,0 @@ -//go:build windows -// +build windows - -package common - -import ( - "errors" - "path/filepath" - "runtime" - "syscall" -) - -func IsHidden(filePath string) (bool, error) { - filename := filepath.Base(filePath) - if runtime.GOOS == "windows" { - if filename[0:1] == "." || filename[0:1] == "~" { - return true, nil - } - pointer, err := syscall.UTF16PtrFromString(filePath) - if err != nil { - return false, err - } - attributes, err := syscall.GetFileAttributes(pointer) - if err != nil { - return false, err - } - return attributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil - } - return false, errors.New("Unable to check if file is hidden under non-Windows OS") -} diff --git a/client/common/logHelper.go b/client/common/logHelper.go deleted file mode 100644 index a117bbc..0000000 --- a/client/common/logHelper.go +++ /dev/null @@ -1,24 +0,0 @@ -package common - -import ( - "encoding/json" - "os" -) - -func LoadFailedLog(path string) (map[string]RetryObject, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var m map[string]RetryObject - if err := json.Unmarshal(data, &m); err != nil { - return nil, err - } - return m, nil -} - -func AlreadySucceededFromFile(filePath string) bool { - // Simple: check if any succeeded log contains this path - // Or just return false — safer to re-upload than skip - return false -} diff --git a/client/g3cmd/auth.go b/client/g3cmd/auth.go deleted file mode 100644 index 2dbd361..0000000 --- a/client/g3cmd/auth.go +++ /dev/null @@ -1,76 +0,0 @@ -package g3cmd - -import ( - "context" - "encoding/json" - "log" - "sort" - - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" -) - -func init() { - var profile string - var authCmd = &cobra.Command{ - Use: "auth", - Short: "Return resource access privileges from profile", - Long: `Gets resource access privileges for specified profile.`, - Example: `./data-client auth --profile=`, - Run: func(cmd *cobra.Command, args []string) { - // don't initialize transmission logs for non-uploading related commands - - logger, logCloser := logs.New(profile, logs.WithConsole()) - defer logCloser() - - g3i, err := client.NewGen3Interface(context.Background(), profile, logger) - if err != nil { - log.Fatalf("Fatal NewGen3Interface error: %s\n", err) - } - - host, resourceAccess, err := g3i.CheckPrivileges() - if err != nil { - g3i.Logger().Fatalf("Fatal authentication error: %s\n", err) - } else { - if len(resourceAccess) == 0 { - g3i.Logger().Printf("\nYou don't currently have access to any resources at %s\n", host) - } else { - g3i.Logger().Printf("\nYou have access to the following resource(s) at %s:\n", host) - - // Sort by resource name - resources := make([]string, 0, len(resourceAccess)) - for resource := range resourceAccess { - resources = append(resources, resource) - } - sort.Strings(resources) - - for _, project := range resources { - // Sort by access name if permissions are from Fence - permissions := resourceAccess[project].([]any) - _, isFencePermission := permissions[0].(string) - if isFencePermission { - access := make([]string, 0, len(permissions)) - for _, permission := range permissions { - access = append(access, permission.(string)) - } - sort.Strings(access) - g3i.Logger().Printf("%s %s\n", project, access) - } else { - // Permissions from Arborist already sorted, just pretty print them - marshalledPermissions, err := json.MarshalIndent(permissions, "", " ") - if err != nil { - g3i.Logger().Printf("%s (error occurred when marshalling permissions): %s\n", project, err) - } - g3i.Logger().Printf("%s %s\n", project, marshalledPermissions) - } - } - } - } - }, - } - - authCmd.Flags().StringVar(&profile, "profile", "", "Specify the profile to check your access privileges") - authCmd.MarkFlagRequired("profile") // nolint: errcheck - RootCmd.AddCommand(authCmd) -} diff --git a/client/g3cmd/delete.go b/client/g3cmd/delete.go deleted file mode 100644 index 5be6795..0000000 --- a/client/g3cmd/delete.go +++ /dev/null @@ -1,34 +0,0 @@ -package g3cmd - -import ( - "log" - - "github.com/spf13/cobra" -) - -//Not support yet, place holder only - -var deleteCmd = &cobra.Command{ // nolint:deadcode,unused,varcheck - Use: "delete", - Short: "Send DELETE HTTP Request for given URI", - Long: `Deletes a given URI from the database. -If no profile is specified, "default" profile is used for authentication.`, - Example: `./data-client delete --uri=v0/submission/bpa/test/entities/example_id - ./data-client delete --profile=user1 --uri=v0/submission/bpa/test/entities/1af1d0ab-efec-4049-98f0-ae0f4bb1bc64`, - Run: func(cmd *cobra.Command, args []string) { - log.Fatalf("Not supported!") - // request := new(jwt.Request) - // configure := new(jwt.Configure) - // function := new(jwt.Functions) - - // function.Config = configure - // function.Request = request - - // fmt.Println(jwt.ResponseToString( - // function.DoRequestWithSignedHeader(RequestDelete, profile, "txt", uri))) - }, -} - -func init() { - // RootCmd.AddCommand(deleteCmd) -} diff --git a/client/g3cmd/download-multiple.go b/client/g3cmd/download-multiple.go deleted file mode 100644 index d8dbca8..0000000 --- a/client/g3cmd/download-multiple.go +++ /dev/null @@ -1,495 +0,0 @@ -package g3cmd - -import ( - "bufio" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "log" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/vbauerster/mpb/v8" - "github.com/vbauerster/mpb/v8/decor" - - "github.com/spf13/cobra" -) - -// mockgen -destination=../mocks/mock_gen3interface.go -package=mocks . Gen3Interface - -func AskGen3ForFileInfo(g3i client.Gen3Interface, guid string, protocol string, downloadPath string, filenameFormat string, rename bool, renamedFiles *[]RenamedOrSkippedFileInfo) (string, int64) { - var fileName string - var fileSize int64 - - // If the commons has the newer Shepherd API deployed, get the filename and file size from the Shepherd API. - // Otherwise, fall back on Indexd and Fence. - hasShepherd, err := g3i.CheckForShepherdAPI() - if err != nil { - g3i.Logger().Println("Error occurred when checking for Shepherd API: " + err.Error()) - g3i.Logger().Println("Falling back to Indexd...") - } - if hasShepherd { - endPointPostfix := common.ShepherdEndpoint + "/objects/" + guid - _, res, err := g3i.GetResponse(endPointPostfix, "GET", "", nil) - if err != nil { - g3i.Logger().Println("Error occurred when querying filename from Shepherd: " + err.Error()) - g3i.Logger().Println("Using GUID for filename instead.") - if filenameFormat != "guid" { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - } - return guid, 0 - } - - decoded := struct { - Record struct { - FileName string `json:"file_name"` - Size int64 `json:"size"` - } - }{} - err = json.NewDecoder(res.Body).Decode(&decoded) - if err != nil { - g3i.Logger().Println("Error occurred when reading response from Shepherd: " + err.Error()) - g3i.Logger().Println("Using GUID for filename instead.") - if filenameFormat != "guid" { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - } - return guid, 0 - } - defer res.Body.Close() - - fileName = decoded.Record.FileName - fileSize = decoded.Record.Size - - } else { - // Attempt to get the filename from Indexd - endPointPostfix := common.IndexdIndexEndpoint + "/" + guid - indexdMsg, err := g3i.DoRequestWithSignedHeader(endPointPostfix, "", nil) - if err != nil { - g3i.Logger().Println("Error occurred when querying filename from IndexD: " + err.Error()) - g3i.Logger().Println("Using GUID for filename instead.") - if filenameFormat != "guid" { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - } - return guid, 0 - } - - if filenameFormat == "guid" { - return guid, indexdMsg.Size - } - - actualFilename := indexdMsg.FileName - if actualFilename == "" { - if len(indexdMsg.URLs) > 0 { - // Indexd record has no file name but does have URLs, try to guess file name from URL - var indexdURL = indexdMsg.URLs[0] - if protocol != "" { - for _, url := range indexdMsg.URLs { - if strings.HasPrefix(url, protocol) { - indexdURL = url - } - } - } - - actualFilename = guessFilenameFromURL(indexdURL) - if actualFilename == "" { - g3i.Logger().Println("Error occurred when guessing filename for object " + guid) - g3i.Logger().Println("Using GUID for filename instead.") - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - return guid, indexdMsg.Size - } - } else { - // Neither file name nor URLs exist in the Indexd record - // Indexd record is busted for that file, just return as we are renaming the file for now - // The download logic will handle the errors - g3i.Logger().Println("Neither file name nor URLs exist in the Indexd record of " + guid) - g3i.Logger().Println("The attempt of downloading file is likely to fail! Check Indexd record!") - g3i.Logger().Println("Using GUID for filename instead.") - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - return guid, indexdMsg.Size - } - } - - fileName = actualFilename - fileSize = indexdMsg.Size - } - - if filenameFormat == "original" { - if !rename { // no renaming in original mode - return fileName, fileSize - } - newFilename := processOriginalFilename(downloadPath, fileName) - if fileName != newFilename { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: fileName, NewFilename: newFilename}) - } - return newFilename, fileSize - } - // filenameFormat == "combined" - combinedFilename := guid + "_" + fileName - return combinedFilename, fileSize -} - -func guessFilenameFromURL(URL string) string { - splittedURLWithFilename := strings.Split(URL, "/") - actualFilename := splittedURLWithFilename[len(splittedURLWithFilename)-1] - return actualFilename -} - -func processOriginalFilename(downloadPath string, actualFilename string) string { - _, err := os.Stat(downloadPath + actualFilename) - if os.IsNotExist(err) { - return actualFilename - } - extension := filepath.Ext(actualFilename) - filename := strings.TrimSuffix(actualFilename, extension) - counter := 2 - for { - newFilename := filename + "_" + strconv.Itoa(counter) + extension - _, err := os.Stat(downloadPath + newFilename) - if os.IsNotExist(err) { - return newFilename - } - counter++ - } -} - -func validateLocalFileStat(logger logs.Logger, downloadPath string, filename string, filesize int64, skipCompleted bool) common.FileDownloadResponseObject { - fi, err := os.Stat(downloadPath + filename) // check filename for local existence - if err != nil { - if os.IsNotExist(err) { - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename} // no local file, normal full length download - } - logger.Printf("Error occurred when getting information for file \"%s\": %s\n", downloadPath+filename, err.Error()) - logger.Println("Will try to download the whole file") - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename} // errorred when trying to get local FI, normal full length download - } - - // have existing local file and may want to skip, check more conditions - if !skipCompleted { - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename, Overwrite: true} // not skipping any local files, normal full length download - } - - localFilesize := fi.Size() - if localFilesize == filesize { - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename, Skip: true} // both filename and filesize matches, consider as completed - } - if localFilesize > filesize { - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename, Overwrite: true} // local filesize is greater than INDEXD record, overwrite local existing - } - // local filesize is less than INDEXD record, try ranged download - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename, Range: localFilesize} -} - -func batchDownload(g3 client.Gen3Interface, progress *mpb.Progress, batchFDRSlice []common.FileDownloadResponseObject, protocolText string, workers int, errCh chan error) int { - fdrs := make([]common.FileDownloadResponseObject, 0) - for _, fdrObject := range batchFDRSlice { - err := GetDownloadResponse(g3, &fdrObject, protocolText) - if err != nil { - errCh <- err - continue - } - - fileFlag := os.O_CREATE | os.O_RDWR - if fdrObject.Range != 0 { - fileFlag = os.O_APPEND | os.O_RDWR - } else if fdrObject.Overwrite { - fileFlag = os.O_TRUNC | os.O_RDWR - } - - subDir := filepath.Dir(fdrObject.Filename) - if subDir != "." && subDir != "/" { - err = os.MkdirAll(fdrObject.DownloadPath+subDir, 0766) - if err != nil { - errCh <- err - continue - } - } - file, err := os.OpenFile(fdrObject.DownloadPath+fdrObject.Filename, fileFlag, 0666) - if err != nil { - errCh <- errors.New("Error occurred during opening local file: " + err.Error()) - continue - } - total := fdrObject.Response.ContentLength + fdrObject.Range - bar := progress.AddBar(total, - mpb.PrependDecorators( - decor.Name(fdrObject.Filename+" "), - decor.CountersKibiByte("% .1f / % .1f"), - ), - mpb.AppendDecorators( - decor.Percentage(), - decor.AverageSpeed(decor.SizeB1024(0), " % .1f"), - ), - ) - if fdrObject.Range > 0 { - bar.SetCurrent(fdrObject.Range) - } - writer := bar.ProxyWriter(file) - fdrObject.Writer = writer - fdrs = append(fdrs, fdrObject) - defer file.Close() - defer fdrObject.Response.Body.Close() - } - - fdrCh := make(chan common.FileDownloadResponseObject, len(fdrs)) - wg := sync.WaitGroup{} - succeeded := 0 - var err error - for range workers { - wg.Add(1) - go func() { - for fdr := range fdrCh { - if _, err = io.Copy(fdr.Writer, fdr.Response.Body); err != nil { - errCh <- errors.New("io.Copy error: " + err.Error()) - return - } - succeeded++ - } - wg.Done() - }() - } - - for _, fdr := range fdrs { - fdrCh <- fdr - } - close(fdrCh) - - wg.Wait() - return succeeded -} - -// AskForConfirmation asks user for confirmation before proceed, will wait if user entered garbage -func AskForConfirmation(logger logs.Logger, s string) bool { - reader := bufio.NewReader(os.Stdin) - - for { - logger.Printf("%s [y/n]: ", s) - - response, err := reader.ReadString('\n') - if err != nil { - logger.Fatal("Error occurred during parsing user's confirmation: " + err.Error()) - } - - switch strings.ToLower(strings.TrimSpace(response)) { - case "y", "yes": - return true - case "n", "no": - return false - default: - return false // Example of defaulting to false - } - } -} - -func downloadFile(g3i client.Gen3Interface, objects []ManifestObject, downloadPath string, filenameFormat string, rename bool, noPrompt bool, protocol string, numParallel int, skipCompleted bool) error { - if numParallel < 1 { - return fmt.Errorf("invalid value for option \"numparallel\": must be a positive integer! Please check your input") - } - - downloadPath, err := common.ParseRootPath(downloadPath) - if err != nil { - return fmt.Errorf("downloadFile Error: %s", err.Error()) - } - if !strings.HasSuffix(downloadPath, "/") { - downloadPath += "/" - } - filenameFormat = strings.ToLower(strings.TrimSpace(filenameFormat)) - if (filenameFormat == "guid" || filenameFormat == "combined") && rename { - g3i.Logger().Println("NOTICE: flag \"rename\" only works if flag \"filename-format\" is \"original\"") - rename = false - } - - if filenameFormat != "original" && filenameFormat != "guid" && filenameFormat != "combined" { - return fmt.Errorf("invalid option found! option \"filename-format\" can either be \"original\", \"guid\" or \"combined\" only") - } - if filenameFormat == "guid" || filenameFormat == "combined" { - g3i.Logger().Printf("WARNING: in \"guid\" or \"combined\" mode, duplicated files under \"%s\" will be overwritten\n", downloadPath) - if !noPrompt && !AskForConfirmation(g3i.Logger(), "Proceed?") { - g3i.Logger().Fatal("Aborted by user") - } - } else if !rename { - g3i.Logger().Printf("WARNING: flag \"rename\" was set to false in \"original\" mode, duplicated files under \"%s\" will be overwritten\n", downloadPath) - if !noPrompt && !AskForConfirmation(g3i.Logger(), "Proceed?") { - g3i.Logger().Fatal("Aborted by user") - } - } else { - g3i.Logger().Printf("NOTICE: flag \"rename\" was set to true in \"original\" mode, duplicated files under \"%s\" will be renamed by appending a counter value to the original filenames\n", downloadPath) - } - - protocolText := "" - if protocol != "" { - protocolText = "?protocol=" + protocol - } - - err = os.MkdirAll(downloadPath, 0766) - if err != nil { - return fmt.Errorf("cannot create folder %s", downloadPath) - } - - renamedFiles := make([]RenamedOrSkippedFileInfo, 0) - skippedFiles := make([]RenamedOrSkippedFileInfo, 0) - fdrObjects := make([]common.FileDownloadResponseObject, 0) - - g3i.Logger().Printf("Total number of objects in manifest: %d\n", len(objects)) - g3i.Logger().Println("Preparing file info for each file, please wait...") - fileInfoProgress := mpb.New(mpb.WithOutput(os.Stdout)) - fileInfoBar := fileInfoProgress.AddBar(int64(len(objects)), - mpb.PrependDecorators( - decor.Name("Preparing files "), - decor.CountersNoUnit("%d / %d"), - ), - mpb.AppendDecorators(decor.Percentage()), - ) - for _, obj := range objects { - if obj.ObjectID == "" { - g3i.Logger().Println("Found empty object_id (GUID), skipping this entry") - continue - } - var fdrObject common.FileDownloadResponseObject - filename := obj.Filename - filesize := obj.Filesize - // only queries Gen3 services if any of these 2 values doesn't exists in manifest - if filename == "" || filesize == 0 { - filename, filesize = AskGen3ForFileInfo(g3i, obj.ObjectID, protocol, downloadPath, filenameFormat, rename, &renamedFiles) - } - fdrObject = common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename} - if !rename { - fdrObject = validateLocalFileStat(g3i.Logger(), downloadPath, filename, filesize, skipCompleted) - } - fdrObject.GUID = obj.ObjectID - fdrObjects = append(fdrObjects, fdrObject) - fileInfoBar.Increment() - } - fileInfoProgress.Wait() - g3i.Logger().Println("File info prepared successfully") - - totalCompeleted := 0 - workers, _, errCh, _ := initBatchUploadChannels(numParallel, len(fdrObjects)) - downloadProgress := mpb.New(mpb.WithOutput(os.Stdout)) - batchFDRSlice := make([]common.FileDownloadResponseObject, 0) - for _, fdrObject := range fdrObjects { - if fdrObject.Skip { - g3i.Logger().Printf("File \"%s\" (GUID: %s) has been skipped because there is a complete local copy\n", fdrObject.Filename, fdrObject.GUID) - skippedFiles = append(skippedFiles, RenamedOrSkippedFileInfo{GUID: fdrObject.GUID, OldFilename: fdrObject.Filename}) - continue - } - - if len(batchFDRSlice) < workers { - batchFDRSlice = append(batchFDRSlice, fdrObject) - } else { - totalCompeleted += batchDownload(g3i, downloadProgress, batchFDRSlice, protocolText, workers, errCh) - batchFDRSlice = make([]common.FileDownloadResponseObject, 0) - batchFDRSlice = append(batchFDRSlice, fdrObject) - } - } - totalCompeleted += batchDownload(g3i, downloadProgress, batchFDRSlice, protocolText, workers, errCh) // download remainders - downloadProgress.Wait() - - g3i.Logger().Printf("%d files downloaded.\n", totalCompeleted) - - if len(renamedFiles) > 0 { - g3i.Logger().Printf("%d files have been renamed as the following:\n", len(renamedFiles)) - for _, rfi := range renamedFiles { - g3i.Logger().Printf("File \"%s\" (GUID: %s) has been renamed as: %s\n", rfi.OldFilename, rfi.GUID, rfi.NewFilename) - } - } - if len(skippedFiles) > 0 { - g3i.Logger().Printf("%d files have been skipped\n", len(skippedFiles)) - } - if len(errCh) > 0 { - close(errCh) - g3i.Logger().Printf("%d files have encountered an error during downloading, detailed error messages are:\n", len(errCh)) - for err := range errCh { - g3i.Logger().Println(err.Error()) - } - } - return nil -} - -func init() { - var manifestPath string - var downloadPath string - var filenameFormat string - var rename bool - var noPrompt bool - var protocol string - var numParallel int - var skipCompleted bool - - var downloadMultipleCmd = &cobra.Command{ - Use: "download-multiple", - Short: "Download multiple of files from a specified manifest", - Long: `Get presigned URLs for multiple of files specified in a manifest file and then download all of them.`, - Example: `./data-client download-multiple --profile= --manifest= --download-path=`, - Run: func(cmd *cobra.Command, args []string) { - // don't initialize transmission logs for non-uploading related commands - - logger, logCloser := logs.New(profile, logs.WithConsole(), logs.WithFailedLog(), logs.WithScoreboard(), logs.WithSucceededLog()) - defer logCloser() - - g3i, err := client.NewGen3Interface(context.Background(), profile, logger) - if err != nil { - log.Fatalf("Failed to parse config on profile %s, %v", profile, err) - } - - manifestPath, _ = common.GetAbsolutePath(manifestPath) - manifestFile, err := os.Open(manifestPath) - if err != nil { - g3i.Logger().Fatalf("Failed to open manifest file %s, %v\n", manifestPath, err) - } - defer manifestFile.Close() - manifestFileStat, err := manifestFile.Stat() - if err != nil { - g3i.Logger().Fatalf("Failed to get manifest file stats %s, %v\n", manifestPath, err) - } - g3i.Logger().Println("Reading manifest...") - manifestFileSize := manifestFileStat.Size() - manifestProgress := mpb.New(mpb.WithOutput(os.Stdout)) - manifestFileBar := manifestProgress.AddBar(manifestFileSize, - mpb.PrependDecorators( - decor.Name("Manifest "), - decor.CountersKibiByte("% .1f / % .1f"), - ), - mpb.AppendDecorators(decor.Percentage()), - ) - - manifestFileReader := manifestFileBar.ProxyReader(manifestFile) - - manifestBytes, err := io.ReadAll(manifestFileReader) - if err != nil { - g3i.Logger().Fatalf("Failed reading manifest %s, %v\n", manifestPath, err) - } - manifestProgress.Wait() - - var objects []ManifestObject - err = json.Unmarshal(manifestBytes, &objects) - if err != nil { - g3i.Logger().Fatalf("Error has occurred during unmarshalling manifest object: %v\n", err) - } - - err = downloadFile(g3i, objects, downloadPath, filenameFormat, rename, noPrompt, protocol, numParallel, skipCompleted) - if err != nil { - g3i.Logger().Fatal(err.Error()) - } - }, - } - - downloadMultipleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - downloadMultipleCmd.MarkFlagRequired("profile") //nolint:errcheck - downloadMultipleCmd.Flags().StringVar(&manifestPath, "manifest", "", "The manifest file to read from. A valid manifest can be acquired by using the \"Download Manifest\" button in Data Explorer from a data common's portal") - downloadMultipleCmd.MarkFlagRequired("manifest") //nolint:errcheck - downloadMultipleCmd.Flags().StringVar(&downloadPath, "download-path", ".", "The directory in which to store the downloaded files") - downloadMultipleCmd.Flags().StringVar(&filenameFormat, "filename-format", "original", "The format of filename to be used, including \"original\", \"guid\" and \"combined\"") - downloadMultipleCmd.Flags().BoolVar(&rename, "rename", false, "Only useful when \"--filename-format=original\", will rename file by appending a counter value to its filename if set to true, otherwise the same filename will be used") - downloadMultipleCmd.Flags().BoolVar(&noPrompt, "no-prompt", false, "If set to true, will not display user prompt message for confirmation") - downloadMultipleCmd.Flags().StringVar(&protocol, "protocol", "", "Specify the preferred protocol with --protocol=s3") - downloadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 1, "Number of downloads to run in parallel") - downloadMultipleCmd.Flags().BoolVar(&skipCompleted, "skip-completed", false, "If set to true, will check for filename and size before download and skip any files in \"download-path\" that matches both") - RootCmd.AddCommand(downloadMultipleCmd) -} diff --git a/client/g3cmd/download-single.go b/client/g3cmd/download-single.go deleted file mode 100644 index 6038f23..0000000 --- a/client/g3cmd/download-single.go +++ /dev/null @@ -1,60 +0,0 @@ -package g3cmd - -import ( - "context" - "log" - - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" -) - -func init() { - var guid string - var downloadPath string - var protocol string - var filenameFormat string - var rename bool - var noPrompt bool - var skipCompleted bool - var profile string - - var downloadSingleCmd = &cobra.Command{ - Use: "download-single", - Short: "Download a single file from a GUID", - Long: `Gets a presigned URL for a file from a GUID and then downloads the specified file.`, - Example: `./data-client download-single --profile= --guid=206dfaa6-bcf1-4bc9-b2d0-77179f0f48fc`, - Run: func(cmd *cobra.Command, args []string) { - // don't initialize transmission logs for non-uploading related commands - - logger, logCloser := logs.New(profile, logs.WithConsole(), logs.WithFailedLog(), logs.WithSucceededLog(), logs.WithScoreboard()) - defer logCloser() - - g3I, err := client.NewGen3Interface(context.Background(), profile, logger) - if err != nil { - log.Fatalf("Failed to parse config on profile %s, %v", profile, err) - } - - obj := ManifestObject{ - ObjectID: guid, - } - objects := []ManifestObject{obj} - err = downloadFile(g3I, objects, downloadPath, filenameFormat, rename, noPrompt, protocol, 1, skipCompleted) - if err != nil { - g3I.Logger().Println(err.Error()) - } - }, - } - - downloadSingleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - downloadSingleCmd.MarkFlagRequired("profile") //nolint:errcheck - downloadSingleCmd.Flags().StringVar(&guid, "guid", "", "Specify the guid for the data you would like to work with") - downloadSingleCmd.MarkFlagRequired("guid") //nolint:errcheck - downloadSingleCmd.Flags().StringVar(&downloadPath, "download-path", ".", "The directory in which to store the downloaded files") - downloadSingleCmd.Flags().StringVar(&filenameFormat, "filename-format", "original", "The format of filename to be used, including \"original\", \"guid\" and \"combined\"") - downloadSingleCmd.Flags().BoolVar(&rename, "rename", false, "Only useful when \"--filename-format=original\", will rename file by appending a counter value to its filename if set to true, otherwise the same filename will be used") - downloadSingleCmd.Flags().BoolVar(&noPrompt, "no-prompt", false, "If set to true, will not display user prompt message for confirmation") - downloadSingleCmd.Flags().StringVar(&protocol, "protocol", "", "Specify the preferred protocol with --protocol=gs") - downloadSingleCmd.Flags().BoolVar(&skipCompleted, "skip-completed", false, "If set to true, will check for filename and size before download and skip any files in \"download-path\" that matches both") - RootCmd.AddCommand(downloadSingleCmd) -} diff --git a/client/g3cmd/generate-tsv.go b/client/g3cmd/generate-tsv.go deleted file mode 100644 index 9abff77..0000000 --- a/client/g3cmd/generate-tsv.go +++ /dev/null @@ -1,17 +0,0 @@ -package g3cmd - -import ( - "github.com/spf13/cobra" -) - -func init() { - var generateTSVCmd = &cobra.Command{ - Use: "generate-tsv", - Short: "Generate a file upload tsv from a template", - Long: `Fills in a Gen3 data file template with information from a directory of files.`, - Deprecated: "please use an older version of data-client", - Run: func(cmd *cobra.Command, args []string) {}, - } - - RootCmd.AddCommand(generateTSVCmd) -} diff --git a/client/g3cmd/gitversion.go b/client/g3cmd/gitversion.go deleted file mode 100644 index cb3a308..0000000 --- a/client/g3cmd/gitversion.go +++ /dev/null @@ -1,6 +0,0 @@ -package g3cmd - -var ( - gitcommit = "N/A" - gitversion = "2023.11" -) diff --git a/client/g3cmd/retry-upload.go b/client/g3cmd/retry-upload.go deleted file mode 100644 index edd5c52..0000000 --- a/client/g3cmd/retry-upload.go +++ /dev/null @@ -1,215 +0,0 @@ -package g3cmd - -import ( - "context" - "os" - "path/filepath" - "time" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - - "github.com/spf13/cobra" -) - -func handleFailedRetry(g3i client.Gen3Interface, ro common.RetryObject, retryObjCh chan common.RetryObject, err error) { - logger := g3i.Logger() - - // Record failure in JSON log - logger.Failed(ro.FilePath, ro.Filename, ro.FileMetadata, ro.GUID, ro.RetryCount, ro.Multipart) - - if err != nil { - logger.Println("Error:", err) - } - - if ro.RetryCount < MaxRetryCount { - retryObjCh <- ro - return - } - - // Max retries reached — clean up - if ro.GUID != "" { - if msg, err := DeleteRecord(g3i, ro.GUID); err == nil { - logger.Println(msg) - } else { - logger.Println("Cleanup failed:", err) - } - } - - // Final failure - sb, err := logs.FromSBContext(context.Background()) - if err != nil { - logger.Println(err) - } - sb.IncrementSB(MaxRetryCount + 1) - - if len(retryObjCh) == 0 { - close(retryObjCh) - logger.Println("Retry channel closed — all done") - } -} - -func retryUpload(g3i client.Gen3Interface, failedLogMap map[string]common.RetryObject) { - logger := g3i.Logger() - - sb, err := logs.FromSBContext(context.Background()) - if err != nil { - logger.Println(err) - } - - if len(failedLogMap) == 0 { - logger.Println("No failed files to retry.") - return - } - - logger.Println("Starting retry-upload...") - retryObjCh := make(chan common.RetryObject, len(failedLogMap)) - - // Load failed entries (skip already succeeded ones) - for _, ro := range failedLogMap { - // Simple check: if succeeded log exists and contains this path, skip - if common.AlreadySucceededFromFile(ro.FilePath) { - logger.Printf("Already uploaded: %s — skipping\n", ro.FilePath) - continue - } - retryObjCh <- ro - } - - if len(retryObjCh) == 0 { - logger.Println("All failed files were already successfully uploaded in a previous run.") - return - } - - for ro := range retryObjCh { - ro.RetryCount++ - logger.Printf("#%d retry — %s\n", ro.RetryCount, ro.FilePath) - logger.Printf("Waiting %.0f seconds...\n", GetWaitTime(ro.RetryCount).Seconds()) - time.Sleep(GetWaitTime(ro.RetryCount)) - - // Optional: delete old record - if ro.GUID != "" { - if msg, err := DeleteRecord(g3i, ro.GUID); err == nil { - logger.Println(msg) - } - } - - // Fix missing filename if needed - if ro.Filename == "" { - absPath, _ := common.GetAbsolutePath(ro.FilePath) - ro.Filename = filepath.Base(absPath) - } - - var err error - if ro.Multipart { - // Multipart retry - req := common.FileUploadRequestObject{ - FilePath: ro.FilePath, - Filename: ro.Filename, - GUID: ro.GUID, - } - err = MultipartUpload(context.Background(), g3i, req, ro.Bucket, true) - if err == nil { - logger.Succeeded(ro.FilePath, req.GUID) - sb.IncrementSB(ro.RetryCount - 1) // success on this retry - continue - } - } else { - // Single-part retry - var presignedURL, guid string - presignedURL, guid, err = GeneratePresignedURL(g3i, ro.Filename, ro.FileMetadata, ro.Bucket) - if err != nil { - handleFailedRetry(g3i, ro, retryObjCh, err) - continue - } - - file, err := os.Open(ro.FilePath) - if err != nil { - handleFailedRetry(g3i, ro, retryObjCh, err) - continue - } - stat, _ := file.Stat() - file.Close() - - if stat.Size() > FileSizeLimit { - ro.Multipart = true - retryObjCh <- ro - continue - } - - fur := common.FileUploadRequestObject{ - FilePath: ro.FilePath, - Filename: ro.Filename, - FileMetadata: ro.FileMetadata, - GUID: guid, - PresignedURL: presignedURL, - } - - fur, err = GenerateUploadRequest(g3i, fur, nil, nil) - if err != nil { - handleFailedRetry(g3i, ro, retryObjCh, err) - continue - } - - err = uploadFile(g3i, fur, ro.RetryCount) - if err != nil { - handleFailedRetry(g3i, ro, retryObjCh, err) - continue - } - - logger.Succeeded(ro.FilePath, fur.GUID) - sb.IncrementSB(ro.RetryCount - 1) - } - - if len(retryObjCh) == 0 { - close(retryObjCh) - } - } -} - -func init() { - var failedLogPath, profile string - - var retryUploadCmd = &cobra.Command{ - Use: "retry-upload", - Short: "Retry failed uploads from a failed_log.json", - Long: `Re-uploads files listed in a failed log using exponential backoff and progress bars.`, - Example: `./data-client retry-upload --profile=myprofile --failed-log-path=/path/to/failed_log.json`, - Run: func(cmd *cobra.Command, args []string) { - Logger, closer := logs.New(profile, - logs.WithConsole(), - logs.WithMessageFile(), - logs.WithFailedLog(), - logs.WithSucceededLog(), - ) - defer closer() - - g3, err := client.NewGen3Interface(context.Background(), profile, Logger) - if err != nil { - Logger.Fatalf("Failed to initialize client: %v", err) - } - - logger := g3.Logger() - - // Create scoreboard with our logger injected - sb := logs.NewSB(MaxRetryCount, logger) - - // Load failed log - failedMap, err := common.LoadFailedLog(failedLogPath) - if err != nil { - logger.Fatalf("Cannot read failed log: %v", err) - } - - retryUpload(g3, failedMap) - sb.PrintSB() - }, - } - - retryUploadCmd.Flags().StringVar(&profile, "profile", "", "Profile to use") - retryUploadCmd.MarkFlagRequired("profile") - - retryUploadCmd.Flags().StringVar(&failedLogPath, "failed-log-path", "", "Path to failed_log.json") - retryUploadCmd.MarkFlagRequired("failed-log-path") - - RootCmd.AddCommand(retryUploadCmd) -} diff --git a/client/g3cmd/root.go b/client/g3cmd/root.go deleted file mode 100644 index 8bc7ab9..0000000 --- a/client/g3cmd/root.go +++ /dev/null @@ -1,124 +0,0 @@ -package g3cmd - -import ( - "encoding/json" - "net/http" - "os" - "strconv" - "time" - - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" - "golang.org/x/mod/semver" -) - -var profile string - -// Package-level variable to hold the closer function -// (Assuming logs.Closer is a type that can hold a function, like func() error) -var logCloser func() - -// Or just: -// var logCloser io.Closer // if closer implements io.Closer - -// RootCmd represents the base command when called without any subcommands -var RootCmd = &cobra.Command{ - Use: "data-client", - Short: "Use the data-client to interact with a Gen3 Data Commons", - Long: "Gen3 Client for downloading, uploading and submitting data to data commons.\ndata-client version: " + gitversion + ", commit: " + gitcommit, - Version: gitversion, -} - -// Execute adds all child commands to the root command sets flags appropriately -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if logCloser != nil { - defer func() { - logCloser() - }() - } - - if err := RootCmd.Execute(); err != nil { - os.Stderr.WriteString("Error: " + err.Error() + "\n") - os.Exit(1) - } -} - -func init() { - cobra.OnInitialize(initConfig) - - // Define flags and configuration settings. - RootCmd.PersistentFlags().StringVar(&profile, "profile", "", "Specify profile to use") - _ = RootCmd.MarkFlagRequired("profile") -} - -type GitHubRelease struct { - TagName string `json:"tag_name"` -} - -func initConfig() { - // The logger is needed throughout the application, so we don't store it here, - // but the closer must be stored. - logger, closer := logs.New(profile, - logs.WithConsole(), - logs.WithMessageFile(), - logs.WithFailedLog(), - logs.WithSucceededLog(), - ) - - // 2. ASSIGN CLOSER TO PACKAGE VARIABLE - logCloser = closer - - // The rest of the function remains the same, except for removing the 'defer resp.Body.Close()' - // from the initConfig body, as that was unrelated to the logs closer. - // The rest of your original logic follows... - - conf := jwt.Configure{} - // init local config file - err := conf.InitConfigFile() - if err != nil { - logger.Fatal("Error occurred when trying to init config file: " + err.Error()) - } - - // version checker - if os.Getenv("GEN3_CLIENT_VERSION_CHECK") != "false" && - gitversion != "" && gitversion != "N/A" { - - const ( - owner = "uc-cdis" - repository = "cdis-data-client" - // The official GitHub API endpoint for the latest release - apiURL = "https://api.github.com/repos/" + owner + "/" + repository + "/releases/latest" - ) - - client := http.Client{Timeout: 5 * time.Second} - resp, err := client.Get(apiURL) - if err != nil { - logger.Println("Error occurred when fetching latest version (HTTP request failed): " + err.Error()) - // Continue execution, as version check failure is non-fatal - return - } - - // This defer is correct and should remain, as it cleans up the HTTP response body - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - logger.Println("Error occurred when fetching latest version (GitHub API returned status " + strconv.Itoa(resp.StatusCode) + ")") - return - } - - var release GitHubRelease - if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { - logger.Println("Error occurred when decoding latest version response: " + err.Error()) - return - } - - latestVersionTag := release.TagName - - if semver.Compare(gitversion, latestVersionTag) < 0 { - logger.Println("A new version of data-client is available! The latest version is " + latestVersionTag + ". You are using version " + gitversion) - logger.Println("Please download the latest data-client release from https://github.com/uc-cdis/cdis-data-client/releases/latest") - } - } -} diff --git a/client/g3cmd/upload-multipart.go b/client/g3cmd/upload-multipart.go deleted file mode 100644 index fa6d26f..0000000 --- a/client/g3cmd/upload-multipart.go +++ /dev/null @@ -1,299 +0,0 @@ -package g3cmd - -import ( - "bytes" - "context" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" - "github.com/vbauerster/mpb/v8" - "github.com/vbauerster/mpb/v8/decor" -) - -const ( - minChunkSize = 5 * 1024 * 1024 // S3 minimum part size - maxMultipartParts = 10000 - maxConcurrentUploads = 10 - maxRetries = 5 -) - -func NewUploadMultipartCmd() *cobra.Command { - var ( - filePath string - guid string - bucketName string - ) - - cmd := &cobra.Command{ - Use: "upload-multipart", - Short: "Upload a single file using multipart upload", - Long: `Uploads a large file to object storage using multipart upload. -This method is resilient to network interruptions and supports resume capability.`, - Example: `./data-client upload-multipart --profile=myprofile --file-path=./large.bam -./data-client upload-multipart --profile=myprofile --file-path=./data.bam --guid=existing-guid`, - RunE: func(cmd *cobra.Command, args []string) error { - profile, _ := cmd.Flags().GetString("profile") - - return UploadSingleFile(profile, bucketName, filePath, guid) - }, - } - - cmd.Flags().StringVar(&filePath, "file-path", "", "Path to the file to upload") - cmd.Flags().StringVar(&guid, "guid", "", "Optional existing GUID (otherwise generated)") - cmd.Flags().StringVar(&bucketName, "bucket", "", "Target bucket (defaults to configured DATA_UPLOAD_BUCKET)") - - _ = cmd.MarkFlagRequired("profile") - _ = cmd.MarkFlagRequired("file-path") - - return cmd -} - -func UploadSingleFile(profile, bucket, filePath, guid string) error { - - logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard()) - defer closer() - g3, err := client.NewGen3Interface( - context.Background(), - profile, - logger, - ) - if err != nil { - return fmt.Errorf("failed to initialize Gen3 interface: %w", err) - } - - absPath, err := common.GetAbsolutePath(filePath) - if err != nil { - return fmt.Errorf("invalid file path: %w", err) - } - - fileInfo := common.FileUploadRequestObject{ - FilePath: absPath, - Filename: filepath.Base(absPath), - GUID: guid, - FileMetadata: common.FileMetadata{}, - } - - return MultipartUpload(context.TODO(), g3, fileInfo, bucket, true) -} - -// MultipartUpload is now clean, context-aware, and uses modern progress bars -func MultipartUpload(ctx context.Context, g3 client.Gen3Interface, req common.FileUploadRequestObject, bucketName string, showProgress bool) error { - g3.Logger().Printf("File Upload Request: %#v\n", req) - - file, err := os.Open(req.FilePath) - if err != nil { - return fmt.Errorf("cannot open file %s: %w", req.FilePath, err) - } - defer file.Close() - - stat, err := file.Stat() - if err != nil { - return fmt.Errorf("cannot stat file: %w", err) - } - - g3.Logger().Printf("File Name: '%s', File Size: '%d'\n", stat.Name(), stat.Size()) - - if stat.Size() == 0 { - return fmt.Errorf("file is empty: %s", req.Filename) - } - - // Initialize multipart upload - uploadID, finalGUID, err := InitMultipartUpload(g3, req, bucketName) - if err != nil { - return fmt.Errorf("failed to initiate multipart upload: %w", err) - } - req.GUID = finalGUID // update with server-provided GUID - - key := finalGUID + "/" + req.Filename - chunkSize := optimalChunkSize(stat.Size()) - - numChunks := int((stat.Size() + chunkSize - 1) / chunkSize) - parts := make([]MultipartPartObject, 0, numChunks) - - // Progress bar setup (modern mpb) - var p *mpb.Progress - var bar *mpb.Bar - if showProgress { - p = mpb.New(mpb.WithOutput(os.Stdout)) - bar = p.AddBar(stat.Size(), - mpb.PrependDecorators( - decor.Name(req.Filename+" "), - decor.CountersKibiByte("%.1f / %.1f"), - ), - mpb.AppendDecorators( - decor.Percentage(), - decor.AverageSpeed(decor.SizeB1024(0), " % .1f"), - ), - ) - } - - // Channel for chunk indices - chunks := make(chan int, numChunks) - for i := 1; i <= numChunks; i++ { - chunks <- i - } - close(chunks) - - var ( - wg sync.WaitGroup - mu sync.Mutex - uploadErrors []error - ) - - worker := func() { - defer wg.Done() - buf := make([]byte, chunkSize) - - for partNum := range chunks { - offset := int64(partNum-1) * chunkSize - end := offset + chunkSize - end = min(end, stat.Size()) - size := end - offset - - // Read chunk - if _, err := file.Seek(offset, io.SeekStart); err != nil { - mu.Lock() - uploadErrors = append(uploadErrors, fmt.Errorf("seek failed for part %d: %w", partNum, err)) - mu.Unlock() - continue - } - n, err := io.ReadFull(file, buf[:size]) - if err != nil && err != io.ErrUnexpectedEOF { - mu.Lock() - uploadErrors = append(uploadErrors, fmt.Errorf("read failed for part %d: %w", partNum, err)) - mu.Unlock() - continue - } - - reader := bytes.NewReader(buf[:n]) - - // Get presigned URL + upload with retry - var etag string - if err := retryWithBackoff(ctx, maxRetries, func() error { - url, err := GenerateMultipartPresignedURL(g3, key, uploadID, partNum, bucketName) - if err != nil { - return err - } - - return uploadPart(url, reader, &etag) - }); err != nil { - mu.Lock() - uploadErrors = append(uploadErrors, fmt.Errorf("part %d failed after retries: %w", partNum, err)) - mu.Unlock() - continue - } - - // Success - mu.Lock() - etag = strings.Trim(etag, `"`) - parts = append(parts, MultipartPartObject{PartNumber: partNum, ETag: etag}) - g3.Logger().Printf("Appended part %d with ETag %s\n", partNum, etag) - if bar != nil { - bar.IncrBy(n) - } - mu.Unlock() - } - } - - // Launch workers - for range maxConcurrentUploads { - wg.Add(1) - go worker() - } - wg.Wait() - - if p != nil { - p.Wait() - } - - if len(uploadErrors) > 0 { - return fmt.Errorf("multipart upload failed: %d parts failed: %v", len(uploadErrors), uploadErrors) - } - - // Sort parts by PartNumber - sort.Slice(parts, func(i, j int) bool { - return parts[i].PartNumber < parts[j].PartNumber - }) - - g3.Logger().Printf("Completing multipart upload with %d parts for file %s\n", len(parts), req.Filename) - for _, part := range parts { - g3.Logger().Printf(" Part %d: ETag=%s\n", part.PartNumber, part.ETag) - } - - if err := CompleteMultipartUpload(g3, key, uploadID, parts, bucketName); err != nil { - return fmt.Errorf("failed to complete multipart upload: %w", err) - } - - g3.Logger().Printf("Successfully uploaded %s as %s (%d)", req.Filename, finalGUID, stat.Size()) - return nil -} - -// Helper: exponential backoff retry -func retryWithBackoff(ctx context.Context, attempts int, fn func() error) error { - var err error - for i := range attempts { - if err = fn(); err == nil { - return nil - } - if i == attempts-1 { - break - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(backoffDuration(i)): - } - } - return fmt.Errorf("after %d attempts: %w", attempts, err) -} - -func backoffDuration(attempt int) time.Duration { - return min(time.Duration(1< --manifest= --upload-path= --bucket= --force-multipart= --include-subdirname= --batch=`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Notice: this is the upload method which requires the user to provide GUIDs. In this method files will be uploaded to specified GUIDs.\nIf your intention is to upload files without pre-existing GUIDs, consider to use \"./data-client upload\" instead.\n\n") - - logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard()) - defer closer() - - // Instantiate interface to Gen3 - g3i, err := client.NewGen3Interface(context.Background(), profile, logger) - if err != nil { - g3i.Logger().Fatalf("Failed to parse config on profile %s, %v", profile, err) - } - - host, err := g3i.GetHost() - if err != nil { - g3i.Logger().Fatal("Error occurred during parsing config file for hostname: " + err.Error()) - } - dataExplorerURL := host.Scheme + "://" + host.Host + "/explorer" - - var objects []ManifestObject - - manifestFile, err := os.Open(manifestPath) - if err != nil { - g3i.Logger().Println("Failed to open manifest file") - g3i.Logger().Fatal("A valid manifest can be acquired by using the \"Download Manifest\" button on " + dataExplorerURL) - } - defer manifestFile.Close() - switch { - case strings.EqualFold(filepath.Ext(manifestPath), ".json"): - manifestBytes, err := os.ReadFile(manifestPath) - if err != nil { - g3i.Logger().Printf("Failed reading manifest %s, %v\n", manifestPath, err) - g3i.Logger().Fatal("A valid manifest can be acquired by using the \"Download Manifest\" button on " + dataExplorerURL) - } - err = json.Unmarshal(manifestBytes, &objects) - if err != nil { - g3i.Logger().Fatal("Unmarshalling manifest failed with error: " + err.Error()) - } - default: - g3i.Logger().Println("Unsupported manifast format") - g3i.Logger().Fatal("A valid manifest can be acquired by using the \"Download Manifest\" button on " + dataExplorerURL) - } - - absUploadPath, err := common.GetAbsolutePath(uploadPath) - if err != nil { - g3i.Logger().Fatalf("Error when parsing file paths: %s", err.Error()) - } - - // Create unified upload request objects - uploadRequestObjects := make([]common.FileUploadRequestObject, 0, len(objects)) - - for _, object := range objects { - var localFilePath string - // Determine the local file path - if object.Filename != "" { - // conform to fence naming convention - localFilePath, err = getFullFilePath(absUploadPath, object.Filename) - } else { - // Otherwise, here we are assuming the local filename will be the same as GUID - localFilePath, err = getFullFilePath(absUploadPath, object.ObjectID) - } - - if err != nil { - g3i.Logger().Println(err.Error()) - continue - } - - fileInfo, err := ProcessFilename(g3i.Logger(), absUploadPath, localFilePath, object.ObjectID, includeSubDirName, false) - if err != nil { - g3i.Logger().Println("Process filename error: " + err.Error()) - g3i.Logger().Failed(localFilePath, filepath.Base(localFilePath), common.FileMetadata{}, object.ObjectID, 0, false) - continue - } - - // Convert FileInfo to the unified common.FileUploadRequestObject - furObject := common.FileUploadRequestObject{ - FilePath: fileInfo.FilePath, - Filename: fileInfo.Filename, - FileMetadata: fileInfo.FileMetadata, - GUID: fileInfo.GUID, - } - uploadRequestObjects = append(uploadRequestObjects, furObject) - } - - // Separate into single-part and multipart objects - singlePartObjects, multipartObjects := separateSingleAndMultipartUploads(g3i, uploadRequestObjects, forceMultipart) - // Pass the unified objects to the upload handlers - if batch { - workers, respCh, errCh, batchFURObjects := initBatchUploadChannels(numParallel, len(singlePartObjects)) - for i, furObject := range singlePartObjects { - // FileInfo processing and path normalization are already done, so we use the object directly - if len(batchFURObjects) < workers { - batchFURObjects = append(batchFURObjects, furObject) - } else { - batchUpload(g3i, batchFURObjects, workers, respCh, errCh, bucketName) - batchFURObjects = []common.FileUploadRequestObject{furObject} - } - if !forceMultipart && i == len(singlePartObjects)-1 && len(batchFURObjects) > 0 { // upload remainders - batchUpload(g3i, batchFURObjects, workers, respCh, errCh, bucketName) - } - } - } else { - processSingleUploads(g3i, singlePartObjects, bucketName, includeSubDirName, absUploadPath) // Assuming updated - } - - if len(multipartObjects) > 0 { - err := processMultipartUpload(g3i, multipartObjects, bucketName, includeSubDirName, absUploadPath) - if err != nil { - g3i.Logger().Fatal(err.Error()) - } - } - - if len(g3i.Logger().GetSucceededLogMap()) == 0 { - retryUpload(g3i, g3i.Logger().GetFailedLogMap()) - } - - g3i.Logger().Scoreboard().PrintSB() - }, - } - - uploadMultipleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - uploadMultipleCmd.MarkFlagRequired("profile") //nolint:errcheck - uploadMultipleCmd.Flags().StringVar(&manifestPath, "manifest", "", "The manifest file to read from. A valid manifest can be acquired by using the \"Download Manifest\" button in Data Explorer for Common portal") - uploadMultipleCmd.MarkFlagRequired("manifest") //nolint:errcheck - uploadMultipleCmd.Flags().StringVar(&uploadPath, "upload-path", "", "The directory in which contains files to be uploaded") - uploadMultipleCmd.MarkFlagRequired("upload-path") //nolint:errcheck - uploadMultipleCmd.Flags().BoolVar(&batch, "batch", true, "Upload in parallel") - uploadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 3, "Number of uploads to run in parallel") - uploadMultipleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") - uploadMultipleCmd.Flags().BoolVar(&forceMultipart, "force-multipart", false, "Force to use multipart upload when possible (file size >= 5MB)") - uploadMultipleCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", true, "Include subdirectory names in file name") - RootCmd.AddCommand(uploadMultipleCmd) -} - -func processSingleUploads(g3i client.Gen3Interface, singleObjects []common.FileUploadRequestObject, bucketName string, includeSubDirName bool, uploadPath string) { - for _, furObject := range singleObjects { - filePath := furObject.FilePath - file, err := os.Open(filePath) - if err != nil { - g3i.Logger().Println("File open error: " + err.Error()) - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - continue - } - startSingleFileUpload(g3i, furObject, file, bucketName) - file.Close() - } -} - -func startSingleFileUpload(g3i client.Gen3Interface, furObject common.FileUploadRequestObject, file *os.File, bucketName string) { - - fi, err := file.Stat() - if err != nil { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - g3i.Logger().Println("File stat error for file" + fi.Name() + ", file may be missing or unreadable because of permissions.\n") - return - } - - respURL, guid, err := GeneratePresignedURL(g3i, furObject.Filename, furObject.FileMetadata, bucketName) - if err != nil { - g3i.Logger().Println(err.Error()) - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, guid, 0, false) - return - } - furObject.GUID = guid - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - furObject.PresignedURL = respURL - - furObject, err = GenerateUploadRequest(g3i, furObject, file, nil) - if err != nil { - file.Close() - g3i.Logger().Printf("Error occurred during request generation: %s\n", err.Error()) - return - } - - err = uploadFile(g3i, furObject, 0) - if err != nil { - g3i.Logger().Println(err.Error()) - } else { - g3i.Logger().Scoreboard().IncrementSB(0) - } - - file.Close() -} - -func processMultipartUpload(g3i client.Gen3Interface, multipartObjects []common.FileUploadRequestObject, bucketName string, includeSubDirName bool, uploadPath string) error { - cred := g3i.GetCredential() - if cred.UseShepherd == "true" || - cred.UseShepherd == "" && common.DefaultUseShepherd == true { - return fmt.Errorf("error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n $ data-client configure --profile=%v --use-shepherd=false\nand try again", cred.Profile) - } - g3i.Logger().Println("Multipart uploading...") - - for _, furObject := range multipartObjects { - // No more redundant ProcessFilename call! - // Pass the complete FileUploadRequestObject to the streamlined multipartUpload. - // Enable progress bar for batch uploads (interactive CLI use) - err := MultipartUpload(context.Background(), g3i, furObject, bucketName, true) - - if err != nil { - g3i.Logger().Println(err.Error()) - } else { - g3i.Logger().Scoreboard().IncrementSB(0) - } - } - return nil -} diff --git a/client/g3cmd/upload-single.go b/client/g3cmd/upload-single.go deleted file mode 100644 index 8395370..0000000 --- a/client/g3cmd/upload-single.go +++ /dev/null @@ -1,122 +0,0 @@ -package g3cmd - -// Deprecated: Use upload instead. -import ( - "context" - "errors" - "fmt" - "log" - "os" - "path/filepath" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" -) - -func init() { - var guid string - var filePath string - var bucketName string - - var uploadSingleCmd = &cobra.Command{ - Use: "upload-single", - Short: "Upload a single file to a GUID", - Long: `Gets a presigned URL for which to upload a file associated with a GUID and then uploads the specified file.`, - Example: `./data-client upload-single --profile= --guid=f6923cf3-xxxx-xxxx-xxxx-14ab3f84f9d6 --file=`, - Run: func(cmd *cobra.Command, args []string) { - // initialize transmission logs - err := UploadSingle(profile, guid, filePath, bucketName, true) - if err != nil { - log.Fatalln(err.Error()) - } - }, - } - uploadSingleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - uploadSingleCmd.MarkFlagRequired("profile") //nolint:errcheck - uploadSingleCmd.Flags().StringVar(&guid, "guid", "", "Specify the guid for the data you would like to work with") - uploadSingleCmd.MarkFlagRequired("guid") //nolint:errcheck - uploadSingleCmd.Flags().StringVar(&filePath, "file", "", "Specify file to upload to with --file=~/path/to/file") - uploadSingleCmd.MarkFlagRequired("file") //nolint:errcheck - uploadSingleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") - RootCmd.AddCommand(uploadSingleCmd) -} - -func UploadSingle(profile string, guid string, filePath string, bucketName string, enableLogs bool) error { - - logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog()) - if enableLogs { - logger, closer = logs.New( - profile, - logs.WithSucceededLog(), - logs.WithFailedLog(), - logs.WithScoreboard(), - logs.WithConsole(), - ) - } - defer closer() - - // Instantiate interface to Gen3 - g3i, err := client.NewGen3Interface( - context.Background(), - profile, - logger, - ) - if err != nil { - return fmt.Errorf("failed to parse config on profile %s: %w", profile, err) - } - - filePaths, err := common.ParseFilePaths(filePath, false) - if len(filePaths) > 1 { - return errors.New("more than 1 file location has been found. Do not use \"*\" in file path or provide a folder as file path") - } - if err != nil { - return errors.New("file path parsing error: " + err.Error()) - } - if len(filePaths) == 1 { - filePath = filePaths[0] - } - filename := filepath.Base(filePath) - if _, err := os.Stat(filePath); os.IsNotExist(err) { - g3i.Logger().Failed(filePath, filename, common.FileMetadata{}, "", 0, false) - sb := g3i.Logger().Scoreboard() - sb.IncrementSB(len(sb.Counts)) - sb.PrintSB() - return fmt.Errorf("[ERROR] The file you specified \"%s\" does not exist locally\n", filePath) - } - - file, err := os.Open(filePath) - if err != nil { - sb := g3i.Logger().Scoreboard() - sb.IncrementSB(len(sb.Counts)) - sb.PrintSB() - g3i.Logger().Failed(filePath, filename, common.FileMetadata{}, "", 0, false) - g3i.Logger().Println("File open error: " + err.Error()) - return fmt.Errorf("[ERROR] when opening file path %s, an error occurred: %s\n", filePath, err.Error()) - } - defer file.Close() - - furObject := common.FileUploadRequestObject{FilePath: filePath, Filename: filename, GUID: guid, Bucket: bucketName} - - furObject, err = GenerateUploadRequest(g3i, furObject, file, nil) - if err != nil { - file.Close() - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, common.FileMetadata{}, furObject.GUID, 0, false) - sb := g3i.Logger().Scoreboard() - sb.IncrementSB(len(sb.Counts)) - sb.PrintSB() - g3i.Logger().Fatalf("Error occurred during request generation: %s", err.Error()) - return fmt.Errorf("[ERROR] Error occurred during request generation for file %s: %s\n", filePath, err.Error()) - } - err = uploadFile(g3i, furObject, 0) - if err != nil { - sb := g3i.Logger().Scoreboard() - sb.IncrementSB(len(sb.Counts)) - return fmt.Errorf("[ERROR] Error uploading file %s: %s\n", filePath, err.Error()) - } else { - g3i.Logger().Scoreboard().IncrementSB(0) - } - g3i.Logger().Scoreboard().PrintSB() - return nil -} diff --git a/client/g3cmd/upload.go b/client/g3cmd/upload.go deleted file mode 100644 index 2e50a87..0000000 --- a/client/g3cmd/upload.go +++ /dev/null @@ -1,155 +0,0 @@ -package g3cmd - -import ( - "context" - "log" - "os" - "path/filepath" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" -) - -func init() { - var bucketName string - var includeSubDirName bool - var uploadPath string - var batch bool - var forceMultipart bool - var numParallel int - var hasMetadata bool - var uploadCmd = &cobra.Command{ - Use: "upload", - Short: "Upload file(s) to object storage.", - Long: `Gets a presigned URL for each file and then uploads the specified file(s).`, - Example: "For uploading a single file:\n./data-client upload --profile= --upload-path=\n" + - "For uploading all files within an folder:\n./data-client upload --profile= --upload-path=\n" + - "Can also support regex such as:\n./data-client upload --profile= --upload-path=\n" + - "Or:\n./data-client upload --profile= --upload-path=\n" + - "This command can also upload file metadata using the --metadata flag. If the --metadata flag is passed, the data-client will look for a file called [filename]_metadata.json in the same folder, which contains the metadata to upload.\n" + - "For example, if uploading the file `folder/my_file.bam`, the data-client will look for a metadata file at `folder/my_file_metadata.json`.\n" + - "For the format of the metadata files, see the README.", - Run: func(cmd *cobra.Command, args []string) { - - Logger, logCloser := logs.New(profile, logs.WithSucceededLog(), logs.WithScoreboard(), logs.WithFailedLog()) - defer logCloser() - // Instantiate interface to Gen3 - g3i, err := client.NewGen3Interface(context.Background(), profile, Logger) - if err != nil { - log.Fatalf("Failed to parse config on profile %s, %v", profile, err) - } - - logger := g3i.Logger() - if hasMetadata { - hasShepherd, err := g3i.CheckForShepherdAPI() - if err != nil { - logger.Printf("WARNING: Error when checking for Shepherd API: %v", err) - } else { - if !hasShepherd { - logger.Fatalf("ERROR: Metadata upload (`--metadata`) is not supported in the environment you are uploading to. Double check that you are uploading to the right profile.") - } - } - } - - uploadPath, _ = common.GetAbsolutePath(uploadPath) - filePaths, err := common.ParseFilePaths(uploadPath, hasMetadata) - if err != nil { - logger.Fatalf("Error when parsing file paths: %s", err.Error()) - } - uploadRequestObjects := make([]common.FileUploadRequestObject, 0, len(filePaths)) - - logger.Println("\nThe following file(s) has been found in path \"" + uploadPath + "\" and will be uploaded:") - for _, filePath := range filePaths { - // Use ProcessFilename to create the unified object (GUID is empty here, as this command requests a new GUID) - // ProcessFilename signature: (uploadPath, filePath, objectId, includeSubDirName, includeMetadata) - furObject, err := ProcessFilename(g3i.Logger(), uploadPath, filePath, "", includeSubDirName, hasMetadata) - - // Handle case where ProcessFilename fails (e.g., metadata parsing error) - if err != nil { - // Use the data available for logging the failure - g3i.Logger().Failed(filePath, filepath.Base(filePath), common.FileMetadata{}, "", 0, false) - logger.Println("Error processing file path or metadata: " + err.Error()) - continue - } - - // Optional: Display file path before proceeding - file, _ := os.Open(filePath) - if fi, _ := file.Stat(); !fi.IsDir() { - logger.Println("\t" + filePath) - } - file.Close() - - uploadRequestObjects = append(uploadRequestObjects, furObject) - } - // fmt.Fprintln(os.Stderr) - logger.Println() - - if len(uploadRequestObjects) == 0 { - logger.Println("No valid file upload requests were created.") - return - } - - singlePartObjects, multipartObjects := separateSingleAndMultipartUploads(g3i, uploadRequestObjects, forceMultipart) - if batch { - workers, respCh, errCh, batchFURObjects := initBatchUploadChannels(numParallel, len(singlePartObjects)) - - for _, furObject := range singlePartObjects { - if len(batchFURObjects) < workers { - batchFURObjects = append(batchFURObjects, furObject) - } else { - batchUpload(g3i, batchFURObjects, workers, respCh, errCh, bucketName) - batchFURObjects = []common.FileUploadRequestObject{furObject} - } - } - if len(batchFURObjects) > 0 { - batchUpload(g3i, batchFURObjects, workers, respCh, errCh, bucketName) - } - - if len(errCh) > 0 { - close(errCh) - for err := range errCh { - if err != nil { - logger.Printf("Error occurred during uploading: %s\n", err.Error()) - } - } - } - } else { - for _, furObject := range singlePartObjects { - file, err := os.Open(furObject.FilePath) - if err != nil { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - logger.Println("File open error: " + err.Error()) - continue - } - startSingleFileUpload(g3i, furObject, file, bucketName) - } - } - - if len(multipartObjects) > 0 { - err := processMultipartUpload(g3i, multipartObjects, bucketName, includeSubDirName, uploadPath) - if err != nil { - logger.Println(err.Error()) - } - } - if len(g3i.Logger().GetSucceededLogMap()) == 0 { - retryUpload(g3i, g3i.Logger().GetFailedLogMap()) - } - - g3i.Logger().Scoreboard().PrintSB() - }, - } - - uploadCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - uploadCmd.MarkFlagRequired("profile") //nolint:errcheck - uploadCmd.Flags().StringVar(&uploadPath, "upload-path", "", "The directory or file in which contains file(s) to be uploaded") - uploadCmd.MarkFlagRequired("upload-path") //nolint:errcheck - uploadCmd.Flags().BoolVar(&batch, "batch", false, "Upload in parallel") - uploadCmd.Flags().IntVar(&numParallel, "numparallel", 3, "Number of uploads to run in parallel") - uploadCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", true, "Include subdirectory names in file name") - uploadCmd.Flags().BoolVar(&forceMultipart, "force-multipart", false, "Force to use multipart upload if possible") - uploadCmd.Flags().BoolVar(&hasMetadata, "metadata", false, "Search for and upload file metadata alongside the file") - uploadCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") - RootCmd.AddCommand(uploadCmd) -} diff --git a/client/g3cmd/utils.go b/client/g3cmd/utils.go deleted file mode 100644 index d65f488..0000000 --- a/client/g3cmd/utils.go +++ /dev/null @@ -1,686 +0,0 @@ -package g3cmd - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "math" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - - "github.com/vbauerster/mpb/v8" - "github.com/vbauerster/mpb/v8/decor" -) - -// ManifestObject represents an object from manifest that downloaded from windmill / data-portal -type ManifestObject struct { - ObjectID string `json:"object_id"` - SubjectID string `json:"subject_id"` - Filename string `json:"file_name"` - Filesize int64 `json:"file_size"` -} - -// InitRequestObject represents the payload that sends to FENCE for getting a singlepart upload presignedURL or init a multipart upload for new object file -type InitRequestObject struct { - Filename string `json:"file_name"` - Bucket string `json:"bucket,omitempty"` - GUID string `json:"guid,omitempty"` -} - -// ShepherdInitRequestObject represents the payload that sends to Shepherd for getting a singlepart upload presignedURL or init a multipart upload for new object file -type ShepherdInitRequestObject struct { - Filename string `json:"file_name"` - Authz struct { - Version string `json:"version"` - ResourcePaths []string `json:"resource_paths"` - } `json:"authz"` - Aliases []string `json:"aliases"` - // Metadata is an encoded JSON string of any arbitrary metadata the user wishes to upload. - Metadata map[string]any `json:"metadata"` -} - -// MultipartUploadRequestObject represents the payload that sends to FENCE for getting a presignedURL for a part -type MultipartUploadRequestObject struct { - Key string `json:"key"` - UploadID string `json:"uploadId"` - PartNumber int `json:"partNumber"` - Bucket string `json:"bucket,omitempty"` -} - -// MultipartCompleteRequestObject represents the payload that sends to FENCE for completeing a multipart upload -type MultipartCompleteRequestObject struct { - Key string `json:"key"` - UploadID string `json:"uploadId"` - Parts []MultipartPartObject `json:"parts"` - Bucket string `json:"bucket,omitempty"` -} - -// MultipartPartObject represents a part object -type MultipartPartObject struct { - PartNumber int `json:"PartNumber"` - ETag string `json:"ETag"` -} - -// FileInfo is a helper struct for including subdirname as filename -type FileInfo struct { - FilePath string - Filename string - FileMetadata common.FileMetadata - ObjectId string -} - -// RenamedOrSkippedFileInfo is a helper struct for recording renamed or skipped files -type RenamedOrSkippedFileInfo struct { - GUID string - OldFilename string - NewFilename string -} - -const ( - // B is bytes - B int64 = iota - // KB is kilobytes - KB int64 = 1 << (10 * iota) - // MB is megabytes - MB - // GB is gigabytes - GB - // TB is terrabytes - TB -) - -var unitMap = map[int64]string{ - B: "B", - KB: "KB", - MB: "MB", - GB: "GB", - TB: "TB", -} - -// FileSizeLimit is the maximun single file size for non-multipart upload (5GB) -const FileSizeLimit = 5 * GB - -// MultipartFileSizeLimit is the maximun single file size for multipart upload (5TB) -const MultipartFileSizeLimit = 5 * TB -const minMultipartChunkSize = 5 * MB - -// MaxRetryCount is the maximum retry number per record -const MaxRetryCount = 5 -const maxWaitTime = 300 - -// InitMultipartUpload helps sending requests to FENCE to init a multipart upload -func InitMultipartUpload(g3 client.Gen3Interface, furObject common.FileUploadRequestObject, bucketName string) (string, string, error) { - // Use Filename and GUID directly from the unified request object - multipartInitObject := InitRequestObject{Filename: furObject.Filename, Bucket: bucketName, GUID: furObject.GUID} - - objectBytes, err := json.Marshal(multipartInitObject) - if err != nil { - return "", "", errors.New("Error has occurred during marshalling data for multipart upload initialization, detailed error message: " + err.Error()) - } - - msg, err := g3.DoRequestWithSignedHeader(common.FenceDataMultipartInitEndpoint, "application/json", objectBytes) - - if err != nil { - if strings.Contains(err.Error(), "404") { - return "", "", errors.New(err.Error() + "\nPlease check to ensure FENCE version is at 2.8.0 or beyond") - } - return "", "", errors.New("Error has occurred during multipart upload initialization, detailed error message: " + err.Error()) - } - if msg.UploadID == "" || msg.GUID == "" { - return "", "", errors.New("unknown error has occurred during multipart upload initialization. Please check logs from Gen3 services") - } - return msg.UploadID, msg.GUID, err -} - -// GenerateMultipartPresignedURL helps sending requests to FENCE to get a presigned URL for a part during a multipart upload -func GenerateMultipartPresignedURL(g3 client.Gen3Interface, key string, uploadID string, partNumber int, bucketName string) (string, error) { - multipartUploadObject := MultipartUploadRequestObject{Key: key, UploadID: uploadID, PartNumber: partNumber, Bucket: bucketName} - objectBytes, err := json.Marshal(multipartUploadObject) - if err != nil { - return "", errors.New("Error has occurred during marshalling data for multipart upload presigned url generation, detailed error message: " + err.Error()) - } - - msg, err := g3.DoRequestWithSignedHeader(common.FenceDataMultipartUploadEndpoint, "application/json", objectBytes) - - if err != nil { - return "", errors.New("Error has occurred during multipart upload presigned url generation, detailed error message: " + err.Error()) - } - if msg.PresignedURL == "" { - return "", errors.New("unknown error has occurred during multipart upload presigned url generation. Please check logs from Gen3 services") - } - return msg.PresignedURL, err -} - -// CompleteMultipartUpload helps sending requests to FENCE to complete a multipart upload -func CompleteMultipartUpload(g3 client.Gen3Interface, key string, uploadID string, parts []MultipartPartObject, bucketName string) error { - multipartCompleteObject := MultipartCompleteRequestObject{Key: key, UploadID: uploadID, Parts: parts, Bucket: bucketName} - objectBytes, err := json.Marshal(multipartCompleteObject) - if err != nil { - return errors.New("Error has occurred during marshalling data for multipart upload, detailed error message: " + err.Error()) - } - - _, err = g3.DoRequestWithSignedHeader(common.FenceDataMultipartCompleteEndpoint, "application/json", objectBytes) - if err != nil { - return errors.New("Error has occurred during completing multipart upload, detailed error message: " + err.Error()) - } - return nil -} - -// GetDownloadResponse helps grabbing a response for downloading a file specified with GUID -func GetDownloadResponse(g3 client.Gen3Interface, fdrObject *common.FileDownloadResponseObject, protocolText string) error { - // Attempt to get the file download URL from Shepherd if it's deployed in this commons, - // otherwise fall back to Fence. - var fileDownloadURL string - hasShepherd, err := g3.CheckForShepherdAPI() - if err != nil { - g3.Logger().Println("Error occurred when checking for Shepherd API: " + err.Error()) - g3.Logger().Println("Falling back to Indexd...") - } else if hasShepherd { - endPointPostfix := common.ShepherdEndpoint + "/objects/" + fdrObject.GUID + "/download" - _, r, err := g3.GetResponse(endPointPostfix, "GET", "", nil) - if err != nil { - return errors.New("Error occurred when getting download URL for object " + fdrObject.GUID + " from endpoint " + endPointPostfix + " . Details: " + err.Error()) - } - defer r.Body.Close() - if r.StatusCode != 200 { - buf := new(bytes.Buffer) - buf.ReadFrom(r.Body) // nolint:errcheck - body := buf.String() - return errors.New("Error when getting download URL at " + endPointPostfix + " for file " + fdrObject.GUID + " : Shepherd returned non-200 status code " + strconv.Itoa(r.StatusCode) + " . Request body: " + body) - } - // Unmarshal into json - urlResponse := struct { - URL string `json:"url"` - }{} - err = json.NewDecoder(r.Body).Decode(&urlResponse) - if err != nil { - return errors.New("Error occurred when getting download URL for object " + fdrObject.GUID + " from endpoint " + endPointPostfix + " . Details: " + err.Error()) - } - fileDownloadURL = urlResponse.URL - if fileDownloadURL == "" { - return errors.New("Unknown error occurred when getting download URL for object " + fdrObject.GUID + " from endpoint " + endPointPostfix + " : No URL found in response body. Check the Shepherd logs") - } - } else { - endPointPostfix := common.FenceDataDownloadEndpoint + "/" + fdrObject.GUID + protocolText - msg, err := g3.DoRequestWithSignedHeader(endPointPostfix, "", nil) - - if err != nil || msg.URL == "" { - errorMsg := "Error occurred when getting download URL for object " + fdrObject.GUID - if err != nil { - errorMsg += "\n Details of error: " + err.Error() - } - return errors.New(errorMsg) - } - fileDownloadURL = msg.URL - } - - // TODO: for now we don't print fdrObject.URL in error messages since it is sensitive - // Later after we had log level we could consider for putting URL into debug logs... - fdrObject.URL = fileDownloadURL - if fdrObject.Range != 0 && !strings.Contains(fdrObject.URL, "X-Amz-Signature") && !strings.Contains(fdrObject.URL, "X-Goog-Signature") { // Not S3 or GS URLs and we want resume, send HEAD req first to check if server supports range - resp, err := http.Head(fdrObject.URL) - if err != nil { - errorMsg := "Error occurred when sending HEAD req to URL associated with GUID " + fdrObject.GUID - errorMsg += "\n Details of error: " + sanitizeErrorMsg(err.Error(), fdrObject.URL) - return errors.New(errorMsg) - } - if resp.Header.Get("Accept-Ranges") != "bytes" { // server does not support range, download without range header - fdrObject.Range = 0 - } - } - - headers := map[string]string{} - if fdrObject.Range != 0 { - headers["Range"] = "bytes=" + strconv.FormatInt(fdrObject.Range, 10) + "-" - } - resp, err := g3.MakeARequest(http.MethodGet, fdrObject.URL, "", "", headers, nil, true) - if err != nil { - errorMsg := "Error occurred when making request to URL associated with GUID " + fdrObject.GUID - errorMsg += "\n Details of error: " + sanitizeErrorMsg(err.Error(), fdrObject.URL) - return errors.New(errorMsg) - } - if resp.StatusCode != 200 && resp.StatusCode != 206 { - errorMsg := "Got a non-200 or non-206 response when making request to URL associated with GUID " + fdrObject.GUID - errorMsg += "\n HTTP status code for response: " + strconv.Itoa(resp.StatusCode) - return errors.New(errorMsg) - } - fdrObject.Response = resp - return nil -} - -func sanitizeErrorMsg(errorMsg string, sensitiveURL string) string { - return strings.ReplaceAll(errorMsg, sensitiveURL, "") -} - -// GeneratePresignedURL helps sending requests to Shepherd/Fence and parsing the response in order to get presigned URL for the new upload flow -func GeneratePresignedURL(g3 client.Gen3Interface, filename string, fileMetadata common.FileMetadata, bucketName string) (string, string, error) { - // Attempt to get the presigned URL of this file from Shepherd if it's deployed, otherwise fall back to Fence. - hasShepherd, err := g3.CheckForShepherdAPI() - if err != nil { - g3.Logger().Println("Error occurred when checking for Shepherd API: " + err.Error()) - g3.Logger().Println("Falling back to Fence...") - } else if hasShepherd { - purObject := ShepherdInitRequestObject{ - Filename: filename, - Authz: struct { - Version string `json:"version"` - ResourcePaths []string `json:"resource_paths"` - }{ - "0", - fileMetadata.Authz, - }, - Aliases: fileMetadata.Aliases, - Metadata: fileMetadata.Metadata, - } - objectBytes, err := json.Marshal(purObject) - if err != nil { - return "", "", errors.New("Error occurred when creating upload request for file " + filename + ". Details: " + err.Error()) - } - endPointPostfix := common.ShepherdEndpoint + "/objects" - _, r, err := g3.GetResponse(endPointPostfix, "POST", "", objectBytes) - if err != nil { - return "", "", errors.New("Error occurred when requesting upload URL from " + endPointPostfix + " for file " + filename + ". Details: " + err.Error()) - } - defer r.Body.Close() - if r.StatusCode != 201 { - buf := new(bytes.Buffer) - buf.ReadFrom(r.Body) // nolint:errcheck - body := buf.String() - return "", "", errors.New("Error when requesting upload URL at " + endPointPostfix + " for file " + filename + ": Shepherd returned non-200 status code " + strconv.Itoa(r.StatusCode) + ". Request body: " + body) - } - res := struct { - GUID string `json:"guid"` - URL string `json:"upload_url"` - }{} - err = json.NewDecoder(r.Body).Decode(&res) - if err != nil { - return "", "", errors.New("Error occurred when creating upload URL for file " + filename + ": . Details: " + err.Error()) - } - if res.URL == "" || res.GUID == "" { - return "", "", errors.New("unknown error has occurred during presigned URL or GUID generation. Please check logs from Gen3 services") - } - return res.URL, res.GUID, nil - } - - // Otherwise, fall back to Fence - purObject := InitRequestObject{Filename: filename, Bucket: bucketName} - objectBytes, err := json.Marshal(purObject) - if err != nil { - return "", "", errors.New("Error occurred when marshalling object: " + err.Error()) - } - msg, err := g3.DoRequestWithSignedHeader(common.FenceDataUploadEndpoint, "application/json", objectBytes) - - if err != nil { - return "", "", errors.New("Something went wrong. Maybe you don't have permission to upload data or Fence is misconfigured. Detailed error message: " + err.Error()) - } - if msg.URL == "" || msg.GUID == "" { - return "", "", errors.New("unknown error has occurred during presigned URL or GUID generation. Please check logs from Gen3 services") - } - return msg.URL, msg.GUID, err -} - -// GenerateUploadRequest helps preparing the HTTP request for upload and the progress bar for single part upload -func GenerateUploadRequest(g3 client.Gen3Interface, furObject common.FileUploadRequestObject, file *os.File, progress *mpb.Progress) (common.FileUploadRequestObject, error) { - if furObject.PresignedURL == "" { - endPointPostfix := common.FenceDataUploadEndpoint + "/" + furObject.GUID + "?file_name=" + url.QueryEscape(furObject.Filename) - - // ensure bucket is set - if furObject.Bucket != "" { - endPointPostfix += "&bucket=" + furObject.Bucket - } - msg, err := g3.DoRequestWithSignedHeader(endPointPostfix, "application/json", nil) - if err != nil && !strings.Contains(err.Error(), "No GUID found") { - return furObject, errors.New("Upload error: " + err.Error()) - } - if msg.URL == "" { - return furObject, errors.New("Upload error: error in generating presigned URL for " + furObject.Filename) - } - furObject.PresignedURL = msg.URL - } - - fi, err := file.Stat() - if err != nil { - return furObject, errors.New("File stat error for file" + furObject.Filename + ", file may be missing or unreadable because of permissions.\n") - } - - if fi.Size() > FileSizeLimit { - return furObject, errors.New("The file size of file " + furObject.Filename + " exceeds the limit allowed and cannot be uploaded. The maximum allowed file size is " + FormatSize(FileSizeLimit) + ".\n") - } - - if progress == nil { - progress = mpb.New(mpb.WithOutput(os.Stdout)) - } - bar := progress.AddBar(fi.Size(), - mpb.PrependDecorators( - decor.Name(furObject.Filename+" "), - decor.CountersKibiByte("% .1f / % .1f"), - ), - mpb.AppendDecorators( - decor.Percentage(), - decor.AverageSpeed(decor.SizeB1024(0), " % .1f"), - ), - ) - pr, pw := io.Pipe() - - go func() { - var writer io.Writer - defer pw.Close() - defer file.Close() - - writer = bar.ProxyWriter(pw) - if _, err = io.Copy(writer, file); err != nil { - err = errors.New("io.Copy error: " + err.Error() + "\n") - } - if err = pw.Close(); err != nil { - err = errors.New("Pipe writer close error: " + err.Error() + "\n") - } - }() - if err != nil { - return furObject, err - } - - req, err := http.NewRequest(http.MethodPut, furObject.PresignedURL, pr) - req.ContentLength = fi.Size() - - furObject.Request = req - furObject.Progress = progress - furObject.Bar = bar - - return furObject, err -} - -// DeleteRecord helps sending requests to FENCE to delete a record from INDEXD as well as its storage locations -func DeleteRecord(g3 client.Gen3Interface, guid string) (string, error) { - return g3.DeleteRecord(guid) -} - -func separateSingleAndMultipartUploads(g3i client.Gen3Interface, objects []common.FileUploadRequestObject, forceMultipart bool) ([]common.FileUploadRequestObject, []common.FileUploadRequestObject) { - fileSizeLimit := FileSizeLimit // 5GB - if forceMultipart { - fileSizeLimit = minMultipartChunkSize // 5MB - } - singlepartObjects := make([]common.FileUploadRequestObject, 0) - multipartObjects := make([]common.FileUploadRequestObject, 0) - - for _, object := range objects { - filePath := object.FilePath - - // Check if file exists locally - if _, err := os.Stat(filePath); os.IsNotExist(err) { - g3i.Logger().Printf("The file you specified \"%s\" does not exist locally\n", filePath) - g3i.Logger().Failed(object.FilePath, object.Filename, object.FileMetadata, object.GUID, 0, false) - continue - } - - // Use a closure to handle file operations and cleanup - func(obj common.FileUploadRequestObject) { - file, err := os.Open(filePath) - if err != nil { - g3i.Logger().Println("File open error occurred when validating file path: " + err.Error()) - g3i.Logger().Failed(obj.FilePath, obj.Filename, obj.FileMetadata, obj.GUID, 0, false) - return - } - defer file.Close() - - fi, err := file.Stat() - if err != nil { - g3i.Logger().Println("File stat error occurred when validating file path: " + err.Error()) - g3i.Logger().Failed(obj.FilePath, obj.Filename, obj.FileMetadata, obj.GUID, 0, false) - return - } - if fi.IsDir() { - return - } - - _, ok := g3i.Logger().GetSucceededLogMap()[filePath] - if ok { - g3i.Logger().Println("File \"" + filePath + "\" has been found in local submission history and has been skipped to prevent duplicated submissions.") - return - } - - // Add to failed log initially, it will be removed on success - // This is an existing pattern, keeping it here. - g3i.Logger().Failed(obj.FilePath, obj.Filename, obj.FileMetadata, obj.GUID, 0, false) - - if fi.Size() > MultipartFileSizeLimit { - g3i.Logger().Printf("The file size of %s has exceeded the limit allowed and cannot be uploaded. The maximum allowed file size is %s\n", fi.Name(), FormatSize(MultipartFileSizeLimit)) - } else if fi.Size() > int64(fileSizeLimit) { - multipartObjects = append(multipartObjects, obj) - } else { - singlepartObjects = append(singlepartObjects, obj) - } - }(object) - } - return singlepartObjects, multipartObjects -} - -// ProcessFilename returns an FileInfo object which has the information about the path and name to be used for upload of a file -func ProcessFilename(logger logs.Logger, uploadPath string, filePath string, objectId string, includeSubDirName bool, includeMetadata bool) (common.FileUploadRequestObject, error) { - var err error - filePath, err = common.GetAbsolutePath(filePath) - if err != nil { - return common.FileUploadRequestObject{}, err - } - - filename := filepath.Base(filePath) // Default to base filename - - var metadata common.FileMetadata - if includeSubDirName { - absUploadPath, err := common.GetAbsolutePath(uploadPath) - if err != nil { - return common.FileUploadRequestObject{}, err - } - - // Ensure absUploadPath is a directory path for relative calculation - // Trim the optional wildcard if present - uploadDir := strings.TrimSuffix(absUploadPath, common.PathSeparator+"*") - fileInfo, err := os.Stat(uploadDir) - if err != nil { - return common.FileUploadRequestObject{}, err - } - if fileInfo.IsDir() { - // Calculate the path of the file relative to the upload directory - relPath, err := filepath.Rel(uploadDir, filePath) - if err != nil { - return common.FileUploadRequestObject{}, err - } - filename = relPath - } - } - - if includeMetadata { - // The metadata path is the file name plus '_metadata.json' - metadataFilePath := strings.TrimSuffix(filePath, filepath.Ext(filePath)) + "_metadata.json" - var metadataFileBytes []byte - if _, err := os.Stat(metadataFilePath); err == nil { - metadataFileBytes, err = os.ReadFile(metadataFilePath) - if err != nil { - return common.FileUploadRequestObject{}, errors.New("Error reading metadata file " + metadataFilePath + ": " + err.Error()) - } - err := json.Unmarshal(metadataFileBytes, &metadata) - if err != nil { - return common.FileUploadRequestObject{}, errors.New("Error parsing metadata file " + metadataFilePath + ": " + err.Error()) - } - } else { - // No metadata file was found for this file -- proceed, but warn the user. - logger.Printf("WARNING: File metadata is enabled, but could not find the metadata file %v for file %v. Execute `data-client upload --help` for more info on file metadata.\n", metadataFilePath, filePath) - } - } - return common.FileUploadRequestObject{FilePath: filePath, Filename: filename, FileMetadata: metadata, GUID: objectId}, nil -} - -func getFullFilePath(filePath string, filename string) (string, error) { - filePath, err := common.GetAbsolutePath(filePath) - if err != nil { - return "", err - } - fi, err := os.Stat(filePath) - if err != nil { - return "", err - } - switch mode := fi.Mode(); { - case mode.IsDir(): - if strings.HasSuffix(filePath, "/") { - return filePath + filename, nil - } - return filePath + "/" + filename, nil - case mode.IsRegular(): - return "", errors.New("in manifest upload mode filePath must be a dir") - default: - return "", errors.New("full file path creation unsuccessful") - } -} - -func uploadFile(g3i client.Gen3Interface, furObject common.FileUploadRequestObject, retryCount int) error { - g3i.Logger().Println("Uploading data ...") - if furObject.Progress != nil { - defer furObject.Progress.Wait() - } - - client := &http.Client{} - resp, err := client.Do(furObject.Request) - if err != nil { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, retryCount, false) - return errors.New("Error occurred during upload: " + err.Error()) - } - if resp.StatusCode != 200 { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, retryCount, false) - return errors.New("Upload request got a non-200 response with status code " + strconv.Itoa(resp.StatusCode)) - } - g3i.Logger().Printf("Successfully uploaded file \"%s\" to GUID %s.\n", furObject.FilePath, furObject.GUID) - g3i.Logger().DeleteFromFailedLog(furObject.FilePath) - g3i.Logger().Succeeded(furObject.FilePath, furObject.GUID) - return nil -} - -func getNumberOfWorkers(numParallel int, inputSliceLen int) int { - workers := numParallel - if workers < 1 || workers > inputSliceLen { - workers = inputSliceLen - } - return workers -} - -func initBatchUploadChannels(numParallel int, inputSliceLen int) (int, chan *http.Response, chan error, []common.FileUploadRequestObject) { - workers := getNumberOfWorkers(numParallel, inputSliceLen) - respCh := make(chan *http.Response, inputSliceLen) - errCh := make(chan error, inputSliceLen) - batchFURSlice := make([]common.FileUploadRequestObject, 0) - return workers, respCh, errCh, batchFURSlice -} - -func batchUpload(g3i client.Gen3Interface, furObjects []common.FileUploadRequestObject, workers int, respCh chan *http.Response, errCh chan error, bucketName string) { - progress := mpb.New(mpb.WithOutput(os.Stdout)) - respURL := "" - var err error - var guid string - - for i := range furObjects { - if furObjects[i].Bucket == "" { - furObjects[i].Bucket = bucketName - } - if furObjects[i].GUID == "" { - respURL, guid, err = GeneratePresignedURL(g3i, furObjects[i].Filename, furObjects[i].FileMetadata, bucketName) - if err != nil { - g3i.Logger().Failed(furObjects[i].FilePath, furObjects[i].Filename, furObjects[i].FileMetadata, guid, 0, false) - errCh <- err - continue - } - furObjects[i].PresignedURL = respURL - furObjects[i].GUID = guid - // update failed log with new guid - g3i.Logger().Failed(furObjects[i].FilePath, furObjects[i].Filename, furObjects[i].FileMetadata, guid, 0, false) - } - file, err := os.Open(furObjects[i].FilePath) - if err != nil { - g3i.Logger().Failed(furObjects[i].FilePath, furObjects[i].Filename, furObjects[i].FileMetadata, furObjects[i].GUID, 0, false) - errCh <- errors.New("File open error: " + err.Error()) - continue - } - defer file.Close() - - furObjects[i], err = GenerateUploadRequest(g3i, furObjects[i], file, progress) - if err != nil { - file.Close() - g3i.Logger().Failed(furObjects[i].FilePath, furObjects[i].Filename, furObjects[i].FileMetadata, furObjects[i].GUID, 0, false) - errCh <- errors.New("Error occurred during request generation: " + err.Error()) - continue - } - } - - furObjectCh := make(chan common.FileUploadRequestObject, len(furObjects)) - - client := &http.Client{} - wg := sync.WaitGroup{} - for range workers { - wg.Add(1) - go func() { - for furObject := range furObjectCh { - if furObject.Request != nil { - resp, err := client.Do(furObject.Request) - if err != nil { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - errCh <- err - } else { - if resp.StatusCode != 200 { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - } else { - respCh <- resp - g3i.Logger().DeleteFromFailedLog(furObject.FilePath) - g3i.Logger().Succeeded(furObject.FilePath, furObject.GUID) - g3i.Logger().Scoreboard().IncrementSB(0) - } - } - } else if furObject.FilePath != "" { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - } - } - wg.Done() - }() - } - - for i := range furObjects { - furObjectCh <- furObjects[i] - } - close(furObjectCh) - - wg.Wait() - progress.Wait() -} - -// GetWaitTime calculates the wait time for the next retry based on retry count -func GetWaitTime(retryCount int) time.Duration { - exponentialWaitTime := math.Pow(2, float64(retryCount)) - return time.Duration(math.Min(exponentialWaitTime, float64(maxWaitTime))) * time.Second -} - -// FormatSize helps to parse a int64 size into string -func FormatSize(size int64) string { - var unitSize int64 - switch { - case size >= TB: - unitSize = TB - case size >= GB: - unitSize = GB - case size >= MB: - unitSize = MB - case size >= KB: - unitSize = KB - default: - unitSize = B - } - - return fmt.Sprintf("%.1f"+unitMap[unitSize], float64(size)/float64(unitSize)) -} diff --git a/client/gen3Client/client.go b/client/gen3Client/client.go deleted file mode 100644 index a4e46c7..0000000 --- a/client/gen3Client/client.go +++ /dev/null @@ -1,120 +0,0 @@ -package client - -import ( - "bytes" - "context" - "errors" - "fmt" - "net/http" - "net/url" - - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/logs" -) - -//go:generate mockgen -destination=../mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/client/gen3Client Gen3Interface - -// Gen3Interface contains methods used to make authorized http requests to Gen3 services. -// The credential is embedded in the implementation, so it doesn't need to be passed to each method. -type Gen3Interface interface { - CheckPrivileges() (string, map[string]any, error) - CheckForShepherdAPI() (bool, error) - GetResponse(endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) - DoRequestWithSignedHeader(endpointPostPrefix string, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) - MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) - GetHost() (*url.URL, error) - GetCredential() *jwt.Credential - DeleteRecord(guid string) (string, error) - - Logger() *logs.TeeLogger -} - -// Gen3Client wraps jwt.FunctionInterface and embeds the credential -type Gen3Client struct { - Ctx context.Context - FunctionInterface jwt.FunctionInterface - credential *jwt.Credential - - logger *logs.TeeLogger -} - -func (g *Gen3Client) Logger() *logs.TeeLogger { - return g.logger -} - -// CheckPrivileges wraps the underlying method with embedded credential -func (g *Gen3Client) CheckPrivileges() (string, map[string]any, error) { - return g.FunctionInterface.CheckPrivileges(g.credential) -} - -// CheckForShepherdAPI wraps the underlying method with embedded credential -func (g *Gen3Client) CheckForShepherdAPI() (bool, error) { - return g.FunctionInterface.CheckForShepherdAPI(g.credential) -} - -// GetResponse wraps the underlying method with embedded credential -func (g *Gen3Client) GetResponse(endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) { - return g.FunctionInterface.GetResponse(g.credential, endpointPostPrefix, method, contentType, bodyBytes) -} - -// DoRequestWithSignedHeader wraps the underlying method with embedded credential -func (g *Gen3Client) DoRequestWithSignedHeader(endpointPostPrefix string, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) { - return g.FunctionInterface.DoRequestWithSignedHeader(g.credential, endpointPostPrefix, contentType, bodyBytes) -} - -// GetHost wraps the underlying method with embedded credential -func (g *Gen3Client) GetHost() (*url.URL, error) { - return g.FunctionInterface.GetHost(g.credential) -} - -// GetCredential returns the embedded credential -func (g *Gen3Client) GetCredential() *jwt.Credential { - return g.credential -} - -// MakeARequest wraps the underlying Request.MakeARequest method -func (g *Gen3Client) MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) { - // Access the underlying Request through the Functions struct - // We need to create a temporary Request instance since we can't access it directly - if functions, ok := g.FunctionInterface.(*jwt.Functions); ok { - return functions.Request.MakeARequest(method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) - } - return nil, errors.New("unable to access MakeARequest method") -} - -// DeleteRecord deletes a record from INDEXD as well as its storage locations -func (g *Gen3Client) DeleteRecord(guid string) (string, error) { - // Use the embedded credential - // Since DeleteRecord is not part of FunctionInterface, we need to access it via type assertion - // or create a new Functions instance. We'll use type assertion first. - if functions, ok := g.FunctionInterface.(*jwt.Functions); ok { - return functions.DeleteRecord(g.credential, guid) - } - - // This should never happen, but handle it gracefully - return "", errors.New("unable to access DeleteRecord method") -} - -// NewGen3Interface returns a Gen3Client that embeds the credential and implements Gen3Interface. -// This eliminates the need to pass credentials around everywhere. -func NewGen3Interface(ctx context.Context, profile string, logger *logs.TeeLogger, opts ...func(*Gen3Client)) (Gen3Interface, error) { - // Note: A tee logger must be passed here otherwise you risk causing panics. - - config := &jwt.Configure{} - request := &jwt.Request{Ctx: ctx, Logs: logger} - client := jwt.NewFunctions(ctx, config, request) - - cred, err := config.ParseConfig(profile) - if err != nil { - return nil, err - } - if valid, err := config.IsValidCredential(cred); !valid { - return nil, fmt.Errorf("invalid credential: %v", err) - } - - return &Gen3Client{ - FunctionInterface: client, - credential: &cred, - logger: logger, - }, nil -} diff --git a/client/jwt/configure.go b/client/jwt/configure.go deleted file mode 100644 index 2b9b7f6..0000000 --- a/client/jwt/configure.go +++ /dev/null @@ -1,321 +0,0 @@ -package jwt - -//go:generate mockgen -destination=../mocks/mock_configure.go -package=mocks github.com/calypr/data-client/client/jwt ConfigureInterface - -import ( - "encoding/json" - "errors" - "fmt" - "net/url" - "os" - "path" - "regexp" - "strings" - "time" - - "github.com/calypr/data-client/client/common" - "github.com/calypr/data-client/client/logs" - "github.com/golang-jwt/jwt/v5" - "gopkg.in/ini.v1" -) - -var ErrProfileNotFound = errors.New("profile not found in config file") - -type Credential struct { - Profile string - KeyId string - APIKey string - AccessToken string - APIEndpoint string - UseShepherd string - MinShepherdVersion string -} - -type Configure struct { - Logs logs.Logger -} - -type ConfigureInterface interface { - ReadFile(string, string) string - ValidateUrl(string) (*url.URL, error) - GetConfigPath() (string, error) - UpdateConfigFile(Credential) error - ParseKeyValue(str string, expr string) (string, error) - ParseConfig(profile string) (Credential, error) - IsValidCredential(Credential) (bool, error) -} - -func (conf *Configure) ReadFile(filePath string, fileType string) string { - //Look in config file - fullFilePath, err := common.GetAbsolutePath(filePath) - if err != nil { - conf.Logs.Println("error occurred when parsing config file path: " + err.Error()) - return "" - } - if _, err := os.Stat(fullFilePath); err != nil { - conf.Logs.Println("File specified at " + fullFilePath + " not found") - return "" - } - - content, err := os.ReadFile(fullFilePath) - if err != nil { - conf.Logs.Println("error occurred when reading file: " + err.Error()) - return "" - } - - contentStr := string(content[:]) - - if fileType == "json" { - contentStr = strings.ReplaceAll(contentStr, "\n", "") - } - return contentStr -} - -func (conf *Configure) ValidateUrl(apiEndpoint string) (*url.URL, error) { - parsedURL, err := url.Parse(apiEndpoint) - if err != nil { - return parsedURL, errors.New("Error occurred when parsing apiendpoint URL: " + err.Error()) - } - if parsedURL.Host == "" { - return parsedURL, errors.New("Invalid endpoint. A valid endpoint looks like: https://www.tests.com") - } - return parsedURL, nil -} - -func (conf *Configure) ReadCredentials(filePath string, fenceToken string) (*Credential, error) { - var profileConfig Credential - if filePath != "" { - jsonContent := conf.ReadFile(filePath, "json") - jsonContent = strings.ReplaceAll(jsonContent, "key_id", "KeyId") - jsonContent = strings.ReplaceAll(jsonContent, "api_key", "APIKey") - err := json.Unmarshal([]byte(jsonContent), &profileConfig) - if err != nil { - errs := fmt.Errorf("Cannot read json file: %s", err.Error()) - conf.Logs.Println(errs.Error()) - return nil, errs - } - } else if fenceToken != "" { - profileConfig.AccessToken = fenceToken - } - return &profileConfig, nil -} - -func (conf *Configure) GetConfigPath() (string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", err - } - configPath := path.Join(homeDir + common.PathSeparator + ".gen3" + common.PathSeparator + "gen3_client_config.ini") - return configPath, nil -} - -func (conf *Configure) InitConfigFile() error { - /* - Make sure the config exists on start up - */ - configPath, err := conf.GetConfigPath() - if err != nil { - return err - } - - if _, err := os.Stat(path.Dir(configPath)); os.IsNotExist(err) { - osErr := os.Mkdir(path.Join(path.Dir(configPath)), os.FileMode(0777)) - if osErr != nil { - return err - } - _, osErr = os.Create(configPath) - if osErr != nil { - return err - } - } - if _, err := os.Stat(configPath); os.IsNotExist(err) { - _, osErr := os.Create(configPath) - if osErr != nil { - return err - } - } - _, err = ini.Load(configPath) - - return err -} - -func (conf *Configure) UpdateConfigFile(profileConfig Credential) error { - /* - Overwrite the config file with new credential - - Args: - profileConfig: Credential object represents config of a profile - configPath: file path to config file - */ - configPath, err := conf.GetConfigPath() - if err != nil { - errs := fmt.Errorf("error occurred when getting config path: %s", err.Error()) - conf.Logs.Println(errs.Error()) - return errs - } - cfg, err := ini.Load(configPath) - if err != nil { - errs := fmt.Errorf("error occurred when loading config file: %s", err.Error()) - conf.Logs.Println(errs.Error()) - return errs - } - - section := cfg.Section(profileConfig.Profile) - if profileConfig.KeyId != "" { - section.Key("key_id").SetValue(profileConfig.KeyId) - } - if profileConfig.APIKey != "" { - section.Key("api_key").SetValue(profileConfig.APIKey) - } - if profileConfig.AccessToken != "" { - section.Key("access_token").SetValue(profileConfig.AccessToken) - } - if profileConfig.APIEndpoint != "" { - section.Key("api_endpoint").SetValue(profileConfig.APIEndpoint) - } - - section.Key("use_shepherd").SetValue(profileConfig.UseShepherd) - section.Key("min_shepherd_version").SetValue(profileConfig.MinShepherdVersion) - err = cfg.SaveTo(configPath) - if err != nil { - errs := fmt.Errorf("error occurred when saving config file: %s", err.Error()) - return errs - } - return nil -} - -func (conf *Configure) ParseKeyValue(str string, expr string) (string, error) { - r, err := regexp.Compile(expr) - if err != nil { - return "", fmt.Errorf("error occurred when parsing key/value: %v", err.Error()) - } - match := r.FindStringSubmatch(str) - if len(match) == 0 { - return "", fmt.Errorf("No match found") - } - return match[1], nil -} - -func (conf *Configure) ParseConfig(profile string) (Credential, error) { - /* - Looking profile in config file. The config file is a text file located at ~/.gen3 directory. It can - contain more than 1 profile. If there is no profile found, the user is asked to run a command to - create the profile - - The format of config file is described as following - - [profile1] - key_id=key_id_example_1 - api_key=api_key_example_1 - access_token=access_token_example_1 - api_endpoint=http://localhost:8000 - use_shepherd=true - min_shepherd_version=2.0.0 - - [profile2] - key_id=key_id_example_2 - api_key=api_key_example_2 - access_token=access_token_example_2 - api_endpoint=http://localhost:8000 - use_shepherd=false - min_shepherd_version= - - Args: - profile: the specific profile in config file - Returns: - An instance of Credential - */ - - homeDir, err := os.UserHomeDir() - if err != nil { - errs := fmt.Errorf("Error occurred when getting home directory: %s", err.Error()) - return Credential{}, errs - } - configPath := path.Join(homeDir + common.PathSeparator + ".gen3" + common.PathSeparator + "gen3_client_config.ini") - profileConfig := Credential{ - Profile: profile, - KeyId: "", - APIKey: "", - AccessToken: "", - APIEndpoint: "", - } - if _, err := os.Stat(configPath); os.IsNotExist(err) { - return Credential{}, fmt.Errorf("%w Run configure command (with a profile if desired) to set up account credentials \n"+ - "Example: ./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org", ErrProfileNotFound) - } - - // If profile not in config file, prompt user to set up config first - cfg, err := ini.Load(configPath) - if err != nil { - errs := fmt.Errorf("Error occurred when reading config file: %s", err.Error()) - return Credential{}, errs - } - sec, err := cfg.GetSection(profile) - if err != nil { - return Credential{}, fmt.Errorf("%w: Need to run \"data-client configure --profile="+profile+" --cred= --apiendpoint=\" first", ErrProfileNotFound) - } - // Read in API key, key ID and endpoint for given profile - profileConfig.KeyId = sec.Key("key_id").String() - profileConfig.APIKey = sec.Key("api_key").String() - profileConfig.AccessToken = sec.Key("access_token").String() - - if profileConfig.KeyId == "" && profileConfig.APIKey == "" && profileConfig.AccessToken == "" { - errs := fmt.Errorf("key_id, api_key and access_token not found in profile.") - return Credential{}, errs - } - profileConfig.APIEndpoint = sec.Key("api_endpoint").String() - if profileConfig.APIEndpoint == "" { - errs := fmt.Errorf("api_endpoint not found in profile.") - return Credential{}, errs - } - // UseShepherd and MinShepherdVersion are optional - profileConfig.UseShepherd = sec.Key("use_shepherd").String() - profileConfig.MinShepherdVersion = sec.Key("min_shepherd_version").String() - - return profileConfig, nil -} - -func (conf *Configure) IsValidCredential(profileConfig Credential) (bool, error) { - /* Checks to see if credential in credential file is still valid */ - const expirationThresholdDays = 10 - // Parse the token without verifying the signature to access the claims. - token, _, err := new(jwt.Parser).ParseUnverified(profileConfig.APIKey, jwt.MapClaims{}) - if err != nil { - return false, fmt.Errorf("ERROR: Invalid token format: %v", err) - } - - claims, ok := token.Claims.(jwt.MapClaims) - if !ok { - return false, fmt.Errorf("Unable to parse claims from provided token %#v", token) - } - - exp, ok := claims["exp"].(float64) - if !ok { - return false, fmt.Errorf("ERROR: 'exp' claim not found or is not a number for claims %s", claims) - } - - iat, ok := claims["iat"].(float64) - if !ok { - return false, fmt.Errorf("ERROR: 'iat' claim not found or is not a number for claims %s", claims) - } - - now := time.Now().UTC() - expTime := time.Unix(int64(exp), 0).UTC() - iatTime := time.Unix(int64(iat), 0).UTC() - - if expTime.Before(now) { - return false, fmt.Errorf("key %s expired %s < %s", profileConfig.APIKey, expTime.Format(time.RFC3339), now.Format(time.RFC3339)) - } - if iatTime.After(now) { - return false, fmt.Errorf("key %s not yet valid %s > %s", profileConfig.APIKey, iatTime.Format(time.RFC3339), now.Format(time.RFC3339)) - } - - delta := expTime.Sub(now) - if delta > 0 && delta.Hours() < float64(expirationThresholdDays*24) { - daysUntilExpiration := int(delta.Hours() / 24) - if daysUntilExpiration > 0 { - return true, fmt.Errorf("WARNING %s: Key will expire in %d days, on %s", profileConfig.APIKey, daysUntilExpiration, expTime.Format(time.RFC3339)) - } - } - return true, nil -} diff --git a/client/jwt/functions.go b/client/jwt/functions.go deleted file mode 100644 index 004d61b..0000000 --- a/client/jwt/functions.go +++ /dev/null @@ -1,370 +0,0 @@ -package jwt - -//go:generate mockgen -destination=../mocks/mock_functions.go -package=mocks github.com/calypr/data-client/client/jwt FunctionInterface -//go:generate mockgen -destination=../mocks/mock_request.go -package=mocks github.com/calypr/data-client/client/jwt RequestInterface - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "strings" - - "github.com/calypr/data-client/client/common" - "github.com/calypr/data-client/client/logs" - "github.com/hashicorp/go-version" -) - -func NewFunctions(ctx context.Context, config ConfigureInterface, request RequestInterface) FunctionInterface { - return &Functions{ - Config: config, - Request: request, - } -} - -type Functions struct { - Request RequestInterface - Config ConfigureInterface -} - -type FunctionInterface interface { - CheckPrivileges(profileConfig *Credential) (string, map[string]any, error) - CheckForShepherdAPI(profileConfig *Credential) (bool, error) - GetResponse(profileConfig *Credential, endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) - DoRequestWithSignedHeader(profileConfig *Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (JsonMessage, error) - ParseFenceURLResponse(resp *http.Response) (JsonMessage, error) - GetHost(profileConfig *Credential) (*url.URL, error) -} - -type Request struct { - Logs logs.Logger - Ctx context.Context -} - -type RequestInterface interface { - MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) - RequestNewAccessToken(accessTokenEndpoint string, profileConfig *Credential) error - Logger() logs.Logger -} - -func (r *Request) Logger() logs.Logger { - return r.Logs -} - -func (r *Request) MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) { - /* - Make http request with header and body - */ - if headers == nil { - headers = make(map[string]string) - } - if accessToken != "" { - headers["Authorization"] = "Bearer " + accessToken - } - if contentType != "" { - headers["Content-Type"] = contentType - } - var client *http.Client - if noTimeout { - client = &http.Client{} - } else { - client = &http.Client{Timeout: common.DefaultTimeout} - } - var req *http.Request - var err error - if body == nil { - req, err = http.NewRequestWithContext(r.Ctx, method, apiEndpoint, nil) - } else { - req, err = http.NewRequestWithContext(r.Ctx, method, apiEndpoint, body) - } - if err != nil { - return nil, errors.New("Error occurred during generating HTTP request: " + err.Error()) - } - for k, v := range headers { - req.Header.Add(k, v) - } - resp, err := client.Do(req) - if err != nil { - return nil, errors.New("Error occurred during making HTTP request: " + err.Error()) - } - return resp, nil -} - -func (r *Request) RequestNewAccessToken(accessTokenEndpoint string, profileConfig *Credential) error { - /* - Request new access token to replace the expired one. - - Args: - accessTokenEndpoint: the api endpoint for request new access token - Returns: - profileConfig: new credential - err: error - - */ - body := bytes.NewBufferString("{\"api_key\": \"" + profileConfig.APIKey + "\"}") - resp, err := r.MakeARequest("POST", accessTokenEndpoint, "", "application/json", nil, body, false) - var m AccessTokenStruct - // parse resp error codes first for profile configuration verification - if resp != nil && resp.StatusCode != 200 { - return errors.New("Error occurred in RequestNewAccessToken with error code " + strconv.Itoa(resp.StatusCode) + ", check FENCE log for more details.") - } - if err != nil { - return errors.New("Error occurred in RequestNewAccessToken: " + err.Error()) - } - defer resp.Body.Close() - - str := ResponseToString(resp) - err = DecodeJsonFromString(str, &m) - if err != nil { - return errors.New("Error occurred in RequestNewAccessToken: " + err.Error()) - } - - if m.AccessToken == "" { - return errors.New("Could not get new access key from response string: " + str) - } - profileConfig.AccessToken = m.AccessToken - return nil -} - -func (f *Functions) ParseFenceURLResponse(resp *http.Response) (JsonMessage, error) { - msg := JsonMessage{} - - if resp == nil { - return msg, errors.New("Nil response received") - } - - // Capture the body for error reporting before we do anything else - // Using your existing ResponseToString helper - bodyStr := ResponseToString(resp) - - if !(resp.StatusCode == 200 || resp.StatusCode == 201) { - // Prepare a base error that includes the body content - errorMessage := fmt.Sprintf("Status: %d | Response: %s", resp.StatusCode, bodyStr) - - switch resp.StatusCode { - case 401: - return msg, fmt.Errorf("401 Unauthorized: %s", errorMessage) - case 403: - return msg, fmt.Errorf("403 Forbidden: %s (URL: %s)", bodyStr, resp.Request.URL.String()) - case 404: - return msg, fmt.Errorf("404 Not Found: %s (URL: %s)", bodyStr, resp.Request.URL.String()) - case 500: - return msg, fmt.Errorf("500 Internal Server Error: %s", bodyStr) - case 503: - return msg, fmt.Errorf("503 Service Unavailable: %s", bodyStr) - default: - return msg, fmt.Errorf("Unexpected Error (%d): %s", resp.StatusCode, bodyStr) - } - } - - // Logic for successful status codes - if strings.Contains(bodyStr, "Can't find a location for the data") { - return msg, errors.New("The provided GUID is not found") - } - - err := DecodeJsonFromString(bodyStr, &msg) - if err != nil { - return msg, fmt.Errorf("failed to decode JSON: %w (Raw body: %s)", err, bodyStr) - } - - return msg, nil -} -func (f *Functions) CheckForShepherdAPI(profileConfig *Credential) (bool, error) { - // Check if Shepherd is enabled - if profileConfig.UseShepherd == "false" { - return false, nil - } - if profileConfig.UseShepherd != "true" && common.DefaultUseShepherd == false { - return false, nil - } - // If Shepherd is enabled, make sure that the commons has a compatible version of Shepherd deployed. - // Compare the version returned from the Shepherd version endpoint with the minimum acceptable Shepherd version. - var minShepherdVersion string - if profileConfig.MinShepherdVersion == "" { - minShepherdVersion = common.DefaultMinShepherdVersion - } else { - minShepherdVersion = profileConfig.MinShepherdVersion - } - - _, res, err := f.GetResponse(profileConfig, common.ShepherdVersionEndpoint, "GET", "", nil) - if err != nil { - return false, errors.New("Error occurred during generating HTTP request: " + err.Error()) - } - defer res.Body.Close() - if res.StatusCode != 200 { - return false, nil - } - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return false, errors.New("Error occurred when reading HTTP request: " + err.Error()) - } - body, err := strconv.Unquote(string(bodyBytes)) - if err != nil { - return false, fmt.Errorf("Error occurred when parsing version from Shepherd: %v: %v", string(body), err) - } - // Compare the version in the response to the target version - ver, err := version.NewVersion(body) - if err != nil { - return false, fmt.Errorf("Error occurred when parsing version from Shepherd: %v: %v", string(body), err) - } - minVer, err := version.NewVersion(minShepherdVersion) - if err != nil { - return false, fmt.Errorf("Error occurred when parsing minimum acceptable Shepherd version: %v: %v", minShepherdVersion, err) - } - if ver.GreaterThanOrEqual(minVer) { - return true, nil - } - return false, fmt.Errorf("Shepherd is enabled, but %v does not have correct Shepherd version. (Need Shepherd version >=%v, got %v)", profileConfig.APIEndpoint, minVer, ver) -} - -func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) { - - var resp *http.Response - var err error - - if profileConfig.APIKey == "" && profileConfig.AccessToken == "" && profileConfig.APIEndpoint == "" { - return "", resp, fmt.Errorf("No credentials found in the configuration file! Please use \"./data-client configure\" to configure your credentials first %s", profileConfig) - } - - host, _ := url.Parse(profileConfig.APIEndpoint) - prefixEndPoint := host.Scheme + "://" + host.Host - apiEndpoint := host.Scheme + "://" + host.Host + endpointPostPrefix - isExpiredToken := false - if profileConfig.AccessToken != "" { - resp, err = f.Request.MakeARequest(method, apiEndpoint, profileConfig.AccessToken, contentType, nil, bytes.NewBuffer(bodyBytes), false) - if err != nil { - return "", resp, fmt.Errorf("Error while requesting user access token at %v: %v", apiEndpoint, err) - } - - // 401 code is general error code from FENCE. the error message is also not clear for the case - // that the token expired. Temporary solution: get new access token and make another attempt. - if resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 503) { - isExpiredToken = true - } else { - return prefixEndPoint, resp, err - } - } - if profileConfig.AccessToken == "" || isExpiredToken { - err := f.Request.RequestNewAccessToken(prefixEndPoint+common.FenceAccessTokenEndpoint, profileConfig) - if err != nil { - return prefixEndPoint, resp, err - } - err = f.Config.UpdateConfigFile(*profileConfig) - if err != nil { - return prefixEndPoint, resp, err - } - - resp, err = f.Request.MakeARequest(method, apiEndpoint, profileConfig.AccessToken, contentType, nil, bytes.NewBuffer(bodyBytes), false) - if err != nil { - return prefixEndPoint, resp, err - } - } - - return prefixEndPoint, resp, nil -} - -func (f *Functions) GetHost(profileConfig *Credential) (*url.URL, error) { - if profileConfig.APIEndpoint == "" { - return nil, errors.New("No APIEndpoint found in the configuration file! Please use \"./data-client configure\" to configure your credentials first") - } - host, _ := url.Parse(profileConfig.APIEndpoint) - return host, nil -} - -func (f *Functions) DoRequestWithSignedHeader(profileConfig *Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (JsonMessage, error) { - /* - Do request with signed header. User may have more than one profile and use a profile to make a request - */ - var err error - var msg JsonMessage - - method := "GET" - if bodyBytes != nil { - method = "POST" - } - - _, resp, err := f.GetResponse(profileConfig, endpointPostPrefix, method, contentType, bodyBytes) - if err != nil { - return msg, err - } - defer resp.Body.Close() - - msg, err = f.ParseFenceURLResponse(resp) - return msg, err -} - -func (f *Functions) CheckPrivileges(profileConfig *Credential) (string, map[string]any, error) { - /* - Return user privileges from specified profile - */ - var err error - var data map[string]any - - host, resp, err := f.GetResponse(profileConfig, common.FenceUserEndpoint, "GET", "", nil) - if err != nil { - return "", nil, errors.New("Error occurred when getting response from remote: " + err.Error()) - } - defer resp.Body.Close() - - str := ResponseToString(resp) - err = json.Unmarshal([]byte(str), &data) - if err != nil { - return "", nil, errors.New("Error occurred when unmarshalling response: " + err.Error()) - } - - resourceAccess, ok := data["authz"].(map[string]any) - - // If the `authz` section (Arborist permissions) is empty or missing, try get `project_access` section (Fence permissions) - if len(resourceAccess) == 0 || !ok { - resourceAccess, ok = data["project_access"].(map[string]any) - if !ok { - return "", nil, errors.New("Not possible to read access privileges of user") - } - } - - return host, resourceAccess, err -} - -func (f *Functions) DeleteRecord(profileConfig *Credential, guid string) (string, error) { - var err error - var msg string - - hasShepherd, err := f.CheckForShepherdAPI(profileConfig) - if err != nil { - f.Request.Logger().Printf("WARNING: Error while checking for Shepherd API: %v. Falling back to Fence to delete record.\n", err) - } else if hasShepherd { - endPointPostfix := common.ShepherdEndpoint + "/objects/" + guid - _, resp, err := f.GetResponse(profileConfig, endPointPostfix, "DELETE", "", nil) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode == 204 { - msg = "Record with GUID " + guid + " has been deleted" - } else if resp.StatusCode == 500 { - err = errors.New("Internal server error occurred when deleting " + guid + "; could not delete stored files, or not able to delete INDEXD record") - } - return msg, err - } - - endPointPostfix := common.FenceDataEndpoint + "/" + guid - - _, resp, err := f.GetResponse(profileConfig, endPointPostfix, "DELETE", "", nil) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode == 204 { - msg = "Record with GUID " + guid + " has been deleted" - } else if resp.StatusCode == 500 { - err = errors.New("Internal server error occurred when deleting " + guid + "; could not delete stored files, or not able to delete INDEXD record") - } - - return msg, err -} diff --git a/client/jwt/update.go b/client/jwt/update.go deleted file mode 100644 index b2d9cc9..0000000 --- a/client/jwt/update.go +++ /dev/null @@ -1,78 +0,0 @@ -package jwt - -import ( - "context" - "errors" - "fmt" - "strings" - - "github.com/calypr/data-client/client/common" - "github.com/calypr/data-client/client/logs" - "github.com/hashicorp/go-version" -) - -func UpdateConfig(logger logs.Logger, cred *Credential) error { - var conf Configure - var req Request = Request{Ctx: context.Background()} - - if cred.Profile == "" { - return fmt.Errorf("profile name is required") - } - if cred.APIEndpoint == "" { - return fmt.Errorf("API endpoint is required") - } - - // Normalize endpoint - cred.APIEndpoint = strings.TrimSpace(cred.APIEndpoint) - cred.APIEndpoint = strings.TrimSuffix(cred.APIEndpoint, "/") - - // Validate URL format - parsedURL, err := conf.ValidateUrl(cred.APIEndpoint) - if err != nil { - return fmt.Errorf("invalid apiendpoint URL: %w", err) - } - fenceBase := parsedURL.Scheme + "://" + parsedURL.Host - if existingCfg, err := conf.ParseConfig(cred.Profile); err == nil { - // Only copy optional fields if the user didn't override them via flags - if cred.UseShepherd == "" { - cred.UseShepherd = existingCfg.UseShepherd - } - if cred.MinShepherdVersion == "" { - cred.MinShepherdVersion = existingCfg.MinShepherdVersion - } - } else if !errors.Is(err, ErrProfileNotFound) { - return err - } - - if cred.APIKey != "" { - // Always refresh the access token — ignore any old one that might be in the struct - err = req.RequestNewAccessToken(fenceBase+common.FenceAccessTokenEndpoint, cred) - if err != nil { - if strings.Contains(err.Error(), "401") { - return fmt.Errorf("authentication failed (401) for %s — your API key is invalid, revoked, or expired", fenceBase) - } - if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "no such host") { - return fmt.Errorf("cannot reach Fence at %s — is this a valid Gen3 commons?", fenceBase) - } - return fmt.Errorf("failed to refresh access token: %w", err) - } - } else { - logger.Printf("WARNING: Your profile will only be valid for 24 hours since you have only provided a refresh token for authentication") - } - - // Clean up shepherd flags - cred.UseShepherd = strings.TrimSpace(cred.UseShepherd) - cred.MinShepherdVersion = strings.TrimSpace(cred.MinShepherdVersion) - - if cred.MinShepherdVersion != "" { - if _, err = version.NewVersion(cred.MinShepherdVersion); err != nil { - return fmt.Errorf("invalid min-shepherd-version: %w", err) - } - } - - if err := conf.UpdateConfigFile(*cred); err != nil { - return fmt.Errorf("failed to write config file: %w", err) - } - - return nil -} diff --git a/client/jwt/utils.go b/client/jwt/utils.go deleted file mode 100644 index 466a3a0..0000000 --- a/client/jwt/utils.go +++ /dev/null @@ -1,38 +0,0 @@ -package jwt - -import ( - "bytes" - "encoding/json" - "net/http" -) - -type Message any - -type Response any - -type AccessTokenStruct struct { - AccessToken string `json:"access_token"` -} - -type JsonMessage struct { - URL string `json:"url"` - GUID string `json:"guid"` - UploadID string `json:"uploadId"` - PresignedURL string `json:"presigned_url"` - FileName string `json:"file_name"` - URLs []string `json:"urls"` - Size int64 `json:"size"` -} - -type DoRequest func(*http.Response) *http.Response - -func ResponseToString(resp *http.Response) string { - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) // nolint: errcheck - return buf.String() -} - -func DecodeJsonFromString(str string, msg Message) error { - err := json.Unmarshal([]byte(str), &msg) - return err -} diff --git a/client/logs/logger.go b/client/logs/logger.go deleted file mode 100644 index 7a6d53b..0000000 --- a/client/logs/logger.go +++ /dev/null @@ -1,41 +0,0 @@ -package logs - -import ( - "io" -) - -type Logger interface { - Printf(format string, v ...any) - Println(v ...any) - Fatalf(format string, v ...any) - Fatal(v ...any) - Writer() io.Writer -} - -type Option func(*config) - -type config struct { - console bool - messageFile bool - failedLog bool - succeededLog bool - enableScoreboard bool - baseLogger Logger -} - -func WithConsole() Option { return func(c *config) { c.console = true } } -func WithMessageFile() Option { return func(c *config) { c.messageFile = true } } -func WithFailedLog() Option { return func(c *config) { c.failedLog = true } } -func WithSucceededLog() Option { return func(c *config) { c.succeededLog = true } } -func WithScoreboard() Option { return func(c *config) { c.enableScoreboard = true } } -func WithBaseLogger(base Logger) Option { return func(c *config) { c.baseLogger = base } } - -func defaults() *config { - return &config{ - console: true, - messageFile: true, - failedLog: true, - succeededLog: true, - baseLogger: nil, - } -} diff --git a/client/logs/tee_logger.go b/client/logs/tee_logger.go deleted file mode 100644 index bd78d8a..0000000 --- a/client/logs/tee_logger.go +++ /dev/null @@ -1,174 +0,0 @@ -package logs - -import ( - "encoding/json" - "fmt" - "io" // Added for standard logging methods like Fatal - "os" - "sync" - - "github.com/calypr/data-client/client/common" -) - -// --- teeLogger Implementation --- -type TeeLogger struct { - mu sync.RWMutex - writers []io.Writer - scoreboard *Scoreboard - - failedMu sync.Mutex - FailedMap map[string]common.RetryObject // Maps filePath to FileMetadata - failedPath string - - succeededMu sync.Mutex - succeededMap map[string]string // Maps filePath to GUID - succeededPath string -} - -// NewTeeLogger combines initialization and log loading (replacing initSyncLogs) -func NewTeeLogger(logDir, profile string, writers ...io.Writer) *TeeLogger { - t := &TeeLogger{ - mu: sync.RWMutex{}, - writers: writers, - scoreboard: nil, - - FailedMap: make(map[string]common.RetryObject), - succeededMap: make(map[string]string), - } - - return t -} - -// Internal helper function (replaces the global loadJSON) -func loadJSON(path string, v any) { - data, _ := os.ReadFile(path) - if len(data) > 0 { - // Error handling for Unmarshal is often omitted in utility code - // but is good practice. We keep the original style for now. - json.Unmarshal(data, v) - } -} - -// --- Public Logger Methods --- - -// Printf implements part of the standard Logger interface. -func (t *TeeLogger) Printf(format string, v ...any) { - t.write(fmt.Sprintf(format, v...)) -} - -// Println implements part of the standard Logger interface. -func (t *TeeLogger) Println(v ...any) { - t.write(fmt.Sprintln(v...)) -} - -// Fatalf implements part of the standard Logger interface and exits the program. -func (t *TeeLogger) Fatalf(format string, v ...any) { - s := fmt.Sprintf(format, v...) - t.write(s) - os.Exit(1) -} - -// Fatal implements part of the standard Logger interface and exits the program. -func (t *TeeLogger) Fatal(v ...any) { - s := fmt.Sprintln(v...) - t.write(s) - os.Exit(1) -} - -// Writer implements part of the standard Logger interface, returning a multi-writer. -func (t *TeeLogger) Writer() io.Writer { - t.mu.RLock() - defer t.mu.RUnlock() - return io.MultiWriter(t.writers...) -} - -// Scoreboard returns the embedded ScoreboardAccess. -func (t *TeeLogger) Scoreboard() *Scoreboard { - return t.scoreboard -} - -// GetSucceededLogMap returns a copy of the succeeded log map. -func (t *TeeLogger) GetSucceededLogMap() map[string]string { - t.succeededMu.Lock() - defer t.succeededMu.Unlock() - // Return a copy to prevent external modification - copiedMap := make(map[string]string, len(t.succeededMap)) - for k, v := range t.succeededMap { - copiedMap[k] = v - } - return copiedMap -} - -// GetFailedLogMap returns a copy of the failed log map. -func (t *TeeLogger) GetFailedLogMap() map[string]common.RetryObject { - t.failedMu.Lock() - defer t.failedMu.Unlock() - // Return a copy to prevent external modification - copiedMap := make(map[string]common.RetryObject, len(t.FailedMap)) - for k, v := range t.FailedMap { - copiedMap[k] = v - } - return copiedMap -} - -func (t *TeeLogger) DeleteFromFailedLog(path string) { - t.failedMu.Lock() - defer t.failedMu.Unlock() - delete(t.FailedMap, path) -} - -// --- Internal Utility Methods --- - -// write handles writing the string to all configured writers. -func (t *TeeLogger) write(s string) { - t.mu.RLock() - defer t.mu.RUnlock() - for _, w := range t.writers { - _, _ = fmt.Fprint(w, s) - } -} - -func (t *TeeLogger) GetSucceededCount() int { - return len(t.succeededMap) -} - -func (t *TeeLogger) writeFailedSync(e common.RetryObject) { - t.failedMu.Lock() - defer t.failedMu.Unlock() - - // Store the FileMetadata part in the map - t.FailedMap[e.FilePath] = e - - data, _ := json.MarshalIndent(t.FailedMap, "", " ") - os.WriteFile(t.failedPath, data, 0644) -} - -func (t *TeeLogger) writeSucceededSync(path, guid string) { - t.succeededMu.Lock() - defer t.succeededMu.Unlock() - t.succeededMap[path] = guid - data, _ := json.MarshalIndent(t.succeededMap, "", " ") - os.WriteFile(t.succeededPath, data, 0644) -} - -// --- Tracking Methods (Part of Logger Interface) --- - -func (t *TeeLogger) Failed(filePath, filename string, metadata common.FileMetadata, guid string, retryCount int, multipart bool) { - if t.failedPath != "" { - t.writeFailedSync(common.RetryObject{ - FilePath: filePath, - Filename: filename, - FileMetadata: metadata, - GUID: guid, - RetryCount: retryCount, - Multipart: multipart, - }) - } -} - -func (t *TeeLogger) Succeeded(filePath, guid string) { - // Use t.succeededPath instead of checking the old global succeededPath - if t.succeededPath != "" { - t.writeSucceededSync(filePath, guid) - } -} diff --git a/client/mocks/mock_configure.go b/client/mocks/mock_configure.go deleted file mode 100644 index 697c3da..0000000 --- a/client/mocks/mock_configure.go +++ /dev/null @@ -1,145 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/data-client/client/jwt (interfaces: ConfigureInterface) -// -// Generated by this command: -// -// mockgen -destination=../mocks/mock_configure.go -package=mocks github.com/calypr/data-client/client/jwt ConfigureInterface -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - url "net/url" - reflect "reflect" - - jwt "github.com/calypr/data-client/client/jwt" - gomock "go.uber.org/mock/gomock" -) - -// MockConfigureInterface is a mock of ConfigureInterface interface. -type MockConfigureInterface struct { - ctrl *gomock.Controller - recorder *MockConfigureInterfaceMockRecorder - isgomock struct{} -} - -// MockConfigureInterfaceMockRecorder is the mock recorder for MockConfigureInterface. -type MockConfigureInterfaceMockRecorder struct { - mock *MockConfigureInterface -} - -// NewMockConfigureInterface creates a new mock instance. -func NewMockConfigureInterface(ctrl *gomock.Controller) *MockConfigureInterface { - mock := &MockConfigureInterface{ctrl: ctrl} - mock.recorder = &MockConfigureInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockConfigureInterface) EXPECT() *MockConfigureInterfaceMockRecorder { - return m.recorder -} - -// GetConfigPath mocks base method. -func (m *MockConfigureInterface) GetConfigPath() (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetConfigPath") - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetConfigPath indicates an expected call of GetConfigPath. -func (mr *MockConfigureInterfaceMockRecorder) GetConfigPath() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigPath", reflect.TypeOf((*MockConfigureInterface)(nil).GetConfigPath)) -} - -// IsValidCredential mocks base method. -func (m *MockConfigureInterface) IsValidCredential(arg0 jwt.Credential) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsValidCredential", arg0) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsValidCredential indicates an expected call of IsValidCredential. -func (mr *MockConfigureInterfaceMockRecorder) IsValidCredential(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsValidCredential", reflect.TypeOf((*MockConfigureInterface)(nil).IsValidCredential), arg0) -} - -// ParseConfig mocks base method. -func (m *MockConfigureInterface) ParseConfig(profile string) (jwt.Credential, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ParseConfig", profile) - ret0, _ := ret[0].(jwt.Credential) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ParseConfig indicates an expected call of ParseConfig. -func (mr *MockConfigureInterfaceMockRecorder) ParseConfig(profile any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseConfig", reflect.TypeOf((*MockConfigureInterface)(nil).ParseConfig), profile) -} - -// ParseKeyValue mocks base method. -func (m *MockConfigureInterface) ParseKeyValue(str, expr string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ParseKeyValue", str, expr) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ParseKeyValue indicates an expected call of ParseKeyValue. -func (mr *MockConfigureInterfaceMockRecorder) ParseKeyValue(str, expr any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseKeyValue", reflect.TypeOf((*MockConfigureInterface)(nil).ParseKeyValue), str, expr) -} - -// ReadFile mocks base method. -func (m *MockConfigureInterface) ReadFile(arg0, arg1 string) string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReadFile", arg0, arg1) - ret0, _ := ret[0].(string) - return ret0 -} - -// ReadFile indicates an expected call of ReadFile. -func (mr *MockConfigureInterfaceMockRecorder) ReadFile(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadFile", reflect.TypeOf((*MockConfigureInterface)(nil).ReadFile), arg0, arg1) -} - -// UpdateConfigFile mocks base method. -func (m *MockConfigureInterface) UpdateConfigFile(arg0 jwt.Credential) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateConfigFile", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateConfigFile indicates an expected call of UpdateConfigFile. -func (mr *MockConfigureInterfaceMockRecorder) UpdateConfigFile(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigFile", reflect.TypeOf((*MockConfigureInterface)(nil).UpdateConfigFile), arg0) -} - -// ValidateUrl mocks base method. -func (m *MockConfigureInterface) ValidateUrl(arg0 string) (*url.URL, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateUrl", arg0) - ret0, _ := ret[0].(*url.URL) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ValidateUrl indicates an expected call of ValidateUrl. -func (mr *MockConfigureInterfaceMockRecorder) ValidateUrl(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateUrl", reflect.TypeOf((*MockConfigureInterface)(nil).ValidateUrl), arg0) -} diff --git a/client/mocks/mock_functions.go b/client/mocks/mock_functions.go deleted file mode 100644 index 6c48765..0000000 --- a/client/mocks/mock_functions.go +++ /dev/null @@ -1,135 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/data-client/client/jwt (interfaces: FunctionInterface) -// -// Generated by this command: -// -// mockgen -destination=../mocks/mock_functions.go -package=mocks github.com/calypr/data-client/client/jwt FunctionInterface -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - http "net/http" - url "net/url" - reflect "reflect" - - jwt "github.com/calypr/data-client/client/jwt" - gomock "go.uber.org/mock/gomock" -) - -// MockFunctionInterface is a mock of FunctionInterface interface. -type MockFunctionInterface struct { - ctrl *gomock.Controller - recorder *MockFunctionInterfaceMockRecorder - isgomock struct{} -} - -// MockFunctionInterfaceMockRecorder is the mock recorder for MockFunctionInterface. -type MockFunctionInterfaceMockRecorder struct { - mock *MockFunctionInterface -} - -// NewMockFunctionInterface creates a new mock instance. -func NewMockFunctionInterface(ctrl *gomock.Controller) *MockFunctionInterface { - mock := &MockFunctionInterface{ctrl: ctrl} - mock.recorder = &MockFunctionInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockFunctionInterface) EXPECT() *MockFunctionInterfaceMockRecorder { - return m.recorder -} - -// CheckForShepherdAPI mocks base method. -func (m *MockFunctionInterface) CheckForShepherdAPI(profileConfig *jwt.Credential) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckForShepherdAPI", profileConfig) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CheckForShepherdAPI indicates an expected call of CheckForShepherdAPI. -func (mr *MockFunctionInterfaceMockRecorder) CheckForShepherdAPI(profileConfig any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckForShepherdAPI", reflect.TypeOf((*MockFunctionInterface)(nil).CheckForShepherdAPI), profileConfig) -} - -// CheckPrivileges mocks base method. -func (m *MockFunctionInterface) CheckPrivileges(profileConfig *jwt.Credential) (string, map[string]any, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckPrivileges", profileConfig) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(map[string]any) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// CheckPrivileges indicates an expected call of CheckPrivileges. -func (mr *MockFunctionInterfaceMockRecorder) CheckPrivileges(profileConfig any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckPrivileges", reflect.TypeOf((*MockFunctionInterface)(nil).CheckPrivileges), profileConfig) -} - -// DoRequestWithSignedHeader mocks base method. -func (m *MockFunctionInterface) DoRequestWithSignedHeader(profileConfig *jwt.Credential, endpointPostPrefix, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DoRequestWithSignedHeader", profileConfig, endpointPostPrefix, contentType, bodyBytes) - ret0, _ := ret[0].(jwt.JsonMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DoRequestWithSignedHeader indicates an expected call of DoRequestWithSignedHeader. -func (mr *MockFunctionInterfaceMockRecorder) DoRequestWithSignedHeader(profileConfig, endpointPostPrefix, contentType, bodyBytes any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DoRequestWithSignedHeader", reflect.TypeOf((*MockFunctionInterface)(nil).DoRequestWithSignedHeader), profileConfig, endpointPostPrefix, contentType, bodyBytes) -} - -// GetHost mocks base method. -func (m *MockFunctionInterface) GetHost(profileConfig *jwt.Credential) (*url.URL, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetHost", profileConfig) - ret0, _ := ret[0].(*url.URL) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetHost indicates an expected call of GetHost. -func (mr *MockFunctionInterfaceMockRecorder) GetHost(profileConfig any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHost", reflect.TypeOf((*MockFunctionInterface)(nil).GetHost), profileConfig) -} - -// GetResponse mocks base method. -func (m *MockFunctionInterface) GetResponse(profileConfig *jwt.Credential, endpointPostPrefix, method, contentType string, bodyBytes []byte) (string, *http.Response, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetResponse", profileConfig, endpointPostPrefix, method, contentType, bodyBytes) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(*http.Response) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetResponse indicates an expected call of GetResponse. -func (mr *MockFunctionInterfaceMockRecorder) GetResponse(profileConfig, endpointPostPrefix, method, contentType, bodyBytes any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResponse", reflect.TypeOf((*MockFunctionInterface)(nil).GetResponse), profileConfig, endpointPostPrefix, method, contentType, bodyBytes) -} - -// ParseFenceURLResponse mocks base method. -func (m *MockFunctionInterface) ParseFenceURLResponse(resp *http.Response) (jwt.JsonMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ParseFenceURLResponse", resp) - ret0, _ := ret[0].(jwt.JsonMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ParseFenceURLResponse indicates an expected call of ParseFenceURLResponse. -func (mr *MockFunctionInterfaceMockRecorder) ParseFenceURLResponse(resp any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseFenceURLResponse", reflect.TypeOf((*MockFunctionInterface)(nil).ParseFenceURLResponse), resp) -} diff --git a/client/mocks/mock_gen3interface.go b/client/mocks/mock_gen3interface.go deleted file mode 100644 index 44dd849..0000000 --- a/client/mocks/mock_gen3interface.go +++ /dev/null @@ -1,180 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/data-client/client/gen3Client (interfaces: Gen3Interface) -// -// Generated by this command: -// -// mockgen -destination=../mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/client/gen3Client Gen3Interface -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - bytes "bytes" - http "net/http" - url "net/url" - reflect "reflect" - - jwt "github.com/calypr/data-client/client/jwt" - logs "github.com/calypr/data-client/client/logs" - gomock "go.uber.org/mock/gomock" -) - -// MockGen3Interface is a mock of Gen3Interface interface. -type MockGen3Interface struct { - ctrl *gomock.Controller - recorder *MockGen3InterfaceMockRecorder - isgomock struct{} -} - -// MockGen3InterfaceMockRecorder is the mock recorder for MockGen3Interface. -type MockGen3InterfaceMockRecorder struct { - mock *MockGen3Interface -} - -// NewMockGen3Interface creates a new mock instance. -func NewMockGen3Interface(ctrl *gomock.Controller) *MockGen3Interface { - mock := &MockGen3Interface{ctrl: ctrl} - mock.recorder = &MockGen3InterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockGen3Interface) EXPECT() *MockGen3InterfaceMockRecorder { - return m.recorder -} - -// CheckForShepherdAPI mocks base method. -func (m *MockGen3Interface) CheckForShepherdAPI() (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckForShepherdAPI") - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CheckForShepherdAPI indicates an expected call of CheckForShepherdAPI. -func (mr *MockGen3InterfaceMockRecorder) CheckForShepherdAPI() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckForShepherdAPI", reflect.TypeOf((*MockGen3Interface)(nil).CheckForShepherdAPI)) -} - -// CheckPrivileges mocks base method. -func (m *MockGen3Interface) CheckPrivileges() (string, map[string]any, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckPrivileges") - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(map[string]any) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// CheckPrivileges indicates an expected call of CheckPrivileges. -func (mr *MockGen3InterfaceMockRecorder) CheckPrivileges() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckPrivileges", reflect.TypeOf((*MockGen3Interface)(nil).CheckPrivileges)) -} - -// DeleteRecord mocks base method. -func (m *MockGen3Interface) DeleteRecord(guid string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteRecord", guid) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DeleteRecord indicates an expected call of DeleteRecord. -func (mr *MockGen3InterfaceMockRecorder) DeleteRecord(guid any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRecord", reflect.TypeOf((*MockGen3Interface)(nil).DeleteRecord), guid) -} - -// DoRequestWithSignedHeader mocks base method. -func (m *MockGen3Interface) DoRequestWithSignedHeader(endpointPostPrefix, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DoRequestWithSignedHeader", endpointPostPrefix, contentType, bodyBytes) - ret0, _ := ret[0].(jwt.JsonMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DoRequestWithSignedHeader indicates an expected call of DoRequestWithSignedHeader. -func (mr *MockGen3InterfaceMockRecorder) DoRequestWithSignedHeader(endpointPostPrefix, contentType, bodyBytes any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DoRequestWithSignedHeader", reflect.TypeOf((*MockGen3Interface)(nil).DoRequestWithSignedHeader), endpointPostPrefix, contentType, bodyBytes) -} - -// GetCredential mocks base method. -func (m *MockGen3Interface) GetCredential() *jwt.Credential { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetCredential") - ret0, _ := ret[0].(*jwt.Credential) - return ret0 -} - -// GetCredential indicates an expected call of GetCredential. -func (mr *MockGen3InterfaceMockRecorder) GetCredential() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCredential", reflect.TypeOf((*MockGen3Interface)(nil).GetCredential)) -} - -// GetHost mocks base method. -func (m *MockGen3Interface) GetHost() (*url.URL, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetHost") - ret0, _ := ret[0].(*url.URL) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetHost indicates an expected call of GetHost. -func (mr *MockGen3InterfaceMockRecorder) GetHost() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHost", reflect.TypeOf((*MockGen3Interface)(nil).GetHost)) -} - -// GetResponse mocks base method. -func (m *MockGen3Interface) GetResponse(endpointPostPrefix, method, contentType string, bodyBytes []byte) (string, *http.Response, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetResponse", endpointPostPrefix, method, contentType, bodyBytes) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(*http.Response) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetResponse indicates an expected call of GetResponse. -func (mr *MockGen3InterfaceMockRecorder) GetResponse(endpointPostPrefix, method, contentType, bodyBytes any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResponse", reflect.TypeOf((*MockGen3Interface)(nil).GetResponse), endpointPostPrefix, method, contentType, bodyBytes) -} - -// Logger mocks base method. -func (m *MockGen3Interface) Logger() *logs.TeeLogger { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Logger") - ret0, _ := ret[0].(*logs.TeeLogger) - return ret0 -} - -// Logger indicates an expected call of Logger. -func (mr *MockGen3InterfaceMockRecorder) Logger() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Logger", reflect.TypeOf((*MockGen3Interface)(nil).Logger)) -} - -// MakeARequest mocks base method. -func (m *MockGen3Interface) MakeARequest(method, apiEndpoint, accessToken, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MakeARequest", method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) - ret0, _ := ret[0].(*http.Response) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// MakeARequest indicates an expected call of MakeARequest. -func (mr *MockGen3InterfaceMockRecorder) MakeARequest(method, apiEndpoint, accessToken, contentType, headers, body, noTimeout any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakeARequest", reflect.TypeOf((*MockGen3Interface)(nil).MakeARequest), method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) -} diff --git a/client/mocks/mock_request.go b/client/mocks/mock_request.go deleted file mode 100644 index 74f87de..0000000 --- a/client/mocks/mock_request.go +++ /dev/null @@ -1,87 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/data-client/client/jwt (interfaces: RequestInterface) -// -// Generated by this command: -// -// mockgen -destination=../mocks/mock_request.go -package=mocks github.com/calypr/data-client/client/jwt RequestInterface -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - bytes "bytes" - http "net/http" - reflect "reflect" - - jwt "github.com/calypr/data-client/client/jwt" - logs "github.com/calypr/data-client/client/logs" - gomock "go.uber.org/mock/gomock" -) - -// MockRequestInterface is a mock of RequestInterface interface. -type MockRequestInterface struct { - ctrl *gomock.Controller - recorder *MockRequestInterfaceMockRecorder - isgomock struct{} -} - -// MockRequestInterfaceMockRecorder is the mock recorder for MockRequestInterface. -type MockRequestInterfaceMockRecorder struct { - mock *MockRequestInterface -} - -// NewMockRequestInterface creates a new mock instance. -func NewMockRequestInterface(ctrl *gomock.Controller) *MockRequestInterface { - mock := &MockRequestInterface{ctrl: ctrl} - mock.recorder = &MockRequestInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRequestInterface) EXPECT() *MockRequestInterfaceMockRecorder { - return m.recorder -} - -// Logger mocks base method. -func (m *MockRequestInterface) Logger() logs.Logger { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Logger") - ret0, _ := ret[0].(logs.Logger) - return ret0 -} - -// Logger indicates an expected call of Logger. -func (mr *MockRequestInterfaceMockRecorder) Logger() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Logger", reflect.TypeOf((*MockRequestInterface)(nil).Logger)) -} - -// MakeARequest mocks base method. -func (m *MockRequestInterface) MakeARequest(method, apiEndpoint, accessToken, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MakeARequest", method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) - ret0, _ := ret[0].(*http.Response) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// MakeARequest indicates an expected call of MakeARequest. -func (mr *MockRequestInterfaceMockRecorder) MakeARequest(method, apiEndpoint, accessToken, contentType, headers, body, noTimeout any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakeARequest", reflect.TypeOf((*MockRequestInterface)(nil).MakeARequest), method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) -} - -// RequestNewAccessToken mocks base method. -func (m *MockRequestInterface) RequestNewAccessToken(accessTokenEndpoint string, profileConfig *jwt.Credential) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RequestNewAccessToken", accessTokenEndpoint, profileConfig) - ret0, _ := ret[0].(error) - return ret0 -} - -// RequestNewAccessToken indicates an expected call of RequestNewAccessToken. -func (mr *MockRequestInterfaceMockRecorder) RequestNewAccessToken(accessTokenEndpoint, profileConfig any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequestNewAccessToken", reflect.TypeOf((*MockRequestInterface)(nil).RequestNewAccessToken), accessTokenEndpoint, profileConfig) -} diff --git a/cmd/arborist.go b/cmd/arborist.go new file mode 100644 index 0000000..8918817 --- /dev/null +++ b/cmd/arborist.go @@ -0,0 +1,290 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/calypr/calypr-cli/arborist" + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + "github.com/spf13/cobra" +) + +type arboristCommandOptions struct { + jsonOutput bool +} + +var arboristOpts arboristCommandOptions + +var arboristCmd = &cobra.Command{ + Use: "permissions", + Aliases: []string{"arborist"}, + Short: "Manage permissions, ownership, and organization access", + Long: "Permissions and ownership power tools. These commands use the Arborist routes exposed through " + + "the configured Gen3 profile.", + RunE: groupHelpOrUnknownError, +} + +func init() { + arboristCmd.PersistentFlags().BoolVar(&arboristOpts.jsonOutput, "json", false, "Output raw JSON responses") + RootCmd.AddCommand(arboristCmd) + + registerArboristAuthCommands() + registerArboristOwnershipCommands() + registerArboristAccessCommands() + registerArboristOrgMembershipCommands() +} + +func getArboristClient() (arborist.ClientInterface, func(), error) { + if strings.TrimSpace(profile) == "" { + return nil, nil, fmt.Errorf("profile is required; use --profile") + } + logger, logCloser := logs.New(profile) + g3i, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + logCloser() + return nil, nil, fmt.Errorf("access Gen3: %w", err) + } + return arborist.NewClient(g3i, g3i.Credentials().Current()), logCloser, nil +} + +func runArborist(cmd *cobra.Command, action func(context.Context, arborist.ClientInterface) (any, string, error)) error { + client, closer, err := getArboristClient() + if err != nil { + return err + } + defer closer() + + result, message, err := action(cmd.Context(), client) + if err != nil { + return err + } + if arboristOpts.jsonOutput { + if result == nil { + result = map[string]string{"message": message} + } + return writeJSON(cmd, result) + } + if strings.TrimSpace(message) != "" { + fmt.Fprintln(cmd.OutOrStdout(), message) + return nil + } + return writeJSON(cmd, result) +} + +func writeJSON(cmd *cobra.Command, value any) error { + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} + +func validEmailArg(value string) (string, error) { + value = strings.ToLower(strings.TrimSpace(value)) + if !emailRegex.MatchString(value) { + return "", fmt.Errorf("invalid email address %q", value) + } + return value, nil +} + +func groupHelpOrUnknownError(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return cmd.Help() + } + cmd.SilenceUsage = true + return fmt.Errorf("unknown command %q for %q", strings.Join(args, " "), cmd.CommandPath()) +} + +func registerArboristAuthCommands() { + authCmd := &cobra.Command{Use: "auth", Short: "Inspect current authorization mapping", RunE: groupHelpOrUnknownError} + authCmd.AddCommand(&cobra.Command{ + Use: "mapping", + Short: "Get auth mapping for the current token", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runArborist(cmd, func(ctx context.Context, client arborist.ClientInterface) (any, string, error) { + var out any + err := client.AuthMapping(ctx, &out) + return out, "", err + }) + }, + }) + arboristCmd.AddCommand(authCmd) +} + +func registerArboristOwnershipCommands() { + ownershipCmd := &cobra.Command{Use: "ownership", Short: "Manage ownership-backed access state", RunE: groupHelpOrUnknownError} + var parentPath, childName, templateName, description string + createDescendantCmd := &cobra.Command{ + Use: "create-descendant", + Short: "Create a missing child resource and owner grants", + RunE: func(cmd *cobra.Command, args []string) error { + if parentPath == "" || childName == "" { + return fmt.Errorf("--parent and --name are required") + } + return runArborist(cmd, func(ctx context.Context, client arborist.ClientInterface) (any, string, error) { + var out any + err := client.CreateOwnedDescendant(ctx, arborist.CreateOwnedDescendantRequest{ + ParentPath: parentPath, + Name: childName, + Template: templateName, + Description: description, + }, &out) + return out, fmt.Sprintf("Created owned descendant %s under %s", childName, parentPath), err + }) + }, + } + createDescendantCmd.Flags().StringVar(&parentPath, "parent", "", "Parent resource path") + createDescendantCmd.Flags().StringVar(&childName, "name", "", "Child resource name") + createDescendantCmd.Flags().StringVar(&templateName, "template", "", "Ownership template") + createDescendantCmd.Flags().StringVar(&description, "description", "", "Resource description") + ownershipCmd.AddCommand(createDescendantCmd) + + ownershipCmd.AddCommand(ownerMutationCmd("add-owner", "Add an owner", true)) + ownershipCmd.AddCommand(ownerMutationCmd("rm-owner", "Remove an owner", false)) + ownershipCmd.AddCommand(ownershipReadCmd()) + arboristCmd.AddCommand(ownershipCmd) +} + +func registerArboristAccessCommands() { + accessCmd := &cobra.Command{ + Use: "access", + Short: "Manage direct non-owner access grants", + RunE: groupHelpOrUnknownError, + } + accessCmd.AddCommand(accessUserMutationCmd("grant-user", "Grant direct user access on a resource", true)) + accessCmd.AddCommand(accessUserMutationCmd("revoke-user", "Revoke direct user access from a resource", false)) + arboristCmd.AddCommand(accessCmd) +} + +func registerArboristOrgMembershipCommands() { + orgCmd := &cobra.Command{ + Use: "org-membership", + Short: "Manage organization membership grants", + Long: "Convenience wrapper around direct access grants. The default role is org-member, which grants " + + "project creation under the organization without granting ownership over existing projects.", + RunE: groupHelpOrUnknownError, + } + orgCmd.AddCommand(orgMembershipMutationCmd("add [email] [organization]", "Grant organization membership", true)) + orgCmd.AddCommand(orgMembershipMutationCmd("rm [email] [organization]", "Remove organization membership", false)) + arboristCmd.AddCommand(orgCmd) +} + +func ownerMutationCmd(use string, short string, add bool) *cobra.Command { + var resourcePath, username string + cmd := &cobra.Command{ + Use: use, + Short: short, + RunE: func(cmd *cobra.Command, args []string) error { + if resourcePath == "" || username == "" { + return fmt.Errorf("--resource and --user are required") + } + username, err := validEmailArg(username) + if err != nil { + return err + } + return runArborist(cmd, func(ctx context.Context, client arborist.ClientInterface) (any, string, error) { + req := arborist.OwnerMutationRequest{ResourcePath: resourcePath, Username: username} + if add { + var out any + err := client.AddOwner(ctx, req, &out) + return out, fmt.Sprintf("Added owner %s on %s", username, resourcePath), err + } + return nil, fmt.Sprintf("Removed owner %s from %s", username, resourcePath), client.RemoveOwner(ctx, req) + }) + }, + } + cmd.Flags().StringVar(&resourcePath, "resource", "", "Owned resource path") + cmd.Flags().StringVar(&username, "user", "", "Username") + return cmd +} + +func ownershipReadCmd() *cobra.Command { + var resourcePath string + var includeChildren bool + var includeAdmins bool + cmd := &cobra.Command{ + Use: "get-resource", + Short: "Read ownership and direct-access state for a resource", + RunE: func(cmd *cobra.Command, args []string) error { + if resourcePath == "" { + return fmt.Errorf("--resource is required") + } + return runArborist(cmd, func(ctx context.Context, client arborist.ClientInterface) (any, string, error) { + var out any + err := client.GetOwnershipResource(ctx, resourcePath, includeChildren, includeAdmins, &out) + return out, "", err + }) + }, + } + cmd.Flags().StringVar(&resourcePath, "resource", "", "Resource path to inspect") + cmd.Flags().BoolVar(&includeChildren, "include-children", false, "Include descendant resources in the ownership view") + cmd.Flags().BoolVar(&includeAdmins, "include-admins", false, "Include protected admin rows in the ownership view") + return cmd +} + +func accessUserMutationCmd(use string, short string, grant bool) *cobra.Command { + var resourcePath, username, roleID string + cmd := &cobra.Command{ + Use: use, + Short: short, + RunE: func(cmd *cobra.Command, args []string) error { + if resourcePath == "" || username == "" || roleID == "" { + return fmt.Errorf("--resource, --user, and --role are required") + } + username, err := validEmailArg(username) + if err != nil { + return err + } + return runArborist(cmd, func(ctx context.Context, client arborist.ClientInterface) (any, string, error) { + req := arborist.AccessUserRequest{ResourcePath: resourcePath, Username: username, RoleID: roleID} + if grant { + var out any + err := client.GrantAccessUser(ctx, req, &out) + return out, fmt.Sprintf("Granted direct access role %s to %s on %s", roleID, username, resourcePath), err + } + return nil, fmt.Sprintf("Revoked direct access role %s from %s on %s", roleID, username, resourcePath), client.RevokeAccessUser(ctx, req) + }) + }, + } + cmd.Flags().StringVar(&resourcePath, "resource", "", "Resource path") + cmd.Flags().StringVar(&username, "user", "", "Username") + cmd.Flags().StringVar(&roleID, "role", "", "Role ID") + return cmd +} + +func orgMembershipMutationCmd(use string, short string, grant bool) *cobra.Command { + var roleID string + cmd := &cobra.Command{ + Use: use, + Short: short, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + username, err := validEmailArg(args[0]) + if err != nil { + return err + } + organization := strings.TrimSpace(args[1]) + return runArborist(cmd, func(ctx context.Context, client arborist.ClientInterface) (any, string, error) { + role := roleIDOrDefault(roleID) + if grant { + err := client.GrantOrgMembership(ctx, organization, username, roleID) + return nil, fmt.Sprintf("Granted %s role %s on /programs/%s", username, role, organization), err + } + err := client.RevokeOrgMembership(ctx, organization, username, roleID) + return nil, fmt.Sprintf("Removed %s role %s from /programs/%s", username, role, organization), err + }) + }, + } + cmd.Flags().StringVar(&roleID, "role", arborist.DefaultOrgMemberRole, "Arborist role to grant on the organization projects container") + return cmd +} + +func roleIDOrDefault(roleID string) string { + roleID = strings.TrimSpace(roleID) + if roleID == "" { + return arborist.DefaultOrgMemberRole + } + return roleID +} diff --git a/cmd/arborist_test.go b/cmd/arborist_test.go new file mode 100644 index 0000000..1ee5df5 --- /dev/null +++ b/cmd/arborist_test.go @@ -0,0 +1,93 @@ +package cmd + +import ( + "bytes" + "strings" + "testing" + + "github.com/spf13/cobra" +) + +func TestPermissionsCommandRegistersOwnershipAndAccessSubcommands(t *testing.T) { + arboristCmd := findSubcommand(t, RootCmd, "permissions") + ownershipCmd := findSubcommand(t, arboristCmd, "ownership") + accessCmd := findSubcommand(t, arboristCmd, "access") + findSubcommand(t, arboristCmd, "auth") + findSubcommand(t, arboristCmd, "org-membership") + + getResourceCmd := findSubcommand(t, ownershipCmd, "get-resource") + if getResourceCmd.Flags().Lookup("resource") == nil { + t.Fatal("ownership get-resource missing --resource flag") + } + if getResourceCmd.Flags().Lookup("include-children") == nil { + t.Fatal("ownership get-resource missing --include-children flag") + } + if getResourceCmd.Flags().Lookup("include-admins") == nil { + t.Fatal("ownership get-resource missing --include-admins flag") + } + + for _, sub := range []string{"grant-user", "revoke-user"} { + accessUserCmd := findSubcommand(t, accessCmd, sub) + if accessUserCmd.Flags().Lookup("resource") == nil { + t.Fatalf("%s missing --resource flag", sub) + } + if accessUserCmd.Flags().Lookup("user") == nil { + t.Fatalf("%s missing --user flag", sub) + } + if accessUserCmd.Flags().Lookup("role") == nil { + t.Fatalf("%s missing --role flag", sub) + } + } + + for _, removed := range []string{"policy", "resource", "role", "user"} { + if hasSubcommand(arboristCmd, removed) { + t.Fatalf("unexpected legacy subcommand %q still registered", removed) + } + } +} + +func TestPermissionsRejectsUnknownLegacySubcommand(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + RootCmd.SetOut(&stdout) + RootCmd.SetErr(&stderr) + RootCmd.SetArgs([]string{"permissions", "policy", "ownership", "--profile", "dev"}) + + _, err := RootCmd.ExecuteC() + if err == nil { + t.Fatal("expected invalid permissions subcommand to fail") + } + if !strings.Contains(err.Error(), "unknown command") && !strings.Contains(err.Error(), "accepts 0 arg(s)") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestPermissionsLegacyAliasStillWorks(t *testing.T) { + cmd, _, err := RootCmd.Find([]string{"arborist", "auth", "mapping"}) + if err != nil { + t.Fatalf("legacy alias lookup failed: %v", err) + } + if cmd == nil || cmd.CommandPath() != "calypr-cli permissions auth mapping" { + t.Fatalf("expected arborist alias to resolve to permissions auth mapping, got %v", cmd) + } +} + +func findSubcommand(t *testing.T, parent *cobra.Command, use string) *cobra.Command { + t.Helper() + for _, child := range parent.Commands() { + if child.Use == use { + return child + } + } + t.Fatalf("subcommand %q not found", use) + return nil +} + +func hasSubcommand(parent *cobra.Command, use string) bool { + for _, child := range parent.Commands() { + if child.Use == use { + return true + } + } + return false +} diff --git a/cmd/auth.go b/cmd/auth.go new file mode 100644 index 0000000..aaa146b --- /dev/null +++ b/cmd/auth.go @@ -0,0 +1,203 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "sort" + "strings" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + "github.com/spf13/cobra" +) + +const authSummaryResourceLimit = 10 + +func init() { + var profile string + var showAll bool + var jsonOutput bool + var authCmd = &cobra.Command{ + Use: "auth", + Short: "Return resource access privileges from profile", + Long: `Gets resource access privileges for specified profile.`, + Example: `./calypr-cli auth --profile= +./calypr-cli auth --profile= --all +./calypr-cli auth --profile= --json`, + RunE: func(cmd *cobra.Command, args []string) error { + // don't initialize transmission logs for non-uploading related commands + + logger, logCloser := logs.New(profile, logs.WithNoConsole()) + defer logCloser() + + g3i, err := g3client.NewGen3Interface( + profile, logger, + g3client.WithClients(g3client.FenceClient), + ) + if err != nil { + return fmt.Errorf("new Gen3 interface: %w", err) + } + + resourceAccess, err := g3i.FenceClient().CheckPrivileges(context.Background()) + if err != nil { + return fmt.Errorf("authentication: %w", err) + } + + if jsonOutput { + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(resourceAccess) + } + + writeAuthSummary(cmd.OutOrStdout(), g3i.Credentials().Current().APIEndpoint, resourceAccess, showAll) + return nil + }, + } + + authCmd.Flags().StringVar(&profile, "profile", "", "Specify the profile to check your access privileges") + authCmd.Flags().BoolVar(&showAll, "all", false, "Show every resource in each permission group") + authCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output raw resource access JSON") + authCmd.MarkFlagRequired("profile") // nolint: errcheck + RootCmd.AddCommand(authCmd) +} + +func writeAuthSummary(w io.Writer, endpoint string, resourceAccess map[string]any, showAll bool) { + if len(resourceAccess) == 0 { + fmt.Fprintf(w, "No resource access found for %s\n", endpoint) + return + } + + groups := make(map[string][]string) + for resource, permissions := range resourceAccess { + signature := authPermissionSignature(permissions) + groups[signature] = append(groups[signature], resource) + } + + type accessGroup struct { + signature string + resources []string + } + + orderedGroups := make([]accessGroup, 0, len(groups)) + for signature, resources := range groups { + sort.Strings(resources) + orderedGroups = append(orderedGroups, accessGroup{ + signature: signature, + resources: resources, + }) + } + + sort.Slice(orderedGroups, func(i, j int) bool { + if len(orderedGroups[i].resources) != len(orderedGroups[j].resources) { + return len(orderedGroups[i].resources) > len(orderedGroups[j].resources) + } + return orderedGroups[i].signature < orderedGroups[j].signature + }) + + fmt.Fprintf(w, "Access for %s\n", endpoint) + fmt.Fprintf(w, "%d resources in %d permission groups\n\n", len(resourceAccess), len(orderedGroups)) + + for _, group := range orderedGroups { + fmt.Fprintf(w, "%d %s: %s\n", len(group.resources), pluralize("resource", len(group.resources)), group.signature) + + limit := len(group.resources) + if !showAll && limit > authSummaryResourceLimit { + limit = authSummaryResourceLimit + } + + for _, resource := range group.resources[:limit] { + fmt.Fprintf(w, " %s\n", resource) + } + if !showAll && len(group.resources) > limit { + fmt.Fprintf(w, " ... %d more (use --all to show every resource)\n", len(group.resources)-limit) + } + fmt.Fprintln(w) + } +} + +func pluralize(word string, count int) string { + if count == 1 { + return word + } + return word + "s" +} + +func authPermissionSignature(value any) string { + permissions, ok := value.([]any) + if !ok { + return compactJSON(value) + } + if len(permissions) == 0 { + return "no permissions" + } + + if _, ok := permissions[0].(string); ok { + access := make([]string, 0, len(permissions)) + for _, permission := range permissions { + access = append(access, fmt.Sprint(permission)) + } + sort.Strings(access) + return strings.Join(access, ", ") + } + + serviceMethods := make(map[string]map[string]struct{}) + unknown := make([]string, 0) + + for _, permission := range permissions { + method, service, ok := authPermissionFields(permission) + if !ok { + unknown = append(unknown, compactJSON(permission)) + continue + } + + if serviceMethods[service] == nil { + serviceMethods[service] = make(map[string]struct{}) + } + serviceMethods[service][method] = struct{}{} + } + + parts := make([]string, 0, len(serviceMethods)+len(unknown)) + services := make([]string, 0, len(serviceMethods)) + for service := range serviceMethods { + services = append(services, service) + } + sort.Strings(services) + + for _, service := range services { + methods := make([]string, 0, len(serviceMethods[service])) + for method := range serviceMethods[service] { + methods = append(methods, method) + } + sort.Strings(methods) + parts = append(parts, fmt.Sprintf("%s: %s", service, strings.Join(methods, ", "))) + } + + sort.Strings(unknown) + parts = append(parts, unknown...) + return strings.Join(parts, "; ") +} + +func authPermissionFields(value any) (method string, service string, ok bool) { + switch permission := value.(type) { + case map[string]any: + method, methodOK := permission["method"].(string) + service, serviceOK := permission["service"].(string) + return method, service, methodOK && serviceOK + case map[string]string: + method, methodOK := permission["method"] + service, serviceOK := permission["service"] + return method, service, methodOK && serviceOK + default: + return "", "", false + } +} + +func compactJSON(value any) string { + data, err := json.Marshal(value) + if err != nil { + return fmt.Sprint(value) + } + return string(data) +} diff --git a/cmd/auth_test.go b/cmd/auth_test.go new file mode 100644 index 0000000..857f74b --- /dev/null +++ b/cmd/auth_test.go @@ -0,0 +1,81 @@ +package cmd + +import ( + "bytes" + "strings" + "testing" +) + +func TestAuthPermissionSignatureGroupsArboristPermissions(t *testing.T) { + signature := authPermissionSignature([]any{ + map[string]any{"method": "read", "service": "*"}, + map[string]any{"method": "create", "service": "*"}, + map[string]any{"method": "*", "service": "indexd"}, + map[string]any{"method": "read", "service": "requestor"}, + }) + + want := "*: create, read; indexd: *; requestor: read" + if signature != want { + t.Fatalf("signature = %q, want %q", signature, want) + } +} + +func TestAuthPermissionSignatureGroupsFenceProjectAccess(t *testing.T) { + signature := authPermissionSignature([]any{"write", "read", "read"}) + + want := "read, read, write" + if signature != want { + t.Fatalf("signature = %q, want %q", signature, want) + } +} + +func TestWriteAuthSummaryCondensesRepeatedPermissionSets(t *testing.T) { + resourceAccess := map[string]any{ + "/programs/a/projects/one": []any{ + map[string]any{"method": "read", "service": "*"}, + map[string]any{"method": "create", "service": "*"}, + }, + "/programs/a/projects/two": []any{ + map[string]any{"method": "create", "service": "*"}, + map[string]any{"method": "read", "service": "*"}, + }, + "/data_file": []any{ + map[string]any{"method": "*", "service": "indexd"}, + }, + } + + var out bytes.Buffer + writeAuthSummary(&out, "https://example.org", resourceAccess, false) + got := out.String() + + for _, want := range []string{ + "Access for https://example.org", + "3 resources in 2 permission groups", + "2 resources: *: create, read", + " /programs/a/projects/one", + " /programs/a/projects/two", + "1 resource: indexd: *", + " /data_file", + } { + if !strings.Contains(got, want) { + t.Fatalf("summary missing %q:\n%s", want, got) + } + } +} + +func TestWriteAuthSummaryCapsLongGroups(t *testing.T) { + resourceAccess := make(map[string]any) + for i := 0; i < authSummaryResourceLimit+2; i++ { + resourceAccess[string(rune('a'+i))] = []any{ + map[string]any{"method": "read", "service": "*"}, + } + } + + var out bytes.Buffer + writeAuthSummary(&out, "https://example.org", resourceAccess, false) + got := out.String() + + if !strings.Contains(got, "... 2 more (use --all to show every resource)") { + t.Fatalf("summary did not cap long group:\n%s", got) + } +} diff --git a/cmd/calypr-cli/main.go b/cmd/calypr-cli/main.go new file mode 100644 index 0000000..c8d9b14 --- /dev/null +++ b/cmd/calypr-cli/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/calypr/calypr-cli/cmd" + +func main() { + cmd.Execute() +} diff --git a/cmd/collaborator.go b/cmd/collaborator.go new file mode 100644 index 0000000..75dc479 --- /dev/null +++ b/cmd/collaborator.go @@ -0,0 +1,324 @@ +package cmd + +import ( + "fmt" + "os" + + "regexp" + "strings" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + "github.com/calypr/calypr-cli/requestor" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +var collaboratorsCmd = &cobra.Command{ + Use: "collaborators", + Short: "Manage collaborators and access requests", +} + +var emailRegex = regexp.MustCompile(`^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}$`) + +func printRequest(r requestor.Request) { + b, err := yaml.Marshal(r) + if err != nil { + fmt.Printf("ID: %s (Error formatting details: %v)\n", r.RequestID, err) + return + } + fmt.Println(string(b)) +} + +func getRequestorClient(localProfile string) (requestor.RequestorInterface, func()) { + if localProfile == "" { + fmt.Println("Error: profile is required.") + os.Exit(1) + } + + // Initialize logger + logger, logCloser := logs.New(localProfile) + + // Initialize base Gen3 interface and build requestor client from it. + g3i, err := g3client.NewGen3Interface(localProfile, logger) + if err != nil { + fmt.Printf("Error accessing Gen3: %v\n", err) + logCloser() + os.Exit(1) + } + + return requestor.NewRequestorClient(g3i, g3i.Credentials().Current()), logCloser +} + +var collaboratorListCmd = &cobra.Command{ + Use: "ls [profile]", + Short: "List requests", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + mine, _ := cmd.Flags().GetBool("mine") + active, _ := cmd.Flags().GetBool("active") + username, _ := cmd.Flags().GetString("username") + + client, closer := getRequestorClient(p) + defer closer() + + requests, err := client.ListRequests(cmd.Context(), mine, active, username) + if err != nil { + fmt.Printf("Error listing requests: %v\n", err) + os.Exit(1) + } + + for _, r := range requests { + printRequest(r) + } + }, +} + +var collaboratorPendingCmd = &cobra.Command{ + Use: "pending [profile]", + Short: "List pending requests", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + client, closer := getRequestorClient(p) + defer closer() + + // Fetch all requests + requests, err := client.ListRequests(cmd.Context(), false, false, "") + if err != nil { + fmt.Printf("Error listing requests: %v\n", err) + os.Exit(1) + } + + fmt.Println("Pending requests:") + for _, r := range requests { + if r.Status != "SIGNED" { + printRequest(r) + } + } + }, +} + +var collaboratorAddUserCmd = &cobra.Command{ + Use: "add [profile] [email] [program] [project]", + Short: "Add a user to a project", + Args: cobra.ExactArgs(4), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + username := args[1] + program := args[2] + project := args[3] + projectID := fmt.Sprintf("%s-%s", program, project) + + if !emailRegex.MatchString(strings.ToLower(username)) { + fmt.Printf("Error: invalid email address '%s'\n", username) + os.Exit(1) + } + + write, _ := cmd.Flags().GetBool("write") + guppy, _ := cmd.Flags().GetBool("guppy") + approve, _ := cmd.Flags().GetBool("approve") + + client, closer := getRequestorClient(p) + defer closer() + + reqs, err := client.AddUser(cmd.Context(), projectID, username, write, guppy) + if err != nil { + fmt.Printf("Error adding user: %v\n", err) + os.Exit(1) + } + + if approve { + fmt.Println("\nAuto-approving requests...") + for _, r := range reqs { + updatedReq, err := client.UpdateRequest(cmd.Context(), r.RequestID, "SIGNED") + if err != nil { + fmt.Printf("Error approving request %s: %v\n", r.RequestID, err) + } else { + fmt.Printf("Approved request %s:\n", updatedReq.RequestID) + printRequest(*updatedReq) + } + } + } else { + fmt.Println("Created requests:") + for _, r := range reqs { + printRequest(r) + } + fmt.Printf("\nAn authorized user must approve these requests to add %s to %s\n", username, projectID) + } + }, +} + +var collaboratorBulkAddUserCmd = &cobra.Command{ + Use: "bulk-add [profile] [email] [resource_paths...]", + Short: "Add a user to multiple project resources", + Long: "Add a user to multiple project resources from a comma-delimited list. Resource paths may be /programs//projects/ or organization/project.", + Args: cobra.MinimumNArgs(3), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + username := args[1] + resourceList := strings.Join(args[2:], " ") + + if !emailRegex.MatchString(strings.ToLower(username)) { + fmt.Printf("Error: invalid email address '%s'\n", username) + os.Exit(1) + } + + resources, err := requestor.ParseProjectResources(resourceList) + if err != nil { + fmt.Printf("Error parsing resource paths: %v\n", err) + os.Exit(1) + } + + write, _ := cmd.Flags().GetBool("write") + guppy, _ := cmd.Flags().GetBool("guppy") + approve, _ := cmd.Flags().GetBool("approve") + + client, closer := getRequestorClient(p) + defer closer() + + fmt.Printf("Creating collaborator requests for %d project resources...\n", len(resources)) + reqs, err := client.AddUserToResources(cmd.Context(), resources, username, write, guppy) + if err != nil { + fmt.Printf("Error adding user to resources: %v\n", err) + os.Exit(1) + } + + if approve { + fmt.Println("\nAuto-approving requests...") + for _, r := range reqs { + updatedReq, err := client.UpdateRequest(cmd.Context(), r.RequestID, "SIGNED") + if err != nil { + fmt.Printf("Error approving request %s: %v\n", r.RequestID, err) + } else { + fmt.Printf("Approved request %s:\n", updatedReq.RequestID) + printRequest(*updatedReq) + } + } + } else { + fmt.Println("Created requests:") + for _, r := range reqs { + printRequest(r) + } + fmt.Printf("\nAn authorized user must approve these requests to add %s to %d project resources\n", username, len(resources)) + } + }, +} + +var collaboratorRemoveUserCmd = &cobra.Command{ + Use: "rm [profile] [email] [program] [project]", + Short: "Remove a user from a project", + Args: cobra.ExactArgs(4), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + username := args[1] + program := args[2] + project := args[3] + projectID := fmt.Sprintf("%s-%s", program, project) + + if !emailRegex.MatchString(strings.ToLower(username)) { + fmt.Printf("Error: invalid email address '%s'\n", username) + os.Exit(1) + } + + approve, _ := cmd.Flags().GetBool("approve") + + client, closer := getRequestorClient(p) + defer closer() + + reqs, err := client.RemoveUser(cmd.Context(), projectID, username) + if err != nil { + fmt.Printf("Error removing user: %v\n", err) + os.Exit(1) + } + + if approve { + fmt.Println("\nAuto-approving revoke requests...") + for _, r := range reqs { + updatedReq, err := client.UpdateRequest(cmd.Context(), r.RequestID, "SIGNED") + if err != nil { + fmt.Printf("Error approving request %s: %v\n", r.RequestID, err) + } else { + fmt.Printf("Approved request %s:\n", updatedReq.RequestID) + printRequest(*updatedReq) + } + } + } else { + fmt.Println("Created revoke requests:") + for _, r := range reqs { + printRequest(r) + } + } + }, +} + +var collaboratorApproveCmd = &cobra.Command{ + Use: "approve [profile] [request_id]", + Short: "Approve a request (sign it)", + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + requestID := args[1] + + client, closer := getRequestorClient(p) + defer closer() + + req, err := client.UpdateRequest(cmd.Context(), requestID, "SIGNED") + if err != nil { + fmt.Printf("Error approving request: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Approved request %s\n", req.RequestID) + printRequest(*req) + }, +} + +var collaboratorUpdateCmd = &cobra.Command{ + Use: "update [profile] [request_id] [status]", + Short: "Update a request status", + Hidden: true, + Args: cobra.ExactArgs(3), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + requestID := args[1] + status := args[2] + + client, closer := getRequestorClient(p) + defer closer() + + req, err := client.UpdateRequest(cmd.Context(), requestID, status) + if err != nil { + fmt.Printf("Error updating request: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Updated request %s to status %s\n", req.RequestID, req.Status) + }, +} + +func init() { + RootCmd.AddCommand(collaboratorsCmd) + collaboratorsCmd.AddCommand(collaboratorListCmd) + collaboratorsCmd.AddCommand(collaboratorPendingCmd) + collaboratorsCmd.AddCommand(collaboratorAddUserCmd) + collaboratorsCmd.AddCommand(collaboratorBulkAddUserCmd) + collaboratorsCmd.AddCommand(collaboratorRemoveUserCmd) + collaboratorsCmd.AddCommand(collaboratorApproveCmd) + collaboratorsCmd.AddCommand(collaboratorUpdateCmd) + + collaboratorListCmd.Flags().Bool("mine", false, "List my requests") + collaboratorListCmd.Flags().Bool("active", false, "List only active requests") + collaboratorListCmd.Flags().String("username", "", "List requests for user") + + collaboratorAddUserCmd.Flags().BoolP("write", "w", false, "Grant write access") + collaboratorAddUserCmd.Flags().BoolP("guppy", "g", false, "Grant guppy admin access") + collaboratorAddUserCmd.Flags().BoolP("approve", "a", false, "Automatically approve the requests") + + collaboratorBulkAddUserCmd.Flags().BoolP("write", "w", false, "Grant write access") + collaboratorBulkAddUserCmd.Flags().BoolP("guppy", "g", false, "Grant guppy admin access") + collaboratorBulkAddUserCmd.Flags().BoolP("approve", "a", false, "Automatically approve the requests") + + collaboratorRemoveUserCmd.Flags().BoolP("approve", "a", false, "Automatically approve the revoke requests") +} diff --git a/client/g3cmd/configure.go b/cmd/configure.go similarity index 68% rename from client/g3cmd/configure.go rename to cmd/configure.go index d9d97e2..128ad8d 100644 --- a/client/g3cmd/configure.go +++ b/cmd/configure.go @@ -1,14 +1,27 @@ -package g3cmd +package cmd import ( + "context" "fmt" - "github.com/calypr/data-client/client/common" - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/logs" + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" "github.com/spf13/cobra" ) +func mergeImportedCredential(target *conf.Credential, imported *conf.Credential) { + if target == nil || imported == nil { + return + } + target.KeyID = imported.KeyID + target.APIKey = imported.APIKey + if target.APIEndpoint == "" && imported.APIEndpoint != "" { + target.APIEndpoint = imported.APIEndpoint + } + target.AccessToken = "" +} + func init() { var profile string var credFile string @@ -21,10 +34,10 @@ func init() { Short: "Add or modify a configuration profile to your config file", Long: `Configuration file located at ~/.gen3/gen3_client_config.ini If a field is left empty, the existing value (if it exists) will remain unchanged`, - Example: `./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org`, + Example: `./calypr-cli configure --profile= --cred= --apiendpoint=https://data.mycommons.org`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands - cred := &jwt.Credential{ + cred := &conf.Credential{ Profile: profile, APIEndpoint: apiEndpoint, AccessToken: fenceToken, @@ -34,21 +47,17 @@ func init() { logger, logCloser := logs.New(profile, logs.WithConsole()) defer logCloser() - conf := jwt.Configure{Logs: logger} - + configure := conf.NewConfigure(logger.Logger) if credFile != "" { - readCred, err := conf.ReadCredentials(credFile, "") + readCred, err := configure.Import(credFile, "") if err != nil { logger.Fatal(err) // or return proper error } - cred.KeyId = readCred.KeyId - cred.APIKey = readCred.APIKey - if readCred.APIEndpoint != "" { - cred.APIEndpoint = readCred.APIEndpoint - } - cred.AccessToken = "" + mergeImportedCredential(cred, readCred) } - err := jwt.UpdateConfig(logger, cred) + + g3i := g3client.NewGen3InterfaceFromCredential(cred, logger, g3client.WithClients()) + err := g3i.Credentials().Export(context.Background(), cred) if err != nil { logger.Println(err.Error()) } @@ -63,7 +72,7 @@ func init() { configureCmd.Flags().StringVar(&fenceToken, "fenceToken", "", "Specify the fence token to use as a substitute for credential file") configureCmd.Flags().StringVar(&apiEndpoint, "apiendpoint", "", "Specify the API endpoint of the data commons") configureCmd.MarkFlagRequired("apiendpoint") //nolint:errcheck - configureCmd.Flags().StringVar(&useShepherd, "use-shepherd", "", fmt.Sprintf("Enables or disables support for the Shepherd API. If enabled, gen3client will use the Shepherd API if available. (Default: %v)", common.DefaultUseShepherd)) - configureCmd.Flags().StringVar(&minShepherdVersion, "min-shepherd-version", "", fmt.Sprintf("Specify the minimum version of Shepherd that the gen3client will use if Shepherd is enabled. (Default: %v)", common.DefaultMinShepherdVersion)) + configureCmd.Flags().StringVar(&useShepherd, "use-shepherd", "", fmt.Sprintf("Enables or disables support for the Shepherd API. If enabled, gen3client will use the Shepherd API if available. (Default: %v)", false)) + configureCmd.Flags().StringVar(&minShepherdVersion, "min-shepherd-version", "", fmt.Sprintf("Specify the minimum version of Shepherd that the gen3client will use if Shepherd is enabled. (Default: %v)", "2.0.0")) RootCmd.AddCommand(configureCmd) } diff --git a/cmd/configure_test.go b/cmd/configure_test.go new file mode 100644 index 0000000..d101609 --- /dev/null +++ b/cmd/configure_test.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "testing" + + "github.com/calypr/calypr-cli/conf" +) + +func TestMergeImportedCredentialPreservesExplicitAPIEndpoint(t *testing.T) { + target := &conf.Credential{ + Profile: "dev", + APIEndpoint: "https://explicit.example.org", + AccessToken: "stale-token", + } + imported := &conf.Credential{ + KeyID: "key-id", + APIKey: "api-key", + APIEndpoint: "https://embedded.example.org", + } + + mergeImportedCredential(target, imported) + + if target.KeyID != "key-id" { + t.Fatalf("KeyID = %q, want %q", target.KeyID, "key-id") + } + if target.APIKey != "api-key" { + t.Fatalf("APIKey = %q, want %q", target.APIKey, "api-key") + } + if target.APIEndpoint != "https://explicit.example.org" { + t.Fatalf("APIEndpoint = %q, want explicit endpoint", target.APIEndpoint) + } + if target.AccessToken != "" { + t.Fatalf("AccessToken = %q, want empty", target.AccessToken) + } +} + +func TestMergeImportedCredentialUsesImportedAPIEndpointWhenMissing(t *testing.T) { + target := &conf.Credential{Profile: "dev"} + imported := &conf.Credential{APIEndpoint: "https://embedded.example.org"} + + mergeImportedCredential(target, imported) + + if target.APIEndpoint != "https://embedded.example.org" { + t.Fatalf("APIEndpoint = %q, want %q", target.APIEndpoint, "https://embedded.example.org") + } +} diff --git a/cmd/download-multipart.go b/cmd/download-multipart.go new file mode 100644 index 0000000..3718720 --- /dev/null +++ b/cmd/download-multipart.go @@ -0,0 +1,261 @@ +package cmd + +/* +// DownloadSignedURL downloads a file from a signed URL with: +// - Resumable single-stream download (if partial file exists) +// - Concurrent multipart download for large files (>1GB) +// - Retries via go-retryablehttp +// - Progress bar support via mpb +func DownloadSignedURL(signedURL, dstPath string) error { + // Setup retryable client + retryClient := retryablehttp.NewClient() + retryClient.RetryMax = 10 + retryClient.RetryWaitMin = 1 * time.Second + retryClient.RetryWaitMax = 30 * time.Second + retryClient.Logger = nil // silent + client := retryClient.StandardClient() + client.Timeout = 0 // no timeout for large downloads + + // HEAD to get size and Accept-Ranges support + headResp, err := client.Head(signedURL) + if err != nil { + return fmt.Errorf("HEAD request failed: %w", err) + } + defer headResp.Body.Close() + + if headResp.StatusCode != http.StatusOK { + return fmt.Errorf("HEAD failed: %s", headResp.Status) + } + + contentLength := headResp.ContentLength + if contentLength <= 0 { + return fmt.Errorf("invalid Content-Length") + } + + acceptRanges := headResp.Header.Get("Accept-Ranges") == "bytes" + if !acceptRanges { + return fmt.Errorf("server does not support range requests") + } + + // Ensure directory exists + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return fmt.Errorf("mkdir failed: %w", err) + } + + // Check if partial file exists + stat, _ := os.Stat(dstPath) + existingSize := int64(0) + if stat != nil { + existingSize = stat.Size() + } + + // If we have a partial file, resume with single stream (safer and simpler) + if existingSize > 0 && existingSize < contentLength { + return downloadResumableSingle(signedURL, dstPath, contentLength, existingSize, client) + } + + // For complete downloads: use multipart if file is large enough + if contentLength >= 5*1024*1024*1024 { + return downloadConcurrentMultipart(signedURL, dstPath, contentLength, client) + } + + // Otherwise: simple single-stream download + return downloadResumableSingle(signedURL, dstPath, contentLength, 0, client) +} + +// downloadResumableSingle handles single-stream (possibly resumed) download +func downloadResumableSingle(signedURL, dstPath string, totalSize, startByte int64, client *http.Client) error { + req, err := http.NewRequest("GET", signedURL, nil) + if err != nil { + return err + } + if startByte > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", startByte)) + } + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("GET failed: %w", err) + } + defer resp.Body.Close() + + if startByte > 0 && resp.StatusCode != http.StatusPartialContent { + return fmt.Errorf("expected 206 Partial Content, got %d", resp.StatusCode) + } + if startByte == 0 && resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected 200 OK, got %d", resp.StatusCode) + } + + file, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + if startByte > 0 { + if _, err := file.Seek(startByte, io.SeekStart); err != nil { + return err + } + } else { + if err := file.Truncate(0); err != nil { + return err + } + } + + var writer io.Writer = file + if progress != nil { + bar := progress.AddBar(totalSize, + mpb.PrependDecorators( + decor.Name(filepath.Base(dstPath)+" "), + decor.CountersKibiByte("% .1f / % .1f"), + ), + mpb.AppendDecorators( + decor.Percentage(), + decor.AverageSpeed(decor.SizeB1024(0), "% .1f"), + ), + ) + if startByte > 0 { + bar.SetCurrent(startByte) + } + writer = bar.ProxyWriter(file) + } + + _, err = io.Copy(writer, resp.Body) + return err +} + +// downloadConcurrentMultipart downloads in parallel chunks +func downloadConcurrentMultipart(signedURL, dstPath string, totalSize int64, client *http.Client) error { + numChunks := int((totalSize + chunkSize - 1) / chunkSize) + if numChunks < defaultConcurrency { + numChunks = defaultConcurrency + } + chunkSizeActual := (totalSize + int64(numChunks) - 1) / int64(numChunks) + + // Pre-allocate file + file, err := os.Create(dstPath) + if err != nil { + return err + } + if err := file.Truncate(totalSize); err != nil { + file.Close() + return err + } + file.Close() + + var wg sync.WaitGroup + var mu sync.Mutex + var downloadErr error + + // Shared progress bar for total + var totalBar *mpb.Bar + if progress != nil { + totalBar = progress.AddBar(totalSize, + mpb.PrependDecorators( + decor.Name(filepath.Base(dstPath)+" (multipart) "), + decor.CountersKibiByte("% .1f / % .1f"), + ), + mpb.AppendDecorators( + decor.Percentage(), + decor.AverageSpeed(decor.SizeB1024(0), "% .1f"), + ), + ) + } + + concurrency := defaultConcurrency + sem := make(chan struct{}, concurrency) + + for i := 0; i < int(numChunks); i++ { + start := int64(i) * chunkSizeActual + end := start + chunkSizeActual - 1 + if end >= totalSize { + end = totalSize - 1 + } + if start > end { + break + } + + wg.Add(1) + sem <- struct{}{} + + go func(start, end int64, chunkIdx int) { + defer wg.Done() + defer func() { <-sem }() + + req, _ := http.NewRequest("GET", signedURL, nil) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + + resp, err := client.Do(req) + if err != nil { + mu.Lock() + if downloadErr == nil { + downloadErr = fmt.Errorf("chunk %d failed: %w", chunkIdx, err) + } + mu.Unlock() + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusPartialContent { + mu.Lock() + if downloadErr == nil { + downloadErr = fmt.Errorf("chunk %d expected 206, got %d", chunkIdx, resp.StatusCode) + } + mu.Unlock() + return + } + + file, err := os.OpenFile(dstPath, os.O_WRONLY, 0644) + if err != nil { + mu.Lock() + downloadErr = err + mu.Unlock() + return + } + file.Seek(start, io.SeekStart) + writer := io.Writer(file) + + var chunkWriter io.Writer = writer + if progress != nil { + chunkBar := progress.AddBar(end-start+1, + mpb.BarRemoveOnComplete(), + mpb.PrependDecorators(decor.Name(fmt.Sprintf("chunk %d ", chunkIdx))), + ) + chunkWriter = chunkBar.ProxyWriter(writer) + defer file.Close() + } + + if _, err := io.Copy(chunkWriter, resp.Body); err != nil { + mu.Lock() + if downloadErr == nil { + downloadErr = fmt.Errorf("chunk %d copy failed: %w", chunkIdx, err) + } + mu.Unlock() + } else { + if totalBar != nil { + totalBar.IncrBy(int(end - start + 1)) + } + } + if progress == nil { + file.Close() + } + }(start, end, i) + } + + wg.Wait() + + if downloadErr != nil { + if totalBar != nil { + totalBar.Abort(true) + } + return downloadErr + } + + if totalBar != nil { + totalBar.SetCurrent(totalSize) + } + + return nil +} + +*/ diff --git a/cmd/download-multiple.go b/cmd/download-multiple.go new file mode 100644 index 0000000..b1ff3ed --- /dev/null +++ b/cmd/download-multiple.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + sydownload "github.com/calypr/syfon/client/transfer/download" + "github.com/spf13/cobra" +) + +func init() { + var manifestPath string + var downloadPath string + var numParallel int + var profile string + + var downloadMultipleCmd = &cobra.Command{ + Hidden: true, + Use: "download-multiple", + Short: "Download multiple of files from a specified manifest", + Long: `Get presigned URLs for multiple of files specified in a manifest file and then download all of them.`, + Example: `./calypr-cli download-multiple --profile --manifest --download-path `, + Run: func(cmd *cobra.Command, args []string) { + logger, logCloser := logs.New(profile, logs.WithConsole(), logs.WithFailedLog(), logs.WithScoreboard(), logs.WithSucceededLog()) + defer logCloser() + + g3i, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + log.Fatalf("Failed to parse config on profile %s, %v", profile, err) + } + + guids, err := loadManifestGuids(manifestPath) + if err != nil { + log.Fatalf("Failed to read manifest %s: %v", manifestPath, err) + } + + syfon := g3i.SyfonClient() + if syfon == nil { + logger.Fatal("failed to initialize syfon client") + } + err = sydownload.DownloadMultiple(context.Background(), syfon.DRS(), syfon.Data(), guids, downloadPath, numParallel, false) + if err != nil { + logger.Fatal(err.Error()) + } + }, + } + + downloadMultipleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") + downloadMultipleCmd.MarkFlagRequired("profile") //nolint:errcheck + downloadMultipleCmd.Flags().StringVar(&manifestPath, "manifest", "", "The manifest file to read from. A valid manifest can be acquired by using the \"Download Manifest\" button in Data Explorer from a data common's portal") + downloadMultipleCmd.MarkFlagRequired("manifest") //nolint:errcheck + downloadMultipleCmd.Flags().StringVar(&downloadPath, "download-path", ".", "The directory in which to store the downloaded files") + downloadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 1, "Number of downloads to run in parallel") + RootCmd.AddCommand(downloadMultipleCmd) +} + +func loadManifestGuids(path string) ([]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var entries []struct { + GUID string `json:"guid"` + } + if err := json.Unmarshal(data, &entries); err != nil { + return nil, fmt.Errorf("parse manifest: %w", err) + } + + guids := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.GUID != "" { + guids = append(guids, entry.GUID) + } + } + if len(guids) == 0 { + return nil, fmt.Errorf("manifest %s did not contain any GUIDs", path) + } + return guids, nil +} diff --git a/cmd/download-single.go b/cmd/download-single.go new file mode 100644 index 0000000..80003f1 --- /dev/null +++ b/cmd/download-single.go @@ -0,0 +1,72 @@ +package cmd + +import ( + "context" + "fmt" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + sydownload "github.com/calypr/syfon/client/transfer/download" + "github.com/spf13/cobra" +) + +func init() { + var guid string + var manifestPath string + var downloadPath string + var numParallel int + + var downloadCmd = &cobra.Command{ + Use: "download", + Aliases: []string{"download-single"}, + Short: "Download file(s) from object storage.", + Long: `Downloads files through the Syfon-backed download surface. + +Use --guid for a single file or --manifest for multiple files.`, + Example: "./calypr-cli download --profile= --guid=206dfaa6-bcf1-4bc9-b2d0-77179f0f48fc\n" + + "./calypr-cli download --profile= --manifest= --download-path=", + RunE: func(cmd *cobra.Command, args []string) error { + if guid == "" && manifestPath == "" { + return fmt.Errorf("one of --guid or --manifest is required") + } + if guid != "" && manifestPath != "" { + return fmt.Errorf("--guid and --manifest cannot be used together") + } + + logger, logCloser := logs.New(profile, logs.WithConsole(), logs.WithFailedLog(), logs.WithSucceededLog(), logs.WithScoreboard()) + defer logCloser() + + g3I, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + return fmt.Errorf("failed to parse config on profile %s: %w", profile, err) + } + + syfon := g3I.SyfonClient() + if syfon == nil { + return fmt.Errorf("failed to initialize syfon client") + } + + if manifestPath != "" { + guids, err := loadManifestGuids(manifestPath) + if err != nil { + return fmt.Errorf("failed to read manifest %s: %w", manifestPath, err) + } + if err := sydownload.DownloadMultiple(context.Background(), syfon.DRS(), syfon.Data(), guids, downloadPath, numParallel, false); err != nil { + return err + } + return nil + } + + if err := sydownload.DownloadSingleWithProgress(context.Background(), syfon.DRS(), syfon.Data(), guid, downloadPath, "original"); err != nil { + return err + } + return nil + }, + } + + downloadCmd.Flags().StringVar(&guid, "guid", "", "Specify the guid for the data you would like to work with") + downloadCmd.Flags().StringVar(&manifestPath, "manifest", "", "Manifest JSON for downloading multiple GUIDs") + downloadCmd.Flags().StringVar(&downloadPath, "download-path", ".", "The directory in which to store the downloaded files") + downloadCmd.Flags().IntVar(&numParallel, "numparallel", 1, "Number of downloads to run in parallel") + RootCmd.AddCommand(downloadCmd) +} diff --git a/cmd/gecko.go b/cmd/gecko.go new file mode 100644 index 0000000..900062c --- /dev/null +++ b/cmd/gecko.go @@ -0,0 +1,282 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/gecko" + "github.com/calypr/calypr-cli/logs" + "github.com/spf13/cobra" +) + +type geckoCommandOptions struct { + jsonOutput bool +} + +var geckoOpts geckoCommandOptions + +var geckoCmd = &cobra.Command{ + Use: "portal", + Aliases: []string{"gecko"}, + Short: "Manage portal configuration, project metadata, and app cards", + Long: "Portal configuration tools for health checks, typed config inspection, project metadata, and app cards.", + RunE: groupHelpOrUnknownError, +} + +func init() { + geckoCmd.PersistentFlags().BoolVar(&geckoOpts.jsonOutput, "json", false, "Output raw JSON responses") + RootCmd.AddCommand(geckoCmd) + + registerGeckoHealthCommands() + registerGeckoConfigCommands() + registerGeckoProjectCommands() + registerGeckoAppCardCommands() +} + +func getGeckoClient() (gecko.GeckoInterface, func(), error) { + if strings.TrimSpace(profile) == "" { + return nil, nil, fmt.Errorf("profile is required; use --profile") + } + logger, logCloser := logs.New(profile) + g3i, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + logCloser() + return nil, nil, fmt.Errorf("access Gen3: %w", err) + } + return g3i.GeckoClient(), logCloser, nil +} + +func runGecko(cmd *cobra.Command, action func(context.Context, gecko.GeckoInterface) (any, string, error)) error { + client, closer, err := getGeckoClient() + if err != nil { + return err + } + defer closer() + + result, message, err := action(cmd.Context(), client) + if err != nil { + return err + } + if geckoOpts.jsonOutput { + if result == nil { + result = map[string]string{"message": message} + } + return writeGeckoJSON(cmd, result) + } + if strings.TrimSpace(message) != "" { + fmt.Fprintln(cmd.OutOrStdout(), message) + return nil + } + return writeGeckoJSON(cmd, result) +} + +func writeGeckoJSON(cmd *cobra.Command, value any) error { + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} + +func readJSONFile[T any](path string) (*T, error) { + if strings.TrimSpace(path) == "" { + return nil, fmt.Errorf("--file is required") + } + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var out T + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return &out, nil +} + +func parseConfigType(raw string) (gecko.ConfigType, error) { + configType := gecko.ConfigType(strings.TrimSpace(raw)) + for _, known := range gecko.KnownConfigTypes() { + if configType == known { + return configType, nil + } + } + return "", fmt.Errorf("unknown Gecko config type %q", raw) +} + +func registerGeckoHealthCommands() { + geckoCmd.AddCommand(&cobra.Command{ + Use: "health", + Short: "Check portal health", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + health, err := client.HealthCheck(ctx) + if err != nil { + return nil, "", err + } + return map[string]string{"status": health}, "", nil + }) + }, + }) +} + +func registerGeckoConfigCommands() { + configCmd := &cobra.Command{ + Use: "config", + Short: "Inspect portal config types and config IDs", + RunE: groupHelpOrUnknownError, + } + configCmd.AddCommand(&cobra.Command{ + Use: "types", + Short: "List known Gecko config types", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + types, err := client.ListConfigTypes(ctx) + if err != nil { + return nil, "", err + } + return types, "", nil + }) + }, + }) + configCmd.AddCommand(&cobra.Command{ + Use: "list [type]", + Short: "List config IDs for a portal config type", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + configType, err := parseConfigType(args[0]) + if err != nil { + return err + } + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + configs, err := client.ListConfigs(ctx, configType) + if err != nil { + return nil, "", err + } + return configs, "", nil + }) + }, + }) + configCmd.AddCommand(&cobra.Command{ + Use: "get [type] [id]", + Short: "Get portal config by type and ID", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + configType, err := parseConfigType(args[0]) + if err != nil { + return err + } + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + var out any + err := client.GetConfig(ctx, configType, args[1], &out) + return out, "", err + }) + }, + }) + geckoCmd.AddCommand(configCmd) +} + +func registerGeckoProjectCommands() { + projectCmd := &cobra.Command{ + Use: "project", + Short: "Manage portal project metadata", + RunE: groupHelpOrUnknownError, + } + projectCmd.AddCommand(&cobra.Command{ + Use: "get [id]", + Short: "Get portal project metadata", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + cfg, err := gecko.GetProjectConfig(ctx, client, args[0]) + return cfg, "", err + }) + }, + }) + + var projectFile string + putProjectCmd := &cobra.Command{ + Use: "put [id]", + Short: "Create or replace portal project metadata from JSON", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + cfg, err := readJSONFile[gecko.ProjectConfig](projectFile) + if err != nil { + return nil, "", err + } + status, err := gecko.PutProjectConfig(ctx, client, args[0], *cfg) + return status, fmt.Sprintf("Upserted portal project metadata %s", args[0]), err + }) + }, + } + putProjectCmd.Flags().StringVar(&projectFile, "file", "", "JSON file containing Gecko project config") + projectCmd.AddCommand(putProjectCmd) + + projectCmd.AddCommand(&cobra.Command{ + Use: "delete [id]", + Short: "Delete portal project metadata", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + status, err := gecko.DeleteProjectConfig(ctx, client, args[0]) + return status, fmt.Sprintf("Deleted portal project metadata %s", args[0]), err + }) + }, + }) + geckoCmd.AddCommand(projectCmd) +} + +func registerGeckoAppCardCommands() { + appCardCmd := &cobra.Command{ + Use: "appcard", + Short: "Manage portal app cards", + RunE: groupHelpOrUnknownError, + } + appCardCmd.AddCommand(&cobra.Command{ + Use: "get [project-id]", + Short: "Get a portal app card", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + card, err := gecko.GetAppCard(ctx, client, args[0]) + return card, "", err + }) + }, + }) + + var appCardFile string + putAppCardCmd := &cobra.Command{ + Use: "put [project-id]", + Short: "Create or replace a portal app card from JSON", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + card, err := readJSONFile[gecko.AppCard](appCardFile) + if err != nil { + return nil, "", err + } + status, err := gecko.UpsertAppCard(ctx, client, args[0], *card) + return status, fmt.Sprintf("Upserted portal app card %s", args[0]), err + }) + }, + } + putAppCardCmd.Flags().StringVar(&appCardFile, "file", "", "JSON file containing Gecko app card") + appCardCmd.AddCommand(putAppCardCmd) + + appCardCmd.AddCommand(&cobra.Command{ + Use: "delete [project-id]", + Short: "Delete a portal app card", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runGecko(cmd, func(ctx context.Context, client gecko.GeckoInterface) (any, string, error) { + status, err := gecko.DeleteAppCard(ctx, client, args[0]) + return status, fmt.Sprintf("Deleted portal app card %s", args[0]), err + }) + }, + }) + geckoCmd.AddCommand(appCardCmd) +} diff --git a/cmd/gecko_test.go b/cmd/gecko_test.go new file mode 100644 index 0000000..d416498 --- /dev/null +++ b/cmd/gecko_test.go @@ -0,0 +1,55 @@ +package cmd + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" +) + +func TestPortalCommandRegistersExpectedSubcommands(t *testing.T) { + geckoCmd := findSubcommand(t, RootCmd, "portal") + findSubcommand(t, geckoCmd, "health") + + configCmd := findSubcommand(t, geckoCmd, "config") + findSubcommand(t, configCmd, "types") + findSubcommandPrefix(t, configCmd, "list") + findSubcommandPrefix(t, configCmd, "get") + + projectCmd := findSubcommand(t, geckoCmd, "project") + findSubcommandPrefix(t, projectCmd, "get") + putProjectCmd := findSubcommandPrefix(t, projectCmd, "put") + findSubcommandPrefix(t, projectCmd, "delete") + if putProjectCmd.Flags().Lookup("file") == nil { + t.Fatal("gecko project put missing --file flag") + } + + appCardCmd := findSubcommand(t, geckoCmd, "appcard") + findSubcommandPrefix(t, appCardCmd, "get") + putAppCardCmd := findSubcommandPrefix(t, appCardCmd, "put") + findSubcommandPrefix(t, appCardCmd, "delete") + if putAppCardCmd.Flags().Lookup("file") == nil { + t.Fatal("gecko appcard put missing --file flag") + } +} + +func TestPortalLegacyAliasStillWorks(t *testing.T) { + cmd, _, err := RootCmd.Find([]string{"gecko", "health"}) + if err != nil { + t.Fatalf("legacy alias lookup failed: %v", err) + } + if cmd == nil || cmd.CommandPath() != "calypr-cli portal health" { + t.Fatalf("expected gecko alias to resolve to portal health, got %v", cmd) + } +} + +func findSubcommandPrefix(t *testing.T, parent interface{ Commands() []*cobra.Command }, prefix string) *cobra.Command { + t.Helper() + for _, child := range parent.Commands() { + if strings.HasPrefix(child.Use, prefix+" ") || child.Use == prefix { + return child + } + } + t.Fatalf("subcommand prefix %q not found", prefix) + return nil +} diff --git a/cmd/gitversion.go b/cmd/gitversion.go new file mode 100644 index 0000000..ce96e41 --- /dev/null +++ b/cmd/gitversion.go @@ -0,0 +1,6 @@ +package cmd + +var ( + gitcommit = "N/A" + gitversion = "2026.2" +) diff --git a/cmd/retry-upload.go b/cmd/retry-upload.go new file mode 100644 index 0000000..1382250 --- /dev/null +++ b/cmd/retry-upload.go @@ -0,0 +1,59 @@ +package cmd + +import ( + "context" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + syclient "github.com/calypr/syfon/client" + sycommon "github.com/calypr/syfon/client/common" + + "github.com/spf13/cobra" +) + +func init() { + var failedLogPath, profile string + + var retryUploadCmd = &cobra.Command{ + Hidden: true, + Use: "retry-upload", + Short: "Retry failed uploads from a failed_log.json", + Long: `Re-uploads files listed in a failed log using exponential backoff and progress bars.`, + Example: `./calypr-cli retry-upload --profile=myprofile --failed-log-path=/path/to/failed_log.json`, + Run: func(cmd *cobra.Command, args []string) { + Logger, closer := logs.New(profile, + logs.WithConsole(), + logs.WithMessageFile(), + logs.WithFailedLog(), + logs.WithSucceededLog(), + ) + defer closer() + + g3, err := g3client.NewGen3Interface(profile, Logger) + if err != nil { + Logger.Fatalf("Failed to initialize client: %v", err) + } + if _, err := sycommon.LoadFailedLog(failedLogPath); err != nil { + Logger.Fatalf("Cannot read failed log: %v", err) + } + syfon := g3.SyfonClient() + if syfon == nil { + Logger.Fatal("failed to initialize syfon client") + } + err = syclient.Upload(context.Background(), syfon.Data(), "", syclient.UploadOptions{ + RetryFailedLogPath: failedLogPath, + }) + if err != nil { + Logger.Fatalf("Retry upload failed: %v", err) + } + }, + } + + retryUploadCmd.Flags().StringVar(&profile, "profile", "", "Profile to use") + retryUploadCmd.MarkFlagRequired("profile") + + retryUploadCmd.Flags().StringVar(&failedLogPath, "failed-log-path", "", "Path to failed_log.json") + retryUploadCmd.MarkFlagRequired("failed-log-path") + + RootCmd.AddCommand(retryUploadCmd) +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..355d88d --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +var profile string +var backendType string + +// RootCmd represents the base command when called without any subcommands +var RootCmd = &cobra.Command{ + Use: "calypr-cli", + Short: "Calypr CLI for data transfer, permissions, collaboration, and portal operations", + Long: "Calypr CLI for data transfer, permissions, collaboration, and portal operations.\ncalypr-cli version: " + gitversion + ", commit: " + gitcommit, + Version: gitversion, + SilenceErrors: true, +} + +// Execute adds all child commands to the root command sets flags appropriately +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + if err := RootCmd.Execute(); err != nil { + os.Stderr.WriteString("Error: " + err.Error() + "\n") + os.Exit(1) + } +} + +func init() { + RootCmd.PersistentFlags().StringVar(&profile, "profile", "", "Specify profile to use") + RootCmd.PersistentFlags().StringVar(&backendType, "backend", "gen3", "Specify backend to use (gen3 or drs)") + _ = RootCmd.MarkFlagRequired("profile") +} diff --git a/cmd/transfer_commands_test.go b/cmd/transfer_commands_test.go new file mode 100644 index 0000000..4883616 --- /dev/null +++ b/cmd/transfer_commands_test.go @@ -0,0 +1,37 @@ +package cmd + +import "testing" + +func TestTransferCommandsExposeCollapsedTopLevelSurface(t *testing.T) { + uploadCmd := findSubcommand(t, RootCmd, "upload") + if uploadCmd.Flags().Lookup("upload-path") == nil { + t.Fatal("upload missing --upload-path flag") + } + for _, flagName := range []string{"file", "file-path", "guid", "manifest", "failed-log-path", "multipart"} { + if uploadCmd.Flags().Lookup(flagName) == nil { + t.Fatalf("upload missing --%s flag", flagName) + } + } + + downloadCmd := findSubcommand(t, RootCmd, "download") + for _, flagName := range []string{"guid", "manifest", "download-path", "numparallel"} { + if downloadCmd.Flags().Lookup(flagName) == nil { + t.Fatalf("download missing --%s flag", flagName) + } + } +} + +func TestLegacyTransferCommandsAreHidden(t *testing.T) { + for _, use := range []string{ + "upload-single", + "upload-multiple", + "upload-multipart", + "retry-upload", + "download-multiple", + } { + cmd := findSubcommand(t, RootCmd, use) + if !cmd.Hidden { + t.Fatalf("legacy command %q should be hidden", use) + } + } +} diff --git a/cmd/upload-multipart.go b/cmd/upload-multipart.go new file mode 100644 index 0000000..b29f055 --- /dev/null +++ b/cmd/upload-multipart.go @@ -0,0 +1,70 @@ +package cmd + +import ( + "context" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + syclient "github.com/calypr/syfon/client" + "github.com/spf13/cobra" +) + +func init() { + var ( + profile string + filePath string + guid string + bucketName string + ) + + var uploadMultipartCmd = &cobra.Command{ + Hidden: true, + Use: "upload-multipart", + Short: "Upload a single file using managed multipart upload", + Long: `Uploads a file to object storage using managed multipart upload +(init -> presigned part URLs -> complete).`, + Example: `./calypr-cli upload-multipart --profile=myprofile --file-path=./large.bam +./calypr-cli upload-multipart --profile=myprofile --file-path=./data.bam --guid=existing-guid`, + Run: func(cmd *cobra.Command, args []string) { + // Initialize logger + logger, logCloser := logs.New(profile, logs.WithConsole()) + defer logCloser() + + logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard()) + defer closer() + + g3, err := g3client.NewGen3Interface( + profile, + logger, + ) + + if err != nil { + logger.Fatalf("failed to initialize Gen3 interface: %v", err) + } + + syfon := g3.SyfonClient() + if syfon == nil { + logger.Fatal("failed to initialize syfon client") + } + err = syclient.Upload(context.Background(), syfon.Data(), filePath, syclient.UploadOptions{ + Bucket: bucketName, + GUID: guid, + ShowProgress: true, + ForceMultipart: true, + }) + if err != nil { + logger.Fatal(err) + } + }, + } + + uploadMultipartCmd.Flags().StringVar(&profile, "profile", "", "Specify the profile to use for upload") + uploadMultipartCmd.Flags().StringVar(&filePath, "file-path", "", "Path to the file to upload") + uploadMultipartCmd.Flags().StringVar(&guid, "guid", "", "Optional existing GUID (otherwise generated)") + uploadMultipartCmd.Flags().StringVar(&bucketName, "bucket", "", "Target bucket (defaults to configured DATA_UPLOAD_BUCKET)") + + _ = uploadMultipartCmd.MarkFlagRequired("profile") + _ = uploadMultipartCmd.MarkFlagRequired("file-path") + + RootCmd.AddCommand(uploadMultipartCmd) +} diff --git a/cmd/upload-multiple.go b/cmd/upload-multiple.go new file mode 100644 index 0000000..9d77a49 --- /dev/null +++ b/cmd/upload-multiple.go @@ -0,0 +1,82 @@ +package cmd + +// Deprecated: Use "upload" instead for new uploads (without pre-existing GUIDs). +import ( + "context" + "fmt" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + syclient "github.com/calypr/syfon/client" + "github.com/spf13/cobra" +) + +func init() { + var bucketName string + var manifestPath string + var uploadPath string + var batch bool + var numParallel int + var includeSubDirName bool + + uploadMultipleCmd := &cobra.Command{ + Hidden: true, + Use: "upload-multiple", + Short: "Upload multiple files from a specified manifest (uses pre-existing GUIDs)", + Long: `Get presigned URLs for multiple files specified in a manifest file and then upload all of them. +This command is for uploading to existing GUIDs (e.g., from a downloaded manifest). +For new uploads (new GUIDs generated), use "calypr-cli upload" instead. + +Options to run multipart uploads for large files and parallel batch uploading are available.`, + Example: `./calypr-cli upload-multiple --profile= --manifest= --upload-path= --bucket= --batch`, + Run: func(cmd *cobra.Command, args []string) { + // Warning message + fmt.Printf("Notice: this command uploads to pre-existing GUIDs from a manifest.\nIf you want to upload new files (new GUIDs generated automatically), use \"./calypr-cli upload\" instead.\n\n") + + ctx := context.Background() + logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard()) + defer closer() + + g3i, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + logger.Fatalf("Failed to parse config on profile %s: %v", profile, err) + } + + syfon := g3i.SyfonClient() + if syfon == nil { + logger.Fatal("failed to initialize syfon client") + } + err = syclient.Upload(ctx, syfon.Data(), uploadPath, syclient.UploadOptions{ + Bucket: bucketName, + IncludeSubDirName: includeSubDirName, + Batch: batch, + NumParallel: numParallel, + ManifestPath: manifestPath, + ShowProgress: true, + }) + if err != nil { + logger.Println("Upload failed:", err) + } + logger.Scoreboard().PrintSB() + }, + } + + // Flags + uploadMultipleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") + uploadMultipleCmd.MarkFlagRequired("profile") + + uploadMultipleCmd.Flags().StringVar(&manifestPath, "manifest", "", "Path to the manifest JSON file") + uploadMultipleCmd.MarkFlagRequired("manifest") + + uploadMultipleCmd.Flags().StringVar(&uploadPath, "upload-path", "", "Directory containing the files to upload") + uploadMultipleCmd.MarkFlagRequired("upload-path") + + uploadMultipleCmd.Flags().BoolVar(&batch, "batch", true, "Upload single-part files in parallel") + uploadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 4, "Number of parallel uploads") + + uploadMultipleCmd.Flags().StringVar(&bucketName, "bucket", "", "Target bucket (defaults to configured DATA_UPLOAD_BUCKET)") + + uploadMultipleCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", true, "Include subdirectory names in object key") + + RootCmd.AddCommand(uploadMultipleCmd) +} diff --git a/cmd/upload-single.go b/cmd/upload-single.go new file mode 100644 index 0000000..0c561f2 --- /dev/null +++ b/cmd/upload-single.go @@ -0,0 +1,55 @@ +package cmd + +// Deprecated: Use upload instead. +import ( + "context" + "log" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + syclient "github.com/calypr/syfon/client" + "github.com/spf13/cobra" +) + +func init() { + var guid string + var filePath string + var bucketName string + + var uploadSingleCmd = &cobra.Command{ + Hidden: true, + Use: "upload-single", + Short: "Upload a single file to a GUID", + Long: `Gets a presigned URL for which to upload a file associated with a GUID and then uploads the specified file.`, + Example: `./calypr-cli upload-single --profile= --guid=f6923cf3-xxxx-xxxx-xxxx-14ab3f84f9d6 --file=`, + Run: func(cmd *cobra.Command, args []string) { + logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard(), logs.WithConsole()) + defer closer() + + g3i, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + log.Fatalf("Failed to parse config on profile %s: %v", profile, err) + } + syfon := g3i.SyfonClient() + if syfon == nil { + log.Fatal("failed to initialize syfon client") + } + err = syclient.Upload(context.Background(), syfon.Data(), filePath, syclient.UploadOptions{ + Bucket: bucketName, + GUID: guid, + ShowProgress: true, + }) + if err != nil { + log.Fatalln(err.Error()) + } + }, + } + uploadSingleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") + uploadSingleCmd.MarkFlagRequired("profile") //nolint:errcheck + uploadSingleCmd.Flags().StringVar(&guid, "guid", "", "Specify the guid for the data you would like to work with") + uploadSingleCmd.MarkFlagRequired("guid") //nolint:errcheck + uploadSingleCmd.Flags().StringVar(&filePath, "file", "", "Specify file to upload to with --file=~/path/to/file") + uploadSingleCmd.MarkFlagRequired("file") //nolint:errcheck + uploadSingleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") + RootCmd.AddCommand(uploadSingleCmd) +} diff --git a/cmd/upload.go b/cmd/upload.go new file mode 100644 index 0000000..81451fc --- /dev/null +++ b/cmd/upload.go @@ -0,0 +1,142 @@ +package cmd + +import ( + "context" + "fmt" + "log" + + "github.com/calypr/calypr-cli/g3client" + "github.com/calypr/calypr-cli/logs" + syclient "github.com/calypr/syfon/client" + sycommon "github.com/calypr/syfon/client/common" + "github.com/spf13/cobra" +) + +func init() { + var bucketName string + var includeSubDirName bool + var uploadPath string + var filePath string + var multipartFilePath string + var batch bool + var numParallel int + var hasMetadata bool + var guid string + var manifestPath string + var failedLogPath string + var multipart bool + var uploadCmd = &cobra.Command{ + Use: "upload", + Short: "Upload file(s) to object storage.", + Long: `Uploads files through the Syfon-backed upload surface. + +This command also covers existing-GUID uploads, manifest uploads, forced +multipart uploads, and failed-log retries so that the CLI only needs one +top-level upload entrypoint.`, + Example: "./calypr-cli upload --profile= --upload-path=\n" + + "./calypr-cli upload --profile= --upload-path=\n" + + "./calypr-cli upload --profile= --guid= --file=\n" + + "./calypr-cli upload --profile= --manifest= --upload-path=\n" + + "./calypr-cli upload --profile= --file-path= --multipart\n" + + "./calypr-cli upload --profile= --failed-log-path=/path/to/failed_log.json", + Run: func(cmd *cobra.Command, args []string) { + resolvedUploadPath, err := resolveUploadInputPath(uploadPath, filePath, multipartFilePath) + if err != nil { + log.Fatalf("Invalid upload flags: %v", err) + } + + ctx := context.Background() + rootLogger, logCloser := logs.New(profile, logs.WithSucceededLog(), logs.WithScoreboard(), logs.WithFailedLog()) + defer logCloser() + // Instantiate interface to Gen3 + g3i, err := g3client.NewGen3Interface(profile, rootLogger) + if err != nil { + log.Fatalf("Failed to parse config on profile %s, %v", profile, err) + } + logger := g3i.Logger() + syfon := g3i.SyfonClient() + if syfon == nil { + logger.Fatalf("failed to initialize syfon client") + } + + uploadOptions := syclient.UploadOptions{ + Bucket: bucketName, + IncludeSubDirName: includeSubDirName, + HasMetadata: hasMetadata, + Batch: batch, + NumParallel: numParallel, + GUID: guid, + ManifestPath: manifestPath, + ShowProgress: guid != "" || manifestPath != "" || multipart, + ForceMultipart: multipart, + } + + switch { + case failedLogPath != "": + if resolvedUploadPath != "" || manifestPath != "" || guid != "" { + logger.Fatalf("--failed-log-path cannot be combined with file, guid, or manifest upload flags") + } + if _, err := sycommon.LoadFailedLog(failedLogPath); err != nil { + logger.Fatalf("Cannot read failed log: %v", err) + } + err = syclient.Upload(ctx, syfon.Data(), "", syclient.UploadOptions{ + RetryFailedLogPath: failedLogPath, + }) + default: + if resolvedUploadPath == "" { + logger.Fatalf("one of --upload-path, --file, --file-path, or --failed-log-path is required") + } + if manifestPath != "" && guid != "" { + logger.Fatalf("--manifest cannot be combined with --guid") + } + if hasMetadata { + hasShepherd, err := g3i.FenceClient().CheckForShepherdAPI(ctx) + if err != nil { + logger.Printf("WARNING: Error when checking for Shepherd API: %v", err) + } else if !hasShepherd { + logger.Fatalf("ERROR: Metadata upload (`--metadata`) is not supported in the environment you are uploading to. Double check that you are uploading to the right profile.") + } + } + err = syclient.Upload(ctx, syfon.Data(), resolvedUploadPath, uploadOptions) + } + if err != nil { + logger.Error("Upload failed", "error", err) + return + } + g3i.Logger().Scoreboard().PrintSB() + }, + } + + uploadCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") + uploadCmd.MarkFlagRequired("profile") //nolint:errcheck + uploadCmd.Flags().StringVar(&uploadPath, "upload-path", "", "The directory or file in which contains file(s) to be uploaded") + uploadCmd.Flags().StringVar(&filePath, "file", "", "Deprecated alias for --upload-path when uploading a single file") + uploadCmd.Flags().StringVar(&multipartFilePath, "file-path", "", "Deprecated alias for --upload-path, used by legacy multipart uploads") + uploadCmd.Flags().StringVar(&guid, "guid", "", "Upload to an existing GUID instead of creating a new one") + uploadCmd.Flags().StringVar(&manifestPath, "manifest", "", "Manifest JSON for uploads to pre-existing GUIDs") + uploadCmd.Flags().StringVar(&failedLogPath, "failed-log-path", "", "Retry uploads listed in a failed_log.json file") + uploadCmd.Flags().BoolVar(&multipart, "multipart", false, "Force managed multipart upload") + uploadCmd.Flags().BoolVar(&batch, "batch", false, "Upload in parallel") + uploadCmd.Flags().IntVar(&numParallel, "numparallel", 3, "Number of uploads to run in parallel") + uploadCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", true, "Include subdirectory names in file name") + uploadCmd.Flags().BoolVar(&hasMetadata, "metadata", false, "Search for and upload file metadata alongside the file") + uploadCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") + RootCmd.AddCommand(uploadCmd) +} + +func resolveUploadInputPath(primaryPath, filePath, multipartPath string) (string, error) { + var resolved string + for _, candidate := range []string{primaryPath, filePath, multipartPath} { + if candidate == "" { + continue + } + if resolved == "" { + resolved = candidate + continue + } + if resolved != candidate { + return "", fmt.Errorf("conflicting upload path flags: %q and %q", resolved, candidate) + } + } + return resolved, nil +} diff --git a/conf/config.go b/conf/config.go new file mode 100644 index 0000000..d123fe2 --- /dev/null +++ b/conf/config.go @@ -0,0 +1,257 @@ +package conf + +//go:generate go run go.uber.org/mock/mockgen@v0.6.0 -destination=../mocks/mock_configure.go -package=mocks github.com/calypr/calypr-cli/conf ManagerInterface + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path" + "strings" + + sycommon "github.com/calypr/syfon/client/common" + syconf "github.com/calypr/syfon/client/config" + "gopkg.in/ini.v1" +) + +var ErrProfileNotFound = errors.New("profile not found in config file") + +type Credential = syconf.Credential + +type Manager struct { + Logger *slog.Logger +} + +func NewConfigure(logs *slog.Logger) ManagerInterface { + return &Manager{ + Logger: logs, + } +} + +type ManagerInterface interface { + // Loads credential from ~/.gen3/ credential file + Import(filePath, fenceToken string) (*Credential, error) + + // Loads credential from ~/.gen3/config.ini + Load(profile string) (*Credential, error) + Save(cred *Credential) error + + EnsureExists() error + IsCredentialValid(*Credential) (bool, error) + IsTokenValid(string) (bool, error) +} + +func (man *Manager) configPath() (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", err + } + configPath := path.Join( + homeDir + + sycommon.PathSeparator + + ".gen3" + + sycommon.PathSeparator + + "gen3_client_config.ini", + ) + return configPath, nil +} + +func (man *Manager) Load(profile string) (*Credential, error) { + /* + Looking profile in config file. The config file is a text file located at ~/.gen3 directory. It can + contain more than 1 profile. If there is no profile found, the user is asked to run a command to + create the profile + + The format of config file is described as following + + [profile1] + key_id=key_id_example_1 + api_key=api_key_example_1 + access_token=access_token_example_1 + api_endpoint=http://localhost:8000 + use_shepherd=true + min_shepherd_version=2.0.0 + + [profile2] + key_id=key_id_example_2 + api_key=api_key_example_2 + access_token=access_token_example_2 + api_endpoint=http://localhost:8000 + use_shepherd=false + min_shepherd_version= + + Args: + profile: the specific profile in config file + Returns: + An instance of Credential + */ + + homeDir, err := os.UserHomeDir() + if err != nil { + errs := fmt.Errorf("Error occurred when getting home directory: %s", err.Error()) + man.Logger.Error(errs.Error()) + return nil, errs + } + configPath := path.Join(homeDir + sycommon.PathSeparator + ".gen3" + sycommon.PathSeparator + "gen3_client_config.ini") + + if _, err := os.Stat(configPath); os.IsNotExist(err) { + return nil, fmt.Errorf("%w Run configure command (with a profile if desired) to set up account credentials \n"+ + "Example: ./calypr-cli configure --profile= --cred= --apiendpoint=https://data.mycommons.org", ErrProfileNotFound) + } + + // If profile not in config file, prompt user to set up config first + cfg, err := ini.Load(configPath) + if err != nil { + errs := fmt.Errorf("Error occurred when reading config file: %s", err.Error()) + return nil, errs + } + sec, err := cfg.GetSection(profile) + if err != nil { + return nil, fmt.Errorf("%w: Need to run \"calypr-cli configure --profile="+profile+" --cred= --apiendpoint=\" first", ErrProfileNotFound) + } + + profileConfig := &Credential{ + Profile: profile, + KeyID: sec.Key("key_id").String(), + APIKey: sec.Key("api_key").String(), + AccessToken: sec.Key("access_token").String(), + APIEndpoint: sec.Key("api_endpoint").String(), + UseShepherd: sec.Key("use_shepherd").String(), + MinShepherdVersion: sec.Key("min_shepherd_version").String(), + Bucket: sec.Key("bucket").String(), + ProjectID: sec.Key("project_id").String(), + } + + if profileConfig.KeyID == "" && profileConfig.APIKey == "" && profileConfig.AccessToken == "" { + errs := fmt.Errorf("key_id, api_key and access_token not found in profile.") + return nil, errs + } + if profileConfig.APIEndpoint == "" { + errs := fmt.Errorf("api_endpoint not found in profile.") + return nil, errs + } + + return profileConfig, nil +} + +func (man *Manager) Save(profileConfig *Credential) error { + /* + Overwrite the config file with new credential + + Args: + profileConfig: Credential object represents config of a profile + configPath: file path to config file + */ + configPath, err := man.configPath() + if err != nil { + errs := fmt.Errorf("error occurred when getting config path: %s", err.Error()) + man.Logger.Error(errs.Error()) + return errs + } + cfg, err := ini.Load(configPath) + if err != nil { + errs := fmt.Errorf("error occurred when loading config file: %s", err.Error()) + man.Logger.Error(errs.Error()) + return errs + } + + section := cfg.Section(profileConfig.Profile) + if profileConfig.KeyID != "" { + section.Key("key_id").SetValue(profileConfig.KeyID) + } + if profileConfig.APIKey != "" { + section.Key("api_key").SetValue(profileConfig.APIKey) + } + if profileConfig.AccessToken != "" { + section.Key("access_token").SetValue(profileConfig.AccessToken) + } + if profileConfig.APIEndpoint != "" { + section.Key("api_endpoint").SetValue(profileConfig.APIEndpoint) + } + + section.Key("use_shepherd").SetValue(profileConfig.UseShepherd) + section.Key("min_shepherd_version").SetValue(profileConfig.MinShepherdVersion) + section.Key("bucket").SetValue(profileConfig.Bucket) + section.Key("project_id").SetValue(profileConfig.ProjectID) + err = cfg.SaveTo(configPath) + if err != nil { + errs := fmt.Errorf("error occurred when saving config file: %s", err.Error()) + man.Logger.Error(errs.Error()) + return fmt.Errorf("error occurred when saving config file: %s", err.Error()) + } + return nil +} + +func (man *Manager) EnsureExists() error { + /* + Make sure the config exists on start up + */ + configPath, err := man.configPath() + if err != nil { + return err + } + + if _, err := os.Stat(path.Dir(configPath)); os.IsNotExist(err) { + osErr := os.Mkdir(path.Join(path.Dir(configPath)), os.FileMode(0777)) + if osErr != nil { + return err + } + _, osErr = os.Create(configPath) + if osErr != nil { + return err + } + } + if _, err := os.Stat(configPath); os.IsNotExist(err) { + _, osErr := os.Create(configPath) + if osErr != nil { + return err + } + } + _, err = ini.Load(configPath) + + return err +} + +func (man *Manager) Import(filePath, fenceToken string) (*Credential, error) { + var cred Credential + + if filePath != "" { + fullPath, err := sycommon.GetAbsolutePath(filePath) + if err != nil { + man.Logger.Error("error parsing credential file path: " + err.Error()) + return nil, err + } + + content, err := os.ReadFile(fullPath) + if err != nil { + if os.IsNotExist(err) { + man.Logger.Error("File not found: " + fullPath) + } else { + man.Logger.Error("error reading file: " + err.Error()) + } + return nil, err + } + + jsonStr := strings.ReplaceAll(string(content), "\n", "") + // Normalize keys from snake_case to CamelCase for unmarshaling + jsonStr = strings.ReplaceAll(jsonStr, "key_id", "KeyID") + jsonStr = strings.ReplaceAll(jsonStr, "api_key", "APIKey") + jsonStr = strings.ReplaceAll(jsonStr, "access_token", "AccessToken") + jsonStr = strings.ReplaceAll(jsonStr, "api_endpoint", "APIEndpoint") + jsonStr = strings.ReplaceAll(jsonStr, "project_id", "ProjectID") + + if err := json.Unmarshal([]byte(jsonStr), &cred); err != nil { + errMsg := fmt.Errorf("cannot parse JSON credential file: %w", err) + man.Logger.Error(errMsg.Error()) + return nil, errMsg + } + } else if fenceToken != "" { + cred.AccessToken = fenceToken + } else { + return nil, errors.New("either credential file or fence token must be provided") + } + + return &cred, nil +} diff --git a/conf/config_test.go b/conf/config_test.go new file mode 100644 index 0000000..24b7ea2 --- /dev/null +++ b/conf/config_test.go @@ -0,0 +1,261 @@ +package conf + +import ( + "log/slog" + "os" + "path" + "testing" +) + +func TestNewConfigure(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := NewConfigure(logger) + + if manager == nil { + t.Fatal("Expected non-nil manager") + } + + // Type assertion to verify it's a *Manager + if _, ok := manager.(*Manager); !ok { + t.Error("Expected manager to be of type *Manager") + } +} + +func TestConfigPath(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + configPath, err := manager.configPath() + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if configPath == "" { + t.Error("Expected non-empty config path") + } + + // Verify path contains expected components + if !contains(configPath, ".gen3") { + t.Error("Expected config path to contain .gen3 directory") + } + + if !contains(configPath, "gen3_client_config.ini") { + t.Error("Expected config path to contain gen3_client_config.ini") + } +} + +func TestImport_WithCredentialFile(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // Create a temporary credential file + tmpDir := t.TempDir() + credFile := path.Join(tmpDir, "cred.json") + + credContent := `{ + "KeyID": "test-key-id", + "APIKey": "test-api-key" + }` + + if err := os.WriteFile(credFile, []byte(credContent), 0644); err != nil { + t.Fatalf("Failed to create test credential file: %v", err) + } + + cred, err := manager.Import(credFile, "") + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if cred == nil { + t.Fatal("Expected non-nil credential") + } + + if cred.KeyID != "test-key-id" { + t.Errorf("Expected KeyID 'test-key-id', got '%s'", cred.KeyID) + } + + if cred.APIKey != "test-api-key" { + t.Errorf("Expected APIKey 'test-api-key', got '%s'", cred.APIKey) + } +} + +func TestImport_WithSnakeCaseCredentialFields(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + tmpDir := t.TempDir() + credFile := path.Join(tmpDir, "cred.json") + + credContent := `{ + "key_id": "test-key-id", + "api_key": "test-api-key", + "access_token": "test-access-token", + "api_endpoint": "https://example.org", + "project_id": "project-1" + }` + + if err := os.WriteFile(credFile, []byte(credContent), 0644); err != nil { + t.Fatalf("Failed to create test credential file: %v", err) + } + + cred, err := manager.Import(credFile, "") + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if cred.KeyID != "test-key-id" { + t.Fatalf("KeyID = %q, want %q", cred.KeyID, "test-key-id") + } + if cred.APIKey != "test-api-key" { + t.Fatalf("APIKey = %q, want %q", cred.APIKey, "test-api-key") + } + if cred.AccessToken != "test-access-token" { + t.Fatalf("AccessToken = %q, want %q", cred.AccessToken, "test-access-token") + } + if cred.APIEndpoint != "https://example.org" { + t.Fatalf("APIEndpoint = %q, want %q", cred.APIEndpoint, "https://example.org") + } + if cred.ProjectID != "project-1" { + t.Fatalf("ProjectID = %q, want %q", cred.ProjectID, "project-1") + } +} + +func TestImport_WithFenceToken(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + token := "test-fence-token-12345" + cred, err := manager.Import("", token) + + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if cred == nil { + t.Fatal("Expected non-nil credential") + } + + if cred.AccessToken != token { + t.Errorf("Expected AccessToken '%s', got '%s'", token, cred.AccessToken) + } +} + +func TestImport_NoCredentialOrToken(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + _, err := manager.Import("", "") + + if err == nil { + t.Fatal("Expected error when neither credential file nor token provided") + } + + if !contains(err.Error(), "either credential file or fence token must be provided") { + t.Errorf("Expected specific error message, got: %v", err) + } +} + +func TestImport_InvalidCredentialFile(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // Test with non-existent file + _, err := manager.Import("/nonexistent/path/cred.json", "") + + if err == nil { + t.Fatal("Expected error for non-existent file") + } +} + +func TestImport_InvalidJSON(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // Create a temporary file with invalid JSON + tmpDir := t.TempDir() + credFile := path.Join(tmpDir, "invalid.json") + + invalidJSON := `{invalid json content` + + if err := os.WriteFile(credFile, []byte(invalidJSON), 0644); err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + _, err := manager.Import(credFile, "") + + if err == nil { + t.Fatal("Expected error for invalid JSON") + } + + if !contains(err.Error(), "cannot parse JSON credential file") { + t.Errorf("Expected JSON parse error, got: %v", err) + } +} + +func TestEnsureExists(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // This test is tricky because it modifies the user's home directory + // We'll just verify it doesn't panic and returns a reasonable error or nil + err := manager.EnsureExists() + + // We accept either success or a reasonable error + if err != nil { + // Just log the error, don't fail the test + t.Logf("EnsureExists returned error (may be expected): %v", err) + } +} + +func TestLoad_ProfileNotFound(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // Try to load a profile that doesn't exist + _, err := manager.Load("nonexistent-profile") + + if err == nil { + t.Fatal("Expected error for non-existent profile") + } + + // Should contain profile not found error + if !contains(err.Error(), "profile not found") && !contains(err.Error(), "Need to run") { + t.Logf("Got error (may be expected): %v", err) + } +} + +func TestIsCredentialValid_RejectsAccessTokenMatchingAPIKey(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + cred := &Credential{ + APIKey: "same-token", + AccessToken: "same-token", + } + + valid, err := manager.IsCredentialValid(cred) + if err == nil { + t.Fatal("expected error when access token matches api key") + } + if valid { + t.Fatal("expected credential to be invalid") + } + if !contains(err.Error(), "access_token matches api_key") { + t.Fatalf("unexpected error: %v", err) + } +} + +// Helper function +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(substr) == 0 || + (len(s) > 0 && len(substr) > 0 && findSubstring(s, substr))) +} + +func findSubstring(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/conf/validate.go b/conf/validate.go new file mode 100644 index 0000000..292493e --- /dev/null +++ b/conf/validate.go @@ -0,0 +1,106 @@ +package conf + +import ( + "errors" + "fmt" + "net/url" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +const tokenExpiryWarningThreshold = 7 * 24 * time.Hour + +func ValidateUrl(apiEndpoint string) (*url.URL, error) { + parsedURL, err := url.Parse(apiEndpoint) + if err != nil { + return parsedURL, errors.New("Error occurred when parsing apiendpoint URL: " + err.Error()) + } + if parsedURL.Host == "" { + return parsedURL, errors.New("Invalid endpoint. A valid endpoint looks like: https://www.tests.com") + } + return parsedURL, nil +} + +func (man *Manager) IsTokenValid(tokenStr string) (bool, error) { + if tokenStr == "" { + return false, fmt.Errorf("token is empty") + } + // Parse the token without verifying the signature to access the claims. + token, _, err := new(jwt.Parser).ParseUnverified(tokenStr, jwt.MapClaims{}) + if err != nil { + return false, fmt.Errorf("invalid token format: %v", err) + } + + claims, ok := token.Claims.(jwt.MapClaims) + if !ok { + return false, fmt.Errorf("unable to parse claims from provided token") + } + + exp, ok := claims["exp"].(float64) + if !ok { + return false, fmt.Errorf("'exp' claim not found or is not a number") + } + + iat, ok := claims["iat"].(float64) + if !ok { + // iat is not strictly required for validity in all cases, but we'll keep it for now as per original code + return false, fmt.Errorf("'iat' claim not found or is not a number") + } + + now := time.Now().UTC() + expTime := time.Unix(int64(exp), 0).UTC() + iatTime := time.Unix(int64(iat), 0).UTC() + + if expTime.Before(now) { + return false, fmt.Errorf("token expired at %s (now: %s)", expTime.Format(time.RFC3339), now.Format(time.RFC3339)) + } + if iatTime.After(now) { + return false, fmt.Errorf("token not yet valid: iat %s > now %s", iatTime.Format(time.RFC3339), now.Format(time.RFC3339)) + } + + delta := expTime.Sub(now) + if delta > 0 && delta <= tokenExpiryWarningThreshold && man.Logger != nil { + if delta < 24*time.Hour { + man.Logger.Warn(fmt.Sprintf("Token will expire in less than 24 hours, on %s", expTime.Format(time.RFC3339))) + } else { + daysUntilExpiration := int(delta.Hours() / 24) + if daysUntilExpiration < 1 { + daysUntilExpiration = 1 + } + man.Logger.Warn(fmt.Sprintf("Token will expire in %d days, on %s", daysUntilExpiration, expTime.Format(time.RFC3339))) + } + } + + return true, nil +} + +func (man *Manager) IsCredentialValid(profileConfig *Credential) (bool, error) { + if profileConfig == nil { + return false, fmt.Errorf("profileConfig is nil") + } + + if profileConfig.AccessToken != "" && + profileConfig.APIKey != "" && + profileConfig.AccessToken == profileConfig.APIKey { + return false, fmt.Errorf("access_token matches api_key and appears to be misconfigured") + } + + accessTokenValid, accessErr := man.IsTokenValid(profileConfig.AccessToken) + apiKeyValid, apiErr := man.IsTokenValid(profileConfig.APIKey) + + if !accessTokenValid && !apiKeyValid { + return false, fmt.Errorf("both access_token and api_key are invalid: %v; %v", accessErr, apiErr) + } + + if !accessTokenValid && apiKeyValid { + return false, fmt.Errorf("access_token is invalid but api_key is valid: %v", accessErr) + } + + return true, nil +} + +func (man *Manager) IsValid(profileConfig *Credential) (bool, error) { + // Maintain backward compatibility by checking APIKey as before, but using the new helper + return man.IsTokenValid(profileConfig.APIKey) +} diff --git a/conf/validate_test.go b/conf/validate_test.go new file mode 100644 index 0000000..71dd5c2 --- /dev/null +++ b/conf/validate_test.go @@ -0,0 +1,181 @@ +package conf + +import ( + "bytes" + "log/slog" + "strings" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +func createTestToken(exp time.Time, iat time.Time) string { + claims := jwt.MapClaims{ + "exp": exp.Unix(), + "iat": iat.Unix(), + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + // We don't need a real signature for ParseUnverified + tokenString, _ := token.SignedString([]byte("secret")) + return tokenString +} + +func TestIsTokenValid(t *testing.T) { + man := &Manager{} + now := time.Now().UTC() + + tests := []struct { + name string + token string + want bool + wantErr bool + }{ + { + name: "Valid Token", + token: createTestToken(now.Add(time.Hour), now.Add(-time.Hour)), + want: true, + wantErr: false, + }, + { + name: "Expired Token", + token: createTestToken(now.Add(-time.Hour), now.Add(-2*time.Hour)), + want: false, + wantErr: true, + }, + { + name: "Not Yet Valid Token", + token: createTestToken(now.Add(2*time.Hour), now.Add(time.Hour)), + want: false, + wantErr: true, + }, + { + name: "Empty Token", + token: "", + want: false, + wantErr: true, + }, + { + name: "Invalid Token Format", + token: "not.a.token", + want: false, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := man.IsTokenValid(tt.token) + if (err != nil) != tt.wantErr { + t.Errorf("IsTokenValid() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("IsTokenValid() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsCredentialValid(t *testing.T) { + man := &Manager{} + now := time.Now().UTC() + validAccessToken := createTestToken(now.Add(time.Hour), now.Add(-time.Hour)) + validAPIKey := createTestToken(now.Add(2*time.Hour), now.Add(-time.Hour)) + expiredToken := createTestToken(now.Add(-time.Hour), now.Add(-2*time.Hour)) + + tests := []struct { + name string + cred *Credential + want bool + wantErr bool + }{ + { + name: "Both Valid", + cred: &Credential{ + AccessToken: validAccessToken, + APIKey: validAPIKey, + }, + want: true, + wantErr: false, + }, + { + name: "AccessToken Invalid, APIKey Valid (Needs Refresh)", + cred: &Credential{ + AccessToken: expiredToken, + APIKey: validAPIKey, + }, + want: false, + wantErr: true, + }, + { + name: "AccessToken Matches APIKey", + cred: &Credential{ + AccessToken: validAccessToken, + APIKey: validAccessToken, + }, + want: false, + wantErr: true, + }, + { + name: "Both Invalid", + cred: &Credential{ + AccessToken: expiredToken, + APIKey: expiredToken, + }, + want: false, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := man.IsCredentialValid(tt.cred) + if (err != nil) != tt.wantErr { + t.Errorf("IsCredentialValid() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("IsCredentialValid() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsTokenValid_WarnsWhenTokenExpiresWithinSevenDays(t *testing.T) { + var out bytes.Buffer + logger := slog.New(slog.NewTextHandler(&out, nil)) + man := &Manager{Logger: logger} + now := time.Now().UTC() + + token := createTestToken(now.Add(6*24*time.Hour+2*time.Hour), now.Add(-time.Hour)) + valid, err := man.IsTokenValid(token) + if err != nil { + t.Fatalf("IsTokenValid returned error: %v", err) + } + if !valid { + t.Fatal("expected token to be valid") + } + if !strings.Contains(out.String(), "Token will expire in") { + t.Fatalf("expected expiration warning, got %q", out.String()) + } +} + +func TestIsTokenValid_DoesNotWarnWhenTokenExpiresAfterSevenDays(t *testing.T) { + var out bytes.Buffer + logger := slog.New(slog.NewTextHandler(&out, nil)) + man := &Manager{Logger: logger} + now := time.Now().UTC() + + token := createTestToken(now.Add(8*24*time.Hour), now.Add(-time.Hour)) + valid, err := man.IsTokenValid(token) + if err != nil { + t.Fatalf("IsTokenValid returned error: %v", err) + } + if !valid { + t.Fatal("expected token to be valid") + } + if strings.Contains(out.String(), "Token will expire in") { + t.Fatalf("did not expect expiration warning, got %q", out.String()) + } +} diff --git a/credentials/refresh.go b/credentials/refresh.go new file mode 100644 index 0000000..95652f0 --- /dev/null +++ b/credentials/refresh.go @@ -0,0 +1,45 @@ +package credentials + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/fence" + "github.com/calypr/calypr-cli/logs" + "github.com/calypr/calypr-cli/request" +) + +// EnsureValidCredential validates a profile credential and refreshes its +// access token from the API key when the token has expired. +func EnsureValidCredential(ctx context.Context, cred *conf.Credential, baseLogger *slog.Logger) error { + manager := conf.NewConfigure(baseLogger) + logger := logs.NewGen3Logger(baseLogger, "", cred.Profile) + + valid, err := manager.IsCredentialValid(cred) + if valid { + return nil + } + if err == nil { + return fmt.Errorf("invalid credential") + } + + if !strings.Contains(err.Error(), "access_token is invalid but api_key is valid") { + return fmt.Errorf("invalid credential: %v", err) + } + + req := request.NewRequestInterface(logger, cred, manager) + fClient := fence.NewFenceClient(req, cred, baseLogger) + newToken, refreshErr := fClient.NewAccessToken(ctx) + if refreshErr != nil { + return fmt.Errorf("failed to refresh access token: %v (original error: %v)", refreshErr, err) + } + + cred.AccessToken = newToken + if saveErr := manager.Save(cred); saveErr != nil { + logger.Warn(fmt.Sprintf("failed to save refreshed token: %v", saveErr)) + } + return nil +} diff --git a/docs/DEVELOPER_DOCS.md b/docs/DEVELOPER_DOCS.md new file mode 100644 index 0000000..ec4a701 --- /dev/null +++ b/docs/DEVELOPER_DOCS.md @@ -0,0 +1,60 @@ +# Developer Docs + +`calypr-cli` started from the Gen3 data-client lineage, but this repo now owns +its own CLI surface and operator workflows. Treat it as a command-oriented Go +application with a few backend-specific client packages, not as a thin mirror of +the old fork. + +## Entry Points + +- `main.go`: root-compatible binary entrypoint used by `go build .` +- `cmd/calypr-cli/main.go`: compatibility wrapper for subdirectory builds +- `cmd/`: Cobra command definitions and CLI orchestration + +The root command is `calypr-cli`, and most subcommands require a named +`--profile`. + +## Package Map + +| Path | Responsibility | +| --- | --- | +| `cmd/` | User-facing commands for configure/auth, transfer flows, collaborators, permissions, and portal management | +| `conf/` | Profile loading, persistence, and config validation | +| `credentials/` | Credential refresh helpers | +| `g3client/` | Shared Gen3 client adapter layer, including Syfon-backed surfaces | +| `fence/` | Fence-specific request and response helpers | +| `gecko/` | Gecko-specific client code for portal operations | +| `arborist/` | Arborist-specific client code for permissions operations | + +## High-Level Command Ownership + +- Transfer commands live in `cmd/` and rely on shared Gen3/Syfon client code. +- `permissions` is the Arborist-backed operator surface. +- `portal` is the Gecko-backed operator surface. +- `collaborators` handles collaboration-request workflows exposed by the target + commons. + +## Build And Test + +Local entrypoints: + +```bash +go build . +go build ./cmd/calypr-cli +go test ./... +``` + +Top-level automation lives under `.github/workflows/`. The intended baseline is: + +- `ci.yaml` for general lint and unit test coverage +- `coverage.yaml` for explicit coverage reporting +- `release.yaml` for tagged releases +- reusable image and integration workflows where the repo still depends on the + shared `uc-cdis/.github` automation + +## Notes + +- The default backend flag is `gen3`, but the CLI also carries backend-specific + behavior behind command flags and adapters. +- This repo still contains some compatibility surfaces from older Gen3 client + behavior; when cleaning up, verify call sites before removing them. diff --git a/docs/optimal-chunk-size.md b/docs/optimal-chunk-size.md new file mode 100644 index 0000000..55cb850 --- /dev/null +++ b/docs/optimal-chunk-size.md @@ -0,0 +1,151 @@ + +# Engineering note — Optimal Chunk Size Calculation for Multipart Uploads + +## OLD: + optimalChunkSize determines the ideal chunk/part size for multipart upload based on file size. + The chunk size (also known as "message size" or "part size") affects upload performance and + must comply with S3 constraints. + + Calculation logic: + - For files ≤ 512 MB: Returns 32 MB chunks for optimal performance + - For files > 512 MB: Calculates fileSize/maxMultipartParts, with minimum of 5 MB + - Enforces minimum of 5 MB (S3 requirement for all parts except the last) + - Rounds up to nearest MB for alignment + + This results in: + - Files ≤ 512 MB: 32 MB chunks + - Files 512 MB - ~49 GB: 5 MB chunks (minimum enforced) + The ~49 GB threshold (10,000 parts × 5 MB) is where files exceed S3's + 10,000 part limit when using the minimum chunk size + - Files > ~49 GB: Dynamically calculated to stay under 10,000 parts + + Examples: + - 100 MB file → 32 MB chunks (4 parts) + - 1 GB file → 5 MB chunks (~205 parts) + - 10 GB file → 5 MB chunks (~2,048 parts) + - 50 GB file → 6 MB chunks (~8,534 parts) + - 100 GB file → 11 MB chunks (~9,310 parts) + - 1 TB file → 105 MB chunks (~9,987 parts) + +## NEW + +OptimalChunkSize determines the ideal chunk/part size for multipart upload based on file size. +The chunk size (also known as "message size" or "part size") affects upload performance and +must comply with S3 constraints. + +Calculation logic: + - For files ≤ 100 MB: Returns the file size itself (single PUT, no multipart) + - For files > 100 MB and ≤ 1 GB: Returns 10 MB chunks + - For files > 1 GB and ≤ 10 GB: Scales linearly between 25 MB and 128 MB + - For files > 10 GB and ≤ 100 GB: Returns 256 MB chunks + - For files > 100 GB: Scales linearly between 512 MB and 1024 MB (capped at 1 TB for ratio purposes) + - All chunk sizes are rounded down to the nearest MB + - Minimum chunk size is 1 MB (for zero or negative input) + +This results in: + - Files ≤ 100 MB: Single PUT upload + - Files 100 MB - 1 GB: 10 MB chunks + - Files 1 GB - 10 GB: 25-128 MB chunks (scaled) + - Files 10 GB - 100 GB: 256 MB chunks + - Files > 100 GB: 512-1024 MB chunks (scaled) + +Examples: + - 100 MB file → 100 MB chunk (1 part, single PUT) + - 500 MB file → 10 MB chunks (50 parts) + - 1 GB file → 10 MB chunks (103 parts) + - 5 GB file → 70 MB chunks (74 parts, scaled) + - 10 GB file → 128 MB chunks (80 parts) + - 50 GB file → 256 MB chunks (200 parts) + - 100 GB file → 256 MB chunks (400 parts) + - 500 GB file → 739 MB chunks (693 parts, scaled) + - 1 TB file → 1024 MB chunks (1024 parts) + +### Testing + + +```bash +go test ./upload -run '^TestOptimalChunkSize$' -v + +``` + +Purpose +- Validate `OptimalChunkSize` behavior and return values (chunk size and number of parts) across thresholds, boundaries and scaled ranges. + +Key behavior to assert +1. Input type and units: sizes are `int64` bytes; tests should use `common.MB` / `common.GB` constants. +2. Parts calculation: `parts = ceil(fileSize / chunk)`; `fileSize == 0` returns `parts == 0`. +3. Scaling: scaled ranges are linear, rounded **down** to the nearest MB and clamped to range. +4. Minimum chunk clamp: result is at least `1 MB`. +5. Boundary semantics: implementation uses `<=` and some ranges start at `X + 1` — include exact, \-1 and \+1 byte checks. + +Parameterized test cases (file size ⇒ expected chunk ⇒ expected parts) +1. `0` bytes + - chunk: `1 MB` (fallback) + - parts: `0` + +2. `1 MB` + - chunk: `1 MB` (<= 100 MB) + - parts: `1` + +3. `100 MB` + - chunk: `100 MB` (<= 100 MB) + - parts: `1` + +4. `100 MB + 1 B` + - chunk: `10 MB` (> 100 MB - <= 1 GB) + - parts: ceil((100 MB + 1 B) / 10 MB) = `11` + +5. `500 MB` + - chunk: `10 MB` + - parts: `50` + +6. `1 GB` (1024 MB) + - chunk: `10 MB` (<= 1 GB) + - parts: ceil(1024 / 10) = `103` + +7. `1 GB + 1 B` + - chunk: `25 MB` (start of 1 GB - 10 GB scaled range) + - parts: ceil((1024 MB + 1 B) / 25 MB) = `41` + +8. `5 GB` (5120 MB) + - chunk: linear between `25 MB` and `128 MB` → ≈ `70 MB` (rounded down) + - parts: ceil(5120 / 70) = `74` + +9. `10 GB` (10240 MB) + - chunk: `128 MB` (end of 1 GB - 10 GB scaled range) + - parts: `80` + +10. `10 GB + 1 B` + - chunk: `256 MB` (> 10 GB - <= 100 GB fixed) + - parts: ceil((10240 MB + 1 B) / 256 MB) = `41` + +11. `50 GB` (51200 MB) + - chunk: `256 MB` + - parts: `200` + +12. `100 GB` (102400 MB) + - chunk: `256 MB` + - parts: `400` + +13. `100 GB + 1 B` + - chunk: `512 MB` (start of > 100 GB scaled range) + - parts: ceil((102400 MB + 1 B) / 512 MB) = `201` + +14. `500 GB` (512000 MB) + - chunk: linear between `512 MB` and `1024 MB` → ≈ `739 MB` (rounded down) + - parts: ceil(512000 / 739) = `693` + +15. `1 TB` (1024 GB = 1,048,576 MB) — note: use project units consistently + - chunk: `1024 MB` (max of scaled range) + - parts: 1,048,576 / 1024 = `1024` + +Test design notes (concise) +1. Use table-driven subtests in `upload/utils_test.go`. Include fields: name, `fileSize int64`, `wantChunk int64`, `wantParts int64`. +2. For scaled cases assert: MB alignment, clamped to min/max, and exact `wantParts`. Use integer arithmetic for parts. +3. Add explicit boundary triples for each threshold: exact, -1 byte, +1 byte. +4. Include negative and zero cases to verify fallback behavior. +5. Keep tests deterministic and fast (no external deps). + +Execution +- Run from repo root: `go test ./upload -v` +- Run single test: `go test ./upload -run '^TestOptimalChunkSize$' -v` \ No newline at end of file diff --git a/docs/permissions-cli.md b/docs/permissions-cli.md new file mode 100644 index 0000000..385db42 --- /dev/null +++ b/docs/permissions-cli.md @@ -0,0 +1,147 @@ +# Permissions CLI + +`calypr-cli permissions` is the operator-facing CLI for the Arborist routes +that are actually exposed through the public Gen3 `/authz` surface. + +This is intentionally not a full Arborist admin client. Raw Arborist catalog +routes like user, role, and resource CRUD are not exposed through revproxy, so +`calypr-cli` does not try to support them. + +Use `calypr-cli collaborators` when a user is asking for access they do not +already have. Use `calypr-cli permissions` when an admin needs to inspect or +change the ownership and direct-access state that Arborist exposes publicly. + +## Basic Shape + +All commands use the normal calypr-cli profile: + +```bash +calypr-cli --profile calypr permissions +``` + +Add `--json` when you need raw output for debugging or scripts: + +```bash +calypr-cli --profile calypr permissions --json auth mapping +``` + +## Current Auth Mapping + +Show the current profile user's Arborist mapping: + +```bash +calypr-cli --profile calypr permissions auth mapping +``` + +This command reads the public `GET /authz/mapping` surface for the current +token. It does not support arbitrary username lookups. + +## Organization Membership + +Organization membership is a convenience wrapper around Arborist direct-access +grants on `/programs//projects`. + +```bash +calypr-cli --profile calypr permissions org-membership add user@ohsu.edu Ellrott_Lab +calypr-cli --profile calypr permissions org-membership rm user@ohsu.edu Ellrott_Lab +``` + +The default role is `org-member`. It carries only +`arborist/create-descendant`, which lets the member create projects under the +existing organization without granting ownership or access on existing projects. + +You can specify another role when needed: + +```bash +calypr-cli --profile calypr permissions org-membership add user@ohsu.edu Ellrott_Lab --role org-member +``` + +Do not pass a resource path to `org-membership`. This is valid: + +```bash +Ellrott_Lab +``` + +This is invalid: + +```bash +/programs/Ellrott_Lab +``` + +## Ownership Power Tools + +Create a new organization resource and make the caller its owner: + +```bash +calypr-cli --profile calypr permissions ownership create-descendant \ + --parent /programs \ + --name Ellrott_Lab \ + --template gen3-program +``` + +Create a new project resource under an organization and make the caller its +owner: + +```bash +calypr-cli --profile calypr permissions ownership create-descendant \ + --parent /programs/Ellrott_Lab/projects \ + --name git_drs_test \ + --template gen3-project +``` + +Add or remove owners: + +```bash +calypr-cli --profile calypr permissions ownership add-owner \ + --resource /programs/Ellrott_Lab \ + --user user@ohsu.edu + +calypr-cli --profile calypr permissions ownership rm-owner \ + --resource /programs/Ellrott_Lab \ + --user user@ohsu.edu +``` + +Read the normalized ownership and direct-access state for a resource: + +```bash +calypr-cli --profile calypr permissions ownership get-resource \ + --resource /programs/Ellrott_Lab/projects/git_drs_test \ + --include-admins \ + --include-children +``` + +## Direct Access Power Tools + +Grant or revoke direct non-owner access on an existing resource: + +```bash +calypr-cli --profile calypr permissions access grant-user \ + --resource /programs/Ellrott_Lab/projects/git_drs_test \ + --user user@ohsu.edu \ + --role writer + +calypr-cli --profile calypr permissions access revoke-user \ + --resource /programs/Ellrott_Lab/projects/git_drs_test \ + --user user@ohsu.edu \ + --role writer +``` + +Use these commands for ordinary direct access. Use ownership add/remove for +owner changes. + +## Deliberate Non-Support + +`calypr-cli permissions` does not support raw Arborist catalog/admin routes such +as: + +- user CRUD +- role CRUD +- resource CRUD +- raw policy mutation +- arbitrary-user auth mapping lookup + +Those routes are not part of the supported public Gen3 revproxy surface, so the +CLI does not expose them. + +The legacy backend-oriented command name `calypr-cli arborist` still works as a +compatibility alias, but `permissions` is the supported user-facing name. diff --git a/docs/portal-cli.md b/docs/portal-cli.md new file mode 100644 index 0000000..07c2041 --- /dev/null +++ b/docs/portal-cli.md @@ -0,0 +1,178 @@ +# Portal CLI + +`calypr-cli portal` is the operator-facing CLI for the portal routes exposed +through the public Gen3 `/gecko` surface. + +This is a configuration and health client. It is not a generic Gecko admin +shell for every internal backend route. + +## Basic Shape + +All commands use a normal calypr-cli profile: + +```bash +calypr-cli --profile calypr portal +``` + +Add `--json` when you want raw output for debugging or scripts: + +```bash +calypr-cli --profile calypr portal --json config types +``` + +## Current Support + +`calypr-cli portal` currently supports: + +- service health checks +- listing Gecko config types +- listing config IDs for a config type +- fetching a config by type and ID +- project config get, put, and delete +- app-card get, put, and delete + +The client currently recognizes these Gecko config-type tokens: + +- `explorer` +- `nav` +- `file_summary` +- `apps_page` +- `project` +- `projects` + +`project` config CRUD should use the dedicated `project` commands below. Those +commands target the public `/gecko/projects/...` route shape. + +## Health Check + +Check whether Gecko is reachable through revproxy: + +```bash +calypr-cli --profile calypr portal health +``` + +## Generic Config Inspection + +List the config types Gecko reports: + +```bash +calypr-cli --profile calypr portal config types +``` + +List config IDs for a type: + +```bash +calypr-cli --profile calypr portal config list explorer +calypr-cli --profile calypr portal config list nav +calypr-cli --profile calypr portal config list apps_page +``` + +Fetch a config by type and ID: + +```bash +calypr-cli --profile calypr portal config get explorer default +calypr-cli --profile calypr portal config get nav default +``` + +Use `config` when you need typed config inspection. Use the dedicated commands +below when you are working with project configs or app cards. + +## Project Configs + +Project config commands use an ID in `ORG/PROJECT` form, not a flat project +name. + +This is valid: + +```bash +HTAN_INT/BForePC +``` + +This is invalid: + +```bash +BForePC +``` + +Get a project config: + +```bash +calypr-cli --profile calypr portal project get HTAN_INT/BForePC +``` + +Create or replace a project config from JSON: + +```bash +calypr-cli --profile calypr portal project put HTAN_INT/BForePC --file project.json +``` + +Delete a project config: + +```bash +calypr-cli --profile calypr portal project delete HTAN_INT/BForePC +``` + +Expected JSON shape: + +```json +{ + "title": "BForePC", + "contact_email": "owner@example.org", + "src_repo": "https://github.com/calypr/gecko", + "org_title": "HTAN INT", + "description": "Example project config", + "project_title": "BForePC", + "icon_name": "folder-git-2" +} +``` + +`src_repo` is validated before the request is sent. The client expects a real +GitHub repository and normalizes the value before writing the config. + +## App Cards + +App-card commands use a project ID, usually in `ORG-PROJECT` form. + +Get an app card: + +```bash +calypr-cli --profile calypr portal appcard get HTAN_INT-BForePC +``` + +Create or replace an app card from JSON: + +```bash +calypr-cli --profile calypr portal appcard put HTAN_INT-BForePC --file appcard.json +``` + +Delete an app card: + +```bash +calypr-cli --profile calypr portal appcard delete HTAN_INT-BForePC +``` + +Expected JSON shape: + +```json +{ + "title": "BForePC", + "description": "Open the BForePC workspace", + "icon": "folder-git-2", + "href": "/workspace/BForePC", + "perms": "read" +} +``` + +## Deliberate Non-Support + +`calypr-cli portal` does not try to expose every internal Gecko backend route. +It is currently scoped to the public revproxy-backed Gecko surface that this +client already knows how to model: + +- health +- config type and config inspection +- project config CRUD +- app-card CRUD + +The legacy backend-oriented command name `calypr-cli gecko` still works as a +compatibility alias, but `portal` is the supported user-facing name. diff --git a/fence/client.go b/fence/client.go new file mode 100644 index 0000000..81770ba --- /dev/null +++ b/fence/client.go @@ -0,0 +1,659 @@ +package fence + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "log/slog" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/request" + sycommon "github.com/calypr/syfon/client/common" + "github.com/hashicorp/go-version" +) + +// FenceBucketEndpoint is the endpoint postfix for FENCE bucket list +const FenceBucketEndpoint = "/data/buckets" + +const ( + fenceAccessTokenEndpoint = "/user/credentials/api/access_token" + fenceUserEndpoint = "/user/user" + fenceDataEndpoint = "/user/data" + fenceDataUploadEndpoint = fenceDataEndpoint + "/upload" + fenceDataDownloadEndpoint = fenceDataEndpoint + "/download" + fenceDataMultipartInitEndpoint = fenceDataEndpoint + "/multipart/init" + fenceDataMultipartUploadEndpoint = fenceDataEndpoint + "/multipart/upload" + fenceDataMultipartCompleteEndpoint = fenceDataEndpoint + "/multipart/complete" + shepherdEndpoint = "/mds" + shepherdVersionEndpoint = "/mds/version" + defaultUseShepherd = false + defaultMinShepherdVersion = "2.0.0" +) + +//go:generate go run go.uber.org/mock/mockgen@v0.6.0 -destination=../mocks/mock_fence.go -package=mocks github.com/calypr/calypr-cli/fence FenceInterface + +// FenceInterface defines the interface for Fence client +type FenceInterface interface { + request.RequestInterface + + NewAccessToken(ctx context.Context) (string, error) + CheckPrivileges(ctx context.Context) (map[string]any, error) + CheckForShepherdAPI(ctx context.Context) (bool, error) + DeleteRecord(ctx context.Context, guid string) (string, error) + GetDownloadPresignedUrl(ctx context.Context, guid, protocolText string) (string, error) + + UserPing(ctx context.Context) (*PingResp, error) + + // Bucket details + GetBucketDetails(ctx context.Context, bucket string) (*S3Bucket, error) + + // Upload methods + InitUpload(ctx context.Context, filename string, bucket string, guid string) (FenceResponse, error) + GetUploadPresignedUrl(ctx context.Context, guid string, filename string, bucket string) (FenceResponse, error) + + // Multipart methods + InitMultipartUpload(ctx context.Context, filename string, bucket string, guid string) (FenceResponse, error) + GenerateMultipartPresignedURL(ctx context.Context, key string, uploadID string, partNumber int, bucket string) (string, error) + CompleteMultipartUpload(ctx context.Context, key string, uploadID string, parts []MultipartPart, bucket string) error + ParseFenceURLResponse(resp *http.Response) (FenceResponse, error) + + RefreshToken(ctx context.Context) error +} + +// FenceClient implements FenceInterface +// FenceClient implements FenceInterface +type FenceClient struct { + request.RequestInterface + cred *conf.Credential + logger *slog.Logger +} + +// NewFenceClient creates a new FenceClient +func NewFenceClient(req request.RequestInterface, cred *conf.Credential, logger *slog.Logger) FenceInterface { + return &FenceClient{ + RequestInterface: req, + cred: cred, + logger: logger, + } +} + +func (f *FenceClient) NewAccessToken(ctx context.Context) (string, error) { + if f.cred.APIKey == "" { + return "", errors.New("APIKey is required to refresh access token") + } + + payload, err := json.Marshal(map[string]string{"api_key": f.cred.APIKey}) + if err != nil { + return "", err + } + bodyReader := bytes.NewReader(payload) + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Method: http.MethodPost, + Url: f.cred.APIEndpoint + fenceAccessTokenEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Body: bodyReader, + }, + ) + + if err != nil { + return "", fmt.Errorf("error when calling Request.Do: %s", err) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", errors.New("failed to refresh token, status: " + strconv.Itoa(resp.StatusCode)) + } + + var result struct { + AccessToken string `json:"access_token"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", errors.New("failed to parse token response: " + err.Error()) + } + + return result.AccessToken, nil +} + +func (f *FenceClient) RefreshToken(ctx context.Context) error { + token, err := f.NewAccessToken(ctx) + if err != nil { + return err + } + f.cred.AccessToken = token + return nil +} + +func (f *FenceClient) CheckPrivileges(ctx context.Context) (map[string]any, error) { + resp, err := f.Do(ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceUserEndpoint, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return nil, errors.New("error occurred when getting response from remote: " + err.Error()) + } + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var data map[string]any + err = json.Unmarshal(bodyBytes, &data) + if err != nil { + return nil, errors.New("error occurred when unmarshalling response: " + err.Error()) + } + + resourceAccess, ok := data["authz"].(map[string]any) + + // If the `authz` section (Arborist permissions) is empty or missing, try get `project_access` section (Fence permissions) + if len(resourceAccess) == 0 || !ok { + resourceAccess, ok = data["project_access"].(map[string]any) + if !ok { + return nil, errors.New("not possible to read access privileges of user") + } + } + + return resourceAccess, nil +} + +func (f *FenceClient) CheckForShepherdAPI(ctx context.Context) (bool, error) { + // Check if Shepherd is enabled + if f.cred.UseShepherd == "false" { + return false, nil + } + if f.cred.UseShepherd != "true" && defaultUseShepherd == false { + return false, nil + } + // If Shepherd is enabled, make sure that the commons has a compatible version of Shepherd deployed. + // Compare the version returned from the Shepherd version endpoint with the minimum acceptable Shepherd version. + var minShepherdVersion string + if f.cred.MinShepherdVersion == "" { + minShepherdVersion = defaultMinShepherdVersion + } else { + minShepherdVersion = f.cred.MinShepherdVersion + } + + res, err := f.Do(ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + shepherdVersionEndpoint, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return false, errors.New("Error occurred during generating HTTP request: " + err.Error()) + } + defer res.Body.Close() + if res.StatusCode != 200 { + return false, nil + } + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, errors.New("Error occurred when reading HTTP request: " + err.Error()) + } + body, err := strconv.Unquote(string(bodyBytes)) + if err != nil { + return false, fmt.Errorf("Error occurred when parsing version from Shepherd: %v: %v", string(body), err) + } + // Compare the version in the response to the target version + ver, err := version.NewVersion(body) + if err != nil { + return false, fmt.Errorf("Error occurred when parsing version from Shepherd: %v: %v", string(body), err) + } + minVer, err := version.NewVersion(minShepherdVersion) + if err != nil { + return false, fmt.Errorf("Error occurred when parsing minimum acceptable Shepherd version: %v: %v", minShepherdVersion, err) + } + if ver.GreaterThanOrEqual(minVer) { + return true, nil + } + return false, fmt.Errorf("Shepherd is enabled, but %v does not have correct Shepherd version. (Need Shepherd version >=%v, got %v)", f.cred.APIEndpoint, minVer, ver) +} + +func (f *FenceClient) DeleteRecord(ctx context.Context, guid string) (string, error) { + hasShepherd, err := f.CheckForShepherdAPI(ctx) + if err != nil { + f.logger.Warn(fmt.Sprintf("WARNING: Error checking Shepherd API: %v. Falling back to Fence.\n", err)) + } else if hasShepherd { + resp, err := f.Do(ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + shepherdEndpoint + "/objects/" + guid, + Method: http.MethodDelete, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return "", fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode == 204 { + return "Record with GUID " + guid + " has been deleted", nil + } + return "", fmt.Errorf("shepherd delete failed: %d", resp.StatusCode) + } + + resp, err := f.Do(ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceDataEndpoint + "/" + guid, + Method: http.MethodDelete, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return "", fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNoContent { + return "Record with GUID " + guid + " has been deleted", nil + } + + _, err = f.ParseFenceURLResponse(resp) + if err != nil { + return "", err + } + return "Record with GUID " + guid + " has been deleted", nil +} + +func (f *FenceClient) GetDownloadPresignedUrl(ctx context.Context, guid, protocolText string) (string, error) { + hasShepherd, err := f.CheckForShepherdAPI(ctx) + if err == nil && hasShepherd { + return f.resolveFromShepherd(ctx, guid) + } + return f.resolveFromFence(ctx, guid, protocolText) +} + +func (f *FenceClient) resolveFromShepherd(ctx context.Context, guid string) (string, error) { + url := fmt.Sprintf("%s%s/objects/%s/download", f.cred.APIEndpoint, shepherdEndpoint, guid) + resp, err := f.Do(ctx, &request.RequestBuilder{ + Url: url, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("shepherd error: %d", resp.StatusCode) + } + + var result struct { + URL string `json:"url"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("failed to decode shepherd response: %w", err) + } + + return result.URL, nil +} + +func (f *FenceClient) resolveFromFence(ctx context.Context, guid, protocolText string) (string, error) { + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceDataDownloadEndpoint + "/" + guid + protocolText, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return "", errors.New("failed to get URL from Fence via Do: " + err.Error()) + } + defer resp.Body.Close() + + msg, err := f.ParseFenceURLResponse(resp) + if err != nil || msg.URL == "" { + return "", errors.New("failed to get URL from Fence via ParseFenceURLResponse: " + err.Error()) + } + + return msg.URL, nil +} + +func (f *FenceClient) GetBucketDetails(ctx context.Context, bucket string) (*S3Bucket, error) { + url := f.cred.APIEndpoint + "/data/buckets" + resp, err := f.Do(ctx, &request.RequestBuilder{ + Method: http.MethodGet, + Url: url, + Token: f.cred.AccessToken, + }) + if err != nil { + return nil, fmt.Errorf("failed to fetch bucket information: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + var bucketInfo S3BucketsResponse + if err := json.NewDecoder(resp.Body).Decode(&bucketInfo); err != nil { + return nil, fmt.Errorf("failed to decode bucket information: %w", err) + } + + if info, exists := bucketInfo.S3Buckets[bucket]; exists { + if info.EndpointURL != "" && info.Region != "" { + return info, nil + } + return nil, errors.New("endpoint_url or region not found for bucket") + } + + return nil, nil +} + +func (f *FenceClient) InitUpload(ctx context.Context, filename string, bucket string, guid string) (FenceResponse, error) { + payload := map[string]string{ + "file_name": filename, + } + if bucket != "" { + payload["bucket"] = bucket + } + if guid != "" { + payload["guid"] = guid + } + + buf, err := sycommon.ToJSONReader(payload) + if err != nil { + return FenceResponse{}, err + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Method: http.MethodPost, + Url: f.cred.APIEndpoint + fenceDataUploadEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Body: buf, + Token: f.cred.AccessToken, + }) + if err != nil { + return FenceResponse{}, err + } + defer resp.Body.Close() + + return f.ParseFenceURLResponse(resp) +} + +func (f *FenceClient) GetUploadPresignedUrl(ctx context.Context, guid string, filename string, bucket string) (FenceResponse, error) { + endPointPostfix := fenceDataUploadEndpoint + "/" + guid + "?file_name=" + url.QueryEscape(filename) + if bucket != "" { + endPointPostfix += "&bucket=" + bucket + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + endPointPostfix, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Token: f.cred.AccessToken, + Method: http.MethodGet, + }, + ) + if err != nil { + return FenceResponse{}, err + } + defer resp.Body.Close() + + return f.ParseFenceURLResponse(resp) +} + +func (f *FenceClient) InitMultipartUpload(ctx context.Context, filename string, bucket string, guid string) (FenceResponse, error) { + reader, err := sycommon.ToJSONReader( + InitRequestObject{ + Filename: filename, + Bucket: bucket, + GUID: guid, + }, + ) + if err != nil { + return FenceResponse{}, err + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Method: http.MethodPost, + Url: f.cred.APIEndpoint + fenceDataMultipartInitEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Body: reader, + Token: f.cred.AccessToken, + }, + ) + + if err != nil { + return FenceResponse{}, err + } + defer resp.Body.Close() + + return f.ParseFenceURLResponse(resp) +} + +func (f *FenceClient) GenerateMultipartPresignedURL(ctx context.Context, key string, uploadID string, partNumber int, bucket string) (string, error) { + reader, err := sycommon.ToJSONReader( + MultipartUploadRequestObject{ + Key: key, + UploadID: uploadID, + PartNumber: partNumber, + Bucket: bucket, + }, + ) + if err != nil { + return "", err + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceDataMultipartUploadEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Method: http.MethodPost, + Body: reader, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return "", err + } + defer resp.Body.Close() + + msg, err := f.ParseFenceURLResponse(resp) + if err != nil { + return "", err + } + + return msg.PresignedURL, nil +} + +func (f *FenceClient) CompleteMultipartUpload(ctx context.Context, key string, uploadID string, parts []MultipartPart, bucket string) error { + multipartCompleteObject := MultipartCompleteRequestObject{Key: key, UploadID: uploadID, Parts: parts, Bucket: bucket} + + reader, err := sycommon.ToJSONReader(multipartCompleteObject) + if err != nil { + return err + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceDataMultipartCompleteEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Body: reader, + Method: http.MethodPost, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusNoContent { + return nil + } + + _, err = f.ParseFenceURLResponse(resp) + return err +} + +func (f *FenceClient) ParseFenceURLResponse(resp *http.Response) (FenceResponse, error) { + msg := FenceResponse{} + if resp == nil { + return msg, errors.New("nil response received") + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return msg, fmt.Errorf("failed to read response body: %w", err) + } + bodyStr := string(bodyBytes) + strURL := "" + if resp.Request != nil && resp.Request.URL != nil { + strURL = resp.Request.URL.String() + } + + // Handle HTTP error statuses first so plain-text error bodies (for example: + // "Unauthorized") are reported accurately instead of as JSON decode failures. + if !(resp.StatusCode == 200 || resp.StatusCode == 201 || resp.StatusCode == 204) { + switch resp.StatusCode { + case http.StatusUnauthorized: + return msg, fmt.Errorf("401 Unauthorized: %s (URL: %s)", bodyStr, strURL) + case http.StatusForbidden: + return msg, fmt.Errorf("403 Forbidden: %s (URL: %s)", bodyStr, strURL) + case http.StatusNotFound: + return msg, fmt.Errorf("404 Not Found: %s (URL: %s)", bodyStr, strURL) + case http.StatusInternalServerError: + return msg, fmt.Errorf("500 Internal Server Error: %s (URL: %s)", bodyStr, strURL) + case http.StatusServiceUnavailable: + return msg, fmt.Errorf("503 Service Unavailable: %s (URL: %s)", bodyStr, strURL) + case http.StatusBadGateway: + return msg, fmt.Errorf("502 Bad Gateway: %s (URL: %s)", bodyStr, strURL) + default: + return msg, fmt.Errorf("unexpected error (%d): %s (URL: %s)", resp.StatusCode, bodyStr, strURL) + } + } + + if len(bodyBytes) > 0 { + err = json.Unmarshal(bodyBytes, &msg) + if err != nil { + return msg, fmt.Errorf("failed to decode JSON response (status=%d, url=%s): %w (raw body: %s)", resp.StatusCode, strURL, err, bodyStr) + } + } + + if strings.Contains(bodyStr, "Can't find a location for the data") { + return msg, errors.New("the provided GUID is not found") + } + + return msg, nil +} + +func (f *FenceClient) UserPing(ctx context.Context) (*PingResp, error) { + resp, err := f.Do(ctx, &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceUserEndpoint, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get user info, status: %d", resp.StatusCode) + } + + var uResp FenceUserResp + if err := json.NewDecoder(resp.Body).Decode(&uResp); err != nil { + return nil, err + } + + bucketResp, err := f.Do(ctx, &request.RequestBuilder{ + Url: f.cred.APIEndpoint + FenceBucketEndpoint, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }) + if err != nil { + return nil, err + } + defer bucketResp.Body.Close() + + if bucketResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get bucket info, status: %d", bucketResp.StatusCode) + } + + var bResp S3BucketsResponse + if err := json.NewDecoder(bucketResp.Body).Decode(&bResp); err != nil { + return nil, err + } + + return &PingResp{ + Profile: f.cred.Profile, + Username: uResp.Username, + Endpoint: f.cred.APIEndpoint, + BucketPrograms: ParseBucketResp(bResp), + YourAccess: ParseUserResp(uResp), + }, nil +} + +func ParseBucketResp(resp S3BucketsResponse) map[string]string { + bucketsByProgram := make(map[string]string) + + // Check both S3_BUCKETS and s3_buckets + s3Buckets := resp.S3Buckets + if len(s3Buckets) == 0 { + s3Buckets = resp.S3BucketsLower + } + + for bucketName, bucketInfo := range s3Buckets { + var programs strings.Builder + if len(bucketInfo.Programs) > 1 { + for i, p := range bucketInfo.Programs { + if i > 0 { + programs.WriteString(",") + } + programs.WriteString(p) + } + } else if len(bucketInfo.Programs) == 1 { + programs.WriteString(bucketInfo.Programs[0]) + } + bucketsByProgram[bucketName] = programs.String() + } + return bucketsByProgram +} + +func ParseUserResp(resp FenceUserResp) map[string]string { + servicesByPath := make(map[string]string) + for path, permissions := range resp.Authz { + var services strings.Builder + seenServices := make(map[string]bool) + for _, p := range permissions { + if !seenServices[p.Method] { + if services.Len() > 0 { + services.WriteString(",") + } + services.WriteString(p.Method) + seenServices[p.Method] = true + } + } + if services.Len() > 0 { + servicesByPath[path] = services.String() + } + } + return servicesByPath +} diff --git a/fence/client_test.go b/fence/client_test.go new file mode 100644 index 0000000..511590e --- /dev/null +++ b/fence/client_test.go @@ -0,0 +1,253 @@ +package fence + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/logs" + "github.com/calypr/calypr-cli/request" +) + +type mockFenceServer struct{} + +func (m *mockFenceServer) handler(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + switch { + case r.Method == http.MethodPost && path == fenceAccessTokenEndpoint: + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]string{"access_token": "new-access-token"}) + return + case r.Method == http.MethodGet && path == fenceUserEndpoint: + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]any{ + "username": "test-user", + "authz": map[string]any{ + "/resource": []map[string]string{ + {"method": "read", "service": "fence"}, + }, + }, + }) + return + case r.Method == http.MethodGet && path == shepherdVersionEndpoint: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`"2.0.0"`)) + return + case r.Method == http.MethodDelete && strings.HasPrefix(path, shepherdEndpoint+"/objects/"): + w.WriteHeader(http.StatusNoContent) + return + case r.Method == http.MethodDelete && strings.HasPrefix(path, fenceDataEndpoint+"/"): + w.WriteHeader(http.StatusNoContent) + return + case r.Method == http.MethodGet && strings.HasSuffix(path, "/download"): + // Shepherd download + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]string{"url": "https://download.url"}) + return + case r.Method == http.MethodGet && strings.Contains(path, fenceDataDownloadEndpoint+"/"): + // Fence download + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(FenceResponse{URL: "https://download.url"}) + return + case r.Method == http.MethodGet && path == "/data/buckets": + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(S3BucketsResponse{ + S3Buckets: map[string]*S3Bucket{ + "test-bucket": { + EndpointURL: "https://s3.amazonaws.com", + Provider: "s3", + Region: "us-east-1", + }, + }, + }) + return + case r.Method == http.MethodPost && path == fenceDataUploadEndpoint: + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(FenceResponse{GUID: "new-guid", URL: "https://upload.url"}) + return + case r.Method == http.MethodGet && strings.HasPrefix(path, fenceDataUploadEndpoint+"/"): + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(FenceResponse{URL: "https://upload.url"}) + return + } + + w.WriteHeader(http.StatusNotFound) + } +} + +func newTestClient(server *httptest.Server) FenceInterface { + cred := &conf.Credential{APIEndpoint: server.URL, Profile: "test", AccessToken: "test-token", APIKey: "test-key"} + logger, _ := logs.New("test") + config := conf.NewConfigure(logger.Logger) + req := request.NewRequestInterface(logger, cred, config) + return NewFenceClient(req, cred, logger.Logger) +} + +func TestFenceClient_NewAccessToken(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + token, err := client.NewAccessToken(context.Background()) + if err != nil { + t.Fatalf("NewAccessToken error: %v", err) + } + if token != "new-access-token" { + t.Errorf("expected token new-access-token, got %s", token) + } +} + +func TestFenceClient_CheckPrivileges(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + privs, err := client.CheckPrivileges(context.Background()) + if err != nil { + t.Fatalf("CheckPrivileges error: %v", err) + } + if _, ok := privs["/resource"]; !ok { + t.Errorf("expected /resource privilege") + } +} + +func TestFenceClient_CheckForShepherdAPI(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + cred := &conf.Credential{ + APIEndpoint: server.URL, + UseShepherd: "true", + } + logger, _ := logs.New("test") + req := request.NewRequestInterface(logger, cred, conf.NewConfigure(logger.Logger)) + client := NewFenceClient(req, cred, logger.Logger) + + hasShepherd, err := client.CheckForShepherdAPI(context.Background()) + if err != nil { + t.Fatalf("CheckForShepherdAPI error: %v", err) + } + if !hasShepherd { + t.Errorf("expected Shepherd to be detected") + } +} + +func TestFenceClient_DeleteRecord(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + // Test Fence fallback (shepherd check returns false or handled by mock behavior) + msg, err := client.DeleteRecord(context.Background(), "guid-1") + if err != nil { + t.Fatalf("DeleteRecord error: %v", err) + } + if !strings.Contains(msg, "has been deleted") { + t.Errorf("unexpected message: %s", msg) + } +} + +func TestFenceClient_GetBucketDetails(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + info, err := client.GetBucketDetails(context.Background(), "test-bucket") + if err != nil { + t.Fatalf("GetBucketDetails error: %v", err) + } + if info.Region != "us-east-1" { + t.Errorf("expected region us-east-1, got %s", info.Region) + } + if info.Provider != "s3" { + t.Errorf("expected provider s3, got %s", info.Provider) + } + + info, err = client.GetBucketDetails(context.Background(), "unknown-bucket") + if err != nil { + t.Fatalf("unexpected error for unknown bucket: %v", err) + } + if info != nil { + t.Errorf("expected nil info for unknown bucket") + } +} + +func TestFenceClient_UploadFlow(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + resp, err := client.InitUpload(context.Background(), "file.txt", "bucket", "") + if err != nil { + t.Fatalf("InitUpload error: %v", err) + } + if resp.URL != "https://upload.url" { + t.Errorf("expected upload URL, got %s", resp.URL) + } + + resp, err = client.GetUploadPresignedUrl(context.Background(), "guid-1", "file.txt", "bucket") + if err != nil { + t.Fatalf("GetUploadPresignedUrl error: %v", err) + } + if resp.URL != "https://upload.url" { + t.Errorf("expected upload URL, got %s", resp.URL) + } +} + +func TestFenceClient_GetDownloadPresignedUrl_Fence(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + url, err := client.GetDownloadPresignedUrl(context.Background(), "guid-1", "") + if err != nil { + t.Fatalf("GetDownloadPresignedUrl error: %v", err) + } + if url != "https://download.url" { + t.Errorf("expected download URL, got %s", url) + } +} + +func TestFenceClient_UserPing(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + resp, err := client.UserPing(context.Background()) + if err != nil { + t.Fatalf("UserPing error: %v", err) + } + + if resp.Username != "test-user" { + t.Errorf("expected username test-user, got %s", resp.Username) + } + + if _, ok := resp.YourAccess["/resource"]; !ok { + t.Errorf("expected /resource access") + } + + if resp.BucketPrograms["test-bucket"] != "" { + // Our mock for /data/buckets returns a bucket but no programs by default unless we update it + // In my update to types.go, I added Programs to S3Bucket. + } +} diff --git a/fence/types.go b/fence/types.go new file mode 100644 index 0000000..ef4956b --- /dev/null +++ b/fence/types.go @@ -0,0 +1,94 @@ +package fence + +// MultipartPart represents a part of a multipart upload +type MultipartPart struct { + PartNumber int `json:"PartNumber"` + ETag string `json:"ETag"` +} + +// FenceResponse represents the standard response from Fence data endpoints +type FenceResponse struct { + URL string `json:"url"` + UploadURL string `json:"upload_url"` // Alias found in some Fence versions + GUID string `json:"guid"` + UploadID string `json:"uploadId"` + PresignedURL string `json:"presigned_url"` + FileName string `json:"file_name"` + URLs []string `json:"urls"` + Size int64 `json:"size"` +} + +// InitRequestObject represents the payload for initializing an upload +type InitRequestObject struct { + Filename string `json:"file_name"` + Bucket string `json:"bucket,omitempty"` + GUID string `json:"guid,omitempty"` +} + +// MultipartUploadRequestObject represents the payload for getting a presigned URL for a part +type MultipartUploadRequestObject struct { + Key string `json:"key"` + UploadID string `json:"uploadId"` + PartNumber int `json:"partNumber"` + Bucket string `json:"bucket,omitempty"` +} + +// MultipartCompleteRequestObject represents the payload for completing a multipart upload +type MultipartCompleteRequestObject struct { + Key string `json:"key"` + UploadID string `json:"uploadId"` + Parts []MultipartPart `json:"parts"` + Bucket string `json:"bucket,omitempty"` +} + +type S3Bucket struct { + EndpointURL string `json:"endpoint_url"` + Provider string `json:"provider,omitempty"` + Programs []string `json:"programs,omitempty"` + Region string `json:"region"` +} + +type S3BucketsResponse struct { + GSBuckets map[string]any `json:"GS_BUCKETS,omitempty"` + S3Buckets map[string]*S3Bucket `json:"S3_BUCKETS,omitempty"` + // Some versions of fence use lowercase + S3BucketsLower map[string]*S3Bucket `json:"s3_buckets,omitempty"` +} + +type UserPermission struct { + Method string `json:"method"` + Service string `json:"service"` +} + +type FenceUserResp struct { + Active bool `json:"active"` + Authz map[string][]UserPermission `json:"authz"` + Azp *string `json:"azp"` + CertificatesUploaded []any `json:"certificates_uploaded"` + DisplayName string `json:"display_name"` + Email string `json:"email"` + Ga4GhPassportV1 []any `json:"ga4gh_passport_v1"` + Groups []any `json:"groups"` + Idp string `json:"idp"` + IsAdmin bool `json:"is_admin"` + Message string `json:"message"` + Name string `json:"name"` + PhoneNumber string `json:"phone_number"` + PreferredUsername string `json:"preferred_username"` + PrimaryGoogleServiceAccount *string `json:"primary_google_service_account"` + ProjectAccess map[string]any `json:"project_access"` + Resources []string `json:"resources"` + ResourcesGranted []any `json:"resources_granted"` + Role string `json:"role"` + Sub string `json:"sub"` + UserID int `json:"user_id"` + Username string `json:"username"` +} + +type PingResp struct { + Profile string `yaml:"profile" json:"profile"` + Username string `yaml:"username" json:"username"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + BucketPrograms map[string]string `yaml:"bucket_programs" json:"bucket_programs"` + YourAccess map[string]string `yaml:"your_access" json:"your_access"` +} diff --git a/g3client/client.go b/g3client/client.go new file mode 100644 index 0000000..5a8b25a --- /dev/null +++ b/g3client/client.go @@ -0,0 +1,274 @@ +package g3client + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/fence" + "github.com/calypr/calypr-cli/gecko" + "github.com/calypr/calypr-cli/logs" + "github.com/calypr/calypr-cli/request" + "github.com/calypr/calypr-cli/requestor" + "github.com/calypr/calypr-cli/sower" + syconfig "github.com/calypr/syfon/client/config" + version "github.com/hashicorp/go-version" +) + +//go:generate go run go.uber.org/mock/mockgen@v0.6.0 -destination=../mocks/mock_gen3interface.go -package=mocks github.com/calypr/calypr-cli/g3client Gen3Interface + +type Gen3Interface interface { + request.RequestInterface + Logger() *logs.Gen3Logger + Credentials() syconfig.CredentialManager + SyfonClient() SyfonClientInterface + FenceClient() fence.FenceInterface + GeckoClient() gecko.GeckoInterface + RequestorClient() requestor.RequestorInterface + SowerClient() sower.SowerInterface +} + +func NewGen3InterfaceFromCredential(cred *conf.Credential, logger *logs.Gen3Logger, opts ...Option) Gen3Interface { + config := conf.NewConfigure(logger.Logger) + reqInterface := request.NewRequestInterface(logger, cred, config) + + client := &Gen3Client{ + config: config, + RequestInterface: reqInterface, + credential: cred, + logger: logger, + } + + for _, opt := range opts { + opt(client) + } + + client.initializeClients() + + return client +} + +func (g *Gen3Client) initializeClients() { + shouldInit := func(ct ClientType) bool { + if len(g.requestedClients) == 0 { + return true + } + for _, c := range g.requestedClients { + if c == ct { + return true + } + } + return false + } + + if shouldInit(FenceClient) { + g.fence = fence.NewFenceClient(g.RequestInterface, g.credential, g.logger.Logger) + } + if shouldInit(SyfonClient) { + g.syfon = buildSyfonClient(g.credential, g.logger, g.RequestInterface) + } + if shouldInit(SowerClient) { + g.sower = sower.NewSowerClient(g.RequestInterface, g.credential.APIEndpoint) + } + if shouldInit(GeckoClient) { + g.gecko = gecko.NewClient(g.RequestInterface, g.credential) + } + if shouldInit(RequestorClient) { + g.requestor = requestor.NewRequestorClient(g.RequestInterface, g.credential) + } +} + +type Gen3Client struct { + Ctx context.Context + fence fence.FenceInterface + syfon SyfonClientInterface + sower sower.SowerInterface + gecko gecko.GeckoInterface + requestor requestor.RequestorInterface + config conf.ManagerInterface + request.RequestInterface + + credential *conf.Credential + creds syconfig.CredentialManager + logger *logs.Gen3Logger + + requestedClients []ClientType +} + +type ClientType string + +const ( + FenceClient ClientType = "fence" + SyfonClient ClientType = "syfon" + SowerClient ClientType = "sower" + GeckoClient ClientType = "gecko" + RequestorClient ClientType = "requestor" +) + +type Option func(*Gen3Client) + +func WithClients(clients ...ClientType) Option { + return func(g *Gen3Client) { + g.requestedClients = clients + } +} + +func (g *Gen3Client) SyfonClient() SyfonClientInterface { + if g.syfon == nil { + g.syfon = buildSyfonClient(g.credential, g.logger, g.RequestInterface) + } + return g.syfon +} + +func (g *Gen3Client) FenceClient() fence.FenceInterface { + return g.fence +} + +func (g *Gen3Client) RequestorClient() requestor.RequestorInterface { + return g.requestor +} + +func (g *Gen3Client) GeckoClient() gecko.GeckoInterface { + return g.gecko +} + +func (g *Gen3Client) SowerClient() sower.SowerInterface { + return g.sower +} + +func (g *Gen3Client) exportCredential(ctx context.Context, cred *conf.Credential) error { + if cred.Profile == "" { + return fmt.Errorf("profile name is required") + } + if cred.APIEndpoint == "" { + return fmt.Errorf("API endpoint is required") + } + + // Normalize endpoint + cred.APIEndpoint = strings.TrimSpace(cred.APIEndpoint) + cred.APIEndpoint = strings.TrimSuffix(cred.APIEndpoint, "/") + + // Validate URL format + parsedURL, err := conf.ValidateUrl(cred.APIEndpoint) + if err != nil { + return fmt.Errorf("invalid apiendpoint URL: %w", err) + } + fenceBase := parsedURL.Scheme + "://" + parsedURL.Host + if _, err := g.config.Load(cred.Profile); err != nil && !errors.Is(err, conf.ErrProfileNotFound) { + return err + } + + if cred.APIKey != "" { + // Always refresh the access token — ignore any old one that might be in the struct + token, err := g.fence.NewAccessToken(ctx) + if err != nil { + if strings.Contains(err.Error(), "401") { + return fmt.Errorf("authentication failed (401) for %s — your API key is invalid, revoked, or expired", fenceBase) + } + if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "no such host") { + return fmt.Errorf("cannot reach Fence at %s — is this a valid Gen3 commons?", fenceBase) + } + return fmt.Errorf("failed to refresh access token: %w", err) + } + g.credential.AccessToken = token + cred.AccessToken = token + } else { + g.logger.Warn("WARNING: Your profile will only be valid for 24 hours since you have only provided a refresh token for authentication") + } + + // Clean up shepherd flags + cred.UseShepherd = strings.TrimSpace(cred.UseShepherd) + cred.MinShepherdVersion = strings.TrimSpace(cred.MinShepherdVersion) + + if cred.MinShepherdVersion != "" { + if _, err = version.NewVersion(cred.MinShepherdVersion); err != nil { + return fmt.Errorf("invalid min-shepherd-version: %w", err) + } + } + + if err := g.config.Save(cred); err != nil { + return fmt.Errorf("failed to write config file: %w", err) + } + + return nil +} + +type gen3Credentials struct { + client *Gen3Client +} + +func (c *gen3Credentials) Current() *conf.Credential { + return c.client.credential +} + +func (c *gen3Credentials) Export(ctx context.Context, cred *conf.Credential) error { + return c.client.exportCredential(ctx, cred) +} + +func (g *Gen3Client) Credentials() syconfig.CredentialManager { + if g.creds == nil { + g.creds = &gen3Credentials{client: g} + } + return g.creds +} + +// EnsureValidCredential checks if the credential is valid and refreshes it if the access token is expired but the API key is valid. +// It accepts an optional fClient; if nil, it will initialize one internally if needed for refresh. +func EnsureValidCredential(ctx context.Context, cred *conf.Credential, config conf.ManagerInterface, logger *logs.Gen3Logger, fClient fence.FenceInterface) error { + if valid, err := config.IsCredentialValid(cred); !valid { + if strings.Contains(err.Error(), "access_token is invalid but api_key is valid") { + // Try to refresh the token + if fClient == nil { + reqInterface := request.NewRequestInterface(logger, cred, config) + fClient = fence.NewFenceClient(reqInterface, cred, logger.Logger) + } + newToken, refreshErr := fClient.NewAccessToken(ctx) + if refreshErr == nil { + cred.AccessToken = newToken + err = config.Save(cred) + if err != nil { + logger.Warn(fmt.Sprintf("Failed to save refreshed token: %v", err)) + } + return nil + } + return fmt.Errorf("failed to refresh access token: %v (original error: %v)", refreshErr, err) + } + return fmt.Errorf("invalid credential: %v", err) + } + return nil +} + +// NewGen3Interface returns a Gen3Client that embeds the credential and implements Gen3Interface. +func NewGen3Interface(profile string, logger *logs.Gen3Logger, opts ...Option) (Gen3Interface, error) { + config := conf.NewConfigure(logger.Logger) + cred, err := config.Load(profile) + if err != nil { + return nil, err + } + + reqInterface := request.NewRequestInterface(logger, cred, config) + + // We need a temporary Fence client to refresh tokens if needed + fClient := fence.NewFenceClient(reqInterface, cred, logger.Logger) + if err := EnsureValidCredential(context.Background(), cred, config, logger, fClient); err != nil { + return nil, err + } + + client := &Gen3Client{ + config: config, + RequestInterface: reqInterface, + credential: cred, + logger: logger, + } + + for _, opt := range opts { + opt(client) + } + + client.initializeClients() + + return client, nil +} +func (g *Gen3Client) Logger() *logs.Gen3Logger { return g.logger } diff --git a/g3client/client_test.go b/g3client/client_test.go new file mode 100644 index 0000000..f1ce5ca --- /dev/null +++ b/g3client/client_test.go @@ -0,0 +1,90 @@ +package g3client + +import ( + "net/http" + "testing" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/logs" + "github.com/calypr/calypr-cli/request" + "github.com/hashicorp/go-retryablehttp" +) + +func TestGen3ClientInitializesSyfonClient(t *testing.T) { + logger, cleanup := logs.New("g3client-test", logs.WithNoConsole(), logs.WithNoMessageFile()) + t.Cleanup(cleanup) + + req := &request.Request{ + RetryClient: &retryablehttp.Client{HTTPClient: &http.Client{}}, + } + cred := &conf.Credential{ + Profile: "test", + APIEndpoint: "https://example.org", + } + client := &Gen3Client{ + RequestInterface: req, + credential: cred, + logger: logger, + requestedClients: []ClientType{SyfonClient}, + } + + client.initializeClients() + + syfon := client.SyfonClient() + if syfon == nil { + t.Fatal("expected syfon client to be initialized") + } + if syfon.Health() == nil || syfon.Data() == nil || syfon.Index() == nil || syfon.DRS() == nil || syfon.Buckets() == nil || syfon.Metrics() == nil || syfon.LFS() == nil { + t.Fatal("expected syfon services to be initialized") + } + + if client.FenceClient() != nil { + t.Fatal("did not expect fence client to be initialized when it was not requested") + } + if client.SowerClient() != nil { + t.Fatal("did not expect sower client to be initialized when it was not requested") + } + if client.GeckoClient() != nil { + t.Fatal("did not expect gecko client to be initialized when it was not requested") + } + if client.RequestorClient() != nil { + t.Fatal("did not expect requestor client to be initialized when it was not requested") + } + + if got := client.Credentials(); got == nil { + t.Fatal("expected credentials manager") + } else if current := got.Current(); current != cred { + t.Fatal("expected credentials manager to return original credential") + } +} + +func TestGen3ClientInitializesGeckoClient(t *testing.T) { + logger, cleanup := logs.New("g3client-test", logs.WithNoConsole(), logs.WithNoMessageFile()) + t.Cleanup(cleanup) + + req := &request.Request{ + RetryClient: &retryablehttp.Client{HTTPClient: &http.Client{}}, + } + cred := &conf.Credential{ + Profile: "test", + APIEndpoint: "https://example.org", + } + client := &Gen3Client{ + RequestInterface: req, + credential: cred, + logger: logger, + requestedClients: []ClientType{GeckoClient}, + } + + client.initializeClients() + + if client.GeckoClient() == nil { + t.Fatal("expected gecko client to be initialized") + } + if client.syfon != nil { + t.Fatal("did not expect syfon client to be initialized when it was not requested") + } + if client.fence != nil { + t.Fatal("did not expect fence client to be initialized when it was not requested") + } +} diff --git a/g3client/syfon_adapter.go b/g3client/syfon_adapter.go new file mode 100644 index 0000000..cc5abad --- /dev/null +++ b/g3client/syfon_adapter.go @@ -0,0 +1,225 @@ +package g3client + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/logs" + "github.com/calypr/calypr-cli/request" + "github.com/calypr/syfon/apigen/client/bucketapi" + "github.com/calypr/syfon/apigen/client/drs" + "github.com/calypr/syfon/apigen/client/internalapi" + "github.com/calypr/syfon/apigen/client/lfsapi" + "github.com/calypr/syfon/apigen/client/metricsapi" + sylogs "github.com/calypr/syfon/client/logs" + syrequest "github.com/calypr/syfon/client/request" + syfonclient "github.com/calypr/syfon/client/services" +) + +// SyfonClientInterface groups the syfon service interfaces that data-client needs. +type SyfonClientInterface interface { + Health() *syfonclient.HealthService + Data() *syfonclient.DataService + Index() *syfonclient.IndexService + DRS() *syfonclient.DRSService + Buckets() *syfonclient.BucketsService + Metrics() *syfonclient.MetricsService + LFS() *syfonclient.LFSService +} + +type syfonClient struct { + health *syfonclient.HealthService + data *syfonclient.DataService + index *syfonclient.IndexService + drs *syfonclient.DRSService + buckets *syfonclient.BucketsService + metrics *syfonclient.MetricsService + lfs *syfonclient.LFSService +} + +func (c *syfonClient) Health() *syfonclient.HealthService { return c.health } +func (c *syfonClient) Data() *syfonclient.DataService { return c.data } +func (c *syfonClient) Index() *syfonclient.IndexService { return c.index } +func (c *syfonClient) DRS() *syfonclient.DRSService { return c.drs } +func (c *syfonClient) Buckets() *syfonclient.BucketsService { return c.buckets } +func (c *syfonClient) Metrics() *syfonclient.MetricsService { return c.metrics } +func (c *syfonClient) LFS() *syfonclient.LFSService { return c.lfs } + +type syfonRequestAdapter struct { + req request.RequestInterface + baseURL string +} + +func (a *syfonRequestAdapter) Do(ctx context.Context, method, path string, body, out any, opts ...syrequest.RequestOption) error { + if a.req == nil { + return fmt.Errorf("request interface is required") + } + + rb := &syrequest.RequestBuilder{ + Method: method, + Url: a.joinURL(path), + Headers: map[string]string{}, + } + for _, opt := range opts { + opt(rb) + } + + if rb.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, rb.Timeout) + defer cancel() + } + + dcRB := a.req.New(method, rb.Url) + for key, value := range rb.Headers { + dcRB.WithHeader(key, value) + } + if rb.SkipAuth { + dcRB.WithSkipAuth(true) + } + if rb.Token != "" { + dcRB.WithToken(rb.Token) + } + if rb.PartSize != 0 { + dcRB.WithPartSize(rb.PartSize) + } + + if body != nil { + if reader, ok := body.(io.Reader); ok { + dcRB.WithBody(reader) + } else { + var err error + dcRB, err = dcRB.WithJSONBody(body) + if err != nil { + return err + } + } + } + + resp, err := a.req.Do(ctx, dcRB) + if err != nil { + return err + } + + switch target := out.(type) { + case **http.Response: + *target = resp + return nil + case *http.Response: + *target = *resp + return nil + } + + defer resp.Body.Close() + data, readErr := io.ReadAll(resp.Body) + if readErr != nil { + return readErr + } + if resp.StatusCode >= http.StatusBadRequest { + return &syrequest.ResponseError{ + Method: method, + URL: resp.Request.URL.String(), + Status: resp.StatusCode, + Body: strings.TrimSpace(string(data)), + Headers: resp.Header.Clone(), + } + } + if out != nil && len(data) > 0 { + if err := json.Unmarshal(data, out); err != nil { + return fmt.Errorf("decode response: %w", err) + } + } + return nil +} + +func (a *syfonRequestAdapter) joinURL(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return a.baseURL + } + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + return path + } + return strings.TrimRight(a.baseURL, "/") + "/" + strings.TrimLeft(path, "/") +} + +func buildSyfonClient(cred *conf.Credential, logger *logs.Gen3Logger, req request.RequestInterface) SyfonClientInterface { + if cred == nil || req == nil { + return nil + } + + baseReq, ok := req.(*request.Request) + if !ok || baseReq.RetryClient == nil { + return nil + } + + httpClient := baseReq.RetryClient.StandardClient() + var slogLogger *slog.Logger + if logger != nil { + slogLogger = logger.Logger + } + syLogger := sylogs.NewGen3Logger(slogLogger, "", "") + syReq := &syfonRequestAdapter{ + req: req, + baseURL: cred.APIEndpoint, + } + + internalClient, err := internalapi.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/"), + internalapi.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + drsClient, err := drs.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/")+"/ga4gh/drs/v1", + drs.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + bucketClient, err := bucketapi.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/"), + bucketapi.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + metricsClient, err := metricsapi.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/"), + metricsapi.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + lfsClient, err := lfsapi.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/"), + lfsapi.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + index := syfonclient.NewIndexService(internalClient, syReq) + drsSvc := syfonclient.NewDRSService(drsClient, index) + dataSvc := syfonclient.NewDataService(internalClient, syReq, syLogger, drsSvc) + return &syfonClient{ + health: syfonclient.NewHealthService(syReq), + data: dataSvc, + index: index, + drs: drsSvc, + buckets: syfonclient.NewBucketsService(bucketClient), + metrics: syfonclient.NewMetricsService(metricsClient), + lfs: syfonclient.NewLFSService(lfsClient), + } +} diff --git a/gecko/client.go b/gecko/client.go new file mode 100644 index 0000000..b53c209 --- /dev/null +++ b/gecko/client.go @@ -0,0 +1,208 @@ +package gecko + +import ( + "context" + "net/http" + "strings" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/githubutil" + "github.com/calypr/calypr-cli/request" +) + +var validateProjectRepoURL = githubutil.ValidateRepositoryURL + +type GeckoInterface interface { + ListConfigTypes(ctx context.Context) ([]ConfigType, error) + ListConfigs(ctx context.Context, configType ConfigType) ([]string, error) + GetConfig(ctx context.Context, configType ConfigType, configID string, out any) error + PutConfig(ctx context.Context, configType ConfigType, configID string, cfg any) (*StatusResponse, error) + DeleteConfig(ctx context.Context, configType ConfigType, configID string) (*StatusResponse, error) + GetAppCard(ctx context.Context, projectID string) (*AppCard, error) + UpsertAppCard(ctx context.Context, projectID string, card AppCard) (*StatusResponse, error) + DeleteAppCard(ctx context.Context, projectID string) (*StatusResponse, error) + HealthCheck(ctx context.Context) (string, error) +} + +type Client struct { + request.RequestInterface + Endpoint string +} + +func NewClient(req request.RequestInterface, creds *conf.Credential) GeckoInterface { + return &Client{ + RequestInterface: req, + Endpoint: creds.APIEndpoint, + } +} + +func (c *Client) ListConfigTypes(ctx context.Context) ([]ConfigType, error) { + var configTypes []ConfigType + if err := request.DoJSON( + ctx, + c.RequestInterface, + c.New(http.MethodGet, request.JoinURL(c.Endpoint, "gecko", "types")), + &configTypes, + request.WithAction("list config types"), + request.WithErrorEnvelope(&ErrorResponse{}), + ); err != nil { + return nil, err + } + return configTypes, nil +} + +func (c *Client) ListConfigs(ctx context.Context, configType ConfigType) ([]string, error) { + var configs []string + if err := request.DoJSON( + ctx, + c.RequestInterface, + c.New(http.MethodGet, c.configListURL(configType)), + &configs, + request.WithAction("list configs"), + request.WithErrorEnvelope(&ErrorResponse{}), + ); err != nil { + return nil, err + } + return configs, nil +} + +func (c *Client) GetConfig(ctx context.Context, configType ConfigType, configID string, out any) error { + return request.DoJSON( + ctx, + c.RequestInterface, + c.New(http.MethodGet, c.configItemURL(configType, configID)), + out, + request.WithAction("get config"), + request.WithErrorEnvelope(&ErrorResponse{}), + ) +} + +func (c *Client) PutConfig(ctx context.Context, configType ConfigType, configID string, cfg any) (*StatusResponse, error) { + rb, err := request.NewJSON(c.RequestInterface, http.MethodPut, c.configItemURL(configType, configID), cfg) + if err != nil { + return nil, err + } + + var status StatusResponse + if err := request.DoJSON( + ctx, + c.RequestInterface, + rb, + &status, + request.WithAction("put config"), + request.WithErrorEnvelope(&ErrorResponse{}), + ); err != nil { + return nil, err + } + return &status, nil +} + +func (c *Client) DeleteConfig(ctx context.Context, configType ConfigType, configID string) (*StatusResponse, error) { + var status StatusResponse + if err := request.DoJSON( + ctx, + c.RequestInterface, + c.New(http.MethodDelete, c.configItemURL(configType, configID)), + &status, + request.WithAction("delete config"), + request.WithErrorEnvelope(&ErrorResponse{}), + ); err != nil { + return nil, err + } + return &status, nil +} + +func (c *Client) GetAppCard(ctx context.Context, projectID string) (*AppCard, error) { + var card AppCard + if err := request.DoJSON( + ctx, + c.RequestInterface, + c.New(http.MethodGet, request.JoinURL(c.Endpoint, "gecko", string(ConfigTypeAppsPage), "appcard", projectID)), + &card, + request.WithAction("get app card"), + request.WithErrorEnvelope(&ErrorResponse{}), + ); err != nil { + return nil, err + } + return &card, nil +} + +func (c *Client) UpsertAppCard(ctx context.Context, projectID string, card AppCard) (*StatusResponse, error) { + rb, err := request.NewJSON( + c.RequestInterface, + http.MethodPost, + request.JoinURL(c.Endpoint, "gecko", string(ConfigTypeAppsPage), "appcard", projectID), + card, + ) + if err != nil { + return nil, err + } + + var status StatusResponse + if err := request.DoJSON( + ctx, + c.RequestInterface, + rb, + &status, + request.WithAction("upsert app card"), + request.WithErrorEnvelope(&ErrorResponse{}), + ); err != nil { + return nil, err + } + return &status, nil +} + +func (c *Client) DeleteAppCard(ctx context.Context, projectID string) (*StatusResponse, error) { + var status StatusResponse + if err := request.DoJSON( + ctx, + c.RequestInterface, + c.New(http.MethodDelete, request.JoinURL(c.Endpoint, "gecko", string(ConfigTypeAppsPage), "appcard", projectID)), + &status, + request.WithAction("delete app card"), + request.WithErrorEnvelope(&ErrorResponse{}), + ); err != nil { + return nil, err + } + return &status, nil +} + +func (c *Client) HealthCheck(ctx context.Context) (string, error) { + var status string + if err := request.DoJSON( + ctx, + c.RequestInterface, + c.New(http.MethodGet, request.JoinURL(c.Endpoint, "gecko", "health")), + &status, + request.WithAction("health check"), + request.WithErrorEnvelope(&ErrorResponse{}), + ); err != nil { + return "", err + } + return status, nil +} + +func (c *Client) configListURL(configType ConfigType) string { + if configType == ConfigTypeProjects { + return request.JoinURL(c.Endpoint, "gecko", "projects", "list") + } + return request.JoinURL(c.Endpoint, "gecko", string(configType), "list") +} + +func (c *Client) configItemURL(configType ConfigType, configID string) string { + if configType == ConfigTypeProjects { + return request.JoinURL(c.Endpoint, append([]string{"gecko", "projects"}, splitConfigID(configID)...)...) + } + return request.JoinURL(c.Endpoint, "gecko", string(configType), configID) +} + +func splitConfigID(configID string) []string { + parts := strings.Split(strings.Trim(configID, "/"), "/") + out := make([]string, 0, len(parts)) + for _, part := range parts { + if trimmed := strings.TrimSpace(part); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} diff --git a/gecko/client_test.go b/gecko/client_test.go new file mode 100644 index 0000000..cd80e24 --- /dev/null +++ b/gecko/client_test.go @@ -0,0 +1,340 @@ +package gecko + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "testing" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/request" + "github.com/hashicorp/go-retryablehttp" +) + +type fakeRequest struct { + doFn func(rb *request.RequestBuilder) (*http.Response, error) +} + +func (f *fakeRequest) New(method, u string) *request.RequestBuilder { + return &request.RequestBuilder{Method: method, Url: u, Headers: map[string]string{}} +} + +func (f *fakeRequest) Do(ctx context.Context, rb *request.RequestBuilder) (*http.Response, error) { + return f.doFn(rb) +} + +func jsonResp(status int, v any) *http.Response { + var body io.ReadCloser = http.NoBody + if v != nil { + buf, _ := json.Marshal(v) + body = io.NopCloser(bytes.NewReader(buf)) + } + return &http.Response{StatusCode: status, Body: body} +} + +func TestGeckoClientExplorerConfigOperations(t *testing.T) { + client := &Client{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + + switch { + case rb.Method == http.MethodGet && u.Path == "/gecko/explorer/list": + return jsonResp(http.StatusOK, []string{"default", "study-a"}), nil + case rb.Method == http.MethodGet && u.Path == "/gecko/explorer/default": + return jsonResp(http.StatusOK, Config{ + ExplorerConfig: []ConfigItem{{TabTitle: "Cases"}}, + }), nil + case rb.Method == http.MethodPut && u.Path == "/gecko/explorer/default": + var payload Config + if err := json.NewDecoder(rb.Body).Decode(&payload); err != nil { + t.Fatalf("decode put payload: %v", err) + } + if len(payload.ExplorerConfig) != 1 || payload.ExplorerConfig[0].TabTitle != "Cases" { + t.Fatalf("unexpected put payload: %+v", payload) + } + return jsonResp(http.StatusOK, StatusResponse{Code: 200, Message: "ACCEPTED"}), nil + case rb.Method == http.MethodDelete && u.Path == "/gecko/explorer/default": + return jsonResp(http.StatusOK, StatusResponse{Code: 200, Message: "DELETED"}), nil + default: + return jsonResp(http.StatusNotFound, ErrorResponse{ + Error: HTTPError{Code: 404, Message: "not found"}, + }), nil + } + }, + }, + Endpoint: "https://example.org", + } + + configs, err := ListExplorerConfigs(context.Background(), client) + if err != nil { + t.Fatalf("ListExplorerConfigs failed: %v", err) + } + if len(configs) != 2 || configs[0] != "default" { + t.Fatalf("unexpected config list: %+v", configs) + } + + cfg, err := GetExplorerConfig(context.Background(), client, "default") + if err != nil { + t.Fatalf("GetExplorerConfig failed: %v", err) + } + if len(cfg.ExplorerConfig) != 1 || cfg.ExplorerConfig[0].TabTitle != "Cases" { + t.Fatalf("unexpected config: %+v", cfg) + } + + putResp, err := PutExplorerConfig(context.Background(), client, "default", *cfg) + if err != nil { + t.Fatalf("PutExplorerConfig failed: %v", err) + } + if putResp.Message != "ACCEPTED" { + t.Fatalf("unexpected put response: %+v", putResp) + } + + deleteResp, err := DeleteExplorerConfig(context.Background(), client, "default") + if err != nil { + t.Fatalf("DeleteExplorerConfig failed: %v", err) + } + if deleteResp.Message != "DELETED" { + t.Fatalf("unexpected delete response: %+v", deleteResp) + } +} + +func TestGeckoClientSupportsAllOfficialConfigTypes(t *testing.T) { + client := &Client{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + switch { + case rb.Method == http.MethodGet && u.Path == "/gecko/types": + return jsonResp(http.StatusOK, []string{"explorer", "nav", "file_summary", "apps_page", "project", "projects"}), nil + case rb.Method == http.MethodGet && u.Path == "/gecko/nav/default": + return jsonResp(http.StatusOK, NavPageLayoutProps{ + HeaderMetadata: HeaderMetadata{Title: "Nav"}, + }), nil + case rb.Method == http.MethodPut && u.Path == "/gecko/apps_page/default": + return jsonResp(http.StatusOK, StatusResponse{Code: 200, Message: "ACCEPTED"}), nil + case rb.Method == http.MethodDelete && u.Path == "/gecko/file_summary/default": + return jsonResp(http.StatusOK, StatusResponse{Code: 200, Message: "DELETED"}), nil + case rb.Method == http.MethodGet && u.Path == "/gecko/projects/HTAN_INT/BForePC": + return jsonResp(http.StatusOK, ProjectConfig{Title: "Project"}), nil + default: + return jsonResp(http.StatusNotFound, ErrorResponse{ + Error: HTTPError{Code: 404, Message: "not found"}, + }), nil + } + }, + }, + Endpoint: "https://example.org", + } + + types, err := client.ListConfigTypes(context.Background()) + if err != nil { + t.Fatalf("ListConfigTypes failed: %v", err) + } + if len(types) != 6 || types[0] != ConfigTypeExplorer || types[3] != ConfigTypeAppsPage || types[4] != ConfigTypeProject || types[5] != ConfigTypeProjects { + t.Fatalf("unexpected config types: %+v", types) + } + + var nav NavPageLayoutProps + if err := client.GetConfig(context.Background(), ConfigTypeNav, "default", &nav); err != nil { + t.Fatalf("GetConfig(nav) failed: %v", err) + } + if nav.HeaderMetadata.Title != "Nav" { + t.Fatalf("unexpected nav config: %+v", nav) + } + + if _, err := client.PutConfig(context.Background(), ConfigTypeAppsPage, "default", AppsConfig{}); err != nil { + t.Fatalf("PutConfig(apps_page) failed: %v", err) + } + + if _, err := client.DeleteConfig(context.Background(), ConfigTypeFileSummary, "default"); err != nil { + t.Fatalf("DeleteConfig(file_summary) failed: %v", err) + } + + project, err := GetTypedConfig[ProjectConfig](context.Background(), client, ConfigTypeProjects, "HTAN_INT/BForePC") + if err != nil { + t.Fatalf("GetTypedConfig(project) failed: %v", err) + } + if project.Title != "Project" { + t.Fatalf("unexpected project config: %+v", project) + } +} + +func TestGeckoClientAppCardAndHealthOperations(t *testing.T) { + client := &Client{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + + switch { + case rb.Method == http.MethodGet && u.Path == "/gecko/apps_page/appcard/PROG-PROJ": + return jsonResp(http.StatusOK, AppCard{ + Title: "Portal", + Description: "Project portal", + Icon: "beaker", + Href: "/portal", + Perms: "PROG-PROJ", + }), nil + case rb.Method == http.MethodPost && u.Path == "/gecko/apps_page/appcard/PROG-PROJ": + var payload AppCard + if err := json.NewDecoder(rb.Body).Decode(&payload); err != nil { + t.Fatalf("decode app card payload: %v", err) + } + if payload.Perms != "PROG-PROJ" || payload.Title != "Portal" { + t.Fatalf("unexpected app card payload: %+v", payload) + } + return jsonResp(http.StatusOK, StatusResponse{Code: 200, Message: "ACCEPTED"}), nil + case rb.Method == http.MethodDelete && u.Path == "/gecko/apps_page/appcard/PROG-PROJ": + return jsonResp(http.StatusOK, StatusResponse{Code: 200, Message: "DELETED"}), nil + case rb.Method == http.MethodGet && u.Path == "/gecko/health": + return jsonResp(http.StatusOK, "Healthy"), nil + default: + return jsonResp(http.StatusNotFound, ErrorResponse{ + Error: HTTPError{Type: ErrorTypeNotFound, Code: 404, Message: "not found"}, + }), nil + } + }, + }, + Endpoint: "https://example.org", + } + + card, err := GetAppCard(context.Background(), client, "PROG-PROJ") + if err != nil { + t.Fatalf("GetAppCard failed: %v", err) + } + if card.Perms != "PROG-PROJ" || card.Title != "Portal" { + t.Fatalf("unexpected app card: %+v", card) + } + + upsertResp, err := UpsertAppCard(context.Background(), client, "PROG-PROJ", *card) + if err != nil { + t.Fatalf("UpsertAppCard failed: %v", err) + } + if upsertResp.Message != "ACCEPTED" { + t.Fatalf("unexpected upsert response: %+v", upsertResp) + } + + deleteResp, err := DeleteAppCard(context.Background(), client, "PROG-PROJ") + if err != nil { + t.Fatalf("DeleteAppCard failed: %v", err) + } + if deleteResp.Message != "DELETED" { + t.Fatalf("unexpected delete response: %+v", deleteResp) + } + + health, err := client.HealthCheck(context.Background()) + if err != nil { + t.Fatalf("HealthCheck failed: %v", err) + } + if health != "Healthy" { + t.Fatalf("unexpected health response: %q", health) + } +} + +func TestGeckoClientPutProjectConfigValidatesRepository(t *testing.T) { + oldValidator := validateProjectRepoURL + validateProjectRepoURL = func(ctx context.Context, raw string, token string) (string, error) { + if raw != "https://github.com/calypr/gecko" { + t.Fatalf("unexpected repo URL passed to validator: %q", raw) + } + return "github.com/calypr/gecko", nil + } + t.Cleanup(func() { + validateProjectRepoURL = oldValidator + }) + + client := &Client{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + if rb.Method != http.MethodPut || u.Path != "/gecko/projects/Org/Project" { + return jsonResp(http.StatusNotFound, nil), nil + } + var payload ProjectConfig + if err := json.NewDecoder(rb.Body).Decode(&payload); err != nil { + t.Fatalf("decode project payload: %v", err) + } + if payload.SrcRepo != "github.com/calypr/gecko" { + t.Fatalf("expected normalized repo, got %q", payload.SrcRepo) + } + return jsonResp(http.StatusOK, StatusResponse{Code: 200, Message: "ACCEPTED"}), nil + }, + }, + Endpoint: "https://example.org", + } + + if _, err := PutProjectConfig(context.Background(), client, "Org/Project", ProjectConfig{ + Title: "Project", + ContactEmail: "user@example.org", + SrcRepo: "https://github.com/calypr/gecko", + OrgTitle: "Org", + Description: "Desc", + ProjectTitle: "Project", + IconName: "beaker", + }); err != nil { + t.Fatalf("PutProjectConfig failed: %v", err) + } +} + +func TestGeckoClientReturnsStructuredErrors(t *testing.T) { + client := &Client{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + return jsonResp(http.StatusNotFound, ErrorResponse{ + Error: HTTPError{ + Type: ErrorTypeConfigNotFound, + Code: 404, + Message: "no config found with configId: missing of type: explorer", + Details: map[string]any{"config_type": "explorer", "config_id": "missing"}, + }, + }), nil + }, + }, + Endpoint: "https://example.org", + } + + _, err := GetExplorerConfig(context.Background(), client, "missing") + if err == nil || err.Error() != "get config: status 404: no config found with configId: missing of type: explorer" { + t.Fatalf("unexpected error: %v", err) + } + + var statusErr *request.HTTPStatusError + if !errors.As(err, &statusErr) { + t.Fatalf("expected request.HTTPStatusError, got %T", err) + } + if statusErr.Type != string(ErrorTypeConfigNotFound) { + t.Fatalf("unexpected error type: %q", statusErr.Type) + } + if statusErr.Details["config_id"] != "missing" { + t.Fatalf("unexpected error details: %+v", statusErr.Details) + } +} + +func TestNewClientBuildsEndpoint(t *testing.T) { + req := &request.Request{ + RetryClient: &retryablehttp.Client{HTTPClient: &http.Client{}}, + } + client := NewClient(req, &conf.Credential{APIEndpoint: "https://example.org/base"}).(*Client) + if got := client.configListURL(ConfigTypeExplorer); got != "https://example.org/base/gecko/explorer/list" { + t.Fatalf("unexpected full URL: %s", got) + } + if got := client.configItemURL(ConfigTypeProjects, "Org/Project"); got != "https://example.org/base/gecko/projects/Org/Project" { + t.Fatalf("unexpected project URL: %s", got) + } +} diff --git a/gecko/helpers.go b/gecko/helpers.go new file mode 100644 index 0000000..b8fb498 --- /dev/null +++ b/gecko/helpers.go @@ -0,0 +1,68 @@ +package gecko + +import "context" + +func GetTypedConfig[T any](ctx context.Context, c GeckoInterface, configType ConfigType, configID string) (*T, error) { + var cfg T + if err := c.GetConfig(ctx, configType, configID, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +func PutTypedConfig[T any](ctx context.Context, c GeckoInterface, configType ConfigType, configID string, cfg T) (*StatusResponse, error) { + return c.PutConfig(ctx, configType, configID, cfg) +} + +func DeleteTypedConfig(ctx context.Context, c GeckoInterface, configType ConfigType, configID string) (*StatusResponse, error) { + return c.DeleteConfig(ctx, configType, configID) +} + +func ListExplorerConfigs(ctx context.Context, c GeckoInterface) ([]string, error) { + return c.ListConfigs(ctx, ConfigTypeExplorer) +} + +func GetExplorerConfig(ctx context.Context, c GeckoInterface, configID string) (*Config, error) { + return GetTypedConfig[Config](ctx, c, ConfigTypeExplorer, configID) +} + +func PutExplorerConfig(ctx context.Context, c GeckoInterface, configID string, cfg Config) (*StatusResponse, error) { + return PutTypedConfig(ctx, c, ConfigTypeExplorer, configID, cfg) +} + +func DeleteExplorerConfig(ctx context.Context, c GeckoInterface, configID string) (*StatusResponse, error) { + return DeleteTypedConfig(ctx, c, ConfigTypeExplorer, configID) +} + +func GetProjectConfig(ctx context.Context, c GeckoInterface, configID string) (*ProjectConfig, error) { + return GetTypedConfig[ProjectConfig](ctx, c, ConfigTypeProjects, configID) +} + +func PutProjectConfig(ctx context.Context, c GeckoInterface, configID string, cfg ProjectConfig) (*StatusResponse, error) { + normalized, err := validateProjectRepoURL(ctx, cfg.SrcRepo, "") + if err != nil { + return nil, err + } + cfg.SrcRepo = normalized + return PutTypedConfig(ctx, c, ConfigTypeProjects, configID, cfg) +} + +func DeleteProjectConfig(ctx context.Context, c GeckoInterface, configID string) (*StatusResponse, error) { + return DeleteTypedConfig(ctx, c, ConfigTypeProjects, configID) +} + +func GetAppCard(ctx context.Context, c GeckoInterface, projectID string) (*AppCard, error) { + return c.GetAppCard(ctx, projectID) +} + +func UpsertAppCard(ctx context.Context, c GeckoInterface, projectID string, card AppCard) (*StatusResponse, error) { + return c.UpsertAppCard(ctx, projectID, card) +} + +func DeleteAppCard(ctx context.Context, c GeckoInterface, projectID string) (*StatusResponse, error) { + return c.DeleteAppCard(ctx, projectID) +} + +func HealthCheck(ctx context.Context, c GeckoInterface) (string, error) { + return c.HealthCheck(ctx) +} diff --git a/gecko/types.go b/gecko/types.go new file mode 100644 index 0000000..b66bf41 --- /dev/null +++ b/gecko/types.go @@ -0,0 +1,370 @@ +package gecko + +import "encoding/json" + +type FieldConfig struct { + Field string `json:"field,omitempty"` + DataField string `json:"dataField,omitempty"` + Index string `json:"index,omitempty"` + Label string `json:"label"` + Type string `json:"type,omitempty"` +} + +type FilterTab struct { + Title string `json:"title,omitempty"` + Fields []string `json:"fields"` + FieldsConfig map[string]FieldConfig `json:"fieldsConfig,omitempty"` +} + +type FiltersConfig struct { + Tabs []FilterTab `json:"tabs"` +} + +type TableConfig struct { + Enabled bool `json:"enabled"` + Fields []string `json:"fields"` + Columns map[string]TableColumnsConfig `json:"columns,omitempty"` + DetailsConfig TableDetailsConfig `json:"detailsConfig"` +} + +type SummaryTableColumnType string + +const ( + SummaryTableColumnTypeString SummaryTableColumnType = "string" + SummaryTableColumnTypeNumber SummaryTableColumnType = "number" + SummaryTableColumnTypeDate SummaryTableColumnType = "date" + SummaryTableColumnTypeArray SummaryTableColumnType = "array" + SummaryTableColumnTypeLink SummaryTableColumnType = "link" + SummaryTableColumnTypeBoolean SummaryTableColumnType = "boolean" + SummaryTableColumnTypeParagraphs SummaryTableColumnType = "paragraphs" +) + +type TableColumnsConfig struct { + Field string `json:"field"` + Title string `json:"title"` + AccessorPath string `json:"accessorPath,omitempty"` + Type SummaryTableColumnType `json:"type,omitempty"` + CellRenderFunction string `json:"cellRenderFunction,omitempty"` + Params map[string]any `json:"params,omitempty"` + Width string `json:"width,omitempty"` + Sortable bool `json:"sortable,omitempty"` + Visable bool `json:"visable,omitempty"` +} + +func (t TableColumnsConfig) MarshalJSON() ([]byte, error) { + type alias TableColumnsConfig + aux := struct { + *alias + Type *SummaryTableColumnType `json:"type,omitempty"` + }{ + alias: (*alias)(&t), + } + if t.Type != "" { + aux.Type = &t.Type + } + return json.Marshal(aux) +} + +type TableDetailsConfig struct { + Panel string `json:"panel,omitempty"` + Mode string `json:"mode,omitempty"` + IDField string `json:"idField,omitempty"` + FilterField string `json:"filterField,omitempty"` + Title string `json:"title,omitempty"` + NodeType string `json:"nodeType,omitempty"` + NodeFields map[string]string `json:"nodeFields,omitempty"` +} + +type GuppyConfig struct { + DataType string `json:"dataType"` + NodeCountTitle string `json:"nodeCountTitle"` + FieldMapping []GuppyFieldMapping `json:"fieldMapping,omitempty"` + AccessibleFieldCheckList []string `json:"accessibleFieldCheckList,omitempty"` + AccessibleValidationField string `json:"accessibleValidationField,omitempty"` + ManifestMapping ManifestMapping `json:"manifestMapping"` +} + +type GuppyFieldMapping struct { + Field string `json:"field,omitempty"` + Name string `json:"name,omitempty"` +} + +type ManifestMapping struct { + ResourceIndexType string `json:"resourceIndexType,omitempty"` + ResourceIdField string `json:"resourceIdField,omitempty"` + ReferenceIdFieldInResourceIndex string `json:"referenceIdFieldInResourceIndex,omitempty"` + ReferenceIdFieldInDataIndex string `json:"referenceIdFieldInDataIndex,omitempty"` +} + +type Chart struct { + ChartType string `json:"chartType"` + Title string `json:"title"` +} + +type ButtonConfig struct { + Enabled bool `json:"enabled,omitempty"` + Type string `json:"type,omitempty"` + Action string `json:"action,omitempty"` + Title string `json:"title,omitempty"` + LeftIcon string `json:"leftIcon,omitempty"` + RightIcon string `json:"rightIcon,omitempty"` + FileName string `json:"fileName,omitempty"` + ActionArgs ButtonActionArgs `json:"actionArgs"` +} + +type ButtonActionArgs struct { + ResourceIndexType string `json:"resourceIndexType,omitempty"` + ResourceIdField string `json:"resourceIdField,omitempty"` + ReferenceIdFieldInDataIndex string `json:"referenceIdFieldInDataIndex,omitempty"` + ReferenceIdFieldInResourceIndex string `json:"referenceIdFieldInResourceIndex,omitempty"` + FileFields []string `json:"fileFields,omitempty"` +} + +type ConfigItem struct { + TabTitle string `json:"tabTitle"` + GuppyConfig GuppyConfig `json:"guppyConfig"` + Charts map[string]Chart `json:"charts,omitempty"` + Filters FiltersConfig `json:"filters"` + Table TableConfig `json:"table"` + Dropdowns map[string]any `json:"dropdowns,omitempty"` + Buttons []ButtonConfig `json:"buttons,omitempty"` + LoginForDownload bool `json:"loginForDownload,omitempty"` + PreFilters map[string]any `json:"preFilters,omitempty"` +} + +type Config struct { + SharedFilters SharedFiltersConfig `json:"sharedFilters"` + ExplorerConfig []ConfigItem `json:"explorerConfig"` + FileActions FileActionsConfig `json:"fileActions,omitempty"` +} + +type FileActionsConfig map[string][]string + +type SharedFiltersConfig struct { + SharedFilter map[string][]FilterPair `json:"defined"` +} + +type FilterPair struct { + Index string `json:"index"` + Field string `json:"field"` +} + +type ErrorResponse struct { + Error HTTPError `json:"error"` +} + +func (e ErrorResponse) ErrorMessage() string { + return e.Error.Message +} + +func (e ErrorResponse) ErrorType() string { + return string(e.Error.Type) +} + +func (e ErrorResponse) ErrorDetails() map[string]any { + return e.Error.Details +} + +type HTTPError struct { + Type ErrorType `json:"type,omitempty"` + Message string `json:"message"` + Code int `json:"code"` + Details map[string]any `json:"details,omitempty"` +} + +type StatusResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type ConfigType string + +const ( + ConfigTypeExplorer ConfigType = "explorer" + ConfigTypeNav ConfigType = "nav" + ConfigTypeFileSummary ConfigType = "file_summary" + ConfigTypeAppsPage ConfigType = "apps_page" + ConfigTypeProject ConfigType = "project" + ConfigTypeProjects ConfigType = "projects" +) + +type ErrorType string + +const ( + ErrorTypeNotFound ErrorType = "not_found" + ErrorTypeUnauthorized ErrorType = "unauthorized" + ErrorTypeMethodNotAllowed ErrorType = "method_not_allowed" + ErrorTypeMissingAuthorization ErrorType = "missing_authorization" + ErrorTypeForbidden ErrorType = "forbidden" + ErrorTypeInvalidConfigType ErrorType = "invalid_config_type" + ErrorTypeConfigNotFound ErrorType = "config_not_found" + ErrorTypeInvalidJSON ErrorType = "invalid_json" + ErrorTypeEmptyRequestBody ErrorType = "empty_request_body" + ErrorTypeInvalidRequestBody ErrorType = "invalid_request_body" + ErrorTypeValidationFailed ErrorType = "validation_failed" + ErrorTypeMissingProjectID ErrorType = "missing_project_id" + ErrorTypeInvalidProjectID ErrorType = "invalid_project_id" + ErrorTypeProjectIDMismatch ErrorType = "project_id_mismatch" + ErrorTypeInvalidDirectory ErrorType = "invalid_directory" + ErrorTypeInvalidUUID ErrorType = "invalid_uuid" + ErrorTypeMissingIdentifier ErrorType = "missing_identifier" + ErrorTypeInvalidQueryParameter ErrorType = "invalid_query_parameter" + ErrorTypeInvalidDistance ErrorType = "invalid_distance" + ErrorTypeInvalidVectorRequest ErrorType = "invalid_vector_request" + ErrorTypeInvalidPointData ErrorType = "invalid_point_data" + ErrorTypePointNotFound ErrorType = "point_not_found" + ErrorTypeVectorCollectionNotFound ErrorType = "vector_collection_not_found" + ErrorTypeVectorCollectionAlreadyExists ErrorType = "vector_collection_already_exists" + ErrorTypeVectorStoreUnavailable ErrorType = "vector_store_unavailable" + ErrorTypeVectorOperationFailed ErrorType = "vector_operation_failed" + ErrorTypeDatabaseError ErrorType = "database_error" + ErrorTypeDatabaseUnavailable ErrorType = "database_unavailable" + ErrorTypeGraphQueryFailed ErrorType = "graph_query_failed" + ErrorTypeInvalidJWTHandler ErrorType = "invalid_jwt_handler" + ErrorTypeInvalidAuthorizationResponse ErrorType = "invalid_authorization_response" + ErrorTypeAuthorizationServiceError ErrorType = "authorization_service_error" + ErrorTypeAppCardNotFound ErrorType = "app_card_not_found" +) + +func KnownConfigTypes() []ConfigType { + return []ConfigType{ + ConfigTypeExplorer, + ConfigTypeNav, + ConfigTypeFileSummary, + ConfigTypeAppsPage, + ConfigTypeProject, + ConfigTypeProjects, + } +} + +type StylingOverrideWithMergeControl struct { + MergeStrategy string `json:"mergeStrategy,omitempty"` + Root string `json:"root,omitempty"` +} + +type NavigationButtonProps struct { + Icon string `json:"icon"` + Tooltip string `json:"tooltip"` + Href string `json:"href"` + NoBasePath *bool `json:"noBasePath,omitempty"` + Name string `json:"name"` + IconHeight string `json:"iconHeight,omitempty"` + Title string `json:"title,omitempty"` + ClassNames *StylingOverrideWithMergeControl `json:"classNames,omitempty"` +} + +type NavigationBarLogo struct { + Src string `json:"src"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + Width float64 `json:"width,omitempty"` + Height float64 `json:"height,omitempty"` + NoBasePath *bool `json:"noBasePath,omitempty"` + Divider *bool `json:"divider,omitempty"` + BasePath string `json:"basePath,omitempty"` + Href string `json:"href"` + OnToggle json.RawMessage `json:"onToggle,omitempty"` + Basepage *bool `json:"basepage,omitempty"` + ClassNames *StylingOverrideWithMergeControl `json:"classNames,omitempty"` +} + +type NavigationProps struct { + Logo *NavigationBarLogo `json:"logo,omitempty"` + Items []NavigationButtonProps `json:"items"` + Title string `json:"title,omitempty"` + LoginIcon json.RawMessage `json:"loginIcon,omitempty"` + ClassNames *StylingOverrideWithMergeControl `json:"classNames"` +} + +type LeftNavBarProps struct { + Title string `json:"title"` + Description string `json:"description"` + Icon string `json:"icon"` + Href string `json:"href"` + Perms any `json:"perms"` +} + +type TopBarItem struct { + ClassNames struct { + Button string `json:"button"` + Label string `json:"label"` + Root string `json:"root"` + } `json:"classNames,omitempty"` + Href string `json:"href,omitempty"` + Name string `json:"name,omitempty"` +} + +type TopBarProps struct { + Items []TopBarItem `json:"items,omitempty"` + LoginButtonVisibility string `json:"loginButtonVisibility,omitempty"` +} + +type HeaderProps struct { + Top TopBarProps `json:"topBar"` + Navigation NavigationProps `json:"navigation"` + LeftNav []LeftNavBarProps `json:"leftnav"` + BasePage bool `json:"basePage,omitempty"` +} + +type FooterColumnLink struct { + Label string `json:"label,omitempty"` + Href string `json:"href,omitempty"` +} + +type FooterColumn struct { + Title string `json:"title,omitempty"` + Links []FooterColumnLink `json:"links,omitempty"` +} + +type FooterRightSection struct { + Columns []FooterColumn `json:"columns,omitempty"` +} + +type FooterProps struct { + RightSection *FooterRightSection `json:"rightSection,omitempty"` + BottomLinks []FooterColumnLink `json:"bottomLinks,omitempty"` + ColumnLinks []FooterColumnLink `json:"columnLinks,omitempty"` +} + +type HeaderMetadata struct { + Title string `json:"title"` + Content string `json:"content"` + Key string `json:"key"` +} + +type NavPageLayoutProps struct { + HeaderProps HeaderProps `json:"headerProps"` + FooterProps FooterProps `json:"footerProps"` + HeaderMetadata HeaderMetadata `json:"headerMetadata"` +} + +type FilesummaryConfig struct { + Config map[string]TableColumnsConfig `json:"config"` + BarChartColor string `json:"barChartColor"` + DefaultProject string `json:"defaultProject"` + BinslicePoints []int `json:"binslicePoints"` + IdField string `json:"idField"` + Index string `json:"index"` +} + +type AppCard struct { + Title string `json:"title"` + Description string `json:"description"` + Icon string `json:"icon"` + Href string `json:"href"` + Perms string `json:"perms"` +} + +type AppsConfig struct { + AppCards []AppCard `json:"appCards"` +} + +type ProjectConfig struct { + Title string `json:"title"` + ContactEmail string `json:"contact_email"` + SrcRepo string `json:"src_repo"` + OrgTitle string `json:"org_title"` + Description string `json:"description"` + ProjectTitle string `json:"project_title"` + IconName string `json:"icon_name"` +} diff --git a/githubutil/repository.go b/githubutil/repository.go new file mode 100644 index 0000000..254570a --- /dev/null +++ b/githubutil/repository.go @@ -0,0 +1,130 @@ +package githubutil + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +var githubSSHHosts = map[string]struct{}{ + "ssh.github.com": {}, + "altssh.github.com": {}, +} + +func NormalizeRepositoryURL(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", fmt.Errorf("repository URL is required") + } + + host, path, err := splitRepositoryURL(raw) + if err != nil { + return "", err + } + + host = strings.ToLower(strings.TrimSpace(host)) + path = strings.Trim(path, "/") + path = strings.TrimSuffix(path, ".git") + + parts := strings.Split(path, "/") + if _, ok := githubSSHHosts[host]; ok { + host = "github.com" + if len(parts) == 3 && parts[0] == "443" { + parts = parts[1:] + } + } + if len(parts) != 2 { + return "", fmt.Errorf("repository URL must point to a GitHub-style owner/repo path") + } + if parts[0] == "" || parts[1] == "" { + return "", fmt.Errorf("repository URL must include both owner and repo") + } + + return host + "/" + parts[0] + "/" + parts[1], nil +} + +func ValidateRepositoryURL(ctx context.Context, raw string, token string) (string, error) { + normalized, err := NormalizeRepositoryURL(raw) + if err != nil { + return "", err + } + if err := ValidateNormalizedRepositoryURL(ctx, normalized, token); err != nil { + return "", err + } + return normalized, nil +} + +func ValidateNormalizedRepositoryURL(ctx context.Context, normalized string, token string) error { + validationURL := "https://" + normalized + ".git/info/refs?service=git-upload-pack" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, validationURL, nil) + if err != nil { + return fmt.Errorf("failed to create validation request: %w", err) + } + req.Header.Set("User-Agent", "git/2.43.0") + if token != "" { + req.SetBasicAuth("x-access-token", token) + } + + client := &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + }, + } + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to reach repository at %s: %w", validationURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return nil + } + + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + return fmt.Errorf("repository exists but is not accessible with the provided credentials") + } + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("repository does not exist: %s", normalized) + } + return fmt.Errorf("repository validation failed for %s: status %s", normalized, resp.Status) +} + +func splitRepositoryURL(raw string) (host string, path string, err error) { + if strings.Contains(raw, "://") { + parsed, parseErr := url.Parse(raw) + if parseErr != nil { + return "", "", fmt.Errorf("invalid repository URL: %w", parseErr) + } + if parsed.Host == "" { + return "", "", fmt.Errorf("repository URL host is required") + } + return parsed.Hostname(), parsed.EscapedPath(), nil + } + + if strings.Contains(raw, "@") && strings.Contains(raw, ":") { + atIdx := strings.LastIndex(raw, "@") + colonIdx := strings.Index(raw[atIdx+1:], ":") + if colonIdx >= 0 { + host := raw[atIdx+1 : atIdx+1+colonIdx] + path := raw[atIdx+1+colonIdx+1:] + return host, path, nil + } + } + + parts := strings.Split(strings.Trim(raw, "/"), "/") + if len(parts) >= 3 { + return parts[0], strings.Join(parts[1:], "/"), nil + } + + return "", "", fmt.Errorf("invalid repository URL: %s", raw) +} diff --git a/githubutil/repository_test.go b/githubutil/repository_test.go new file mode 100644 index 0000000..61f5c63 --- /dev/null +++ b/githubutil/repository_test.go @@ -0,0 +1,42 @@ +package githubutil + +import "testing" + +func TestNormalizeRepositoryURL(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"https://github.com/calypr/gecko", "github.com/calypr/gecko"}, + {"https://github.com/calypr/gecko.git", "github.com/calypr/gecko"}, + {"git@github.com:calypr/gecko.git", "github.com/calypr/gecko"}, + {"github.com/calypr/gecko", "github.com/calypr/gecko"}, + {"ssh://git@ssh.github.com:443/calypr/gecko.git", "github.com/calypr/gecko"}, + {"source.ohsu.edu/calypr/gecko", "source.ohsu.edu/calypr/gecko"}, + } + + for _, tt := range tests { + got, err := NormalizeRepositoryURL(tt.in) + if err != nil { + t.Fatalf("NormalizeRepositoryURL(%q) returned error: %v", tt.in, err) + } + if got != tt.want { + t.Fatalf("NormalizeRepositoryURL(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestNormalizeRepositoryURLRejectsInvalidPaths(t *testing.T) { + bad := []string{ + "", + "https://github.com/calypr", + "github.com/calypr", + "https://github.com/calypr/gecko/extra", + } + + for _, input := range bad { + if _, err := NormalizeRepositoryURL(input); err == nil { + t.Fatalf("expected NormalizeRepositoryURL(%q) to fail", input) + } + } +} diff --git a/go.mod b/go.mod index 6515b39..17d53d7 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,49 @@ -module github.com/calypr/data-client +module github.com/calypr/calypr-cli -go 1.24.2 +go 1.26.2 require ( - github.com/golang-jwt/jwt/v5 v5.3.0 - github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/go-version v1.8.0 + github.com/calypr/syfon/apigen v0.2.6-0.20260503003649-cda722e27216 + github.com/calypr/syfon/client v0.2.7-0.20260503003649-cda722e27216 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/hashicorp/go-retryablehttp v0.7.8 + github.com/hashicorp/go-version v1.9.0 github.com/spf13/cobra v1.10.2 - github.com/vbauerster/mpb/v8 v8.11.2 go.uber.org/mock v0.6.0 - golang.org/x/mod v0.31.0 - gopkg.in/ini.v1 v1.67.0 + gopkg.in/ini.v1 v1.67.1 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/clipperhouse/stringish v0.1.1 // indirect - github.com/clipperhouse/uax29/v2 v2.3.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/calypr/syfon v0.2.8-0.20260503003649-cda722e27216 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/gofiber/fiber/v3 v3.1.0 // indirect + github.com/gofiber/schema v1.7.0 // indirect + github.com/gofiber/utils/v2 v2.0.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.21 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/oapi-codegen/runtime v1.4.0 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/spf13/pflag v1.0.10 // indirect - golang.org/x/sys v0.39.0 // indirect + github.com/tinylib/msgp v1.6.3 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.69.0 // indirect + github.com/vbauerster/mpb/v8 v8.12.0 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index bae303b..714a133 100644 --- a/go.sum +++ b/go.sum @@ -1,48 +1,127 @@ +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/calypr/syfon v0.2.8-0.20260503003649-cda722e27216 h1:f/fr9r4N3yKARlX+RPiZmg6fJG3xuSLpeTvk6as6AxM= +github.com/calypr/syfon v0.2.8-0.20260503003649-cda722e27216/go.mod h1:m2jd8Snb+Gjc32AcOTdioZVjLmhsAXw7F4uzHQTGBtg= +github.com/calypr/syfon/apigen v0.2.6-0.20260503003649-cda722e27216 h1:MuPHiYQXGX7frvN3EQZ4ZM5RjRZ1eGVY0D4iXCNAj0s= +github.com/calypr/syfon/apigen v0.2.6-0.20260503003649-cda722e27216/go.mod h1:9JNwTgR57yKJlWqZpdqP+/l4zCNzH1EIFrW+e20PyMQ= +github.com/calypr/syfon/client v0.2.7-0.20260503003649-cda722e27216 h1:1LFo3dMc4ZQHml9zQsBE5/k2ZEcHT4EpqxL+Hr/ROjo= +github.com/calypr/syfon/client v0.2.7-0.20260503003649-cda722e27216/go.mod h1:DQSqNkxl9V3w08BiMconcJh3xtc+/Je7Xeo7qRH7wto= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= -github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gofiber/fiber/v3 v3.1.0 h1:1p4I820pIa+FGxfwWuQZ5rAyX0WlGZbGT6Hnuxt6hKY= +github.com/gofiber/fiber/v3 v3.1.0/go.mod h1:n2nYQovvL9z3Too/FGOfgtERjW3GQcAUqgfoezGBZdU= +github.com/gofiber/schema v1.7.0 h1:yNM+FNRZjyYEli9Ey0AXRBrAY9jTnb+kmGs3lJGPvKg= +github.com/gofiber/schema v1.7.0/go.mod h1:A/X5Ffyru4p9eBdp99qu+nzviHzQiZ7odLT+TwxWhbk= +github.com/gofiber/utils/v2 v2.0.2 h1:ShRRssz0F3AhTlAQcuEj54OEDtWF7+HJDwEi/aa6QLI= +github.com/gofiber/utils/v2 v2.0.2/go.mod h1:+9Ub4NqQ+IaJoTliq5LfdmOJAA/Hzwf4pXOxOa3RrJ0= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= +github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/oapi-codegen/runtime v1.4.0 h1:KLOSFOp7UzkbS7Cs1ms6NBEKYr0WmH2wZG0KKbd2er4= +github.com/oapi-codegen/runtime v1.4.0/go.mod h1:5sw5fxCDmnOzKNYmkVNF8d34kyUeejJEY8HNT2WaPec= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shamaton/msgpack/v3 v3.1.0 h1:jsk0vEAqVvvS9+fTZ5/EcQ9tz860c9pWxJ4Iwecz8gU= +github.com/shamaton/msgpack/v3 v3.1.0/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/vbauerster/mpb/v8 v8.11.2 h1:OqLoHznUVU7SKS/WV+1dB5/hm20YLheYupiHhL5+M1Y= -github.com/vbauerster/mpb/v8 v8.11.2/go.mod h1:mEB/M353al1a7wMUNtiymmPsEkGlJgeJmtlbY5adCJ8= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= +github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI= +github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw= +github.com/vbauerster/mpb/v8 v8.12.0 h1:+gneY3ifzc88tKDzOtfG8k8gfngCx615S2ZmFM4liWg= +github.com/vbauerster/mpb/v8 v8.12.0/go.mod h1:V02YIuMVo301Y1VE9VtZlD8s84OMsk+EKN6mwvf/588= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= +gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/client/logs/factory.go b/logs/factory.go similarity index 64% rename from client/logs/factory.go rename to logs/factory.go index f63a63b..ac44b54 100644 --- a/client/logs/factory.go +++ b/logs/factory.go @@ -2,14 +2,14 @@ package logs import ( "fmt" - "io" + "log/slog" "os" "os/user" "path/filepath" "time" ) -func New(profile string, opts ...Option) (*TeeLogger, func()) { +func New(profile string, opts ...Option) (*Gen3Logger, func()) { cfg := defaults() for _, o := range opts { o(cfg) @@ -19,15 +19,15 @@ func New(profile string, opts ...Option) (*TeeLogger, func()) { logDir := filepath.Join(usr.HomeDir, ".gen3", "logs") os.MkdirAll(logDir, 0755) - var writers []io.Writer + var handlers []slog.Handler var messageFile *os.File if cfg.baseLogger != nil { - writers = append(writers, cfg.baseLogger.Writer()) + handlers = append(handlers, cfg.baseLogger.Handler()) } if cfg.console { - writers = append(writers, os.Stderr) + handlers = append(handlers, slog.NewTextHandler(os.Stderr, nil)) } if cfg.messageFile { @@ -39,15 +39,26 @@ func New(profile string, opts ...Option) (*TeeLogger, func()) { f, err := os.OpenFile(filepath.Join(logDir, filename), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err == nil { messageFile = f - writers = append(writers, f) + handlers = append(handlers, slog.NewTextHandler(f, nil)) fmt.Fprintf(f, "[%s] Message log started\n", time.Now().Format(time.RFC3339)) } } - t := NewTeeLogger(logDir, profile, writers...) + var rootHandler slog.Handler + if len(handlers) == 0 { + rootHandler = slog.NewTextHandler(os.Stderr, nil) + } else if len(handlers) == 1 { + rootHandler = handlers[0] + } else { + rootHandler = NewTeeHandler(handlers...) + } + + sl := slog.New(NewProgressHandler(rootHandler)) + + t := NewGen3Logger(sl, logDir, profile) if cfg.enableScoreboard { - t.scoreboard = NewSB(5, t) + t.scoreboard = NewSB(5, t.Logger) } if cfg.failedLog { diff --git a/logs/handler.go b/logs/handler.go new file mode 100644 index 0000000..ac7ed48 --- /dev/null +++ b/logs/handler.go @@ -0,0 +1,102 @@ +package logs + +import ( + "context" + "log/slog" + + sycommon "github.com/calypr/syfon/client/common" +) + +// ProgressHandler is a slog.Handler that captures log messages and +// forwards them to a ProgressCallback if one is present in the context. +type ProgressHandler struct { + next slog.Handler +} + +func NewProgressHandler(next slog.Handler) *ProgressHandler { + if next == nil { + next = slog.Default().Handler() + } + return &ProgressHandler{next: next} +} + +func (h *ProgressHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.next.Enabled(ctx, level) +} + +func (h *ProgressHandler) Handle(ctx context.Context, r slog.Record) error { + // Call the next handler in the chain (original logging) + err := h.next.Handle(ctx, r) + + // In addition, try to bubble up to progress callback + cb := sycommon.GetProgress(ctx) + if cb != nil { + oid := sycommon.GetOid(ctx) + // We send an event of type "log" + attrs := make(map[string]any) + r.Attrs(func(a slog.Attr) bool { + attrs[a.Key] = a.Value.Any() + return true + }) + _ = cb(sycommon.ProgressEvent{ + Event: "log", + Oid: oid, + Message: r.Message, + Level: r.Level.String(), + Attrs: attrs, + }) + } + + return err +} + +func (h *ProgressHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &ProgressHandler{next: h.next.WithAttrs(attrs)} +} + +func (h *ProgressHandler) WithGroup(name string) slog.Handler { + return &ProgressHandler{next: h.next.WithGroup(name)} +} + +// TeeHandler fans out log records to multiple handlers +type TeeHandler struct { + handlers []slog.Handler +} + +func NewTeeHandler(handlers ...slog.Handler) slog.Handler { + return &TeeHandler{handlers: handlers} +} + +func (h *TeeHandler) Enabled(ctx context.Context, level slog.Level) bool { + for _, hand := range h.handlers { + if hand.Enabled(ctx, level) { + return true + } + } + return false +} + +func (h *TeeHandler) Handle(ctx context.Context, r slog.Record) error { + for _, hand := range h.handlers { + if hand.Enabled(ctx, r.Level) { + _ = hand.Handle(ctx, r) + } + } + return nil +} + +func (h *TeeHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + newHandlers := make([]slog.Handler, len(h.handlers)) + for i, hand := range h.handlers { + newHandlers[i] = hand.WithAttrs(attrs) + } + return &TeeHandler{handlers: newHandlers} +} + +func (h *TeeHandler) WithGroup(name string) slog.Handler { + newHandlers := make([]slog.Handler, len(h.handlers)) + for i, hand := range h.handlers { + newHandlers[i] = hand.WithGroup(name) + } + return &TeeHandler{handlers: newHandlers} +} diff --git a/logs/logger.go b/logs/logger.go new file mode 100644 index 0000000..f6a55f0 --- /dev/null +++ b/logs/logger.go @@ -0,0 +1,35 @@ +package logs + +import ( + "log/slog" +) + +type Option func(*config) + +type config struct { + console bool + messageFile bool + failedLog bool + succeededLog bool + enableScoreboard bool + baseLogger *slog.Logger +} + +func WithConsole() Option { return func(c *config) { c.console = true } } +func WithNoConsole() Option { return func(c *config) { c.console = false } } +func WithMessageFile() Option { return func(c *config) { c.messageFile = true } } +func WithNoMessageFile() Option { return func(c *config) { c.messageFile = false } } +func WithFailedLog() Option { return func(c *config) { c.failedLog = true } } +func WithSucceededLog() Option { return func(c *config) { c.succeededLog = true } } +func WithScoreboard() Option { return func(c *config) { c.enableScoreboard = true } } +func WithBaseLogger(base *slog.Logger) Option { return func(c *config) { c.baseLogger = base } } + +func defaults() *config { + return &config{ + console: true, + messageFile: true, + failedLog: true, + succeededLog: true, + baseLogger: nil, + } +} diff --git a/logs/logger_test.go b/logs/logger_test.go new file mode 100644 index 0000000..382d2b5 --- /dev/null +++ b/logs/logger_test.go @@ -0,0 +1,208 @@ +package logs + +import ( + "io" + "log/slog" + "os" + "testing" +) + +func TestNewSlogNoOpLogger(t *testing.T) { + logger := NewSlogNoOpLogger() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Verify it's a valid slog.Logger + logger.Info("test message") // Should not panic + logger.Error("test error") // Should not panic +} + +func TestNew_WithDefaults(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + if logger.Logger == nil { + t.Error("Expected non-nil embedded slog logger") + } +} + +func TestNew_WithConsoleOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithConsole()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Test that we can log without errors + logger.Info("test console message") +} + +func TestNew_WithMessageFileOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithMessageFile()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Test that we can log without errors + logger.Info("test file message") +} + +func TestNew_WithScoreboardOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithScoreboard()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + if logger.Scoreboard() == nil { + t.Error("Expected non-nil scoreboard when WithScoreboard option is used") + } +} + +func TestNew_WithFailedLogOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithFailedLog()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Ensure failed-log helpers remain callable with syfon-backed logger. + _ = logger.GetFailedLogMap() +} + +func TestNew_WithSucceededLogOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithSucceededLog()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Ensure succeeded-log helpers remain callable with syfon-backed logger. + _ = logger.GetSucceededLogMap() +} + +func TestNew_WithBaseLogger(t *testing.T) { + profile := "test-profile" + baseLogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + logger, cleanup := New(profile, WithBaseLogger(baseLogger)) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Test that we can log without errors + logger.Info("test with base logger") +} + +func TestNew_WithMultipleOptions(t *testing.T) { + profile := "test-profile" + baseLogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + logger, cleanup := New(profile, + WithBaseLogger(baseLogger), + WithConsole(), + WithMessageFile(), + WithScoreboard(), + ) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + if logger.Logger == nil { + t.Error("Expected non-nil embedded slog logger") + } + + if logger.Scoreboard() == nil { + t.Error("Expected non-nil scoreboard") + } + + // Test that we can log without errors + logger.Info("test with multiple options") +} + +func TestGen3Logger_Info(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Info("test info message") +} + +func TestGen3Logger_Error(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Error("test error message") +} + +func TestGen3Logger_Warn(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Warn("test warning message") +} + +func TestGen3Logger_Debug(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Debug("test debug message") +} + +func TestGen3Logger_Printf(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Printf("test printf message: %s", "value") +} + +func TestGen3Logger_Println(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Println("test println message") +} + +// testLogger implements the Logger interface for testing +type testLogger struct { + writer io.Writer +} + +func (l *testLogger) Printf(format string, v ...any) {} +func (l *testLogger) Println(v ...any) {} +func (l *testLogger) Fatalf(format string, v ...any) {} +func (l *testLogger) Fatal(v ...any) {} +func (l *testLogger) Writer() io.Writer { return l.writer } diff --git a/logs/noop.go b/logs/noop.go new file mode 100644 index 0000000..f705772 --- /dev/null +++ b/logs/noop.go @@ -0,0 +1,11 @@ +package logs + +import ( + "io" + "log/slog" +) + +// NewSlogNoOpLogger creates a no-op slog logger for testing. +func NewSlogNoOpLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} diff --git a/client/logs/scoreboard.go b/logs/scoreboard.go similarity index 62% rename from client/logs/scoreboard.go rename to logs/scoreboard.go index a73117c..738a47a 100644 --- a/client/logs/scoreboard.go +++ b/logs/scoreboard.go @@ -1,33 +1,33 @@ package logs import ( - "context" "fmt" + "io" + "log/slog" + "os" "sync" "text/tabwriter" ) -type key int - -const scoreboardKey key = 0 - // Scoreboard holds retry statistics type Scoreboard struct { mu sync.Mutex Counts []int // index 0 = success on first try, 1 = after 1 retry, ..., last = failed - log Logger + logger *slog.Logger + writer io.Writer } -// New creates a new scoreboard +// NewSB creates a new scoreboard // maxRetryCount = how many retries you allow before giving up -func NewSB(maxRetryCount int, log Logger) *Scoreboard { +func NewSB(maxRetryCount int, logger *slog.Logger) *Scoreboard { return &Scoreboard{ Counts: make([]int, maxRetryCount+2), // +2: one for success-on-first, one for final failure - log: log, + logger: logger, + writer: os.Stderr, } } -// Increment records a result after `retryCount` attempts +// IncrementSB records a result after `retryCount` attempts // retryCount == 0 → succeeded on first try // retryCount == max → final failure func (s *Scoreboard) IncrementSB(retryCount int) { @@ -43,7 +43,7 @@ func (s *Scoreboard) IncrementSB(retryCount int) { s.Counts[retryCount]++ } -// Print the beautiful table at the end +// PrintSB prints the beautiful table at the end func (s *Scoreboard) PrintSB() { s.mu.Lock() defer s.mu.Unlock() @@ -56,8 +56,8 @@ func (s *Scoreboard) PrintSB() { return } - s.log.Println("\n\nSubmission Results") - w := tabwriter.NewWriter(s.log.Writer(), 0, 0, 2, ' ', 0) + s.logger.Info("Submission Results") + w := tabwriter.NewWriter(s.writer, 0, 0, 2, ' ', 0) for i, count := range s.Counts { if i == 0 { @@ -73,16 +73,3 @@ func (s *Scoreboard) PrintSB() { fmt.Fprintf(w, "TOTAL\t%d\n", total) w.Flush() } - -// Context helpers — so you don't have to pass scoreboard around - -func NewSBContext(parent context.Context, sb *Scoreboard) context.Context { - return context.WithValue(parent, scoreboardKey, sb) -} - -func FromSBContext(ctx context.Context) (*Scoreboard, error) { - if sb, ok := ctx.Value(scoreboardKey).(*Scoreboard); ok { - return sb, nil - } - return nil, fmt.Errorf("Scoreboard is not of type Scoreboard") -} diff --git a/logs/tee_logger.go b/logs/tee_logger.go new file mode 100644 index 0000000..998a195 --- /dev/null +++ b/logs/tee_logger.go @@ -0,0 +1,217 @@ +package logs + +import ( + "context" + "encoding/json" + "fmt" + "io" + "maps" + "os" + "runtime" + "sync" + "time" + + "log/slog" + + sycommon "github.com/calypr/syfon/client/common" +) + +// --- Gen3Logger Implementation --- +type Gen3Logger struct { + *slog.Logger + mu sync.RWMutex + scoreboard *Scoreboard + + failedMu sync.Mutex + FailedMap map[string]sycommon.RetryObject // Maps filePath to FileMetadata + failedPath string + + succeededMu sync.Mutex + succeededMap map[string]string // Maps filePath to GUID + succeededPath string +} + +// NewGen3Logger creates a new Gen3Logger wrapping the provided slog.Logger. +func NewGen3Logger(logger *slog.Logger, logDir, profile string) *Gen3Logger { + if logger == nil { + logger = slog.New(slog.NewTextHandler(os.Stdout, nil)) + } + return &Gen3Logger{ + Logger: logger, + FailedMap: make(map[string]sycommon.RetryObject), + succeededMap: make(map[string]string), + } +} + +// loadJSON is an internal helper to load JSON from a file path. +func loadJSON(path string, v any) { + data, _ := os.ReadFile(path) + if len(data) > 0 { + json.Unmarshal(data, v) + } +} + +// --- Core logging helper --- + +// logWithSkip logs a message at the given level, skipping `skip` stack frames for source attribution. +func (t *Gen3Logger) logWithSkip(ctx context.Context, level slog.Level, skip int, msg string, args ...any) { + if !t.Enabled(ctx, level) { + return + } + var pcs [1]uintptr + runtime.Callers(skip, pcs[:]) + r := slog.NewRecord(time.Now(), level, msg, pcs[0]) + r.Add(args...) + _ = t.Handler().Handle(ctx, r) +} + +// --- slog.Logger Method Overrides for accurate source attribution --- + +func (t *Gen3Logger) Info(msg string, args ...any) { + t.logWithSkip(context.Background(), slog.LevelInfo, 3, msg, args...) +} + +func (t *Gen3Logger) InfoContext(ctx context.Context, msg string, args ...any) { + t.logWithSkip(ctx, slog.LevelInfo, 3, msg, args...) +} + +func (t *Gen3Logger) Error(msg string, args ...any) { + t.logWithSkip(context.Background(), slog.LevelError, 3, msg, args...) +} + +func (t *Gen3Logger) ErrorContext(ctx context.Context, msg string, args ...any) { + t.logWithSkip(ctx, slog.LevelError, 3, msg, args...) +} + +func (t *Gen3Logger) Warn(msg string, args ...any) { + t.logWithSkip(context.Background(), slog.LevelWarn, 3, msg, args...) +} + +func (t *Gen3Logger) WarnContext(ctx context.Context, msg string, args ...any) { + t.logWithSkip(ctx, slog.LevelWarn, 3, msg, args...) +} + +func (t *Gen3Logger) Debug(msg string, args ...any) { + t.logWithSkip(context.Background(), slog.LevelDebug, 3, msg, args...) +} + +func (t *Gen3Logger) DebugContext(ctx context.Context, msg string, args ...any) { + t.logWithSkip(ctx, slog.LevelDebug, 3, msg, args...) +} + +// --- Legacy fmt-style methods --- + +func (t *Gen3Logger) Printf(format string, v ...any) { + t.logWithSkip(context.Background(), slog.LevelInfo, 3, fmt.Sprintf(format, v...)) +} + +func (t *Gen3Logger) Println(v ...any) { + t.logWithSkip(context.Background(), slog.LevelInfo, 3, fmt.Sprint(v...)) +} + +func (t *Gen3Logger) Fatalf(format string, v ...any) { + t.logWithSkip(context.Background(), slog.LevelError, 3, fmt.Sprintf(format, v...)) + os.Exit(1) +} + +func (t *Gen3Logger) Fatal(v ...any) { + t.logWithSkip(context.Background(), slog.LevelError, 3, fmt.Sprint(v...)) + os.Exit(1) +} + +// Writer returns os.Stderr for legacy compatibility (used by Scoreboard's tabwriter). +func (t *Gen3Logger) Writer() io.Writer { + return os.Stderr +} + +// Scoreboard returns the embedded Scoreboard. +func (t *Gen3Logger) Scoreboard() *Scoreboard { + return t.scoreboard +} + +// --- Succeeded/Failed log map methods --- + +func (t *Gen3Logger) GetSucceededLogMap() map[string]string { + t.succeededMu.Lock() + defer t.succeededMu.Unlock() + copiedMap := make(map[string]string, len(t.succeededMap)) + maps.Copy(copiedMap, t.succeededMap) + return copiedMap +} + +func (t *Gen3Logger) GetFailedLogMap() map[string]sycommon.RetryObject { + t.failedMu.Lock() + defer t.failedMu.Unlock() + copiedMap := make(map[string]sycommon.RetryObject, len(t.FailedMap)) + maps.Copy(copiedMap, t.FailedMap) + return copiedMap +} + +func (t *Gen3Logger) DeleteFromFailedLog(path string) { + t.failedMu.Lock() + defer t.failedMu.Unlock() + delete(t.FailedMap, path) +} + +func (t *Gen3Logger) GetSucceededCount() int { + return len(t.succeededMap) +} + +func (t *Gen3Logger) writeFailedSync(e sycommon.RetryObject) { + t.failedMu.Lock() + defer t.failedMu.Unlock() + t.FailedMap[e.SourcePath] = e + data, _ := json.MarshalIndent(t.FailedMap, "", " ") + os.WriteFile(t.failedPath, data, 0644) +} + +func (t *Gen3Logger) writeSucceededSync(path, guid string) { + t.succeededMu.Lock() + defer t.succeededMu.Unlock() + t.succeededMap[path] = guid + data, _ := json.MarshalIndent(t.succeededMap, "", " ") + os.WriteFile(t.succeededPath, data, 0644) +} + +// --- Tracking Methods --- + +// --- Tracking Methods --- + +func (t *Gen3Logger) Failed(filePath, filename string, metadata sycommon.FileMetadata, guid string, retryCount int, multipart bool) { + t.failedHelper(context.Background(), filePath, filename, metadata, guid, retryCount, multipart, 4) +} + +func (t *Gen3Logger) FailedContext(ctx context.Context, filePath, filename string, metadata sycommon.FileMetadata, guid string, retryCount int, multipart bool) { + t.failedHelper(ctx, filePath, filename, metadata, guid, retryCount, multipart, 4) +} + +func (t *Gen3Logger) failedHelper(ctx context.Context, filePath, filename string, metadata sycommon.FileMetadata, guid string, retryCount int, multipart bool, skip int) { + msg := fmt.Sprintf("Failed: %s (GUID: %s, Retry: %d)", filePath, guid, retryCount) + t.logWithSkip(ctx, slog.LevelError, skip, msg) + if t.failedPath != "" { + t.writeFailedSync(sycommon.RetryObject{ + SourcePath: filePath, + ObjectKey: filename, + FileMetadata: metadata, + GUID: guid, + RetryCount: retryCount, + Multipart: multipart, + }) + } +} + +func (t *Gen3Logger) Succeeded(filePath, guid string) { + t.succeededHelper(context.Background(), filePath, guid, 4) +} + +func (t *Gen3Logger) SucceededContext(ctx context.Context, filePath, guid string) { + t.succeededHelper(ctx, filePath, guid, 4) +} + +func (t *Gen3Logger) succeededHelper(ctx context.Context, filePath, guid string, skip int) { + msg := fmt.Sprintf("Succeeded: %s (GUID: %s)", filePath, guid) + t.logWithSkip(ctx, slog.LevelDebug, skip, msg) + if t.succeededPath != "" { + t.writeSucceededSync(filePath, guid) + } +} diff --git a/main.go b/main.go index 00bb0f7..5c039bf 100644 --- a/main.go +++ b/main.go @@ -1,9 +1,9 @@ package main import ( - "github.com/calypr/data-client/client/g3cmd" + "github.com/calypr/calypr-cli/cmd" ) func main() { - g3cmd.Execute() + cmd.Execute() } diff --git a/mocks/mock_configure.go b/mocks/mock_configure.go new file mode 100644 index 0000000..0677003 --- /dev/null +++ b/mocks/mock_configure.go @@ -0,0 +1,129 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/calypr/calypr-cli/conf (interfaces: ManagerInterface) +// +// Generated by this command: +// +// mockgen -destination=../mocks/mock_configure.go -package=mocks github.com/calypr/calypr-cli/conf ManagerInterface +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + conf "github.com/calypr/calypr-cli/conf" + gomock "go.uber.org/mock/gomock" +) + +// MockManagerInterface is a mock of ManagerInterface interface. +type MockManagerInterface struct { + ctrl *gomock.Controller + recorder *MockManagerInterfaceMockRecorder + isgomock struct{} +} + +// MockManagerInterfaceMockRecorder is the mock recorder for MockManagerInterface. +type MockManagerInterfaceMockRecorder struct { + mock *MockManagerInterface +} + +// NewMockManagerInterface creates a new mock instance. +func NewMockManagerInterface(ctrl *gomock.Controller) *MockManagerInterface { + mock := &MockManagerInterface{ctrl: ctrl} + mock.recorder = &MockManagerInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockManagerInterface) EXPECT() *MockManagerInterfaceMockRecorder { + return m.recorder +} + +// EnsureExists mocks base method. +func (m *MockManagerInterface) EnsureExists() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureExists") + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureExists indicates an expected call of EnsureExists. +func (mr *MockManagerInterfaceMockRecorder) EnsureExists() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureExists", reflect.TypeOf((*MockManagerInterface)(nil).EnsureExists)) +} + +// Import mocks base method. +func (m *MockManagerInterface) Import(filePath, fenceToken string) (*conf.Credential, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Import", filePath, fenceToken) + ret0, _ := ret[0].(*conf.Credential) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Import indicates an expected call of Import. +func (mr *MockManagerInterfaceMockRecorder) Import(filePath, fenceToken any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Import", reflect.TypeOf((*MockManagerInterface)(nil).Import), filePath, fenceToken) +} + +// IsCredentialValid mocks base method. +func (m *MockManagerInterface) IsCredentialValid(arg0 *conf.Credential) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsCredentialValid", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsCredentialValid indicates an expected call of IsCredentialValid. +func (mr *MockManagerInterfaceMockRecorder) IsCredentialValid(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCredentialValid", reflect.TypeOf((*MockManagerInterface)(nil).IsCredentialValid), arg0) +} + +// IsTokenValid mocks base method. +func (m *MockManagerInterface) IsTokenValid(arg0 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsTokenValid", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsTokenValid indicates an expected call of IsTokenValid. +func (mr *MockManagerInterfaceMockRecorder) IsTokenValid(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsTokenValid", reflect.TypeOf((*MockManagerInterface)(nil).IsTokenValid), arg0) +} + +// Load mocks base method. +func (m *MockManagerInterface) Load(profile string) (*conf.Credential, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load", profile) + ret0, _ := ret[0].(*conf.Credential) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Load indicates an expected call of Load. +func (mr *MockManagerInterfaceMockRecorder) Load(profile any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockManagerInterface)(nil).Load), profile) +} + +// Save mocks base method. +func (m *MockManagerInterface) Save(cred *conf.Credential) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", cred) + ret0, _ := ret[0].(error) + return ret0 +} + +// Save indicates an expected call of Save. +func (mr *MockManagerInterfaceMockRecorder) Save(cred any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockManagerInterface)(nil).Save), cred) +} diff --git a/mocks/mock_fence.go b/mocks/mock_fence.go new file mode 100644 index 0000000..646da5f --- /dev/null +++ b/mocks/mock_fence.go @@ -0,0 +1,281 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/calypr/calypr-cli/fence (interfaces: FenceInterface) +// +// Generated by this command: +// +// mockgen -destination=../mocks/mock_fence.go -package=mocks github.com/calypr/calypr-cli/fence FenceInterface +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + http "net/http" + reflect "reflect" + + fence "github.com/calypr/calypr-cli/fence" + request "github.com/calypr/calypr-cli/request" + gomock "go.uber.org/mock/gomock" +) + +// MockFenceInterface is a mock of FenceInterface interface. +type MockFenceInterface struct { + ctrl *gomock.Controller + recorder *MockFenceInterfaceMockRecorder + isgomock struct{} +} + +// MockFenceInterfaceMockRecorder is the mock recorder for MockFenceInterface. +type MockFenceInterfaceMockRecorder struct { + mock *MockFenceInterface +} + +// NewMockFenceInterface creates a new mock instance. +func NewMockFenceInterface(ctrl *gomock.Controller) *MockFenceInterface { + mock := &MockFenceInterface{ctrl: ctrl} + mock.recorder = &MockFenceInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFenceInterface) EXPECT() *MockFenceInterfaceMockRecorder { + return m.recorder +} + +// CheckForShepherdAPI mocks base method. +func (m *MockFenceInterface) CheckForShepherdAPI(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckForShepherdAPI", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckForShepherdAPI indicates an expected call of CheckForShepherdAPI. +func (mr *MockFenceInterfaceMockRecorder) CheckForShepherdAPI(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckForShepherdAPI", reflect.TypeOf((*MockFenceInterface)(nil).CheckForShepherdAPI), ctx) +} + +// CheckPrivileges mocks base method. +func (m *MockFenceInterface) CheckPrivileges(ctx context.Context) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckPrivileges", ctx) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckPrivileges indicates an expected call of CheckPrivileges. +func (mr *MockFenceInterfaceMockRecorder) CheckPrivileges(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckPrivileges", reflect.TypeOf((*MockFenceInterface)(nil).CheckPrivileges), ctx) +} + +// CompleteMultipartUpload mocks base method. +func (m *MockFenceInterface) CompleteMultipartUpload(ctx context.Context, key, uploadID string, parts []fence.MultipartPart, bucket string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteMultipartUpload", ctx, key, uploadID, parts, bucket) + ret0, _ := ret[0].(error) + return ret0 +} + +// CompleteMultipartUpload indicates an expected call of CompleteMultipartUpload. +func (mr *MockFenceInterfaceMockRecorder) CompleteMultipartUpload(ctx, key, uploadID, parts, bucket any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteMultipartUpload", reflect.TypeOf((*MockFenceInterface)(nil).CompleteMultipartUpload), ctx, key, uploadID, parts, bucket) +} + +// DeleteRecord mocks base method. +func (m *MockFenceInterface) DeleteRecord(ctx context.Context, guid string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRecord", ctx, guid) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteRecord indicates an expected call of DeleteRecord. +func (mr *MockFenceInterfaceMockRecorder) DeleteRecord(ctx, guid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRecord", reflect.TypeOf((*MockFenceInterface)(nil).DeleteRecord), ctx, guid) +} + +// Do mocks base method. +func (m *MockFenceInterface) Do(ctx context.Context, req *request.RequestBuilder) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Do", ctx, req) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Do indicates an expected call of Do. +func (mr *MockFenceInterfaceMockRecorder) Do(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockFenceInterface)(nil).Do), ctx, req) +} + +// GenerateMultipartPresignedURL mocks base method. +func (m *MockFenceInterface) GenerateMultipartPresignedURL(ctx context.Context, key, uploadID string, partNumber int, bucket string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateMultipartPresignedURL", ctx, key, uploadID, partNumber, bucket) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateMultipartPresignedURL indicates an expected call of GenerateMultipartPresignedURL. +func (mr *MockFenceInterfaceMockRecorder) GenerateMultipartPresignedURL(ctx, key, uploadID, partNumber, bucket any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateMultipartPresignedURL", reflect.TypeOf((*MockFenceInterface)(nil).GenerateMultipartPresignedURL), ctx, key, uploadID, partNumber, bucket) +} + +// GetBucketDetails mocks base method. +func (m *MockFenceInterface) GetBucketDetails(ctx context.Context, bucket string) (*fence.S3Bucket, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBucketDetails", ctx, bucket) + ret0, _ := ret[0].(*fence.S3Bucket) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBucketDetails indicates an expected call of GetBucketDetails. +func (mr *MockFenceInterfaceMockRecorder) GetBucketDetails(ctx, bucket any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBucketDetails", reflect.TypeOf((*MockFenceInterface)(nil).GetBucketDetails), ctx, bucket) +} + +// GetDownloadPresignedUrl mocks base method. +func (m *MockFenceInterface) GetDownloadPresignedUrl(ctx context.Context, guid, protocolText string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDownloadPresignedUrl", ctx, guid, protocolText) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDownloadPresignedUrl indicates an expected call of GetDownloadPresignedUrl. +func (mr *MockFenceInterfaceMockRecorder) GetDownloadPresignedUrl(ctx, guid, protocolText any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDownloadPresignedUrl", reflect.TypeOf((*MockFenceInterface)(nil).GetDownloadPresignedUrl), ctx, guid, protocolText) +} + +// GetUploadPresignedUrl mocks base method. +func (m *MockFenceInterface) GetUploadPresignedUrl(ctx context.Context, guid, filename, bucket string) (fence.FenceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUploadPresignedUrl", ctx, guid, filename, bucket) + ret0, _ := ret[0].(fence.FenceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetUploadPresignedUrl indicates an expected call of GetUploadPresignedUrl. +func (mr *MockFenceInterfaceMockRecorder) GetUploadPresignedUrl(ctx, guid, filename, bucket any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUploadPresignedUrl", reflect.TypeOf((*MockFenceInterface)(nil).GetUploadPresignedUrl), ctx, guid, filename, bucket) +} + +// InitMultipartUpload mocks base method. +func (m *MockFenceInterface) InitMultipartUpload(ctx context.Context, filename, bucket, guid string) (fence.FenceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitMultipartUpload", ctx, filename, bucket, guid) + ret0, _ := ret[0].(fence.FenceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InitMultipartUpload indicates an expected call of InitMultipartUpload. +func (mr *MockFenceInterfaceMockRecorder) InitMultipartUpload(ctx, filename, bucket, guid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitMultipartUpload", reflect.TypeOf((*MockFenceInterface)(nil).InitMultipartUpload), ctx, filename, bucket, guid) +} + +// InitUpload mocks base method. +func (m *MockFenceInterface) InitUpload(ctx context.Context, filename, bucket, guid string) (fence.FenceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitUpload", ctx, filename, bucket, guid) + ret0, _ := ret[0].(fence.FenceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InitUpload indicates an expected call of InitUpload. +func (mr *MockFenceInterfaceMockRecorder) InitUpload(ctx, filename, bucket, guid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitUpload", reflect.TypeOf((*MockFenceInterface)(nil).InitUpload), ctx, filename, bucket, guid) +} + +// New mocks base method. +func (m *MockFenceInterface) New(method, url string) *request.RequestBuilder { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "New", method, url) + ret0, _ := ret[0].(*request.RequestBuilder) + return ret0 +} + +// New indicates an expected call of New. +func (mr *MockFenceInterfaceMockRecorder) New(method, url any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "New", reflect.TypeOf((*MockFenceInterface)(nil).New), method, url) +} + +// NewAccessToken mocks base method. +func (m *MockFenceInterface) NewAccessToken(ctx context.Context) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewAccessToken", ctx) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewAccessToken indicates an expected call of NewAccessToken. +func (mr *MockFenceInterfaceMockRecorder) NewAccessToken(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewAccessToken", reflect.TypeOf((*MockFenceInterface)(nil).NewAccessToken), ctx) +} + +// ParseFenceURLResponse mocks base method. +func (m *MockFenceInterface) ParseFenceURLResponse(resp *http.Response) (fence.FenceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ParseFenceURLResponse", resp) + ret0, _ := ret[0].(fence.FenceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ParseFenceURLResponse indicates an expected call of ParseFenceURLResponse. +func (mr *MockFenceInterfaceMockRecorder) ParseFenceURLResponse(resp any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseFenceURLResponse", reflect.TypeOf((*MockFenceInterface)(nil).ParseFenceURLResponse), resp) +} + +// RefreshToken mocks base method. +func (m *MockFenceInterface) RefreshToken(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RefreshToken", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// RefreshToken indicates an expected call of RefreshToken. +func (mr *MockFenceInterfaceMockRecorder) RefreshToken(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshToken", reflect.TypeOf((*MockFenceInterface)(nil).RefreshToken), ctx) +} + +// UserPing mocks base method. +func (m *MockFenceInterface) UserPing(ctx context.Context) (*fence.PingResp, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UserPing", ctx) + ret0, _ := ret[0].(*fence.PingResp) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UserPing indicates an expected call of UserPing. +func (mr *MockFenceInterfaceMockRecorder) UserPing(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UserPing", reflect.TypeOf((*MockFenceInterface)(nil).UserPing), ctx) +} diff --git a/mocks/mock_gen3interface.go b/mocks/mock_gen3interface.go new file mode 100644 index 0000000..f03f195 --- /dev/null +++ b/mocks/mock_gen3interface.go @@ -0,0 +1,177 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/calypr/calypr-cli/g3client (interfaces: Gen3Interface) +// +// Generated by this command: +// +// mockgen -destination=../mocks/mock_gen3interface.go -package=mocks github.com/calypr/calypr-cli/g3client Gen3Interface +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + http "net/http" + reflect "reflect" + + fence "github.com/calypr/calypr-cli/fence" + g3client "github.com/calypr/calypr-cli/g3client" + gecko "github.com/calypr/calypr-cli/gecko" + logs "github.com/calypr/calypr-cli/logs" + request "github.com/calypr/calypr-cli/request" + requestor "github.com/calypr/calypr-cli/requestor" + sower "github.com/calypr/calypr-cli/sower" + syconfig "github.com/calypr/syfon/client/config" + gomock "go.uber.org/mock/gomock" +) + +// MockGen3Interface is a mock of Gen3Interface interface. +type MockGen3Interface struct { + ctrl *gomock.Controller + recorder *MockGen3InterfaceMockRecorder + isgomock struct{} +} + +// MockGen3InterfaceMockRecorder is the mock recorder for MockGen3Interface. +type MockGen3InterfaceMockRecorder struct { + mock *MockGen3Interface +} + +// NewMockGen3Interface creates a new mock instance. +func NewMockGen3Interface(ctrl *gomock.Controller) *MockGen3Interface { + mock := &MockGen3Interface{ctrl: ctrl} + mock.recorder = &MockGen3InterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGen3Interface) EXPECT() *MockGen3InterfaceMockRecorder { + return m.recorder +} + +// Credentials mocks base method. +func (m *MockGen3Interface) Credentials() syconfig.CredentialManager { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Credentials") + ret0, _ := ret[0].(syconfig.CredentialManager) + return ret0 +} + +// Credentials indicates an expected call of Credentials. +func (mr *MockGen3InterfaceMockRecorder) Credentials() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Credentials", reflect.TypeOf((*MockGen3Interface)(nil).Credentials)) +} + +// SyfonClient mocks base method. +func (m *MockGen3Interface) SyfonClient() g3client.SyfonClientInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SyfonClient") + ret0, _ := ret[0].(g3client.SyfonClientInterface) + return ret0 +} + +// SyfonClient indicates an expected call of SyfonClient. +func (mr *MockGen3InterfaceMockRecorder) SyfonClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyfonClient", reflect.TypeOf((*MockGen3Interface)(nil).SyfonClient)) +} + +// Do mocks base method. +func (m *MockGen3Interface) Do(ctx context.Context, req *request.RequestBuilder) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Do", ctx, req) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Do indicates an expected call of Do. +func (mr *MockGen3InterfaceMockRecorder) Do(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockGen3Interface)(nil).Do), ctx, req) +} + +// FenceClient mocks base method. +func (m *MockGen3Interface) FenceClient() fence.FenceInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FenceClient") + ret0, _ := ret[0].(fence.FenceInterface) + return ret0 +} + +// FenceClient indicates an expected call of FenceClient. +func (mr *MockGen3InterfaceMockRecorder) FenceClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FenceClient", reflect.TypeOf((*MockGen3Interface)(nil).FenceClient)) +} + +// GeckoClient mocks base method. +func (m *MockGen3Interface) GeckoClient() gecko.GeckoInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GeckoClient") + ret0, _ := ret[0].(gecko.GeckoInterface) + return ret0 +} + +// GeckoClient indicates an expected call of GeckoClient. +func (mr *MockGen3InterfaceMockRecorder) GeckoClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GeckoClient", reflect.TypeOf((*MockGen3Interface)(nil).GeckoClient)) +} + +// Logger mocks base method. +func (m *MockGen3Interface) Logger() *logs.Gen3Logger { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Logger") + ret0, _ := ret[0].(*logs.Gen3Logger) + return ret0 +} + +// Logger indicates an expected call of Logger. +func (mr *MockGen3InterfaceMockRecorder) Logger() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Logger", reflect.TypeOf((*MockGen3Interface)(nil).Logger)) +} + +// New mocks base method. +func (m *MockGen3Interface) New(method, url string) *request.RequestBuilder { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "New", method, url) + ret0, _ := ret[0].(*request.RequestBuilder) + return ret0 +} + +// New indicates an expected call of New. +func (mr *MockGen3InterfaceMockRecorder) New(method, url any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "New", reflect.TypeOf((*MockGen3Interface)(nil).New), method, url) +} + +// RequestorClient mocks base method. +func (m *MockGen3Interface) RequestorClient() requestor.RequestorInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RequestorClient") + ret0, _ := ret[0].(requestor.RequestorInterface) + return ret0 +} + +// RequestorClient indicates an expected call of RequestorClient. +func (mr *MockGen3InterfaceMockRecorder) RequestorClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequestorClient", reflect.TypeOf((*MockGen3Interface)(nil).RequestorClient)) +} + +// SowerClient mocks base method. +func (m *MockGen3Interface) SowerClient() sower.SowerInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SowerClient") + ret0, _ := ret[0].(sower.SowerInterface) + return ret0 +} + +// SowerClient indicates an expected call of SowerClient. +func (mr *MockGen3InterfaceMockRecorder) SowerClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SowerClient", reflect.TypeOf((*MockGen3Interface)(nil).SowerClient)) +} diff --git a/mocks/mock_request.go b/mocks/mock_request.go new file mode 100644 index 0000000..eb58e59 --- /dev/null +++ b/mocks/mock_request.go @@ -0,0 +1,72 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/calypr/calypr-cli/request (interfaces: RequestInterface) +// +// Generated by this command: +// +// mockgen -destination=./mocks/mock_request.go -package=mocks github.com/calypr/calypr-cli/request RequestInterface +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + http "net/http" + reflect "reflect" + + request "github.com/calypr/calypr-cli/request" + gomock "go.uber.org/mock/gomock" +) + +// MockRequestInterface is a mock of RequestInterface interface. +type MockRequestInterface struct { + ctrl *gomock.Controller + recorder *MockRequestInterfaceMockRecorder + isgomock struct{} +} + +// MockRequestInterfaceMockRecorder is the mock recorder for MockRequestInterface. +type MockRequestInterfaceMockRecorder struct { + mock *MockRequestInterface +} + +// NewMockRequestInterface creates a new mock instance. +func NewMockRequestInterface(ctrl *gomock.Controller) *MockRequestInterface { + mock := &MockRequestInterface{ctrl: ctrl} + mock.recorder = &MockRequestInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRequestInterface) EXPECT() *MockRequestInterfaceMockRecorder { + return m.recorder +} + +// Do mocks base method. +func (m *MockRequestInterface) Do(ctx context.Context, req *request.RequestBuilder) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Do", ctx, req) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Do indicates an expected call of Do. +func (mr *MockRequestInterfaceMockRecorder) Do(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockRequestInterface)(nil).Do), ctx, req) +} + +// New mocks base method. +func (m *MockRequestInterface) New(method, url string) *request.RequestBuilder { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "New", method, url) + ret0, _ := ret[0].(*request.RequestBuilder) + return ret0 +} + +// New indicates an expected call of New. +func (mr *MockRequestInterfaceMockRecorder) New(method, url any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "New", reflect.TypeOf((*MockRequestInterface)(nil).New), method, url) +} diff --git a/request/auth.go b/request/auth.go new file mode 100644 index 0000000..4b5c828 --- /dev/null +++ b/request/auth.go @@ -0,0 +1,116 @@ +package request + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "sync" + + "github.com/calypr/calypr-cli/conf" + sycommon "github.com/calypr/syfon/client/common" +) + +func (t *AuthTransport) NewAccessToken(ctx context.Context) error { + if t.Cred.APIKey == "" { + return errors.New("APIKey is required to refresh access token") + } + + refreshClient := &http.Client{Transport: t.Base} + + payload := map[string]string{"api_key": t.Cred.APIKey} + reader, err := sycommon.ToJSONReader(payload) + if err != nil { + return err + } + + refreshUrl := t.Cred.APIEndpoint + sycommon.DataAccessTokenEndpoint + req, err := http.NewRequestWithContext(ctx, http.MethodPost, refreshUrl, reader) + if err != nil { + return err + } + req.Header.Set(sycommon.HeaderContentType, sycommon.MIMEApplicationJSON) + + resp, err := refreshClient.Do(req) + if err != nil { + return fmt.Errorf("refresh request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return errors.New("failed to refresh token, status: " + strconv.Itoa(resp.StatusCode)) + } + + var result struct { + AccessToken string `json:"access_token"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return err + } + + t.mu.Lock() + t.Cred.AccessToken = result.AccessToken + if t.Manager != nil { + t.Manager.Save(t.Cred) + } + t.mu.Unlock() + return nil +} + +type AuthTransport struct { + Manager conf.ManagerInterface + Base http.RoundTripper + Cred *conf.Credential + mu sync.RWMutex + refreshMu sync.Mutex +} + +func (t *AuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Header.Get("X-Skip-Auth") == "true" { + req.Header.Del("X-Skip-Auth") + return t.Base.RoundTrip(req) + } + + t.mu.RLock() + token := t.Cred.AccessToken + t.mu.RUnlock() + + // Just add the header and pass it down + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + return t.Base.RoundTrip(req) +} + +func extractBearerToken(header string) string { + if !strings.HasPrefix(strings.ToLower(header), "bearer ") { + return "" + } + return strings.TrimSpace(header[len("Bearer "):]) +} + +func (t *AuthTransport) refreshOnce(ctx context.Context, failedToken string) error { + t.refreshMu.Lock() + defer t.refreshMu.Unlock() + + t.mu.RLock() + currentToken := t.Cred.AccessToken + t.mu.RUnlock() + + // Another goroutine may already have refreshed the token after the failing + // request was sent. If the credential token has changed, use that one and + // avoid a redundant refresh call. + if failedToken != "" && currentToken != "" && currentToken != failedToken { + return nil + } + + if currentToken != "" && failedToken == "" { + // Without the failed token, we cannot distinguish a stale token from a + // freshly refreshed one, so fall through and refresh explicitly. + } + + return t.NewAccessToken(ctx) +} diff --git a/request/auth_test.go b/request/auth_test.go new file mode 100644 index 0000000..50744cc --- /dev/null +++ b/request/auth_test.go @@ -0,0 +1,98 @@ +package request + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/logs" +) + +type trackingConfigManager struct { + mockConfigManager + saveCalls int + lastSaved *conf.Credential +} + +func (m *trackingConfigManager) Save(cred *conf.Credential) error { + m.saveCalls++ + if cred != nil { + copyCred := *cred + m.lastSaved = ©Cred + } + return nil +} + +func TestRequestRefreshesExpiredBearerTokenOnUnauthorized(t *testing.T) { + const ( + oldToken = "expired-token" + newToken = "fresh-token" + apiKey = "refresh-api-key" + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/user/credentials/api/access_token": + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read refresh body: %v", err) + } + if !strings.Contains(string(body), apiKey) { + t.Fatalf("refresh request missing api key: %s", string(body)) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"access_token":"` + newToken + `"}`)) + case "/protected": + switch r.Header.Get("Authorization") { + case "Bearer " + oldToken: + http.Error(w, "expired", http.StatusUnauthorized) + case "Bearer " + newToken: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + default: + t.Fatalf("unexpected authorization header: %q", r.Header.Get("Authorization")) + } + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + defer server.Close() + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + Profile: "test", + APIKey: apiKey, + AccessToken: oldToken, + APIEndpoint: server.URL, + } + cfg := &trackingConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, cfg) + req := reqInterface.(*Request) + builder := req.New(http.MethodGet, server.URL+"/protected") + + resp, err := req.Do(context.Background(), builder) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if cred.AccessToken != newToken { + t.Fatalf("credential token = %q, want %q", cred.AccessToken, newToken) + } + if cfg.saveCalls != 1 { + t.Fatalf("save calls = %d, want 1", cfg.saveCalls) + } + if cfg.lastSaved == nil || cfg.lastSaved.AccessToken != newToken { + t.Fatalf("saved credential token = %#v, want %q", cfg.lastSaved, newToken) + } +} diff --git a/request/builder.go b/request/builder.go new file mode 100644 index 0000000..104ac47 --- /dev/null +++ b/request/builder.go @@ -0,0 +1,65 @@ +package request + +import ( + "io" + + sycommon "github.com/calypr/syfon/client/common" +) + +// New addition to your request package +type RequestBuilder struct { + //Req *Request // the underlying retry client holder + Method string + Url string + Body io.Reader // store as []byte for easy reuse + Headers map[string]string + Token string + PartSize int64 + SkipAuth bool +} + +func (r *Request) New(method, url string) *RequestBuilder { + return &RequestBuilder{ + //Req: r, + Method: method, + Url: url, + Headers: make(map[string]string), + } +} + +func (ar *RequestBuilder) WithToken(token string) *RequestBuilder { + ar.Token = token + return ar +} + +func (ar *RequestBuilder) WithJSONBody(v any) (*RequestBuilder, error) { + reader, err := sycommon.ToJSONReader(v) + if err != nil { + return nil, err + } + + ar.Body = reader + ar.Headers[sycommon.HeaderContentType] = sycommon.MIMEApplicationJSON + return ar, nil + +} + +func (ar *RequestBuilder) WithBody(body io.Reader) *RequestBuilder { + ar.Body = body + return ar +} + +func (ar *RequestBuilder) WithHeader(key, value string) *RequestBuilder { + ar.Headers[key] = value + return ar +} + +func (ar *RequestBuilder) WithSkipAuth(skip bool) *RequestBuilder { + ar.SkipAuth = skip + return ar +} + +func (ar *RequestBuilder) WithPartSize(size int64) *RequestBuilder { + ar.PartSize = size + return ar +} diff --git a/request/helpers.go b/request/helpers.go new file mode 100644 index 0000000..a08d518 --- /dev/null +++ b/request/helpers.go @@ -0,0 +1,176 @@ +package request + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +type ErrorMessageProvider interface { + ErrorMessage() string +} + +type ErrorTypeProvider interface { + ErrorType() string +} + +type ErrorDetailsProvider interface { + ErrorDetails() map[string]any +} + +type HTTPStatusError struct { + Action string + StatusCode int + Message string + Type string + Details map[string]any + RawBody string +} + +func (e *HTTPStatusError) Error() string { + if e.Message != "" { + return fmt.Sprintf("%s: status %d: %s", e.Action, e.StatusCode, e.Message) + } + if e.RawBody == "" { + return fmt.Sprintf("%s: status %d", e.Action, e.StatusCode) + } + return fmt.Sprintf("%s: status %d, body: %s", e.Action, e.StatusCode, e.RawBody) +} + +type JSONOption func(*jsonOptions) + +type jsonOptions struct { + action string + errorEnvelope ErrorMessageProvider + expectedCodes map[int]struct{} +} + +func WithAction(action string) JSONOption { + return func(opts *jsonOptions) { + opts.action = action + } +} + +func WithErrorEnvelope(envelope ErrorMessageProvider) JSONOption { + return func(opts *jsonOptions) { + opts.errorEnvelope = envelope + } +} + +func WithExpectedStatus(codes ...int) JSONOption { + return func(opts *jsonOptions) { + opts.expectedCodes = make(map[int]struct{}, len(codes)) + for _, code := range codes { + opts.expectedCodes[code] = struct{}{} + } + } +} + +func JoinURL(base string, parts ...string) string { + u, _ := url.Parse(strings.TrimSpace(base)) + + joined := make([]string, 0, len(parts)+1) + if basePath := strings.Trim(u.Path, "/"); basePath != "" { + joined = append(joined, basePath) + } + for _, part := range parts { + if trimmed := strings.Trim(part, "/"); trimmed != "" { + joined = append(joined, trimmed) + } + } + + if len(joined) == 0 { + u.Path = "" + return u.String() + } + + u.Path = "/" + strings.Join(joined, "/") + return u.String() +} + +func NewJSON(req RequestInterface, method, url string, body any) (*RequestBuilder, error) { + rb := req.New(method, url) + if body == nil { + return rb, nil + } + return rb.WithJSONBody(body) +} + +func DoJSON(ctx context.Context, req RequestInterface, rb *RequestBuilder, out any, opts ...JSONOption) error { + options := jsonOptions{ + action: "request failed", + } + for _, opt := range opts { + opt(&options) + } + + resp, err := req.Do(ctx, rb) + if err != nil { + return err + } + defer resp.Body.Close() + + if !statusAccepted(resp.StatusCode, options.expectedCodes) { + if options.errorEnvelope != nil { + return StatusErrorJSON(resp, options.action, options.errorEnvelope) + } + return StatusError(resp, options.action) + } + + if out == nil { + return nil + } + return DecodeJSON(resp, out) +} + +func DecodeJSON(resp *http.Response, out any) error { + return json.NewDecoder(resp.Body).Decode(out) +} + +func StatusError(resp *http.Response, action string) error { + bodyBytes, _ := io.ReadAll(resp.Body) + return statusErrorFromBody(resp.StatusCode, action, bodyBytes) +} + +func StatusErrorJSON(resp *http.Response, action string, out ErrorMessageProvider) error { + bodyBytes, _ := io.ReadAll(resp.Body) + if err := json.Unmarshal(bodyBytes, out); err == nil { + if message := strings.TrimSpace(out.ErrorMessage()); message != "" { + statusErr := &HTTPStatusError{ + Action: action, + StatusCode: resp.StatusCode, + Message: message, + RawBody: strings.TrimSpace(string(bodyBytes)), + } + if typed, ok := out.(ErrorTypeProvider); ok { + statusErr.Type = strings.TrimSpace(typed.ErrorType()) + } + if detailed, ok := out.(ErrorDetailsProvider); ok { + statusErr.Details = detailed.ErrorDetails() + } + return statusErr + } + } + return statusErrorFromBody(resp.StatusCode, action, bodyBytes) +} + +func statusAccepted(statusCode int, expectedCodes map[int]struct{}) bool { + if len(expectedCodes) == 0 { + return statusCode < http.StatusBadRequest + } + _, ok := expectedCodes[statusCode] + return ok +} + +func statusErrorFromBody(statusCode int, action string, bodyBytes []byte) error { + body := strings.TrimSpace(string(bodyBytes)) + return &HTTPStatusError{ + Action: action, + StatusCode: statusCode, + RawBody: body, + } +} diff --git a/request/request.go b/request/request.go new file mode 100644 index 0000000..9857a2d --- /dev/null +++ b/request/request.go @@ -0,0 +1,119 @@ +package request + +//go:generate mockgen -destination=../mocks/mock_request.go -package=mocks github.com/calypr/calypr-cli/request RequestInterface + +import ( + "context" + "errors" + "net" + "net/http" + "strings" + "time" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/logs" + "github.com/hashicorp/go-retryablehttp" +) + +type Request struct { + Logs *logs.Gen3Logger + RetryClient *retryablehttp.Client +} + +type RequestInterface interface { + New(method, url string) *RequestBuilder + Do(ctx context.Context, req *RequestBuilder) (*http.Response, error) +} + +func NewRequestInterface( + logger *logs.Gen3Logger, + cred *conf.Credential, + conf conf.ManagerInterface, +) RequestInterface { + retryClient := retryablehttp.NewClient() + retryClient.RetryMax = 5 + retryClient.Logger = logger + retryClient.RetryWaitMin = 5 * time.Second + retryClient.RetryWaitMax = 15 * time.Second + baseTransport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + TLSHandshakeTimeout: 30 * time.Second, + ResponseHeaderTimeout: 60 * time.Second, + } + + authTransport := &AuthTransport{ + Base: baseTransport, + Cred: cred, + Manager: conf, + } + retryClient.HTTPClient = &http.Client{ + Timeout: 0, + Transport: authTransport, + } + + retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { + shouldRetry, retryErr := + retryablehttp.DefaultRetryPolicy(ctx, resp, err) + + if resp != nil && + (resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusBadGateway) { + failedToken := "" + if resp.Request != nil { + failedToken = extractBearerToken(resp.Request.Header.Get("Authorization")) + } + err := authTransport.refreshOnce(ctx, strings.TrimSpace(failedToken)) + if err != nil { + return false, err + } + return true, nil + } + return shouldRetry, retryErr + } + + return &Request{ + RetryClient: retryClient, + Logs: logger, + } +} + +func (r *Request) Do(ctx context.Context, rb *RequestBuilder) (*http.Response, error) { + // Prepare body reader + + httpReq, err := http.NewRequestWithContext(ctx, rb.Method, rb.Url, rb.Body) + if err != nil { + return nil, errors.New("failed to create HTTP request: " + err.Error()) + } + + for key, value := range rb.Headers { + httpReq.Header.Add(key, value) + } + + if rb.SkipAuth { + httpReq.Header.Set("X-Skip-Auth", "true") + } + + if rb.Token != "" { + httpReq.Header.Set("Authorization", "Bearer "+rb.Token) + } + + if rb.PartSize != 0 { + httpReq.ContentLength = rb.PartSize + } + // Convert to retryablehttp.Request + retryReq, err := retryablehttp.FromRequest(httpReq) + if err != nil { + return nil, err + } + + resp, err := r.RetryClient.Do(retryReq) + if err != nil { + return resp, errors.New("request failed after retries: " + err.Error()) + } + + return resp, nil +} diff --git a/request/request_test.go b/request/request_test.go new file mode 100644 index 0000000..1e49af9 --- /dev/null +++ b/request/request_test.go @@ -0,0 +1,379 @@ +package request + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/logs" +) + +func TestNewRequestInterface(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + + // Create a mock config manager + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + + if reqInterface == nil { + t.Fatal("Expected non-nil request interface") + } + + req, ok := reqInterface.(*Request) + if !ok { + t.Fatal("Expected request interface to be of type *Request") + } + + if req.RetryClient == nil { + t.Error("Expected non-nil retry client") + } + + if req.Logs == nil { + t.Error("Expected non-nil logger") + } +} + +func TestRequestBuilder_New(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + builder := req.New("GET", "https://example.com/api/test") + + if builder == nil { + t.Fatal("Expected non-nil request builder") + } + + if builder.Method != "GET" { + t.Errorf("Expected method 'GET', got '%s'", builder.Method) + } + + if builder.Url != "https://example.com/api/test" { + t.Errorf("Expected URL 'https://example.com/api/test', got '%s'", builder.Url) + } +} + +func TestRequestBuilder_WithHeaders(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + builder := req.New("GET", "https://example.com/api/test") + builder = builder.WithHeader("Content-Type", "application/json") + builder = builder.WithHeader("X-Custom-Header", "test-value") + + if len(builder.Headers) != 2 { + t.Errorf("Expected 2 headers, got %d", len(builder.Headers)) + } + + if builder.Headers["Content-Type"] != "application/json" { + t.Error("Expected Content-Type header to be set") + } + + if builder.Headers["X-Custom-Header"] != "test-value" { + t.Error("Expected X-Custom-Header to be set") + } +} + +func TestRequestBuilder_WithToken(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + token := "test-bearer-token-12345" + builder := req.New("GET", "https://example.com/api/test") + builder = builder.WithToken(token) + + if builder.Token != token { + t.Errorf("Expected token '%s', got '%s'", token, builder.Token) + } +} + +func TestRequestBuilder_WithBody(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + body := strings.NewReader("test body content") + builder := req.New("POST", "https://example.com/api/test") + builder = builder.WithBody(body) + + if builder.Body == nil { + t.Error("Expected non-nil body") + } +} + +func TestRequest_Do_Success(t *testing.T) { + // Create a test server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify the request + if r.Method != "GET" { + t.Errorf("Expected GET method, got %s", r.Method) + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "success"}`)) + })) + defer server.Close() + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: server.URL, + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + builder := req.New("GET", server.URL+"/api/test") + builder = builder.WithToken("test-token") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + resp, err := req.Do(ctx, builder) + + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if resp == nil { + t.Fatal("Expected non-nil response") + } + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if !strings.Contains(string(body), "success") { + t.Error("Expected response body to contain 'success'") + } +} + +func TestRequest_Do_WithCustomHeaders(t *testing.T) { + // Create a test server that checks for custom headers + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + customHeader := r.Header.Get("X-Custom-Header") + if customHeader != "test-value" { + t.Errorf("Expected X-Custom-Header 'test-value', got '%s'", customHeader) + } + + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: server.URL, + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + builder := req.New("GET", server.URL+"/api/test") + builder = builder.WithHeader("X-Custom-Header", "test-value") + + ctx := context.Background() + resp, err := req.Do(ctx, builder) + + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + resp.Body.Close() +} + +func TestRequest_JoinURL(t *testing.T) { + got := JoinURL("https://example.org/base", "config", "explorer", "default") + if got != "https://example.org/base/config/explorer/default" { + t.Fatalf("unexpected joined URL: %s", got) + } +} + +func TestRequest_DecodeJSON(t *testing.T) { + resp := &http.Response{ + Body: io.NopCloser(strings.NewReader(`{"status":"ok"}`)), + } + + var decoded struct { + Status string `json:"status"` + } + if err := DecodeJSON(resp, &decoded); err != nil { + t.Fatalf("DecodeJSON failed: %v", err) + } + if decoded.Status != "ok" { + t.Fatalf("unexpected decoded payload: %+v", decoded) + } +} + +func TestRequest_StatusError(t *testing.T) { + resp := &http.Response{ + StatusCode: http.StatusBadRequest, + Body: io.NopCloser(strings.NewReader(`{"error":"bad request"}`)), + } + + err := StatusError(resp, "request failed") + if err == nil { + t.Fatal("expected error") + } + if got := err.Error(); got != `request failed: status 400, body: {"error":"bad request"}` { + t.Fatalf("unexpected error: %s", got) + } +} + +type testErrorEnvelope struct { + Type string `json:"type,omitempty"` + Message string `json:"message"` + Details map[string]any `json:"details,omitempty"` +} + +func (t testErrorEnvelope) ErrorMessage() string { + return t.Message +} + +func (t testErrorEnvelope) ErrorType() string { + return t.Type +} + +func (t testErrorEnvelope) ErrorDetails() map[string]any { + return t.Details +} + +func TestRequest_StatusErrorJSON(t *testing.T) { + resp := &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(strings.NewReader(`{"type":"config_not_found","message":"missing","details":{"config_id":"default"}}`)), + } + + err := StatusErrorJSON(resp, "request failed", &testErrorEnvelope{}) + if err == nil { + t.Fatal("expected error") + } + if got := err.Error(); got != `request failed: status 404: missing` { + t.Fatalf("unexpected error: %s", got) + } + + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) { + t.Fatalf("expected HTTPStatusError, got %T", err) + } + if statusErr.Type != "config_not_found" { + t.Fatalf("unexpected error type: %q", statusErr.Type) + } + if statusErr.Details["config_id"] != "default" { + t.Fatalf("unexpected error details: %+v", statusErr.Details) + } +} + +func TestRequest_NewJSONAndDoJSON(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Fatalf("expected POST, got %s", r.Method) + } + if got := r.Header.Get("Content-Type"); got != "application/json" { + t.Fatalf("expected application/json content type, got %q", got) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"status":"ok"}`)) + })) + defer server.Close() + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{KeyID: "test-key", APIKey: "test-secret", APIEndpoint: server.URL} + req := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, &mockConfigManager{}) + + rb, err := NewJSON(req, http.MethodPost, server.URL+"/api/test", map[string]string{"hello": "world"}) + if err != nil { + t.Fatalf("NewJSON failed: %v", err) + } + + var decoded struct { + Status string `json:"status"` + } + if err := DoJSON(context.Background(), req, rb, &decoded, WithExpectedStatus(http.StatusOK), WithAction("post test")); err != nil { + t.Fatalf("DoJSON failed: %v", err) + } + if decoded.Status != "ok" { + t.Fatalf("unexpected decoded payload: %+v", decoded) + } +} + +// Mock config manager for testing +type mockConfigManager struct{} + +func (m *mockConfigManager) Import(filePath, fenceToken string) (*conf.Credential, error) { + return &conf.Credential{}, nil +} + +func (m *mockConfigManager) Load(profile string) (*conf.Credential, error) { + return &conf.Credential{}, nil +} + +func (m *mockConfigManager) Save(cred *conf.Credential) error { + return nil +} + +func (m *mockConfigManager) EnsureExists() error { + return nil +} + +func (m *mockConfigManager) IsCredentialValid(cred *conf.Credential) (bool, error) { + return true, nil +} + +func (m *mockConfigManager) IsTokenValid(token string) (bool, error) { + return true, nil +} diff --git a/requestor/client.go b/requestor/client.go new file mode 100644 index 0000000..47e4383 --- /dev/null +++ b/requestor/client.go @@ -0,0 +1,419 @@ +package requestor + +import ( + "context" + "embed" + "fmt" + "net/http" + "net/url" + "sort" + "strings" + + "github.com/calypr/calypr-cli/conf" + "github.com/calypr/calypr-cli/request" + "gopkg.in/yaml.v3" +) + +//go:embed policies/*.yaml +var policyFS embed.FS + +type RequestorClient struct { + request.RequestInterface + Endpoint string +} + +func NewRequestorClient(req request.RequestInterface, creds *conf.Credential) *RequestorClient { + return &RequestorClient{ + RequestInterface: req, + Endpoint: creds.APIEndpoint, + } +} + +// Ensure interface compliance +var _ RequestorInterface = &RequestorClient{} + +type RequestorInterface interface { + ListRequests(ctx context.Context, mine bool, active bool, username string) ([]Request, error) + CreateRequest(ctx context.Context, req CreateRequestRequest, revoke bool) (*Request, error) + UpdateRequest(ctx context.Context, requestID string, status string) (*Request, error) + AddUser(ctx context.Context, projectID string, username string, write bool, guppy bool) ([]Request, error) + AddUserToResources(ctx context.Context, resources []ProjectResource, username string, write bool, guppy bool) ([]Request, error) + RemoveUser(ctx context.Context, projectID string, username string) ([]Request, error) +} + +func (c *RequestorClient) ListRequests(ctx context.Context, mine bool, active bool, username string) ([]Request, error) { + url := request.JoinURL(c.Endpoint, "requestor", "request") + if mine { + url = request.JoinURL(c.Endpoint, "requestor", "request", "user") + } + + params := []string{} + if active { + params = append(params, "active") + } + if username != "" && !mine { + params = append(params, fmt.Sprintf("username=%s", username)) + } + + if len(params) > 0 { + url += "?" + strings.Join(params, "&") + } + + var requests []Request + if err := request.DoJSON( + ctx, + c.RequestInterface, + c.New(http.MethodGet, url), + &requests, + request.WithAction("failed to list requests"), + request.WithExpectedStatus(http.StatusOK), + ); err != nil { + return nil, err + } + return requests, nil +} + +func (c *RequestorClient) CreateRequest(ctx context.Context, reqPayload CreateRequestRequest, revoke bool) (*Request, error) { + url := request.JoinURL(c.Endpoint, "requestor", "request") + if revoke { + url += "?revoke" + } + + rb, err := request.NewJSON(c.RequestInterface, http.MethodPost, url, reqPayload) + if err != nil { + return nil, err + } + + var createdRequest Request + if err := request.DoJSON( + ctx, + c.RequestInterface, + rb, + &createdRequest, + request.WithAction("failed to create request"), + ); err != nil { + return nil, err + } + return &createdRequest, nil +} + +func (c *RequestorClient) UpdateRequest(ctx context.Context, requestID string, status string) (*Request, error) { + url := request.JoinURL(c.Endpoint, "requestor", "request", requestID) + payload := UpdateRequestRequest{Status: status} + + rb, err := request.NewJSON(c.RequestInterface, http.MethodPut, url, payload) + if err != nil { + return nil, err + } + + var updatedRequest Request + if err := request.DoJSON( + ctx, + c.RequestInterface, + rb, + &updatedRequest, + request.WithAction("failed to update request"), + ); err != nil { + return nil, err + } + return &updatedRequest, nil +} + +func loadPolicies(filename string) ([]CreateRequestRequest, error) { + content, err := policyFS.ReadFile("policies/" + filename) + if err != nil { + return nil, err + } + + var config PolicyConfig + if err := yaml.Unmarshal(content, &config); err != nil { + return nil, err + } + return config.Policies, nil +} + +func formatPolicy(policy CreateRequestRequest, projectID string, username string) CreateRequestRequest { + p := policy + if username != "" { + p.Username = username + } + + if projectID != "" { + parts := strings.SplitN(projectID, "-", 2) + if len(parts) == 2 { + program := parts[0] + project := parts[1] + + newPaths := make([]string, len(p.ResourcePaths)) + for i, path := range p.ResourcePaths { + r := strings.ReplaceAll(path, "PROGRAM", program) + r = strings.ReplaceAll(r, "PROJECT", project) + newPaths[i] = r + } + p.ResourcePaths = newPaths + } + p.ResourceDisplayName = projectID + } + return p +} + +func formatPolicyForResource(policy CreateRequestRequest, resource ProjectResource, username string) CreateRequestRequest { + p := policy + if username != "" { + p.Username = username + } + + if resource.ResourcePath != "" { + newPaths := make([]string, len(p.ResourcePaths)) + for i, path := range p.ResourcePaths { + if strings.Contains(path, "PROGRAM") || strings.Contains(path, "PROJECT") { + newPaths[i] = resource.ResourcePath + continue + } + newPaths[i] = path + } + p.ResourcePaths = newPaths + p.ResourceDisplayName = resource.DisplayName() + } + return p +} + +func ParseProjectResources(raw string) ([]ProjectResource, error) { + raw = extractProjectResourceList(raw) + parts := strings.FieldsFunc(raw, func(r rune) bool { + return r == ',' || r == '\n' || r == '\r' + }) + + seen := map[string]struct{}{} + resources := make([]ProjectResource, 0, len(parts)) + for _, part := range parts { + trimmed := strings.TrimSpace(strings.Trim(part, `"'`)) + if trimmed == "" || strings.HasPrefix(trimmed, "(") { + continue + } + resource, err := ParseProjectResource(part) + if err != nil { + return nil, err + } + if _, ok := seen[resource.ResourcePath]; ok { + continue + } + seen[resource.ResourcePath] = struct{}{} + resources = append(resources, resource) + } + if len(resources) == 0 { + return nil, fmt.Errorf("no project resources found") + } + return resources, nil +} + +func extractProjectResourceList(raw string) string { + markers := []string{ + "copy/paste scope list:", + "missing organization/project scopes:", + "denied organization/project scopes:", + "denied resource paths:", + } + lower := strings.ToLower(raw) + start := -1 + markerLen := 0 + for _, marker := range markers { + if idx := strings.LastIndex(lower, marker); idx >= 0 && idx >= start { + start = idx + markerLen = len(marker) + } + } + if start >= 0 { + raw = raw[start+markerLen:] + } + raw = strings.TrimSpace(raw) + if idx := strings.Index(raw, "\r\n\r\n"); idx >= 0 { + raw = raw[:idx] + } + if idx := strings.Index(raw, "\n\n"); idx >= 0 { + raw = raw[:idx] + } + return raw +} + +func ParseProjectResource(raw string) (ProjectResource, error) { + candidate := strings.TrimSpace(raw) + candidate = strings.Trim(candidate, `"'`) + if idx := strings.Index(candidate, "/programs/"); idx >= 0 { + candidate = candidate[idx:] + } else if idx := strings.LastIndex(candidate, ":"); idx >= 0 { + candidate = candidate[idx+1:] + } + candidate = strings.TrimSpace(candidate) + candidate = strings.Trim(candidate, `"'`) + candidate = strings.TrimSuffix(candidate, ".") + if candidate == "" || strings.HasPrefix(candidate, "(") { + return ProjectResource{}, fmt.Errorf("empty resource path") + } + + if parsed, err := url.Parse(candidate); err == nil && parsed.Path != "" { + candidate = parsed.Path + } + + trimmed := strings.Trim(candidate, "/") + if !strings.HasPrefix(trimmed, "programs/") && strings.Count(trimmed, "/") == 1 { + pair := strings.SplitN(trimmed, "/", 2) + return projectResource(pair[0], pair[1]) + } + + parts := strings.Split(trimmed, "/") + if len(parts) < 4 || parts[0] != "programs" || parts[2] != "projects" { + return ProjectResource{}, fmt.Errorf("invalid project resource %q; expected /programs//projects/", raw) + } + return projectResource(parts[1], parts[3]) +} + +func projectResource(program, project string) (ProjectResource, error) { + program = strings.TrimSpace(program) + project = strings.TrimSpace(project) + if program == "" || project == "" { + return ProjectResource{}, fmt.Errorf("project resource requires both organization and project") + } + if strings.ContainsAny(program, " \t") || strings.ContainsAny(project, " \t") { + return ProjectResource{}, fmt.Errorf("project resource contains whitespace: %s/%s", program, project) + } + return ProjectResource{ + Program: program, + Project: project, + ResourcePath: "/programs/" + program + "/projects/" + project, + }, nil +} + +func (c *RequestorClient) getPolicyKey(p CreateRequestRequest) string { + roles := make([]string, len(p.RoleIDs)) + copy(roles, p.RoleIDs) + sort.Strings(roles) + + paths := make([]string, len(p.ResourcePaths)) + copy(paths, p.ResourcePaths) + sort.Strings(paths) + + return fmt.Sprintf("%s:%s:%s", p.PolicyID, strings.Join(roles, ","), strings.Join(paths, ",")) +} + +func (c *RequestorClient) AddUser(ctx context.Context, projectID string, username string, write bool, guppy bool) ([]Request, error) { + uniquePolicies := make(map[string]CreateRequestRequest) + + addFrom := func(fileName string) error { + pols, err := loadPolicies(fileName) + if err != nil { + return err + } + for _, p := range pols { + formatted := formatPolicy(p, projectID, username) + key := c.getPolicyKey(formatted) + uniquePolicies[key] = formatted + } + return nil + } + + // Always add read + if err := addFrom("add-user-read.yaml"); err != nil { + return nil, fmt.Errorf("failed to load read policy: %w", err) + } + + if write { + if err := addFrom("add-user-write.yaml"); err != nil { + return nil, fmt.Errorf("failed to load write policy: %w", err) + } + } + if guppy { + if err := addFrom("add-user-guppy-admin.yaml"); err != nil { + return nil, fmt.Errorf("failed to load guppy policy: %w", err) + } + } + + var createdRequests []Request + for _, formatted := range uniquePolicies { + req, err := c.CreateRequest(ctx, formatted, false) + if err != nil { + return createdRequests, fmt.Errorf("failed to create request for policy %v: %w", formatted, err) + } + createdRequests = append(createdRequests, *req) + } + return createdRequests, nil +} + +func (c *RequestorClient) AddUserToResources(ctx context.Context, resources []ProjectResource, username string, write bool, guppy bool) ([]Request, error) { + uniquePolicies := make(map[string]CreateRequestRequest) + + addFrom := func(fileName string, resource ProjectResource) error { + pols, err := loadPolicies(fileName) + if err != nil { + return err + } + for _, p := range pols { + formatted := formatPolicyForResource(p, resource, username) + key := c.getPolicyKey(formatted) + uniquePolicies[key] = formatted + } + return nil + } + + for _, resource := range resources { + if err := addFrom("add-user-read.yaml", resource); err != nil { + return nil, fmt.Errorf("failed to load read policy for %s: %w", resource.DisplayName(), err) + } + if write { + if err := addFrom("add-user-write.yaml", resource); err != nil { + return nil, fmt.Errorf("failed to load write policy for %s: %w", resource.DisplayName(), err) + } + } + if guppy { + if err := addFrom("add-user-guppy-admin.yaml", resource); err != nil { + return nil, fmt.Errorf("failed to load guppy policy for %s: %w", resource.DisplayName(), err) + } + } + } + + var createdRequests []Request + for _, formatted := range uniquePolicies { + req, err := c.CreateRequest(ctx, formatted, false) + if err != nil { + return createdRequests, fmt.Errorf("failed to create request for policy %v: %w", formatted, err) + } + createdRequests = append(createdRequests, *req) + } + return createdRequests, nil +} + +func (c *RequestorClient) RemoveUser(ctx context.Context, projectID string, username string) ([]Request, error) { + uniquePolicies := make(map[string]CreateRequestRequest) + + addFrom := func(fileName string) error { + pols, err := loadPolicies(fileName) + if err != nil { + return err + } + for _, p := range pols { + formatted := formatPolicy(p, projectID, username) + key := c.getPolicyKey(formatted) + uniquePolicies[key] = formatted + } + return nil + } + + // Revoke read and write + if err := addFrom("add-user-read.yaml"); err != nil { + return nil, fmt.Errorf("failed to load read policy: %w", err) + } + + if err := addFrom("add-user-write.yaml"); err != nil { + return nil, fmt.Errorf("failed to load write policy: %w", err) + } + + var createdRequests []Request + for _, formatted := range uniquePolicies { + req, err := c.CreateRequest(ctx, formatted, true) // revoke=true + if err != nil { + return createdRequests, fmt.Errorf("failed to revoke request: %w", err) + } + createdRequests = append(createdRequests, *req) + } + return createdRequests, nil +} diff --git a/requestor/client_test.go b/requestor/client_test.go new file mode 100644 index 0000000..5d85c5f --- /dev/null +++ b/requestor/client_test.go @@ -0,0 +1,419 @@ +package requestor + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/calypr/calypr-cli/request" +) + +type fakeRequest struct { + doFn func(rb *request.RequestBuilder) (*http.Response, error) +} + +func (f *fakeRequest) New(method, u string) *request.RequestBuilder { + return &request.RequestBuilder{Method: method, Url: u, Headers: map[string]string{}} +} + +func (f *fakeRequest) Do(ctx context.Context, rb *request.RequestBuilder) (*http.Response, error) { + return f.doFn(rb) +} + +func jsonResponse(status int, v any) *http.Response { + var body io.ReadCloser = http.NoBody + if v != nil { + buf, _ := json.Marshal(v) + body = io.NopCloser(bytes.NewReader(buf)) + } + return &http.Response{StatusCode: status, Body: body} +} + +func TestRequestorClientStatusError(t *testing.T) { + err := request.StatusError(jsonResponse(http.StatusBadRequest, map[string]string{"error": "bad request"}), "failed") + if err == nil || !strings.Contains(err.Error(), "failed: status 400") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestGetPolicyKey(t *testing.T) { + c := &RequestorClient{} + + p1 := CreateRequestRequest{ + PolicyID: "p1", + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + p2 := CreateRequestRequest{ + PolicyID: "p1", + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + p3 := CreateRequestRequest{ + PolicyID: "p2", + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + + if c.getPolicyKey(p1) != c.getPolicyKey(p2) { + t.Errorf("Expected p1 and p2 to have same key") + } + if c.getPolicyKey(p1) == c.getPolicyKey(p3) { + t.Errorf("Expected p1 and p3 to have different keys (PolicyID differs)") + } + + p4 := CreateRequestRequest{ + RoleIDs: []string{"a", "b"}, + ResourcePaths: []string{"/p1", "/p2"}, + } + p5 := CreateRequestRequest{ + RoleIDs: []string{"b", "a"}, + ResourcePaths: []string{"/p2", "/p1"}, + } + if c.getPolicyKey(p4) != c.getPolicyKey(p5) { + t.Errorf("Expected p4 and p5 to have same key (sorting check)") + } + + p6 := CreateRequestRequest{ + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + p7 := CreateRequestRequest{ + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + if c.getPolicyKey(p6) != c.getPolicyKey(p7) { + t.Errorf("Expected p6 and p7 (empty PolicyID) to have same key") + } +} + +func TestLoadPoliciesAndFormatPolicy(t *testing.T) { + policies, err := loadPolicies("add-user-read.yaml") + if err != nil { + t.Fatalf("loadPolicies returned error: %v", err) + } + if len(policies) == 0 { + t.Fatal("expected add-user-read.yaml to contain at least one policy") + } + + if _, err := loadPolicies("missing.yaml"); err == nil { + t.Fatal("expected missing policy file to return an error") + } + + formatted := formatPolicy( + CreateRequestRequest{ + PolicyID: "policy-1", + ResourcePaths: []string{"/PROGRAM/projects/PROJECT/data"}, + ResourceDisplayName: "original", + }, + "demo-program-demo-project", + "alice", + ) + if formatted.Username != "alice" { + t.Fatalf("expected username to be replaced, got %q", formatted.Username) + } + if formatted.ResourceDisplayName != "demo-program-demo-project" { + t.Fatalf("expected display name to be replaced, got %q", formatted.ResourceDisplayName) + } + if got := formatted.ResourcePaths[0]; got != "/demo/projects/program-demo-project/data" { + t.Fatalf("unexpected formatted path: %q", got) + } +} + +func TestParseProjectResources(t *testing.T) { + resources, err := ParseProjectResources(`denied resource paths: /programs/HTAN_INT/projects/BForePC, /programs/cbds/projects/git_drs_test, aced/evotypes`) + if err != nil { + t.Fatalf("ParseProjectResources returned error: %v", err) + } + if len(resources) != 3 { + t.Fatalf("expected 3 resources, got %+v", resources) + } + want := []string{ + "/programs/HTAN_INT/projects/BForePC", + "/programs/cbds/projects/git_drs_test", + "/programs/aced/projects/evotypes", + } + for i, expected := range want { + if resources[i].ResourcePath != expected { + t.Fatalf("resource %d = %q, want %q", i, resources[i].ResourcePath, expected) + } + } +} + +func TestParseProjectResourcesFromScopeMessage(t *testing.T) { + resources, err := ParseProjectResources(`denied organization/project scopes: HTAN_INT/BForePC, cbds/git_drs_test`) + if err != nil { + t.Fatalf("ParseProjectResources returned error: %v", err) + } + if len(resources) != 2 { + t.Fatalf("expected 2 resources, got %+v", resources) + } + if got := resources[0].ResourcePath; got != "/programs/HTAN_INT/projects/BForePC" { + t.Fatalf("unexpected first resource: %q", got) + } +} + +func TestParseProjectResourcesFromPreflightCopyPasteBlock(t *testing.T) { + raw := `migration import preflight failed: missing create access for 1320/82628 records across 323 scopes +first denied record: 04ae835f-9a18-5867-8a63-fa527dedf425 + +Copy/paste scope list: +Ellrott_Lab/embedding_rotation, HTAN_INT/TestUpload1, cbds/017549802cd04d108524ae3f196ffacd + +2026/05/01 12:01:42 ERROR command execution failed err="target authorization preflight failed: missing create access"` + + resources, err := ParseProjectResources(raw) + if err != nil { + t.Fatalf("ParseProjectResources returned error: %v", err) + } + if len(resources) != 3 { + t.Fatalf("expected 3 resources, got %+v", resources) + } + want := []string{ + "/programs/Ellrott_Lab/projects/embedding_rotation", + "/programs/HTAN_INT/projects/TestUpload1", + "/programs/cbds/projects/017549802cd04d108524ae3f196ffacd", + } + for i, expected := range want { + if resources[i].ResourcePath != expected { + t.Fatalf("resource %d = %q, want %q", i, resources[i].ResourcePath, expected) + } + } +} + +func TestRequestorClientListCreateAndUpdate(t *testing.T) { + client := &RequestorClient{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + + switch { + case rb.Method == http.MethodGet && u.Path == "/requestor/request": + if !u.Query().Has("active") { + t.Fatalf("expected active query parameter") + } + if got := u.Query().Get("username"); got != "bob" { + t.Fatalf("expected username query bob, got %q", got) + } + return jsonResponse(http.StatusOK, []Request{{RequestID: "req-list", Status: "active"}}), nil + case rb.Method == http.MethodPost && u.Path == "/requestor/request": + body, _ := io.ReadAll(rb.Body) + var payload CreateRequestRequest + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode create payload: %v", err) + } + if payload.PolicyID == "" { + t.Fatal("expected create payload policy id") + } + return jsonResponse(http.StatusOK, Request{RequestID: "req-create", Status: "open"}), nil + case rb.Method == http.MethodPut && strings.HasPrefix(u.Path, "/requestor/request/"): + body, _ := io.ReadAll(rb.Body) + var payload UpdateRequestRequest + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode update payload: %v", err) + } + if payload.Status != "approved" { + t.Fatalf("expected approved status, got %q", payload.Status) + } + return jsonResponse(http.StatusOK, Request{RequestID: "req-update", Status: payload.Status}), nil + default: + return jsonResponse(http.StatusNotFound, nil), nil + } + }, + }, + Endpoint: "https://example.org", + } + + list, err := client.ListRequests(context.Background(), false, true, "bob") + if err != nil { + t.Fatalf("ListRequests failed: %v", err) + } + if len(list) != 1 || list[0].RequestID != "req-list" { + t.Fatalf("unexpected list response: %+v", list) + } + + created, err := client.CreateRequest(context.Background(), CreateRequestRequest{ + PolicyID: "policy-1", + ResourcePaths: []string{"/path"}, + RoleIDs: []string{"reader"}, + }, false) + if err != nil { + t.Fatalf("CreateRequest failed: %v", err) + } + if created.RequestID != "req-create" { + t.Fatalf("unexpected create response: %+v", created) + } + + updated, err := client.UpdateRequest(context.Background(), "req-1", "approved") + if err != nil { + t.Fatalf("UpdateRequest failed: %v", err) + } + if updated.Status != "approved" { + t.Fatalf("unexpected update response: %+v", updated) + } +} + +func TestRequestorClientAddAndRemoveUser(t *testing.T) { + var revokeCount int + + client := &RequestorClient{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + if rb.Method != http.MethodPost || u.Path != "/requestor/request" { + return jsonResponse(http.StatusNotFound, nil), nil + } + if u.Query().Has("revoke") { + revokeCount++ + } + return jsonResponse(http.StatusOK, Request{RequestID: "req-ok", Status: "open"}), nil + }, + }, + Endpoint: "https://example.org", + } + + added, err := client.AddUser(context.Background(), "demo-program-demo-project", "bob", true, true) + if err != nil { + t.Fatalf("AddUser failed: %v", err) + } + if len(added) == 0 { + t.Fatal("expected AddUser to create at least one request") + } + + revoked, err := client.RemoveUser(context.Background(), "demo-program-demo-project", "bob") + if err != nil { + t.Fatalf("RemoveUser failed: %v", err) + } + if len(revoked) == 0 { + t.Fatal("expected RemoveUser to create at least one revocation request") + } + if revokeCount == 0 { + t.Fatal("expected revoke query parameter to be seen for revocation requests") + } +} + +func TestRequestorClientAddUserPreservesEmbeddedDashInProject(t *testing.T) { + var payloads []CreateRequestRequest + + client := &RequestorClient{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + if rb.Method != http.MethodPost || u.Path != "/requestor/request" { + return jsonResponse(http.StatusNotFound, nil), nil + } + + var payload CreateRequestRequest + if err := json.NewDecoder(rb.Body).Decode(&payload); err != nil { + return nil, err + } + payloads = append(payloads, payload) + return jsonResponse(http.StatusOK, Request{RequestID: "req-ok", Status: "open"}), nil + }, + }, + Endpoint: "https://example.org", + } + + created, err := client.AddUser(context.Background(), "cbds-aaa-bbb", "bob@example.org", true, true) + if err != nil { + t.Fatalf("AddUser failed: %v", err) + } + if len(created) == 0 { + t.Fatal("expected AddUser to create requests") + } + if len(payloads) == 0 { + t.Fatal("expected request payloads to be captured") + } + + wantPath := "/programs/cbds/projects/aaa-bbb" + foundProjectScopedPayload := false + for _, payload := range payloads { + if payload.ResourceDisplayName != "cbds-aaa-bbb" { + t.Fatalf("unexpected display name: %+v", payload) + } + + hasProjectPlaceholderPath := false + found := false + for _, path := range payload.ResourcePaths { + if strings.Contains(path, "/programs/") { + hasProjectPlaceholderPath = true + } + if path == wantPath || strings.Contains(path, wantPath+"/") { + found = true + break + } + } + if hasProjectPlaceholderPath && !found { + t.Fatalf("expected payload to target %q, got %+v", wantPath, payload.ResourcePaths) + } + if found { + foundProjectScopedPayload = true + } + } + if !foundProjectScopedPayload { + t.Fatalf("expected at least one project-scoped payload to target %q", wantPath) + } +} + +func TestAddUserToResourcesCreatesRequestsPerResource(t *testing.T) { + var payloads []CreateRequestRequest + client := &RequestorClient{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + if rb.Method != http.MethodPost { + return jsonResponse(http.StatusMethodNotAllowed, nil), nil + } + var payload CreateRequestRequest + if err := json.NewDecoder(rb.Body).Decode(&payload); err != nil { + return nil, err + } + payloads = append(payloads, payload) + return jsonResponse(http.StatusOK, Request{RequestID: "req-ok", Status: "open"}), nil + }, + }, + Endpoint: "https://example.org", + } + + resources, err := ParseProjectResources("/programs/cbds/projects/git_drs_test, /programs/aced/projects/evotypes") + if err != nil { + t.Fatalf("ParseProjectResources returned error: %v", err) + } + created, err := client.AddUserToResources(context.Background(), resources, "bob@example.org", true, true) + if err != nil { + t.Fatalf("AddUserToResources failed: %v", err) + } + if len(created) != 5 { + t.Fatalf("expected 5 deduplicated requests, got %d", len(created)) + } + + seen := map[string]bool{} + for _, payload := range payloads { + if payload.Username != "bob@example.org" { + t.Fatalf("unexpected username in payload: %+v", payload) + } + seen[strings.Join(payload.RoleIDs, ",")+"|"+strings.Join(payload.ResourcePaths, ",")] = true + } + if !seen["reader|/programs/cbds/projects/git_drs_test"] { + t.Fatalf("missing reader request for cbds/git_drs_test: %+v", payloads) + } + if !seen["writer|/programs/aced/projects/evotypes"] { + t.Fatalf("missing writer request for aced/evotypes: %+v", payloads) + } + if !seen["guppy_admin_user|/guppy_admin"] { + t.Fatalf("missing deduplicated guppy admin request: %+v", payloads) + } +} diff --git a/requestor/policies/add-user-guppy-admin.yaml b/requestor/policies/add-user-guppy-admin.yaml new file mode 100644 index 0000000..fb544df --- /dev/null +++ b/requestor/policies/add-user-guppy-admin.yaml @@ -0,0 +1,9 @@ +policies: +- role_ids: + - writer + resource_paths: + - /programs/PROGRAM/projects/PROJECT +- role_ids: + - guppy_admin_user + resource_paths: + - /guppy_admin diff --git a/requestor/policies/add-user-read.yaml b/requestor/policies/add-user-read.yaml new file mode 100644 index 0000000..e7acb34 --- /dev/null +++ b/requestor/policies/add-user-read.yaml @@ -0,0 +1,5 @@ +policies: +- role_ids: + - reader + resource_paths: + - /programs/PROGRAM/projects/PROJECT diff --git a/requestor/policies/add-user-write.yaml b/requestor/policies/add-user-write.yaml new file mode 100644 index 0000000..8fda383 --- /dev/null +++ b/requestor/policies/add-user-write.yaml @@ -0,0 +1,5 @@ +policies: +- role_ids: + - writer + resource_paths: + - /programs/PROGRAM/projects/PROJECT diff --git a/requestor/types.go b/requestor/types.go new file mode 100644 index 0000000..c3c39c0 --- /dev/null +++ b/requestor/types.go @@ -0,0 +1,47 @@ +package requestor + +// Request represents a requestor request object +type Request struct { + RequestID string `json:"request_id,omitempty" yaml:"request_id,omitempty"` + Username string `json:"username,omitempty" yaml:"username,omitempty"` + PolicyID string `json:"policy_id,omitempty" yaml:"policy_id,omitempty"` + ResourcePaths []string `json:"resource_paths,omitempty" yaml:"resource_paths,omitempty"` + RoleIDs []string `json:"role_ids,omitempty" yaml:"role_ids,omitempty"` + ResourceID string `json:"resource_id,omitempty" yaml:"resource_id,omitempty"` + ResourceDisplay string `json:"resource_display_name,omitempty" yaml:"resource_display_name,omitempty"` + Status string `json:"status,omitempty" yaml:"status,omitempty"` + CreatedTime string `json:"created_time,omitempty" yaml:"created_time,omitempty"` + UpdatedTime string `json:"updated_time,omitempty" yaml:"updated_time,omitempty"` + Revoke bool `json:"revoke,omitempty" yaml:"revoke,omitempty"` +} + +// CreateRequestRequest represents the payload to create a request +type CreateRequestRequest struct { + Username string `json:"username,omitempty" yaml:"username,omitempty"` + PolicyID string `json:"policy_id,omitempty" yaml:"policy_id,omitempty"` + ResourcePaths []string `json:"resource_paths,omitempty" yaml:"resource_paths,omitempty"` + RoleIDs []string `json:"role_ids,omitempty" yaml:"role_ids,omitempty"` + ResourceDisplayName string `json:"resource_display_name,omitempty" yaml:"resource_display_name,omitempty"` +} + +// UpdateRequestRequest represents the payload to update a request +type UpdateRequestRequest struct { + Status string `json:"status" yaml:"status"` +} + +type PolicyConfig struct { + Policies []CreateRequestRequest `yaml:"policies"` +} + +type ProjectResource struct { + Program string + Project string + ResourcePath string +} + +func (r ProjectResource) DisplayName() string { + if r.Program == "" || r.Project == "" { + return r.ResourcePath + } + return r.Program + "/" + r.Project +} diff --git a/sower/client.go b/sower/client.go new file mode 100644 index 0000000..99f0901 --- /dev/null +++ b/sower/client.go @@ -0,0 +1,119 @@ +package sower + +import ( + "context" + "net/http" + "net/url" + + "github.com/calypr/calypr-cli/request" +) + +const ( + sowerDispatch = "/job/dispatch" + sowerStatus = "/job/status" + sowerList = "/job/list" + sowerJobOutput = "/job/output" +) + +type SowerInterface interface { + DispatchJob(ctx context.Context, name string, args *DispatchArgs) (*StatusResp, error) + Status(ctx context.Context, uid string) (*StatusResp, error) + List(ctx context.Context) ([]StatusResp, error) + Output(ctx context.Context, uid string) (*OutputResp, error) +} + +type SowerClient struct { + request.RequestInterface + Endpoint string +} + +func NewSowerClient(req request.RequestInterface, endpoint string) *SowerClient { + return &SowerClient{ + RequestInterface: req, + Endpoint: endpoint, + } +} + +func (sc *SowerClient) fullURL(path string) string { + return request.JoinURL(sc.Endpoint, path) +} + +func (sc *SowerClient) DispatchJob(ctx context.Context, name string, args *DispatchArgs) (*StatusResp, error) { + body := JobArgs{ + Action: name, + Input: *args, + } + + rb, err := request.NewJSON(sc.RequestInterface, http.MethodPost, sc.fullURL(sowerDispatch), body) + if err != nil { + return nil, err + } + + statusResp := &StatusResp{} + if err := request.DoJSON( + ctx, + sc.RequestInterface, + rb, + statusResp, + request.WithAction("sower dispatch failed"), + request.WithExpectedStatus(http.StatusOK), + ); err != nil { + return nil, err + } + return statusResp, nil +} + +func (sc *SowerClient) Status(ctx context.Context, uid string) (*StatusResp, error) { + u, _ := url.Parse(sc.fullURL(sowerStatus)) + q := u.Query() + q.Add("UID", uid) + u.RawQuery = q.Encode() + + statusResp := &StatusResp{} + if err := request.DoJSON( + ctx, + sc.RequestInterface, + sc.New(http.MethodGet, u.String()), + statusResp, + request.WithAction("sower status failed"), + request.WithExpectedStatus(http.StatusOK), + ); err != nil { + return nil, err + } + return statusResp, nil +} + +func (sc *SowerClient) Output(ctx context.Context, uid string) (*OutputResp, error) { + u, _ := url.Parse(sc.fullURL(sowerJobOutput)) + q := u.Query() + q.Add("UID", uid) + u.RawQuery = q.Encode() + + var outputResp OutputResp + if err := request.DoJSON( + ctx, + sc.RequestInterface, + sc.New(http.MethodGet, u.String()), + &outputResp, + request.WithAction("sower output failed"), + request.WithExpectedStatus(http.StatusOK), + ); err != nil { + return nil, err + } + return &outputResp, nil +} + +func (sc *SowerClient) List(ctx context.Context) ([]StatusResp, error) { + var listResp []StatusResp + if err := request.DoJSON( + ctx, + sc.RequestInterface, + sc.New(http.MethodGet, sc.fullURL(sowerList)), + &listResp, + request.WithAction("sower list failed"), + request.WithExpectedStatus(http.StatusOK), + ); err != nil { + return nil, err + } + return listResp, nil +} diff --git a/sower/client_test.go b/sower/client_test.go new file mode 100644 index 0000000..fa5085b --- /dev/null +++ b/sower/client_test.go @@ -0,0 +1,122 @@ +package sower + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "testing" + + "github.com/calypr/calypr-cli/request" + "github.com/hashicorp/go-retryablehttp" +) + +type fakeSowerRequest struct { + doFn func(rb *request.RequestBuilder) (*http.Response, error) +} + +func (f *fakeSowerRequest) New(method, u string) *request.RequestBuilder { + return &request.RequestBuilder{Method: method, Url: u, Headers: map[string]string{}} +} + +func (f *fakeSowerRequest) Do(ctx context.Context, rb *request.RequestBuilder) (*http.Response, error) { + return f.doFn(rb) +} + +func jsonResp(status int, v any) *http.Response { + var body io.ReadCloser = http.NoBody + if v != nil { + buf, _ := json.Marshal(v) + body = io.NopCloser(bytes.NewReader(buf)) + } + return &http.Response{StatusCode: status, Body: body} +} + +func TestSowerClientOperations(t *testing.T) { + client := &SowerClient{ + RequestInterface: &fakeSowerRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + switch { + case rb.Method == http.MethodPost && u.Path == sowerDispatch: + var payload JobArgs + if err := json.NewDecoder(rb.Body).Decode(&payload); err != nil { + t.Fatalf("decode dispatch payload: %v", err) + } + if payload.Action != "dispatch" { + t.Fatalf("unexpected dispatch action: %q", payload.Action) + } + if payload.Input.Profile != "profile-a" { + t.Fatalf("unexpected profile: %q", payload.Input.Profile) + } + return jsonResp(http.StatusOK, StatusResp{Uid: "uid-1", Name: "job-1", Status: "running"}), nil + case rb.Method == http.MethodGet && u.Path == sowerStatus: + if got := u.Query().Get("UID"); got != "uid-1" { + t.Fatalf("unexpected UID query: %q", got) + } + return jsonResp(http.StatusOK, StatusResp{Uid: "uid-1", Name: "job-1", Status: "complete"}), nil + case rb.Method == http.MethodGet && u.Path == sowerJobOutput: + if got := u.Query().Get("UID"); got != "uid-1" { + t.Fatalf("unexpected UID query: %q", got) + } + return jsonResp(http.StatusOK, OutputResp{Output: "done"}), nil + case rb.Method == http.MethodGet && u.Path == sowerList: + return jsonResp(http.StatusOK, []StatusResp{{Uid: "uid-1", Name: "job-1", Status: "complete"}}), nil + default: + return jsonResp(http.StatusNotFound, nil), nil + } + }, + }, + Endpoint: "https://example.org", + } + + status, err := client.DispatchJob(context.Background(), "dispatch", &DispatchArgs{ + Profile: "profile-a", + APIEndpoint: "https://example.org", + }) + if err != nil { + t.Fatalf("DispatchJob failed: %v", err) + } + if status.Uid != "uid-1" { + t.Fatalf("unexpected dispatch result: %+v", status) + } + + gotStatus, err := client.Status(context.Background(), "uid-1") + if err != nil { + t.Fatalf("Status failed: %v", err) + } + if gotStatus.Status != "complete" { + t.Fatalf("unexpected status result: %+v", gotStatus) + } + + output, err := client.Output(context.Background(), "uid-1") + if err != nil { + t.Fatalf("Output failed: %v", err) + } + if output.Output != "done" { + t.Fatalf("unexpected output result: %+v", output) + } + + list, err := client.List(context.Background()) + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(list) != 1 || list[0].Uid != "uid-1" { + t.Fatalf("unexpected list result: %+v", list) + } +} + +func TestNewSowerClientBuildsEndpoint(t *testing.T) { + req := &request.Request{ + RetryClient: &retryablehttp.Client{HTTPClient: &http.Client{}}, + } + client := NewSowerClient(req, "https://example.org") + if client.fullURL(sowerDispatch) != "https://example.org/job/dispatch" { + t.Fatalf("unexpected full URL: %s", client.fullURL(sowerDispatch)) + } +} diff --git a/sower/types.go b/sower/types.go new file mode 100644 index 0000000..7b735a7 --- /dev/null +++ b/sower/types.go @@ -0,0 +1,33 @@ +package sower + +type StatusResp struct { + Uid string `json:"uid"` + Name string `json:"name"` + Status string `json:"status"` +} + +type OutputResp struct { + Output string `json:"output"` +} + +type File struct { + FileTitle string `json:"fileTitle,omitempty"` + FilePath string `json:"filePath"` +} + +type DispatchArgs struct { + Method string `json:"method"` + ProjectId string `json:"projectId"` + Profile string `json:"profile"` + BucketName string `json:"bucketName"` + APIEndpoint string `json:"APIEndpoint"` + GHCommitHash string `json:"ghCommitHash"` + GHPAccessToken string `json:"ghToken"` + GHUserName string `json:"ghUserName"` + GHRepoURL string `json:"ghRepoUrl"` +} + +type JobArgs struct { + Input DispatchArgs `json:"input"` + Action string `json:"action"` +} diff --git a/tests/download-multiple_test.go b/tests/download-multiple_test.go index 0113935..aca1d49 100644 --- a/tests/download-multiple_test.go +++ b/tests/download-multiple_test.go @@ -1,183 +1,94 @@ package tests import ( + "context" "fmt" - "io" - "net/http" - "os" - "strings" "testing" - "github.com/calypr/data-client/client/common" - g3cmd "github.com/calypr/data-client/client/g3cmd" - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/logs" - "github.com/calypr/data-client/client/mocks" - "go.uber.org/mock/gomock" + sylogs "github.com/calypr/syfon/client/logs" + "github.com/calypr/syfon/client/transfer" + "github.com/calypr/syfon/client/transfer/download" ) -// Add all other methods required by your logs.Logger interface! +type fakeResolver struct { + obj *transfer.ResolvedObject + err error +} + +func (f *fakeResolver) Resolve(ctx context.Context, id string) (*transfer.ResolvedObject, error) { + return f.obj, f.err +} -// If Shepherd is deployed, attempt to get the filename from the Shepherd API. func Test_askGen3ForFileInfo_withShepherd(t *testing.T) { - // -- SETUP -- testGUID := "000000-0000000-0000000-000000" testFileName := "test-file" testFileSize := int64(120) - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Expect AskGen3ForFileInfo to call shepherd looking for testGUID: respond with a valid file. - testBody := `{ - "record": { - "file_name": "test-file", - "size": 120, - "did": "000000-0000000-0000000-000000" - }, - "metadata": { - "_file_type": "PFB", - "_resource_paths": ["/open"], - "_uploader_id": 42, - "_bucket": "s3://gen3-bucket" + + logger := sylogs.NewGen3Logger(nil, "", "test") + + skipped := []download.RenamedOrSkippedFileInfo{} + resolver := &fakeResolver{obj: &transfer.ResolvedObject{Id: testGUID, Name: testFileName, Size: testFileSize}} + info, err := download.GetFileInfo(context.Background(), resolver, logger, testGUID, "", "", "original", true, &skipped) + if err != nil { + t.Error(err) } -}` - testResponse := http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader(testBody)), + + if info.Name != testFileName { + t.Errorf("Wanted filename %v, got %v", testFileName, info.Name) } - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(true, nil) - mockGen3Interface. - EXPECT(). - GetResponse(common.ShepherdEndpoint+"/objects/"+testGUID, "GET", "", nil). - Return("", &testResponse, nil) - // ---------- - - // Expect AskGen3ForFileInfo to return the correct filename and filesize from shepherd. - fileName, fileSize := g3cmd.AskGen3ForFileInfo(mockGen3Interface, testGUID, "", "", "original", true, &[]g3cmd.RenamedOrSkippedFileInfo{}) - if fileName != testFileName { - t.Errorf("Wanted filename %v, got %v", testFileName, fileName) + if info.Size != testFileSize { + t.Errorf("Wanted filesize %v, got %v", testFileSize, info.Size) } - if fileSize != testFileSize { - t.Errorf("Wanted filesize %v, got %v", testFileSize, fileSize) + if len(skipped) != 0 { + t.Errorf("Expected no skipped files, got %v", skipped) } } -// If there's an error while getting the filename from Shepherd, add the guid -// to *renamedFiles, which tracks which files have errored. func Test_askGen3ForFileInfo_withShepherd_shepherdError(t *testing.T) { - // -- SETUP -- testGUID := "000000-0000000-0000000-000000" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Expect AskGen3ForFileInfo to call indexd looking for testGUID: - // Respond with an error. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(true, nil) - mockGen3Interface. - EXPECT(). - GetResponse(common.ShepherdEndpoint+"/objects/"+testGUID, "GET", "", nil). - Return("", nil, fmt.Errorf("Error getting metadata from Shepherd")) - // ---------- - - mockGen3Interface. - EXPECT(). - Logger(). - Return(logs.NewTeeLogger("", "test", os.Stdout)). // Or your appropriate dummy logger - AnyTimes() - - // Expect AskGen3ForFileInfo to add this file's GUID to the renamedOrSkippedFiles array. - skipped := []g3cmd.RenamedOrSkippedFileInfo{} - fileName, _ := g3cmd.AskGen3ForFileInfo(mockGen3Interface, testGUID, "", "", "original", true, &skipped) - expected := g3cmd.RenamedOrSkippedFileInfo{GUID: testGUID, OldFilename: "N/A", NewFilename: testGUID} - if skipped[0] != expected { - t.Errorf("Wanted skipped files list to contain %v, got %v", expected, skipped) + + logger := sylogs.NewGen3Logger(nil, "", "test") + + skipped := []download.RenamedOrSkippedFileInfo{} + resolver := &fakeResolver{err: fmt.Errorf("Indexd error")} + info, err := download.GetFileInfo(context.Background(), resolver, logger, testGUID, "", "", "original", true, &skipped) + if err != nil { + t.Fatal(err) + } + + if info == nil { + t.Fatal("AskGen3ForFileInfo returned nil when both Shepherd and Indexd failed. Expected fallback FileInfo with Name = GUID") } - // Expect the returned filename to be the file's GUID. - if fileName != testGUID { - t.Errorf("Wanted filename %v, got %v", testGUID, fileName) + + if info.Name != testGUID { + t.Errorf("Wanted fallback filename %v, got %v", testGUID, info.Name) + } + + if len(skipped) != 1 { + t.Errorf("Expected exactly 1 skipped file, got %d", len(skipped)) + } else if skipped[0].GUID != testGUID || skipped[0].NewFilename != testGUID { + t.Errorf("Skipped entry mismatch: %+v", skipped[0]) } } -// If Shepherd is not deployed, attempt to get the filename from indexd. func Test_askGen3ForFileInfo_noShepherd(t *testing.T) { - // -- SETUP -- testGUID := "000000-0000000-0000000-000000" testFileName := "test-file" testFileSize := int64(120) - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Expect AskGen3ForFileInfo to call indexd looking for testGUID: respond with a valid file. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(false, nil) - mockGen3Interface. - EXPECT(). - DoRequestWithSignedHeader(common.IndexdIndexEndpoint+"/"+testGUID, "", nil). - Return(jwt.JsonMessage{FileName: testFileName, Size: testFileSize}, nil) - // ---------- - - mockGen3Interface. - EXPECT(). - Logger(). - Return(logs.NewTeeLogger("", "test", os.Stdout)). // Or your appropriate dummy logger - AnyTimes() - - // Expect AskGen3ForFileInfo to return the correct filename and filesize from indexd. - fileName, fileSize := g3cmd.AskGen3ForFileInfo(mockGen3Interface, testGUID, "", "", "original", true, &[]g3cmd.RenamedOrSkippedFileInfo{}) - if fileName != testFileName { - t.Errorf("Wanted filename %v, got %v", testFileName, fileName) - } - if fileSize != testFileSize { - t.Errorf("Wanted filesize %v, got %v", testFileSize, fileSize) + + logger := sylogs.NewGen3Logger(nil, "", "test") + + skipped := []download.RenamedOrSkippedFileInfo{} + resolver := &fakeResolver{obj: &transfer.ResolvedObject{Id: testGUID, Name: testFileName, Size: testFileSize}} + info, err := download.GetFileInfo(context.Background(), resolver, logger, testGUID, "", "", "original", true, &skipped) + if err != nil { + t.Fatal(err) } -} -// If there's an error while getting the filename from indexd, add the guid -// to *renamedFiles, which tracks which files have errored. -func Test_askGen3ForFileInfo_noShepherd_indexdError(t *testing.T) { - // -- SETUP -- - testGUID := "000000-0000000-0000000-000000" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Expect AskGen3ForFileInfo to call indexd looking for testGUID: - // Respond with an error. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(false, nil) - mockGen3Interface. - EXPECT(). - DoRequestWithSignedHeader(common.IndexdIndexEndpoint+"/"+testGUID, "", nil). - Return(jwt.JsonMessage{}, fmt.Errorf("Error downloading file from Indexd")) - // ---------- - mockGen3Interface. - EXPECT(). - Logger(). - Return(logs.NewTeeLogger("", "test", os.Stdout)). // Or your appropriate dummy logger - AnyTimes() - - // Expect AskGen3ForFileInfo to add this file's GUID to the renamedOrSkippedFiles array. - skipped := []g3cmd.RenamedOrSkippedFileInfo{} - fileName, _ := g3cmd.AskGen3ForFileInfo(mockGen3Interface, testGUID, "", "", "original", true, &skipped) - expected := g3cmd.RenamedOrSkippedFileInfo{GUID: testGUID, OldFilename: "N/A", NewFilename: testGUID} - if skipped[0] != expected { - t.Errorf("Wanted skipped files list to contain %v, got %v", expected, skipped) + if info.Name != testFileName { + t.Errorf("Wanted filename %v, got %v", testFileName, info.Name) } - // Expect the returned filename to be the file's GUID. - if fileName != testGUID { - t.Errorf("Wanted filename %v, got %v", testGUID, fileName) + if info.Size != testFileSize { + t.Errorf("Wanted filesize %v, got %v", testFileSize, info.Size) } } diff --git a/tests/functions_test.go b/tests/functions_test.go deleted file mode 100755 index d1e0982..0000000 --- a/tests/functions_test.go +++ /dev/null @@ -1,254 +0,0 @@ -package tests - -import ( - "bytes" - "fmt" - "io" - "net/http" - "reflect" - "strings" - "testing" - - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/mocks" - "go.uber.org/mock/gomock" -) - -func TestDoRequestWithSignedHeaderNoProfile(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "", AccessToken: "", APIEndpoint: ""} - - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) - - if err == nil { - t.Fail() - } -} - -func TestDoRequestWithSignedHeaderGoodToken(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{Profile: "test", KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com", UseShepherd: "false", MinShepherdVersion: ""} - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"http://www.test.com/user/data/download/test_uuid\"}")), - StatusCode: 200, - } - - mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/data/download/test_uuid", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) - - if err != nil { - t.Fail() - } -} - -func TestDoRequestWithSignedHeaderCreateNewToken(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "", APIEndpoint: "http://www.test.com"} - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), - StatusCode: 200, - } - - mockConfig.EXPECT().UpdateConfigFile(profileConfig).Times(1) - mockRequest.EXPECT().RequestNewAccessToken("http://www.test.com/user/credentials/api/access_token", &profileConfig).Return(nil).Times(1) - mockRequest.EXPECT().MakeARequest(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) - - if err != nil { - t.Fail() - } -} - -func TestDoRequestWithSignedHeaderRefreshToken(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "expired_token", APIEndpoint: "http://www.test.com"} - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), - StatusCode: 401, - } - - mockConfig.EXPECT().UpdateConfigFile(profileConfig).Times(1) - mockRequest.EXPECT().RequestNewAccessToken("http://www.test.com/user/credentials/api/access_token", &profileConfig).Return(nil).Times(1) - mockRequest.EXPECT().MakeARequest(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(2) - - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) - - if err != nil && !strings.Contains(err.Error(), "401") { - t.Fail() - } - -} - -func TestCheckPrivilegesNoProfile(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "", AccessToken: "", APIEndpoint: ""} - - _, _, err := testFunction.CheckPrivileges(&profileConfig) - - if err == nil { - t.Errorf("Expected an error on missing credentials in configuration, but not received") - } -} - -func TestCheckPrivilegesNoAccess(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com"} - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString("{\"project_access\": {}}")), - StatusCode: 200, - } - - mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/user", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, receivedAccess, err := testFunction.CheckPrivileges(&profileConfig) - - expectedAccess := make(map[string]any) - - if err != nil { - t.Errorf("Expected no errors, received an error \"%v\"", err) - } else if !reflect.DeepEqual(receivedAccess, expectedAccess) { - t.Errorf("Expected no user access, received %v", receivedAccess) - } -} - -func TestCheckPrivilegesGrantedAccess(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com"} - - grantedAccessJSON := `{ - "project_access": - { - "test_project": ["read", "create","read-storage","update","delete"] - } - }` - - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString(grantedAccessJSON)), - StatusCode: 200, - } - - mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/user", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, expectedAccess, err := testFunction.CheckPrivileges(&profileConfig) - - receivedAccess := make(map[string]any) - receivedAccess["test_project"] = []any{ - "read", - "create", - "read-storage", - "update", - "delete"} - - if err != nil { - t.Errorf("Expected no errors, received an error \"%v\"", err) - } else if !reflect.DeepEqual(expectedAccess, receivedAccess) { - t.Errorf(`Expected user access and received user access are not the same. - Expected: %v - Received: %v`, expectedAccess, receivedAccess) - } -} - -// If both `authz` and `project_access` section exists, `authz` takes precedence -func TestCheckPrivilegesGrantedAccessAuthz(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com"} - - grantedAccessJSON := `{ - "authz": { - "test_project":[ - {"method":"create", "service":"*"}, - {"method":"delete", "service":"*"}, - {"method":"read", "service":"*"}, - {"method":"read-storage", "service":"*"}, - {"method":"update", "service":"*"}, - {"method":"upload", "service":"*"} - ] - }, - "project_access": { - "test_project": ["read", "create","read-storage","update","delete"] - } - }` - - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString(grantedAccessJSON)), - StatusCode: 200, - } - - mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/user", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, expectedAccess, err := testFunction.CheckPrivileges(&profileConfig) - - receivedAccess := make(map[string]any) - receivedAccess["test_project"] = []map[string]any{ - {"method": "create", "service": "*"}, - {"method": "delete", "service": "*"}, - {"method": "read", "service": "*"}, - {"method": "read-storage", "service": "*"}, - {"method": "update", "service": "*"}, - {"method": "upload", "service": "*"}, - } - - if err != nil { - t.Errorf("Expected no errors, received an error \"%v\"", err) - // don't use DeepEqual since expectedAccess is []interface {} and receivedAccess is []map[string]interface {}, just check for contents - } else if fmt.Sprint(expectedAccess) != fmt.Sprint(receivedAccess) { - t.Errorf(`Expected user access and received user access are not the same. - Expected: %v - Received: %v`, expectedAccess, receivedAccess) - } -} diff --git a/tests/utils_test.go b/tests/utils_test.go index ae2c387..4b80f88 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -1,241 +1,88 @@ package tests import ( - "encoding/json" - "fmt" + "context" "io" "net/http" - "strings" "testing" - "github.com/calypr/data-client/client/common" - g3cmd "github.com/calypr/data-client/client/g3cmd" - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/mocks" - "go.uber.org/mock/gomock" + internalapi "github.com/calypr/syfon/apigen/client/internalapi" + sycommon "github.com/calypr/syfon/client/common" + sylogs "github.com/calypr/syfon/client/logs" + "github.com/calypr/syfon/client/transfer" + "github.com/calypr/syfon/client/transfer/upload" ) -// Expect GetDownloadResponse to: -// 1. get the file download URL from Shepherd if it's deployed -// 2. add the file download URL to the FileDownloadResponseObject -// 3. GET the file download URL, and add the response to the FileDownloadResponseObject -func TestGetDownloadResponse_withShepherd(t *testing.T) { - // -- SETUP -- - testGUID := "000000-0000000-0000000-000000" - testFilename := "test-file" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Mock the request that checks if Shepherd is deployed. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(true, nil) - - // Mock the request to Shepherd for the download URL of this file. - mockDownloadURL := "https://example.com/example.pfb" - downloadURLBody := fmt.Sprintf(`{ - "url": "%v" - }`, mockDownloadURL) - mockDownloadURLResponse := http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader(downloadURLBody)), - } - mockGen3Interface. - EXPECT(). - GetResponse(common.ShepherdEndpoint+"/objects/"+testGUID+"/download", "GET", "", nil). - Return("", &mockDownloadURLResponse, nil) - - // Mock the request for the file at mockDownloadURL. - mockFileResponse := http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader("It work")), - } - mockGen3Interface. - EXPECT(). - MakeARequest(http.MethodGet, mockDownloadURL, "", "", map[string]string{}, nil, true). - Return(&mockFileResponse, nil) - // ---------- - - mockFDRObj := common.FileDownloadResponseObject{ - Filename: testFilename, - GUID: testGUID, - Range: 0, - } - err := g3cmd.GetDownloadResponse(mockGen3Interface, &mockFDRObj, "") - if err != nil { - t.Error(err) - } - if mockFDRObj.URL != mockDownloadURL { - t.Errorf("Wanted the DownloadPath to be set to %v, got %v", mockDownloadURL, mockFDRObj.DownloadPath) - } - if mockFDRObj.Response != &mockFileResponse { - t.Errorf("Wanted download response to be %v, got %v", mockFileResponse, mockFDRObj.Response) - } +type fakeDownloader struct { + resolveFn func(ctx context.Context, guid, accessID string) (string, error) + downloadFn func(ctx context.Context, url string, rangeStart, rangeEnd *int64) (*http.Response, error) } -// Expect GetDownloadResponse to: -// 1. get the file download URL from Fence if Shepherd is not deployed -// 2. add the file download URL to the FileDownloadResponseObject -// 3. GET the file download URL, and add the response to the FileDownloadResponseObject -func TestGetDownloadResponse_noShepherd(t *testing.T) { - // -- SETUP -- - testGUID := "000000-0000000-0000000-000000" - testFilename := "test-file" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Mock the request that checks if Shepherd is deployed. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(false, nil) - - // Mock the request to Fence for the download URL of this file. - mockDownloadURL := "https://example.com/example.pfb" - mockDownloadURLResponse := jwt.JsonMessage{ - URL: mockDownloadURL, - } - mockGen3Interface. - EXPECT(). - DoRequestWithSignedHeader(common.FenceDataDownloadEndpoint+"/"+testGUID, "", nil). - Return(mockDownloadURLResponse, nil) - - // Mock the request for the file at mockDownloadURL. - mockFileResponse := http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader("It work")), - } - mockGen3Interface. - EXPECT(). - MakeARequest(http.MethodGet, mockDownloadURL, "", "", map[string]string{}, nil, true). - Return(&mockFileResponse, nil) - // ---------- - - mockFDRObj := common.FileDownloadResponseObject{ - Filename: testFilename, - GUID: testGUID, - Range: 0, - } - err := g3cmd.GetDownloadResponse(mockGen3Interface, &mockFDRObj, "") - if err != nil { - t.Error(err) - } - if mockFDRObj.URL != mockDownloadURL { - t.Errorf("Wanted the DownloadPath to be set to %v, got %v", mockDownloadURL, mockFDRObj.DownloadPath) - } - if mockFDRObj.Response != &mockFileResponse { - t.Errorf("Wanted download response to be %v, got %v", mockFileResponse, mockFDRObj.Response) - } +func (f *fakeDownloader) Name() string { return "fake-downloader" } +func (f *fakeDownloader) Logger() transfer.TransferLogger { + return sylogs.NewGen3Logger(nil, "", "test") +} +func (f *fakeDownloader) ResolveDownloadURL(ctx context.Context, guid, accessID string) (string, error) { + return f.resolveFn(ctx, guid, accessID) +} +func (f *fakeDownloader) Download(ctx context.Context, url string, rangeStart, rangeEnd *int64) (*http.Response, error) { + return f.downloadFn(ctx, url, rangeStart, rangeEnd) } -// If Shepherd is not deployed, expect GeneratePresignedURL to hit fence's data upload -// endpoint and return the presigned URL and guid. -func TestGeneratePresignedURL_noShepherd(t *testing.T) { - // -- SETUP -- - testFilename := "test-file" - testBucketname := "test-bucket" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Mock the request that checks if Shepherd is deployed. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(false, nil) +type fakeUploader struct { + resolveFn func(ctx context.Context, guid, filename string, metadata sycommon.FileMetadata, bucket string) (string, error) +} - // Mock the request to Fence's data upload endpoint to create a presigned url for this file name. - expectedReqBody := []byte(fmt.Sprintf(`{"file_name":"%v","bucket":"%v"}`, testFilename, testBucketname)) - mockPresignedURL := "https://example.com/example.pfb" - mockGUID := "000000-0000000-0000000-000000" - mockUploadURLResponse := jwt.JsonMessage{ - URL: mockPresignedURL, - GUID: mockGUID, - } - mockGen3Interface. - EXPECT(). - DoRequestWithSignedHeader(common.FenceDataUploadEndpoint, "application/json", expectedReqBody). - Return(mockUploadURLResponse, nil) - // ---------- +func (f *fakeUploader) Name() string { return "fake-uploader" } +func (f *fakeUploader) Logger() transfer.TransferLogger { return sylogs.NewGen3Logger(nil, "", "test") } - url, guid, err := g3cmd.GeneratePresignedURL(mockGen3Interface, testFilename, common.FileMetadata{}, testBucketname) - if err != nil { - t.Error(err) - } - if url != mockPresignedURL { - t.Errorf("Wanted the presignedURL to be set to %v, got %v", mockPresignedURL, url) - } - if guid != mockGUID { - t.Errorf("Wanted generated GUID to be %v, got %v", mockGUID, guid) - } +func (f *fakeUploader) ResolveUploadURL(ctx context.Context, guid, filename string, metadata sycommon.FileMetadata, bucket string) (string, error) { + return f.resolveFn(ctx, guid, filename, metadata, bucket) +} +func (f *fakeUploader) InitMultipartUpload(ctx context.Context, guid string, filename string, bucket string) (string, string, error) { + return "", "", nil +} +func (f *fakeUploader) GetMultipartUploadURL(ctx context.Context, key string, uploadID string, partNumber int32, bucket string) (string, error) { + return "", nil +} +func (f *fakeUploader) CompleteMultipartUpload(ctx context.Context, key string, uploadID string, parts []internalapi.InternalMultipartPart, bucket string) error { + return nil +} +func (f *fakeUploader) Upload(ctx context.Context, url string, body io.Reader, size int64) error { + return nil +} +func (f *fakeUploader) UploadPart(ctx context.Context, url string, body io.Reader, size int64) (string, error) { + return "", nil +} +func (f *fakeUploader) DeleteFile(ctx context.Context, guid string) (string, error) { + return "", nil +} +func (f *fakeUploader) CanonicalObjectURL(signedURL, bucketHint, fallbackDID string) (string, error) { + return signedURL, nil } -// If Shepherd is deployed, expect GeneratePresignedURL to hit Shepherd's data upload -// endpoint with the file name and file metadata. GeneratePresignedURL should then -// return the guid and file name that it gets from the endpoint. -func TestGeneratePresignedURL_withShepherd(t *testing.T) { - // -- SETUP -- +func TestGeneratePresignedUploadURL(t *testing.T) { testFilename := "test-file" - testBucketname := "test-bucket" - testMetadata := common.FileMetadata{ - Aliases: []string{"test-alias-1", "test-alias-2"}, - Authz: []string{"authz-resource-1", "authz-resource-2"}, - Metadata: map[string]any{"arbitrary": "metadata"}, - } - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Mock the request that checks if Shepherd is deployed. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(true, nil) - - // Mock the request to Fence's data upload endpoint to create a presigned url for this file name. - expectedReq := g3cmd.ShepherdInitRequestObject{ - Filename: testFilename, - Authz: struct { - Version string `json:"version"` - ResourcePaths []string `json:"resource_paths"` - }{ - "0", - testMetadata.Authz, + testBucket := "test-bucket" + mockUploadURL := "https://example.com/upload" + + bk := &fakeUploader{ + resolveFn: func(ctx context.Context, guid, filename string, metadata sycommon.FileMetadata, bucket string) (string, error) { + if filename != testFilename { + t.Fatalf("unexpected filename: %s", filename) + } + if bucket != testBucket { + t.Fatalf("unexpected bucket: %s", bucket) + } + return mockUploadURL, nil }, - Aliases: testMetadata.Aliases, - Metadata: testMetadata.Metadata, - } - expectedReqBody, err := json.Marshal(expectedReq) - if err != nil { - t.Error(err) - } - mockPresignedURL := "https://example.com/example.pfb" - mockGUID := "000000-0000000-0000000-000000" - presignedURLBody := fmt.Sprintf(`{ - "guid": "%v", - "upload_url": "%v" - }`, mockGUID, mockPresignedURL) - mockUploadURLResponse := http.Response{ - StatusCode: 201, - Body: io.NopCloser(strings.NewReader(presignedURLBody)), } - mockGen3Interface. - EXPECT(). - GetResponse(common.ShepherdEndpoint+"/objects", "POST", "", expectedReqBody). - Return("", &mockUploadURLResponse, nil) - // ---------- - url, guid, err := g3cmd.GeneratePresignedURL(mockGen3Interface, testFilename, testMetadata, testBucketname) + resp, err := upload.GeneratePresignedUploadURL(context.Background(), bk, testFilename, sycommon.FileMetadata{}, testBucket) if err != nil { - t.Error(err) - } - if url != mockPresignedURL { - t.Errorf("Wanted the presignedURL to be set to %v, got %v", mockPresignedURL, url) + t.Fatalf("unexpected error: %v", err) } - if guid != mockGUID { - t.Errorf("Wanted generated GUID to be %v, got %v", mockGUID, guid) + if resp != mockUploadURL { + t.Errorf("wanted URL %s, got %s", mockUploadURL, resp) } }