Skip to content

chore: sync from internal repo #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
calummoore merged 1 commit into from
May 15, 2026
Merged

chore: sync from internal repo #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .github/actions/cloud-cache/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,28 @@
name: 'Cloud Cache'
description: 'Restores and saves a directory to cloud storage as a cache (tar.zst compressed), using a primary key and fallback keys. Supports GCS and S3.'
description: 'Restores and saves a directory to cloud storage as a cache (tar.zst compressed), using a primary key and fallback keys. Supports GCS and Cloudflare R2 via its S3-compatible API.'
inputs:
provider:
description: 'Storage provider: gcs or s3'
description: 'Storage provider: gcs or s3 (Cloudflare R2 only for s3)'
required: false
default: 'gcs'
default: 's3'
gcs-key:
description: 'Base64 encoded JSON service account key for GCS'
required: false
s3-access-key:
description: 'Access key for S3'
description: 'Cloudflare R2 access key ID for the s3 backend'
required: false
s3-secret-key:
description: 'Secret key for S3'
description: 'Cloudflare R2 secret access key for the s3 backend'
required: false
s3-endpoint:
description: 'Endpoint URL for S3 (for non-AWS S3-compatible services)'
description: 'Cloudflare R2 endpoint URL for the s3 backend; required when provider=s3'
required: false
s3-region:
description: 'Region for S3 (default: auto)'
description: 'Region for the R2 s3 backend (default: auto)'
required: false
default: 'auto'
bucket:
description: 'Bucket name (GCS or S3)'
description: 'Bucket name (GCS or Cloudflare R2)'
required: true
prefix:
description: 'Prefix path in the bucket (optional)'
Expand Down
29 changes: 25 additions & 4 deletions .github/actions/cloud-cache/dist/index.js
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 14 additions & 6 deletions .github/actions/cloud-cache/dist/post/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108416,6 +108416,11 @@ const exec = util.promisify(cp.exec);
function sleep(ms) {
return new Promise((res) => setTimeout(res, ms));
}

function isStrictR2Context() {
return process.env.GITHUB_REF === "refs/heads/main";
}

async function run() {
try {
const skipUpload = core.getInput("skip-upload") === "true";
Expand Down Expand Up @@ -108480,11 +108485,16 @@ async function run() {
} else if (provider === "s3") {
const s3AccessKey = core.getState("s3AccessKey");
const s3SecretKey = core.getState("s3SecretKey");
if (!s3AccessKey || !s3SecretKey) {
console.log("S3 credentials not available, skipping cache upload");
const s3Endpoint = core.getState("s3Endpoint");
if (!s3AccessKey || !s3SecretKey || !s3Endpoint) {
if (isStrictR2Context()) {
throw new Error(
"Cloudflare R2 credentials and endpoint are required for provider=s3 on refs/heads/main"
);
}
console.log("Cloudflare R2 credentials or endpoint not available for provider=s3, skipping cache upload");
return;
}
const s3Endpoint = core.getState("s3Endpoint");
const s3Region = core.getState("s3Region");
// Initialize S3 client
const s3Config = {
Expand All @@ -108494,10 +108504,8 @@ async function run() {
secretAccessKey: s3SecretKey,
},
forcePathStyle: true,
endpoint: s3Endpoint,
};
if (s3Endpoint) {
s3Config.endpoint = s3Endpoint;
}
s3Client = new S3Client(s3Config);
}
const primaryWithExt = primaryKey + ".tar.zst";
Expand Down
29 changes: 25 additions & 4 deletions .github/actions/cloud-cache/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ function expandTilde(inputPath) {
}
return inputPath;
}

function isStrictR2Context() {
return process.env.GITHUB_REF === "refs/heads/main";
}

async function run() {
try {
// Get inputs
Expand Down Expand Up @@ -92,12 +97,30 @@ async function run() {
const s3SecretKey = core.getInput("s3-secret-key");
const s3Endpoint = core.getInput("s3-endpoint");
const s3Region = core.getInput("s3-region");
const strictR2Context = isStrictR2Context();

if (!s3Endpoint) {
if (strictR2Context) {
throw new Error("s3-endpoint is required when provider=s3 on refs/heads/main");
}
if (needsCredentials) {
throw new Error("s3-endpoint is required when provider=s3 and skip-download is false");
}
core.warning("Cloudflare R2 endpoint not provided for provider=s3. Cache operations will be skipped.");
core.setOutput("cache-hit", "false");
core.setOutput("restored", "false");
core.setOutput("path", localPath);
return;
}

if (!s3AccessKey || !s3SecretKey) {
if (strictR2Context) {
throw new Error("s3-access-key and s3-secret-key are required when provider=s3 on refs/heads/main");
}
if (needsCredentials) {
throw new Error("s3-access-key and s3-secret-key are required when skip-download is false");
}
core.warning("S3 credentials not provided. Cache operations will be skipped.");
core.warning("Cloudflare R2 credentials not provided for provider=s3. Cache operations will be skipped.");
core.setOutput("cache-hit", "false");
core.setOutput("restored", "false");
core.setOutput("path", localPath);
Expand All @@ -116,10 +139,8 @@ async function run() {
secretAccessKey: s3SecretKey,
},
forcePathStyle: true,
endpoint: s3Endpoint,
};
if (s3Endpoint) {
s3Config.endpoint = s3Endpoint;
}
s3Client = new S3Client(s3Config);
} else {
throw new Error(`Unsupported provider: ${provider}`);
Expand Down
20 changes: 14 additions & 6 deletions .github/actions/cloud-cache/post.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ const exec = util.promisify(cp.exec);
function sleep(ms) {
return new Promise((res) => setTimeout(res, ms));
}

function isStrictR2Context() {
return process.env.GITHUB_REF === "refs/heads/main";
}

async function run() {
try {
const skipUpload = core.getInput("skip-upload") === "true";
Expand Down Expand Up @@ -82,11 +87,16 @@ async function run() {
} else if (provider === "s3") {
const s3AccessKey = core.getState("s3AccessKey");
const s3SecretKey = core.getState("s3SecretKey");
if (!s3AccessKey || !s3SecretKey) {
console.log("S3 credentials not available, skipping cache upload");
const s3Endpoint = core.getState("s3Endpoint");
if (!s3AccessKey || !s3SecretKey || !s3Endpoint) {
if (isStrictR2Context()) {
throw new Error(
"Cloudflare R2 credentials and endpoint are required for provider=s3 on refs/heads/main"
);
}
console.log("Cloudflare R2 credentials or endpoint not available for provider=s3, skipping cache upload");
return;
}
const s3Endpoint = core.getState("s3Endpoint");
const s3Region = core.getState("s3Region");
// Initialize S3 client
const s3Config = {
Expand All @@ -96,10 +106,8 @@ async function run() {
secretAccessKey: s3SecretKey,
},
forcePathStyle: true,
endpoint: s3Endpoint,
};
if (s3Endpoint) {
s3Config.endpoint = s3Endpoint;
}
s3Client = new S3Client(s3Config);
}
const primaryWithExt = primaryKey + ".tar.zst";
Expand Down
108 changes: 66 additions & 42 deletions .github/actions/setup-sccache/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
name: Setup sccache
description: Authenticates to GCS or S3, installs sccache, and sets environment variables for sccache
description: Authenticates to Cloudflare R2 or GCS, installs sccache, and sets environment variables for sccache

inputs:
backend:
description: 'The backend to use: gcs or s3'
description: 'The backend to use: gcs or s3 (Cloudflare R2 only for s3)'
required: true
default: 'gcs'
default: 's3'
sccache_bucket:
description: 'The bucket name for GCS or S3'
description: 'The bucket name for GCS or Cloudflare R2'
required: false
default: 'polybase-sccache-eu'
sccache_key_prefix:
Expand All @@ -18,19 +18,20 @@ inputs:
description: 'The GCS service account key in base64, required for GCS backend'
required: false
sccache_s3_region:
description: 'The S3 region, required for S3 backend (or auto if endpoint is set)'
description: 'The region for the Cloudflare R2 s3 backend (usually auto)'
required: false
default: 'auto'
sccache_s3_access_key_id:
description: 'The S3 access key ID, required for S3 backend'
description: 'The Cloudflare R2 access key ID, required for the s3 backend'
required: false
sccache_s3_secret_access_key:
description: 'The S3 secret access key, required for S3 backend'
description: 'The Cloudflare R2 secret access key, required for the s3 backend'
required: false
sccache_s3_endpoint:
description: 'The S3 endpoint URL for generic S3 (e.g., https://s3.example.com or minio:9000)'
description: 'The Cloudflare R2 endpoint URL, required for the s3 backend'
required: false
sccache_s3_use_ssl:
description: 'Use SSL for S3 endpoint (true/false)'
description: 'Use SSL for the Cloudflare R2 endpoint (true/false)'
required: false
default: 'true'

Expand All @@ -47,27 +48,45 @@ runs:
backend="${{ inputs.backend }}"
bucket="${{ inputs.sccache_bucket }}"
prefix="${{ inputs.sccache_key_prefix }}"
strict_r2_config="false"
if [ "${GITHUB_REF:-}" = "refs/heads/main" ]; then
strict_r2_config="true"
fi
if [ "$backend" = "gcs" ]; then
echo "SCCACHE_GCS_BUCKET=$bucket" >> $GITHUB_ENV
echo "SCCACHE_GCS_KEY_PREFIX=$prefix" >> $GITHUB_ENV
echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> $GITHUB_ENV
base64 -d <<< "${{ inputs.sccache_gcs_sa_key_base64 }}" > /tmp/sccache-gcs-sa-key.json
chmod 660 /tmp/sccache-gcs-sa-key.json
echo "SCCACHE_GCS_KEY_PATH=/tmp/sccache-gcs-sa-key.json" >> $GITHUB_ENV
elif [ "$backend" = "s3" ]; then
echo "SCCACHE_BUCKET=$bucket" >> $GITHUB_ENV
echo "SCCACHE_S3_KEY_PREFIX=$prefix" >> $GITHUB_ENV
region="${{ inputs.sccache_s3_region }}"
if [ -n "${{ inputs.sccache_s3_endpoint }}" ] && [ -z "$region" ]; then
region="auto"
if [ -z "${{ inputs.sccache_gcs_sa_key_base64 }}" ]; then
echo "::warning::setup-sccache: backend=gcs but sccache_gcs_sa_key_base64 is unavailable; continuing without remote sccache."
else
echo "SCCACHE_GCS_BUCKET=$bucket" >> $GITHUB_ENV
echo "SCCACHE_GCS_KEY_PREFIX=$prefix" >> $GITHUB_ENV
echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> $GITHUB_ENV
base64 -d <<< "${{ inputs.sccache_gcs_sa_key_base64 }}" > /tmp/sccache-gcs-sa-key.json
chmod 660 /tmp/sccache-gcs-sa-key.json
echo "SCCACHE_GCS_KEY_PATH=/tmp/sccache-gcs-sa-key.json" >> $GITHUB_ENV
fi
echo "SCCACHE_REGION=$region" >> $GITHUB_ENV
if [ -n "${{ inputs.sccache_s3_endpoint }}" ]; then
echo "SCCACHE_ENDPOINT=${{ inputs.sccache_s3_endpoint }}" >> $GITHUB_ENV
elif [ "$backend" = "s3" ]; then
if [ -z "${{ inputs.sccache_s3_access_key_id }}" ] || [ -z "${{ inputs.sccache_s3_secret_access_key }}" ] || [ -z "${{ inputs.sccache_s3_endpoint }}" ]; then
if [ "$strict_r2_config" = "true" ]; then
echo "setup-sccache: backend=s3 (Cloudflare R2) requires credentials and endpoint on ${GITHUB_REF}" >&2
exit 1
fi
echo "::warning::setup-sccache: backend=s3 (Cloudflare R2) but credentials or endpoint are unavailable; continuing without remote sccache."
else
echo "SCCACHE_BUCKET=$bucket" >> $GITHUB_ENV
echo "SCCACHE_S3_KEY_PREFIX=$prefix" >> $GITHUB_ENV
region="${{ inputs.sccache_s3_region }}"
echo "SCCACHE_REGION=$region" >> $GITHUB_ENV
if [ -n "${{ inputs.sccache_s3_endpoint }}" ]; then
echo "SCCACHE_ENDPOINT=${{ inputs.sccache_s3_endpoint }}" >> $GITHUB_ENV
fi
echo "SCCACHE_S3_USE_SSL=${{ inputs.sccache_s3_use_ssl }}" >> $GITHUB_ENV
{
echo "[default]"
echo "aws_access_key_id=${{ inputs.sccache_s3_access_key_id }}"
echo "aws_secret_access_key=${{ inputs.sccache_s3_secret_access_key }}"
} > /tmp/sccache-aws-credentials
chmod 600 /tmp/sccache-aws-credentials
echo "AWS_SHARED_CREDENTIALS_FILE=/tmp/sccache-aws-credentials" >> $GITHUB_ENV
fi
echo "SCCACHE_S3_USE_SSL=${{ inputs.sccache_s3_use_ssl }}" >> $GITHUB_ENV
echo "AWS_ACCESS_KEY_ID=${{ inputs.sccache_s3_access_key_id }}" >> $GITHUB_ENV
echo "AWS_SECRET_ACCESS_KEY=${{ inputs.sccache_s3_secret_access_key }}" >> $GITHUB_ENV
else
echo "Unsupported backend: $backend" >&2
exit 1
Expand All @@ -81,27 +100,32 @@ runs:
prefix="${{ inputs.sccache_key_prefix }}"
echo "SCCACHE_BUILD_ARGS<<EOF" >> $GITHUB_ENV
if [ "$backend" = "gcs" ]; then
echo "SCCACHE_GCS_BUCKET=$bucket" >> $GITHUB_ENV
echo "SCCACHE_GCS_KEY_PREFIX=$prefix" >> $GITHUB_ENV
elif [ "$backend" = "s3" ]; then
echo "SCCACHE_BUCKET=$bucket" >> $GITHUB_ENV
echo "SCCACHE_S3_KEY_PREFIX=$prefix" >> $GITHUB_ENV
region="${{ inputs.sccache_s3_region }}"
if [ -n "${{ inputs.sccache_s3_endpoint }}" ] && [ -z "$region" ]; then
region="auto"
if [ -n "${{ inputs.sccache_gcs_sa_key_base64 }}" ]; then
echo "SCCACHE_GCS_BUCKET=$bucket" >> $GITHUB_ENV
echo "SCCACHE_GCS_KEY_PREFIX=$prefix" >> $GITHUB_ENV
fi
echo "SCCACHE_REGION=$region" >> $GITHUB_ENV
if [ -n "${{ inputs.sccache_s3_endpoint }}" ]; then
echo "SCCACHE_ENDPOINT=${{ inputs.sccache_s3_endpoint }}" >> $GITHUB_ENV
elif [ "$backend" = "s3" ]; then
if [ -n "${{ inputs.sccache_s3_access_key_id }}" ] && [ -n "${{ inputs.sccache_s3_secret_access_key }}" ] && [ -n "${{ inputs.sccache_s3_endpoint }}" ]; then
echo "SCCACHE_BUCKET=$bucket" >> $GITHUB_ENV
echo "SCCACHE_S3_KEY_PREFIX=$prefix" >> $GITHUB_ENV
region="${{ inputs.sccache_s3_region }}"
echo "SCCACHE_REGION=$region" >> $GITHUB_ENV
if [ -n "${{ inputs.sccache_s3_endpoint }}" ]; then
echo "SCCACHE_ENDPOINT=${{ inputs.sccache_s3_endpoint }}" >> $GITHUB_ENV
fi
echo "SCCACHE_S3_USE_SSL=${{ inputs.sccache_s3_use_ssl }}" >> $GITHUB_ENV
fi
echo "SCCACHE_S3_USE_SSL=${{ inputs.sccache_s3_use_ssl }}" >> $GITHUB_ENV
fi
echo "EOF" >> $GITHUB_ENV
echo "SCCACHE_BUILD_SECRETS<<EOF" >> $GITHUB_ENV
if [ "$backend" = "gcs" ]; then
echo "gcs_sa_key_base64=${{ inputs.sccache_gcs_sa_key_base64 }}" >> $GITHUB_ENV
if [ -n "${{ inputs.sccache_gcs_sa_key_base64 }}" ]; then
echo "gcs_sa_key_base64=${{ inputs.sccache_gcs_sa_key_base64 }}" >> $GITHUB_ENV
fi
elif [ "$backend" = "s3" ]; then
echo "aws_access_key_id=${{ inputs.sccache_s3_access_key_id }}" >> $GITHUB_ENV
echo "aws_secret_access_key=${{ inputs.sccache_s3_secret_access_key }}" >> $GITHUB_ENV
if [ -n "${{ inputs.sccache_s3_access_key_id }}" ] && [ -n "${{ inputs.sccache_s3_secret_access_key }}" ] && [ -n "${{ inputs.sccache_s3_endpoint }}" ]; then
echo "aws_access_key_id=${{ inputs.sccache_s3_access_key_id }}" >> $GITHUB_ENV
echo "aws_secret_access_key=${{ inputs.sccache_s3_secret_access_key }}" >> $GITHUB_ENV
fi
fi
echo "EOF" >> $GITHUB_ENV
Loading
Loading