Sync upstream main

.asf.yaml

@@ -50,6 +50,79 @@ github:
     main:
       required_pull_request_reviews:
         required_approving_review_count: 1
+    # needs to be updated as part of the release process
+    # .asf.yaml doesn't support wildcard branch protection rules, only exact branch names
+    # https://github.com/apache/infrastructure-asfyaml?tab=readme-ov-file#branch-protection
+    # Keeping set of protected branches for future releases
+    # Meanwhile creating a prerelease script that will update the branch protection names
+    # automatically. Keep track on it https://github.com/apache/datafusion/issues/17134
+    branch-50:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-51:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-52:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-53:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-54:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-55:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-56:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-57:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-58:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-59:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-60:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-61:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-62:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-63:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-64:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-65:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-66:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-67:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-68:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-69:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-70:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+  pull_requests:
+    # enable updating head branches of pull requests
+    allow_update_branch: true
+    allow_auto_merge: true
 
 # publishes the content of the `asf-site` branch to
 # https://datafusion.apache.org/

.devcontainer/Dockerfile

@@ -4,10 +4,12 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
     && apt-get purge -y imagemagick imagemagick-6-common
 
-# Add protoc
-# https://datafusion.apache.org/contributor-guide/getting_started.html#protoc-installation
-RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protoc-25.1-linux-x86_64.zip \
-    && unzip protoc-25.1-linux-x86_64.zip -d $HOME/.local \
-    && rm protoc-25.1-linux-x86_64.zip
+# setup the containers WORKDIR so npm install works
+# https://stackoverflow.com/questions/57534295/npm-err-tracker-idealtree-already-exists-while-creating-the-docker-image-for
+WORKDIR /root
 
-ENV PATH="$PATH:$HOME/.local/bin"
+# Add protoc, npm, prettier
+# https://datafusion.apache.org/contributor-guide/development_environment.html#protoc-installation
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends protobuf-compiler libprotobuf-dev npm nodejs\
+    && rm -rf /var/lib/apt/lists/*

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,5 +1,6 @@
 name: Bug report
 description: Create a report to help us improve
+type: Bug
 labels: bug
 body:
   - type: textarea

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,5 +1,6 @@
 name: Feature request
 description: Suggest an idea for this project
+type: Feature
 labels: enhancement
 body:
   - type: textarea

.github/actions/setup-macos-aarch64-builder/action.yaml

@@ -44,6 +44,8 @@ runs:
         rustup default stable
         rustup component add rustfmt
     - name: Setup rust cache
-      uses: Swatinem/rust-cache@v2
+      uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1  # v2.8.1
+      with:
+           save-if: ${{ github.ref_name == 'main' }}
     - name: Configure rust runtime env
       uses: ./.github/actions/setup-rust-runtime        

.github/actions/setup-rust-runtime/action.yaml

@@ -20,8 +20,6 @@ description: 'Setup Rust Runtime Environment'
 runs:
   using: "composite"
   steps:
-    - name: Run sccache-cache
-      uses: mozilla-actions/sccache-action@v0.0.4
     - name: Configure runtime env
       shell: bash
       # do not produce debug symbols to keep memory usage down
@@ -31,8 +29,5 @@ runs:
       # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
       # See for more details: https://github.com/rust-lang/rust/issues/119560
       run: |
-        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV  
-        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV 
         echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
         echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV
-

.github/dependabot.yml

@@ -20,7 +20,7 @@ updates:
   - package-ecosystem: cargo
     directory: "/"
     schedule:
-      interval: daily
+      interval: weekly
     target-branch: main
     labels: [auto-dependencies]
     ignore:
@@ -50,3 +50,8 @@ updates:
       interval: "daily"
     open-pull-requests-limit: 10
     labels: [auto-dependencies]
+  - package-ecosystem: "pip"
+    directory: "/docs"
+    schedule:
+      interval: "weekly"
+    labels: [auto-dependencies]

.github/workflows/audit.yml

@@ -23,6 +23,8 @@ concurrency:
 
 on:
   push:
+    branches:
+      - main
     paths:
       - "**/Cargo.toml"
       - "**/Cargo.lock"
@@ -31,13 +33,17 @@ on:
     paths:
       - "**/Cargo.toml"
       - "**/Cargo.lock"
+
+  merge_group:
 
 jobs:
   security_audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
       - name: Install cargo-audit
-        run: cargo install cargo-audit
+        uses: taiki-e/install-action@e7ef886cf8f69c25ecef6bbc2858a42e273496ec  # v2.62.28
+        with:
+          tool: cargo-audit
       - name: Run audit check
         run: cargo audit

.github/workflows/dependencies.yml

@@ -23,13 +23,16 @@ concurrency:
 
 on:
   push:
+    branches-ignore:
+      - 'gh-readonly-queue/**'
     paths:
       - "**/Cargo.toml"
       - "**/Cargo.lock"
   pull_request:
     paths:
       - "**/Cargo.toml"
       - "**/Cargo.lock"
+  merge_group:
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
@@ -41,7 +44,7 @@ jobs:
     container:
       image: amd64/rust
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
         with:
           submodules: true
           fetch-depth: 1
@@ -53,3 +56,14 @@ jobs:
         run: |
           cd dev/depcheck
           cargo run
+
+  detect-unused-dependencies:
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      - name: Install cargo-machete
+        run: cargo install cargo-machete --version ^0.9 --locked
+      - name: Detect unused dependencies
+        run: cargo machete --with-metadata

.github/workflows/dev.yml

@@ -16,7 +16,12 @@
 # under the License.
 
 name: Dev
-on: [push, pull_request]
+on:
+  push:
+    branches-ignore:
+      - 'gh-readonly-queue/**'
+  pull_request:
+  merge_group:
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -27,15 +32,18 @@ jobs:
     runs-on: ubuntu-latest
     name: Check License Header
     steps:
-      - uses: actions/checkout@v4
-      - uses: korandoru/hawkeye@v6
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      - name: Install HawkEye
+        run: cargo install hawkeye --version 6.2.0 --locked --profile dev
+      - name: Run license header check
+        run: ci/scripts/license_header.sh
 
   prettier:
     name: Use prettier to check formatting of documents
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903  # v6.0.0
         with:
           node-version: "20"
       - name: Prettier check

.github/workflows/docs.yaml

@@ -32,16 +32,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout docs sources
-        uses: actions/checkout@v4
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
 
       - name: Checkout asf-site branch
-        uses: actions/checkout@v4
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
         with:
           ref: asf-site
           path: asf-site
 
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c  # v6.0.0
         with:
           python-version: "3.12"
 

.github/workflows/docs_pr.yaml

@@ -34,37 +34,18 @@ on:
   workflow_dispatch:
 
 jobs:
-  # Run doc tests
-  linux-test-doc:
-    name: cargo doctest (amd64)
-    runs-on: ubuntu-latest
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
-      - name: Run doctests (embedded rust examples)
-        run: cargo test --doc --features avro,json
-      - name: Verify Working Directory Clean
-        run: git diff --exit-code
-
+
   # Test doc build
   linux-test-doc-build:
     name: Test doc build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
         with:
           submodules: true
           fetch-depth: 1
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c  # v6.0.0
         with:
           python-version: "3.12"
       - name: Install doc dependencies