Feat: Unified ir pipeline

.amalgam-manifest.toml

@@ -1,38 +1,98 @@
-# Minimal Amalgam Package Generation Manifest for Testing
-# Tests universal dependency detection without special casing
-
 [config]
 # Output directory for generated packages
 output_base = "examples/pkgs"
-# Enable package mode for nickel-mine compatibility
+
+# Enable package mode (generates Nickel-pkg.ncl files)
 package_mode = true
+
 # Base package ID for dependencies
 base_package_id = "github:seryl/nickel-pkgs/pkgs"
-# Local package prefix for development - generates Path dependencies
-local_package_prefix = "examples/pkgs"
 
-# Core Kubernetes types - foundation for other packages
 [[packages]]
-name = "k8s-io"
-type = "k8s-core"
-version = "v1.33.4"  # Latest stable Kubernetes as of 2025-09-01
-output = "k8s_io"
-description = "Kubernetes core type definitions including apimachinery"
-keywords = ["kubernetes", "k8s", "types", "api", "core"]
-# No dependencies - this is the base package
-dependencies = {}
-
-# CrossPlane - should auto-detect k8s_io dependency
+# Kubernetes OpenAPI - domain inferred as k8s.io from definitions
+# NOTE: K8s uses Swagger 2.0 format which is now automatically converted to OpenAPI 3.0
+source = "https://raw.githubusercontent.com/kubernetes/kubernetes/v1.33.4/api/openapi-spec/swagger.json"
+# Everything else is automatic:
+# - Type: detected as OpenAPI from content
+# - Domain: extracted as k8s.io from definition keys like "io.k8s.api.core.v1.Pod"
+# - Output: k8s_io/ (from domain)
+# - Structure: determined by parsing
+
 [[packages]]
-name = "crossplane"
-type = "url"
-url = "https://github.com/crossplane/crossplane/tree/v2.0.2/cluster/crds"
-git_ref = "v2.0.2"  # Latest stable Crossplane as of 2025-09-01
-version = "2.0.2"   # Package version
-output = "crossplane"
-description = "Crossplane CRD type definitions for infrastructure as code"
-keywords = ["crossplane", "kubernetes", "infrastructure", "gitops"]
-# Explicit dependency constraints (auto-detected dependencies will use these)
-[packages.dependencies]
-k8s_io = { version = "1.33.4" }  # Match the k8s version above
+# CrossPlane CRDs have multiple domains, so we need separate entries
+# This will pick up apiextensions.crossplane.io (alphabetically first)
+source = "https://github.com/crossplane/crossplane/tree/v2.0.2/cluster/crds"
+# Domain: apiextensions.crossplane.io (from CRD spec.group)
+# TODO: Split into multiple manifest entries for each domain, or enhance system to handle multi-domain sources
+
+# =================== #
+# ADDITIONAL EXAMPLES #
+# =================== #
+
+# [[packages]]
+# # Cert Manager - multiple CRD files forming one package
+# source = [
+#   "https://raw.githubusercontent.com/cert-manager/cert-manager/v1.12.0/deploy/crds/crd-certificates.yaml",
+#   "https://raw.githubusercontent.com/cert-manager/cert-manager/v1.12.0/deploy/crds/crd-issuers.yaml",
+#   "https://raw.githubusercontent.com/cert-manager/cert-manager/v1.12.0/deploy/crds/crd-clusterissuers.yaml",
+# ]
+# # All CRDs have same domain (cert-manager.io) so they merge into one package
+
+# [[packages]]
+# # Istio Service Mesh CRDs
+# source = "https://github.com/istio/istio/tree/1.20.1/manifests/charts/base/crds"
+# # Domain: inferred as networking.istio.io, security.istio.io, etc.
+
+# [[packages]]
+# # ArgoCD CRDs
+# source = [
+#   "https://raw.githubusercontent.com/argoproj/argo-cd/v2.9.0/manifests/crds/application-crd.yaml",
+#   "https://raw.githubusercontent.com/argoproj/argo-cd/v2.9.0/manifests/crds/applicationset-crd.yaml",
+#   "https://raw.githubusercontent.com/argoproj/argo-cd/v2.9.0/manifests/crds/appproject-crd.yaml",
+# ]
+# # Domain: inferred as argoproj.io
+
+# [[packages]]
+# # Prometheus Operator CRDs
+# source = "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml"
+# # Domain: inferred as monitoring.coreos.com
+
+# [[packages]]
+# # Local OpenAPI file
+# source = "file:///path/to/local/swagger.json"
+# # Type: auto-detected from content
+
+# [[packages]]
+# # Example with domain override (rarely needed)
+# source = "https://example.com/api/schema.json"
+# domain = "example.com"  # Only if it can't be inferred
+
+# [[packages]]
+# # Disabled package example
+# source = "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/jsonnet/prometheus-operator/crds.yaml"
+# enabled = false  # Skip this one
+
+# [[packages]]
+# # Tekton Pipelines CRDs
+# source = [
+#   "https://raw.githubusercontent.com/tektoncd/pipeline/v0.53.0/config/300-crds/300-pipeline.yaml",
+#   "https://raw.githubusercontent.com/tektoncd/pipeline/v0.53.0/config/300-crds/300-pipelinerun.yaml",
+#   "https://raw.githubusercontent.com/tektoncd/pipeline/v0.53.0/config/300-crds/300-task.yaml",
+#   "https://raw.githubusercontent.com/tektoncd/pipeline/v0.53.0/config/300-crds/300-taskrun.yaml",
+# ]
+# # Domain: inferred as tekton.dev
+
+# [[packages]]
+# # Knative Serving CRDs
+# source = "https://github.com/knative/serving/tree/v1.12.0/config/core/300-crds"
+# # Domain: inferred as serving.knative.dev
+
+# [[packages]]
+# # Velero Backup CRDs
+# source = [
+#   "https://raw.githubusercontent.com/vmware-tanzu/velero/v1.12.0/config/crd/v1/bases/velero.io_backups.yaml",
+#   "https://raw.githubusercontent.com/vmware-tanzu/velero/v1.12.0/config/crd/v1/bases/velero.io_restores.yaml",
+#   "https://raw.githubusercontent.com/vmware-tanzu/velero/v1.12.0/config/crd/v1/bases/velero.io_schedules.yaml",
+# ]
+# # Domain: inferred as velero.io
 

.github/workflows/devshell-ci.yml

@@ -19,7 +19,7 @@ jobs:
     # Skip CI if ENABLE_CI is not set to 'true'
     if: github.event_name == 'workflow_dispatch' || vars.ENABLE_CI == 'true'
     runs-on: ubuntu-latest
-    
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -39,13 +39,13 @@ jobs:
         run: |
           # Install direnv
           nix profile install nixpkgs#direnv nixpkgs#nix-direnv
-          
+
           # Configure direnv
           echo "source_url \"https://raw.githubusercontent.com/nix-community/nix-direnv/master/direnvrc\" \"sha256-zelF0vLbEl5uaqrfIzbgNzJWGmLzCmYAkInj/LNxvKs=\"" > ~/.direnvrc
-          
+
           # Allow the .envrc
           direnv allow
-          
+
           # Export direnv hooks for bash
           eval "$(direnv hook bash)"
 
@@ -59,11 +59,11 @@ jobs:
         run: |
           # Load the dev shell via direnv
           eval "$(direnv export bash)"
-          
+
           if [ -n "$ATTIC_TOKEN" ]; then
             attic login "$ATTIC_SERVER" "$ATTIC_TOKEN"
           fi
-          
+
           attic use "$ATTIC_CACHE"
           attic watch-store "$ATTIC_CACHE" &
 
@@ -75,10 +75,10 @@ jobs:
         run: |
           # Load the dev shell environment via direnv
           eval "$(direnv export bash)"
-          
+
           # Now we're in the dev shell with all tools available
           echo "Running CI pipeline with target: ${{ github.event.inputs.target || 'ci' }}"
-          
+
           # Use ci-runner command from the devshell
           ci-runner ${{ github.event.inputs.target || 'ci' }}
 
@@ -97,7 +97,7 @@ jobs:
     # Skip CI if ENABLE_CI is not set to 'true'
     if: github.event_name == 'workflow_dispatch' || vars.ENABLE_CI == 'true'
     runs-on: ubuntu-latest
-    
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -111,4 +111,4 @@ jobs:
 
       - name: Run flake checks
         run: |
-          nix flake check --print-build-logs
+          nix flake check --print-build-logs

.github/workflows/nix-docker-build.yml

@@ -0,0 +1,147 @@
+name: Build and Push Docker Images with Nix
+
+on:
+  push:
+    branches: [main]
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      push:
+        description: 'Push images to registry'
+        type: boolean
+        default: false
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write  # For ghcr.io
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+
+      - name: Setup Magic Nix Cache
+        uses: DeterminateSystems/magic-nix-cache-action@v2
+
+      - name: Build Docker images with Nix
+        run: |
+          echo "Building amalgam compiler image..."
+          nix build .#amalgam-image
+
+          echo "Building packages distribution image..."
+          nix build .#packages-image
+
+          echo "Building layered image (better caching)..."
+          nix build .#amalgam-layered
+
+      - name: Load images into Docker
+        run: |
+          echo "Loading amalgam image..."
+          docker load < result
+
+          # The result symlink points to the last built item
+          # We need to load each separately
+          docker load < $(nix build .#amalgam-image --print-out-paths)
+          docker load < $(nix build .#packages-image --print-out-paths)
+
+      - name: Tag images for registry
+        id: meta
+        run: |
+          # Determine tags based on trigger
+          if [[ "${{ github.event_name }}" == "release" ]]; then
+            VERSION="${{ github.event.release.tag_name }}"
+            TAGS="latest,$VERSION"
+          else
+            TAGS="latest,sha-${{ github.sha }}"
+          fi
+
+          echo "tags=$TAGS" >> $GITHUB_OUTPUT
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request' && (github.event.inputs.push == 'true' || github.event_name == 'release')
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push images to ghcr.io
+        if: github.event_name != 'pull_request' && (github.event.inputs.push == 'true' || github.event_name == 'release')
+        run: |
+          # Get the image names from the built images
+          AMALGAM_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep amalgam-compiler | head -1)
+          PACKAGES_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep nickel-packages | head -1)
+
+          # Tag and push amalgam compiler
+          for TAG in $(echo "${{ steps.meta.outputs.tags }}" | tr ',' ' '); do
+            docker tag $AMALGAM_IMAGE ghcr.io/${{ github.repository }}/amalgam:$TAG
+            docker push ghcr.io/${{ github.repository }}/amalgam:$TAG
+          done
+
+          # Tag and push packages
+          for TAG in $(echo "${{ steps.meta.outputs.tags }}" | tr ',' ' '); do
+            docker tag $PACKAGES_IMAGE ghcr.io/${{ github.repository }}/packages:$TAG
+            docker push ghcr.io/${{ github.repository }}/packages:$TAG
+          done
+
+          echo "✅ Images pushed to ghcr.io/${{ github.repository }}"
+
+      - name: Push using Nix helper (alternative method)
+        if: false  # Enable this to use the Nix-based push script
+        env:
+          REGISTRY: ghcr.io
+          REPO: ${{ github.repository }}
+          TAG: ${{ steps.meta.outputs.tags }}
+          REGISTRY_USER: ${{ github.actor }}
+          REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          nix run .#push-with-skopeo
+
+  # Build multi-platform images
+  build-multiplatform:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release'
+    strategy:
+      matrix:
+        platform: [linux/amd64, linux/arm64]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+
+      - name: Setup QEMU for cross-compilation
+        uses: docker/setup-qemu-action@v3
+
+      - name: Build for ${{ matrix.platform }}
+        run: |
+          # This would require cross-compilation support in the flake
+          echo "Building for ${{ matrix.platform }}..."
+          # nix build .#amalgam-image-${{ matrix.platform }}
+
+  # Streaming image build (most efficient)
+  build-stream:
+    runs-on: ubuntu-latest
+    if: false  # Enable when you add streaming support
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+
+      - name: Stream image directly to registry
+        env:
+          REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Stream the image directly without loading into Docker
+          nix build .#amalgam-stream --print-out-paths | \
+            skopeo copy docker-archive:/dev/stdin \
+            docker://ghcr.io/${{ github.repository }}/amalgam:stream \
+            --dest-username ${{ github.actor }} \
+            --dest-password $REGISTRY_PASSWORD