Release Opcore 0.2.0

.github/workflows/release.yml

@@ -107,11 +107,16 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           RELEASE_SHA: ${{ github.event.workflow_run.head_sha }}
         run: |
+          release_notes="docs/release/v${RELEASE_VERSION}.md"
+          if [[ ! -f "$release_notes" ]]; then
+            echo "::error::Missing release notes: $release_notes"
+            exit 1
+          fi
           if gh release view "v${RELEASE_VERSION}" >/dev/null 2>&1; then
             echo "GitHub release v${RELEASE_VERSION} already exists."
           else
             gh release create "v${RELEASE_VERSION}" \
               --target "$RELEASE_SHA" \
               --title "Opcore ${RELEASE_VERSION}" \
-              --notes "Initial Opcore alpha release."
+              --notes-file "$release_notes"
           fi

crates/graph-core/src/clone/analysis.rs

@@ -1,10 +1,14 @@
-use super::{CloneAnalysisRequest, CloneError, CloneFinding, CloneOverlay, CloneReportMode};
+use super::{
+    CloneAnalysisRequest, CloneError, CloneFinding, CloneOverlay, CloneReportMode,
+    CloneSourceReadMode,
+};
 use crate::extraction::{
     discover_sources_for_options, normalize_repo_relative_path, ExtractionOptions, SourceLanguage,
 };
 use sha2::{Digest, Sha256};
 use std::collections::{BTreeMap, BTreeSet};
 use std::path::Path;
+use std::process::Command;
 
 #[derive(Debug, Clone)]
 pub(super) struct CloneSource {
@@ -34,7 +38,10 @@ pub(super) fn clone_sources(
     repo_root: &Path,
     request: &CloneAnalysisRequest,
 ) -> Result<Vec<CloneSource>, CloneError> {
-    let mut sources = discovered_clone_sources(repo_root)?;
+    let mut sources = match request.source_paths.as_deref() {
+        Some(paths) => sparse_clone_sources(repo_root, request, paths)?,
+        None => discovered_clone_sources(repo_root)?,
+    };
     apply_overlays(&mut sources, repo_root, &request.overlays)?;
     if !request.exclude.is_empty() {
         sources.retain(|source| {
@@ -203,6 +210,100 @@ fn discovered_clone_sources(repo_root: &Path) -> Result<Vec<CloneSource>, CloneE
     Ok(sources)
 }
 
+fn sparse_clone_sources(
+    repo_root: &Path,
+    request: &CloneAnalysisRequest,
+    paths: &[String],
+) -> Result<Vec<CloneSource>, CloneError> {
+    let mut sources = Vec::new();
+    let read_mode = request
+        .source_read_mode
+        .unwrap_or(CloneSourceReadMode::Disk);
+    let introduced_paths = request.paths.iter().cloned().collect::<BTreeSet<_>>();
+    for path in paths {
+        let path = normalize_repo_relative_path(path, "clone request source path")
+            .map_err(|message| CloneError::InvalidRequest(message.to_string()))?;
+        let Some(language) = SourceLanguage::from_path(Path::new(&path)) else {
+            continue;
+        };
+        let Some(content) = read_sparse_source(
+            repo_root,
+            &path,
+            read_mode,
+            request.source_tree_ref.as_deref(),
+        )?
+        else {
+            continue;
+        };
+        let introduced = introduced_paths.contains(&path);
+        sources.push(CloneSource {
+            path,
+            language: language.as_str().to_string(),
+            sha256: sha256_hex(content.as_bytes()),
+            content,
+            introduced,
+        });
+    }
+    Ok(sources)
+}
+
+fn read_sparse_source(
+    repo_root: &Path,
+    path: &str,
+    read_mode: CloneSourceReadMode,
+    source_tree_ref: Option<&str>,
+) -> Result<Option<String>, CloneError> {
+    match read_mode {
+        CloneSourceReadMode::Disk => read_disk_sparse_source(repo_root, path),
+        CloneSourceReadMode::GitIndex => read_git_sparse_source(repo_root, &format!(":{path}")),
+        CloneSourceReadMode::GitTree => {
+            let tree_ref = source_tree_ref.ok_or_else(|| {
+                CloneError::InvalidRequest(
+                    "sourceTreeRef is required when sourceReadMode is gitTree".to_string(),
+                )
+            })?;
+            read_git_sparse_source(repo_root, &format!("{tree_ref}:{path}"))
+        }
+    }
+}
+
+fn read_disk_sparse_source(repo_root: &Path, path: &str) -> Result<Option<String>, CloneError> {
+    match std::fs::read_to_string(repo_root.join(path)) {
+        Ok(content) => Ok(Some(content)),
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(error) => Err(CloneError::Canonicalize(error)),
+    }
+}
+
+fn read_git_sparse_source(repo_root: &Path, spec: &str) -> Result<Option<String>, CloneError> {
+    let output = Command::new("git")
+        .arg("-C")
+        .arg(repo_root)
+        .arg("show")
+        .arg(spec)
+        .output()
+        .map_err(CloneError::Canonicalize)?;
+    if output.status.success() {
+        return Ok(Some(String::from_utf8_lossy(&output.stdout).into_owned()));
+    }
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    if missing_git_sparse_source(spec, &stderr) {
+        return Ok(None);
+    }
+    Err(CloneError::InvalidRequest(format!(
+        "clone sparse git source read failed for {spec}: {}",
+        stderr.trim()
+    )))
+}
+
+fn missing_git_sparse_source(spec: &str, stderr: &str) -> bool {
+    stderr.contains("exists on disk, but not in")
+        || stderr.contains("does not exist")
+        || stderr.contains("exists, but not")
+        || ((spec.starts_with("HEAD:") || spec.starts_with("@:"))
+            && stderr.contains("invalid object name"))
+}
+
 fn apply_overlays(
     sources: &mut Vec<CloneSource>,
     repo_root: &Path,

crates/graph-core/src/clone/mod.rs

@@ -44,6 +44,12 @@ pub struct CloneAnalysisRequest {
     pub report_mode: CloneReportMode,
     #[serde(skip_serializing_if = "Vec::is_empty", default)]
     pub paths: Vec<String>,
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub source_paths: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub source_read_mode: Option<CloneSourceReadMode>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub source_tree_ref: Option<String>,
     #[serde(skip_serializing_if = "Vec::is_empty", default)]
     pub overlays: Vec<CloneOverlay>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -69,6 +75,14 @@ pub enum CloneReportMode {
     Introduced,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum CloneSourceReadMode {
+    Disk,
+    GitIndex,
+    GitTree,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(tag = "action", rename_all = "camelCase")]
 pub enum CloneOverlay {
@@ -207,6 +221,7 @@ fn validate_request(request: &CloneAnalysisRequest) -> Result<(), CloneError> {
     }
     validate_positive_options(request)?;
     validate_request_paths(request)?;
+    validate_source_read_mode(request)?;
     validate_modes(&request.modes)?;
     Ok(())
 }
@@ -246,6 +261,12 @@ fn validate_request_paths(request: &CloneAnalysisRequest) -> Result<(), CloneErr
         normalize_repo_relative_path(path, "clone request path")
             .map_err(|message| CloneError::InvalidRequest(message.to_string()))?;
     }
+    if let Some(paths) = &request.source_paths {
+        for path in paths {
+            normalize_repo_relative_path(path, "clone request source path")
+                .map_err(|message| CloneError::InvalidRequest(message.to_string()))?;
+        }
+    }
     for (index, partition) in request.partitions.iter().enumerate() {
         if partition.is_empty() {
             return Err(CloneError::InvalidRequest(format!(
@@ -266,6 +287,29 @@ fn validate_request_paths(request: &CloneAnalysisRequest) -> Result<(), CloneErr
     Ok(())
 }
 
+fn validate_source_read_mode(request: &CloneAnalysisRequest) -> Result<(), CloneError> {
+    match request
+        .source_read_mode
+        .unwrap_or(CloneSourceReadMode::Disk)
+    {
+        CloneSourceReadMode::GitTree => {
+            if request.source_tree_ref.as_deref().is_none_or(str::is_empty) {
+                return Err(CloneError::InvalidRequest(
+                    "sourceTreeRef is required when sourceReadMode is gitTree".to_string(),
+                ));
+            }
+        }
+        CloneSourceReadMode::Disk | CloneSourceReadMode::GitIndex => {
+            if request.source_tree_ref.is_some() {
+                return Err(CloneError::InvalidRequest(
+                    "sourceTreeRef is only valid when sourceReadMode is gitTree".to_string(),
+                ));
+            }
+        }
+    }
+    Ok(())
+}
+
 fn validate_modes(modes: &[String]) -> Result<(), CloneError> {
     for mode in modes {
         if mode.is_empty() {

crates/graph-core/src/clone/tests.rs

@@ -156,6 +156,51 @@ fn scoped_no_overlay_requests_are_ephemeral() -> TestResult {
     Ok(())
 }
 
+#[test]
+fn sparse_path_list_request_reports_committed_peer_and_applies_write_delete_overlays() -> TestResult
+{
+    let fixture = clone_fixture()?;
+    let duplicate = duplicate_block();
+    write_source(fixture.path(), "src/peer.ts", &duplicate)?;
+    write_source(fixture.path(), "src/deleted.ts", &duplicate)?;
+    init_git_snapshot(fixture.path(), &["src/peer.ts", "src/deleted.ts"])?;
+
+    let mut clone_request = request(
+        fixture.path(),
+        CloneReportMode::Introduced,
+        vec![
+            CloneOverlay::Write {
+                path: "src/new.ts".to_string(),
+                content: duplicate,
+                checksum_before: None,
+            },
+            CloneOverlay::Delete {
+                path: "src/deleted.ts".to_string(),
+                checksum_before: None,
+            },
+        ],
+    )?;
+    clone_request.paths = vec!["src/new.ts".to_string(), "src/deleted.ts".to_string()];
+    clone_request.source_paths = Some(vec![
+        "src/peer.ts".to_string(),
+        "src/deleted.ts".to_string(),
+        "src/new.ts".to_string(),
+    ]);
+
+    let result = analyze_clones(clone_request)?;
+
+    assert!(!result.persisted);
+    assert!(result.db_path.is_none());
+    assert!(result.findings.iter().any(|finding| {
+        finding.path == "src/new.ts" && finding.peer_path == "src/peer.ts" && finding.introduced
+    }));
+    assert!(result
+        .findings
+        .iter()
+        .all(|finding| finding.path != "src/deleted.ts" && finding.peer_path != "src/deleted.ts"));
+    Ok(())
+}
+
 #[test]
 fn clone_exclude_patterns_remove_sources_from_analysis() -> TestResult {
     let fixture = clone_fixture()?;
@@ -263,6 +308,9 @@ fn request(
         },
         report_mode,
         paths: Vec::new(),
+        source_paths: None,
+        source_read_mode: None,
+        source_tree_ref: None,
         overlays,
         window_size: None,
         min_lines: Some(5),

docs/release/v0.2.0.md

@@ -0,0 +1,27 @@
+Opcore 0.2.0 makes the default robustness loop lighter on large repositories and strengthens validation correctness. This release reduces several major sources of unnecessary in-memory data while keeping coverage and degraded checks explicit.
+
+## Highlights
+
+- Bounds product scan validation output to summaries, counts, failed check IDs, and diagnostic samples instead of retaining and serializing an unrestricted result payload.
+- Makes validation file views lazy so path-specific checks do not enumerate every visible file unless they need the complete file universe.
+- Sends committed clone-analysis inputs as path references and reserves content-bearing overlays for changed or hypothetical files.
+- Replaces the Python syntax heuristic with parser-backed `ast.parse` validation, including structured locations and an honest unsupported result when no Python interpreter is available.
+- Adds native validation-policy parity and tightens the package guardrails around it.
+
+## Memory scope
+
+These changes materially reduce memory pressure in scan and validation paths, but they do not complete the broader large-monorepo memory program. Graph build/query working sets, inspect/edit language-service isolation, bounded history and sidecar payloads, and regression harness work remain tracked separately.
+
+## Upgrade
+
+```sh
+npm install --global opcore@0.2.0
+```
+
+Or run it without a global install:
+
+```sh
+npx opcore@0.2.0 --version
+```
+
+Full comparison: https://github.com/the-open-engine/opcore/compare/v0.1.0...v0.2.0

packages/contracts/schemas/opcore-contracts.schema.json

@@ -2039,6 +2039,23 @@
             "$ref": "#/$defs/RepoRelativePath"
           }
         },
+        "sourcePaths": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/RepoRelativePath"
+          }
+        },
+        "sourceReadMode": {
+          "enum": [
+            "disk",
+            "gitIndex",
+            "gitTree"
+          ]
+        },
+        "sourceTreeRef": {
+          "type": "string",
+          "minLength": 1
+        },
         "overlays": {
           "type": "array",
           "items": {

packages/contracts/src/index.ts

@@ -1061,13 +1061,19 @@ export type HypotheticalOverlay =
 export const cloneReportModes = ["all", "introduced"] as const;
 export type CloneReportMode = (typeof cloneReportModes)[number];
 
+export const cloneSourceReadModes = ["disk", "gitIndex", "gitTree"] as const;
+export type CloneSourceReadMode = (typeof cloneSourceReadModes)[number];
+
 export interface CloneAnalysisRequest {
   protocol: typeof CLONE_PROTOCOL;
   requestId?: string;
   schemaVersion: 1;
   repo: RepoIdentity;
   reportMode: CloneReportMode;
   paths?: readonly string[];
+  sourcePaths?: readonly string[];
+  sourceReadMode?: CloneSourceReadMode;
+  sourceTreeRef?: string;
   overlays: readonly HypotheticalOverlay[];
   windowSize?: number;
   minLines?: number;
@@ -6251,6 +6257,9 @@ export function validateCloneAnalysisRequest(request: CloneAnalysisRequest): Clo
   validateRepoIdentity(request.repo);
   validateCloneReportMode(request.reportMode, "Clone analysis request reportMode");
   if (request.paths !== undefined) validateRepoRelativePaths(request.paths, "Clone analysis request paths");
+  if (request.sourcePaths !== undefined) validateRepoRelativePaths(request.sourcePaths, "Clone analysis request sourcePaths");
+  if (request.sourceReadMode !== undefined) validateCloneSourceReadMode(request.sourceReadMode, "Clone analysis request sourceReadMode");
+  if (request.sourceTreeRef !== undefined) validateNonEmptyString(request.sourceTreeRef, "Clone analysis request sourceTreeRef");
   validateHypotheticalOverlays(request.overlays);
   if (request.windowSize !== undefined) validatePositiveInteger(request.windowSize, "Clone analysis request windowSize");
   if (request.minLines !== undefined) validatePositiveInteger(request.minLines, "Clone analysis request minLines");
@@ -6262,6 +6271,12 @@ export function validateCloneAnalysisRequest(request: CloneAnalysisRequest): Clo
   return request;
 }
 
+function validateCloneSourceReadMode(value: unknown, label: string): asserts value is CloneSourceReadMode {
+  if (!cloneSourceReadModes.includes(value as CloneSourceReadMode)) {
+    throw new Error(`${label} must be one of ${cloneSourceReadModes.join(", ")}`);
+  }
+}
+
 function validateCloneAnalysisPartitions(partitions: readonly (readonly string[])[]): void {
   if (!Array.isArray(partitions)) throw new Error("Clone analysis request partitions must be an array");
   for (const [index, partition] of partitions.entries()) {

packages/opcore-graph-core-darwin-arm64/metadata.json

@@ -4,6 +4,6 @@
   "targetPlatform": "darwin-arm64",
   "binaryPath": "opcore-graph-core",
   "checksumPath": "opcore-graph-core.sha256",
-  "checksumSha256": "c222610c492e628fa48ff50be22310e5bf8ebcd62bffbde1903d1fa91c45f6cb",
+  "checksumSha256": "3fb851c8e057f085c0dd1fb15a7e3a9796a6176974a8fb6e1f34a5222b1271ae",
   "buildProfile": "release"
 }

packages/opcore-graph-core-darwin-arm64/opcore-graph-core.sha256

@@ -1 +1 @@
-c222610c492e628fa48ff50be22310e5bf8ebcd62bffbde1903d1fa91c45f6cb  opcore-graph-core
+3fb851c8e057f085c0dd1fb15a7e3a9796a6176974a8fb6e1f34a5222b1271ae  opcore-graph-core