Add PackageCache support for base packages

crates/ark/src/lsp/diagnostics.rs

@@ -1647,6 +1647,7 @@ foo
                 version: "1.0.0".to_string(),
                 depends: vec![],
                 repository: None,
+                priority: None,
                 fields: Dcf::new(),
             };
             let package = Package::from_parts(PathBuf::from("/mock/path"), description, namespace);
@@ -1744,6 +1745,7 @@ foo
                 version: "1.0.0".to_string(),
                 depends: vec![],
                 repository: None,
+                priority: None,
                 fields: Dcf::new(),
             };
             let package1 =
@@ -1760,6 +1762,7 @@ foo
                 version: "1.0.0".to_string(),
                 depends: vec![],
                 repository: None,
+                priority: None,
                 fields: Dcf::new(),
             };
             let package2 =
@@ -1818,6 +1821,7 @@ foo
                 version: "1.0.0".to_string(),
                 depends: vec![],
                 repository: None,
+                priority: None,
                 fields: Dcf::new(),
             };
             let package = Package::from_parts(PathBuf::from("/mock/path"), description, namespace);

crates/oak_package/src/package_description.rs

@@ -40,6 +40,8 @@ pub struct Description {
 
     pub repository: Option<Repository>,
 
+    pub priority: Option<Priority>,
+
     /// Raw DCF fields
     pub fields: Dcf,
 }
@@ -49,6 +51,12 @@ pub enum Repository {
     CRAN,
 }
 
+#[derive(Clone, Debug, PartialEq)]
+pub enum Priority {
+    Base,
+    Recommended,
+}
+
 impl Description {
     /// Parse a DESCRIPTION file in DCF format
     pub fn parse(contents: &str) -> anyhow::Result<Self> {
@@ -84,11 +92,22 @@ impl Description {
             None
         });
 
+        let priority = fields.get("Priority").and_then(|priority| {
+            if priority == "base" {
+                return Some(Priority::Base);
+            }
+            if priority == "recommended" {
+                return Some(Priority::Recommended);
+            }
+            None
+        });
+
         Ok(Description {
             name,
             version,
             depends,
             repository,
+            priority,
             fields,
         })
     }
@@ -214,6 +233,26 @@ Repository: CRAN"#;
         assert_eq!(parsed.repository, Some(Repository::CRAN));
     }
 
+    #[test]
+    fn parses_description_with_priority() {
+        let desc = r#"Package: utils
+Version: 4.5.0
+Priority: base"#;
+        let parsed = Description::parse(desc).unwrap();
+        assert_eq!(parsed.priority, Some(Priority::Base));
+
+        let desc = r#"Package: MASS
+Version: 7.3-65
+Priority: recommended"#;
+        let parsed = Description::parse(desc).unwrap();
+        assert_eq!(parsed.priority, Some(Priority::Recommended));
+
+        let desc = r#"Package: mypkg
+Version: 1.0.0"#;
+        let parsed = Description::parse(desc).unwrap();
+        assert!(parsed.priority.is_none());
+    }
+
     #[test]
     fn parses_description_with_unknown_repository() {
         let desc = r#"Package: mypackage

crates/oak_sources/src/base.rs

@@ -0,0 +1,138 @@
+use std::io::Cursor;
+use std::io::Read;
+
+use flate2::read::GzDecoder;
+use oak_fs::file_lock::FileLock;
+
+use crate::download::Outcome;
+
+/// Names of the R base packages, i.e. everything that ships with R and carries
+/// `Priority: base` in its DESCRIPTION.
+pub(crate) const BASE_PACKAGES: &[&str] = &[
+    "base",
+    "compiler",
+    "datasets",
+    "graphics",
+    "grDevices",
+    "grid",
+    "methods",
+    "parallel",
+    "splines",
+    "stats",
+    "stats4",
+    "tcltk",
+    "tools",
+    "utils",
+];
+
+/// Download the R source tarball for R {version} from CRAN's archive.
+///
+/// Base R packages (e.g. `base`, `utils`, `stats`) are not distributed at the standard
+/// `src/contrib/` location on CRAN. Instead, we must retrieve them from the base R
+/// sources themselves, which lives at `src/base/R-{major}/R-{version}.tar.gz`. Each
+/// package is located inside that tarball at `src/library/{package}/`.
+///
+/// Returns `Ok(None)` if the tarball is not on CRAN (e.g. a development R version), which
+/// we treat as "source unavailable" rather than an error.
+pub(crate) fn download(version: &str) -> anyhow::Result<Option<Vec<u8>>> {
+    let major = version
+        .split('.')
+        .next()
+        .ok_or_else(|| anyhow::anyhow!("Invalid R version for base source download: {version}"))?;
+
+    let mirrors = ["https://cran.r-project.org", "https://cran.rstudio.com"];
+    let suffix = format!("src/base/R-{major}/R-{version}.tar.gz");
+
+    match crate::download::download_with_mirrors(&suffix, &mirrors)? {
+        Outcome::Success(response) => {
+            let mut bytes = Vec::new();
+            response.into_body().into_reader().read_to_end(&mut bytes)?;
+            Ok(Some(bytes))
+        },
+        Outcome::NotFound => Ok(None),
+    }
+}
+
+/// Extract a single base package's R files from the R source tarball bytes.
+///
+/// Writes `R-{version}/src/library/{package}/R/*.R` entries into an `R/` folder inside
+/// the directory `destination_lock` lives in. Files are marked read only to match the
+/// rest of the cache.
+pub(crate) fn extract(
+    package: &str,
+    version: &str,
+    bytes: &[u8],
+    destination_lock: &FileLock,
+) -> anyhow::Result<()> {
+    let destination = destination_lock.parent().join("R");
+    std::fs::create_dir(&destination)?;
+
+    let cursor = Cursor::new(bytes);
+    let gz = GzDecoder::new(cursor);
+    let mut archive = tar::Archive::new(gz);
+
+    let prefix = format!("R-{version}/src/library/{package}/R/");
+
+    for entry in archive.entries()? {
+        let mut entry = entry?;
+        let path = entry.path()?;
+
+        let Some(relative) = path.strip_prefix(&prefix).ok() else {
+            continue;
+        };
+
+        if relative
+            .extension()
+            .is_none_or(|ext| ext != "R" && ext != "r")
+        {
+            continue;
+        }
+
+        let absolute = destination.join(relative);
+
+        // Some base packages (e.g. `utils`) have platform-specific subdirs under `R/`
+        // like `R/windows/` and `R/unix/` (their `Makefile` handles them at install
+        // time). Create parents if one is required so `unpack()` can write nested files.
+        if let Some(parent) = relative.parent().filter(|p| !p.as_os_str().is_empty()) {
+            std::fs::create_dir_all(destination.join(parent))?;
+        }
+
+        entry.unpack(&absolute)?;
+        crate::fs::set_readonly(&absolute)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use oak_fs::file_lock::Filesystem;
+    use tempfile::TempDir;
+
+    use crate::base::download;
+    use crate::base::extract;
+
+    /// Requires internet access and downloads a large tarball of the R sources
+    #[ignore = "Downloads a 40mb tarball"]
+    #[test]
+    fn test_base_download_and_extract() {
+        let bytes = download("4.5.0").unwrap().expect("R 4.5.0 source to exist");
+
+        let destination_tempdir = TempDir::new().unwrap();
+        let destination = Filesystem::new(destination_tempdir.path().to_path_buf());
+        let destination_lock = destination.open_rw_exclusive_create(".lock").unwrap();
+
+        extract("utils", "4.5.0", &bytes, &destination_lock).unwrap();
+
+        // Spot check: `utils` has a well-known `help.R` file
+        let help = destination_lock.parent().join("R").join("help.R");
+        assert!(help.exists());
+        assert!(help.metadata().unwrap().permissions().readonly());
+    }
+
+    #[test]
+    fn test_base_download_unknown_version_returns_none() {
+        let bytes = download("0.0.0").unwrap();
+        assert!(bytes.is_none());
+    }
+}

crates/oak_sources/src/hash.rs

@@ -0,0 +1,9 @@
+use sha2::Digest;
+use sha2::Sha256;
+
+/// Retain 8 ASCII characters for each hash fragment
+pub(crate) fn hash(contents: &str) -> String {
+    let mut hash = hex::encode(Sha256::digest(contents));
+    hash.truncate(8);
+    hash
+}

crates/oak_sources/src/installed_package.rs

@@ -0,0 +1,98 @@
+use std::fs::read_to_string;
+use std::path::Path;
+use std::path::PathBuf;
+
+use oak_package::package_description::Description;
+
+pub(crate) struct InstalledPackage {
+    key: String,
+    name: String,
+    library_path: PathBuf,
+    description: Description,
+    description_hash: String,
+}
+
+impl InstalledPackage {
+    pub(crate) fn find(package: &str, library_paths: &[PathBuf]) -> anyhow::Result<Option<Self>> {
+        let mut library_path = None;
+
+        for library_path_candidate in library_paths {
+            if library_path_candidate.join(package).exists() {
+                library_path = Some(library_path_candidate);
+                break;
+            }
+        }
+
+        let Some(library_path) = library_path else {
+            // Not installed
+            return Ok(None);
+        };
+
+        let package_path = library_path.join(package);
+
+        let description_path = package_path.join("DESCRIPTION");
+        let description_contents = read_to_string(&description_path)?;
+        let description = Description::parse(&description_contents)?;
+
+        let library_path_hash = crate::hash::hash(library_path.to_string_lossy().as_ref());
+        let description_hash = crate::hash::hash(&description_contents);
+
+        // Flat key unique enough to handle:
+        // - The same R package across multiple libpaths
+        // - Reinstalling a dev R package without changing the version (0.1.0.9000)
+        let key = format!(
+            "{name}_{version}_libpath-{library_path_hash}_description-{description_hash}",
+            name = package,
+            version = &description.version,
+            library_path_hash = &library_path_hash,
+            description_hash = &description_hash
+        );
+
+        Ok(Some(Self {
+            key,
+            name: package.to_string(),
+            library_path: library_path.clone(),
+            description,
+            description_hash,
+        }))
+    }
+
+    pub(crate) fn name(&self) -> &str {
+        &self.name
+    }
+
+    pub(crate) fn version(&self) -> &str {
+        &self.description().version
+    }
+
+    pub(crate) fn description(&self) -> &Description {
+        &self.description
+    }
+
+    // Flat key unique enough to handle:
+    // - The same R package across multiple libpaths
+    // - Reinstalling a dev R package without changing the version (0.1.0.9000)
+    pub(crate) fn key(&self) -> &str {
+        &self.key
+    }
+
+    pub(crate) fn library_path(&self) -> &Path {
+        self.library_path.as_path()
+    }
+
+    pub(crate) fn package_path(&self) -> PathBuf {
+        self.library_path.join(&self.name)
+    }
+
+    pub(crate) fn description_path(&self) -> PathBuf {
+        self.package_path().join("DESCRIPTION")
+    }
+
+    pub(crate) fn namespace_path(&self) -> PathBuf {
+        self.package_path().join("NAMESPACE")
+    }
+
+    pub(crate) fn description_hash(&self) -> &str {
+        &self.description_hash
+    }
+}