feat: fetch data

.github/workflows/ci.yml

@@ -8,8 +8,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
-      - name: Get Data 
-        run: ./get_data.sh
       - uses: icepuma/rust-action@master
         with:
           args: cargo fmt -- --check && cargo clippy -- -Dwarnings && cargo test -- --include-ignored

.github/workflows/publish.yml

@@ -11,8 +11,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
-      - name: Get Data 
-        run: ./get_data.sh
       - uses: icepuma/rust-action@master
         with:
           args: cargo fmt -- --check && cargo clippy -- -Dwarnings && cargo test -- --include-ignored && cargo publish --token ${CARGO_REGISTRY_TOKEN}

Cargo.toml

@@ -13,5 +13,7 @@ exclude = ["data/*"]
 
 [dependencies]
 chrono = "0.4"
+flate2 = "1.0"
+reqwest = { version = "0", features = ["blocking"] }
 auto_ops = "0.3"
 assert_float_eq = "1"

src/catalog/hipparcos.rs

@@ -1,7 +1,5 @@
 /*! # [Hipparcos Catalog](https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html) parser
 
-> NOTE: run the `get_data.sh` script to get the tests to pass.
-
 ## [The Hipparcos and Tycho Catalogues](https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html) (ESA 1997)
 
 - ESA 1997
@@ -377,6 +375,7 @@ H77         r_SpType   Spect_Type_Source  /Source of spectral type
 use super::ValidParse;
 use crate::angle::{Angle, Dms, Hms, Sign};
 use crate::coord::{Declination, RightAscension};
+use crate::utils::{fetch_url, FetchResult};
 
 use std::convert::TryFrom;
 
@@ -768,6 +767,25 @@ impl ValidParse for HipparcosStar {
     }
 }
 
+pub fn fetch_hipparcos() -> FetchResult<()> {
+    if !std::path::Path::new("data/Hipparcos/hip_main.dat").exists() {
+        let dir = "data/Hipparcos/";
+
+        std::fs::create_dir_all(dir).unwrap();
+
+        fetch_url(
+            "https://cdsarc.cds.unistra.fr/ftp/cats/I/239/hip_main.dat".to_string(),
+            "data/Hipparcos/hip_main.dat".to_string(),
+        )?;
+        fetch_url(
+            "https://cdsarc.cds.unistra.fr/ftp/cats/I/239/ReadMe".to_string(),
+            "data/Hipparcos/ReadMe".to_string(),
+        )?;
+    }
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use crate::catalog::hipparcos::*;
@@ -782,6 +800,7 @@ mod tests {
     #[test]
     #[ignore]
     fn test_catalog() {
+        fetch_hipparcos().unwrap();
         let _stars = parse_catalog!(
             HipparcosStar,
             Path::new("data/Hipparcos/hip_main.dat"),

src/catalog/osbsc.rs

@@ -1,11 +1,10 @@
 /*!
 [Open Source Bright Star Catalog](https://github.com/johanley/star-catalog) parser
-
-> NOTE: run the `get_data.sh` script to get the tests to pass.
 */
 use super::ValidParse;
 use crate::angle::{Dms, Hms, Sign};
 use crate::parse_trim;
+use crate::utils::{fetch_url, FetchResult};
 
 /// Parse an arc/minute/second field.
 macro_rules! parse_ams {
@@ -239,6 +238,24 @@ impl ValidParse for OSBSCStar {
     }
 }
 
+pub fn fetch_osbsc() -> FetchResult<()> {
+    if !std::path::Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8").exists() {
+        let dir = "data/OSBSC/";
+
+        std::fs::create_dir_all(dir).unwrap();
+
+        fetch_url(
+        "https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/ReadMe.utf8".to_string(),
+        "data/OSBSC/ReadMe.utf8".to_string(),
+    )?;
+        fetch_url(
+        "https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/os-bright-star-catalog-hip.utf8".to_string(),
+        "data/OSBSC/os-bright-star-catalog-hip.utf8".to_string(),
+    )?;
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use crate::catalog::osbsc::*;
@@ -253,6 +270,7 @@ mod tests {
     #[test]
     #[ignore]
     fn test_catalog() {
+        fetch_osbsc().unwrap();
         let _stars = parse_catalog!(
             OSBSCStar,
             Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8"),

src/catalog/yale.rs

@@ -1,7 +1,5 @@
 /*!  [Yale Bright Star Catalog](http://tdc-www.harvard.edu/catalogs/bsc5.html) parser
 
-> NOTE: run the `get_data.sh` script to get the tests to pass.
-
 The Bright Star Catalogue,  5th Revised Ed. (Preliminary Version)
 
 - Hoffleit D., Warren Jr W.H.
@@ -159,7 +157,10 @@ Note on n_RadVel:
 
 */
 use super::ValidParse;
-use crate::parse_trim;
+use crate::{
+    parse_trim,
+    utils::{fetch_gz_url, fetch_url, FetchResult},
+};
 
 #[allow(non_snake_case)] // Copying field names from original data source
 #[derive(Debug, Clone)]
@@ -404,6 +405,29 @@ impl ValidParse for YaleStar {
     }
 }
 
+pub fn fetch_yale() -> FetchResult<()> {
+    if !std::path::Path::new("data/Yale/bsc5.dat").exists() {
+        let dir = "data/Yale/";
+
+        std::fs::create_dir_all(dir).unwrap();
+
+        fetch_gz_url(
+            "http://tdc-www.harvard.edu/catalogs/bsc5.dat.gz".to_string(),
+            "data/Yale/bsc5.dat".to_string(),
+        )?;
+        fetch_url(
+            "http://tdc-www.harvard.edu/catalogs/bsc5.readme".to_string(),
+            "data/Yale/bsc5.readme".to_string(),
+        )?;
+        fetch_gz_url(
+            "http://tdc-www.harvard.edu/catalogs/bsc5.notes.gz".to_string(),
+            "data/Yale/bsc5.notes".to_string(),
+        )?;
+        // TODO: gunzip the files
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use crate::catalog::yale::*;
@@ -418,6 +442,7 @@ mod tests {
     #[test]
     #[ignore]
     fn test_catalog() {
+        fetch_yale().unwrap();
         let _stars = parse_catalog!(YaleStar, Path::new("data/Yale/bsc5.dat"), Some(197));
         println!("Number of stars: {}", _stars.len());
         println!("Last Star: {:?}", _stars.last().unwrap());

src/constellation.rs

@@ -4,7 +4,10 @@ WIP Open Source Constellation Catalog Parser
 > NOTE: still under construction!
  */
 
-use crate::catalog::osbsc::OSBSCStar;
+use crate::{
+    catalog::osbsc::OSBSCStar,
+    utils::{fetch_url, FetchResult},
+};
 
 /// Polyline
 #[allow(dead_code)] // FIXME
@@ -72,6 +75,20 @@ macro_rules! parse_constellation_catalog {
     }};
 }
 
+pub fn fetch_constellations() -> FetchResult<()> {
+    if !std::path::Path::new("data/OSBSC/constellation-lines-hip.utf8").exists() {
+        let dir = "data/OSBSC/";
+
+        std::fs::create_dir_all(dir).unwrap();
+
+        fetch_url(
+        "https://github.com/johanley/constellation-lines/raw/master/output/constellation-lines-hip.utf8".to_string(),
+        "data/OSBSC/constellation-lines-hip.utf8".to_string(),
+    )?;
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
@@ -83,6 +100,8 @@ mod tests {
     #[test]
     #[ignore]
     fn test_constellations_1() {
+        crate::catalog::osbsc::fetch_osbsc().unwrap();
+
         let _stars = parse_catalog!(
             OSBSCStar,
             Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8"),
@@ -124,6 +143,8 @@ mod tests {
     #[test]
     #[ignore]
     fn test_constellations_2() {
+        crate::catalog::osbsc::fetch_osbsc().unwrap();
+
         let _stars = parse_catalog!(
             OSBSCStar,
             Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8"),
@@ -138,6 +159,8 @@ mod tests {
             _star_map.insert(star.Hipparcos_id.unwrap(), star);
         }
 
+        fetch_constellations().unwrap();
+
         let constells = parse_constellation_catalog!(
             Path::new("data/OSBSC/constellation-lines-hip.utf8"),
             _star_map

src/lib.rs

@@ -6,7 +6,6 @@
 A simple star catalog parser and primitive types for star coordinates and astronomical times.
 
 > Note: currently supports the [Yale Bright Stars Catalog](http://tdc-www.harvard.edu/catalogs/bsc5.html),  the [Hipparcos Catalog](https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html), and the [Open Source Bright Star Catalog](https://github.com/johanley/star-catalog).
-  Run the `get_data.sh` script to fetch the catalogs.
  */
 
 #[macro_use]
@@ -18,3 +17,4 @@ pub mod constellation;
 pub mod coord;
 pub mod star;
 pub mod time;
+pub mod utils;

src/utils.rs

@@ -0,0 +1,37 @@
+/*!
+Utility functions
+*/
+
+use flate2::read::GzDecoder;
+
+/**
+Fetch function result.
+Allows a dynamic error type so we don't have to explicitly coerce upstream errors to the local one.
+*/
+pub type FetchResult<T> = std::result::Result<T, Box<dyn std::error::Error + Send + Sync>>;
+
+/** Fetch a file from url.
+ Adapted from <https://georgik.rocks/how-to-download-binary-file-in-rust-by-reqwest/>.
+*/
+pub fn fetch_url(url: String, file_name: String) -> FetchResult<()> {
+    let response = reqwest::blocking::get(url)?;
+    let mut file = std::fs::File::create(file_name)?;
+    let mut content = std::io::Cursor::new(response.bytes()?);
+    std::io::copy(&mut content, &mut file)?;
+    Ok(())
+}
+
+/** Fetch a gz file from url and decompress it.
+ Adapted from <https://georgik.rocks/how-to-download-binary-file-in-rust-by-reqwest/>.
+*/
+pub fn fetch_gz_url(url: String, file_name: String) -> FetchResult<()> {
+    let response = reqwest::blocking::get(url)?;
+    // let resp_text = response.text()?;
+    let mut d = GzDecoder::new(response);
+    let mut file = std::fs::File::create(file_name)?;
+    // let mut content = std::io::Cursor::new(d.bytes());
+    std::io::copy(&mut d, &mut file)?;
+    Ok(())
+}
+
+// TODO: add unit tests for fetching functions.