diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a00a79..cb1f584 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,6 @@ jobs: steps: - uses: actions/checkout@v1 - - name: Get Data - run: ./get_data.sh - uses: icepuma/rust-action@master with: args: cargo fmt -- --check && cargo clippy -- -Dwarnings && cargo test -- --include-ignored \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e959d48..ed1d4f6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,8 +11,6 @@ jobs: steps: - uses: actions/checkout@v1 - - name: Get Data - run: ./get_data.sh - uses: icepuma/rust-action@master with: args: cargo fmt -- --check && cargo clippy -- -Dwarnings && cargo test -- --include-ignored && cargo publish --token ${CARGO_REGISTRY_TOKEN} diff --git a/Cargo.toml b/Cargo.toml index 8954629..9e0d587 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,5 +13,7 @@ exclude = ["data/*"] [dependencies] chrono = "0.4" +flate2 = "1.0" +reqwest = { version = "0", features = ["blocking"] } auto_ops = "0.3" assert_float_eq = "1" diff --git a/get_data.sh b/get_data.sh deleted file mode 100755 index b31e3e5..0000000 --- a/get_data.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -########################################################################### -# Download star data from Yale Bright Stars Catalog and Hipparcos Catalog # -# Yale: http://tdc-www.harvard.edu/catalogs/bsc5.html # -# Hipparcos: https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html # -# Open Source BSC: https://github.com/johanley/star-catalog/ # -########################################################################### - -# TODO: make a simple rust cli to replace this script - -mkdir -p data/{Hipparcos,Yale,OSBSC} - -( - cd data/Hipparcos - wget -cv 'https://cdsarc.cds.unistra.fr/ftp/cats/I/239/hip_main.dat' - wget -cv 'https://cdsarc.cds.unistra.fr/ftp/cats/I/239/ReadMe' -) - -( - cd data/Yale - wget -cv 'http://tdc-www.harvard.edu/catalogs/bsc5.dat.gz' - wget -cv 'http://tdc-www.harvard.edu/catalogs/bsc5.readme' - wget -cv 'http://tdc-www.harvard.edu/catalogs/bsc5.notes.gz' - gunzip bsc5.dat.gz - gunzip bsc5.notes.gz -) - -( - cd data/OSBSC - wget -cv 'https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/ReadMe.utf8' - wget -cv 'https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/os-bright-star-catalog-hip.utf8' - wget -cv 'https://github.com/johanley/constellation-lines/raw/master/output/constellation-lines-hip.utf8' -) \ No newline at end of file diff --git a/src/catalog/hipparcos.rs b/src/catalog/hipparcos.rs index 0501a20..a671769 100644 --- a/src/catalog/hipparcos.rs +++ b/src/catalog/hipparcos.rs @@ -1,7 +1,5 @@ /*! # [Hipparcos Catalog](https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html) parser -> NOTE: run the `get_data.sh` script to get the tests to pass. - ## [The Hipparcos and Tycho Catalogues](https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html) (ESA 1997) - ESA 1997 @@ -377,6 +375,7 @@ H77 r_SpType Spect_Type_Source /Source of spectral type use super::ValidParse; use crate::angle::{Angle, Dms, Hms, Sign}; use crate::coord::{Declination, RightAscension}; +use crate::utils::{fetch_url, FetchResult}; use std::convert::TryFrom; @@ -768,6 +767,25 @@ impl ValidParse for HipparcosStar { } } +pub fn fetch_hipparcos() -> FetchResult<()> { + if !std::path::Path::new("data/Hipparcos/hip_main.dat").exists() { + let dir = "data/Hipparcos/"; + + std::fs::create_dir_all(dir).unwrap(); + + fetch_url( + "https://cdsarc.cds.unistra.fr/ftp/cats/I/239/hip_main.dat".to_string(), + "data/Hipparcos/hip_main.dat".to_string(), + )?; + fetch_url( + "https://cdsarc.cds.unistra.fr/ftp/cats/I/239/ReadMe".to_string(), + "data/Hipparcos/ReadMe".to_string(), + )?; + } + + Ok(()) +} + #[cfg(test)] mod tests { use crate::catalog::hipparcos::*; @@ -782,6 +800,7 @@ mod tests { #[test] #[ignore] fn test_catalog() { + fetch_hipparcos().unwrap(); let _stars = parse_catalog!( HipparcosStar, Path::new("data/Hipparcos/hip_main.dat"), diff --git a/src/catalog/osbsc.rs b/src/catalog/osbsc.rs index cf259c3..fcf6b8b 100644 --- a/src/catalog/osbsc.rs +++ b/src/catalog/osbsc.rs @@ -1,11 +1,10 @@ /*! [Open Source Bright Star Catalog](https://github.com/johanley/star-catalog) parser - -> NOTE: run the `get_data.sh` script to get the tests to pass. */ use super::ValidParse; use crate::angle::{Dms, Hms, Sign}; use crate::parse_trim; +use crate::utils::{fetch_url, FetchResult}; /// Parse an arc/minute/second field. macro_rules! parse_ams { @@ -239,6 +238,24 @@ impl ValidParse for OSBSCStar { } } +pub fn fetch_osbsc() -> FetchResult<()> { + if !std::path::Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8").exists() { + let dir = "data/OSBSC/"; + + std::fs::create_dir_all(dir).unwrap(); + + fetch_url( + "https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/ReadMe.utf8".to_string(), + "data/OSBSC/ReadMe.utf8".to_string(), + )?; + fetch_url( + "https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/os-bright-star-catalog-hip.utf8".to_string(), + "data/OSBSC/os-bright-star-catalog-hip.utf8".to_string(), + )?; + } + Ok(()) +} + #[cfg(test)] mod tests { use crate::catalog::osbsc::*; @@ -253,6 +270,7 @@ mod tests { #[test] #[ignore] fn test_catalog() { + fetch_osbsc().unwrap(); let _stars = parse_catalog!( OSBSCStar, Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8"), diff --git a/src/catalog/yale.rs b/src/catalog/yale.rs index 9d4c777..62fe836 100644 --- a/src/catalog/yale.rs +++ b/src/catalog/yale.rs @@ -1,7 +1,5 @@ /*! [Yale Bright Star Catalog](http://tdc-www.harvard.edu/catalogs/bsc5.html) parser -> NOTE: run the `get_data.sh` script to get the tests to pass. - The Bright Star Catalogue, 5th Revised Ed. (Preliminary Version) - Hoffleit D., Warren Jr W.H. @@ -159,7 +157,10 @@ Note on n_RadVel: */ use super::ValidParse; -use crate::parse_trim; +use crate::{ + parse_trim, + utils::{fetch_gz_url, fetch_url, FetchResult}, +}; #[allow(non_snake_case)] // Copying field names from original data source #[derive(Debug, Clone)] @@ -404,6 +405,29 @@ impl ValidParse for YaleStar { } } +pub fn fetch_yale() -> FetchResult<()> { + if !std::path::Path::new("data/Yale/bsc5.dat").exists() { + let dir = "data/Yale/"; + + std::fs::create_dir_all(dir).unwrap(); + + fetch_gz_url( + "http://tdc-www.harvard.edu/catalogs/bsc5.dat.gz".to_string(), + "data/Yale/bsc5.dat".to_string(), + )?; + fetch_url( + "http://tdc-www.harvard.edu/catalogs/bsc5.readme".to_string(), + "data/Yale/bsc5.readme".to_string(), + )?; + fetch_gz_url( + "http://tdc-www.harvard.edu/catalogs/bsc5.notes.gz".to_string(), + "data/Yale/bsc5.notes".to_string(), + )?; + // TODO: gunzip the files + } + Ok(()) +} + #[cfg(test)] mod tests { use crate::catalog::yale::*; @@ -418,6 +442,7 @@ mod tests { #[test] #[ignore] fn test_catalog() { + fetch_yale().unwrap(); let _stars = parse_catalog!(YaleStar, Path::new("data/Yale/bsc5.dat"), Some(197)); println!("Number of stars: {}", _stars.len()); println!("Last Star: {:?}", _stars.last().unwrap()); diff --git a/src/constellation.rs b/src/constellation.rs index 4b91005..3cd8108 100644 --- a/src/constellation.rs +++ b/src/constellation.rs @@ -4,7 +4,10 @@ WIP Open Source Constellation Catalog Parser > NOTE: still under construction! */ -use crate::catalog::osbsc::OSBSCStar; +use crate::{ + catalog::osbsc::OSBSCStar, + utils::{fetch_url, FetchResult}, +}; /// Polyline #[allow(dead_code)] // FIXME @@ -72,6 +75,20 @@ macro_rules! parse_constellation_catalog { }}; } +pub fn fetch_constellations() -> FetchResult<()> { + if !std::path::Path::new("data/OSBSC/constellation-lines-hip.utf8").exists() { + let dir = "data/OSBSC/"; + + std::fs::create_dir_all(dir).unwrap(); + + fetch_url( + "https://github.com/johanley/constellation-lines/raw/master/output/constellation-lines-hip.utf8".to_string(), + "data/OSBSC/constellation-lines-hip.utf8".to_string(), + )?; + } + Ok(()) +} + #[cfg(test)] mod tests { use std::collections::HashMap; @@ -83,6 +100,8 @@ mod tests { #[test] #[ignore] fn test_constellations_1() { + crate::catalog::osbsc::fetch_osbsc().unwrap(); + let _stars = parse_catalog!( OSBSCStar, Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8"), @@ -124,6 +143,8 @@ mod tests { #[test] #[ignore] fn test_constellations_2() { + crate::catalog::osbsc::fetch_osbsc().unwrap(); + let _stars = parse_catalog!( OSBSCStar, Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8"), @@ -138,6 +159,8 @@ mod tests { _star_map.insert(star.Hipparcos_id.unwrap(), star); } + fetch_constellations().unwrap(); + let constells = parse_constellation_catalog!( Path::new("data/OSBSC/constellation-lines-hip.utf8"), _star_map diff --git a/src/lib.rs b/src/lib.rs index b02d394..7eed83f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,6 @@ A simple star catalog parser and primitive types for star coordinates and astronomical times. > Note: currently supports the [Yale Bright Stars Catalog](http://tdc-www.harvard.edu/catalogs/bsc5.html), the [Hipparcos Catalog](https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html), and the [Open Source Bright Star Catalog](https://github.com/johanley/star-catalog). - Run the `get_data.sh` script to fetch the catalogs. */ #[macro_use] @@ -18,3 +17,4 @@ pub mod constellation; pub mod coord; pub mod star; pub mod time; +pub mod utils; diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..0f5f064 --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,37 @@ +/*! +Utility functions +*/ + +use flate2::read::GzDecoder; + +/** +Fetch function result. +Allows a dynamic error type so we don't have to explicitly coerce upstream errors to the local one. +*/ +pub type FetchResult = std::result::Result>; + +/** Fetch a file from url. + Adapted from . +*/ +pub fn fetch_url(url: String, file_name: String) -> FetchResult<()> { + let response = reqwest::blocking::get(url)?; + let mut file = std::fs::File::create(file_name)?; + let mut content = std::io::Cursor::new(response.bytes()?); + std::io::copy(&mut content, &mut file)?; + Ok(()) +} + +/** Fetch a gz file from url and decompress it. + Adapted from . +*/ +pub fn fetch_gz_url(url: String, file_name: String) -> FetchResult<()> { + let response = reqwest::blocking::get(url)?; + // let resp_text = response.text()?; + let mut d = GzDecoder::new(response); + let mut file = std::fs::File::create(file_name)?; + // let mut content = std::io::Cursor::new(d.bytes()); + std::io::copy(&mut d, &mut file)?; + Ok(()) +} + +// TODO: add unit tests for fetching functions.