From e4e2055fd7f6ca384c9ccccb3a4910687bfff884 Mon Sep 17 00:00:00 2001 From: Matt Anderson Date: Fri, 23 Dec 2022 19:10:43 -0600 Subject: [PATCH 1/4] feat: fetch data --- .github/workflows/ci.yml | 2 -- Cargo.toml | 2 ++ get_data.sh | 34 ---------------------------------- src/catalog/hipparcos.rs | 23 +++++++++++++++++++++-- src/catalog/osbsc.rs | 23 +++++++++++++++++++++-- src/catalog/yale.rs | 28 +++++++++++++++++++++++++--- src/constellation.rs | 26 +++++++++++++++++++++++++- src/lib.rs | 2 +- src/utils.rs | 37 +++++++++++++++++++++++++++++++++++++ 9 files changed, 132 insertions(+), 45 deletions(-) delete mode 100755 get_data.sh create mode 100644 src/utils.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a00a79..cb1f584 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,6 @@ jobs: steps: - uses: actions/checkout@v1 - - name: Get Data - run: ./get_data.sh - uses: icepuma/rust-action@master with: args: cargo fmt -- --check && cargo clippy -- -Dwarnings && cargo test -- --include-ignored \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 8954629..9e0d587 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,5 +13,7 @@ exclude = ["data/*"] [dependencies] chrono = "0.4" +flate2 = "1.0" +reqwest = { version = "0", features = ["blocking"] } auto_ops = "0.3" assert_float_eq = "1" diff --git a/get_data.sh b/get_data.sh deleted file mode 100755 index b31e3e5..0000000 --- a/get_data.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -########################################################################### -# Download star data from Yale Bright Stars Catalog and Hipparcos Catalog # -# Yale: http://tdc-www.harvard.edu/catalogs/bsc5.html # -# Hipparcos: https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html # -# Open Source BSC: https://github.com/johanley/star-catalog/ # -########################################################################### - -# TODO: make a simple rust cli to replace this script - -mkdir -p data/{Hipparcos,Yale,OSBSC} - -( - cd data/Hipparcos - wget -cv 'https://cdsarc.cds.unistra.fr/ftp/cats/I/239/hip_main.dat' - wget -cv 'https://cdsarc.cds.unistra.fr/ftp/cats/I/239/ReadMe' -) - -( - cd data/Yale - wget -cv 'http://tdc-www.harvard.edu/catalogs/bsc5.dat.gz' - wget -cv 'http://tdc-www.harvard.edu/catalogs/bsc5.readme' - wget -cv 'http://tdc-www.harvard.edu/catalogs/bsc5.notes.gz' - gunzip bsc5.dat.gz - gunzip bsc5.notes.gz -) - -( - cd data/OSBSC - wget -cv 'https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/ReadMe.utf8' - wget -cv 'https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/os-bright-star-catalog-hip.utf8' - wget -cv 'https://github.com/johanley/constellation-lines/raw/master/output/constellation-lines-hip.utf8' -) \ No newline at end of file diff --git a/src/catalog/hipparcos.rs b/src/catalog/hipparcos.rs index 0501a20..a671769 100644 --- a/src/catalog/hipparcos.rs +++ b/src/catalog/hipparcos.rs @@ -1,7 +1,5 @@ /*! # [Hipparcos Catalog](https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html) parser -> NOTE: run the `get_data.sh` script to get the tests to pass. - ## [The Hipparcos and Tycho Catalogues](https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html) (ESA 1997) - ESA 1997 @@ -377,6 +375,7 @@ H77 r_SpType Spect_Type_Source /Source of spectral type use super::ValidParse; use crate::angle::{Angle, Dms, Hms, Sign}; use crate::coord::{Declination, RightAscension}; +use crate::utils::{fetch_url, FetchResult}; use std::convert::TryFrom; @@ -768,6 +767,25 @@ impl ValidParse for HipparcosStar { } } +pub fn fetch_hipparcos() -> FetchResult<()> { + if !std::path::Path::new("data/Hipparcos/hip_main.dat").exists() { + let dir = "data/Hipparcos/"; + + std::fs::create_dir_all(dir).unwrap(); + + fetch_url( + "https://cdsarc.cds.unistra.fr/ftp/cats/I/239/hip_main.dat".to_string(), + "data/Hipparcos/hip_main.dat".to_string(), + )?; + fetch_url( + "https://cdsarc.cds.unistra.fr/ftp/cats/I/239/ReadMe".to_string(), + "data/Hipparcos/ReadMe".to_string(), + )?; + } + + Ok(()) +} + #[cfg(test)] mod tests { use crate::catalog::hipparcos::*; @@ -782,6 +800,7 @@ mod tests { #[test] #[ignore] fn test_catalog() { + fetch_hipparcos().unwrap(); let _stars = parse_catalog!( HipparcosStar, Path::new("data/Hipparcos/hip_main.dat"), diff --git a/src/catalog/osbsc.rs b/src/catalog/osbsc.rs index cf259c3..c680de9 100644 --- a/src/catalog/osbsc.rs +++ b/src/catalog/osbsc.rs @@ -1,11 +1,10 @@ /*! [Open Source Bright Star Catalog](https://github.com/johanley/star-catalog) parser - -> NOTE: run the `get_data.sh` script to get the tests to pass. */ use super::ValidParse; use crate::angle::{Dms, Hms, Sign}; use crate::parse_trim; +use crate::utils::{FetchResult, fetch_url}; /// Parse an arc/minute/second field. macro_rules! parse_ams { @@ -239,6 +238,25 @@ impl ValidParse for OSBSCStar { } } +pub fn fetch_osbsc() -> FetchResult<()> { + if !std::path::Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8").exists() + { + let dir = "data/OSBSC/"; + + std::fs::create_dir_all(dir).unwrap(); + + fetch_url( + "https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/ReadMe.utf8".to_string(), + "data/OSBSC/ReadMe.utf8".to_string(), + )?; + fetch_url( + "https://github.com/johanley/star-catalog/raw/master/catalogs/output/open-source-bsc/os-bright-star-catalog-hip.utf8".to_string(), + "data/OSBSC/os-bright-star-catalog-hip.utf8".to_string(), + )?; + } + Ok(()) +} + #[cfg(test)] mod tests { use crate::catalog::osbsc::*; @@ -253,6 +271,7 @@ mod tests { #[test] #[ignore] fn test_catalog() { + fetch_osbsc().unwrap(); let _stars = parse_catalog!( OSBSCStar, Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8"), diff --git a/src/catalog/yale.rs b/src/catalog/yale.rs index 9d4c777..5d0b987 100644 --- a/src/catalog/yale.rs +++ b/src/catalog/yale.rs @@ -1,7 +1,5 @@ /*! [Yale Bright Star Catalog](http://tdc-www.harvard.edu/catalogs/bsc5.html) parser -> NOTE: run the `get_data.sh` script to get the tests to pass. - The Bright Star Catalogue, 5th Revised Ed. (Preliminary Version) - Hoffleit D., Warren Jr W.H. @@ -159,7 +157,7 @@ Note on n_RadVel: */ use super::ValidParse; -use crate::parse_trim; +use crate::{parse_trim, utils::{FetchResult, fetch_gz_url, fetch_url}}; #[allow(non_snake_case)] // Copying field names from original data source #[derive(Debug, Clone)] @@ -404,6 +402,29 @@ impl ValidParse for YaleStar { } } +pub fn fetch_yale() -> FetchResult<()> { + if !std::path::Path::new("data/Yale/bsc5.dat").exists() { + let dir = "data/Yale/"; + + std::fs::create_dir_all(dir).unwrap(); + + fetch_gz_url( + "http://tdc-www.harvard.edu/catalogs/bsc5.dat.gz".to_string(), + "data/Yale/bsc5.dat".to_string(), + )?; + fetch_url( + "http://tdc-www.harvard.edu/catalogs/bsc5.readme".to_string(), + "data/Yale/bsc5.readme".to_string(), + )?; + fetch_gz_url( + "http://tdc-www.harvard.edu/catalogs/bsc5.notes.gz".to_string(), + "data/Yale/bsc5.notes".to_string(), + )?; + // TODO: gunzip the files + } + Ok(()) +} + #[cfg(test)] mod tests { use crate::catalog::yale::*; @@ -418,6 +439,7 @@ mod tests { #[test] #[ignore] fn test_catalog() { + fetch_yale().unwrap(); let _stars = parse_catalog!(YaleStar, Path::new("data/Yale/bsc5.dat"), Some(197)); println!("Number of stars: {}", _stars.len()); println!("Last Star: {:?}", _stars.last().unwrap()); diff --git a/src/constellation.rs b/src/constellation.rs index 4b91005..d8bc53a 100644 --- a/src/constellation.rs +++ b/src/constellation.rs @@ -4,7 +4,7 @@ WIP Open Source Constellation Catalog Parser > NOTE: still under construction! */ -use crate::catalog::osbsc::OSBSCStar; +use crate::{catalog::osbsc::OSBSCStar, utils::{FetchResult, fetch_url}}; /// Polyline #[allow(dead_code)] // FIXME @@ -72,6 +72,22 @@ macro_rules! parse_constellation_catalog { }}; } +pub fn fetch_constellations() -> FetchResult<()> { + if + !std::path::Path::new("data/OSBSC/constellation-lines-hip.utf8").exists() + { + let dir = "data/OSBSC/"; + + std::fs::create_dir_all(dir).unwrap(); + + fetch_url( + "https://github.com/johanley/constellation-lines/raw/master/output/constellation-lines-hip.utf8".to_string(), + "data/OSBSC/constellation-lines-hip.utf8".to_string(), + )?; + } + Ok(()) +} + #[cfg(test)] mod tests { use std::collections::HashMap; @@ -83,6 +99,9 @@ mod tests { #[test] #[ignore] fn test_constellations_1() { + + crate::catalog::osbsc::fetch_osbsc().unwrap(); + let _stars = parse_catalog!( OSBSCStar, Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8"), @@ -124,6 +143,9 @@ mod tests { #[test] #[ignore] fn test_constellations_2() { + + crate::catalog::osbsc::fetch_osbsc().unwrap(); + let _stars = parse_catalog!( OSBSCStar, Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8"), @@ -138,6 +160,8 @@ mod tests { _star_map.insert(star.Hipparcos_id.unwrap(), star); } + fetch_constellations().unwrap(); + let constells = parse_constellation_catalog!( Path::new("data/OSBSC/constellation-lines-hip.utf8"), _star_map diff --git a/src/lib.rs b/src/lib.rs index b02d394..7eed83f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,6 @@ A simple star catalog parser and primitive types for star coordinates and astronomical times. > Note: currently supports the [Yale Bright Stars Catalog](http://tdc-www.harvard.edu/catalogs/bsc5.html), the [Hipparcos Catalog](https://heasarc.gsfc.nasa.gov/W3Browse/all/hipparcos.html), and the [Open Source Bright Star Catalog](https://github.com/johanley/star-catalog). - Run the `get_data.sh` script to fetch the catalogs. */ #[macro_use] @@ -18,3 +17,4 @@ pub mod constellation; pub mod coord; pub mod star; pub mod time; +pub mod utils; diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..b6d7e67 --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,37 @@ +/*! + Utility functions + */ + +use flate2::read::GzDecoder; + +/** + Fetch function result. + Allows a dynamic error type so we don't have to explicitly coerce upstream errors to the local one. + */ +pub type FetchResult = std::result::Result>; + +/** Fetch a file from url. + Adapted from . +*/ +pub fn fetch_url(url: String, file_name: String) -> FetchResult<()> { + let response = reqwest::blocking::get(url)?; + let mut file = std::fs::File::create(file_name)?; + let mut content = std::io::Cursor::new(response.bytes()?); + std::io::copy(&mut content, &mut file)?; + Ok(()) +} + +/** Fetch a gz file from url and decompress it. + Adapted from . +*/ +pub fn fetch_gz_url(url: String, file_name: String) -> FetchResult<()> { + let response = reqwest::blocking::get(url)?; + // let resp_text = response.text()?; + let mut d = GzDecoder::new(response); + let mut file = std::fs::File::create(file_name)?; + // let mut content = std::io::Cursor::new(d.bytes()); + std::io::copy(&mut d, &mut file)?; + Ok(()) +} + +// TODO: add unit tests for fetching functions. From 194846307e2672c5cae60a50be75d9b7e7a997cd Mon Sep 17 00:00:00 2001 From: Matt Anderson Date: Fri, 23 Dec 2022 19:17:01 -0600 Subject: [PATCH 2/4] fix: formatting --- src/utils.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index b6d7e67..0f5f064 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,13 +1,13 @@ /*! - Utility functions - */ +Utility functions +*/ use flate2::read::GzDecoder; /** - Fetch function result. - Allows a dynamic error type so we don't have to explicitly coerce upstream errors to the local one. - */ +Fetch function result. +Allows a dynamic error type so we don't have to explicitly coerce upstream errors to the local one. +*/ pub type FetchResult = std::result::Result>; /** Fetch a file from url. From 371bc3184a27c34870c4f4cb2f1f94bac0a5b915 Mon Sep 17 00:00:00 2001 From: Matt Anderson Date: Fri, 23 Dec 2022 19:28:11 -0600 Subject: [PATCH 3/4] fix: formatting --- src/catalog/osbsc.rs | 5 ++--- src/catalog/yale.rs | 5 ++++- src/constellation.rs | 11 +++++------ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/catalog/osbsc.rs b/src/catalog/osbsc.rs index c680de9..fcf6b8b 100644 --- a/src/catalog/osbsc.rs +++ b/src/catalog/osbsc.rs @@ -4,7 +4,7 @@ use super::ValidParse; use crate::angle::{Dms, Hms, Sign}; use crate::parse_trim; -use crate::utils::{FetchResult, fetch_url}; +use crate::utils::{fetch_url, FetchResult}; /// Parse an arc/minute/second field. macro_rules! parse_ams { @@ -239,8 +239,7 @@ impl ValidParse for OSBSCStar { } pub fn fetch_osbsc() -> FetchResult<()> { - if !std::path::Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8").exists() - { + if !std::path::Path::new("data/OSBSC/os-bright-star-catalog-hip.utf8").exists() { let dir = "data/OSBSC/"; std::fs::create_dir_all(dir).unwrap(); diff --git a/src/catalog/yale.rs b/src/catalog/yale.rs index 5d0b987..62fe836 100644 --- a/src/catalog/yale.rs +++ b/src/catalog/yale.rs @@ -157,7 +157,10 @@ Note on n_RadVel: */ use super::ValidParse; -use crate::{parse_trim, utils::{FetchResult, fetch_gz_url, fetch_url}}; +use crate::{ + parse_trim, + utils::{fetch_gz_url, fetch_url, FetchResult}, +}; #[allow(non_snake_case)] // Copying field names from original data source #[derive(Debug, Clone)] diff --git a/src/constellation.rs b/src/constellation.rs index d8bc53a..3cd8108 100644 --- a/src/constellation.rs +++ b/src/constellation.rs @@ -4,7 +4,10 @@ WIP Open Source Constellation Catalog Parser > NOTE: still under construction! */ -use crate::{catalog::osbsc::OSBSCStar, utils::{FetchResult, fetch_url}}; +use crate::{ + catalog::osbsc::OSBSCStar, + utils::{fetch_url, FetchResult}, +}; /// Polyline #[allow(dead_code)] // FIXME @@ -73,9 +76,7 @@ macro_rules! parse_constellation_catalog { } pub fn fetch_constellations() -> FetchResult<()> { - if - !std::path::Path::new("data/OSBSC/constellation-lines-hip.utf8").exists() - { + if !std::path::Path::new("data/OSBSC/constellation-lines-hip.utf8").exists() { let dir = "data/OSBSC/"; std::fs::create_dir_all(dir).unwrap(); @@ -99,7 +100,6 @@ mod tests { #[test] #[ignore] fn test_constellations_1() { - crate::catalog::osbsc::fetch_osbsc().unwrap(); let _stars = parse_catalog!( @@ -143,7 +143,6 @@ mod tests { #[test] #[ignore] fn test_constellations_2() { - crate::catalog::osbsc::fetch_osbsc().unwrap(); let _stars = parse_catalog!( From 2bc5fd9c73af46811e6d9a90c19a5350b6c09408 Mon Sep 17 00:00:00 2001 From: Matt Anderson Date: Fri, 23 Dec 2022 19:33:53 -0600 Subject: [PATCH 4/4] fix: remove unnecessary step in publish --- .github/workflows/publish.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e959d48..ed1d4f6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,8 +11,6 @@ jobs: steps: - uses: actions/checkout@v1 - - name: Get Data - run: ./get_data.sh - uses: icepuma/rust-action@master with: args: cargo fmt -- --check && cargo clippy -- -Dwarnings && cargo test -- --include-ignored && cargo publish --token ${CARGO_REGISTRY_TOKEN}