From fdfc49ede38d598301bb952f0795da05aa093e54 Mon Sep 17 00:00:00 2001 From: Jan Marvin Garbuszus Date: Thu, 3 Jul 2025 00:55:40 +0200 Subject: [PATCH 1/3] revive pdfetch_ONS() --- R/pdfetch.R | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/R/pdfetch.R b/R/pdfetch.R index 8952510..efc8b92 100644 --- a/R/pdfetch.R +++ b/R/pdfetch.R @@ -460,20 +460,48 @@ pdfetch_INSEE <- function(identifiers) { #' pdfetch_ONS(c("LF24","LF2G"), "lms") #' } pdfetch_ONS <- function(identifiers, dataset) { - identifiers <- tolower(identifiers) + identifiers <- toupper(identifiers) results <- list() - for (id in identifiers) { - url <- paste0("https://api.beta.ons.gov.uk/v1/datasets/",dataset,"/editions/time-series/versions/1/observations?time=*&aggregate=",id) + ons_api <- "https://api.beta.ons.gov.uk/v1" + search_api <- paste0( + ons_api, "/search?q=", paste0(identifiers, collapse = "+") + ) + + if (!missing(dataset)) { + search_api <- paste0(search_api, "&dataset_ids=", toupper(dataset)) + } + + raw <- httr::content(httr::GET(search_api)) + + res <- lapply(raw$items, function(x) { + c( + title = x$title, + uri = x$uri, + cdid = x$cdid + ) + }) + res <- as.data.frame(do.call("rbind", res)) + + # output order might differ from input + res <- res[match(identifiers, res$cdid), ] + + for (id in seq_len(nrow(res))) { + # url <- paste0("https://www.ons.gov.uk/generator?format=csv&uri=", res$uri[id]) + # csv <- tempfile(fileext = ".csv") + # download.file(url, csv) + # raw <- read.csv(csv) + + url <- paste0("https://api.beta.ons.gov.uk/v1/data?uri=", res$uri[id]) + raw <- httr::content(httr::GET(url)) - raw <- content(GET(url)) if (is.null(raw)) { warning(paste0('Series ', id, ' in dataset ', dataset, ' not found.')) next } - print(raw) - + # message(res$cdid[id], ": ", res$title[id]) + if (length(raw$months) > 0) { month <- sapply(raw$months, function(x) match(x$month, c("January","February","March","April","May","June","July","August","September","October","November","December"))) year <- sapply(raw$months, function(x) as.numeric(x$year)) @@ -487,16 +515,16 @@ pdfetch_ONS <- function(identifiers, dataset) { year <- sapply(raw$years, function(x) as.numeric(x$year)) values <- sapply(raw$years, function(x) as.numeric(x$value)) } - + dates <- month_end(as.Date(ISOdate(year, month, 1))) x <- xts(values, dates) - colnames(x) <- toupper(id) + colnames(x) <- toupper(res$cdid[id]) results[[id]] <- x } if (length(results) == 0) return(NULL) - + na.trim(do.call(merge.xts, results), is.na="all") } From d20176a05f970ea762d61a0d48673a94c9708701 Mon Sep 17 00:00:00 2001 From: Jan Marvin Garbuszus Date: Thu, 3 Jul 2025 07:37:49 +0200 Subject: [PATCH 2/3] cleanup --- R/pdfetch.R | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/R/pdfetch.R b/R/pdfetch.R index efc8b92..4f71e55 100644 --- a/R/pdfetch.R +++ b/R/pdfetch.R @@ -460,9 +460,10 @@ pdfetch_INSEE <- function(identifiers) { #' pdfetch_ONS(c("LF24","LF2G"), "lms") #' } pdfetch_ONS <- function(identifiers, dataset) { + orig_identifiers <- identifiers identifiers <- toupper(identifiers) - results <- list() + results <- vector("list", length(identifiers)) ons_api <- "https://api.beta.ons.gov.uk/v1" search_api <- paste0( @@ -477,31 +478,37 @@ pdfetch_ONS <- function(identifiers, dataset) { res <- lapply(raw$items, function(x) { c( - title = x$title, + title = x$title, # series label uri = x$uri, cdid = x$cdid ) }) res <- as.data.frame(do.call("rbind", res)) + ## if the input is not found in res, res$count can be 0. But if some inputs + ## are found, only these are returned by the search api + found_identifiers <- match(identifiers, res$cdid) + if (anyNA(found_identifiers) || raw$count < length(identifiers)) { + miss_identifiers <- paste( + orig_identifiers[is.na(found_identifiers)], + collapse = ", " + ) + msg <- paste0('Series ', miss_identifiers) + if (!missing(dataset)) { + msg <- paste0(msg, ' in dataset ', dataset) + } + msg <- paste0(msg, ' not found.') + warning(msg, call. = FALSE) + } + # output order might differ from input - res <- res[match(identifiers, res$cdid), ] + res <- res[found_identifiers[!is.na(found_identifiers)], ] for (id in seq_len(nrow(res))) { - # url <- paste0("https://www.ons.gov.uk/generator?format=csv&uri=", res$uri[id]) - # csv <- tempfile(fileext = ".csv") - # download.file(url, csv) - # raw <- read.csv(csv) - url <- paste0("https://api.beta.ons.gov.uk/v1/data?uri=", res$uri[id]) + url <- paste0(ons_api, "/data?uri=", res$uri[id]) raw <- httr::content(httr::GET(url)) - if (is.null(raw)) { - warning(paste0('Series ', id, ' in dataset ', dataset, ' not found.')) - next - } - # message(res$cdid[id], ": ", res$title[id]) - if (length(raw$months) > 0) { month <- sapply(raw$months, function(x) match(x$month, c("January","February","March","April","May","June","July","August","September","October","November","December"))) year <- sapply(raw$months, function(x) as.numeric(x$year)) From b0416a933a2c999819597cd1c9402fd29ed0bde2 Mon Sep 17 00:00:00 2001 From: Jan Marvin Garbuszus Date: Thu, 3 Jul 2025 10:54:10 +0200 Subject: [PATCH 3/3] update EUROSTAT examples --- R/pdfetch.R | 15 ++++++++------- man/pdfetch_EUROSTAT.Rd | 4 ++-- man/pdfetch_EUROSTAT_DSD.Rd | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/R/pdfetch.R b/R/pdfetch.R index 4f71e55..7cec91d 100644 --- a/R/pdfetch.R +++ b/R/pdfetch.R @@ -142,7 +142,7 @@ pdfetch_EUROSTAT_GETDSD <- function(flowRef) { #' @export #' @examples #' \dontrun{ -#' pdfetch_EUROSTAT_DSD("namq_gdp_c") +#' pdfetch_EUROSTAT_DSD("namq_10_gdp") #' } pdfetch_EUROSTAT_DSD <- function(flowRef) { results <- NULL @@ -183,8 +183,8 @@ pdfetch_EUROSTAT_DSD <- function(flowRef) { #' @export #' @examples #' \dontrun{ -#' pdfetch_EUROSTAT("namq_gdp_c", FREQ="Q", S_ADJ="NSA", UNIT="MIO_EUR", -#' INDIC_NA="B1GM", GEO=c("DE","UK")) +#' pdfetch_EUROSTAT("namq_10_gdp", FREQ="Q", S_ADJ="NSA", UNIT="CP_MEUR", +#' INDIC_NA="B1GM", GEO=c("DE","UK")) #' } pdfetch_EUROSTAT <- function(flowRef, from, to, ...) { arguments <- list(...) @@ -226,12 +226,13 @@ pdfetch_EUROSTAT <- function(flowRef, from, to, ...) { doc <- doc %>% select(c("freq","ID_COLUMN","TIME_PERIOD","OBS_VALUE")) %>% tidyr::pivot_wider(names_from="ID_COLUMN", values_from="OBS_VALUE") %>% + mutate(TIME_PERIOD = as.character(.data$TIME_PERIOD)) %>% mutate( date=case_when( - freq == 'A' ~ as.Date(ISOdate(TIME_PERIOD,12,31)), - freq == 'Q' ~ quarter_end(as.Date(lubridate::parse_date_time2(TIME_PERIOD, orders="Y-q"))), - freq == 'M' ~ month_end(as.Date(lubridate::parse_date_time2(TIME_PERIOD, orders="Y-m"))), - TRUE ~ as.Date(TIME_PERIOD, format="%Y-%m-%d") + freq == 'A' ~ as.Date(ISOdate(.data$TIME_PERIOD,12,31)), + freq == 'Q' ~ quarter_end(as.Date(lubridate::parse_date_time2(.data$TIME_PERIOD, orders="Y-q"))), + freq == 'M' ~ month_end(as.Date(lubridate::parse_date_time2(.data$TIME_PERIOD, orders="Y-m"))), + TRUE ~ as.Date(.data$TIME_PERIOD, format="%Y-%m-%d") ) ) diff --git a/man/pdfetch_EUROSTAT.Rd b/man/pdfetch_EUROSTAT.Rd index b7fc6d2..21935f2 100644 --- a/man/pdfetch_EUROSTAT.Rd +++ b/man/pdfetch_EUROSTAT.Rd @@ -24,7 +24,7 @@ Eurostat stores its statistics in data cubes, which can be browsed at } \examples{ \dontrun{ -pdfetch_EUROSTAT("namq_gdp_c", FREQ="Q", S_ADJ="NSA", UNIT="MIO_EUR", - INDIC_NA="B1GM", GEO=c("DE","UK")) +pdfetch_EUROSTAT("namq_10_gdp", FREQ="Q", S_ADJ="NSA", UNIT="CP_MEUR", + INDIC_NA="B1GM", GEO=c("DE","UK")) } } diff --git a/man/pdfetch_EUROSTAT_DSD.Rd b/man/pdfetch_EUROSTAT_DSD.Rd index 5521071..d09f59f 100644 --- a/man/pdfetch_EUROSTAT_DSD.Rd +++ b/man/pdfetch_EUROSTAT_DSD.Rd @@ -14,6 +14,6 @@ Fetch description for a Eurostat dataset } \examples{ \dontrun{ -pdfetch_EUROSTAT_DSD("namq_gdp_c") +pdfetch_EUROSTAT_DSD("namq_10_gdp") } }