Merge pull request #19 from scarnecchia/DEV-13

DEV-13: Update tool to accomadate Oryx site changes.

R/scrape_data.R

@@ -3,20 +3,20 @@
 #'
 #' @return a tibble
 #' @export
-scrape_data <- function() {
+scrape_data <- function(country) {
+  if (country == "Russia") {
+    url <-
+      russia_url
+  } else {
+    url <-
+      ukraine_url
+  }
+
   materiel <-
-    get_data(
-      "https://www.oryxspioenkop.com/2022/02/attack-on-europe-documenting-equipment.html",
-      "article"
-    ) %>%
+    get_data(url,
+             "article") %>%
     rvest::html_elements("li")
 
-  # Retreive the start position of each country
-  country_pos <- materiel %>% rvest::html_text2() %>%
-    # T-64BV is the first row in the tank list and marks the beginning of each country
-    stringr::str_which("T-64BV")
-
-  #' Run Program
   data <-
     tibble::tibble(
       country = character(),
@@ -31,8 +31,7 @@ scrape_data <- function() {
     status <- materiel[[a]] %>% rvest::html_elements("a")
     for (b in seq_along(status)) {
       counter = counter + 1
-      data[counter, 1] <-
-        ifelse(a < country_pos[2], "Russia", "Ukraine")
+      data[counter, 1] <- country
       data[counter, 2] <- extract_origin(materiel, a)
       data[counter, 3] <- extract_system(materiel, a)
       data[counter, 4] <- extract_status(status, b)
@@ -45,6 +44,15 @@ scrape_data <- function() {
     tidyr::unnest_longer(status) %>%
     dplyr::mutate(date_recorded = as.Date(lubridate::today())) %>%
     trim_all()
+}
+
+create_data <- function() {
+  russia <- scrape_data("Russia")
+  ukraine <- scrape_data("Ukraine")
+
+  data <- russia %>%
+    dplyr::bind_rows(ukraine) %>%
+    dplyr::select(country, origin, system, status, url, date_recorded)
 
   previous <- get_inputfile("totals_by_system") %>%
     trim_all() %>%
@@ -61,12 +69,14 @@ scrape_data <- function() {
       )) %>%
       dplyr::arrange(country, system, date_recorded)
 
-  data <- check %>% dplyr::bind_rows(get_inputfile("totals_by_system")) %>%
-    dplyr::arrange(country, system, date_recorded)
+    data <- check %>% dplyr::bind_rows(previous, .id = NULL) %>%
+      dplyr::arrange(country, system, date_recorded)
 
-  previous %>% readr::write_csv("inputfiles/totals_by_system.csv.bak")
+    previous %>% readr::write_csv("inputfiles/totals_by_system.csv.bak")
 
-  data %>% readr::write_csv(glue::glue("inputfiles/totals_by_system{lubridate::today()+1}.csv"))
+    data %>% readr::write_csv(glue::glue(
+      "inputfiles/totals_by_system{lubridate::today()+1}.csv"
+    ))
 
   } else {
     logr::put("No new data")

R/totals_by_type.R

@@ -2,10 +2,18 @@
 #' @description Gets data by system category.
 #'
 #' @return a tibble
-totals_by_type <- function() {
+create_by_type <- function(country) {
+  if (country == "Russia") {
+    url <-
+      russia_url
+  } else {
+    url <-
+      ukraine_url
+  }
+
   heads <-
     get_data(
-      "https://www.oryxspioenkop.com/2022/02/attack-on-europe-documenting-equipment.html",
+      url,
       "article div"
     ) %>%
     rvest::html_elements("h3") %>%
@@ -15,8 +23,7 @@ totals_by_type <- function() {
   heads <- heads[nchar(heads) > 0]
 
   # Get the positons of the Russia and Ukraine headers
-  rus_pos <- heads %>% stringr::str_which("Russia") %>% as.double()
-  ukr_pos <- heads %>% stringr::str_which("Ukraine") %>% as.double()
+  pos <- heads %>% stringr::str_which(country) %>% as.double()
 
   totals <- tibble(
     country = character(),
@@ -45,20 +52,31 @@ totals_by_type <- function() {
   }
 
 
-  totals_df <- totals %>%
+  country_df <- totals %>%
     dplyr::mutate(
       dplyr::across(destroyed:damaged, ~ as.double(tidyr::replace_na(.x, "0"))),
       type_total = destroyed + abandoned + captured + damaged,
-      row_id = 1:n(),
-      country = dplyr::case_when(row_id < ukr_pos ~ "Russia",
-                                 row_id >= ukr_pos ~ "Ukraine")
+      row_id = 1:n()
     ) %>%
+    dplyr::mutate(country = tidyr::replace_na(country, !!!country)) %>%
     select(-row_id) %>%
     dplyr::mutate(
-      equipment = replace(equipment, rus_pos, "All Types"),
-      equipment = replace(equipment, ukr_pos, "All Types")
+      equipment = replace(equipment, pos, "All Types"),
     ) %>%
     dplyr::rename(equipment_type = equipment)
 
+  return(country_df)
+}
+
+totals_by_type <- function() {
+  russia <- create_by_type("Russia")
+  ukraine <- create_by_type("Ukraine")
+
+  totals_df <- russia %>%
+    dplyr::bind_rows(ukraine, .id=NULL)
+
   return(totals_df)
 }
+
+
+

scrape_oryx.R

@@ -35,13 +35,16 @@ source("R/totals_by_type.R")
 source("R/per_event.R")
 source("R/daily_count.R")
 
+russia_url <- "https://www.oryxspioenkop.com/2022/02/attack-on-europe-documenting-equipment.html"
+ukraine_url <- "https://www.oryxspioenkop.com/2022/02/attack-on-europe-documenting-ukrainian.html"
+
 tmp <-
   file.path("outputfiles", sprintf("scrape_oryx_%s.log", format(Sys.time(), "%Y%m%dT%H%M%S")))
 lf <- logr::log_open(tmp)
 today <- format(Sys.Date(), "%Y-%m-%d")
 
 
-totals_by_system <- scrape_data() %>%
+totals_by_system <- create_data() %>%
   readr::write_csv(., file = glue::glue("outputfiles/totals_by_system.csv"))
 
 #' Write Event Tables