From e566f893be32cc6154333b7b611acd75825ca002 Mon Sep 17 00:00:00 2001 From: cathaypacific8747 <58929011+cathaypacific8747@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:58:13 +0800 Subject: [PATCH] refactor: versioned data files --- Cargo.lock | 1 + am4-web/Cargo.toml | 19 +++- am4-web/src/components/aircraft.rs | 6 +- am4-web/src/components/airport.rs | 15 ++- am4-web/src/db.rs | 91 ++++++++++++++----- am4-web/src/lib.rs | 1 + am4/data/{aircrafts.bin => aircrafts-v0.bin} | 0 am4/data/{airports.bin => airports-v0.bin} | Bin am4/data/{routes0.bin => demands0-v0.bin} | Bin am4/data/{routes1.bin => demands1-v0.bin} | Bin am4/src/aircraft/db.rs | 1 - am4/src/lib.rs | 34 ++++++- am4/src/route/db.rs | 26 +++++- am4/src/utils.rs | 2 + am4/tests/db.rs | 13 ++- misc/scripts/prepare_data/src/main.rs | 12 +-- 16 files changed, 162 insertions(+), 59 deletions(-) rename am4/data/{aircrafts.bin => aircrafts-v0.bin} (100%) rename am4/data/{airports.bin => airports-v0.bin} (100%) rename am4/data/{routes0.bin => demands0-v0.bin} (100%) rename am4/data/{routes1.bin => demands1-v0.bin} (100%) diff --git a/Cargo.lock b/Cargo.lock index 7520d6a..4220d26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,6 +113,7 @@ dependencies = [ "serde", "thiserror", "wasm-bindgen-futures", + "web-sys", ] [[package]] diff --git a/am4-web/Cargo.toml b/am4-web/Cargo.toml index f86bed9..bd1884c 100644 --- a/am4-web/Cargo.toml +++ b/am4-web/Cargo.toml @@ -5,12 +5,23 @@ edition = "2021" [dependencies] am4 = { path = "../am4" } -thiserror = "1.0.61" -leptos = { version = "0.6.12", features = ["csr"] } -console_error_panic_hook = "0.1.7" +serde = "1.0" +thiserror = "1.0" indexed_db_futures = "0.4.1" -serde = "1.0.203" wasm-bindgen-futures = "0.4.42" +leptos = { version = "0.6.12", features = ["csr"] } +console_error_panic_hook = "0.1.7" + +[dependencies.web-sys] +version = "0.3" +features = [ + "Document", + "DomException", + "HtmlInputElement", + "Response", + "Blob", + "BlobPropertyBag", +] [package.metadata.leptos] lib-profile-release = "wasm-release" diff --git a/am4-web/src/components/aircraft.rs b/am4-web/src/components/aircraft.rs index 2f84aef..97764b5 100644 --- a/am4-web/src/components/aircraft.rs +++ b/am4-web/src/components/aircraft.rs @@ -2,9 +2,9 @@ use crate::db::Database; use am4::aircraft::db::AircraftSearchError; use am4::aircraft::{Aircraft, AircraftType}; use leptos::{wasm_bindgen::JsCast, *}; +use web_sys::HtmlInputElement; #[component] -#[allow(non_snake_case)] pub fn ACSearch() -> impl IntoView { let (search_term, set_search_term) = create_signal("".to_string()); let database = expect_context::>>(); @@ -30,7 +30,7 @@ pub fn ACSearch() -> impl IntoView { placeholder="Search an aircraft..." on:input=move |event| { let target = event.target().unwrap(); - let value = target.unchecked_into::().value(); + let value = target.unchecked_into::().value(); set_search_term.set(value); } /> @@ -41,7 +41,6 @@ pub fn ACSearch() -> impl IntoView { } #[component] -#[allow(non_snake_case)] fn ACErr(e: AircraftSearchError) -> impl IntoView { let database = expect_context::>>(); @@ -79,7 +78,6 @@ fn ACErr(e: AircraftSearchError) -> impl IntoView { } #[component] -#[allow(non_snake_case)] fn Ac(aircraft: Aircraft) -> impl IntoView { let ac_type = move || match aircraft.ac_type { AircraftType::Pax => "Pax", diff --git a/am4-web/src/components/airport.rs b/am4-web/src/components/airport.rs index 89896f0..a8bb015 100644 --- a/am4-web/src/components/airport.rs +++ b/am4-web/src/components/airport.rs @@ -1,10 +1,11 @@ use crate::db::Database; use am4::airport::db::AirportSearchError; use am4::airport::Airport; +use leptos::prelude::*; use leptos::{wasm_bindgen::JsCast, *}; +use web_sys::HtmlInputElement; #[component] -#[allow(non_snake_case)] pub fn APSearch() -> impl IntoView { let (search_term, set_search_term) = create_signal("".to_string()); let database = expect_context::>>(); @@ -16,10 +17,7 @@ pub fn APSearch() -> impl IntoView { .unwrap() .airports .search(s.as_str()) - .map_or_else( - |e| view! { }, - |ap| view! { }, - ) + .map_or_else(|e| view! { }, |ap| view! { }) }) }; @@ -30,7 +28,7 @@ pub fn APSearch() -> impl IntoView { placeholder="Search an airport..." on:input=move |event| { let target = event.target().unwrap(); - let value = target.unchecked_into::().value(); + let value = target.unchecked_into::().value(); set_search_term.set(value); } /> @@ -41,7 +39,6 @@ pub fn APSearch() -> impl IntoView { } #[component] -#[allow(non_snake_case)] fn APErr(e: AirportSearchError) -> impl IntoView { let database = expect_context::>>(); @@ -82,8 +79,8 @@ fn APErr(e: AirportSearchError) -> impl IntoView { } #[component] -#[allow(non_snake_case)] -fn Ap(airport: Airport) -> impl IntoView { +#[allow(clippy::needless_lifetimes)] +fn Ap<'a>(airport: &'a Airport) -> impl IntoView { view! {

diff --git a/am4-web/src/db.rs b/am4-web/src/db.rs index 55472ea..9793bf0 100644 --- a/am4-web/src/db.rs +++ b/am4-web/src/db.rs @@ -2,22 +2,24 @@ use indexed_db_futures::prelude::*; use leptos::{logging::log, wasm_bindgen::{prelude::*, JsValue}, web_sys}; use thiserror::Error; use wasm_bindgen_futures::JsFuture; -use web_sys::{window, Response, js_sys::{Uint8Array}, Blob}; +use web_sys::{window, Response, js_sys::{Uint8Array, Array}, Blob, BlobPropertyBag}; use am4::aircraft::db::Aircrafts; -use am4::airport::db::Airports; -// use am4::route::db::Routes; +use am4::airport::{Airport, db::Airports}; +use am4::route::db::Distances; +use am4::{AC_FILENAME, AP_FILENAME, DIST_FILENAME}; pub struct Idb { database: IdbDatabase, } +// TODO: replace unwrap with proper handling with rkvy parse errors. impl Idb { - /// connect to the database and ensure that `am4help/static`` object store exists + /// connect to the database and ensure that `am4help/data` object store exists pub async fn connect() -> Result { let mut db_req = IdbDatabase::open("am4help")?; db_req.set_on_upgrade_needed(Some(move |evt: &IdbVersionChangeEvent| { - if !evt.db().object_store_names().any(|n| &n == "static") { - evt.db().create_object_store("static")?; + if !evt.db().object_store_names().any(|n| &n == "data") { + evt.db().create_object_store("data")?; } Ok(()) })); @@ -25,23 +27,35 @@ impl Idb { Ok(Self { database: db }) } - /// Load a binary file from the IndexedDb. If the blob is not found*, fetch it from the server - /// and cache it in the IndexedDb. - /// `Response`* -> `Blob`* -> IndexedDb -> `Blob` -> `ArrayBuffer` -> `Vec` - /// https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put - pub async fn fetch(&self, k: &str, url: &str, set_progress: &dyn Fn(LoadDbProgress)) -> Result, GenericError> { + async fn get(&self, k: &str) -> Result, GenericError> { + let tx = self.database.transaction_on_one_with_mode("data", IdbTransactionMode::Readonly)?; + tx.object_store("data")?.get_owned(k)?.await.map_err(|e| e.into()) + } + + async fn write(&self, k: &str, v: &JsValue) -> Result<(), GenericError> { + let tx = self.database.transaction_on_one_with_mode("data", IdbTransactionMode::Readwrite).unwrap(); + tx.object_store("data")?.put_key_val_owned(k, v)?; + Ok(()) + } + + pub async fn clear(&self) -> Result<(), GenericError> { + let tx = self.database.transaction_on_one_with_mode("data", IdbTransactionMode::Readwrite)?; + tx.object_store("data")?.clear()?; + Ok(()) + } + + /// Load a binary file from the IndexedDb. If the blob is not found*, + /// fetch it from the server and cache it in the IndexedDb. + /// not found: `Response`* -> `Blob`* -> IndexedDb -> `Blob` -> `ArrayBuffer` -> `Vec` + pub async fn get_blob(&self, k: &str, url: &str, set_progress: &dyn Fn(LoadDbProgress)) -> Result, GenericError> { set_progress(LoadDbProgress::IDBRead(k.to_string())); - let tx = self.database.transaction_on_one_with_mode("static", IdbTransactionMode::Readonly)?; - let jsb = tx.object_store("static")?.get_owned(k)?.await?; - - let jsb = match jsb { + let jsb = match self.get(k).await? { Some(b) => b, None => { set_progress(LoadDbProgress::Fetching(k.to_string())); let b = fetch_bytes(url).await?; set_progress(LoadDbProgress::IDBWrite(k.to_string())); - let tx = self.database.transaction_on_one_with_mode("static", IdbTransactionMode::Readwrite)?; - tx.object_store("static")?.put_key_val_owned(k, &b)?; + let _ = self.write(k, &b).await; b } }; @@ -49,24 +63,53 @@ impl Idb { Ok(Uint8Array::new(&ab).to_vec()) } - pub async fn clear(&self) -> Result<(), GenericError> { - let tx = self.database.transaction_on_one_with_mode("static", IdbTransactionMode::Readwrite)?; - let store = tx.object_store("static")?; - store.clear()?; - Ok(()) + /// Load the flat distances from the indexeddb. If the blob is not found*, + /// generate it from the slice of airports and cache it in the indexeddb. + /// not found: `&[Airport]`* -> `Distances`* (return this) -> `Blob`* -> IndexedDb + /// found: IndexedDb -> `Blob` -> `ArrayBuffer` -> `Distances` + async fn get_distances(&self, aps: &[Airport], set_progress: &dyn Fn(LoadDbProgress)) -> Result { + let k = DIST_FILENAME; + set_progress(LoadDbProgress::IDBRead(k.to_string())); + match self.get(k).await? { + Some(jsb) => { + set_progress(LoadDbProgress::Parsing(k.to_string())); + let ab = JsFuture::from(jsb.dyn_into::()?.array_buffer()).await?; + let bytes = Uint8Array::new(&ab).to_vec(); + Ok(Distances::from_bytes(&bytes).unwrap()) + }, + None => { + set_progress(LoadDbProgress::Parsing(k.to_string())); + let distances = Distances::from_airports(aps); + let b = distances.to_bytes().unwrap(); + + // https://github.com/rustwasm/wasm-bindgen/issues/1693 + // effectively, this is `new Blob([new Uint8Array(b)], {type: 'application/octet-stream'})` + let ja = Array::new(); + ja.push(&Uint8Array::from(b.as_slice()).buffer()); + let mut opts = BlobPropertyBag::new(); + opts.type_("application/octet-stream"); + let blob = Blob::new_with_u8_array_sequence_and_options(&ja, &opts)?; + let _ = self.write(k, &blob).await; + Ok(distances) + } + } } pub async fn init_db(&self, set_progress: &dyn Fn(LoadDbProgress)) -> Result { - let bytes = self.fetch("airports", "data/airports.bin", set_progress).await?; + let bytes = self.get_blob(AP_FILENAME, format!("data/{}", AP_FILENAME).as_str(), set_progress).await?; set_progress(LoadDbProgress::Parsing("airports".to_string())); let airports = Airports::from_bytes(&bytes).unwrap(); log!("airports: {}", airports.data().len()); - let bytes = self.fetch("aircrafts", "data/aircrafts.bin", set_progress).await?; + let bytes = self.get_blob(AC_FILENAME, format!("data/{}", AC_FILENAME).as_str(), set_progress).await?; set_progress(LoadDbProgress::Parsing("aircrafts".to_string())); let aircrafts = Aircrafts::from_bytes(&bytes).unwrap(); log!("aircrafts: {}", aircrafts.data().len()); + set_progress(LoadDbProgress::Parsing("distances".to_string())); + let distances = self.get_distances(airports.data(), set_progress).await?; + log!("distances: {}", distances.data().len()); + // let distances = Distances::from_airports(&(airports.data())); // let bytes = self.fetch("routes", "data/routes.bin", set_progress).await?; // set_progress(LoadDbProgress::Parsing("routes".to_string())); // let routes = Routes::from_bytes(&bytes).unwrap(); diff --git a/am4-web/src/lib.rs b/am4-web/src/lib.rs index 5b54697..687ae56 100644 --- a/am4-web/src/lib.rs +++ b/am4-web/src/lib.rs @@ -1,3 +1,4 @@ +#[allow(non_snake_case)] mod components; mod db; diff --git a/am4/data/aircrafts.bin b/am4/data/aircrafts-v0.bin similarity index 100% rename from am4/data/aircrafts.bin rename to am4/data/aircrafts-v0.bin diff --git a/am4/data/airports.bin b/am4/data/airports-v0.bin similarity index 100% rename from am4/data/airports.bin rename to am4/data/airports-v0.bin diff --git a/am4/data/routes0.bin b/am4/data/demands0-v0.bin similarity index 100% rename from am4/data/routes0.bin rename to am4/data/demands0-v0.bin diff --git a/am4/data/routes1.bin b/am4/data/demands1-v0.bin similarity index 100% rename from am4/data/routes1.bin rename to am4/data/demands1-v0.bin diff --git a/am4/src/aircraft/db.rs b/am4/src/aircraft/db.rs index 3cbb954..3409a27 100644 --- a/am4/src/aircraft/db.rs +++ b/am4/src/aircraft/db.rs @@ -7,7 +7,6 @@ use rkyv::{self, Deserialize}; use std::collections::BinaryHeap; use std::collections::HashMap; use std::str::FromStr; - use thiserror::Error; #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/am4/src/lib.rs b/am4/src/lib.rs index f78ce39..2b73b6e 100644 --- a/am4/src/lib.rs +++ b/am4/src/lib.rs @@ -1,14 +1,38 @@ -// stage 1 pub mod airport; pub mod utils; -// stage 2 pub mod campaign; pub mod ticket; pub mod user; -// stage 3 pub mod aircraft; -// stage 4 -pub mod route; +pub mod route; // under development + +// to keep track of changes for data files in `../data` +#[macro_export] +macro_rules! ac_version { + () => { + "0" + }; +} +pub const AC_FILENAME: &str = concat!("aircrafts-v", ac_version!(), ".bin"); + +#[macro_export] +macro_rules! ap_version { + () => { + "0" + }; +} +pub const AP_FILENAME: &str = concat!("airports-v", ap_version!(), ".bin"); +// distances are generated from the airports +pub const DIST_FILENAME: &str = concat!("distances-v", ap_version!(), ".bin"); + +#[macro_export] +macro_rules! demand_version { + () => { + "0" + }; +} +pub const DEM_FILENAME0: &str = concat!("demands0-v", demand_version!(), ".bin"); +pub const DEM_FILENAME1: &str = concat!("demands1-v", demand_version!(), ".bin"); diff --git a/am4/src/route/db.rs b/am4/src/route/db.rs index 56155dd..e7add97 100644 --- a/am4/src/route/db.rs +++ b/am4/src/route/db.rs @@ -1,7 +1,7 @@ use crate::airport::{db::AIRPORT_COUNT, Airport}; use crate::route::{demand::pax::PaxDemand, leg::calculate_distance}; use crate::utils::ParseError; -use rkyv::{self, Deserialize}; +use rkyv::{self, AlignedVec, Deserialize}; pub const ROUTE_COUNT: usize = AIRPORT_COUNT * (AIRPORT_COUNT - 1) / 2; @@ -37,6 +37,24 @@ impl Demands { pub struct Distances(Vec); impl Distances { + pub fn from_bytes(buffer: &[u8]) -> Result { + let archived = rkyv::check_archived_root::>(buffer) + .map_err(|e| ParseError::ArchiveError(e.to_string()))?; + + let distances: Vec = archived + .deserialize(&mut rkyv::Infallible) + .map_err(|e| ParseError::DeserialiseError(e.to_string()))?; + + if distances.len() != ROUTE_COUNT { + return Err(ParseError::InvalidDataLength { + expected: ROUTE_COUNT, + actual: distances.len(), + }); + } + + Ok(Distances(distances)) + } + pub fn from_airports(aps: &[Airport]) -> Self { assert!(aps.len() == AIRPORT_COUNT); // compiler optimisation let mut d = Vec::::with_capacity(ROUTE_COUNT); @@ -54,6 +72,12 @@ impl Distances { Distances(d) } + pub fn to_bytes(&self) -> Result { + let av = rkyv::to_bytes::, 30_521_492>(&self.0) + .map_err(|e| ParseError::SerialiseError(e.to_string()))?; + Ok(av) + } + pub fn data(&self) -> &Vec { &self.0 } diff --git a/am4/src/utils.rs b/am4/src/utils.rs index bd77d36..779d7b7 100644 --- a/am4/src/utils.rs +++ b/am4/src/utils.rs @@ -44,6 +44,8 @@ pub enum ParseError { ArchiveError(String), #[error("Deserialise error: {0}")] DeserialiseError(String), + #[error("Serialise error: {0}")] + SerialiseError(String), #[error("Invalid data length: expected {expected} routes, got {actual}")] InvalidDataLength { expected: usize, actual: usize }, } diff --git a/am4/tests/db.rs b/am4/tests/db.rs index b136fc9..6b4799a 100644 --- a/am4/tests/db.rs +++ b/am4/tests/db.rs @@ -1,12 +1,14 @@ use am4::aircraft::db::Aircrafts; use am4::airport::db::Airports; use am4::route::db::{Demands, Distances}; +use am4::{AC_FILENAME, AP_FILENAME, DEM_FILENAME0, DEM_FILENAME1}; use once_cell::sync::Lazy; use std::fs::File; use std::io::Read; pub fn get_bytes(path: &str) -> Result, std::io::Error> { - let mut file = File::open(path)?; + let fp = "./data".to_string() + path; + let mut file = File::open(fp)?; let mut buffer = Vec::::new(); file.read_to_end(&mut buffer)?; Ok(buffer) @@ -14,18 +16,19 @@ pub fn get_bytes(path: &str) -> Result, std::io::Error> { #[allow(dead_code)] pub static AIRCRAFTS: Lazy = - Lazy::new(|| Aircrafts::from_bytes(&get_bytes("./data/aircrafts.bin").unwrap()).unwrap()); + Lazy::new(|| Aircrafts::from_bytes(&get_bytes(AC_FILENAME).unwrap()).unwrap()); #[allow(dead_code)] pub static AIRPORTS: Lazy = - Lazy::new(|| Airports::from_bytes(&get_bytes("./data/airports.bin").unwrap()).unwrap()); + Lazy::new(|| Airports::from_bytes(&get_bytes(AP_FILENAME).unwrap()).unwrap()); #[allow(dead_code)] pub static ROUTES: Lazy = Lazy::new(|| { - let mut buf = get_bytes("./data/routes0.bin").unwrap(); - let b1 = get_bytes("./data/routes1.bin").unwrap(); + let mut buf = get_bytes(DEM_FILENAME0).unwrap(); + let b1 = get_bytes(DEM_FILENAME1).unwrap(); buf.extend(b1); Demands::from_bytes(&buf).unwrap() }); +#[allow(dead_code)] pub static DISTANCES: Lazy = Lazy::new(|| Distances::from_airports(&(*AIRPORTS.data()))); diff --git a/misc/scripts/prepare_data/src/main.rs b/misc/scripts/prepare_data/src/main.rs index b908f3b..3319232 100644 --- a/misc/scripts/prepare_data/src/main.rs +++ b/misc/scripts/prepare_data/src/main.rs @@ -1,5 +1,6 @@ mod utils; +use am4::{AC_FILENAME, AP_FILENAME, DEM_FILENAME0, DEM_FILENAME1}; use polars::frame::row::Row; use polars::prelude::*; use std::io::Write; @@ -60,18 +61,17 @@ fn convert_routes() { let buf = rkyv::to_bytes::, 45_782_236>(&dems).unwrap(); let spl = buf.len() / 2; - let mut file0 = std::fs::File::create("routes0.bin").unwrap(); + let mut file0 = std::fs::File::create(DEM_FILENAME0).unwrap(); file0.write_all(&buf[..spl]).unwrap(); println!("wrote ..{:} to {:?}", spl, file0); - let mut file1 = std::fs::File::create("routes1.bin").unwrap(); + let mut file1 = std::fs::File::create(DEM_FILENAME1).unwrap(); file1.write_all(&buf[spl..]).unwrap(); println!("wrote {:}.. to {:?}", spl, file1); } fn convert_airports() { - use am4::airport::Point; - use am4::airport::{Airport, Iata, Icao, Id, Name}; + use am4::airport::{Airport, Iata, Icao, Id, Name, Point}; let mut schema = Schema::new(); schema.with_column("id".into(), DataType::UInt16); @@ -130,7 +130,7 @@ fn convert_airports() { }); } println!("{:?}", airports.len()); - let mut file = std::fs::File::create("airports.bin").unwrap(); + let mut file = std::fs::File::create(AP_FILENAME).unwrap(); let b = rkyv::to_bytes::, 502684>(&airports).unwrap(); file.write_all(&b).unwrap(); @@ -213,7 +213,7 @@ fn convert_aircrafts() { length: get_u8(r[24].clone()), }); } - let mut file = std::fs::File::create("aircrafts.bin").unwrap(); + let mut file = std::fs::File::create(AC_FILENAME).unwrap(); let b = rkyv::to_bytes::, 68200>(&aircrafts).unwrap(); file.write_all(&b).unwrap();