refactor: versioned data files

am4-web/Cargo.toml

@@ -5,12 +5,23 @@ edition = "2021"
 
 [dependencies]
 am4 = { path = "../am4" }
-thiserror = "1.0.61"
-leptos = { version = "0.6.12", features = ["csr"] }
-console_error_panic_hook = "0.1.7"
+serde = "1.0"
+thiserror = "1.0"
 indexed_db_futures = "0.4.1"
-serde = "1.0.203"
 wasm-bindgen-futures = "0.4.42"
+leptos = { version = "0.6.12", features = ["csr"] }
+console_error_panic_hook = "0.1.7"
+
+[dependencies.web-sys]
+version = "0.3"
+features = [
+    "Document",
+    "DomException",
+    "HtmlInputElement",
+    "Response",
+    "Blob",
+    "BlobPropertyBag",
+]
 
 [package.metadata.leptos]
 lib-profile-release = "wasm-release"

am4-web/src/components/aircraft.rs

@@ -2,9 +2,9 @@ use crate::db::Database;
 use am4::aircraft::db::AircraftSearchError;
 use am4::aircraft::{Aircraft, AircraftType};
 use leptos::{wasm_bindgen::JsCast, *};
+use web_sys::HtmlInputElement;
 
 #[component]
-#[allow(non_snake_case)]
 pub fn ACSearch() -> impl IntoView {
     let (search_term, set_search_term) = create_signal("".to_string());
     let database = expect_context::<StoredValue<Option<Database>>>();
@@ -30,7 +30,7 @@ pub fn ACSearch() -> impl IntoView {
                 placeholder="Search an aircraft..."
                 on:input=move |event| {
                     let target = event.target().unwrap();
-                    let value = target.unchecked_into::<web_sys::HtmlInputElement>().value();
+                    let value = target.unchecked_into::<HtmlInputElement>().value();
                     set_search_term.set(value);
                 }
             />
@@ -41,7 +41,6 @@ pub fn ACSearch() -> impl IntoView {
 }
 
 #[component]
-#[allow(non_snake_case)]
 fn ACErr(e: AircraftSearchError) -> impl IntoView {
     let database = expect_context::<StoredValue<Option<Database>>>();
 
@@ -79,7 +78,6 @@ fn ACErr(e: AircraftSearchError) -> impl IntoView {
 }
 
 #[component]
-#[allow(non_snake_case)]
 fn Ac(aircraft: Aircraft) -> impl IntoView {
     let ac_type = move || match aircraft.ac_type {
         AircraftType::Pax => "Pax",

am4-web/src/components/airport.rs

@@ -1,10 +1,11 @@
 use crate::db::Database;
 use am4::airport::db::AirportSearchError;
 use am4::airport::Airport;
+use leptos::prelude::*;
 use leptos::{wasm_bindgen::JsCast, *};
+use web_sys::HtmlInputElement;
 
 #[component]
-#[allow(non_snake_case)]
 pub fn APSearch() -> impl IntoView {
     let (search_term, set_search_term) = create_signal("".to_string());
     let database = expect_context::<StoredValue<Option<Database>>>();
@@ -16,10 +17,7 @@ pub fn APSearch() -> impl IntoView {
                 .unwrap()
                 .airports
                 .search(s.as_str())
-                .map_or_else(
-                    |e| view! { <APErr e/> },
-                    |ap| view! { <Ap airport=ap.clone()/> },
-                )
+                .map_or_else(|e| view! { <APErr e/> }, |ap| view! { <Ap airport=ap/> })
         })
     };
 
@@ -30,7 +28,7 @@ pub fn APSearch() -> impl IntoView {
                 placeholder="Search an airport..."
                 on:input=move |event| {
                     let target = event.target().unwrap();
-                    let value = target.unchecked_into::<web_sys::HtmlInputElement>().value();
+                    let value = target.unchecked_into::<HtmlInputElement>().value();
                     set_search_term.set(value);
                 }
             />
@@ -41,7 +39,6 @@ pub fn APSearch() -> impl IntoView {
 }
 
 #[component]
-#[allow(non_snake_case)]
 fn APErr(e: AirportSearchError) -> impl IntoView {
     let database = expect_context::<StoredValue<Option<Database>>>();
 
@@ -82,8 +79,8 @@ fn APErr(e: AirportSearchError) -> impl IntoView {
 }
 
 #[component]
-#[allow(non_snake_case)]
-fn Ap(airport: Airport) -> impl IntoView {
+#[allow(clippy::needless_lifetimes)]
+fn Ap<'a>(airport: &'a Airport) -> impl IntoView {
     view! {
         <div class="ap-card">
             <h3>

am4-web/src/db.rs

@@ -2,71 +2,114 @@ use indexed_db_futures::prelude::*;
 use leptos::{logging::log, wasm_bindgen::{prelude::*, JsValue}, web_sys};
 use thiserror::Error;
 use wasm_bindgen_futures::JsFuture;
-use web_sys::{window, Response, js_sys::{Uint8Array}, Blob};
+use web_sys::{window, Response, js_sys::{Uint8Array, Array}, Blob, BlobPropertyBag};
 use am4::aircraft::db::Aircrafts;
-use am4::airport::db::Airports;
-// use am4::route::db::Routes;
+use am4::airport::{Airport, db::Airports};
+use am4::route::db::Distances;
+use am4::{AC_FILENAME, AP_FILENAME, DIST_FILENAME};
 
 pub struct Idb {
     database: IdbDatabase,
 }
 
+// TODO: replace unwrap with proper handling with rkvy parse errors.
 impl Idb {
-    /// connect to the database and ensure that `am4help/static`` object store exists
+    /// connect to the database and ensure that `am4help/data` object store exists
     pub async fn connect() -> Result<Self, GenericError> {
         let mut db_req = IdbDatabase::open("am4help")?;
         db_req.set_on_upgrade_needed(Some(move |evt: &IdbVersionChangeEvent| {
-            if !evt.db().object_store_names().any(|n| &n == "static") {
-                evt.db().create_object_store("static")?;
+            if !evt.db().object_store_names().any(|n| &n == "data") {
+                evt.db().create_object_store("data")?;
             }
             Ok(())
         }));
         let db = db_req.await?;
         Ok(Self { database: db })
     }
 
-    /// Load a binary file from the IndexedDb. If the blob is not found*, fetch it from the server
-    /// and cache it in the IndexedDb.
-    /// `Response`* -> `Blob`* -> IndexedDb -> `Blob` -> `ArrayBuffer` -> `Vec<u8>`
-    /// https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put
-    pub async fn fetch(&self, k: &str, url: &str, set_progress: &dyn Fn(LoadDbProgress)) -> Result<Vec<u8>, GenericError> {
+    async fn get(&self, k: &str) -> Result<Option<JsValue>, GenericError> {
+        let tx = self.database.transaction_on_one_with_mode("data", IdbTransactionMode::Readonly)?;
+        tx.object_store("data")?.get_owned(k)?.await.map_err(|e| e.into())
+    }
+
+    async fn write(&self, k: &str, v: &JsValue) -> Result<(), GenericError> {
+        let tx = self.database.transaction_on_one_with_mode("data", IdbTransactionMode::Readwrite).unwrap();
+        tx.object_store("data")?.put_key_val_owned(k, v)?;
+        Ok(())
+    }
+
+    pub async fn clear(&self) -> Result<(), GenericError> {
+        let tx = self.database.transaction_on_one_with_mode("data", IdbTransactionMode::Readwrite)?;
+        tx.object_store("data")?.clear()?;
+        Ok(())
+    }
+
+    /// Load a binary file from the IndexedDb. If the blob is not found*, 
+    /// fetch it from the server and cache it in the IndexedDb.
+    /// not found: `Response`* -> `Blob`* -> IndexedDb -> `Blob` -> `ArrayBuffer` -> `Vec<u8>`
+    pub async fn get_blob(&self, k: &str, url: &str, set_progress: &dyn Fn(LoadDbProgress)) -> Result<Vec<u8>, GenericError> {
         set_progress(LoadDbProgress::IDBRead(k.to_string()));
-        let tx = self.database.transaction_on_one_with_mode("static", IdbTransactionMode::Readonly)?;
-        let jsb = tx.object_store("static")?.get_owned(k)?.await?;
-
-        let jsb = match jsb {
+        let jsb = match self.get(k).await? {
             Some(b) => b,
             None => {
                 set_progress(LoadDbProgress::Fetching(k.to_string()));
                 let b = fetch_bytes(url).await?;
                 set_progress(LoadDbProgress::IDBWrite(k.to_string()));
-                let tx = self.database.transaction_on_one_with_mode("static", IdbTransactionMode::Readwrite)?;
-                tx.object_store("static")?.put_key_val_owned(k, &b)?;
+                let _ = self.write(k, &b).await;
                 b
             }
         };
         let ab = JsFuture::from(jsb.dyn_into::<Blob>()?.array_buffer()).await?;
         Ok(Uint8Array::new(&ab).to_vec())
     }
 
-    pub async fn clear(&self) -> Result<(), GenericError> {
-        let tx = self.database.transaction_on_one_with_mode("static", IdbTransactionMode::Readwrite)?;
-        let store = tx.object_store("static")?;
-        store.clear()?;
-        Ok(())
+    /// Load the flat distances from the indexeddb. If the blob is not found*,
+    /// generate it from the slice of airports and cache it in the indexeddb.
+    /// not found: `&[Airport]`* -> `Distances`* (return this) -> `Blob`* -> IndexedDb
+    /// found: IndexedDb -> `Blob` -> `ArrayBuffer` -> `Distances`
+    async fn get_distances(&self, aps: &[Airport], set_progress: &dyn Fn(LoadDbProgress)) -> Result<Distances, GenericError> {
+        let k = DIST_FILENAME;
+        set_progress(LoadDbProgress::IDBRead(k.to_string()));
+        match self.get(k).await? {
+            Some(jsb) => {
+                set_progress(LoadDbProgress::Parsing(k.to_string()));
+                let ab = JsFuture::from(jsb.dyn_into::<Blob>()?.array_buffer()).await?;
+                let bytes = Uint8Array::new(&ab).to_vec();
+                Ok(Distances::from_bytes(&bytes).unwrap())
+            },
+            None => {
+                set_progress(LoadDbProgress::Parsing(k.to_string()));
+                let distances = Distances::from_airports(aps);
+                let b = distances.to_bytes().unwrap();
+
+                // https://github.com/rustwasm/wasm-bindgen/issues/1693
+                // effectively, this is `new Blob([new Uint8Array(b)], {type: 'application/octet-stream'})`
+                let ja = Array::new();
+                ja.push(&Uint8Array::from(b.as_slice()).buffer());
+                let mut opts = BlobPropertyBag::new();
+                opts.type_("application/octet-stream");
+                let blob = Blob::new_with_u8_array_sequence_and_options(&ja, &opts)?;
+                let _ = self.write(k, &blob).await;
+                Ok(distances)
+            }
+        }
     }
 
     pub async fn init_db(&self, set_progress: &dyn Fn(LoadDbProgress)) -> Result<Database, GenericError> {
-        let bytes = self.fetch("airports", "data/airports.bin", set_progress).await?;
+        let bytes = self.get_blob(AP_FILENAME, format!("data/{}", AP_FILENAME).as_str(), set_progress).await?;
         set_progress(LoadDbProgress::Parsing("airports".to_string()));
         let airports = Airports::from_bytes(&bytes).unwrap();
         log!("airports: {}", airports.data().len());
 
-        let bytes = self.fetch("aircrafts", "data/aircrafts.bin", set_progress).await?;
+        let bytes = self.get_blob(AC_FILENAME, format!("data/{}", AC_FILENAME).as_str(), set_progress).await?;
         set_progress(LoadDbProgress::Parsing("aircrafts".to_string()));
         let aircrafts = Aircrafts::from_bytes(&bytes).unwrap();
         log!("aircrafts: {}", aircrafts.data().len());
 
+        set_progress(LoadDbProgress::Parsing("distances".to_string()));
+        let distances = self.get_distances(airports.data(), set_progress).await?;
+        log!("distances: {}", distances.data().len());
+        // let distances = Distances::from_airports(&(airports.data()));
         // let bytes = self.fetch("routes", "data/routes.bin", set_progress).await?;
         // set_progress(LoadDbProgress::Parsing("routes".to_string()));
         // let routes = Routes::from_bytes(&bytes).unwrap();

am4-web/src/lib.rs

@@ -1,3 +1,4 @@
+#[allow(non_snake_case)]
 mod components;
 mod db;
 

am4/src/aircraft/db.rs

@@ -7,7 +7,6 @@ use rkyv::{self, Deserialize};
 use std::collections::BinaryHeap;
 use std::collections::HashMap;
 use std::str::FromStr;
-
 use thiserror::Error;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]

am4/src/lib.rs

@@ -1,14 +1,38 @@
-// stage 1
 pub mod airport;
 pub mod utils;
 
-// stage 2
 pub mod campaign;
 pub mod ticket;
 pub mod user;
 
-// stage 3
 pub mod aircraft;
 
-// stage 4
-pub mod route;
+pub mod route; // under development
+
+// to keep track of changes for data files in `../data`
+#[macro_export]
+macro_rules! ac_version {
+    () => {
+        "0"
+    };
+}
+pub const AC_FILENAME: &str = concat!("aircrafts-v", ac_version!(), ".bin");
+
+#[macro_export]
+macro_rules! ap_version {
+    () => {
+        "0"
+    };
+}
+pub const AP_FILENAME: &str = concat!("airports-v", ap_version!(), ".bin");
+// distances are generated from the airports
+pub const DIST_FILENAME: &str = concat!("distances-v", ap_version!(), ".bin");
+
+#[macro_export]
+macro_rules! demand_version {
+    () => {
+        "0"
+    };
+}
+pub const DEM_FILENAME0: &str = concat!("demands0-v", demand_version!(), ".bin");
+pub const DEM_FILENAME1: &str = concat!("demands1-v", demand_version!(), ".bin");

am4/src/route/db.rs

@@ -1,7 +1,7 @@
 use crate::airport::{db::AIRPORT_COUNT, Airport};
 use crate::route::{demand::pax::PaxDemand, leg::calculate_distance};
 use crate::utils::ParseError;
-use rkyv::{self, Deserialize};
+use rkyv::{self, AlignedVec, Deserialize};
 
 pub const ROUTE_COUNT: usize = AIRPORT_COUNT * (AIRPORT_COUNT - 1) / 2;
 
@@ -37,6 +37,24 @@ impl Demands {
 pub struct Distances(Vec<f32>);
 
 impl Distances {
+    pub fn from_bytes(buffer: &[u8]) -> Result<Self, ParseError> {
+        let archived = rkyv::check_archived_root::<Vec<f32>>(buffer)
+            .map_err(|e| ParseError::ArchiveError(e.to_string()))?;
+
+        let distances: Vec<f32> = archived
+            .deserialize(&mut rkyv::Infallible)
+            .map_err(|e| ParseError::DeserialiseError(e.to_string()))?;
+
+        if distances.len() != ROUTE_COUNT {
+            return Err(ParseError::InvalidDataLength {
+                expected: ROUTE_COUNT,
+                actual: distances.len(),
+            });
+        }
+
+        Ok(Distances(distances))
+    }
+
     pub fn from_airports(aps: &[Airport]) -> Self {
         assert!(aps.len() == AIRPORT_COUNT); // compiler optimisation
         let mut d = Vec::<f32>::with_capacity(ROUTE_COUNT);
@@ -54,6 +72,12 @@ impl Distances {
         Distances(d)
     }
 
+    pub fn to_bytes(&self) -> Result<AlignedVec, ParseError> {
+        let av = rkyv::to_bytes::<Vec<f32>, 30_521_492>(&self.0)
+            .map_err(|e| ParseError::SerialiseError(e.to_string()))?;
+        Ok(av)
+    }
+
     pub fn data(&self) -> &Vec<f32> {
         &self.0
     }

am4/src/utils.rs

@@ -44,6 +44,8 @@ pub enum ParseError {
     ArchiveError(String),
     #[error("Deserialise error: {0}")]
     DeserialiseError(String),
+    #[error("Serialise error: {0}")]
+    SerialiseError(String),
     #[error("Invalid data length: expected {expected} routes, got {actual}")]
     InvalidDataLength { expected: usize, actual: usize },
 }