From d60a76214fc638757e276f94430864330b468dc1 Mon Sep 17 00:00:00 2001 From: Jeromos Kovacs Date: Wed, 17 Jul 2024 01:21:12 +0200 Subject: [PATCH] fix: use srtm --- Cargo.lock | 55 ++++++++++++++++++- Cargo.toml | 5 +- README.md | 51 ++++++++++++++++-- src/main.rs | 152 ++++++++++++++++++++++++++++++++++------------------ 4 files changed, 204 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e54fe00..9be8c45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,6 +118,31 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "csv" version = "1.3.0" @@ -148,6 +173,12 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "fit2gpx" version = "0.3.0" @@ -156,6 +187,7 @@ dependencies = [ "fit_file", "geo-types", "gpx", + "rayon", "srtm", "time", ] @@ -262,6 +294,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "ryu" version = "1.0.18" @@ -290,7 +342,8 @@ dependencies = [ [[package]] name = "srtm" -version = "0.1.1" +version = "0.2.1" +source = "git+https://github.com/jeromeschmied/srtm#20ed65362e6aadae0ba8be46b1a7cf42f11ab073" dependencies = [ "byteorder", ] diff --git a/Cargo.toml b/Cargo.toml index ee77f5a..47b0a58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" authors = ["Jeromos Kovács "] description = ".fit to .gpx converter" -keywords = ["garmin", "fit", "cli"] +keywords = ["garmin", "fit", "cli", "gpx"] categories = ["command-line-interface"] repository = "https://github.com/jeromeschmied/fit2gpx-rs" license = "MIT" @@ -17,5 +17,6 @@ clap = { version = "4.5.8", features = ["derive"] } fit_file = "0.6.0" geo-types = "0.7.13" gpx = "0.10.0" -srtm = { version = "0.1.1", path = "../srtm" } +rayon = "1.10.0" +srtm = { git = "https://github.com/jeromeschmied/srtm", version = "0.2.1" } time = { version = "0.3.36", default-features = false } diff --git a/README.md b/README.md index 7ca6f08..1a9427a 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,67 @@ # fit2gpx-rs +## Installation + +- have [Rust](https://rustup.rs) + +- with `cargo` from [crates.io](https://crates.io): `cargo install fit2gpx` +- with `cargo` from [github](https://github.com/jeromeschmied): `cargo install --git "https://github.com/jeromeschmied/fit2gpx-rs"` + +- with `cargo` and `git` from [github](https://github.com/jeromeschmied): + +```sh +git clone --depth 1 "https://github.com/jeromeschmied/fit2gpx-rs" +cd fit2gpx-rs +cargo install --locked --path . +``` + +## Usage + +see `fit2gpx --help` + +let's say you want to convert `a_lovely_evening_walk.fit` to `a_lovely_evening_walk.gpx` +in that case, you'd do the following +`fit2gpx a_lovely_evening_walk.fit` +if you also want to add elevation data, as the `.fit` file didn't contain any, follow [these steps](#how-to-add-elevation-data) + ## Purpose This is a simple Rust library and binary for converting .FIT files to .GPX files. -A **_faster_** alternative to [fit2gpx](https://github.com/dodo-saba/fit2gpx) +A **_faster_** alternative to the great [**_fit2gpx_**](https://github.com/dodo-saba/fit2gpx) - [FIT](https://developer.garmin.com/fit/overview/) is a GIS data file format used by Garmin GPS sport devices and Garmin software - [GPX](https://docs.fileformat.com/gis/gpx/) is an XML based format for GPS tracks. +## Is it any good? + +Yes. + ## Why another one - it's about 80 times as fast (single file, no elevation added) -- it can add elevation data (though it isn't very precise) +- it's way faster with multi-file execution too +- it can add elevation data - it's fun -## Why not this one +## How to add elevation data -- it doesn't support strava bulk-export stuff +- first of all, have srtm data: `.hgt` files downloaded + one reliable source is [Sonny's collection](https://sonny.4lima.de/), it's only for Europe though +- then unzip everything, place all of the `.hgt` files to a single directory +- set `$ELEV_DATA_DIR` to that very directory or pass `--elev_data_dir ~/my_elevation_data_dir` +- pass the `--add_altitude | -a` flag to `fit2gpx` + +## Why might this one not be the right choice + +- it doesn't support strava bulk-export stuff: unzipping `.gz` files, + which you can do in the shell with 1 command ## Direct dependencies -- [coordinate-altitude](https://github.com/jeromeschmied/coordinate-altitude) + + +- [srtm](https://github.com/jeromeschmied/srtm) - [fit_file](https://crates.io/crates/fit_file) - [gpx](https://crates.io/crates/gpx) - [clap](https://crates.io/crates/clap) +- [rayon](https://crates.io/crates/rayon) diff --git a/src/main.rs b/src/main.rs index b591be2..c1f6a1d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ use fit_file as fit; use fit_file::{fit_file, FitFieldValue, FitRecordMsg, FitSessionMsg}; use geo_types::{coord, Point}; use gpx::{Gpx, GpxVersion, Track, TrackSegment, Waypoint}; +use rayon::prelude::*; use std::{collections::HashMap, fs::File, io::BufWriter, path::PathBuf}; use time::OffsetDateTime; @@ -103,18 +104,23 @@ fn callback( } /// Context structure. An instance of this will be passed to the parser and ultimately to the callback function so we can use it for whatever. -#[derive(Default)] +#[derive(Default, Clone)] struct Context { + file_name: String, sum00: u32, num_records_processed: u16, track_segment: TrackSegment, } -fn fit2gpx(f_in: &str, config: &Args) -> Res<()> { - let file = std::fs::File::open(f_in)?; +fn read_fit(fit: &str) -> Res { + let mut cx = Context { + file_name: fit.to_owned(), + ..Context::default() + }; + let file = std::fs::File::open(fit)?; let mut reader = std::io::BufReader::new(file); - let mut cx = Context::default(); + // let mut cx = Context::default(); fit_file::read(&mut reader, callback, &mut cx)?; let percent_00 = cx.sum00 as f32 / cx.track_segment.points.len() as f32; @@ -129,21 +135,12 @@ fn fit2gpx(f_in: &str, config: &Args) -> Res<()> { && (-90. ..90.).contains(&y) && (-180. ..180.).contains(&x) }); - if config.add_altitude { - add_altitude(&mut cx.track_segment.points, &config.elev_data_dir); - - // coordinate_altitude::add_altitude(&mut coords)?; - - // for (i, point) in cx.track_segment.points.iter_mut().enumerate() { - // if point.elevation.is_none() { - // point.elevation = coords.get(i).map(|c| c.altitude); - // } - // } - } - + Ok(cx) +} +fn fit2gpx(cx: Context) -> Res<()> { // Instantiate Gpx struct let track = Track { - segments: vec![cx.track_segment], + segments: vec![cx.track_segment.clone()], ..Track::default() }; let gpx = Gpx { @@ -152,7 +149,7 @@ fn fit2gpx(f_in: &str, config: &Args) -> Res<()> { ..Gpx::default() }; - let f_out = f_in.replace(".fit", ".gpx"); + let f_out = cx.file_name.replace(".fit", ".gpx"); // Create file at path let gpx_file = File::create(f_out)?; let buf = BufWriter::new(gpx_file); @@ -162,31 +159,29 @@ fn fit2gpx(f_in: &str, config: &Args) -> Res<()> { println!("{} records processed", cx.num_records_processed); Ok(()) } - fn is_00(wp: &Waypoint) -> bool { wp.point().x_y() == (0., 0.) } -fn add_altitude(wps: &mut [Waypoint], elev_data_dir: &Option) { - // coord is x,y but we need y,x - let xy_yx = |wp: &Waypoint| -> srtm::Coord { - let (x, y) = wp.point().x_y(); - (y, x).into() - }; +fn needed_tile_coords(wps: &[Waypoint]) -> Vec<(i32, i32)> { // kinda Waypoint to (i32, i32) let trunc = |wp: &Waypoint| -> (i32, i32) { let (x, y) = wp.point().x_y(); (y.trunc() as i32, x.trunc() as i32) }; // tiles we need - let mut needs: Vec = Vec::new(); + let mut needs = Vec::new(); for wp in wps.iter().filter(|wp| !is_00(wp)).map(trunc) { - if !needs.contains(&wp.into()) { - needs.push(wp.into()); + if !needs.contains(&wp) { + needs.push(wp); } } + needs +} + +fn needed_tiles(needs: &[(i32, i32)], elev_data_dir: &Option) -> Vec { if needs.is_empty() { - return; + return vec![]; } let elev_data_dir = if let Some(arg_data_dir) = &elev_data_dir { @@ -196,39 +191,94 @@ fn add_altitude(wps: &mut [Waypoint], elev_data_dir: &Option) { } else { panic!("no elevation data dir is passed as an arg or set as an environment variable: elev_data_dir"); }; - let tiles = needs - .iter() + needs + .par_iter() .map(|c| srtm::get_filename(*c)) .map(|t| elev_data_dir.join(t)) .map(|p| srtm::Tile::from_file(p).unwrap()) - .collect::>(); + .collect::>() +} +fn get_all_elev_data<'a>( + needs: &'a [(i32, i32)], + tiles: &'a [srtm::Tile], +) -> HashMap<&'a (i32, i32), &'a srtm::Tile> { + assert_eq!(needs.len(), tiles.len()); + if needs.is_empty() { + return HashMap::new(); + } let all_elev_data = needs - .iter() + .par_iter() .enumerate() - .map(|(i, coord)| (coord.trunc(), tiles.get(i).unwrap())) + .map(|(i, coord)| (coord, tiles.get(i).unwrap())) .collect::>(); eprintln!("loaded elevation data: {:?}", all_elev_data.keys()); - for wp in wps - .iter_mut() + all_elev_data +} + +fn add_elev(wps: &mut [Waypoint], elev_data: &HashMap<&(i32, i32), &srtm::Tile>) -> Res<()> { + // coord is x,y but we need y,x + let xy_yx = |wp: &Waypoint| -> srtm::Coord { + let (x, y) = wp.point().x_y(); + (y, x).into() + }; + wps.par_iter_mut() .filter(|wp| wp.elevation.is_none() && !is_00(wp)) - { - let coord: srtm::Coord = xy_yx(wp); - let elev_data = all_elev_data.get(&coord.trunc()).unwrap(); - wp.elevation = Some(elev_data.get(coord) as f64); - } + .for_each(|wp| { + let coord = xy_yx(wp); + let elev_data = elev_data.get(&coord.trunc()).unwrap(); + wp.elevation = Some(elev_data.get(coord) as f64); + }); + Ok(()) } -fn main() { +fn main() -> Res<()> { // collecting cli args - let args = Args::parse(); + let conf = Args::parse(); + + let all_fit = conf + .files + .par_iter() + .filter(|f| { + f.ends_with(".fit") + && (if !conf.overwrite { + !PathBuf::from(f.replace(".fit", ".gpx")).exists() + } else { + true + }) + }) + .flat_map(|f| read_fit(f).inspect_err(|e| eprintln!("read error: {e:?}"))) + .collect::>(); + + let all_needed_tile_coords = if conf.add_altitude { + let mut all = all_fit + .par_iter() + .flat_map(|cx| needed_tile_coords(&cx.track_segment.points)) + .collect::>(); + all.sort_unstable(); + all.dedup(); - for file in args.files.iter().filter(|f| f.ends_with(".fit")) { - if !args.overwrite { - let as_gpx = PathBuf::from(&file.clone().replace(".fit", ".gpx")); - if as_gpx.exists() { - continue; + all + } else { + vec![] + }; + let all_needed_tiles = needed_tiles(&all_needed_tile_coords, &conf.elev_data_dir); + let all_elev_data = get_all_elev_data(&all_needed_tile_coords, &all_needed_tiles); + // coordinate_altitude::add_altitude(&mut coords)?; + // for (i, point) in cx.track_segment.points.iter_mut().enumerate() { + // if point.elevation.is_none() { + // point.elevation = coords.get(i).map(|c| c.altitude); + // } + // } + + all_fit + .into_iter() + .try_for_each(|mut cx: Context| -> Res<()> { + if conf.add_altitude { + let _ = add_elev(&mut cx.track_segment.points, &all_elev_data) + .inspect_err(|e| eprintln!("elevation error: {e:?}")); } - } - let _ = fit2gpx(file, &args).inspect_err(|e| eprintln!("error: {e:#?}")); - } + fit2gpx(cx).inspect_err(|e| eprintln!("convertion error: {e:?}")) + })?; + + Ok(()) }