From 89f136b661ccfdf306ebca8725330982ff345ad2 Mon Sep 17 00:00:00 2001 From: Yannik Sander <7040031+ysndr@users.noreply.github.com> Date: Mon, 27 Mar 2023 18:50:02 +0200 Subject: [PATCH] feat: redesign flake-ref parsing (#15) This PR implements nix supported flakerefs. Unlike the previous implementation all flakerefs are individual (generic) types. This allows implementing converting to and from string representations on a per-type basis, rather than necessarily covering all possible references in a single match statement. As a side effect, implementing flake references one by one, allowed us to match nix' behavior more closely (#3, #16). Some flakerefs allow multiple protocols, but are essentially equivalent. This includes `file`, `tarball`, `git` and "git forges" (e.g. `github`). Such flakerefs have been implemented as generic types in order to share parsing logic, and at the same time retain the ability to enforce individual origins statically. For instance, it is now possible to define a composite type that requires local files (i.e. `git+file`, `path` `[file+]file://` or `[tarball+]file://`). All tests that existed with the old flake_ref work with the new ones and I added a couple of "roundtrips" to assure, we do not lose information on the way. I stumbled over a very annoying [`serde` bug](https://github.com/serde-rs/serde/issues/1547#issuecomment-705744778) that basically says, `deny_unknown_fields` and `flatten` cannot be used together. That and the fact that `url` queries are not self describing structures (which [triggers](https://github.com/nox/serde_urlencoded/issues/33) another [`serde` bug](https://github.com/serde-rs/serde/issues/1183) related to flattening), lets me wonder if we should use serde at all for query parsing or do the parsing manually, at the cost of legibility (https://github.com/flox/runix/pull/12/files#diff-fa82b2796286fd4622869172f2187af6691578ffbdf972e853826db2d4277fbcR200-R226). --------- Co-authored-by: Matthew Kenigsberg --- crates/runix/Cargo.toml | 6 +- crates/runix/src/flake_ref.rs | 511 ---------------------- crates/runix/src/flake_ref/file.rs | 362 +++++++++++++++ crates/runix/src/flake_ref/git.rs | 141 ++++++ crates/runix/src/flake_ref/git_service.rs | 312 +++++++++++++ crates/runix/src/flake_ref/indirect.rs | 127 ++++++ crates/runix/src/flake_ref/lock.rs | 75 ++++ crates/runix/src/flake_ref/mod.rs | 290 ++++++++++++ crates/runix/src/flake_ref/path.rs | 169 +++++++ crates/runix/src/flake_ref/protocol.rs | 85 ++++ crates/runix/src/registry.rs | 21 +- 11 files changed, 1576 insertions(+), 523 deletions(-) delete mode 100644 crates/runix/src/flake_ref.rs create mode 100644 crates/runix/src/flake_ref/file.rs create mode 100644 crates/runix/src/flake_ref/git.rs create mode 100644 crates/runix/src/flake_ref/git_service.rs create mode 100644 crates/runix/src/flake_ref/indirect.rs create mode 100644 crates/runix/src/flake_ref/lock.rs create mode 100644 crates/runix/src/flake_ref/mod.rs create mode 100644 crates/runix/src/flake_ref/path.rs create mode 100644 crates/runix/src/flake_ref/protocol.rs diff --git a/crates/runix/Cargo.toml b/crates/runix/Cargo.toml index 48bc2cd..2392f6e 100644 --- a/crates/runix/Cargo.toml +++ b/crates/runix/Cargo.toml @@ -18,8 +18,12 @@ runix-derive = "0.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_with = "2.2.0" +serde_urlencoded = "0.7.1" +url = { version = "2.3", features = ["serde"] } shell-escape = "0.1.5" tokio = { version = "1.21", features = ["full"] } tokio-stream = { version = "0.1.11", features = ["tokio-util", "io-util"] } thiserror = "1.0" -url = "2.3" +chrono = { version = "0.4.24", features = ["serde"] } +regex = "1.7.2" +once_cell = "1.17.1" diff --git a/crates/runix/src/flake_ref.rs b/crates/runix/src/flake_ref.rs deleted file mode 100644 index 5e2d12f..0000000 --- a/crates/runix/src/flake_ref.rs +++ /dev/null @@ -1,511 +0,0 @@ -//! A work in progress FlakeRef implementation -//! -//! Parses flake reference Urls as defined by the Nix reference implementation. - -use std::borrow::Cow; -use std::num::ParseIntError; -use std::path::PathBuf; -use std::str::FromStr; - -use log::info; -use serde::{Deserialize, Serialize}; -use serde_with::skip_serializing_none; -use thiserror::Error; -use url::form_urlencoded::Serializer; -use url::{Url, UrlQuery}; - -#[derive(Debug, Error)] -pub enum UrlError { - #[error("Could not extract path from Url")] - ExtractPath(()), -} - -#[derive(Debug, Error)] -pub enum FlakeRefError { - #[error("Url action failed: {0}")] - UrlAccess(#[from] UrlError), - - #[error("Invalid FlakeRef Url: {0}")] - FlakeRefUrl(Url), - - #[error("Could not parse flakeRef {0} as Url: {1}")] - ParseUrl(String, url::ParseError), - - #[error("Could not parse `lastModified` field as Integer")] - ParseLastModified(ParseIntError), - #[error("Could not parse `revCount` field as Integer")] - ParseRevCount(ParseIntError), -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct IndirectFlake { - pub id: FlakeId, -} - -/// Flake ref definitions -/// TODO: make sure to conform with https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html -#[skip_serializing_none] -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -#[serde(rename_all = "lowercase")] -#[serde(tag = "type")] - -pub enum ToFlakeRef { - GitHub(GitService), - GitLab(GitService), - Sourcehut(GitService), - /// https://cs.github.com/NixOS/nix/blob/f225f4307662fe9a57543d0c86c28aa9fddaf0d2/src/libfetchers/path.cc#L46 - Path { - path: PathBuf, - - #[serde(rename = "revCount")] - rev_count: Option, - - #[serde(flatten)] - pinned: Option, - }, - /// https://cs.github.com/NixOS/nix/blob/f225f4307662fe9a57543d0c86c28aa9fddaf0d2/src/libfetchers/git.cc#L287 - Git { - url: GitUrl, - shallow: Option, - submodules: Option, - #[serde(rename = "allRefs")] - all_refs: Option, - - #[serde(rename = "ref")] - commit_ref: CommitRef, - - #[serde(rename = "revCount")] - rev_count: Option, - - #[serde(flatten)] - pinned: Pinned, - }, - /// https://cs.github.com/NixOS/nix/blob/f225f4307662fe9a57543d0c86c28aa9fddaf0d2/src/libfetchers/tarball.cc#L206 - Tarball { - url: TarUrl, - unpack: Option, - #[serde(rename = "narHash")] - nar_hash: NarHash, - }, - Indirect(IndirectFlake), -} - -impl ToFlakeRef { - pub fn to_url(&self) -> Result { - let url = match self { - ToFlakeRef::GitHub(e) | ToFlakeRef::GitLab(e) | ToFlakeRef::Sourcehut(e) => { - let service = match self { - ToFlakeRef::GitHub(_) => "github", - ToFlakeRef::GitLab(_) => "gitlab", - ToFlakeRef::Sourcehut(_) => "sourcehut", - _ => unreachable!(), - }; - - let mut url = Url::parse(&format!("{service}:/")) - .expect("Failed initializing `{service}:` url"); - e.add_to_url(&mut url); - url - }, - ToFlakeRef::Path { - path, - rev_count, - pinned, - } => { - // Ugly way to force "path" scheme and correct path - let mut url = Url::parse("path:/").expect("Failed initializing `path:` url"); - - // set the path part - url.set_path(&path.to_string_lossy()); - - // get the query handle - let mut query = url.query_pairs_mut(); - - if let Some(count) = rev_count { - query.append_pair("revCount", &count.to_string()); - } - - // add common `pin` attrbutes to query - for pin in pinned { - pin.add_to_query(&mut query); - } - - let url = query.finish().to_owned(); - - debug_assert_eq!(url.scheme(), "path"); - url - }, - ToFlakeRef::Git { - url: _, - shallow: _, - submodules: _, - all_refs: _, - commit_ref: _, - rev_count: _, - pinned: _, - } => todo!(), - ToFlakeRef::Tarball { - url: _, - unpack: _, - nar_hash: _, - } => todo!(), - ToFlakeRef::Indirect(IndirectFlake { id }) => { - Url::parse(&format!("flake:{id}")).expect("Failed to create indirect reference") - }, - }; - Ok(url) - } - - pub fn from_url(url: &FlakeUrl) -> Result { - let flake_ref = match url.scheme() { - // https://cs.github.com/NixOS/nix/blob/f225f4307662fe9a57543d0c86c28aa9fddaf0d2/src/libfetchers/path.cc#L11 - "path" | "file" => ToFlakeRef::Path { - path: PathBuf::from(url.path()), - rev_count: url - .query_pairs() - .find(|(name, _)| name == "revCount") - .map(|(c, _)| c.parse().map_err(FlakeRefError::ParseRevCount)) - .transpose()?, - pinned: Pinned::from_query(url)?, - }, - "github" => ToFlakeRef::GitHub(GitService::from_url(url)?), - "flake" => ToFlakeRef::Indirect(IndirectFlake { - id: url.path().to_string(), - }), - _ => todo!(), - }; - Ok(flake_ref) - } -} - -impl FromStr for ToFlakeRef { - type Err = FlakeRefError; - - fn from_str(s: &str) -> Result { - let url = Url::parse(s).or_else(|e| { - info!("could not parse '{s}' as qualified url, trying to parse as `path:` ({e})",); - Url::parse(&format!("path:{}", s)) - .map_err(|e| FlakeRefError::ParseUrl(s.to_string(), e)) - })?; - - ToFlakeRef::from_url(&url) - } -} - -#[skip_serializing_none] -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum Pinned { - NarAndRev { - #[serde(rename = "narHash")] - nar_hash: NarHash, - #[serde(rename = "lastModified")] - last_modified: LastModified, - #[serde(rename = "rev")] - commit_rev: CommitRev, - }, - Nar { - #[serde(rename = "narHash")] - nar_hash: NarHash, - #[serde(rename = "lastModified")] - last_modified: LastModified, - }, - - Rev { - #[serde(rename = "rev")] - commit_rev: CommitRev, - - #[serde(rename = "lastModified")] - last_modified: LastModified, - }, -} - -impl Pinned { - fn add_to_query(&self, query: &mut Serializer) { - match self { - Pinned::Nar { - nar_hash, - last_modified: _, - } => { - query.append_pair("narHash", &nar_hash.clone()); - }, - Pinned::Rev { - commit_rev, - last_modified: _, - } => { - query.append_pair("rev", &commit_rev.clone()); - }, - Pinned::NarAndRev { - nar_hash, - last_modified: _, - commit_rev, - } => { - query.append_pair("narHash", &nar_hash.clone()); - query.append_pair("rev", &commit_rev.clone()); - }, - }; - - match self { - Pinned::NarAndRev { last_modified, .. } - | Pinned::Nar { last_modified, .. } - | Pinned::Rev { last_modified, .. } => { - query.append_pair("lastModified", &last_modified.to_string()) - }, - }; - } - - fn from_query(url: &Url) -> Result, FlakeRefError> { - let nar_hash = url - .query_pairs() - .find(|(name, _)| name == "narHash") - .map(|(_, value)| value); - let last_modified = url - .query_pairs() - .find(|(name, _)| name == "lastModified") - .map(|(_, value)| value); - let rev = url - .query_pairs() - .find(|(name, _)| name == "rev") - .map(|(_, value)| value); - - fn parse_last_modified(modified: Option>) -> u64 { - modified - .map(|s| s.parse().unwrap_or_default()) - .unwrap_or_default() - } - - let pinned = match (nar_hash, rev, last_modified) { - (None, None, _) => None, - (None, Some(rev), modified) => Some(Self::Rev { - commit_rev: rev.into_owned(), - last_modified: parse_last_modified(modified), - }), - (Some(nar), None, modified) => Some(Self::Nar { - nar_hash: nar.into_owned(), - last_modified: parse_last_modified(modified), - }), - (Some(nar), Some(rev), modified) => Some(Self::NarAndRev { - nar_hash: nar.into_owned(), - last_modified: parse_last_modified(modified), - - commit_rev: rev.into_owned(), - }), - }; - - Ok(pinned) - } -} - -/// Encodes type github type (+ gitlab + sourcehut = git service) data -/// -/// https://cs.github.com/NixOS/nix/blob/f225f4307662fe9a57543d0c86c28aa9fddaf0d2/src/libfetchers/github.cc#L108 -#[skip_serializing_none] -#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, Eq)] -pub struct GitService { - owner: RepoOwner, - repo: RepoName, - host: Option, - #[serde(rename = "ref")] - commit_ref: Option, - #[serde(flatten)] - pinned: Option, -} - -impl GitService { - fn add_to_url(&self, url: &mut Url) { - let path = format!("{}/{}", self.owner, self.repo); - url.set_path(&path); - let mut query = url.query_pairs_mut(); - if let Some(ref commit_ref) = self.commit_ref { - query.append_pair("ref", commit_ref); - } - for pin in &self.pinned { - pin.add_to_query(&mut query); - } - query.finish(); - } - - fn from_url(url: &Url) -> Result { - let mut service = match url - .path() - .splitn(3, '/') - // .cloned() - .collect::>()[..] - { - ["", owner, repo] | [owner, repo] => GitService { - owner: owner.to_string(), - repo: repo.to_string(), - ..Default::default() - }, - ["", owner, repo, commit_ref] | [owner, repo, commit_ref] => GitService { - owner: owner.to_string(), - repo: repo.to_string(), - commit_ref: Some(commit_ref.to_string()), - ..Default::default() - }, - _ => Err(FlakeRefError::FlakeRefUrl(url.clone()))?, - }; - - if let Some((_, commit_ref)) = url.query_pairs().find(|(name, _)| name == "ref") { - let _ = service.commit_ref.insert(commit_ref.to_string()); - } - - Ok(service) - } -} - -pub type FlakeId = String; -pub type RepoOwner = String; -pub type RepoName = String; -pub type CommitRef = String; -pub type CommitRev = String; -pub type RepoHost = String; -pub type NarHash = String; -pub type LastModified = u64; -pub type RevCount = u64; -pub type GitUrl = String; -pub type TarUrl = String; -pub type FlakeUrl = Url; - -#[cfg(test)] -mod tests { - - use url::Url; - - use super::*; - - #[test] - fn parses_dot_flakeref() { - ToFlakeRef::from_str(".").unwrap(); - ToFlakeRef::from_str("path:.").unwrap(); - } - - #[test] - fn parses_path_flakeref() { - dbg!(ToFlakeRef::from_str("/").unwrap()); - - assert_eq!( - ToFlakeRef::from_str("/some/where").unwrap(), - ToFlakeRef::from_str("path:///some/where").unwrap() - ); - assert_eq!( - ToFlakeRef::from_str("/some/where").unwrap(), - ToFlakeRef::from_str("path:/some/where").unwrap() - ); - } - - #[test] - fn parses_github_flakeref() { - let flakeref = serde_json::from_str::( - r#" -{ - "owner": "flox", - "ref": "unstable", - "repo": "nixpkgs", - "type": "github" -} - "#, - ) - .expect("should parse"); - - assert_eq!( - flakeref, - ToFlakeRef::GitHub(GitService { - owner: "flox".into(), - repo: "nixpkgs".into(), - host: None, - commit_ref: Some("unstable".into()), - pinned: None - }) - ) - } - - #[test] - fn parses_pinned_path() { - let flakeref = serde_json::from_str::( - r#" -{ - "lastModified": 1666570118, - "narHash": "sha256-MTXmIYowHM1wyIYyqPdBLia5SjGnxETv0YkIbDsbkx4=", - "path": "/nix/store/083m43hjhry94cvfmqdv7kjpvsl3zzvi-source", - "rev": "1e684b371cf05300bc2b432f958f285855bac8fb", - "type": "path" -} - "#, - ) - .expect("should parse pin"); - - assert_eq!(flakeref, ToFlakeRef::Path { - path: "/nix/store/083m43hjhry94cvfmqdv7kjpvsl3zzvi-source".into(), - rev_count: None, - pinned: Some(Pinned::NarAndRev { - nar_hash: "sha256-MTXmIYowHM1wyIYyqPdBLia5SjGnxETv0YkIbDsbkx4=".into(), - last_modified: 1666570118, - commit_rev: "1e684b371cf05300bc2b432f958f285855bac8fb".into() - }) - }) - } - - /// Ensure that a path flake ref serializes without information loss - #[test] - fn path_to_from_url() { - let flake_ref = ToFlakeRef::Path { - path: "/nix/store/083m43hjhry94cvfmqdv7kjpvsl3zzvi-source".into(), - rev_count: None, - pinned: Some(Pinned::NarAndRev { - nar_hash: "sha256-MTXmIYowHM1wyIYyqPdBLia5SjGnxETv0YkIbDsbkx4=".into(), - last_modified: 1666570118, - commit_rev: "1e684b371cf05300bc2b432f958f285855bac8fb".into(), - }), - }; - - let parsed = ToFlakeRef::from_url(&flake_ref.to_url().expect("should serialize to url")) - .expect("should deserialize from url"); - - assert_eq!(flake_ref, parsed) - } - - /// Ensure that paths with `path` and `file` scheme parse - #[test] - fn path_from_path_url() { - let qualified = Url::parse("path:/my/directory"); - assert!(dbg!(&qualified).is_ok()); - - let unqualified = Url::from_file_path("/my/directory"); - assert!(dbg!(&unqualified).is_ok()); - - assert_eq!( - ToFlakeRef::from_url(&qualified.unwrap()).expect("Should parse qualified path"), - ToFlakeRef::from_url(&unqualified.unwrap()).expect("Should parse unqualified path") - ) - } - - /// Ensure that a github flake ref serializes without information loss - #[test] - fn github_to_from_url() { - let flake_ref = ToFlakeRef::GitHub(GitService { - owner: "flox".into(), - repo: "nixpkgs".into(), - host: None, - commit_ref: Some("unstable".into()), - pinned: None, - }); - - let parsed = ToFlakeRef::from_url(&flake_ref.to_url().expect("should serialize to url")) - .expect("should deserialize from url"); - - assert_eq!(flake_ref, parsed) - } - - /// Ensure that an indirect flake ref serializes without information loss - #[test] - fn indirect_to_from_url() { - let flake_ref = ToFlakeRef::Indirect(IndirectFlake { - id: "nixpkgs-flox".into(), - }); - - let parsed = ToFlakeRef::from_url(&flake_ref.to_url().expect("should serialize to url")) - .expect("should deserialize from url"); - - assert_eq!(flake_ref, parsed) - } -} diff --git a/crates/runix/src/flake_ref/file.rs b/crates/runix/src/flake_ref/file.rs new file mode 100644 index 0000000..d6f8d40 --- /dev/null +++ b/crates/runix/src/flake_ref/file.rs @@ -0,0 +1,362 @@ +use std::borrow::Cow; +use std::fmt::Display; +use std::str::FromStr; + +use derive_more::Deref; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use url::Url; + +use self::application::{Application, ApplicationProtocol}; +use super::lock::NarHash; +use super::protocol::{self, Protocol, WrappedUrl, WrappedUrlParseError}; +use super::{BoolReprs, FlakeRefSource}; + +pub type FileUrl = WrappedUrl; + +/// +#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone)] +#[serde(deny_unknown_fields)] +pub struct FileBasedRef { + pub url: WrappedUrl, + + #[serde(rename = "type")] + #[serde(bound(deserialize = "Application: Deserialize<'de>"))] + #[serde(bound(serialize = "Application: Serialize"))] + _type: Application, + + #[serde(flatten)] + pub attributes: FileAttributes, +} + +pub type FileRef = FileBasedRef; +pub type TarballRef = FileBasedRef; + +#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone)] +#[serde(deny_unknown_fields)] +pub struct FileAttributes { + #[serde(rename = "narHash")] + pub nar_hash: Option, + + pub unpack: Option, + + pub name: Option, +} + +#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Deref, Clone)] +pub struct Unpack(#[serde(deserialize_with = "BoolReprs::deserialize_bool")] bool); + +pub mod application { + use std::borrow::Cow; + use std::fmt::Display; + use std::path::Path; + use std::str::FromStr; + + use serde_with::{DeserializeFromStr, SerializeDisplay}; + use thiserror::Error; + use url::Url; + + pub trait ApplicationProtocol: Default { + /// Describes the the application name, ie. + /// + /// Applications are prepended to a url as `+`, + /// to denote how the url should be parsed. + /// + /// ``` + /// # use runix::flake_ref::file::application::{ApplicationProtocol, File, Tarball}; + /// + /// assert_eq!(Tarball::protocol(), "tarball"); + /// assert_eq!(File::protocol(), "file"); + /// ``` + fn protocol() -> Cow<'static, str>; + + /// Determines whether the application has to be provided for a given [Url] + /// or can be implied. + /// + /// # Example + /// + /// The url + /// implies a [Tarball]. + /// This it is not required to specify the application with `tarball+`. + /// If the url is supoosed to be parsed as a [File] instead, + /// the `file+` application must be added. + /// + /// ``` + /// use runix::flake_ref::file::application::{ApplicationProtocol, File, Tarball}; + /// use url::Url; + /// let url = Url::parse("https://github.com/flox/runix/archive/refs/heads/main.tar.gz").unwrap(); + /// + /// assert!(!Tarball::required(&url)); // application not required to parse [Tarball] + /// assert!(File::required(&url)); // application required to parse as [File] + /// ``` + fn required(url: &Url) -> bool; + } + + #[derive(Debug, Default, PartialEq, Eq, DeserializeFromStr, SerializeDisplay, Clone)] + pub struct Application { + inner: P, + } + + #[derive(Debug, Error)] + #[error("Invalid Application '{0}', expected {1}")] + pub struct InvalidApplication(String, Application); + + impl FromStr for Application { + type Err = InvalidApplication; + + fn from_str(s: &str) -> Result { + if s != T::protocol() { + return Err(InvalidApplication(s.to_string(), Self::default())); + } + Ok(Self::default()) + } + } + + impl Display for Application { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", T::protocol()) + } + } + + fn is_tarball_url(url: &Url) -> bool { + let is_tarball_url = Path::new(url.path()) + .file_name() + .map(|name| { + [ + ".zip", ".tar", ".tgz", ".tar.gz", ".tar.xz", ".tar.bz2", ".tar.zst", + ] + .iter() + .any(|ext| name.to_string_lossy().ends_with(ext)) + }) + .unwrap_or(false); + + is_tarball_url + } + + #[derive(Default, Debug, Clone, PartialEq, Eq)] + pub struct File; + impl ApplicationProtocol for File { + fn protocol() -> Cow<'static, str> { + "file".into() + } + + fn required(url: &Url) -> bool { + is_tarball_url(url) + } + } + + #[derive(Default, Debug, Clone, PartialEq, Eq)] + pub struct Tarball; + impl ApplicationProtocol for Tarball { + fn protocol() -> Cow<'static, str> { + "tarball".into() + } + + fn required(url: &Url) -> bool { + !is_tarball_url(url) + } + } +} + +pub trait FileProtocol: Protocol {} +impl FileProtocol for protocol::File {} +impl FileProtocol for protocol::HTTP {} +impl FileProtocol for protocol::HTTPS {} + +impl FlakeRefSource + for FileBasedRef +{ + fn scheme() -> Cow<'static, str> { + format!( + "{outer}+{inner}", + outer = Type::protocol(), + inner = Protocol::scheme() + ) + .into() + } + + /// A [FileBasedRef] has to hold three properties: + /// + /// 1. It has to be a [Url], + /// we cannot deduce a file from just a path (that would be a [PathRef]) + /// 2. the schema has to be a supported file protocol, i.e. one of + /// [`file://`, `http://`, `https://`] + /// 3. the schema may contain an application, a hint as what the file should be parsed + /// Nix supports tarballs as their own filetype. + /// In some cases this application can be deduced from the url, + /// e.g. by looking at the path and file extension. (See [ApplicationProtocol::required]) + fn parses(maybe_ref: &str) -> bool { + if maybe_ref.starts_with(&format!( + "{scheme_with_application}:", + scheme_with_application = Self::scheme() + )) { + return true; + } + + if !Url::parse(maybe_ref) + .map(|url| Type::required(&url)) + .unwrap_or(false) + { + return maybe_ref.starts_with(&format!("{scheme}:", scheme = Protocol::scheme())); + } + + false + } +} + +impl Display for FileBasedRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut url: Url = Url::parse(&format!( + "{application}+{url}", + application = self._type, + url = self.url + )) + .unwrap(); + + url.set_query( + serde_urlencoded::to_string(&self.attributes) + .ok() + .filter(|s| !s.is_empty()) + .as_deref(), + ); + + write!(f, "{url}") + } +} + +impl FromStr for FileBasedRef { + type Err = ParseFileError; + + fn from_str(s: &str) -> Result { + let url = Url::parse(s)?; + + // resolve the application part from the url schema. + // report missing application if required ( see [ApplicationProtocol::required]) + // deduce from url + // - `app` will be the (implied) application + // - `proto` the transport protocol from the url scheme + // - `url` the url with application prepended + // - `wrapped` the parsed url with guaranteed scheme + let (app, proto, url, wrapped) = if let Some((app, proto)) = &url.scheme().split_once('+') { + let wrapped = FileUrl::::from_str(s.trim_start_matches(&format!("{app}+")))?; + + (app.to_string(), proto.to_string(), url, wrapped) + } else if App::required(&url) { + return Err(ParseFileError::InvalidScheme( + Self::scheme().to_string(), + url.scheme().to_string(), + )); + } else { + let app = App::protocol(); + let proto = url.scheme(); + let fixed = Url::parse(&format!("{app}+{url}"))?; + ( + app.to_string(), + proto.to_string(), + fixed, + FileUrl::::from_str(s)?, + ) + }; + + if app != App::protocol() || proto != Protocol::scheme() { + return Err(ParseFileError::InvalidScheme( + Self::scheme().to_string(), + url.scheme().to_string(), + )); + }; + + let attributes: FileAttributes = + serde_urlencoded::from_str(url.query().unwrap_or_default())?; + + Ok(FileBasedRef { + url: wrapped, + attributes, + _type: Application::default(), + }) + } +} + +#[derive(Debug, Error)] +pub enum ParseFileError { + #[error(transparent)] + Url(#[from] url::ParseError), + #[error(transparent)] + FileUrl(#[from] WrappedUrlParseError), + #[error("Invalid scheme (expected: '{0}:', found '{1}:')")] + InvalidScheme(String, String), + #[error("No repo specified")] + NoRepo, + #[error("Couldn't parse query: {0}")] + Query(#[from] serde_urlencoded::de::Error), +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::flake_ref::tests::{roundtrip, roundtrip_to}; + + type FileFileRef = super::FileRef; + type HttpFileRef = super::FileRef; + type HttpsFileRef = super::FileRef; + + type FileTarballRef = super::TarballRef; + type HttpTarballRef = super::TarballRef; + type HttpsTarballRef = super::TarballRef; + + #[test] + fn file_file_roundtrips() { + roundtrip::("file+file:///somewhere/there"); + roundtrip_to::("file:///somewhere/there", "file+file:///somewhere/there"); + roundtrip::("file+file:///somewhere/there?unpack=true"); + } + + #[test] + fn file_http_roundtrips() { + roundtrip::("file+http://somewhere/there"); + roundtrip_to::("http://somewhere/there", "file+http://somewhere/there"); + roundtrip::("file+http://somewhere/there?unpack=true"); + } + + #[test] + fn file_https_roundtrips() { + roundtrip::("file+https://somewhere/there"); + roundtrip_to::("https://somewhere/there", "file+https://somewhere/there"); + roundtrip::("file+https://somewhere/there?unpack=true"); + } + + #[test] + fn tarball_file_roundtrips() { + roundtrip::("tarball+file:///somewhere/there"); + roundtrip_to::( + "file:///somewhere/there.tar.gz", + "tarball+file:///somewhere/there.tar.gz", + ); + roundtrip::("tarball+file:///somewhere/there?unpack=true"); + } + + #[test] + fn tarball_http_roundtrips() { + roundtrip::("tarball+http://somewhere/there"); + roundtrip_to::( + "http://somewhere/there.tar.gz", + "tarball+http://somewhere/there.tar.gz", + ); + roundtrip::("tarball+http://somewhere/there?unpack=true"); + } + + #[test] + fn tarball_https_roundtrips() { + roundtrip::("tarball+https://somewhere/there"); + roundtrip_to::( + "https://somewhere/there.tar.gz", + "tarball+https://somewhere/there.tar.gz", + ); + roundtrip::("tarball+https://somewhere/there?unpack=true"); + } + + #[test] + fn test_parse_nar_hash() { + roundtrip::("file+file:///somewhere/there?narHash=sha256-MjeRjunqfGTBGU401nxIjs7PC9PZZ1FBCZp%2FbRB3C2M%3D") + } +} diff --git a/crates/runix/src/flake_ref/git.rs b/crates/runix/src/flake_ref/git.rs new file mode 100644 index 0000000..65e55ff --- /dev/null +++ b/crates/runix/src/flake_ref/git.rs @@ -0,0 +1,141 @@ +use std::borrow::Cow; +use std::fmt::Display; +use std::path::PathBuf; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use url::Url; + +use super::lock::{Rev, RevCount}; +use super::protocol::{self, Protocol, WrappedUrl, WrappedUrlParseError}; +use super::FlakeRefSource; + +pub type GitUrl = WrappedUrl; + +/// +#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone)] +#[serde(tag = "git")] +pub struct GitRef { + pub url: GitUrl, + + #[serde(flatten)] + pub attributes: GitAttributes, +} + +#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone)] +pub struct GitAttributes { + pub shallow: Option, + pub submodules: Option, + #[serde(rename = "allRefs")] + pub all_refs: Option, + + #[serde(rename = "revCount")] + pub rev_count: Option, + + #[serde(flatten)] + pub rev: Option, + + #[serde(rename = "ref")] + pub reference: Option, + + pub dir: Option, +} + +pub trait GitProtocol: Protocol {} +impl GitProtocol for protocol::File {} +impl GitProtocol for protocol::SSH {} +impl GitProtocol for protocol::HTTP {} +impl GitProtocol for protocol::HTTPS {} + +impl FlakeRefSource for GitRef { + fn scheme() -> Cow<'static, str> { + format!("git+{inner}", inner = Protcol::scheme()).into() + } +} + +impl Display for GitRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut url = Url::parse(&format!("git+{url}", url = self.url)).unwrap(); + url.set_query( + serde_urlencoded::to_string(&self.attributes) + .ok() + .as_deref(), + ); + + write!(f, "{url}") + } +} + +impl FromStr for GitRef { + type Err = ParseGitError; + + fn from_str(s: &str) -> Result { + let url = Url::parse(s)?; + + if url.scheme() != Self::scheme() { + return Err(ParseGitError::InvalidScheme( + Self::scheme().to_string(), + url.scheme().to_string(), + )); + } + + let attributes: GitAttributes = + serde_urlencoded::from_str(url.query().unwrap_or_default())?; + + let url = GitUrl::::from_str(s.trim_start_matches("git+"))?; + + Ok(GitRef { url, attributes }) + } +} + +#[derive(Debug, Error)] +pub enum ParseGitError { + #[error(transparent)] + Url(#[from] url::ParseError), + #[error(transparent)] + GitUrl(#[from] WrappedUrlParseError), + #[error("Invalid scheme (expected: '{0}:', found '{1}:'")] + InvalidScheme(String, String), + #[error("No repo specified")] + NoRepo, + #[error("Couldn't parse query: {0}")] + Query(#[from] serde_urlencoded::de::Error), +} + +#[cfg(test)] +mod tests { + + use super::*; + + static FLAKE_REF: &'_ str = "git+file:///somewhere/on/the/drive?shallow=false&submodules=false&ref=feature%2Fxyz&dir=abc"; + + #[test] + fn parses_git_path_flakeref() { + let expected: GitRef = GitRef { + url: "file:///somewhere/on/the/drive".parse().unwrap(), + attributes: GitAttributes { + shallow: Some(false), + submodules: Some(false), + all_refs: None, + rev_count: None, + dir: Some("abc".into()), + rev: None, + reference: Some("feature/xyz".to_string()), + }, + }; + + assert_eq!(GitRef::from_str(FLAKE_REF).unwrap(), expected); + assert_eq!(expected.to_string(), FLAKE_REF); + } + + #[test] + fn parses_unsescaped_qs() { + assert_eq!( + "git+file:///somewhere/on/the/drive?shallow=false&submodules=false&ref=feature/xyz&dir=abc" + .parse::>() + .unwrap() + .to_string(), + FLAKE_REF) + } +} diff --git a/crates/runix/src/flake_ref/git_service.rs b/crates/runix/src/flake_ref/git_service.rs new file mode 100644 index 0000000..86ee949 --- /dev/null +++ b/crates/runix/src/flake_ref/git_service.rs @@ -0,0 +1,312 @@ +use std::borrow::Cow; +use std::fmt::Display; +use std::path::PathBuf; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use url::Url; + +use self::service::GitService; +use super::lock::{LastModified, NarHash, Rev, RevOrRef}; +use super::FlakeRefSource; + +#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone)] +pub struct GitServiceRef { + pub owner: String, + pub repo: String, + + #[serde(flatten)] + pub attributes: GitServiceAttributes, + + #[serde(rename = "type")] + #[serde(bound(deserialize = "GitService: Deserialize<'de>"))] + #[serde(bound(serialize = "GitService: Serialize"))] + _type: GitService, +} + +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] +#[serde_with::skip_serializing_none] +#[serde(deny_unknown_fields)] +pub struct GitServiceAttributes { + pub host: Option, + pub dir: Option, + + #[serde(rename = "ref")] + pub reference: Option, + + pub rev: Option, + + #[serde(flatten)] + pub nar_hash: Option, + + #[serde(flatten)] + pub last_modified: Option, +} + +pub mod service { + use std::borrow::Cow; + + use derive_more::From; + use serde::{Deserialize, Serialize}; + + #[derive(Default, Debug, PartialEq, Eq, From, Clone)] + pub struct GitService(Service); + + impl Serialize for GitService { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&Service::scheme()) + } + } + + impl<'de, Service: GitServiceHost> Deserialize<'de> for GitService { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + if s != Service::scheme() { + return Err(serde::de::Error::invalid_value( + serde::de::Unexpected::Str(&s), + &&*Service::scheme(), + )); + } + + Ok(Default::default()) + } + } + + pub(crate) trait GitServiceHost: Default + Eq { + fn scheme() -> Cow<'static, str>; + } + + #[derive(Default, Debug, PartialEq, Eq, Clone)] + pub struct Github; + impl GitServiceHost for Github { + fn scheme() -> Cow<'static, str> { + "github".into() + } + } + + #[derive(Default, Debug, PartialEq, Eq, Clone)] + pub struct Gitlab; + impl GitServiceHost for Gitlab { + fn scheme() -> Cow<'static, str> { + "gitlab".into() + } + } +} + +impl FlakeRefSource for GitServiceRef { + fn scheme() -> Cow<'static, str> { + Service::scheme() + } +} +impl FromStr for GitServiceRef { + type Err = ParseGitServiceError; + + fn from_str(s: &str) -> Result { + let url = Url::parse(s)?; + if url.scheme() != Self::scheme() { + return Err(ParseGitServiceError::InvalidScheme( + Self::scheme().to_string(), + url.scheme().to_string(), + )); + } + let (owner, rest) = url + .path() + .split_once('/') + .ok_or(ParseGitServiceError::NoRepo)?; + let (repo, rev_or_ref) = match rest.split_once('/') { + Some((repo, rev_or_ref)) => (repo, Some(rev_or_ref.to_owned().into())), + None => (rest, None), + }; + + let mut attributes: GitServiceAttributes = + serde_urlencoded::from_str(url.query().unwrap_or_default())?; + + if attributes.rev.is_some() && attributes.reference.is_some() { + Err(ParseGitServiceError::TwoRevs)?; + } + + if (attributes.rev.is_some() || attributes.reference.is_some()) && rev_or_ref.is_some() { + Err(ParseGitServiceError::TwoRevs)?; + } + + match rev_or_ref { + Some(RevOrRef::Rev { rev }) => attributes.rev = Some(rev), + Some(RevOrRef::Ref { reference }) => attributes.reference = Some(reference), + None => {}, + } + + Ok(GitServiceRef { + owner: owner.to_string(), + repo: repo.to_string(), + attributes, + _type: Default::default(), + }) + } +} + +impl Display for GitServiceRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut attributes = self.attributes.clone(); + + write!( + f, + "{schema}:{owner}/{repo}", + schema = Self::scheme(), + owner = self.owner, + repo = self.repo + )?; + + if let Some(part) = attributes + .rev + .take() + .map(|rev| rev.to_string()) + .or_else(|| attributes.reference.take()) + { + write!(f, "/{part}")?; + }; + + let query = serde_urlencoded::to_string(attributes).unwrap_or_default(); + if !query.is_empty() { + write!(f, "?{query}",)?; + } + + Ok(()) + } +} + +#[derive(Debug, Error)] +pub enum ParseGitServiceError { + #[error(transparent)] + Url(#[from] url::ParseError), + #[error("Url contains multiple commit hashes")] + TwoRevs, + #[error("Couldn't parse query: {0}")] + Query(#[from] serde_urlencoded::de::Error), + #[error("Invalid scheme (expected: '{0}:', found '{1}:'")] + InvalidScheme(String, String), + #[error("No repo specified")] + NoRepo, + #[error("Unkown Attribute: {0}")] + UnkownAttribute(String), +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + use crate::flake_ref::tests::{roundtrip, roundtrip_to}; + + #[test] + fn parse_github_simple() { + roundtrip::>("github:owner/repo"); + roundtrip::>("github:owner/repo/feature-branch"); + roundtrip_to::>( + "github:owner/repo?ref=feature-branch", + "github:owner/repo/feature-branch", + ); + roundtrip_to::>( + "github:owner/repo?rev=50500a744e3c2af9d89123ae17b71406b428c3ab", + "github:owner/repo/50500a744e3c2af9d89123ae17b71406b428c3ab", + ); + } + + #[test] + fn parse_invalid_simple() { + GitServiceRef::::from_str("github:owner/repo/feature?ref=another-feature") + .unwrap_err(); + GitServiceRef::::from_str( + "github:owner/repo/feature?rev=50500a744e3c2af9d89123ae17b71406b428c3ab", + ) + .unwrap_err(); + GitServiceRef::::from_str( + "github:owner/repo?ref=feature&rev=50500a744e3c2af9d89123ae17b71406b428c3ab", + ) + .unwrap_err(); + } + + #[test] + fn fail_parse_github_no_repo() { + assert!(matches!( + GitServiceRef::::from_str("github:owner"), + Err(ParseGitServiceError::NoRepo) + )) + } + + #[test] + fn parse_attributes() { + assert_eq!( + GitServiceRef::::from_str("github:owner/repo/unstable?dir=subdir") + .unwrap(), + GitServiceRef { + owner: "owner".to_string(), + repo: "repo".to_string(), + attributes: GitServiceAttributes { + host: None, + dir: Some(Path::new("subdir").to_path_buf()), + reference: Some("unstable".into()), + rev: None, + nar_hash: None, + last_modified: None, + }, + _type: GitService::default() + } + ); + } + + #[test] + fn parses_github_flakeref() { + println!( + "{}", + serde_json::to_string_pretty(&GitServiceRef:: { + owner: "flox".into(), + repo: "nixpkgs".into(), + attributes: GitServiceAttributes { + host: None, + dir: None, + reference: Some("unstable".into()), + rev: None, + nar_hash: None, + last_modified: None, + }, + _type: GitService::default(), + }) + .unwrap() + ); + + let flakeref = serde_json::from_str::>( + r#" +{ + "owner": "flox", + "repo": "nixpkgs", + "ref": "unstable", + "type": "github" +} + "#, + ) + .expect("should parse"); + + dbg!(&flakeref); + + assert_eq!(flakeref, GitServiceRef { + owner: "flox".into(), + repo: "nixpkgs".into(), + attributes: GitServiceAttributes { + host: None, + dir: None, + reference: Some("unstable".into()), + rev: None, + nar_hash: None, + last_modified: None, + }, + _type: GitService::default(), + }); + } +} diff --git a/crates/runix/src/flake_ref/indirect.rs b/crates/runix/src/flake_ref/indirect.rs new file mode 100644 index 0000000..497d9fe --- /dev/null +++ b/crates/runix/src/flake_ref/indirect.rs @@ -0,0 +1,127 @@ +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::fmt::Display; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use url::Url; + +use super::FlakeRefSource; + +/// +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone, PartialOrd, Ord)] +#[serde(tag = "indirect")] +pub struct IndirectRef { + pub id: String, + #[serde(flatten)] + pub attributes: BTreeMap, +} + +impl FlakeRefSource for IndirectRef { + fn scheme() -> Cow<'static, str> { + "flake".into() + } + + fn parses(maybe_ref: &str) -> bool { + if maybe_ref.starts_with("flake:") { + return true; + } + + if maybe_ref.contains(':') { + return false; + } + + ('a'..='z').any(|prefix| maybe_ref.starts_with(prefix)) + || ('A'..='Z').any(|prefix| maybe_ref.starts_with(prefix)) + } +} + +impl Display for IndirectRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{prefix}:{id}", prefix = Self::scheme(), id = self.id)?; + if !self.attributes.is_empty() { + write!( + f, + "?{attributes}", + attributes = serde_urlencoded::to_string(&self.attributes).unwrap_or_default() + )? + } + + Ok(()) + } +} + +impl FromStr for IndirectRef { + type Err = ParseIndirectError; + + fn from_str(s: &str) -> Result { + let url = match Url::parse(s) { + Ok(url) if url.scheme() == Self::scheme() => url, + Ok(url_bad_scheme) => Err(ParseIndirectError::InvalidScheme( + url_bad_scheme.scheme().to_string(), + Self::scheme().into_owned(), + ))?, + Err(_) if Self::parses(s) && !s.starts_with(&*Self::scheme()) => { + Url::parse(&format!("{scheme}:{s}", scheme = Self::scheme()))? + }, + e => e?, + }; + + let id = url.path().to_string(); + let attributes = serde_urlencoded::from_str(url.query().unwrap_or_default())?; + Ok(IndirectRef { id, attributes }) + } +} + +#[derive(Debug, Error)] +pub enum ParseIndirectError { + #[error(transparent)] + Url(#[from] url::ParseError), + #[error("Invalid scheme (expected: '{0}:', found '{1}:'")] + InvalidScheme(String, String), + #[error("Couldn't parse query: {0}")] + Query(#[from] serde_urlencoded::de::Error), +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::flake_ref::tests::roundtrip_to; + + /// Ensure that an indirect flake ref serializes without information loss + #[test] + fn indirect_to_from_url() { + let expect = IndirectRef { + id: "nixpkgs-flox".into(), + attributes: BTreeMap::default(), + }; + + let flakeref = "flake:nixpkgs-flox"; + + assert_eq!(flakeref.parse::().unwrap(), expect); + assert_eq!(expect.to_string(), flakeref); + } + + #[test] + fn parses_registry_flakeref() { + let expected = IndirectRef { + id: "nixpkgs".to_string(), + attributes: BTreeMap::default(), + }; + + assert_eq!(IndirectRef::from_str("flake:nixpkgs").unwrap(), expected); + assert_eq!(IndirectRef::from_str("nixpkgs").unwrap(), expected); + } + + #[test] + fn does_not_parse_other() { + IndirectRef::from_str("github:nixpkgs").unwrap_err(); + } + + #[test] + fn roundtrip_attributes() { + roundtrip_to::("nixpkgs?ref=master&dir=1", "flake:nixpkgs?dir=1&ref=master"); + } +} diff --git a/crates/runix/src/flake_ref/lock.rs b/crates/runix/src/flake_ref/lock.rs new file mode 100644 index 0000000..29e47f6 --- /dev/null +++ b/crates/runix/src/flake_ref/lock.rs @@ -0,0 +1,75 @@ +use std::str::FromStr; + +use derive_more::Deref; +use once_cell::sync::Lazy; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use serde_with::DeserializeFromStr; +use thiserror::Error; + +use super::Timestamp; + +static HASH_REGEX: Lazy = Lazy::new(|| Regex::new("[a-f0-9]{40}").unwrap()); + +/// todo: parse/validate narHash? +pub type NarHash = String; +pub type LastModified = Timestamp; + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +#[serde(untagged)] +pub enum RevOrRef { + Rev { + rev: Rev, + }, + Ref { + #[serde(rename = "ref")] + reference: String, + }, +} + +impl From for RevOrRef { + fn from(value: String) -> Self { + Rev::from_str(&value) + .map(|rev| RevOrRef::Rev { rev }) + .unwrap_or(RevOrRef::Ref { reference: value }) + } +} + +#[derive(DeserializeFromStr, Serialize, Clone, Debug, PartialEq, Eq, Deref)] +pub struct Rev(String); +impl FromStr for Rev { + type Err = InvalidRev; + + fn from_str(s: &str) -> Result { + if !HASH_REGEX.is_match(s) { + Err(InvalidRev) + } else { + Ok(Rev(s.to_string())) + } + } +} + +#[derive(Error, Debug)] +#[error("Invalid revision hash")] +pub struct InvalidRev; + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +#[serde(try_from = "StringOrInt")] +pub struct RevCount(u64); + +#[derive(Debug, Deserialize)] +enum StringOrInt { + String(String), + Int(u64), +} + +impl TryFrom for RevCount { + type Error = ::Err; + + fn try_from(value: StringOrInt) -> Result { + match value { + StringOrInt::String(s) => Ok(RevCount(s.parse()?)), + StringOrInt::Int(i) => Ok(RevCount(i)), + } + } +} diff --git a/crates/runix/src/flake_ref/mod.rs b/crates/runix/src/flake_ref/mod.rs new file mode 100644 index 0000000..9ffd323 --- /dev/null +++ b/crates/runix/src/flake_ref/mod.rs @@ -0,0 +1,290 @@ +use std::borrow::Cow; +use std::fmt::Display; +use std::str::FromStr; + +use chrono::{NaiveDateTime, TimeZone, Utc}; +use derive_more::{Display, From}; +use serde::{Deserialize, Deserializer, Serialize}; +use thiserror::Error; + +use self::file::{FileRef, TarballRef}; +use self::git::GitRef; +use self::git_service::{service, GitServiceRef}; +use self::indirect::IndirectRef; +use self::path::PathRef; + +pub mod file; +pub mod git; +pub mod git_service; +pub mod indirect; +pub mod lock; +pub mod path; +pub mod protocol; + +pub trait FlakeRefSource: FromStr + Display { + fn scheme() -> Cow<'static, str>; + fn parses(maybe_ref: &str) -> bool { + maybe_ref.starts_with(&format!("{}:", Self::scheme())) + } +} + +#[derive(Serialize, Deserialize, Display, From, Debug, Clone, PartialEq, Eq)] +#[serde(untagged)] +pub enum FlakeRef { + FileFile(FileRef), + FileHTTP(FileRef), + FileHTTPS(FileRef), + TarballFile(TarballRef), + TarballHTTP(TarballRef), + TarballHTTPS(TarballRef), + Github(GitServiceRef), + Gitlab(GitServiceRef), + Path(PathRef), + GitPath(GitRef), + GitSsh(GitRef), + GitHttps(GitRef), + GitHttp(GitRef), + Indirect(IndirectRef), + // /// https://cs.github.com/NixOS/nix/blob/f225f4307662fe9a57543d0c86c28aa9fddaf0d2/src/libfetchers/tarball.cc#L206 + // Tarball(TarballRef), +} + +impl FromStr for FlakeRef { + type Err = ParseFlakeRefError; + + fn from_str(s: &str) -> Result { + let flake_ref = match s { + _ if FileRef::::parses(s) => { + s.parse::>()?.into() + }, + _ if FileRef::::parses(s) => { + s.parse::>()?.into() + }, + _ if FileRef::::parses(s) => { + s.parse::>()?.into() + }, + _ if TarballRef::::parses(s) => { + s.parse::>()?.into() + }, + _ if TarballRef::::parses(s) => { + s.parse::>()?.into() + }, + _ if TarballRef::::parses(s) => { + s.parse::>()?.into() + }, + _ if GitServiceRef::::parses(s) => { + s.parse::>()?.into() + }, + _ if GitServiceRef::::parses(s) => { + s.parse::>()?.into() + }, + _ if PathRef::parses(s) => s.parse::()?.into(), + _ if GitRef::::parses(s) => s.parse::>()?.into(), + _ if GitRef::::parses(s) => s.parse::>()?.into(), + _ if GitRef::::parses(s) => s.parse::>()?.into(), + _ if GitRef::::parses(s) => { + s.parse::>()?.into() + }, + _ if IndirectRef::parses(s) => s.parse::()?.into(), + _ => Err(ParseFlakeRefError::Invalid)?, + }; + Ok(flake_ref) + } +} + +#[derive(Debug, Error)] +pub enum ParseFlakeRefError { + #[error(transparent)] + File(#[from] file::ParseFileError), + #[error(transparent)] + GitService(#[from] git_service::ParseGitServiceError), + #[error(transparent)] + Git(#[from] git::ParseGitError), + #[error(transparent)] + Indirect(#[from] indirect::ParseIndirectError), + #[error(transparent)] + Path(#[from] path::ParsePathRefError), + + #[error("Invalid flakeref")] + Invalid, +} + +#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, From, Clone)] +#[serde(try_from = "TimestampDeserialize")] +pub struct Timestamp( + #[serde(serialize_with = "chrono::serde::ts_seconds::serialize")] chrono::DateTime, +); + +#[derive(Deserialize, Debug)] +#[serde(untagged)] +pub enum TimestampDeserialize { + TsI64(i64), + TsString(String), +} + +impl TryFrom for Timestamp { + type Error = ParseTimeError; + + fn try_from(value: TimestampDeserialize) -> Result { + let ts = match value { + TimestampDeserialize::TsI64(t) => Utc + .timestamp_opt(t, 0) + .earliest() + .ok_or(ParseTimeError::FromInt(t))?, + // per + TimestampDeserialize::TsString(s) => NaiveDateTime::parse_from_str(&s, "%s")? + .and_local_timezone(Utc) + .earliest() + .unwrap(), + }; + Ok(Timestamp(ts)) + } +} + +#[derive(Debug, Error)] +pub enum ParseTimeError { + #[error("Could not parse {0} to UTC date")] + FromInt(i64), + #[error(transparent)] + FromString(#[from] chrono::ParseError), +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub enum BoolReprs { + String(String), + Bool(bool), +} + +impl TryFrom for bool { + type Error = ::Err; + + fn try_from(value: BoolReprs) -> Result { + match value { + BoolReprs::String(s) => s.parse::(), + BoolReprs::Bool(b) => Ok(b), + } + } +} + +impl BoolReprs { + pub fn deserialize_bool<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + BoolReprs::deserialize(deserializer)? + .try_into() + .map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +pub(super) mod tests { + + pub(super) fn roundtrip_to(input: &str, output: &str) + where + T: FromStr + Display, + ::Err: Debug + Display, + { + let parsed = input + .parse::() + .unwrap_or_else(|e| panic!("'{input}' should parse: \n{e}\n{e:#?}")); + assert_eq!(parsed.to_string(), output); + } + + pub(super) fn roundtrip(input: &str) + where + T: FromStr + Display, + ::Err: Debug + Display, + { + roundtrip_to::(input, input) + } + + use std::fmt::Debug; + + use super::*; + + #[test] + fn test_all_parsing() { + assert!(matches!( + dbg!(FlakeRef::from_str("file+file:///somewhere/there")).unwrap(), + FlakeRef::FileFile(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("file:///somewhere/there")).unwrap(), + FlakeRef::FileFile(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("file+http://my.de/path/to/file")).unwrap(), + FlakeRef::FileHTTP(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("http://my.de/path/to/file")).unwrap(), + FlakeRef::FileHTTP(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("file+https://my.de/path/to/file")).unwrap(), + FlakeRef::FileHTTPS(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("https://my.de/path/to/file")).unwrap(), + FlakeRef::FileHTTPS(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("tarball+file:///somewhere/there")).unwrap(), + FlakeRef::TarballFile(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("file:///somewhere/there.tar.gz")).unwrap(), + FlakeRef::TarballFile(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("tarball+http://my.de/path/to/file")).unwrap(), + FlakeRef::TarballHTTP(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("http://my.de/path/to/file.tar.gz")).unwrap(), + FlakeRef::TarballHTTP(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("tarball+https://my.de/path/to/file")).unwrap(), + FlakeRef::TarballHTTPS(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("https://my.de/path/to/file.tar.gz")).unwrap(), + FlakeRef::TarballHTTPS(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("github:flox/runix")).unwrap(), + FlakeRef::Github(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("gitlab:flox/runix")).unwrap(), + FlakeRef::Gitlab(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("path:/somewhere/there")).unwrap(), + FlakeRef::Path(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("git+file:///somewhere/there")).unwrap(), + FlakeRef::GitPath(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("git+ssh://github.com/flox/runix")).unwrap(), + FlakeRef::GitSsh(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("git+https://github.com/flox/runix")).unwrap(), + FlakeRef::GitHttps(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("git+http://github.com/flox/runix")).unwrap(), + FlakeRef::GitHttp(_) + )); + assert!(matches!( + dbg!(FlakeRef::from_str("flake:nixpkgs")).unwrap(), + FlakeRef::Indirect(_) + )); + } +} diff --git a/crates/runix/src/flake_ref/path.rs b/crates/runix/src/flake_ref/path.rs new file mode 100644 index 0000000..104ef14 --- /dev/null +++ b/crates/runix/src/flake_ref/path.rs @@ -0,0 +1,169 @@ +use std::borrow::Cow; +use std::fmt::Display; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use url::Url; + +use super::lock::{LastModified, NarHash, Rev, RevCount}; +use super::FlakeRefSource; + +/// +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +#[serde(tag = "path")] +pub struct PathRef { + pub path: PathBuf, + #[serde(flatten)] + pub attributes: PathAttributes, +} + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +pub struct PathAttributes { + #[serde(rename = "revCount")] + pub rev_count: Option, + + #[serde(rename = "narHash")] + pub nar_hash: Option, + + #[serde(rename = "lastModified")] + pub last_modified: Option, + + pub rev: Option, +} + +impl FlakeRefSource for PathRef { + fn scheme() -> Cow<'static, str> { + "path".into() + } + + fn parses(maybe_ref: &str) -> bool { + ["path:", "/", "."] + .iter() + .any(|prefix| maybe_ref.starts_with(prefix)) + } +} + +impl FromStr for PathRef { + type Err = ParsePathRefError; + + fn from_str(s: &str) -> Result { + let url = match Url::parse(s) { + Ok(url) => url, + Err(_) if Self::parses(s) && !s.starts_with(&*Self::scheme()) => { + Url::parse(&format!("path:{s}"))? + }, + e => e?, + }; + + if url.scheme() != Self::scheme() { + return Err(ParsePathRefError::InvalidScheme( + Self::scheme().to_string(), + url.scheme().to_string(), + )); + } + let path = Path::new(url.path()).to_path_buf(); + let attributes: PathAttributes = + serde_urlencoded::from_str(url.query().unwrap_or_default())?; + + Ok(PathRef { path, attributes }) + } +} + +impl Display for PathRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut url: Url = format!( + "{scheme}:{path}", + scheme = Self::scheme(), + path = self.path.to_string_lossy(), + ) + .parse() + .unwrap(); + + url.set_query( + serde_urlencoded::to_string(dbg!(&self.attributes)) + .ok() + .as_deref(), + ); + + write!(f, "{url}") + } +} + +#[derive(Debug, Error)] +pub enum ParsePathRefError { + #[error(transparent)] + Url(#[from] url::ParseError), + #[error("Invalid scheme (expected: '{0}:', found '{1}:'")] + InvalidScheme(String, String), + #[error("Couldn't parse query: {0}")] + Query(#[from] serde_urlencoded::de::Error), +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + + use super::*; + + #[test] + fn parses_path_flakeref() { + assert_eq!( + PathRef::from_str("/some/where").unwrap(), + PathRef::from_str("path:///some/where").unwrap() + ); + assert_eq!( + PathRef::from_str("./some/where").unwrap(), + PathRef::from_str("path:./some/where").unwrap() + ); + } + + #[test] + fn parses_pinned_path() { + let flakeref = serde_json::from_str::( + r#" +{ + "lastModified": 1666570118, + "narHash": "sha256-MTXmIYowHM1wyIYyqPdBLia5SjGnxETv0YkIbDsbkx4=", + "path": "/nix/store/083m43hjhry94cvfmqdv7kjpvsl3zzvi-source", + "rev": "1e684b371cf05300bc2b432f958f285855bac8fb", + "type": "path" +} + "#, + ) + .expect("should parse pin"); + + assert_eq!(flakeref, PathRef { + path: "/nix/store/083m43hjhry94cvfmqdv7kjpvsl3zzvi-source".into(), + attributes: PathAttributes { + rev_count: None, + nar_hash: Some("sha256-MTXmIYowHM1wyIYyqPdBLia5SjGnxETv0YkIbDsbkx4=".into()), + last_modified: Some(Utc.timestamp_opt(1666570118, 0).unwrap().into()), + rev: Some("1e684b371cf05300bc2b432f958f285855bac8fb".parse().unwrap()) + } + }) + } + + /// Ensure that a path flake ref serializes without information loss + #[test] + fn path_to_from_url() { + let flake_ref = PathRef { + path: "/nix/store/083m43hjhry94cvfmqdv7kjpvsl3zzvi-source".into(), + attributes: PathAttributes { + rev_count: None, + nar_hash: Some("sha256-MTXmIYowHM1wyIYyqPdBLia5SjGnxETv0YkIbDsbkx4=".into()), + last_modified: Some(Utc.timestamp_opt(1666570118, 0).unwrap().into()), + rev: Some("1e684b371cf05300bc2b432f958f285855bac8fb".parse().unwrap()), + }, + }; + + dbg!(&flake_ref); + + let serialized = flake_ref.to_string(); + dbg!(&serialized); + let parsed = serialized.parse().unwrap(); + + assert_eq!(flake_ref, parsed); + } +} diff --git a/crates/runix/src/flake_ref/protocol.rs b/crates/runix/src/flake_ref/protocol.rs new file mode 100644 index 0000000..d49a83e --- /dev/null +++ b/crates/runix/src/flake_ref/protocol.rs @@ -0,0 +1,85 @@ +use std::borrow::Cow; +use std::marker::PhantomData; +use std::str::FromStr; + +use derive_more::{Deref, Display}; +use serde::{Deserialize, Serialize}; +use serde_with::{DeserializeFromStr, SerializeDisplay}; +use thiserror::Error; +use url::Url; + +pub trait Protocol { + fn scheme() -> Cow<'static, str>; +} + +#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone)] +pub struct File; +impl Protocol for File { + fn scheme() -> Cow<'static, str> { + "file".into() + } +} + +#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone)] +pub struct SSH; +impl Protocol for SSH { + fn scheme() -> Cow<'static, str> { + "ssh".into() + } +} + +#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone)] +pub struct HTTPS; +impl Protocol for HTTPS { + fn scheme() -> Cow<'static, str> { + "https".into() + } +} + +#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone)] +pub struct HTTP; +impl Protocol for HTTP { + fn scheme() -> Cow<'static, str> { + "http".into() + } +} + +#[derive(DeserializeFromStr, SerializeDisplay, Display, Debug, PartialEq, Eq, Deref, Clone)] +#[display(fmt = "{inner}")] +pub struct WrappedUrl { + #[deref] + inner: Url, + _protocol: PhantomData, +} + +impl TryFrom for WrappedUrl

{ + type Error = WrappedUrlParseError; + + fn try_from(mut url: Url) -> Result { + if url.scheme() != P::scheme() { + return Err(WrappedUrlParseError::Protocol(url.scheme().to_string())); + } + url.set_fragment(None); + url.set_query(None); + Ok(WrappedUrl { + inner: url, + _protocol: PhantomData, + }) + } +} + +impl FromStr for WrappedUrl

{ + type Err = WrappedUrlParseError; + + fn from_str(s: &str) -> Result { + Url::parse(s)?.try_into() + } +} + +#[derive(Debug, Error)] +pub enum WrappedUrlParseError { + #[error(transparent)] + Url(#[from] url::ParseError), + #[error("Unsupported protocol: {0}")] + Protocol(String), +} diff --git a/crates/runix/src/registry.rs b/crates/runix/src/registry.rs index 87c47ef..6bb88dc 100644 --- a/crates/runix/src/registry.rs +++ b/crates/runix/src/registry.rs @@ -6,13 +6,11 @@ use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; use thiserror::Error; -use super::flake_ref::{FlakeRefError, IndirectFlake, ToFlakeRef}; +use crate::flake_ref::indirect::IndirectRef; +use crate::flake_ref::FlakeRef; #[derive(Error, Debug)] -pub enum RegistryError { - #[error(transparent)] - FlakeRef(#[from] FlakeRefError), -} +pub enum RegistryError {} #[derive(Debug, Clone, Deserialize, Serialize, Default, PartialEq, Eq)] pub struct Registry { @@ -27,11 +25,12 @@ pub struct Registry { } impl Registry { - pub fn set(&mut self, name: impl ToString, to: ToFlakeRef) { + pub fn set(&mut self, name: impl ToString, to: FlakeRef) { let entry = RegistryEntry { - from: FromFlakeRef::Indirect(IndirectFlake { + from: IndirectRef { id: name.to_string(), - }), + attributes: Default::default(), + }, to, exact: None, }; @@ -55,8 +54,8 @@ impl Default for Version { #[skip_serializing_none] #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] struct RegistryEntry { - from: FromFlakeRef, // TODO merge into single flakeRef type @notgne2? - to: ToFlakeRef, + from: IndirectRef, // TODO merge into single flakeRef type @notgne2? + to: FlakeRef, exact: Option, } @@ -76,7 +75,7 @@ impl PartialOrd for RegistryEntry { #[serde(tag = "type")] #[serde(rename_all = "lowercase")] enum FromFlakeRef { - Indirect(IndirectFlake), + Indirect(IndirectRef), } #[cfg(test)]