From ea79c0c4984a5d68ba0a3d4386a2a4ef9dd1122a Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Sat, 9 Sep 2023 20:53:44 +0200 Subject: [PATCH 01/11] Correct usage of cargo-get in .gitlab-ci snippet --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7dcf5f5..6868369 100644 --- a/README.md +++ b/README.md @@ -34,13 +34,13 @@ To release your package from CI, add a new pipeline step: ```yaml release-crate: - image: rust:1.62 + image: rust:latest stage: deploy only: # release when a tag is pushed - tags before_script: - cargo install cargo-get - - export CRATE_NAME=$(cargo get --name) CRATE_VERSION=$(cargo get version) + - export CRATE_NAME=$(cargo-get package.name) CRATE_VERSION=$(cargo-get package.version) - export CRATE_FILE=${CRATE_NAME}-${CRATE_VERSION}.crate script: - cargo package From c006cb77b9acc5506bd693b03d9f70bcdb6cad5c Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Mon, 11 Sep 2023 00:39:18 +0200 Subject: [PATCH 02/11] Add note reg. incompatibility of cargo and thrussh --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6868369..6909b6f 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,14 @@ Users are identified by their SSH keys from GitLab when connecting to the server To publish run `cargo package` and push the resulting `.crate` file to the GitLab package repository with a semver-compatible version string, to consume the package configure your `.cargo/config.toml` and `Cargo.toml` accordingly. +At time of writing, `libssh2`, which `cargo` implicitly uses for communicating with the registry by SSH, is incompatible with rust's `thrussh`, due to non-overlapping ciphers. Hence, activating `net.git-fetch-with-cli` is necessary. + ```toml # .cargo/config.toml [registries] -my-gitlab-project = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group/my-gitlab-project" } +my-gitlab-project = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group/my-gitlab-project/" } +[net] +git-fetch-with-cli = true # Cargo.toml [dependencies] From 1306f8633f18230bded169ecf48cb0058d409da3 Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Sun, 17 Sep 2023 17:13:44 +0200 Subject: [PATCH 03/11] README: Add .ssh/config for personal token usage --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6909b6f..ff1248c 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,10 @@ git-fetch-with-cli = true # Cargo.toml [dependencies] my-crate = { version = "0.1", registry = "my-gitlab-project" } + +# .ssh/config (only if authentication by personal token is requires) +Host gitlab-cargo-shim.local + User personal-token: ``` In your CI build, setup a `before_script` step to replace the connection string with one containing the CI token: From f535d9bf425a5a24d7ff36a68f95c9b9f64a76e7 Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Sun, 17 Sep 2023 17:15:40 +0200 Subject: [PATCH 04/11] README: Newer cargo versions do not allow URL-embedded passwords --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ff1248c..f85b047 100644 --- a/README.md +++ b/README.md @@ -62,4 +62,4 @@ It's that easy. Go forth and enjoy your newfound quality of life improvements, R [gitlab-package-registry]: https://docs.gitlab.com/ee/user/packages/package_registry/index.html [imp-token]: https://docs.gitlab.com/ee/api/index.html#impersonation-tokens [envvar]: https://doc.rust-lang.org/cargo/reference/registries.html#using-an-alternate-registry -[example-configuration]: https://github.com/w4/gitlab-cargo-shim/blob/main/config.toml \ No newline at end of file +[example-configuration]: https://github.com/w4/gitlab-cargo-shim/blob/main/config.toml From 58a56d1806017a9d0bf652d826c35d0f69bd05c2 Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Mon, 11 Sep 2023 00:53:47 +0200 Subject: [PATCH 05/11] Refactor: Add function to build client with token --- src/providers/gitlab.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index 3faf752..68e3296 100644 --- a/src/providers/gitlab.rs +++ b/src/providers/gitlab.rs @@ -11,6 +11,7 @@ use std::{borrow::Cow, sync::Arc}; use time::{Duration, OffsetDateTime}; use tracing::{info_span, instrument, Instrument}; use url::Url; +use std::str::FromStr; pub struct Gitlab { client: reqwest::Client, @@ -46,6 +47,19 @@ impl Gitlab { ssl_cert, }) } + + pub fn build_client_with_token(&self, token_field: &str, token: &str) -> anyhow::Result { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::HeaderName::from_str(token_field)?, + header::HeaderValue::from_str(token)?, + ); + let mut client_builder = reqwest::ClientBuilder::new().default_headers(headers); + if let Some(cert) = &self.ssl_cert { + client_builder = client_builder.add_root_certificate(cert.clone()); + } + Ok(client_builder.build()?) + } } #[async_trait] @@ -63,11 +77,7 @@ impl super::UserProvider for Gitlab { if username == "gitlab-ci-token" { // we're purposely not using `self.client` here as we don't // want to use our admin token for this request but still want to use any ssl cert provided. - let mut client_builder = reqwest::Client::builder(); - if let Some(cert) = &self.ssl_cert { - client_builder = client_builder.add_root_certificate(cert.clone()); - } - let client = client_builder.build(); + let client = self.build_client_with_token("JOB-TOKEN", password); let res: GitlabJobResponse = handle_error( client? .get(self.base_url.join("job/")?) From 5283255f36cb7a8e80dc7bc3b9659d0771b8b271 Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Mon, 11 Sep 2023 00:55:27 +0200 Subject: [PATCH 06/11] Allow authentication with personal token --- src/main.rs | 2 +- src/providers/gitlab.rs | 50 +++++++++++++++++++++++++++-------------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/main.rs b/src/main.rs index 70b99d6..be3aa20 100644 --- a/src/main.rs +++ b/src/main.rs @@ -402,7 +402,7 @@ impl thrussh::server: info!( "Successfully authenticated for GitLab user `{}` by {}", &user.username, - if by_ssh_key { "SSH Key" } else { "Build Token" }, + if by_ssh_key { "SSH Key" } else { "Build or Personal Token" }, ); self.user = Some(Arc::new(user)); self.finished_auth(Auth::Accept).await diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index 68e3296..7126842 100644 --- a/src/providers/gitlab.rs +++ b/src/providers/gitlab.rs @@ -74,25 +74,41 @@ impl super::UserProvider for Gitlab { return Ok(None); }; - if username == "gitlab-ci-token" { + if username == "gitlab-ci-token" || username == "personal-token" { // we're purposely not using `self.client` here as we don't // want to use our admin token for this request but still want to use any ssl cert provided. - let client = self.build_client_with_token("JOB-TOKEN", password); - let res: GitlabJobResponse = handle_error( - client? - .get(self.base_url.join("job/")?) - .header("JOB-TOKEN", password) - .send() - .await?, - ) - .await? - .json() - .await?; - - Ok(Some(User { - id: res.user.id, - username: res.user.username, - })) + let client = self.build_client_with_token(if username == "gitlab-ci-token" { "JOB-TOKEN" } else { "PRIVATE-TOKEN" }, password); + if username == "gitlab-ci-token" { + let res: GitlabJobResponse = handle_error( + client? + .get(self.base_url.join("job/")?) + .send() + .await?, + ) + .await? + .json() + .await?; + + Ok(Some(User { + id: res.user.id, + username: res.user.username, + })) + } else { + let res: GitlabUserResponse = handle_error( + client? + .get(self.base_url.join("user/")?) + .send() + .await?, + ) + .await? + .json() + .await?; + + Ok(Some(User { + id: res.id, + username: res.username, + })) + } } else { Ok(None) } From c55a2bfd71f1a46feb3c92e2e70e9c162c53fe7f Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Mon, 11 Sep 2023 00:57:37 +0200 Subject: [PATCH 07/11] Keep personal token in User --- src/providers/gitlab.rs | 3 +++ src/providers/mod.rs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index 7126842..cf2035a 100644 --- a/src/providers/gitlab.rs +++ b/src/providers/gitlab.rs @@ -92,6 +92,7 @@ impl super::UserProvider for Gitlab { Ok(Some(User { id: res.user.id, username: res.user.username, + ..Default::default() })) } else { let res: GitlabUserResponse = handle_error( @@ -107,6 +108,7 @@ impl super::UserProvider for Gitlab { Ok(Some(User { id: res.id, username: res.username, + token: Some(password.to_string()), })) } } else { @@ -127,6 +129,7 @@ impl super::UserProvider for Gitlab { Ok(res.user.map(|u| User { id: u.id, username: u.username, + ..Default::default() })) } diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 031e63e..b658838 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -36,10 +36,11 @@ pub trait PackageProvider { fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result; } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct User { pub id: u64, pub username: String, + pub token: Option, } pub type ReleaseName = Arc; From 0c44af358214bad858d0c9619e54b5a74a13c61d Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Mon, 11 Sep 2023 01:00:57 +0200 Subject: [PATCH 08/11] Optionally use personal token for reading packages --- src/main.rs | 2 +- src/providers/gitlab.rs | 22 ++++++++++++++++++---- src/providers/mod.rs | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index be3aa20..3c9a9f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -241,7 +241,7 @@ impl Handler { // fetch metadata from the provider let metadata = Arc::clone(&self.gitlab) - .fetch_metadata_for_release(path, crate_version) + .fetch_metadata_for_release(path, crate_version, self.user()?) .await?; // transform the `cargo metadata` output to the cargo index diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index cf2035a..80230a0 100644 --- a/src/providers/gitlab.rs +++ b/src/providers/gitlab.rs @@ -178,15 +178,23 @@ impl super::PackageProvider for Gitlab { query.append_pair("per_page", itoa::Buffer::new().format(100u16)); query.append_pair("pagination", "keyset"); query.append_pair("sort", "asc"); - query.append_pair("sudo", itoa::Buffer::new().format(do_as.id)); + if do_as.token.is_none() { + query.append_pair("sudo", itoa::Buffer::new().format(do_as.id)); + } } uri }); let futures = FuturesUnordered::new(); + let client = match &do_as.token { + None => self.client.clone(), + Some(token) => self.build_client_with_token("PRIVATE-TOKEN", token)? + }; + let client = Arc::new(client); + while let Some(uri) = next_uri.take() { - let res = handle_error(self.client.get(uri).send().await?).await?; + let res = handle_error(client.get(uri).send().await?).await?; if let Some(link_header) = res.headers().get(header::LINK) { let mut link_header = parse_link_header::parse_with_rel(link_header.to_str()?)?; @@ -200,6 +208,7 @@ impl super::PackageProvider for Gitlab { for release in res { let this = Arc::clone(&self); + let client = Arc::clone(&client); futures.push(tokio::spawn( async move { @@ -218,7 +227,7 @@ impl super::PackageProvider for Gitlab { }); let package_files: Vec = handle_error( - this.client + client .get(format!( "{}/projects/{}/packages/{}/package_files", this.base_url, @@ -268,10 +277,15 @@ impl super::PackageProvider for Gitlab { &self, path: &Self::CratePath, version: &str, + do_as: &User, ) -> anyhow::Result { let uri = self.base_url.join(&path.metadata_uri(version))?; + let client = match &do_as.token { + None => self.client.clone(), + Some(token) => self.build_client_with_token("PRIVATE-TOKEN", token)? + }; - Ok(handle_error(self.client.get(uri).send().await?) + Ok(handle_error(client.get(uri).send().await?) .await? .json() .await?) diff --git a/src/providers/mod.rs b/src/providers/mod.rs index b658838..079e720 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -31,6 +31,7 @@ pub trait PackageProvider { &self, path: &Self::CratePath, version: &str, + do_as: &User, ) -> anyhow::Result; fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result; From e7f09603c219d47bb33eaa2afcedcab72e75c93d Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Mon, 11 Sep 2023 01:06:10 +0200 Subject: [PATCH 09/11] Optionally use personal token for DL urls --- src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 3c9a9f1..65c753a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -285,7 +285,10 @@ impl Handler { // fetch the impersonation token for the user we'll embed // the `dl` string. - let token = self.gitlab.fetch_token_for_user(self.user()?).await?; + let token = match &self.user()?.token { + None => self.gitlab.fetch_token_for_user(self.user()?).await?, + Some(token) => token.clone(), + }; // generate the config for the user, containing the download // url template from gitlab and the impersonation token embedded From 63031db3b27a0cee2fadedf0857426a73542513a Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Mon, 11 Sep 2023 01:05:15 +0200 Subject: [PATCH 10/11] README: Add usage of personal token authentication --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f85b047..f1b9c15 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ Say goodbye to your Git dependencies, `gitlab-cargo-shim` is a stateless SSH ser Access controls work like they do in GitLab, builds are scoped to users - if they don't have permission to the dependency they can't build it, it's that simple. -Users are identified by their SSH keys from GitLab when connecting to the server and an [impersonation token][imp-token] will be generated for that run in order to pull available versions. Builds will insert their token as a username to the SSH server and the shim will use that to call the GitLab API. +Users are either identified by their SSH keys from GitLab when connecting to the server or by an Gitlab personal-token. If no token is given, an [impersonation token][imp-token] will be generated for that run in order to pull available versions. Doing so requires ad admin personal token. -To publish run `cargo package` and push the resulting `.crate` file to the GitLab package repository with a semver-compatible version string, to consume the package configure your `.cargo/config.toml` and `Cargo.toml` accordingly. +To publish run `cargo package` and push the resulting `.crate` file to the GitLab package repository with a semver-compatible version string, to consume the package configure your `.cargo/config.toml`, `Cargo.toml` and, optionally, `.ssh/config` accordingly. At time of writing, `libssh2`, which `cargo` implicitly uses for communicating with the registry by SSH, is incompatible with rust's `thrussh`, due to non-overlapping ciphers. Hence, activating `net.git-fetch-with-cli` is necessary. @@ -22,7 +22,8 @@ git-fetch-with-cli = true # Cargo.toml [dependencies] my-crate = { version = "0.1", registry = "my-gitlab-project" } - +``` +```ssh-config # .ssh/config (only if authentication by personal token is requires) Host gitlab-cargo-shim.local User personal-token: From 273d5f1161b2f82eb7a089f3e4d6a7631699ae08 Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Mon, 11 Sep 2023 01:03:45 +0200 Subject: [PATCH 11/11] Consider only packages with type generic --- src/providers/gitlab.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index 80230a0..9751442 100644 --- a/src/providers/gitlab.rs +++ b/src/providers/gitlab.rs @@ -204,7 +204,11 @@ impl super::PackageProvider for Gitlab { } } - let res: Vec = res.json().await?; + let res: Vec = res.json::>() + .await? + .into_iter() + .filter(|release| release.package_type == "generic") + .collect(); for release in res { let this = Arc::clone(&self); @@ -358,6 +362,7 @@ pub struct GitlabPackageResponse { pub id: u64, pub name: String, pub version: String, + pub package_type: String, #[serde(rename = "_links")] pub links: GitlabPackageLinksResponse, }