From bc7b330c9424515e36e7bb842a304a06f2e5fca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Thu, 6 Jun 2024 17:14:41 -0700 Subject: [PATCH 01/70] Version bump --- CHANGELOG.md | 4 ++++ lib/azure_blob/version.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c697f1e..090a92f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## [Unreleased] +## [0.4.2] 2024-06-06 + +Documentation + ## [0.4.1] 2024-05-27 First working release. diff --git a/lib/azure_blob/version.rb b/lib/azure_blob/version.rb index 5f0e2ef..92cd35d 100644 --- a/lib/azure_blob/version.rb +++ b/lib/azure_blob/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module AzureBlob - VERSION = "0.4.1" + VERSION = "0.4.2" end From 29a99a2476c2dd981f04d76431801ed9e229295d Mon Sep 17 00:00:00 2001 From: Dan Corneanu Date: Fri, 7 Jun 2024 18:04:34 +1200 Subject: [PATCH 02/70] Make Signer#sign private --- Gemfile.lock | 6 +++++- README.md | 11 +++++++++++ lib/azure_blob/signer.rb | 4 ++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 60cb58c..243f485 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -97,7 +97,7 @@ GIT PATH remote: . specs: - azure-blob (0.4.1) + azure-blob (0.4.2) rexml GEM @@ -152,6 +152,8 @@ GEM net-smtp (0.5.0) net-protocol nio4r (2.7.3) + nokogiri (1.16.5-arm64-darwin) + racc (~> 1.4) nokogiri (1.16.5-x86_64-linux) racc (~> 1.4) parallel (1.24.0) @@ -222,6 +224,7 @@ GEM ruby-progressbar (1.13.0) ruby-vips (2.2.1) ffi (~> 1.12) + sqlite3 (2.0.2-arm64-darwin) sqlite3 (2.0.2-x86_64-linux-gnu) stringio (3.1.0) strscan (3.1.0) @@ -238,6 +241,7 @@ GEM zeitwerk (2.6.14) PLATFORMS + arm64-darwin-23 x86_64-linux DEPENDENCIES diff --git a/README.md b/README.md index cf6fca1..997e6b9 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,17 @@ To migrate from azure-storage-blob to azure-blob: 3. Change the `AzureStorage` service to `AzureBlob` in your Active Storage config (`config/storage.yml`) 4. Restart or deploy the app. +## Authenricate using Managed Identity + +Get an access token for Azure Storage from the the local Managed Identity endpoint. +```bash +curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fstorage.azure.com%2F' -H Metadata:true +``` + +Now use the access token to access Azure Storage. +```bash +curl 'https://.blob.core.windows.net//' -H "x-ms-version: 2017-11-09" -H "Authorization: Bearer " +``` ## Standalone diff --git a/lib/azure_blob/signer.rb b/lib/azure_blob/signer.rb index 5921278..6f8bd15 100644 --- a/lib/azure_blob/signer.rb +++ b/lib/azure_blob/signer.rb @@ -71,12 +71,12 @@ def sas_token(uri, options = {}) URI.encode_www_form(**query) end + private + def sign(body) Base64.strict_encode64(OpenSSL::HMAC.digest("sha256", access_key, body)) end - private - def sanitize_headers(headers) headers = headers.dup headers[:"Content-Length"] = nil if headers[:"Content-Length"].to_i == 0 From f9360b985473dd36d1cf409c8f92c2e1b4dd5555 Mon Sep 17 00:00:00 2001 From: Dan Corneanu Date: Mon, 10 Jun 2024 12:15:04 +1200 Subject: [PATCH 03/70] Add support for additional MSI config options --- README.md | 7 +++++- Rakefile | 4 ++-- .../service/azure_blob_service.rb | 24 +++++++++++++++++-- lib/azure_blob/client.rb | 4 ++-- test/test_client.rb | 2 +- 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 997e6b9..3e50460 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,11 @@ To migrate from azure-storage-blob to azure-blob: ## Authenricate using Managed Identity +As of 04/04/2018, there are 2 supported ways to get MSI Token. + +- Using the extension installed locally and accessing http://localhost:50342/oauth2/token to get the MSI Token +- Accessing the http://169.254.169.254/metadata/identity/oauth2/token to get the MSI Token (default) + Get an access token for Azure Storage from the the local Managed Identity endpoint. ```bash curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fstorage.azure.com%2F' -H Metadata:true @@ -31,7 +36,7 @@ Instantiate a client with your account name, an access key and the container nam ```ruby client = AzureBlob::Client.new( account_name: @account_name, - access_key: @access_key, + storage_access_key: @access_key, container: @container, ) diff --git a/Rakefile b/Rakefile index 56d2819..a4acbf9 100644 --- a/Rakefile +++ b/Rakefile @@ -11,12 +11,12 @@ task default: %i[test] task :flush_test_container do |t| AzureBlob::Client.new( account_name: ENV["AZURE_ACCOUNT_NAME"], - access_key: ENV["AZURE_ACCESS_KEY"], + storage_access_key: ENV["AZURE_ACCESS_KEY"], container: ENV["AZURE_PRIVATE_CONTAINER"], ).delete_prefix '' AzureBlob::Client.new( account_name: ENV["AZURE_ACCOUNT_NAME"], - access_key: ENV["AZURE_ACCESS_KEY"], + storage_access_key: ENV["AZURE_ACCESS_KEY"], container: ENV["AZURE_PUBLIC_CONTAINER"], ).delete_prefix '' end diff --git a/lib/active_storage/service/azure_blob_service.rb b/lib/active_storage/service/azure_blob_service.rb index 4eaf8ce..105280e 100644 --- a/lib/active_storage/service/azure_blob_service.rb +++ b/lib/active_storage/service/azure_blob_service.rb @@ -36,13 +36,33 @@ class Service::AzureBlobService < Service attr_reader :client, :container, :signer def initialize(storage_account_name:, storage_access_key:, container:, public: false, **options) + # Either use storage_access_key or msi authentication + access_key_options = options.slice( + :storage_access_key, + ) + + msi_options = options.slice( + :subscription_id, + :tenant_id, + :resource_group_name, + :msi_identity_uri, + ) + + rest_options = options.except( + :storage_access_key, + :subscription_id, + :tenant_id, + :resource_group_name, + :msi_identity_uri, + ) + @container = container @public = public @client = AzureBlob::Client.new( account_name: storage_account_name, - access_key: storage_access_key, container: container, - **options) + **access_key_options, + **rest_options) end def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, custom_metadata: {}, **) diff --git a/lib/azure_blob/client.rb b/lib/azure_blob/client.rb index 18f81f3..48350ab 100644 --- a/lib/azure_blob/client.rb +++ b/lib/azure_blob/client.rb @@ -12,10 +12,10 @@ module AzureBlob # AzureBlob Client class. You interact with the Azure Blob api # through an instance of this class. class Client - def initialize(account_name:, access_key:, container:) + def initialize(account_name:, storage_access_key:, container:) @account_name = account_name @container = container - @signer = Signer.new(account_name:, access_key:) + @signer = Signer.new(account_name:, access_key: storage_access_key) end # Create a blob of type block. Will automatically split the the blob in multiple block and send the blob in pieces (blocks) if the blob is too big. diff --git a/test/test_client.rb b/test/test_client.rb index 9bf42ff..34af7ab 100644 --- a/test/test_client.rb +++ b/test/test_client.rb @@ -14,7 +14,7 @@ def setup @container = ENV["AZURE_PRIVATE_CONTAINER"] @client = AzureBlob::Client.new( account_name: @account_name, - access_key: @access_key, + storage_access_key: @access_key, container: @container, ) @key = "test client##{name}" From 0f3d8dfcd9fe86faf9a8ffd5434ed3b02c41cf62 Mon Sep 17 00:00:00 2001 From: Dan Corneanu Date: Mon, 10 Jun 2024 12:15:18 +1200 Subject: [PATCH 04/70] Implement a MSI token provider --- .../service/azure_blob_service.rb | 2 +- lib/azure_blob.rb | 1 + lib/azure_blob/auth.rb | 9 +++ lib/azure_blob/auth/token_credentials.rb | 6 ++ lib/azure_blob/auth/token_provider.rb | 75 +++++++++++++++++++ 5 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 lib/azure_blob/auth.rb create mode 100644 lib/azure_blob/auth/token_credentials.rb create mode 100644 lib/azure_blob/auth/token_provider.rb diff --git a/lib/active_storage/service/azure_blob_service.rb b/lib/active_storage/service/azure_blob_service.rb index 105280e..f7e3164 100644 --- a/lib/active_storage/service/azure_blob_service.rb +++ b/lib/active_storage/service/azure_blob_service.rb @@ -35,7 +35,7 @@ module ActiveStorage class Service::AzureBlobService < Service attr_reader :client, :container, :signer - def initialize(storage_account_name:, storage_access_key:, container:, public: false, **options) + def initialize(storage_account_name:, container:, public: false, **options) # Either use storage_access_key or msi authentication access_key_options = options.slice( :storage_access_key, diff --git a/lib/azure_blob.rb b/lib/azure_blob.rb index 34e3813..6649241 100644 --- a/lib/azure_blob.rb +++ b/lib/azure_blob.rb @@ -4,6 +4,7 @@ require_relative "azure_blob/client" require_relative "azure_blob/const" require_relative "azure_blob/errors" +require_relative "azure_blob/auth" module AzureBlob end diff --git a/lib/azure_blob/auth.rb b/lib/azure_blob/auth.rb new file mode 100644 index 0000000..ea6ea6b --- /dev/null +++ b/lib/azure_blob/auth.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require_relative "auth/token_credentials" +require_relative "auth/token_provider" + +module AzureBlob + module Auth + end +end \ No newline at end of file diff --git a/lib/azure_blob/auth/token_credentials.rb b/lib/azure_blob/auth/token_credentials.rb new file mode 100644 index 0000000..b0eefde --- /dev/null +++ b/lib/azure_blob/auth/token_credentials.rb @@ -0,0 +1,6 @@ +module AzureBlob + module Auth + class TokenCredentials + end + end +end \ No newline at end of file diff --git a/lib/azure_blob/auth/token_provider.rb b/lib/azure_blob/auth/token_provider.rb new file mode 100644 index 0000000..93a998d --- /dev/null +++ b/lib/azure_blob/auth/token_provider.rb @@ -0,0 +1,75 @@ +# +# As of 04/04/2018, there are 2 supported ways to get MSI Token. +# +# - Using the extension installed locally and accessing +# http://localhost:50342/oauth2/token to get the MSI Token +# - Accessing the http://169.254.169.254/metadata/identity/oauth2/token to get +# the MSI Token (default) +# +# Find further details here +# https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/tutorial-linux-vm-access-storage +# +# [How to use managed identities for Azure resources on an Azure VM to acquire an access token](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token) +# +# Azure Instance Metadata Service (IMDS) endpoint +# http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/ + +require "net/http" + +# +# Class that provides access to authentication token via Managed Service +# Identity. +# +module AzureBlob + module Auth + class TokenProvider + + RESOURCE_URI_STORAGE = 'https://storage.azure.com/' + API_VERSION = '2018-02-01' + + attr_reader :token + attr_reader :token_expires_on + + attr_reader :expiration_threshold + attr_reader :msi_identity_uri + + def initialize(msi_identity_uri:, resource_uri:, expiration_threshold: 10.minutes) + @msi_identity_uri = URI.parse(msi_identity_uri) + params = { + :'api-version' => AzureBlob::Auth::TokenProvider::API_VERSION, + :resource => resource_uri, + } + @msi_identity_uri.query = URI.encode_www_form(params) + @expiration_threshold = expiration_threshold + end + + def token_expired? + @token.nil? || Time.now >= (@token_expires_on + expiration_threshold) + end + + def token + if(self.token_expired?) + self.get_new_token() + end + @token + end + + private + + def get_new_token + # curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://storage.azure.com/' -H Metadata:true + res = Net::HTTP.get_response(msi_identity_uri, {'Metadata' => 'true'}) + json_res = JSON.parse(res.body) + + @token = json_res['access_token'] + # The number of seconds from "1970-01-01T0:0:0Z UTC" (corresponds to the token's exp claim). + @token_expires_on = Time.at(json_res['expires_on'].to_i) + end + end + end +end + +# subscription_id = ENV['AZURE_SUBSCRIPTION_ID'] +# tenant_id = ENV['AZURE_TENANT_ID'] +# resource_group_name = ENV['RESOURCE_GROUP_NAME'] +# msi_identity_uri = ENV['MSI_IDENTITY_URI'] || 'http://169.254.169.254/metadata/identity/oauth2/token' \ No newline at end of file From ecbdff960d9ed99f25f7476a4f5c273a54910ba6 Mon Sep 17 00:00:00 2001 From: Dan Corneanu Date: Mon, 10 Jun 2024 18:11:20 +1200 Subject: [PATCH 05/70] Refactor to extract signer config params out of Client --- README.md | 3 +- Rakefile | 6 ++- .../service/azure_blob_service.rb | 37 ++++++++----------- lib/azure_blob/auth.rb | 3 +- lib/azure_blob/auth/error.rb | 5 +++ ...oken_provider.rb => msi_token_provider.rb} | 31 +++++++++++----- lib/azure_blob/auth/token_credentials.rb | 6 --- lib/azure_blob/client.rb | 5 +-- lib/azure_blob/entra_id_signer.rb | 16 ++++++++ .../{signer.rb => shared_key_signer.rb} | 12 +++++- test/test_client.rb | 4 +- 11 files changed, 82 insertions(+), 46 deletions(-) create mode 100644 lib/azure_blob/auth/error.rb rename lib/azure_blob/auth/{token_provider.rb => msi_token_provider.rb} (67%) delete mode 100644 lib/azure_blob/auth/token_credentials.rb create mode 100644 lib/azure_blob/entra_id_signer.rb rename lib/azure_blob/{signer.rb => shared_key_signer.rb} (86%) diff --git a/README.md b/README.md index 3e50460..58a3a7e 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,11 @@ curl 'https://.blob.core.windows.net// AzureBlob::Auth::TokenProvider::API_VERSION, + :'api-version' => AzureBlob::Auth::MsiTokenProvider::IMDS_API_VERSION, :resource => resource_uri, } @msi_identity_uri.query = URI.encode_www_form(params) @@ -49,17 +55,24 @@ def token_expired? def token if(self.token_expired?) - self.get_new_token() + self.get_new_token_from_imds() end @token end private - def get_new_token + def get_new_token_from_imds # curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://storage.azure.com/' -H Metadata:true - res = Net::HTTP.get_response(msi_identity_uri, {'Metadata' => 'true'}) - json_res = JSON.parse(res.body) + response = Net::HTTP.get_response(msi_identity_uri, {'Metadata' => 'true'}) + + # TODO implement some retry strategies as per the documentation. + # https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling + unless response.is_a?(Net::HTTPSuccess) + raise AzureBlob::Auth::Error.new(response.body) + end + + json_res = JSON.parse(response.body) @token = json_res['access_token'] # The number of seconds from "1970-01-01T0:0:0Z UTC" (corresponds to the token's exp claim). diff --git a/lib/azure_blob/auth/token_credentials.rb b/lib/azure_blob/auth/token_credentials.rb deleted file mode 100644 index b0eefde..0000000 --- a/lib/azure_blob/auth/token_credentials.rb +++ /dev/null @@ -1,6 +0,0 @@ -module AzureBlob - module Auth - class TokenCredentials - end - end -end \ No newline at end of file diff --git a/lib/azure_blob/client.rb b/lib/azure_blob/client.rb index 48350ab..2710559 100644 --- a/lib/azure_blob/client.rb +++ b/lib/azure_blob/client.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require_relative "signer" require_relative "block_list" require_relative "blob_list" require_relative "blob" @@ -12,10 +11,10 @@ module AzureBlob # AzureBlob Client class. You interact with the Azure Blob api # through an instance of this class. class Client - def initialize(account_name:, storage_access_key:, container:) + def initialize(account_name:, container:, signer:) @account_name = account_name @container = container - @signer = Signer.new(account_name:, access_key: storage_access_key) + @signer = signer end # Create a blob of type block. Will automatically split the the blob in multiple block and send the blob in pieces (blocks) if the blob is too big. diff --git a/lib/azure_blob/entra_id_signer.rb b/lib/azure_blob/entra_id_signer.rb new file mode 100644 index 0000000..a8569e5 --- /dev/null +++ b/lib/azure_blob/entra_id_signer.rb @@ -0,0 +1,16 @@ +module AzureBlob + class EntraIdSigner + attr_reader :token_provider + + def initialize(token_provider) + @token_provider = token_provider + end + + def authorization_header(uri:, verb:, headers: {}) + "Bearer #{token_provider.token}" + end + + def sas_token(uri, options = {}) + end + end +end \ No newline at end of file diff --git a/lib/azure_blob/signer.rb b/lib/azure_blob/shared_key_signer.rb similarity index 86% rename from lib/azure_blob/signer.rb rename to lib/azure_blob/shared_key_signer.rb index 6f8bd15..ce2bfb2 100644 --- a/lib/azure_blob/signer.rb +++ b/lib/azure_blob/shared_key_signer.rb @@ -6,7 +6,12 @@ require_relative "canonicalized_resource" module AzureBlob - class Signer # :nodoc: + # + # This implementatio uses a Shared Key to + # - generate the authorisation header + # - generate SAS for download and upload URLs + # + class SharedKeySigner def initialize(account_name:, access_key:) @account_name = account_name @access_key = Base64.decode64(access_key) @@ -38,6 +43,11 @@ def authorization_header(uri:, verb:, headers: {}) "SharedKey #{account_name}:#{sign(to_sign)}" end + # + # Generate a Shared Access Signature. + # See https://learn.microsoft.com/en-us/rest/api/storageservices/create-service-sas + # See https://learn.microsoft.com/en-us/rest/api/storageservices/delegate-access-with-shared-access-signature + # def sas_token(uri, options = {}) to_sign = [ options[:permissions], diff --git a/test/test_client.rb b/test/test_client.rb index 34af7ab..238de4a 100644 --- a/test/test_client.rb +++ b/test/test_client.rb @@ -12,10 +12,12 @@ def setup @account_name = ENV["AZURE_ACCOUNT_NAME"] @access_key = ENV["AZURE_ACCESS_KEY"] @container = ENV["AZURE_PRIVATE_CONTAINER"] + + signer = AzureBlob::SharedKeySigner.new(account_name: @account_name, access_key: @access_key) @client = AzureBlob::Client.new( account_name: @account_name, - storage_access_key: @access_key, container: @container, + signer: signer ) @key = "test client##{name}" @content = "Some random content #{Random.rand(200)}" From 29411039d4ee44c3ef38242af5f8d4a774545404 Mon Sep 17 00:00:00 2001 From: Dan Corneanu Date: Tue, 11 Jun 2024 10:53:53 +1200 Subject: [PATCH 06/70] Implement EntraIdSigner#sas_token --- README.md | 2 +- .../service/azure_blob_service.rb | 3 +- lib/azure_blob/entra_id_signer.rb | 154 +++++++++++++++++- lib/azure_blob/http.rb | 9 + lib/azure_blob/shared_key_signer.rb | 3 +- 5 files changed, 167 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 58a3a7e..e7a8ee3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ To migrate from azure-storage-blob to azure-blob: 3. Change the `AzureStorage` service to `AzureBlob` in your Active Storage config (`config/storage.yml`) 4. Restart or deploy the app. -## Authenricate using Managed Identity +## Authenticate using Managed Identity As of 04/04/2018, there are 2 supported ways to get MSI Token. diff --git a/lib/active_storage/service/azure_blob_service.rb b/lib/active_storage/service/azure_blob_service.rb index c204639..6d0fbe3 100644 --- a/lib/active_storage/service/azure_blob_service.rb +++ b/lib/active_storage/service/azure_blob_service.rb @@ -50,7 +50,8 @@ def initialize(storage_account_name:, container:, public: false, **options) AzureBlob::EntraIdSigner.new( AzureBlob::Auth::MsiTokenProvider.new( resource_uri: AzureBlob::Auth::MsiTokenProvider::RESOURCE_URI_STORAGE - ) + ), + account_name: storage_account_name ) @client = AzureBlob::Client.new( diff --git a/lib/azure_blob/entra_id_signer.rb b/lib/azure_blob/entra_id_signer.rb index a8569e5..5bdf4d8 100644 --- a/lib/azure_blob/entra_id_signer.rb +++ b/lib/azure_blob/entra_id_signer.rb @@ -1,16 +1,168 @@ +require "base64" +require "openssl" +require "net/http" +require "rexml/document" + +require_relative "canonicalized_resource" + module AzureBlob + # + # This implementatio uses Microsoft Entra ID to + # - generate the authorisation header + # See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-azure-active-directory + # - generate SAS for download and upload URLs + # class EntraIdSigner attr_reader :token_provider + attr_reader :account_name - def initialize(token_provider) + def initialize(token_provider, account_name:) @token_provider = token_provider + @account_name = account_name end def authorization_header(uri:, verb:, headers: {}) "Bearer #{token_provider.token}" end + # + # Generate a user delegation Shared Access Signature. + # See https://learn.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas + # See https://learn.microsoft.com/en-us/rest/api/storageservices/delegate-access-with-shared-access-signature + # See https://learn.microsoft.com/en-us/rest/api/storageservices/get-user-delegation-key + # def sas_token(uri, options = {}) + # 1. Acquire an OAuth 2.0 token from Microsoft Entra ID. + # 2. Use the token to request the user delegation key by calling the Get User Delegation Key operation. + + # TODO: move the user delegation key handling into it's own + # UserDelegationKeyProvider and reuse the same key if not expired + user_delegation_key_uri = URI.parse( + "https://#{account_name}.blob.core.windows.net/?restype=service&comp=userdelegationkey" + ) + now = Time.now.utc + + key_start = now.iso8601 + key_expiry = (now + 7.hours).iso8601 + + content = <<-XML.squish + + + #{key_start} + #{key_expiry} + + XML + http = AzureBlob::Http.new(user_delegation_key_uri, signer: self) + response = http.post(content) + + # 3. Use the user delegation key to construct the SAS token with the appropriate fields. + doc = REXML::Document.new(response) + # + # + # String containing a GUID value + # String containing a GUID value + # String formatted as ISO date + # String formatted as ISO date + # b + # String specifying REST api version to use to create the user delegation key + # String containing the user delegation key + # + signed_oid = doc.get_elements("/UserDelegationKey/SignedOid").first.get_text.to_s + signed_tid = doc.get_elements("/UserDelegationKey/SignedTid").first.get_text.to_s + signed_start = doc.get_elements("/UserDelegationKey/SignedStart").first.get_text.to_s + signed_expiry = doc.get_elements("/UserDelegationKey/SignedExpiry").first.get_text.to_s + signed_service = doc.get_elements("/UserDelegationKey/SignedService").first.get_text.to_s + signed_version = doc.get_elements("/UserDelegationKey/SignedVersion").first.get_text.to_s + user_delegation_key = Base64.decode64(doc.get_elements("/UserDelegationKey/Value").first.get_text.to_s) + + # :start and :expiry, if present, are already in iso8601 format + start = options[:start] || now.iso8601 + expiry = options[:expiry] || (now + 5.minutes).iso8601 + + canonicalized_resource = CanonicalizedResource.new(uri, account_name, url_safe: false, service_name: :blob) + to_sign = [ + options[:permissions], # signedPermissions + "\n" + + start, # signedStart + "\n" + + expiry, # signedExpiry + "\n" + + canonicalized_resource, # canonicalizedResource + "\n" + + signed_oid, # signedKeyObjectId + "\n" + + signed_tid, # signedKeyTenantId + "\n" + + signed_start, # signedKeyStart + "\n" + + signed_expiry, # signedKeyExpiry + "\n" + + signed_service, # signedKeyService + "\n" + + signed_version, # signedKeyVersion + "\n" + + nil, # signedAuthorizedUserObjectId + "\n" + + nil, # signedUnauthorizedUserObjectId + "\n" + + nil, # signedCorrelationId + "\n" + + options[:ip], # signedIP + "\n" + + options[:protocol], # signedProtocol + "\n" + + SAS::Version, # signedVersion + "\n" + + SAS::Resources::Blob, # signedResource + "\n" + + nil, # signedSnapshotTime + "\n" + + nil, # signedEncryptionScope + "\n" + + nil, # rscc + "\n" + + options[:content_disposition], # rscd + "\n" + + nil, # rsce + "\n" + + nil, # rscl + "\n" + + options[:content_type], # rsct + ].join("\n") + + query = { + SAS::Fields::Permissions => options[:permissions], + SAS::Fields::Start => start, + SAS::Fields::Expiry => expiry, + + SAS::Fields::SignedObjectId => signed_oid, + SAS::Fields::SignedTenantId => signed_tid, + SAS::Fields::SignedKeyStartTime => signed_start, + SAS::Fields::SignedKeyExpiryTime => signed_expiry, + SAS::Fields::SignedKeyService => signed_service, + SAS::Fields::Signedkeyversion => signed_version, + + + SAS::Fields::SignedIp => options[:ip], + SAS::Fields::SignedProtocol => options[:protocol], + SAS::Fields::Version => SAS::Version, + SAS::Fields::Resource => SAS::Resources::Blob, + + SAS::Fields::Disposition => options[:content_disposition], + SAS::Fields::Type => options[:content_type], + SAS::Fields::Signature => sign(to_sign, key: user_delegation_key), + + }.reject { |_, value| value.nil? } + + URI.encode_www_form(**query) + end + + private + + def sign(body, key:) + Base64.strict_encode64(OpenSSL::HMAC.digest("sha256", key, body)) + end + + module SAS # :nodoc: + Version = "2024-05-04" + module Fields # :nodoc: + Permissions = :sp + Version = :sv + Start = :st + Expiry = :se + Resource = :sr + Signature = :sig + Disposition = :rscd + Type = :rsct + SignedObjectId = :skoid + SignedTenantId = :sktid + SignedKeyStartTime = :skt + SignedKeyExpiryTime = :ske + SignedKeyService = :sks + Signedkeyversion = :skv + SignedIp = :sip + SignedProtocol = :spr + end + module Resources # :nodoc: + Blob = :b + end end end end \ No newline at end of file diff --git a/lib/azure_blob/http.rb b/lib/azure_blob/http.rb index 4f67f7f..01bfe7c 100644 --- a/lib/azure_blob/http.rb +++ b/lib/azure_blob/http.rb @@ -44,6 +44,15 @@ def put(content) true end + def post(content) + sign_request("POST") if signer + @response = http.start do |http| + http.post(uri, content, headers) + end + raise_error unless success? + response.body + end + def head sign_request("HEAD") if signer @response = http.start do |http| diff --git a/lib/azure_blob/shared_key_signer.rb b/lib/azure_blob/shared_key_signer.rb index ce2bfb2..40c922a 100644 --- a/lib/azure_blob/shared_key_signer.rb +++ b/lib/azure_blob/shared_key_signer.rb @@ -9,6 +9,7 @@ module AzureBlob # # This implementatio uses a Shared Key to # - generate the authorisation header + # See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key # - generate SAS for download and upload URLs # class SharedKeySigner @@ -44,7 +45,7 @@ def authorization_header(uri:, verb:, headers: {}) end # - # Generate a Shared Access Signature. + # Generate a service Shared Access Signature. # See https://learn.microsoft.com/en-us/rest/api/storageservices/create-service-sas # See https://learn.microsoft.com/en-us/rest/api/storageservices/delegate-access-with-shared-access-signature # From 2801fbb49142f1844bf595ad8ff15e31e4d3791e Mon Sep 17 00:00:00 2001 From: Dan Corneanu Date: Wed, 12 Jun 2024 18:26:23 +1200 Subject: [PATCH 07/70] Remove section on Managed Identity internals --- README.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/README.md b/README.md index e7a8ee3..a5e4500 100644 --- a/README.md +++ b/README.md @@ -12,23 +12,6 @@ To migrate from azure-storage-blob to azure-blob: 3. Change the `AzureStorage` service to `AzureBlob` in your Active Storage config (`config/storage.yml`) 4. Restart or deploy the app. -## Authenticate using Managed Identity - -As of 04/04/2018, there are 2 supported ways to get MSI Token. - -- Using the extension installed locally and accessing http://localhost:50342/oauth2/token to get the MSI Token -- Accessing the http://169.254.169.254/metadata/identity/oauth2/token to get the MSI Token (default) - -Get an access token for Azure Storage from the the local Managed Identity endpoint. -```bash -curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fstorage.azure.com%2F' -H Metadata:true -``` - -Now use the access token to access Azure Storage. -```bash -curl 'https://.blob.core.windows.net//' -H "x-ms-version: 2017-11-09" -H "Authorization: Bearer " -``` - ## Standalone Instantiate a client with your account name, an access key and the container name: From b8e2a86e7d891beef97d02761ba27af9222b4872 Mon Sep 17 00:00:00 2001 From: Dan Corneanu Date: Wed, 12 Jun 2024 18:41:38 +1200 Subject: [PATCH 08/70] Let the Client figure out which signer to instantiate --- CHANGELOG.md | 1 + README.md | 4 ++-- Rakefile | 6 ++--- .../service/azure_blob_service.rb | 23 ++++--------------- lib/azure_blob/client.rb | 14 +++++++++-- test/test_client.rb | 4 +--- 6 files changed, 22 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3319dba..85d7cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Documentation - Fix an issue with integrity check on multi block upload + ## [0.4.1] 2024-05-27 First working release. diff --git a/README.md b/README.md index a5e4500..cf6fca1 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,16 @@ To migrate from azure-storage-blob to azure-blob: 3. Change the `AzureStorage` service to `AzureBlob` in your Active Storage config (`config/storage.yml`) 4. Restart or deploy the app. + ## Standalone Instantiate a client with your account name, an access key and the container name: ```ruby -signer = AzureBlob::SharedKeySigner.new(account_name: @account_name, access_key: @access_key) client = AzureBlob::Client.new( account_name: @account_name, + access_key: @access_key, container: @container, - signer: signer ) path = "some/new/file" diff --git a/Rakefile b/Rakefile index 64c3094..56d2819 100644 --- a/Rakefile +++ b/Rakefile @@ -9,16 +9,14 @@ Minitest::TestTask.create task default: %i[test] task :flush_test_container do |t| - signer = AzureBlob::SharedKeySigner.new(account_name: ENV["AZURE_ACCOUNT_NAME"], access_key: ENV["AZURE_ACCESS_KEY"]) - AzureBlob::Client.new( account_name: ENV["AZURE_ACCOUNT_NAME"], + access_key: ENV["AZURE_ACCESS_KEY"], container: ENV["AZURE_PRIVATE_CONTAINER"], - signer: ).delete_prefix '' AzureBlob::Client.new( account_name: ENV["AZURE_ACCOUNT_NAME"], + access_key: ENV["AZURE_ACCESS_KEY"], container: ENV["AZURE_PUBLIC_CONTAINER"], - signer: ).delete_prefix '' end diff --git a/lib/active_storage/service/azure_blob_service.rb b/lib/active_storage/service/azure_blob_service.rb index 6d0fbe3..24d83f3 100644 --- a/lib/active_storage/service/azure_blob_service.rb +++ b/lib/active_storage/service/azure_blob_service.rb @@ -24,8 +24,6 @@ require "active_support/core_ext/numeric/bytes" require "active_storage/service" -require "azure_blob/shared_key_signer" -require "azure_blob/entra_id_signer" require "azure_blob/auth/msi_token_provider.rb" require "azure_blob" @@ -38,27 +36,14 @@ module ActiveStorage class Service::AzureBlobService < Service attr_reader :client, :container, :signer - def initialize(storage_account_name:, container:, public: false, **options) - # Either use storage_access_key or msi authentication - storage_access_key = options[:storage_access_key] - + def initialize(storage_account_name:, storage_access_key: nil, container:, public: false, **options) @container = container @public = public - - signer = storage_access_key.present? ? - AzureBlob::SharedKeySigner.new(account_name: storage_account_name, access_key: storage_access_key) : - AzureBlob::EntraIdSigner.new( - AzureBlob::Auth::MsiTokenProvider.new( - resource_uri: AzureBlob::Auth::MsiTokenProvider::RESOURCE_URI_STORAGE - ), - account_name: storage_account_name - ) - @client = AzureBlob::Client.new( account_name: storage_account_name, - container:, - signer: - ) + access_key: storage_access_key, + container: container, + **options) end def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, custom_metadata: {}, **) diff --git a/lib/azure_blob/client.rb b/lib/azure_blob/client.rb index 2710559..a5503f4 100644 --- a/lib/azure_blob/client.rb +++ b/lib/azure_blob/client.rb @@ -4,6 +4,8 @@ require_relative "blob_list" require_relative "blob" require_relative "http" +require_relative "shared_key_signer" +require_relative "entra_id_signer" require "time" require "base64" @@ -11,10 +13,18 @@ module AzureBlob # AzureBlob Client class. You interact with the Azure Blob api # through an instance of this class. class Client - def initialize(account_name:, container:, signer:) + def initialize(account_name:, access_key:, container:) @account_name = account_name @container = container - @signer = signer + + @signer = access_key.present? ? + AzureBlob::SharedKeySigner.new(account_name:, access_key:) : + AzureBlob::EntraIdSigner.new( + AzureBlob::Auth::MsiTokenProvider.new( + resource_uri: AzureBlob::Auth::MsiTokenProvider::RESOURCE_URI_STORAGE + ), + account_name: + ) end # Create a blob of type block. Will automatically split the the blob in multiple block and send the blob in pieces (blocks) if the blob is too big. diff --git a/test/test_client.rb b/test/test_client.rb index 238de4a..9bf42ff 100644 --- a/test/test_client.rb +++ b/test/test_client.rb @@ -12,12 +12,10 @@ def setup @account_name = ENV["AZURE_ACCOUNT_NAME"] @access_key = ENV["AZURE_ACCESS_KEY"] @container = ENV["AZURE_PRIVATE_CONTAINER"] - - signer = AzureBlob::SharedKeySigner.new(account_name: @account_name, access_key: @access_key) @client = AzureBlob::Client.new( account_name: @account_name, + access_key: @access_key, container: @container, - signer: signer ) @key = "test client##{name}" @content = "Some random content #{Random.rand(200)}" From e747b0626f962ecf3e501ce45f1de7e3f6a66643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Thu, 15 Aug 2024 16:02:20 -0700 Subject: [PATCH 09/70] Remove an active support reference --- lib/azure_blob/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/azure_blob/client.rb b/lib/azure_blob/client.rb index a5503f4..37b508b 100644 --- a/lib/azure_blob/client.rb +++ b/lib/azure_blob/client.rb @@ -17,7 +17,7 @@ def initialize(account_name:, access_key:, container:) @account_name = account_name @container = container - @signer = access_key.present? ? + @signer = !access_key.nil? && !access_key.empty? ? AzureBlob::SharedKeySigner.new(account_name:, access_key:) : AzureBlob::EntraIdSigner.new( AzureBlob::Auth::MsiTokenProvider.new( From cfa9f84e74227c82af1895efc6d816781b15e3c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Thu, 15 Aug 2024 16:03:52 -0700 Subject: [PATCH 10/70] Script to generate the devenv.local.nix file --- devenv.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/devenv.nix b/devenv.nix index dfd810c..323456e 100644 --- a/devenv.nix +++ b/devenv.nix @@ -9,6 +9,10 @@ ]; + scripts.generate-env-file.exec = '' + terraform output -raw devenv_local_nix > devenv.local.nix + ''; + languages.ruby.enable = true; languages.ruby.version = "3.1.5"; } From 70ca9cce279c2700d5fe620f6124500c61d7a8f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Thu, 15 Aug 2024 16:04:15 -0700 Subject: [PATCH 11/70] Script to rsync the working directory to the test vm --- devenv.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/devenv.nix b/devenv.nix index 323456e..3358362 100644 --- a/devenv.nix +++ b/devenv.nix @@ -8,6 +8,11 @@ azure-cli ]; + scripts.sync-vm.exec = '' + vm_username=$(terraform output --raw "vm_username") + vm_ip=$(terraform output --raw "vm_ip") + rsync -avx --progress --exclude .devenv . $vm_username@$vm_ip:azure-blob/ + ''; scripts.generate-env-file.exec = '' terraform output -raw devenv_local_nix > devenv.local.nix From f01826c674575559da046fb32c7826691824b198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Thu, 15 Aug 2024 16:04:43 -0700 Subject: [PATCH 12/70] Use the default ruby to avoid long rebuild on the vm --- devenv.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/devenv.nix b/devenv.nix index 3358362..5e706a8 100644 --- a/devenv.nix +++ b/devenv.nix @@ -6,6 +6,11 @@ libyaml terraform azure-cli + ruby + + glib + glibc + vips ]; scripts.sync-vm.exec = '' @@ -17,7 +22,4 @@ scripts.generate-env-file.exec = '' terraform output -raw devenv_local_nix > devenv.local.nix ''; - - languages.ruby.enable = true; - languages.ruby.version = "3.1.5"; } From 8a43c98b663b78793bbed30fb716567e70db9646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Thu, 15 Aug 2024 16:06:58 -0700 Subject: [PATCH 13/70] Add VM support --- input.tf | 35 +++++++++++++++++ main.tf | 110 ++++++++++++++++++++++++++++++++++++++++++++++-------- output.tf | 21 +++++++++++ 3 files changed, 150 insertions(+), 16 deletions(-) create mode 100644 output.tf diff --git a/input.tf b/input.tf index b1ce808..624ee1a 100644 --- a/input.tf +++ b/input.tf @@ -2,3 +2,38 @@ variable "location" { type = string default = "westus2" } + +variable "prefix" { + type = string + default = "azure-blob" +} + +variable "storage_account_name" { + type = string + default = "azureblobrubygemdev" +} + +variable "create_vm" { + type = bool + default = false +} + +variable "vm_size" { + type = string + default = "Standard_B2s" +} + +variable "vm_username" { + type = string + default = "azureblob" +} + +variable "vm_password" { + type = string + default = "azureblob" +} + +variable "ssh_key" { + type = string + default = "" +} diff --git a/main.tf b/main.tf index 3118d13..39399b0 100644 --- a/main.tf +++ b/main.tf @@ -11,8 +11,8 @@ provider "azurerm" { features {} } -resource "azurerm_resource_group" "rg" { - name = "azure-blob" +resource "azurerm_resource_group" "main" { + name = var.prefix location = var.location tags = { source = "Terraform" @@ -20,9 +20,9 @@ resource "azurerm_resource_group" "rg" { } resource "azurerm_storage_account" "main" { - name = "azureblobrubygemdev" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location + name = var.storage_account_name + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location account_tier = "Standard" account_replication_type = "LRS" @@ -43,16 +43,94 @@ resource "azurerm_storage_container" "public" { container_access_type = "blob" } -output "devenv_local_nix" { - sensitive = true - value = < Date: Thu, 15 Aug 2024 16:42:35 -0700 Subject: [PATCH 14/70] Use a azurerm_linux_virtual_machine block instead --- cloudinit.cfg | 10 ++++++ input.tf | 2 +- main.tf | 84 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 71 insertions(+), 25 deletions(-) create mode 100644 cloudinit.cfg diff --git a/cloudinit.cfg b/cloudinit.cfg new file mode 100644 index 0000000..30d25e2 --- /dev/null +++ b/cloudinit.cfg @@ -0,0 +1,10 @@ +# cloud-config +package_upgrade: true +packages: + - httpd + - glib2.0 +runcmd: + - "sh <(curl -L https://nixos.org/nix/install) --daemon --yes" + - "nix-env -iA devenv -f https://github.com/NixOS/nixpkgs/tarball/nixpkgs-unstable" + - 'echo "trusted-users = root ${vm_username}" >> /etc/nix/nix.conf' + - 'systemctl restart nix-daemon' diff --git a/input.tf b/input.tf index 624ee1a..85312c1 100644 --- a/input.tf +++ b/input.tf @@ -30,7 +30,7 @@ variable "vm_username" { variable "vm_password" { type = string - default = "azureblob" + default = "qwe123QWE!@#" } variable "ssh_key" { diff --git a/main.tf b/main.tf index 39399b0..384f00e 100644 --- a/main.tf +++ b/main.tf @@ -94,43 +94,79 @@ resource "azurerm_public_ip" "main" { } } - -resource "azurerm_virtual_machine" "main" { +resource "azurerm_linux_virtual_machine" "main" { count = var.create_vm ? 1 : 0 name = "${var.prefix}-vm" - location = azurerm_resource_group.main.location + computer_name = var.prefix resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location + size = var.vm_size + admin_username = var.vm_username + admin_password = var.vm_password + custom_data = base64encode(templatefile("./cloudinit.cfg", { vm_username = var.vm_username})) + disable_password_authentication = true network_interface_ids = [azurerm_network_interface.main[0].id] - vm_size = var.vm_size - delete_os_disk_on_termination = true - delete_data_disks_on_termination = true - storage_image_reference { + admin_ssh_key { + username = var.vm_username + public_key = var.ssh_key != "" ? var.ssh_key : file("~/.ssh/id_rsa.pub") + } + + source_image_reference { publisher = "Canonical" offer = "0001-com-ubuntu-server-jammy" sku = "22_04-lts" version = "latest" } - storage_os_disk { - name = "myosdisk1" - caching = "ReadWrite" - create_option = "FromImage" - managed_disk_type = "Standard_LRS" - } - os_profile { - computer_name = var.prefix - admin_username = var.vm_username - admin_password = var.vm_password - } - os_profile_linux_config { - disable_password_authentication = true - ssh_keys { - path = "/home/${var.vm_username}/.ssh/authorized_keys" - key_data = var.ssh_key != "" ? var.ssh_key : file("~/.ssh/id_rsa.pub") - } + + os_disk { + caching = "ReadWrite" + storage_account_type = "Standard_LRS" } tags = { source = "Terraform" } } + + +# resource "azurerm_virtual_machine" "main" { +# count = var.create_vm ? 1 : 0 +# name = "${var.prefix}-vm" +# location = azurerm_resource_group.main.location +# resource_group_name = azurerm_resource_group.main.name +# network_interface_ids = [azurerm_network_interface.main[0].id] +# vm_size = var.vm_size +# delete_os_disk_on_termination = true +# delete_data_disks_on_termination = true + +# storage_image_reference { +# publisher = "Canonical" +# offer = "0001-com-ubuntu-server-jammy" +# sku = "22_04-lts" +# version = "latest" +# } +# storage_os_disk { +# name = "myosdisk1" +# caching = "ReadWrite" +# create_option = "FromImage" +# managed_disk_type = "Standard_LRS" +# } +# os_profile { +# computer_name = var.prefix +# admin_username = var.vm_username +# admin_password = var.vm_password +# custom_data = base64encode(templatefile("./cloudinit.cfg", { vm_username = var.vm_username})) +# } +# os_profile_linux_config { +# disable_password_authentication = true +# ssh_keys { +# path = "/home/${var.vm_username}/.ssh/authorized_keys" +# key_data = var.ssh_key != "" ? var.ssh_key : file("~/.ssh/id_rsa.pub") +# } +# } + +# tags = { +# source = "Terraform" +# } +# } From e297175c9ee905a8e08fe7534ab38d56fad8f420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Thu, 15 Aug 2024 18:37:57 -0700 Subject: [PATCH 15/70] Remove deads code --- main.tf | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/main.tf b/main.tf index 384f00e..ad0cca3 100644 --- a/main.tf +++ b/main.tf @@ -128,45 +128,3 @@ resource "azurerm_linux_virtual_machine" "main" { source = "Terraform" } } - - -# resource "azurerm_virtual_machine" "main" { -# count = var.create_vm ? 1 : 0 -# name = "${var.prefix}-vm" -# location = azurerm_resource_group.main.location -# resource_group_name = azurerm_resource_group.main.name -# network_interface_ids = [azurerm_network_interface.main[0].id] -# vm_size = var.vm_size -# delete_os_disk_on_termination = true -# delete_data_disks_on_termination = true - -# storage_image_reference { -# publisher = "Canonical" -# offer = "0001-com-ubuntu-server-jammy" -# sku = "22_04-lts" -# version = "latest" -# } -# storage_os_disk { -# name = "myosdisk1" -# caching = "ReadWrite" -# create_option = "FromImage" -# managed_disk_type = "Standard_LRS" -# } -# os_profile { -# computer_name = var.prefix -# admin_username = var.vm_username -# admin_password = var.vm_password -# custom_data = base64encode(templatefile("./cloudinit.cfg", { vm_username = var.vm_username})) -# } -# os_profile_linux_config { -# disable_password_authentication = true -# ssh_keys { -# path = "/home/${var.vm_username}/.ssh/authorized_keys" -# key_data = var.ssh_key != "" ? var.ssh_key : file("~/.ssh/id_rsa.pub") -# } -# } - -# tags = { -# source = "Terraform" -# } -# } From d91c4cc0fffd00a2a2640a11ca63b41f0005d8f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Thu, 15 Aug 2024 18:38:12 -0700 Subject: [PATCH 16/70] Attach role to the vm --- main.tf | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/main.tf b/main.tf index ad0cca3..a5211bb 100644 --- a/main.tf +++ b/main.tf @@ -94,6 +94,25 @@ resource "azurerm_public_ip" "main" { } } +resource "azurerm_user_assigned_identity" "vm" { + location = azurerm_resource_group.main.location + name = "${var.prefix}-vm" + resource_group_name = azurerm_resource_group.main.name +} + +resource "azurerm_role_assignment" "vm-private" { + scope = azurerm_storage_container.private.resource_manager_id + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_user_assigned_identity.vm.principal_id +} + +resource "azurerm_role_assignment" "vm-public" { + scope = azurerm_storage_container.public.resource_manager_id + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_user_assigned_identity.vm.principal_id +} + + resource "azurerm_linux_virtual_machine" "main" { count = var.create_vm ? 1 : 0 name = "${var.prefix}-vm" @@ -107,6 +126,11 @@ resource "azurerm_linux_virtual_machine" "main" { disable_password_authentication = true network_interface_ids = [azurerm_network_interface.main[0].id] + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.vm.id] + } + admin_ssh_key { username = var.vm_username public_key = var.ssh_key != "" ? var.ssh_key : file("~/.ssh/id_rsa.pub") From 3ac644b3daaff058d2d3a9d7dfff84e28a511432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Thu, 15 Aug 2024 18:38:52 -0700 Subject: [PATCH 17/70] Exclude the terraform folder --- devenv.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devenv.nix b/devenv.nix index 5e706a8..12b1a7f 100644 --- a/devenv.nix +++ b/devenv.nix @@ -16,7 +16,7 @@ scripts.sync-vm.exec = '' vm_username=$(terraform output --raw "vm_username") vm_ip=$(terraform output --raw "vm_ip") - rsync -avx --progress --exclude .devenv . $vm_username@$vm_ip:azure-blob/ + rsync -avx --progress --exclude .devenv --exclude .terraform . $vm_username@$vm_ip:azure-blob/ ''; scripts.generate-env-file.exec = '' From 8830281aea889698bd79346360c4b851a4b50a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 18 Aug 2024 15:23:31 -0700 Subject: [PATCH 18/70] Remove glibc --- devenv.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/devenv.nix b/devenv.nix index 12b1a7f..dbc7127 100644 --- a/devenv.nix +++ b/devenv.nix @@ -7,9 +7,7 @@ terraform azure-cli ruby - glib - glibc vips ]; From 0ff29d149171c0819dd6aac98ec5fab231ecaea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 18 Aug 2024 15:24:00 -0700 Subject: [PATCH 19/70] Set LD_LIBRARY_PATH for FFI --- devenv.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/devenv.nix b/devenv.nix index dbc7127..4d4116e 100644 --- a/devenv.nix +++ b/devenv.nix @@ -1,6 +1,10 @@ -{ pkgs, ... }: +{ pkgs, config, ... }: { + env = { + LD_LIBRARY_PATH = "${config.devenv.profile}/lib"; + }; + packages = with pkgs; [ git libyaml From 71aaaeb668261ff72875f35f6acbdec4a7137736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 18 Aug 2024 15:25:03 -0700 Subject: [PATCH 20/70] Ignore some terraform files --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ba232bf..d723847 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,6 @@ devenv.local.nix /.terraform terraform.tfstate -terraform.tfstate.backup \ No newline at end of file +terraform.tfstate.backup +.terraform.tfstate.lock.info +*.tfvars \ No newline at end of file From 6dd25bf394cc8e7ba4e9161bd1a8118f0c90820c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 18 Aug 2024 18:35:03 -0700 Subject: [PATCH 21/70] Set the role on the account instead of the containers --- main.tf | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/main.tf b/main.tf index a5211bb..06e4cc8 100644 --- a/main.tf +++ b/main.tf @@ -100,19 +100,13 @@ resource "azurerm_user_assigned_identity" "vm" { resource_group_name = azurerm_resource_group.main.name } -resource "azurerm_role_assignment" "vm-private" { - scope = azurerm_storage_container.private.resource_manager_id - role_definition_name = "Storage Blob Data Contributor" - principal_id = azurerm_user_assigned_identity.vm.principal_id -} -resource "azurerm_role_assignment" "vm-public" { - scope = azurerm_storage_container.public.resource_manager_id +resource "azurerm_role_assignment" "vm" { + scope = azurerm_storage_account.main.id role_definition_name = "Storage Blob Data Contributor" principal_id = azurerm_user_assigned_identity.vm.principal_id } - resource "azurerm_linux_virtual_machine" "main" { count = var.create_vm ? 1 : 0 name = "${var.prefix}-vm" From a6e0e0dc6c900ba5267e791d831b0e9fb81d4f09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Fri, 23 Aug 2024 16:44:06 -0700 Subject: [PATCH 22/70] Remove unused require --- lib/active_storage/service/azure_blob_service.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/active_storage/service/azure_blob_service.rb b/lib/active_storage/service/azure_blob_service.rb index 24d83f3..bc56c42 100644 --- a/lib/active_storage/service/azure_blob_service.rb +++ b/lib/active_storage/service/azure_blob_service.rb @@ -24,7 +24,6 @@ require "active_support/core_ext/numeric/bytes" require "active_storage/service" -require "azure_blob/auth/msi_token_provider.rb" require "azure_blob" From b843a2d6d1bfad53e09c12eeb48e1835f406e3a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Fri, 23 Aug 2024 16:46:53 -0700 Subject: [PATCH 23/70] Remove Signer documentation (private API) --- lib/azure_blob/entra_id_signer.rb | 24 ++++++------------------ lib/azure_blob/shared_key_signer.rb | 13 +------------ 2 files changed, 7 insertions(+), 30 deletions(-) diff --git a/lib/azure_blob/entra_id_signer.rb b/lib/azure_blob/entra_id_signer.rb index 5bdf4d8..cc66f9e 100644 --- a/lib/azure_blob/entra_id_signer.rb +++ b/lib/azure_blob/entra_id_signer.rb @@ -6,13 +6,7 @@ require_relative "canonicalized_resource" module AzureBlob - # - # This implementatio uses Microsoft Entra ID to - # - generate the authorisation header - # See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-azure-active-directory - # - generate SAS for download and upload URLs - # - class EntraIdSigner + class EntraIdSigner # :nodoc: attr_reader :token_provider attr_reader :account_name @@ -25,12 +19,6 @@ def authorization_header(uri:, verb:, headers: {}) "Bearer #{token_provider.token}" end - # - # Generate a user delegation Shared Access Signature. - # See https://learn.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas - # See https://learn.microsoft.com/en-us/rest/api/storageservices/delegate-access-with-shared-access-signature - # See https://learn.microsoft.com/en-us/rest/api/storageservices/get-user-delegation-key - # def sas_token(uri, options = {}) # 1. Acquire an OAuth 2.0 token from Microsoft Entra ID. # 2. Use the token to request the user delegation key by calling the Get User Delegation Key operation. @@ -46,8 +34,8 @@ def sas_token(uri, options = {}) key_expiry = (now + 7.hours).iso8601 content = <<-XML.squish - - + + #{key_start} #{key_expiry} @@ -74,7 +62,7 @@ def sas_token(uri, options = {}) signed_service = doc.get_elements("/UserDelegationKey/SignedService").first.get_text.to_s signed_version = doc.get_elements("/UserDelegationKey/SignedVersion").first.get_text.to_s user_delegation_key = Base64.decode64(doc.get_elements("/UserDelegationKey/Value").first.get_text.to_s) - + # :start and :expiry, if present, are already in iso8601 format start = options[:start] || now.iso8601 expiry = options[:expiry] || (now + 5.minutes).iso8601 @@ -106,7 +94,7 @@ def sas_token(uri, options = {}) nil, # rscl + "\n" + options[:content_type], # rsct ].join("\n") - + query = { SAS::Fields::Permissions => options[:permissions], SAS::Fields::Start => start, @@ -165,4 +153,4 @@ module Resources # :nodoc: end end end -end \ No newline at end of file +end diff --git a/lib/azure_blob/shared_key_signer.rb b/lib/azure_blob/shared_key_signer.rb index 40c922a..907954c 100644 --- a/lib/azure_blob/shared_key_signer.rb +++ b/lib/azure_blob/shared_key_signer.rb @@ -6,13 +6,7 @@ require_relative "canonicalized_resource" module AzureBlob - # - # This implementatio uses a Shared Key to - # - generate the authorisation header - # See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key - # - generate SAS for download and upload URLs - # - class SharedKeySigner + class SharedKeySigner # :nodoc: def initialize(account_name:, access_key:) @account_name = account_name @access_key = Base64.decode64(access_key) @@ -44,11 +38,6 @@ def authorization_header(uri:, verb:, headers: {}) "SharedKey #{account_name}:#{sign(to_sign)}" end - # - # Generate a service Shared Access Signature. - # See https://learn.microsoft.com/en-us/rest/api/storageservices/create-service-sas - # See https://learn.microsoft.com/en-us/rest/api/storageservices/delegate-access-with-shared-access-signature - # def sas_token(uri, options = {}) to_sign = [ options[:permissions], From 198ec29232106b3065954a973152c6f723508e6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Tue, 3 Sep 2024 16:28:02 -0700 Subject: [PATCH 24/70] Use stock ruby --- devenv.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/devenv.yaml b/devenv.yaml index fe251b1..90b9a08 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -2,5 +2,3 @@ allowUnfree: true inputs: nixpkgs: url: github:NixOS/nixpkgs/nixos-23.11 - nixpkgs-ruby: - url: github:bobvanderlinden/nixpkgs-ruby From 7cdbb1658cf18301f8c10ce7294d77de72ce7ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Tue, 3 Sep 2024 16:28:32 -0700 Subject: [PATCH 25/70] Add nix2container and mk-shell-bin --- devenv.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/devenv.yaml b/devenv.yaml index 90b9a08..11ee558 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -1,4 +1,11 @@ -allowUnfree: true inputs: + nix2container: + url: github:nlewo/nix2container + inputs: + nixpkgs: + follows: nixpkgs + mk-shell-bin: + url: github:rrbutani/nix-mk-shell-bin nixpkgs: url: github:NixOS/nixpkgs/nixos-23.11 +allowUnfree: true From fe3751da6e2437cfe2f2d1515d112386067a130c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Tue, 3 Sep 2024 17:08:06 -0700 Subject: [PATCH 26/70] Can't have nice things --- devenv.lock | 97 ++++++++++++++++++++++++++++++++++++++++++++++------- devenv.nix | 4 ++- devenv.yaml | 6 ++-- 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/devenv.lock b/devenv.lock index df9976e..d2b5e4e 100644 --- a/devenv.lock +++ b/devenv.lock @@ -71,6 +71,24 @@ "inputs": { "systems": "systems_2" }, + "locked": { + "lastModified": 1710146030, + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "treeHash": "bd263f021e345cb4a39d80c126ab650bebc3c10c", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_3": { + "inputs": { + "systems": "systems_3" + }, "locked": { "lastModified": 1710146030, "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", @@ -106,18 +124,54 @@ "type": "github" } }, + "mk-shell-bin": { + "locked": { + "lastModified": 1677004959, + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "rev": "ff5d8bd4d68a347be5042e2f16caee391cd75887", + "treeHash": "496327dabdc787353a29987f492dd4939151baad", + "type": "github" + }, + "original": { + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "type": "github" + } + }, + "nix2container": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1724996935, + "owner": "nlewo", + "repo": "nix2container", + "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401", + "treeHash": "a934d246fadcf8b36d28f3577fad413f5ab3f7d3", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1715542476, + "lastModified": 1725001927, "owner": "NixOS", "repo": "nixpkgs", - "rev": "44072e24566c5bcc0b7aa9178a0104f4cfffab19", - "treeHash": "3f9021e4c33de6fe59b88ac8c3019fc49136dc2a", + "rev": "6e99f2a27d600612004fbd2c3282d614bfee6421", + "treeHash": "1e85443cc9f0ba302df2cf61cacb8014943e2d19", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } @@ -125,15 +179,15 @@ "nixpkgs-ruby": { "inputs": { "flake-compat": "flake-compat", - "flake-utils": "flake-utils", + "flake-utils": "flake-utils_2", "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1713939467, + "lastModified": 1724737223, "owner": "bobvanderlinden", "repo": "nixpkgs-ruby", - "rev": "c1ba161adf31119cfdbb24489766a7bcd4dbe881", - "treeHash": "0d32620317b29f94d6718684f030dd2fc2f30cb2", + "rev": "175b5867babcbc471b94be9fd5576f2973bbdb6d", + "treeHash": "2fe3404ac0eeb7bcb7ac7b5f5f8b9b6a7e460147", "type": "github" }, "original": { @@ -160,16 +214,16 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1715542476, + "lastModified": 1725001927, "owner": "NixOS", "repo": "nixpkgs", - "rev": "44072e24566c5bcc0b7aa9178a0104f4cfffab19", - "treeHash": "3f9021e4c33de6fe59b88ac8c3019fc49136dc2a", + "rev": "6e99f2a27d600612004fbd2c3282d614bfee6421", + "treeHash": "1e85443cc9f0ba302df2cf61cacb8014943e2d19", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } @@ -177,7 +231,7 @@ "pre-commit-hooks": { "inputs": { "flake-compat": "flake-compat_2", - "flake-utils": "flake-utils_2", + "flake-utils": "flake-utils_3", "gitignore": "gitignore", "nixpkgs": [ "nixpkgs" @@ -201,6 +255,8 @@ "root": { "inputs": { "devenv": "devenv", + "mk-shell-bin": "mk-shell-bin", + "nix2container": "nix2container", "nixpkgs": "nixpkgs", "nixpkgs-ruby": "nixpkgs-ruby", "pre-commit-hooks": "pre-commit-hooks" @@ -222,6 +278,21 @@ } }, "systems_2": { + "locked": { + "lastModified": 1681028828, + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "treeHash": "cce81f2a0f0743b2eb61bc2eb6c7adbe2f2c6beb", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { "locked": { "lastModified": 1681028828, "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", diff --git a/devenv.nix b/devenv.nix index 4d4116e..8530bb1 100644 --- a/devenv.nix +++ b/devenv.nix @@ -10,11 +10,13 @@ libyaml terraform azure-cli - ruby glib vips ]; + languages.ruby.enable = true; + languages.ruby.version = "3.2.1"; + scripts.sync-vm.exec = '' vm_username=$(terraform output --raw "vm_username") vm_ip=$(terraform output --raw "vm_ip") diff --git a/devenv.yaml b/devenv.yaml index 11ee558..7e5376c 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -1,3 +1,4 @@ +allowUnfree: true inputs: nix2container: url: github:nlewo/nix2container @@ -7,5 +8,6 @@ inputs: mk-shell-bin: url: github:rrbutani/nix-mk-shell-bin nixpkgs: - url: github:NixOS/nixpkgs/nixos-23.11 -allowUnfree: true + url: github:NixOS/nixpkgs/nixos-24.05 + nixpkgs-ruby: + url: github:bobvanderlinden/nixpkgs-ruby From 76157a66f84ef4660cff162b16a93372c87f327e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Tue, 3 Sep 2024 17:08:22 -0700 Subject: [PATCH 27/70] sshuttle --- devenv.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/devenv.nix b/devenv.nix index 8530bb1..285d62e 100644 --- a/devenv.nix +++ b/devenv.nix @@ -12,6 +12,7 @@ azure-cli glib vips + sshuttle ]; languages.ruby.enable = true; From 3fd0fa78ed0729c76f7fded8d1f46847589c87db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Tue, 3 Sep 2024 17:19:10 -0700 Subject: [PATCH 28/70] Update bundler and stringio --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 243f485..58d9b0c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -226,7 +226,7 @@ GEM ffi (~> 1.12) sqlite3 (2.0.2-arm64-darwin) sqlite3 (2.0.2-x86_64-linux-gnu) - stringio (3.1.0) + stringio (3.1.1) strscan (3.1.0) thor (1.3.1) timeout (0.4.1) @@ -256,4 +256,4 @@ DEPENDENCIES sqlite3 (>= 1.6.6) BUNDLED WITH - 2.4.22 + 2.5.9 From d97e54df6fce3be9c81c999e62eb668f77c1a0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Tue, 3 Sep 2024 17:19:52 -0700 Subject: [PATCH 29/70] No need for cloud config with sshuttle --- cloudinit.cfg | 10 ---------- main.tf | 1 - 2 files changed, 11 deletions(-) delete mode 100644 cloudinit.cfg diff --git a/cloudinit.cfg b/cloudinit.cfg deleted file mode 100644 index 30d25e2..0000000 --- a/cloudinit.cfg +++ /dev/null @@ -1,10 +0,0 @@ -# cloud-config -package_upgrade: true -packages: - - httpd - - glib2.0 -runcmd: - - "sh <(curl -L https://nixos.org/nix/install) --daemon --yes" - - "nix-env -iA devenv -f https://github.com/NixOS/nixpkgs/tarball/nixpkgs-unstable" - - 'echo "trusted-users = root ${vm_username}" >> /etc/nix/nix.conf' - - 'systemctl restart nix-daemon' diff --git a/main.tf b/main.tf index 06e4cc8..d59146c 100644 --- a/main.tf +++ b/main.tf @@ -116,7 +116,6 @@ resource "azurerm_linux_virtual_machine" "main" { size = var.vm_size admin_username = var.vm_username admin_password = var.vm_password - custom_data = base64encode(templatefile("./cloudinit.cfg", { vm_username = var.vm_username})) disable_password_authentication = true network_interface_ids = [azurerm_network_interface.main[0].id] From 0aedc9eca7071bac1e134459fbe141a6cb694bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Tue, 3 Sep 2024 17:20:09 -0700 Subject: [PATCH 30/70] vps proxy script --- devenv.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/devenv.nix b/devenv.nix index 285d62e..de72ac3 100644 --- a/devenv.nix +++ b/devenv.nix @@ -27,4 +27,8 @@ scripts.generate-env-file.exec = '' terraform output -raw devenv_local_nix > devenv.local.nix ''; + + scripts.proxy-vps.exec = '' + sshuttle -r "$(terraform output --raw vm_username)@$(terraform output --raw vm_ip)" 0/0 + ''; } From 440eef3eb37636aa87d6b7fdc0efa3ba9c01477e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Tue, 3 Sep 2024 17:24:15 -0700 Subject: [PATCH 31/70] Rake task to test with entra id --- Rakefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Rakefile b/Rakefile index 56d2819..e127709 100644 --- a/Rakefile +++ b/Rakefile @@ -8,6 +8,11 @@ Minitest::TestTask.create task default: %i[test] +task :test_entra_id do |t| + ENV["AZURE_ACCESS_KEY"] = nil + Rake::Task["test"].execute +end + task :flush_test_container do |t| AzureBlob::Client.new( account_name: ENV["AZURE_ACCOUNT_NAME"], From 6a19fdbb99d56a3d397a88df065e128075cbcf02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Tue, 3 Sep 2024 17:42:22 -0700 Subject: [PATCH 32/70] VPS instructions --- README.md | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 124ed2a..615ebe2 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,36 @@ A dev environment is supplied through Nix with [devenv](https://devenv.sh/). 2. Enter the dev environment by cd into the repo and running `devenv shell` (or `direnv allow` if you are a direnv user). 3. Log into azure CLI with `az login` 4. `terraform init` -5. `terraform apply` This will generate the necessary infrastructure on azure +5. `terraform apply` This will generate the necessary infrastructure on azure. 6. Generate devenv.local.nix with your private key and container information: `terraform output -raw devenv_local_nix > devenv.local.nix` 7. If you are using direnv, the environment will reload automatically. If not, exit the shell and reopen it by hitting and running `devenv shell` again. +#### Entra ID + +To test with Entra ID, the `AZURE_ACCESS_KEY` environment variable must be unset and the code must be ran or proxied through a VPS with the proper roles. + +For cost saving, the terraform variable `create_vm` is false by default. +To create the VPS, Create a var file `var.tfvars` containing: + +``` +create_vm = true +``` +and re-apply terraform: `terraform apply -var-file=var.tfvars`. + +This will create the VPS and required roles. + +Use `proxy-vps` to proxy all network requests through the vps with sshuttle. sshuttle will likely ask for a sudo password. + +Then use `bin/rake test_entra_id` to run the tests with Entra ID. + +After you are done, running terraform again without the var file (`terraform apply`) it should destroy the VPS. + +#### Cleanup + +Some test copied over from Rails codebase don't clean after themselves. A rake task is provided to empty your containers and keep cost low: `bin/rake flush_test_container` + +#### Run without devenv/nix + If you prefer not using devenv/nix: Ensure your version of Ruby fit the minimum version in `azure-blob.gemspec` @@ -63,11 +89,6 @@ and setup those Env variables: - `AZURE_PRIVATE_CONTAINER` - `AZURE_PUBLIC_CONTAINER` - -### Tests - -`bin/rake test` - ## License The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). From 98540ba2c419bf4b56917019915720169a25682a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Wed, 4 Sep 2024 19:00:56 -0700 Subject: [PATCH 33/70] Create an app service app --- input.tf | 5 +++++ main.tf | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/input.tf b/input.tf index 85312c1..1b1500c 100644 --- a/input.tf +++ b/input.tf @@ -33,6 +33,11 @@ variable "vm_password" { default = "qwe123QWE!@#" } +variable "create_app_service" { + type = bool + default = false +} + variable "ssh_key" { type = string default = "" diff --git a/main.tf b/main.tf index d59146c..646de24 100644 --- a/main.tf +++ b/main.tf @@ -145,3 +145,37 @@ resource "azurerm_linux_virtual_machine" "main" { source = "Terraform" } } + + +resource "azurerm_service_plan" "main" { + count = var.create_app_service ? 1 : 0 + name = "${var.prefix}-appserviceplan" + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location + os_type = "Linux" + sku_name = "B1" +} + + +resource "azurerm_linux_web_app" "main" { + count = var.create_app_service ? 1 : 0 + name = "${var.prefix}-app" + service_plan_id = azurerm_service_plan.main[0].id + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location + + site_config { + application_stack { + node_version = "20-lts" + } + } +} + +resource "azurerm_app_service_source_control" "main" { + count = var.create_app_service ? 1 : 0 + app_id = azurerm_linux_web_app.main[0].id + repo_url = "https://github.com/Azure-Samples/nodejs-docs-hello-world" + branch = "master" + use_manual_integration = true + use_mercurial = false +} From 3b7e70a9a510d3bcc964c135f9912e91ef307ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Wed, 4 Sep 2024 19:01:56 -0700 Subject: [PATCH 34/70] Put public key in local --- main.tf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 646de24..b0a770c 100644 --- a/main.tf +++ b/main.tf @@ -11,6 +11,10 @@ provider "azurerm" { features {} } +locals { + public_ssh_key = var.ssh_key != "" ? var.ssh_key : file("~/.ssh/id_rsa.pub") +} + resource "azurerm_resource_group" "main" { name = var.prefix location = var.location @@ -126,7 +130,7 @@ resource "azurerm_linux_virtual_machine" "main" { admin_ssh_key { username = var.vm_username - public_key = var.ssh_key != "" ? var.ssh_key : file("~/.ssh/id_rsa.pub") + public_key = local.public_ssh_key } source_image_reference { From da979b01545de80713c04a6bcadb34acae963d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sat, 7 Sep 2024 15:18:25 -0700 Subject: [PATCH 35/70] Establish VPN to App Service --- Gemfile | 5 ++ Gemfile.lock | 7 +++ Rakefile | 8 ++++ devenv.nix | 8 ++++ main.tf | 5 ++ output.tf | 8 ++++ test/support/app_service_vpn.rb | 82 +++++++++++++++++++++++++++++++++ 7 files changed, 123 insertions(+) create mode 100644 test/support/app_service_vpn.rb diff --git a/Gemfile b/Gemfile index 89a7435..2359e2c 100644 --- a/Gemfile +++ b/Gemfile @@ -19,3 +19,8 @@ gem "rails", github: "rails/rails", branch: "main" gem "propshaft", ">= 0.1.7" gem "image_processing", "~> 1.2" gem "sqlite3", ">= 1.6.6" + + +gem "net-ssh" +gem "ed25519" +gem "bcrypt_pbkdf" diff --git a/Gemfile.lock b/Gemfile.lock index 58d9b0c..f9f7f97 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -105,6 +105,8 @@ GEM specs: ast (2.4.2) base64 (0.2.0) + bcrypt_pbkdf (1.1.1) + bcrypt_pbkdf (1.1.1-arm64-darwin) bigdecimal (3.1.8) builder (3.2.4) concurrent-ruby (1.2.3) @@ -115,6 +117,7 @@ GEM irb (~> 1.10) reline (>= 0.3.8) drb (2.2.1) + ed25519 (1.3.0) erubi (1.12.0) ffi (1.16.3) globalid (1.2.1) @@ -151,6 +154,7 @@ GEM timeout net-smtp (0.5.0) net-protocol + net-ssh (7.2.3) nio4r (2.7.3) nokogiri (1.16.5-arm64-darwin) racc (~> 1.4) @@ -246,9 +250,12 @@ PLATFORMS DEPENDENCIES azure-blob! + bcrypt_pbkdf debug + ed25519 image_processing (~> 1.2) minitest + net-ssh propshaft (>= 0.1.7) rails! rake diff --git a/Rakefile b/Rakefile index e127709..7505a3e 100644 --- a/Rakefile +++ b/Rakefile @@ -3,11 +3,19 @@ require "bundler/gem_tasks" require "minitest/test_task" require 'azure_blob' +require_relative 'test/support/app_service_vpn' Minitest::TestTask.create task default: %i[test] +task :test_app_service do |t| + vpn = AppServiceVPN.new(verbose: true) + ENV["IDENTITY_ENDPOINT"] = vpn.endpoint + ENV["IDENTITY_HEADER"] = vpn.header + Rake::Task["test_entra_id"].execute +end + task :test_entra_id do |t| ENV["AZURE_ACCESS_KEY"] = nil Rake::Task["test"].execute diff --git a/devenv.nix b/devenv.nix index de72ac3..408dabf 100644 --- a/devenv.nix +++ b/devenv.nix @@ -13,6 +13,8 @@ glib vips sshuttle + sshpass + rsync ]; languages.ruby.enable = true; @@ -31,4 +33,10 @@ scripts.proxy-vps.exec = '' sshuttle -r "$(terraform output --raw vm_username)@$(terraform output --raw vm_ip)" 0/0 ''; + + scripts.start-app-service-ssh.exec = '' + resource_group=$(terraform output --raw "resource_group") + app_name=$(terraform output --raw "app_service_app_name") + az webapp create-remote-connection --resource-group $resource_group --name $app_name + ''; } diff --git a/main.tf b/main.tf index b0a770c..002a6ce 100644 --- a/main.tf +++ b/main.tf @@ -168,6 +168,11 @@ resource "azurerm_linux_web_app" "main" { resource_group_name = azurerm_resource_group.main.name location = azurerm_resource_group.main.location + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.vm.id] + } + site_config { application_stack { node_version = "20-lts" diff --git a/output.tf b/output.tf index 5ae49b9..0717ece 100644 --- a/output.tf +++ b/output.tf @@ -19,3 +19,11 @@ output "vm_ip" { output "vm_username" { value = var.vm_username } + +output "app_service_app_name" { + value = var.create_app_service ? azurerm_linux_web_app.main[0].name : "" +} + +output "resource_group" { + value = azurerm_resource_group.main.name +} diff --git a/test/support/app_service_vpn.rb b/test/support/app_service_vpn.rb new file mode 100644 index 0000000..83de06e --- /dev/null +++ b/test/support/app_service_vpn.rb @@ -0,0 +1,82 @@ +require 'open3' +require 'net/ssh' +require 'shellwords' + +class AppServiceVPN + HOST = '127.0.0.1' + + attr_reader :header, :endpoint + + def initialize verbose: false + @verbose = verbose + establish_vpn_connection + end + + private + + def establish_vpn_connection + establish_app_service_tunnel + extract_msi_info + + puts "Establishing VPN connection..." + + tunnel_stdin, tunnel_stdout, tunnel_wait_thread = Open3.popen2e(["sshuttle", "-e", "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null", "-r", "#{username}:#{password}@#{HOST}:#{port}", "0/0"].shelljoin) + + connection_successful = false + tunnel_stdout.each do |line| + puts "(sshuttle) #{line}" if verbose + if line.include?("Connected to server") + connection_successful = true + puts "Connection successful!" + break + end + end + + raise "Could not establish VPN connection to app service" unless connection_successful + end + + def establish_app_service_tunnel + puts 'Establishing tunnel connection to app service...' + connection_stdin, connection_stdout, connection_wait_thread = Open3.popen2e("start-app-service-ssh") + + port = nil + username = nil + password = nil + + connection_stdout.each do |line| + puts "(start-app-service-ssh) #{line}" if verbose + if line =~ /WARNING: Opening tunnel on port: (\d+)/ + port = $1.to_i + end + + if line =~ /WARNING: SSH is available \{ username: (\w+), password: ([^\s]+) \}/ + username = $1 + password = $2 + end + + if port && username && password + break + end + end + + raise "Could not establish tunnel connection to app service" unless port && username && password + @port = port + @username = username + @password = password + end + + def extract_msi_info + puts "Extracting MSI endpoint info..." + endpoint = nil + header = nil + Net::SSH.start(HOST, username, password:, port:) do |ssh| + endpoint = ssh.exec! ["bash", "-l", "-c", "echo -n $IDENTITY_ENDPOINT"].shelljoin + header = ssh.exec! ["bash", "-l", "-c", "echo -n $IDENTITY_HEADER"].shelljoin + end + raise "Could not extract MSI endpoint information" unless endpoint && header + @endpoint = endpoint + @header = header + end + + attr_reader :port, :username, :password, :verbose +end From 74e32508ce6216eeff090cd64fbe8de03c8b4aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sat, 7 Sep 2024 15:26:03 -0700 Subject: [PATCH 36/70] Temp support for both app service and VM --- lib/azure_blob/auth/msi_token_provider.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/azure_blob/auth/msi_token_provider.rb b/lib/azure_blob/auth/msi_token_provider.rb index def9e5d..528016f 100644 --- a/lib/azure_blob/auth/msi_token_provider.rb +++ b/lib/azure_blob/auth/msi_token_provider.rb @@ -29,8 +29,8 @@ class MsiTokenProvider RESOURCE_URI_STORAGE = 'https://storage.azure.com/' - IMDS_URI = 'http://169.254.169.254/metadata/identity/oauth2/token' - IMDS_API_VERSION = '2018-02-01' + IMDS_URI = ENV["IDENTITY_ENDPOINT"] || 'http://169.254.169.254/metadata/identity/oauth2/token' + IMDS_API_VERSION = ENV["IDENTITY_ENDPOINT"] ? '2019-08-01' : '2018-02-01' attr_reader :token attr_reader :token_expires_on @@ -64,7 +64,7 @@ def token def get_new_token_from_imds # curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://storage.azure.com/' -H Metadata:true - response = Net::HTTP.get_response(msi_identity_uri, {'Metadata' => 'true'}) + response = Net::HTTP.get_response(msi_identity_uri, {'Metadata' => 'true', 'X-IDENTITY-HEADER' => ENV['IDENTITY_HEADER']}) # TODO implement some retry strategies as per the documentation. # https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling From 55dfc7b5ca54f81229b3e2fc06ea412da972e9f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sat, 7 Sep 2024 15:34:12 -0700 Subject: [PATCH 37/70] Kill the vpn --- Rakefile | 1 + lib/azure_blob/auth/msi_token_provider.rb | 2 +- test/support/app_service_vpn.rb | 11 ++++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Rakefile b/Rakefile index 7505a3e..dcd1a34 100644 --- a/Rakefile +++ b/Rakefile @@ -14,6 +14,7 @@ task :test_app_service do |t| ENV["IDENTITY_ENDPOINT"] = vpn.endpoint ENV["IDENTITY_HEADER"] = vpn.header Rake::Task["test_entra_id"].execute + vpn.kill end task :test_entra_id do |t| diff --git a/lib/azure_blob/auth/msi_token_provider.rb b/lib/azure_blob/auth/msi_token_provider.rb index 528016f..07a955d 100644 --- a/lib/azure_blob/auth/msi_token_provider.rb +++ b/lib/azure_blob/auth/msi_token_provider.rb @@ -13,7 +13,7 @@ # # Azure Instance Metadata Service (IMDS) endpoint # http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/ -# +# # Using the extension installed locally and accessing # http://localhost:50342/oauth2/token diff --git a/test/support/app_service_vpn.rb b/test/support/app_service_vpn.rb index 83de06e..a282703 100644 --- a/test/support/app_service_vpn.rb +++ b/test/support/app_service_vpn.rb @@ -12,6 +12,11 @@ def initialize verbose: false establish_vpn_connection end + def kill + Process.kill("KILL", tunnel_wait_thread.pid) + Process.kill("KILL", connection_wait_thread.pid) + end + private def establish_vpn_connection @@ -20,7 +25,7 @@ def establish_vpn_connection puts "Establishing VPN connection..." - tunnel_stdin, tunnel_stdout, tunnel_wait_thread = Open3.popen2e(["sshuttle", "-e", "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null", "-r", "#{username}:#{password}@#{HOST}:#{port}", "0/0"].shelljoin) + tunnel_stdin, tunnel_stdout, @tunnel_wait_thread = Open3.popen2e(["sshuttle", "-e", "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null", "-r", "#{username}:#{password}@#{HOST}:#{port}", "0/0"].shelljoin) connection_successful = false tunnel_stdout.each do |line| @@ -37,7 +42,7 @@ def establish_vpn_connection def establish_app_service_tunnel puts 'Establishing tunnel connection to app service...' - connection_stdin, connection_stdout, connection_wait_thread = Open3.popen2e("start-app-service-ssh") + connection_stdin, connection_stdout, @connection_wait_thread = Open3.popen2e("start-app-service-ssh") port = nil username = nil @@ -78,5 +83,5 @@ def extract_msi_info @header = header end - attr_reader :port, :username, :password, :verbose + attr_reader :port, :username, :password, :verbose, :tunnel_wait_thread, :connection_wait_thread end From 487e5b7ab8f4e1b3572177b50ed395a643f0bf10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sat, 7 Sep 2024 15:58:10 -0700 Subject: [PATCH 38/70] Allow setting the role principal id --- lib/azure_blob/auth/msi_token_provider.rb | 5 +++-- lib/azure_blob/client.rb | 5 +++-- output.tf | 1 + test/rails/service/configurations.yml | 1 + test/test_client.rb | 2 ++ 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/azure_blob/auth/msi_token_provider.rb b/lib/azure_blob/auth/msi_token_provider.rb index 07a955d..14e8942 100644 --- a/lib/azure_blob/auth/msi_token_provider.rb +++ b/lib/azure_blob/auth/msi_token_provider.rb @@ -38,13 +38,14 @@ class MsiTokenProvider attr_reader :expiration_threshold attr_reader :msi_identity_uri - def initialize(resource_uri:, expiration_threshold: 10.minutes) + def initialize(resource_uri:, expiration_threshold: 10.minutes, principal_id: nil) # TODO Decide if we are going to use the IMDS uri or the localhost URI @msi_identity_uri = URI.parse(AzureBlob::Auth::MsiTokenProvider::IMDS_URI) params = { :'api-version' => AzureBlob::Auth::MsiTokenProvider::IMDS_API_VERSION, :resource => resource_uri, } + params[:principal_id] = principal_id if principal_id @msi_identity_uri.query = URI.encode_www_form(params) @expiration_threshold = expiration_threshold end @@ -85,4 +86,4 @@ def get_new_token_from_imds # subscription_id = ENV['AZURE_SUBSCRIPTION_ID'] # tenant_id = ENV['AZURE_TENANT_ID'] # resource_group_name = ENV['RESOURCE_GROUP_NAME'] -# msi_identity_uri = ENV['MSI_IDENTITY_URI'] || 'http://169.254.169.254/metadata/identity/oauth2/token' \ No newline at end of file +# msi_identity_uri = ENV['MSI_IDENTITY_URI'] || 'http://169.254.169.254/metadata/identity/oauth2/token' diff --git a/lib/azure_blob/client.rb b/lib/azure_blob/client.rb index 37b508b..db40ee4 100644 --- a/lib/azure_blob/client.rb +++ b/lib/azure_blob/client.rb @@ -13,7 +13,7 @@ module AzureBlob # AzureBlob Client class. You interact with the Azure Blob api # through an instance of this class. class Client - def initialize(account_name:, access_key:, container:) + def initialize(account_name:, access_key:, container:, **options) @account_name = account_name @container = container @@ -21,7 +21,8 @@ def initialize(account_name:, access_key:, container:) AzureBlob::SharedKeySigner.new(account_name:, access_key:) : AzureBlob::EntraIdSigner.new( AzureBlob::Auth::MsiTokenProvider.new( - resource_uri: AzureBlob::Auth::MsiTokenProvider::RESOURCE_URI_STORAGE + resource_uri: AzureBlob::Auth::MsiTokenProvider::RESOURCE_URI_STORAGE, + **options.slice(:principal_id) ), account_name: ) diff --git a/output.tf b/output.tf index 0717ece..e955d08 100644 --- a/output.tf +++ b/output.tf @@ -7,6 +7,7 @@ output "devenv_local_nix" { AZURE_ACCESS_KEY = "${azurerm_storage_account.main.primary_access_key}"; AZURE_PRIVATE_CONTAINER = "${azurerm_storage_container.private.name}"; AZURE_PUBLIC_CONTAINER = "${azurerm_storage_container.public.name}"; + AZURE_PRINCIPAL_ID = "${azurerm_user_assigned_identity.vm.principal_id}"; }; } EOT diff --git a/test/rails/service/configurations.yml b/test/rails/service/configurations.yml index 23e8f70..2acc30d 100644 --- a/test/rails/service/configurations.yml +++ b/test/rails/service/configurations.yml @@ -2,6 +2,7 @@ DEFAULT: &default service: AzureBlob storage_account_name: <%= ENV["AZURE_ACCOUNT_NAME"] %> storage_access_key: <%= ENV["AZURE_ACCESS_KEY"] %> + principal_id: <%= ENV["AZURE_PRINCIPAL_ID"]%> azure: <<: *default diff --git a/test/test_client.rb b/test/test_client.rb index 9bf42ff..2fc1d28 100644 --- a/test/test_client.rb +++ b/test/test_client.rb @@ -12,10 +12,12 @@ def setup @account_name = ENV["AZURE_ACCOUNT_NAME"] @access_key = ENV["AZURE_ACCESS_KEY"] @container = ENV["AZURE_PRIVATE_CONTAINER"] + @principal_id = ENV["AZURE_PRINCIPAL_ID"] @client = AzureBlob::Client.new( account_name: @account_name, access_key: @access_key, container: @container, + principal_id: @principal_id, ) @key = "test client##{name}" @content = "Some random content #{Random.rand(200)}" From 1f8d37312e720fcb4d3f9ceeaff7a50d77536fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sat, 7 Sep 2024 16:04:58 -0700 Subject: [PATCH 39/70] Auto start the azure vm vpn --- Rakefile | 9 ++++++++- devenv.nix | 2 +- test/support/app_service_vpn.rb | 2 +- test/support/azure_vm_vpn.rb | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 test/support/azure_vm_vpn.rb diff --git a/Rakefile b/Rakefile index dcd1a34..c1283ef 100644 --- a/Rakefile +++ b/Rakefile @@ -4,19 +4,26 @@ require "bundler/gem_tasks" require "minitest/test_task" require 'azure_blob' require_relative 'test/support/app_service_vpn' +require_relative 'test/support/azure_vm_vpn' Minitest::TestTask.create task default: %i[test] task :test_app_service do |t| - vpn = AppServiceVPN.new(verbose: true) + vpn = AppServiceVpn.new ENV["IDENTITY_ENDPOINT"] = vpn.endpoint ENV["IDENTITY_HEADER"] = vpn.header Rake::Task["test_entra_id"].execute vpn.kill end +task :test_azure_vm do |t| + vpn = AzureVmVpn.new + Rake::Task["test_entra_id"].execute + vpn.kill +end + task :test_entra_id do |t| ENV["AZURE_ACCESS_KEY"] = nil Rake::Task["test"].execute diff --git a/devenv.nix b/devenv.nix index 408dabf..ffc7bf7 100644 --- a/devenv.nix +++ b/devenv.nix @@ -31,7 +31,7 @@ ''; scripts.proxy-vps.exec = '' - sshuttle -r "$(terraform output --raw vm_username)@$(terraform output --raw vm_ip)" 0/0 + sshuttle -e "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -r "$(terraform output --raw vm_username)@$(terraform output --raw vm_ip)" 0/0 ''; scripts.start-app-service-ssh.exec = '' diff --git a/test/support/app_service_vpn.rb b/test/support/app_service_vpn.rb index a282703..a2f7d20 100644 --- a/test/support/app_service_vpn.rb +++ b/test/support/app_service_vpn.rb @@ -2,7 +2,7 @@ require 'net/ssh' require 'shellwords' -class AppServiceVPN +class AppServiceVpn HOST = '127.0.0.1' attr_reader :header, :endpoint diff --git a/test/support/azure_vm_vpn.rb b/test/support/azure_vm_vpn.rb new file mode 100644 index 0000000..b69b363 --- /dev/null +++ b/test/support/azure_vm_vpn.rb @@ -0,0 +1,17 @@ +require 'open3' +require 'shellwords' + +class AzureVmVpn + def initialize verbose: false + @verbose = verbose + stdin, stdout, @wait_thread = Open3.popen2e("proxy-vps") + end + + def kill + Process.kill("KILL", wait_thread.pid) + end + + private + + attr_reader :wait_thread +end From 35e95ff89ea89e1d50eed8ce82fcbd0d2f84e66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 11:43:31 -0700 Subject: [PATCH 40/70] Ensure vpn kill --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index c1283ef..4c66903 100644 --- a/Rakefile +++ b/Rakefile @@ -15,12 +15,14 @@ task :test_app_service do |t| ENV["IDENTITY_ENDPOINT"] = vpn.endpoint ENV["IDENTITY_HEADER"] = vpn.header Rake::Task["test_entra_id"].execute +ensure vpn.kill end task :test_azure_vm do |t| vpn = AzureVmVpn.new Rake::Task["test_entra_id"].execute +ensure vpn.kill end From 7b9776358cb3f0c2068aa436132c3311a64f4b2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 11:43:42 -0700 Subject: [PATCH 41/70] Wait for the vpn to connect --- test/support/azure_vm_vpn.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/support/azure_vm_vpn.rb b/test/support/azure_vm_vpn.rb index b69b363..604b3f1 100644 --- a/test/support/azure_vm_vpn.rb +++ b/test/support/azure_vm_vpn.rb @@ -5,6 +5,9 @@ class AzureVmVpn def initialize verbose: false @verbose = verbose stdin, stdout, @wait_thread = Open3.popen2e("proxy-vps") + stdout.each do |line| + break if line.include?("Connected to server") + end end def kill From 8a6efcd9b15df3b7f10db5d1cd109691eeeba4ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 11:57:08 -0700 Subject: [PATCH 42/70] Do not expose the token to the client --- lib/azure_blob/auth.rb | 4 +- lib/azure_blob/auth/msi_token_provider.rb | 89 ----------------------- lib/azure_blob/client.rb | 8 +- lib/azure_blob/entra_id_signer.rb | 5 +- lib/azure_blob/msi_token_provider.rb | 89 +++++++++++++++++++++++ 5 files changed, 94 insertions(+), 101 deletions(-) delete mode 100644 lib/azure_blob/auth/msi_token_provider.rb create mode 100644 lib/azure_blob/msi_token_provider.rb diff --git a/lib/azure_blob/auth.rb b/lib/azure_blob/auth.rb index 3444af5..7dac2c6 100644 --- a/lib/azure_blob/auth.rb +++ b/lib/azure_blob/auth.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true -require_relative "auth/msi_token_provider" - module AzureBlob module Auth end -end \ No newline at end of file +end diff --git a/lib/azure_blob/auth/msi_token_provider.rb b/lib/azure_blob/auth/msi_token_provider.rb deleted file mode 100644 index 14e8942..0000000 --- a/lib/azure_blob/auth/msi_token_provider.rb +++ /dev/null @@ -1,89 +0,0 @@ -# -# As of 04/04/2018, there are 2 supported ways to get MSI Token. -# -# - Using the extension installed locally and accessing -# http://localhost:50342/oauth2/token to get the MSI Token -# - Accessing the http://169.254.169.254/metadata/identity/oauth2/token to get -# the MSI Token (default) -# -# Find further details here -# https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/tutorial-linux-vm-access-storage -# -# [How to use managed identities for Azure resources on an Azure VM to acquire an access token](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token) -# -# Azure Instance Metadata Service (IMDS) endpoint -# http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/ -# -# Using the extension installed locally and accessing -# http://localhost:50342/oauth2/token - -require "net/http" - -# -# Class that provides access to authentication token via Managed Service -# Identity. -# -module AzureBlob - module Auth - class MsiTokenProvider - - RESOURCE_URI_STORAGE = 'https://storage.azure.com/' - - IMDS_URI = ENV["IDENTITY_ENDPOINT"] || 'http://169.254.169.254/metadata/identity/oauth2/token' - IMDS_API_VERSION = ENV["IDENTITY_ENDPOINT"] ? '2019-08-01' : '2018-02-01' - - attr_reader :token - attr_reader :token_expires_on - - attr_reader :expiration_threshold - attr_reader :msi_identity_uri - - def initialize(resource_uri:, expiration_threshold: 10.minutes, principal_id: nil) - # TODO Decide if we are going to use the IMDS uri or the localhost URI - @msi_identity_uri = URI.parse(AzureBlob::Auth::MsiTokenProvider::IMDS_URI) - params = { - :'api-version' => AzureBlob::Auth::MsiTokenProvider::IMDS_API_VERSION, - :resource => resource_uri, - } - params[:principal_id] = principal_id if principal_id - @msi_identity_uri.query = URI.encode_www_form(params) - @expiration_threshold = expiration_threshold - end - - def token_expired? - @token.nil? || Time.now >= (@token_expires_on + expiration_threshold) - end - - def token - if(self.token_expired?) - self.get_new_token_from_imds() - end - @token - end - - private - - def get_new_token_from_imds - # curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://storage.azure.com/' -H Metadata:true - response = Net::HTTP.get_response(msi_identity_uri, {'Metadata' => 'true', 'X-IDENTITY-HEADER' => ENV['IDENTITY_HEADER']}) - - # TODO implement some retry strategies as per the documentation. - # https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling - unless response.is_a?(Net::HTTPSuccess) - raise AzureBlob::Auth::Error.new(response.body) - end - - json_res = JSON.parse(response.body) - - @token = json_res['access_token'] - # The number of seconds from "1970-01-01T0:0:0Z UTC" (corresponds to the token's exp claim). - @token_expires_on = Time.at(json_res['expires_on'].to_i) - end - end - end -end - -# subscription_id = ENV['AZURE_SUBSCRIPTION_ID'] -# tenant_id = ENV['AZURE_TENANT_ID'] -# resource_group_name = ENV['RESOURCE_GROUP_NAME'] -# msi_identity_uri = ENV['MSI_IDENTITY_URI'] || 'http://169.254.169.254/metadata/identity/oauth2/token' diff --git a/lib/azure_blob/client.rb b/lib/azure_blob/client.rb index db40ee4..fb8802e 100644 --- a/lib/azure_blob/client.rb +++ b/lib/azure_blob/client.rb @@ -19,13 +19,7 @@ def initialize(account_name:, access_key:, container:, **options) @signer = !access_key.nil? && !access_key.empty? ? AzureBlob::SharedKeySigner.new(account_name:, access_key:) : - AzureBlob::EntraIdSigner.new( - AzureBlob::Auth::MsiTokenProvider.new( - resource_uri: AzureBlob::Auth::MsiTokenProvider::RESOURCE_URI_STORAGE, - **options.slice(:principal_id) - ), - account_name: - ) + AzureBlob::EntraIdSigner.new(account_name:, **options.slice(:principal_id)) end # Create a blob of type block. Will automatically split the the blob in multiple block and send the blob in pieces (blocks) if the blob is too big. diff --git a/lib/azure_blob/entra_id_signer.rb b/lib/azure_blob/entra_id_signer.rb index cc66f9e..a9fe10e 100644 --- a/lib/azure_blob/entra_id_signer.rb +++ b/lib/azure_blob/entra_id_signer.rb @@ -4,14 +4,15 @@ require "rexml/document" require_relative "canonicalized_resource" +require_relative "msi_token_provider" module AzureBlob class EntraIdSigner # :nodoc: attr_reader :token_provider attr_reader :account_name - def initialize(token_provider, account_name:) - @token_provider = token_provider + def initialize(account_name:, principal_id: nil) + @token_provider = AzureBlob::MsiTokenProvider.new(principal_id:) @account_name = account_name end diff --git a/lib/azure_blob/msi_token_provider.rb b/lib/azure_blob/msi_token_provider.rb new file mode 100644 index 0000000..7b900f6 --- /dev/null +++ b/lib/azure_blob/msi_token_provider.rb @@ -0,0 +1,89 @@ +# +# As of 04/04/2018, there are 2 supported ways to get MSI Token. +# +# - Using the extension installed locally and accessing +# http://localhost:50342/oauth2/token to get the MSI Token +# - Accessing the http://169.254.169.254/metadata/identity/oauth2/token to get +# the MSI Token (default) +# +# Find further details here +# https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/tutorial-linux-vm-access-storage +# +# [How to use managed identities for Azure resources on an Azure VM to acquire an access token](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token) +# +# Azure Instance Metadata Service (IMDS) endpoint +# http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/ +# +# Using the extension installed locally and accessing +# http://localhost:50342/oauth2/token + +require "net/http" + +# +# Class that provides access to authentication token via Managed Service +# Identity. +# +module AzureBlob + class MsiTokenProvider + + RESOURCE_URI = 'https://storage.azure.com/' + + IMDS_URI = ENV["IDENTITY_ENDPOINT"] || 'http://169.254.169.254/metadata/identity/oauth2/token' + IMDS_API_VERSION = ENV["IDENTITY_ENDPOINT"] ? '2019-08-01' : '2018-02-01' + + attr_reader :token + attr_reader :token_expires_on + + attr_reader :expiration_threshold + attr_reader :msi_identity_uri + + def initialize(expiration_threshold: 10.minutes, principal_id: nil) + # TODO Decide if we are going to use the IMDS uri or the localhost URI + @msi_identity_uri = URI.parse(IMDS_URI) + params = { + 'api-version': IMDS_API_VERSION, + resource: RESOURCE_URI, + } + params[:principal_id] = principal_id if principal_id + @msi_identity_uri.query = URI.encode_www_form(params) + @expiration_threshold = expiration_threshold + end + + def token_expired? + @token.nil? || Time.now >= (@token_expires_on + expiration_threshold) + end + + def token + if(self.token_expired?) + self.get_new_token_from_imds() + end + @token + end + + private + + def get_new_token_from_imds + # curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://storage.azure.com/' -H Metadata:true + headers = {'Metadata' => 'true'} + headers['X-IDENTITY-HEADER'] = ENV['IDENTITY_HEADER'] if ENV['IDENTITY_HEADER'] + response = Net::HTTP.get_response(msi_identity_uri, headers) + + # TODO implement some retry strategies as per the documentation. + # https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling + unless response.is_a?(Net::HTTPSuccess) + raise AzureBlob::Auth::Error.new(response.body) + end + + json_res = JSON.parse(response.body) + + @token = json_res['access_token'] + # The number of seconds from "1970-01-01T0:0:0Z UTC" (corresponds to the token's exp claim). + @token_expires_on = Time.at(json_res['expires_on'].to_i) + end + end +end + +# subscription_id = ENV['AZURE_SUBSCRIPTION_ID'] +# tenant_id = ENV['AZURE_TENANT_ID'] +# resource_group_name = ENV['RESOURCE_GROUP_NAME'] +# msi_identity_uri = ENV['MSI_IDENTITY_URI'] || 'http://169.254.169.254/metadata/identity/oauth2/token' From b7a31885515e88c18d986e63e4d19db93d7f2ba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 14:01:54 -0700 Subject: [PATCH 43/70] Refactor the msi token provider --- lib/azure_blob.rb | 1 - lib/azure_blob/auth.rb | 6 -- lib/azure_blob/auth/error.rb | 5 -- lib/azure_blob/entra_id_signer.rb | 8 +-- lib/azure_blob/identity_token.rb | 46 ++++++++++++++ lib/azure_blob/msi_token_provider.rb | 89 ---------------------------- 6 files changed, 50 insertions(+), 105 deletions(-) delete mode 100644 lib/azure_blob/auth.rb delete mode 100644 lib/azure_blob/auth/error.rb create mode 100644 lib/azure_blob/identity_token.rb delete mode 100644 lib/azure_blob/msi_token_provider.rb diff --git a/lib/azure_blob.rb b/lib/azure_blob.rb index 6649241..34e3813 100644 --- a/lib/azure_blob.rb +++ b/lib/azure_blob.rb @@ -4,7 +4,6 @@ require_relative "azure_blob/client" require_relative "azure_blob/const" require_relative "azure_blob/errors" -require_relative "azure_blob/auth" module AzureBlob end diff --git a/lib/azure_blob/auth.rb b/lib/azure_blob/auth.rb deleted file mode 100644 index 7dac2c6..0000000 --- a/lib/azure_blob/auth.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module AzureBlob - module Auth - end -end diff --git a/lib/azure_blob/auth/error.rb b/lib/azure_blob/auth/error.rb deleted file mode 100644 index 5dfb8cf..0000000 --- a/lib/azure_blob/auth/error.rb +++ /dev/null @@ -1,5 +0,0 @@ -module AzureBlob - module Auth - class Error < StandardError; end - end -end diff --git a/lib/azure_blob/entra_id_signer.rb b/lib/azure_blob/entra_id_signer.rb index a9fe10e..9df5b8b 100644 --- a/lib/azure_blob/entra_id_signer.rb +++ b/lib/azure_blob/entra_id_signer.rb @@ -4,20 +4,20 @@ require "rexml/document" require_relative "canonicalized_resource" -require_relative "msi_token_provider" +require_relative "identity_token" module AzureBlob class EntraIdSigner # :nodoc: - attr_reader :token_provider + attr_reader :token attr_reader :account_name def initialize(account_name:, principal_id: nil) - @token_provider = AzureBlob::MsiTokenProvider.new(principal_id:) + @token = AzureBlob::IdentityToken.new(principal_id:) @account_name = account_name end def authorization_header(uri:, verb:, headers: {}) - "Bearer #{token_provider.token}" + "Bearer #{token}" end def sas_token(uri, options = {}) diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb new file mode 100644 index 0000000..82e57d6 --- /dev/null +++ b/lib/azure_blob/identity_token.rb @@ -0,0 +1,46 @@ +require "net/http" + +module AzureBlob + class IdentityToken + + RESOURCE_URI = 'https://storage.azure.com/' + EXPIRATION_BUFFER = 600 # 10 minutes + + IDENTITY_ENDPOINT = ENV["IDENTITY_ENDPOINT"] || 'http://169.254.169.254/metadata/identity/oauth2/token' + API_VERSION = ENV["IDENTITY_ENDPOINT"] ? '2019-08-01' : '2018-02-01' + + def initialize(principal_id: nil) + @identity_uri = URI.parse(IDENTITY_ENDPOINT) + params = { + 'api-version': API_VERSION, + resource: RESOURCE_URI, + } + params[:principal_id] = principal_id if principal_id + @identity_uri.query = URI.encode_www_form(params) + end + + def to_s + refresh if expired? + token + end + + private + + def expired? + token.nil? || Time.now >= (expiration + EXPIRATION_BUFFER) + end + + def refresh + headers = {'Metadata' => 'true'} + headers['X-IDENTITY-HEADER'] = ENV['IDENTITY_HEADER'] if ENV['IDENTITY_HEADER'] + # TODO implement some retry strategies as per the documentation. + # https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling + response = JSON.parse(Http.new(identity_uri, headers).get) + + @token = response['access_token'] + @expiration = Time.at(response['expires_on'].to_i) + end + + attr_reader :identity_uri, :expiration, :token + end +end diff --git a/lib/azure_blob/msi_token_provider.rb b/lib/azure_blob/msi_token_provider.rb deleted file mode 100644 index 7b900f6..0000000 --- a/lib/azure_blob/msi_token_provider.rb +++ /dev/null @@ -1,89 +0,0 @@ -# -# As of 04/04/2018, there are 2 supported ways to get MSI Token. -# -# - Using the extension installed locally and accessing -# http://localhost:50342/oauth2/token to get the MSI Token -# - Accessing the http://169.254.169.254/metadata/identity/oauth2/token to get -# the MSI Token (default) -# -# Find further details here -# https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/tutorial-linux-vm-access-storage -# -# [How to use managed identities for Azure resources on an Azure VM to acquire an access token](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token) -# -# Azure Instance Metadata Service (IMDS) endpoint -# http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/ -# -# Using the extension installed locally and accessing -# http://localhost:50342/oauth2/token - -require "net/http" - -# -# Class that provides access to authentication token via Managed Service -# Identity. -# -module AzureBlob - class MsiTokenProvider - - RESOURCE_URI = 'https://storage.azure.com/' - - IMDS_URI = ENV["IDENTITY_ENDPOINT"] || 'http://169.254.169.254/metadata/identity/oauth2/token' - IMDS_API_VERSION = ENV["IDENTITY_ENDPOINT"] ? '2019-08-01' : '2018-02-01' - - attr_reader :token - attr_reader :token_expires_on - - attr_reader :expiration_threshold - attr_reader :msi_identity_uri - - def initialize(expiration_threshold: 10.minutes, principal_id: nil) - # TODO Decide if we are going to use the IMDS uri or the localhost URI - @msi_identity_uri = URI.parse(IMDS_URI) - params = { - 'api-version': IMDS_API_VERSION, - resource: RESOURCE_URI, - } - params[:principal_id] = principal_id if principal_id - @msi_identity_uri.query = URI.encode_www_form(params) - @expiration_threshold = expiration_threshold - end - - def token_expired? - @token.nil? || Time.now >= (@token_expires_on + expiration_threshold) - end - - def token - if(self.token_expired?) - self.get_new_token_from_imds() - end - @token - end - - private - - def get_new_token_from_imds - # curl 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://storage.azure.com/' -H Metadata:true - headers = {'Metadata' => 'true'} - headers['X-IDENTITY-HEADER'] = ENV['IDENTITY_HEADER'] if ENV['IDENTITY_HEADER'] - response = Net::HTTP.get_response(msi_identity_uri, headers) - - # TODO implement some retry strategies as per the documentation. - # https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling - unless response.is_a?(Net::HTTPSuccess) - raise AzureBlob::Auth::Error.new(response.body) - end - - json_res = JSON.parse(response.body) - - @token = json_res['access_token'] - # The number of seconds from "1970-01-01T0:0:0Z UTC" (corresponds to the token's exp claim). - @token_expires_on = Time.at(json_res['expires_on'].to_i) - end - end -end - -# subscription_id = ENV['AZURE_SUBSCRIPTION_ID'] -# tenant_id = ENV['AZURE_TENANT_ID'] -# resource_group_name = ENV['RESOURCE_GROUP_NAME'] -# msi_identity_uri = ENV['MSI_IDENTITY_URI'] || 'http://169.254.169.254/metadata/identity/oauth2/token' From 3f3aacf08efe747706ff6ca226fea042c134b759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 14:32:38 -0700 Subject: [PATCH 44/70] Extract user delegation key --- lib/azure_blob/entra_id_signer.rb | 108 ++++++++------------------ lib/azure_blob/user_delegation_key.rb | 55 +++++++++++++ 2 files changed, 89 insertions(+), 74 deletions(-) create mode 100644 lib/azure_blob/user_delegation_key.rb diff --git a/lib/azure_blob/entra_id_signer.rb b/lib/azure_blob/entra_id_signer.rb index 9df5b8b..fa8dc23 100644 --- a/lib/azure_blob/entra_id_signer.rb +++ b/lib/azure_blob/entra_id_signer.rb @@ -6,6 +6,8 @@ require_relative "canonicalized_resource" require_relative "identity_token" +require_relative 'user_delegation_key' + module AzureBlob class EntraIdSigner # :nodoc: attr_reader :token @@ -21,79 +23,37 @@ def authorization_header(uri:, verb:, headers: {}) end def sas_token(uri, options = {}) - # 1. Acquire an OAuth 2.0 token from Microsoft Entra ID. - # 2. Use the token to request the user delegation key by calling the Get User Delegation Key operation. - - # TODO: move the user delegation key handling into it's own - # UserDelegationKeyProvider and reuse the same key if not expired - user_delegation_key_uri = URI.parse( - "https://#{account_name}.blob.core.windows.net/?restype=service&comp=userdelegationkey" - ) + delegation_key = UserDelegationKey.new(account_name:, signer: self) now = Time.now.utc - - key_start = now.iso8601 - key_expiry = (now + 7.hours).iso8601 - - content = <<-XML.squish - - - #{key_start} - #{key_expiry} - - XML - http = AzureBlob::Http.new(user_delegation_key_uri, signer: self) - response = http.post(content) - - # 3. Use the user delegation key to construct the SAS token with the appropriate fields. - doc = REXML::Document.new(response) - # - # - # String containing a GUID value - # String containing a GUID value - # String formatted as ISO date - # String formatted as ISO date - # b - # String specifying REST api version to use to create the user delegation key - # String containing the user delegation key - # - signed_oid = doc.get_elements("/UserDelegationKey/SignedOid").first.get_text.to_s - signed_tid = doc.get_elements("/UserDelegationKey/SignedTid").first.get_text.to_s - signed_start = doc.get_elements("/UserDelegationKey/SignedStart").first.get_text.to_s - signed_expiry = doc.get_elements("/UserDelegationKey/SignedExpiry").first.get_text.to_s - signed_service = doc.get_elements("/UserDelegationKey/SignedService").first.get_text.to_s - signed_version = doc.get_elements("/UserDelegationKey/SignedVersion").first.get_text.to_s - user_delegation_key = Base64.decode64(doc.get_elements("/UserDelegationKey/Value").first.get_text.to_s) - - # :start and :expiry, if present, are already in iso8601 format start = options[:start] || now.iso8601 expiry = options[:expiry] || (now + 5.minutes).iso8601 canonicalized_resource = CanonicalizedResource.new(uri, account_name, url_safe: false, service_name: :blob) to_sign = [ - options[:permissions], # signedPermissions + "\n" + - start, # signedStart + "\n" + - expiry, # signedExpiry + "\n" + - canonicalized_resource, # canonicalizedResource + "\n" + - signed_oid, # signedKeyObjectId + "\n" + - signed_tid, # signedKeyTenantId + "\n" + - signed_start, # signedKeyStart + "\n" + - signed_expiry, # signedKeyExpiry + "\n" + - signed_service, # signedKeyService + "\n" + - signed_version, # signedKeyVersion + "\n" + - nil, # signedAuthorizedUserObjectId + "\n" + - nil, # signedUnauthorizedUserObjectId + "\n" + - nil, # signedCorrelationId + "\n" + - options[:ip], # signedIP + "\n" + - options[:protocol], # signedProtocol + "\n" + - SAS::Version, # signedVersion + "\n" + - SAS::Resources::Blob, # signedResource + "\n" + - nil, # signedSnapshotTime + "\n" + - nil, # signedEncryptionScope + "\n" + - nil, # rscc + "\n" + - options[:content_disposition], # rscd + "\n" + - nil, # rsce + "\n" + - nil, # rscl + "\n" + - options[:content_type], # rsct + options[:permissions], + start, + expiry, + canonicalized_resource, + delegation_key.signed_oid, + delegation_key.signed_tid, + delegation_key.signed_start, + delegation_key.signed_expiry, + delegation_key.signed_service, + delegation_key.signed_version, + nil, + nil, + nil, + options[:ip], + options[:protocol], + SAS::Version, + SAS::Resources::Blob, + nil, + nil, + nil, + options[:content_disposition], + nil, + nil, + options[:content_type], ].join("\n") query = { @@ -101,12 +61,12 @@ def sas_token(uri, options = {}) SAS::Fields::Start => start, SAS::Fields::Expiry => expiry, - SAS::Fields::SignedObjectId => signed_oid, - SAS::Fields::SignedTenantId => signed_tid, - SAS::Fields::SignedKeyStartTime => signed_start, - SAS::Fields::SignedKeyExpiryTime => signed_expiry, - SAS::Fields::SignedKeyService => signed_service, - SAS::Fields::Signedkeyversion => signed_version, + SAS::Fields::SignedObjectId => delegation_key.signed_oid, + SAS::Fields::SignedTenantId => delegation_key.signed_tid, + SAS::Fields::SignedKeyStartTime => delegation_key.signed_start, + SAS::Fields::SignedKeyExpiryTime => delegation_key.signed_expiry, + SAS::Fields::SignedKeyService => delegation_key.signed_service, + SAS::Fields::Signedkeyversion => delegation_key.signed_version, SAS::Fields::SignedIp => options[:ip], @@ -116,7 +76,7 @@ def sas_token(uri, options = {}) SAS::Fields::Disposition => options[:content_disposition], SAS::Fields::Type => options[:content_type], - SAS::Fields::Signature => sign(to_sign, key: user_delegation_key), + SAS::Fields::Signature => sign(to_sign, key: delegation_key.to_s), }.reject { |_, value| value.nil? } diff --git a/lib/azure_blob/user_delegation_key.rb b/lib/azure_blob/user_delegation_key.rb new file mode 100644 index 0000000..f5eacf1 --- /dev/null +++ b/lib/azure_blob/user_delegation_key.rb @@ -0,0 +1,55 @@ +require_relative 'http' + +module AzureBlob + class UserDelegationKey # :nodoc: + def initialize(account_name:, signer:) + # TODO: reuse the same key if not expired + @uri = URI.parse( + "https://#{account_name}.blob.core.windows.net/?restype=service&comp=userdelegationkey" + ) + + @signer = signer + + now = Time.now.utc + + key_start = now.iso8601 + key_expiry = (now + 7.hours).iso8601 + + content = <<-XML.squish + + + #{key_start} + #{key_expiry} + + XML + + response = Http.new(uri, signer:).post(content) + + doc = REXML::Document.new(response) + + @signed_oid = doc.get_elements("/UserDelegationKey/SignedOid").first.get_text.to_s + @signed_tid = doc.get_elements("/UserDelegationKey/SignedTid").first.get_text.to_s + @signed_start = doc.get_elements("/UserDelegationKey/SignedStart").first.get_text.to_s + @signed_expiry = doc.get_elements("/UserDelegationKey/SignedExpiry").first.get_text.to_s + @signed_service = doc.get_elements("/UserDelegationKey/SignedService").first.get_text.to_s + @signed_version = doc.get_elements("/UserDelegationKey/SignedVersion").first.get_text.to_s + @user_delegation_key = Base64.decode64(doc.get_elements("/UserDelegationKey/Value").first.get_text.to_s) + end + + def to_s + user_delegation_key + end + + attr_reader :signed_oid, + :signed_tid, + :signed_start, + :signed_expiry, + :signed_service, + :signed_version, + :user_delegation_key + + + private + attr_reader :uri, :user_delegation_key, :signer + end +end From 024c55ef6bfb4b402879e98b5c6da05da21c81c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 14:33:09 -0700 Subject: [PATCH 45/70] Minitest fail fast --- Gemfile | 1 + Gemfile.lock | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Gemfile b/Gemfile index 2359e2c..c3e24cf 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ gemspec gem "rake" gem "minitest" +gem "minitest-fail-fast" gem "rubocop-rails-omakase" diff --git a/Gemfile.lock b/Gemfile.lock index f9f7f97..a8224d7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -145,6 +145,8 @@ GEM mini_magick (4.12.0) mini_mime (1.1.5) minitest (5.23.1) + minitest-fail-fast (0.1.0) + minitest (~> 5) net-imap (0.4.11) date net-protocol @@ -255,6 +257,7 @@ DEPENDENCIES ed25519 image_processing (~> 1.2) minitest + minitest-fail-fast net-ssh propshaft (>= 0.1.7) rails! From e0145a78344e22df5663f49f888023551dc7efcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 15:44:19 -0700 Subject: [PATCH 46/70] Try sigint instead of kill --- test/support/app_service_vpn.rb | 4 ++-- test/support/azure_vm_vpn.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/support/app_service_vpn.rb b/test/support/app_service_vpn.rb index a2f7d20..dab1b9c 100644 --- a/test/support/app_service_vpn.rb +++ b/test/support/app_service_vpn.rb @@ -13,8 +13,8 @@ def initialize verbose: false end def kill - Process.kill("KILL", tunnel_wait_thread.pid) - Process.kill("KILL", connection_wait_thread.pid) + Process.kill("INT", tunnel_wait_thread.pid) + Process.kill("INT", connection_wait_thread.pid) end private diff --git a/test/support/azure_vm_vpn.rb b/test/support/azure_vm_vpn.rb index 604b3f1..6afba58 100644 --- a/test/support/azure_vm_vpn.rb +++ b/test/support/azure_vm_vpn.rb @@ -11,7 +11,7 @@ def initialize verbose: false end def kill - Process.kill("KILL", wait_thread.pid) + Process.kill("INT", wait_thread.pid) end private From afaac9366a9b2371d8270428f6ede2f870133acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 15:44:43 -0700 Subject: [PATCH 47/70] Ensure we don't load rails before running the client tests --- Rakefile | 15 ++++++++++++++- test/{ => client}/test_client.rb | 2 +- test/{ => client}/test_helper.rb | 0 3 files changed, 15 insertions(+), 2 deletions(-) rename test/{ => client}/test_client.rb (99%) rename test/{ => client}/test_helper.rb (100%) diff --git a/Rakefile b/Rakefile index 4c66903..274ba3e 100644 --- a/Rakefile +++ b/Rakefile @@ -6,10 +6,23 @@ require 'azure_blob' require_relative 'test/support/app_service_vpn' require_relative 'test/support/azure_vm_vpn' -Minitest::TestTask.create +Minitest::TestTask.create(:test_rails) do + self.test_globs = ["test/rails/**/test_*.rb", + "test/rails/**/*_test.rb"] +end + +Minitest::TestTask.create(:test_client) do + self.test_globs = ["test/client/**/test_*.rb", + "test/client/**/*_test.rb"] +end task default: %i[test] +task :test do + Rake::Task["test_client"].execute + Rake::Task["test_rails"].execute +end + task :test_app_service do |t| vpn = AppServiceVpn.new ENV["IDENTITY_ENDPOINT"] = vpn.endpoint diff --git a/test/test_client.rb b/test/client/test_client.rb similarity index 99% rename from test/test_client.rb rename to test/client/test_client.rb index 2fc1d28..4bd7870 100644 --- a/test/test_client.rb +++ b/test/client/test_client.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "test_helper" +require_relative "test_helper" require "securerandom" class TestClient < TestCase diff --git a/test/test_helper.rb b/test/client/test_helper.rb similarity index 100% rename from test/test_helper.rb rename to test/client/test_helper.rb From 4c7d2fb11db39b33539e0b62c28ff1a541d937ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 15:52:37 -0700 Subject: [PATCH 48/70] Exec the proxy commands --- devenv.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devenv.nix b/devenv.nix index ffc7bf7..1fd9687 100644 --- a/devenv.nix +++ b/devenv.nix @@ -31,12 +31,12 @@ ''; scripts.proxy-vps.exec = '' - sshuttle -e "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -r "$(terraform output --raw vm_username)@$(terraform output --raw vm_ip)" 0/0 + exec sshuttle -e "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -r "$(terraform output --raw vm_username)@$(terraform output --raw vm_ip)" 0/0 ''; scripts.start-app-service-ssh.exec = '' resource_group=$(terraform output --raw "resource_group") app_name=$(terraform output --raw "app_service_app_name") - az webapp create-remote-connection --resource-group $resource_group --name $app_name + exec az webapp create-remote-connection --resource-group $resource_group --name $app_name ''; } From 5f2edf5a46afddad219bd19b8d39c25b639d2bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 15:54:00 -0700 Subject: [PATCH 49/70] Fix missing require --- lib/azure_blob/identity_token.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb index 82e57d6..4f5c9db 100644 --- a/lib/azure_blob/identity_token.rb +++ b/lib/azure_blob/identity_token.rb @@ -1,4 +1,5 @@ require "net/http" +require "json" module AzureBlob class IdentityToken From 48940b4c2074597c10ae6275aa7b07576cc8b049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 16:10:49 -0700 Subject: [PATCH 50/70] Extract delegation key from the sas_token method --- lib/azure_blob/entra_id_signer.rb | 20 ++++++------ lib/azure_blob/user_delegation_key.rb | 45 +++++++++++++++------------ 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/lib/azure_blob/entra_id_signer.rb b/lib/azure_blob/entra_id_signer.rb index fa8dc23..58aa20f 100644 --- a/lib/azure_blob/entra_id_signer.rb +++ b/lib/azure_blob/entra_id_signer.rb @@ -23,17 +23,11 @@ def authorization_header(uri:, verb:, headers: {}) end def sas_token(uri, options = {}) - delegation_key = UserDelegationKey.new(account_name:, signer: self) - now = Time.now.utc - start = options[:start] || now.iso8601 - expiry = options[:expiry] || (now + 5.minutes).iso8601 - - canonicalized_resource = CanonicalizedResource.new(uri, account_name, url_safe: false, service_name: :blob) to_sign = [ options[:permissions], - start, - expiry, - canonicalized_resource, + options[:start], + options[:expiry], + CanonicalizedResource.new(uri, account_name, url_safe: false, service_name: :blob), delegation_key.signed_oid, delegation_key.signed_tid, delegation_key.signed_start, @@ -58,8 +52,8 @@ def sas_token(uri, options = {}) query = { SAS::Fields::Permissions => options[:permissions], - SAS::Fields::Start => start, - SAS::Fields::Expiry => expiry, + SAS::Fields::Start => options[:start], + SAS::Fields::Expiry => options[:expiry], SAS::Fields::SignedObjectId => delegation_key.signed_oid, SAS::Fields::SignedTenantId => delegation_key.signed_tid, @@ -85,6 +79,10 @@ def sas_token(uri, options = {}) private + def delegation_key + @delegation_key ||= UserDelegationKey.new(account_name:, signer: self) + end + def sign(body, key:) Base64.strict_encode64(OpenSSL::HMAC.digest("sha256", key, body)) end diff --git a/lib/azure_blob/user_delegation_key.rb b/lib/azure_blob/user_delegation_key.rb index f5eacf1..ef9a155 100644 --- a/lib/azure_blob/user_delegation_key.rb +++ b/lib/azure_blob/user_delegation_key.rb @@ -2,6 +2,7 @@ module AzureBlob class UserDelegationKey # :nodoc: + EXPIRATION = 25200 # 7 hours def initialize(account_name:, signer:) # TODO: reuse the same key if not expired @uri = URI.parse( @@ -10,16 +11,34 @@ def initialize(account_name:, signer:) @signer = signer - now = Time.now.utc + refresh + end + + def to_s + user_delegation_key + end + + attr_reader :signed_oid, + :signed_tid, + :signed_start, + :signed_expiry, + :signed_service, + :signed_version, + :user_delegation_key + + + private - key_start = now.iso8601 - key_expiry = (now + 7.hours).iso8601 + def refresh + now = Time.now.utc + start = now.iso8601 + expiry = (now + EXPIRATION).iso8601 - content = <<-XML.squish + content = <<-XML.gsub!(/[[:space:]]+/, " ").strip! - #{key_start} - #{key_expiry} + #{start} + #{expiry} XML @@ -36,20 +55,6 @@ def initialize(account_name:, signer:) @user_delegation_key = Base64.decode64(doc.get_elements("/UserDelegationKey/Value").first.get_text.to_s) end - def to_s - user_delegation_key - end - - attr_reader :signed_oid, - :signed_tid, - :signed_start, - :signed_expiry, - :signed_service, - :signed_version, - :user_delegation_key - - - private attr_reader :uri, :user_delegation_key, :signer end end From ae53cc2fe1c35ed4d06f88e14fb43c2fd37c7e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 16:13:23 -0700 Subject: [PATCH 51/70] Consistency --- lib/azure_blob/identity_token.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb index 4f5c9db..d3c5c91 100644 --- a/lib/azure_blob/identity_token.rb +++ b/lib/azure_blob/identity_token.rb @@ -28,7 +28,7 @@ def to_s private def expired? - token.nil? || Time.now >= (expiration + EXPIRATION_BUFFER) + token.nil? || Time.now >= (expiry + EXPIRATION_BUFFER) end def refresh @@ -39,9 +39,9 @@ def refresh response = JSON.parse(Http.new(identity_uri, headers).get) @token = response['access_token'] - @expiration = Time.at(response['expires_on'].to_i) + @expiry = Time.at(response['expires_on'].to_i) end - attr_reader :identity_uri, :expiration, :token + attr_reader :identity_uri, :expiry, :token end end From 7e95ade5692e36f4e3440f7740baf25f7c8c2aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 18:16:42 -0700 Subject: [PATCH 52/70] Fix expiration buffer --- lib/azure_blob/identity_token.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb index d3c5c91..39b2878 100644 --- a/lib/azure_blob/identity_token.rb +++ b/lib/azure_blob/identity_token.rb @@ -28,7 +28,7 @@ def to_s private def expired? - token.nil? || Time.now >= (expiry + EXPIRATION_BUFFER) + token.nil? || Time.now >= (expiration - EXPIRATION_BUFFER) end def refresh From 9f97768db1b25a13ad489e0d90fbc74612eca7f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 18:17:52 -0700 Subject: [PATCH 53/70] Test identity token and user delegation key --- lib/azure_blob/identity_token.rb | 7 ++- lib/azure_blob/user_delegation_key.rb | 34 ++++++++----- test/client/test_helper.rb | 4 ++ test/client/test_identity_token.rb | 64 +++++++++++++++++++++++++ test/client/test_user_delegation_key.rb | 44 +++++++++++++++++ 5 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 test/client/test_identity_token.rb create mode 100644 test/client/test_user_delegation_key.rb diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb index 39b2878..746df3d 100644 --- a/lib/azure_blob/identity_token.rb +++ b/lib/azure_blob/identity_token.rb @@ -1,4 +1,3 @@ -require "net/http" require "json" module AzureBlob @@ -36,12 +35,12 @@ def refresh headers['X-IDENTITY-HEADER'] = ENV['IDENTITY_HEADER'] if ENV['IDENTITY_HEADER'] # TODO implement some retry strategies as per the documentation. # https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling - response = JSON.parse(Http.new(identity_uri, headers).get) + response = JSON.parse(AzureBlob::Http.new(identity_uri, headers).get) @token = response['access_token'] - @expiry = Time.at(response['expires_on'].to_i) + @expiration = Time.at(response['expires_on'].to_i) end - attr_reader :identity_uri, :expiry, :token + attr_reader :identity_uri, :expiration, :token end end diff --git a/lib/azure_blob/user_delegation_key.rb b/lib/azure_blob/user_delegation_key.rb index ef9a155..c5ba986 100644 --- a/lib/azure_blob/user_delegation_key.rb +++ b/lib/azure_blob/user_delegation_key.rb @@ -3,6 +3,7 @@ module AzureBlob class UserDelegationKey # :nodoc: EXPIRATION = 25200 # 7 hours + EXPIRATION_BUFFER = 3600 # 1 hours def initialize(account_name:, signer:) # TODO: reuse the same key if not expired @uri = URI.parse( @@ -18,21 +19,14 @@ def to_s user_delegation_key end - attr_reader :signed_oid, - :signed_tid, - :signed_start, - :signed_expiry, - :signed_service, - :signed_version, - :user_delegation_key - - - private - def refresh + return unless expired? now = Time.now.utc + + start = now.iso8601 - expiry = (now + EXPIRATION).iso8601 + @expiration = (now + EXPIRATION) + expiry = @expiration.iso8601 content = <<-XML.gsub!(/[[:space:]]+/, " ").strip! @@ -55,6 +49,20 @@ def refresh @user_delegation_key = Base64.decode64(doc.get_elements("/UserDelegationKey/Value").first.get_text.to_s) end - attr_reader :uri, :user_delegation_key, :signer + attr_reader :signed_oid, + :signed_tid, + :signed_start, + :signed_expiry, + :signed_service, + :signed_version, + :user_delegation_key + + private + + def expired? + expiration.nil? || Time.now >= (expiration - EXPIRATION_BUFFER) + end + + attr_reader :uri, :user_delegation_key, :signer, :expiration end end diff --git a/test/client/test_helper.rb b/test/client/test_helper.rb index b9062ac..74358c9 100644 --- a/test/client/test_helper.rb +++ b/test/client/test_helper.rb @@ -13,4 +13,8 @@ def assert_match_content(expected, received) assert received.include?(element) end end + + def using_shared_key + !(ENV["AZURE_ACCESS_KEY"].nil? || ENV["AZURE_ACCESS_KEY"].empty?) + end end diff --git a/test/client/test_identity_token.rb b/test/client/test_identity_token.rb new file mode 100644 index 0000000..b10997a --- /dev/null +++ b/test/client/test_identity_token.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +class TestIdentityToken < TestCase + attr_reader :identity_token + def setup + @principal_id = ENV["AZURE_PRINCIPAL_ID"] + end + + def test_do_not_refresh_under_expiration_buffer + http_mock = Minitest::Mock.new + now = Time.now + expiration = now.to_i + 3600 # Expire in 1 hour + http_mock.expect :get, JSON.generate({access_token: '123', expires_on: expiration}) + + token = nil + new_token = nil + + + AzureBlob::Http.stub :new, http_mock do + @identity_token = AzureBlob::IdentityToken.new(principal_id: @principal_id) + Time.stub :now, Time.now do + token = identity_token.to_s + end + + http_mock.expect :get, JSON.generate({access_token: '321', expires_on: expiration}) + + Time.stub :now, Time.at(now.to_i + 1000) do + new_token = identity_token.to_s + end + end + + assert_equal '123', token + assert_equal '123', new_token + end + + def test_refresh_when_over_expiration_buffer + http_mock = Minitest::Mock.new + now = Time.now + expiration = now.to_i + 3600 # Expire in 1 hour + http_mock.expect :get, JSON.generate({access_token: '123', expires_on: expiration}) + + token = nil + new_token = nil + + + AzureBlob::Http.stub :new, http_mock do + @identity_token = AzureBlob::IdentityToken.new(principal_id: @principal_id) + Time.stub :now, Time.now do + token = identity_token.to_s + end + + http_mock.expect :get, JSON.generate({access_token: '321', expires_on: expiration}) + + Time.stub :now, Time.at(expiration - 10) do + new_token = identity_token.to_s + end + end + + assert_equal '123', token + assert_equal '321', new_token + end +end diff --git a/test/client/test_user_delegation_key.rb b/test/client/test_user_delegation_key.rb new file mode 100644 index 0000000..7ba0558 --- /dev/null +++ b/test/client/test_user_delegation_key.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +class TestUserDelegationKey < TestCase + attr_reader :delegation_key + def setup + skip if using_shared_key + @account_name = ENV["AZURE_ACCOUNT_NAME"] + @principal_id = ENV["AZURE_PRINCIPAL_ID"] + @signer = AzureBlob::EntraIdSigner.new(account_name: @account_name, principal_id: @principal_id) + @delegation_key = AzureBlob::UserDelegationKey.new(account_name: @account_name, signer: @signer) + end + + def test_do_not_refresh_under_expiration_buffer + now = Time.now + five_hours = 18000 + Time.stub :now, now do + @delegation_key = AzureBlob::UserDelegationKey.new(account_name: @account_name, signer: @signer) + end + initial_expiry = @delegation_key.signed_expiry + + Time.stub :now, now + five_hours do + @delegation_key.refresh + end + + assert_equal initial_expiry, @delegation_key.signed_expiry + end + + def test_refresh_when_over_expiration_buffer + now = Time.now + after_expiration_buffer = now + 21601 + Time.stub :now, now do + @delegation_key = AzureBlob::UserDelegationKey.new(account_name: @account_name, signer: @signer) + end + + initial_expiry = delegation_key.signed_expiry + Time.stub :now, after_expiration_buffer do + delegation_key.refresh + end + + refute_equal initial_expiry, delegation_key.signed_expiry + end +end From 17b4474746c3afdd9a0ff738a71c4e35accbbcdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 20:11:13 -0700 Subject: [PATCH 54/70] Exponential backoff --- lib/azure_blob/http.rb | 11 +++++-- lib/azure_blob/identity_token.rb | 27 ++++++++++++++++-- lib/azure_blob/user_delegation_key.rb | 1 - test/client/test_identity_token.rb | 41 +++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 6 deletions(-) diff --git a/lib/azure_blob/http.rb b/lib/azure_blob/http.rb index 01bfe7c..3460d14 100644 --- a/lib/azure_blob/http.rb +++ b/lib/azure_blob/http.rb @@ -7,7 +7,14 @@ module AzureBlob class Http # :nodoc: - class Error < AzureBlob::Error; end + class Error < AzureBlob::Error + attr_reader :body, :status + def initialize body: nil, status: nil + @body = body + @status = status + super(body) + end + end class FileNotFoundError < Error; end class ForbidenError < Error; end class IntegrityError < Error; end @@ -100,7 +107,7 @@ def sign_request(method) end def raise_error - raise error_from_response.new(@response.body) + raise error_from_response.new(body: @response.body, status: @response.code&.to_i) end def status diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb index 746df3d..f33a711 100644 --- a/lib/azure_blob/identity_token.rb +++ b/lib/azure_blob/identity_token.rb @@ -33,14 +33,35 @@ def expired? def refresh headers = {'Metadata' => 'true'} headers['X-IDENTITY-HEADER'] = ENV['IDENTITY_HEADER'] if ENV['IDENTITY_HEADER'] - # TODO implement some retry strategies as per the documentation. - # https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling - response = JSON.parse(AzureBlob::Http.new(identity_uri, headers).get) + attempt = 0 + begin + attempt += 1 + response = JSON.parse(AzureBlob::Http.new(identity_uri, headers).get) + rescue AzureBlob::Http::Error => error + if should_retry?(error, attempt) + attempt = 1 if error.status == 410 + delay = exponential_backoff(error, attempt) + Kernel.sleep(delay) + retry + end + raise + end @token = response['access_token'] @expiration = Time.at(response['expires_on'].to_i) end + + def should_retry?(error, attempt) + is_500 = error.status/500 == 1 + (is_500 || [404, 408, 410, 429 ].include?(error.status)) && attempt < 5 + end + + def exponential_backoff(error, attempt) + EXPONENTIAL_BACKOFF[attempt -1] || raise(AzureBlob::Error.new("Exponential backoff out of bounds!")) + end + EXPONENTIAL_BACKOFF = [2, 6, 14, 30] + attr_reader :identity_uri, :expiration, :token end end diff --git a/lib/azure_blob/user_delegation_key.rb b/lib/azure_blob/user_delegation_key.rb index c5ba986..98f04f7 100644 --- a/lib/azure_blob/user_delegation_key.rb +++ b/lib/azure_blob/user_delegation_key.rb @@ -5,7 +5,6 @@ class UserDelegationKey # :nodoc: EXPIRATION = 25200 # 7 hours EXPIRATION_BUFFER = 3600 # 1 hours def initialize(account_name:, signer:) - # TODO: reuse the same key if not expired @uri = URI.parse( "https://#{account_name}.blob.core.windows.net/?restype=service&comp=userdelegationkey" ) diff --git a/test/client/test_identity_token.rb b/test/client/test_identity_token.rb index b10997a..e81a3c5 100644 --- a/test/client/test_identity_token.rb +++ b/test/client/test_identity_token.rb @@ -61,4 +61,45 @@ def test_refresh_when_over_expiration_buffer assert_equal '123', token assert_equal '321', new_token end + + def test_exponential_backoff + #https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling + http_mock = Minitest::Mock.new + def http_mock.get; raise AzureBlob::Http::Error.new(status: 404) end + slept = [] + sleep_lambda = ->(time){ slept << time} + AzureBlob::Http.stub :new, http_mock do + Kernel.stub :sleep, sleep_lambda do + + @identity_token = AzureBlob::IdentityToken.new(principal_id: @principal_id) + assert_raises(AzureBlob::Http::Error){ identity_token.to_s } + end + end + + assert_equal [2,6,14,30], slept + end + + + def test_410_retry + http_mock = Minitest::Mock.new + def http_mock.get; raise AzureBlob::Http::Error.new(status: 410) end + attempt = 0 + slept = [] + sleep_lambda = ->(time) do + attempt += 1 + slept << time + if attempt > 3 + def http_mock.get; raise AzureBlob::Http::Error.new(status: 404) end + end + end + AzureBlob::Http.stub :new, http_mock do + Kernel.stub :sleep, sleep_lambda do + + @identity_token = AzureBlob::IdentityToken.new(principal_id: @principal_id) + assert_raises(AzureBlob::Http::Error){ identity_token.to_s } + end + end + + assert_equal [2, 2, 2, 2, 6, 14, 30], slept + end end From 1f56a0dc8af36ca208ee3c474bab81cd62399197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Sun, 8 Sep 2024 20:22:32 -0700 Subject: [PATCH 55/70] Managed identity readme --- README.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 615ebe2..2539b9f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This gem was built to replace azure-storage-blob (deprecated) in Active Storage, ## Active Storage -## Migration +### Migration To migrate from azure-storage-blob to azure-blob: 1. Replace `azure-storage-blob` in your Gemfile with `azure-blob` @@ -12,6 +12,29 @@ To migrate from azure-storage-blob to azure-blob: 3. Change the `AzureStorage` service to `AzureBlob` in your Active Storage config (`config/storage.yml`) 4. Restart or deploy the app. +### Managed Identity (Entra ID) + +AzureBlob supports managed identities on : +- Azure VM +- App Service +- Azure Functions (Untested but should work) +- Azure Containers (Untested but should work) + +AKS support will likely require more work. Contributions are welcome. + +To authenticate through managed identities instead of a shared key, omit `storage_access_key` from your `storage.yml` file. + +It is recommended to add the role `principal_id` to the config. + +ActiveStorage config example: + +``` +prod: + service: AzureBlob + container: container_name + storage_account_name: account_name + principal_id: 71b34410-4c50-451d-b456-95ead1b18cce +``` ## Standalone From 117927ad0fc49937062cb0dde52e095ef86e722f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 16:19:20 -0700 Subject: [PATCH 56/70] Test with the minimum ruby version --- devenv.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devenv.nix b/devenv.nix index 1fd9687..7c4b1ab 100644 --- a/devenv.nix +++ b/devenv.nix @@ -18,7 +18,7 @@ ]; languages.ruby.enable = true; - languages.ruby.version = "3.2.1"; + languages.ruby.version = "3.1.6"; scripts.sync-vm.exec = '' vm_username=$(terraform output --raw "vm_username") From c9857fe1da98e6476985303c37ca3c0bfeddc65c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 16:19:41 -0700 Subject: [PATCH 57/70] Enforce MFA --- azure-blob.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-blob.gemspec b/azure-blob.gemspec index dee8966..30dec42 100644 --- a/azure-blob.gemspec +++ b/azure-blob.gemspec @@ -13,6 +13,7 @@ Gem::Specification.new do |spec| spec.license = "MIT" spec.required_ruby_version = ">= 3.1" + spec.metadata["rubygems_mfa_required"] = "true" spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = spec.homepage spec.metadata["changelog_uri"] = "https://github.com/testdouble/azure-blob/blob/main/CHANGELOG.md" From f21e939a8db7d691500a23e1b2153f63308d546b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 16:39:44 -0700 Subject: [PATCH 58/70] Lint --- Rakefile | 8 +++---- lib/azure_blob/entra_id_signer.rb | 2 +- lib/azure_blob/http.rb | 2 +- lib/azure_blob/identity_token.rb | 19 ++++++++--------- lib/azure_blob/user_delegation_key.rb | 2 +- test/client/test_identity_token.rb | 30 +++++++++++++-------------- test/support/app_service_vpn.rb | 18 ++++++++-------- test/support/azure_vm_vpn.rb | 6 +++--- 8 files changed, 42 insertions(+), 45 deletions(-) diff --git a/Rakefile b/Rakefile index 274ba3e..768227f 100644 --- a/Rakefile +++ b/Rakefile @@ -7,13 +7,13 @@ require_relative 'test/support/app_service_vpn' require_relative 'test/support/azure_vm_vpn' Minitest::TestTask.create(:test_rails) do - self.test_globs = ["test/rails/**/test_*.rb", - "test/rails/**/*_test.rb"] + self.test_globs = [ "test/rails/**/test_*.rb", + "test/rails/**/*_test.rb", ] end Minitest::TestTask.create(:test_client) do - self.test_globs = ["test/client/**/test_*.rb", - "test/client/**/*_test.rb"] + self.test_globs = [ "test/client/**/test_*.rb", + "test/client/**/*_test.rb", ] end task default: %i[test] diff --git a/lib/azure_blob/entra_id_signer.rb b/lib/azure_blob/entra_id_signer.rb index 58aa20f..93d827d 100644 --- a/lib/azure_blob/entra_id_signer.rb +++ b/lib/azure_blob/entra_id_signer.rb @@ -6,7 +6,7 @@ require_relative "canonicalized_resource" require_relative "identity_token" -require_relative 'user_delegation_key' +require_relative "user_delegation_key" module AzureBlob class EntraIdSigner # :nodoc: diff --git a/lib/azure_blob/http.rb b/lib/azure_blob/http.rb index 3460d14..5e52c24 100644 --- a/lib/azure_blob/http.rb +++ b/lib/azure_blob/http.rb @@ -9,7 +9,7 @@ module AzureBlob class Http # :nodoc: class Error < AzureBlob::Error attr_reader :body, :status - def initialize body: nil, status: nil + def initialize(body: nil, status: nil) @body = body @status = status super(body) diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb index f33a711..44d2523 100644 --- a/lib/azure_blob/identity_token.rb +++ b/lib/azure_blob/identity_token.rb @@ -2,12 +2,11 @@ module AzureBlob class IdentityToken - - RESOURCE_URI = 'https://storage.azure.com/' + RESOURCE_URI = "https://storage.azure.com/" EXPIRATION_BUFFER = 600 # 10 minutes - IDENTITY_ENDPOINT = ENV["IDENTITY_ENDPOINT"] || 'http://169.254.169.254/metadata/identity/oauth2/token' - API_VERSION = ENV["IDENTITY_ENDPOINT"] ? '2019-08-01' : '2018-02-01' + IDENTITY_ENDPOINT = ENV["IDENTITY_ENDPOINT"] || "http://169.254.169.254/metadata/identity/oauth2/token" + API_VERSION = ENV["IDENTITY_ENDPOINT"] ? "2019-08-01" : "2018-02-01" def initialize(principal_id: nil) @identity_uri = URI.parse(IDENTITY_ENDPOINT) @@ -31,8 +30,8 @@ def expired? end def refresh - headers = {'Metadata' => 'true'} - headers['X-IDENTITY-HEADER'] = ENV['IDENTITY_HEADER'] if ENV['IDENTITY_HEADER'] + headers = { "Metadata" => "true" } + headers["X-IDENTITY-HEADER"] = ENV["IDENTITY_HEADER"] if ENV["IDENTITY_HEADER"] attempt = 0 begin @@ -47,20 +46,20 @@ def refresh end raise end - @token = response['access_token'] - @expiration = Time.at(response['expires_on'].to_i) + @token = response["access_token"] + @expiration = Time.at(response["expires_on"].to_i) end def should_retry?(error, attempt) is_500 = error.status/500 == 1 - (is_500 || [404, 408, 410, 429 ].include?(error.status)) && attempt < 5 + (is_500 || [ 404, 408, 410, 429 ].include?(error.status)) && attempt < 5 end def exponential_backoff(error, attempt) EXPONENTIAL_BACKOFF[attempt -1] || raise(AzureBlob::Error.new("Exponential backoff out of bounds!")) end - EXPONENTIAL_BACKOFF = [2, 6, 14, 30] + EXPONENTIAL_BACKOFF = [ 2, 6, 14, 30 ] attr_reader :identity_uri, :expiration, :token end diff --git a/lib/azure_blob/user_delegation_key.rb b/lib/azure_blob/user_delegation_key.rb index 98f04f7..d374164 100644 --- a/lib/azure_blob/user_delegation_key.rb +++ b/lib/azure_blob/user_delegation_key.rb @@ -1,4 +1,4 @@ -require_relative 'http' +require_relative "http" module AzureBlob class UserDelegationKey # :nodoc: diff --git a/test/client/test_identity_token.rb b/test/client/test_identity_token.rb index e81a3c5..0a23a8d 100644 --- a/test/client/test_identity_token.rb +++ b/test/client/test_identity_token.rb @@ -12,7 +12,7 @@ def test_do_not_refresh_under_expiration_buffer http_mock = Minitest::Mock.new now = Time.now expiration = now.to_i + 3600 # Expire in 1 hour - http_mock.expect :get, JSON.generate({access_token: '123', expires_on: expiration}) + http_mock.expect :get, JSON.generate({ access_token: "123", expires_on: expiration }) token = nil new_token = nil @@ -24,22 +24,22 @@ def test_do_not_refresh_under_expiration_buffer token = identity_token.to_s end - http_mock.expect :get, JSON.generate({access_token: '321', expires_on: expiration}) + http_mock.expect :get, JSON.generate({ access_token: "321", expires_on: expiration }) Time.stub :now, Time.at(now.to_i + 1000) do new_token = identity_token.to_s end end - assert_equal '123', token - assert_equal '123', new_token + assert_equal "123", token + assert_equal "123", new_token end def test_refresh_when_over_expiration_buffer http_mock = Minitest::Mock.new now = Time.now expiration = now.to_i + 3600 # Expire in 1 hour - http_mock.expect :get, JSON.generate({access_token: '123', expires_on: expiration}) + http_mock.expect :get, JSON.generate({ access_token: "123", expires_on: expiration }) token = nil new_token = nil @@ -51,32 +51,31 @@ def test_refresh_when_over_expiration_buffer token = identity_token.to_s end - http_mock.expect :get, JSON.generate({access_token: '321', expires_on: expiration}) + http_mock.expect :get, JSON.generate({ access_token: "321", expires_on: expiration }) Time.stub :now, Time.at(expiration - 10) do new_token = identity_token.to_s end end - assert_equal '123', token - assert_equal '321', new_token + assert_equal "123", token + assert_equal "321", new_token end def test_exponential_backoff - #https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling + # https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#error-handling http_mock = Minitest::Mock.new def http_mock.get; raise AzureBlob::Http::Error.new(status: 404) end slept = [] - sleep_lambda = ->(time){ slept << time} + sleep_lambda = ->(time) { slept << time } AzureBlob::Http.stub :new, http_mock do Kernel.stub :sleep, sleep_lambda do - @identity_token = AzureBlob::IdentityToken.new(principal_id: @principal_id) - assert_raises(AzureBlob::Http::Error){ identity_token.to_s } + assert_raises(AzureBlob::Http::Error) { identity_token.to_s } end end - assert_equal [2,6,14,30], slept + assert_equal [ 2, 6, 14, 30 ], slept end @@ -94,12 +93,11 @@ def http_mock.get; raise AzureBlob::Http::Error.new(status: 404) end end AzureBlob::Http.stub :new, http_mock do Kernel.stub :sleep, sleep_lambda do - @identity_token = AzureBlob::IdentityToken.new(principal_id: @principal_id) - assert_raises(AzureBlob::Http::Error){ identity_token.to_s } + assert_raises(AzureBlob::Http::Error) { identity_token.to_s } end end - assert_equal [2, 2, 2, 2, 6, 14, 30], slept + assert_equal [ 2, 2, 2, 2, 6, 14, 30 ], slept end end diff --git a/test/support/app_service_vpn.rb b/test/support/app_service_vpn.rb index dab1b9c..852ab5c 100644 --- a/test/support/app_service_vpn.rb +++ b/test/support/app_service_vpn.rb @@ -1,13 +1,13 @@ -require 'open3' -require 'net/ssh' -require 'shellwords' +require "open3" +require "net/ssh" +require "shellwords" class AppServiceVpn - HOST = '127.0.0.1' + HOST = "127.0.0.1" attr_reader :header, :endpoint - def initialize verbose: false + def initialize(verbose: false) @verbose = verbose establish_vpn_connection end @@ -25,7 +25,7 @@ def establish_vpn_connection puts "Establishing VPN connection..." - tunnel_stdin, tunnel_stdout, @tunnel_wait_thread = Open3.popen2e(["sshuttle", "-e", "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null", "-r", "#{username}:#{password}@#{HOST}:#{port}", "0/0"].shelljoin) + tunnel_stdin, tunnel_stdout, @tunnel_wait_thread = Open3.popen2e([ "sshuttle", "-e", "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null", "-r", "#{username}:#{password}@#{HOST}:#{port}", "0/0" ].shelljoin) connection_successful = false tunnel_stdout.each do |line| @@ -41,7 +41,7 @@ def establish_vpn_connection end def establish_app_service_tunnel - puts 'Establishing tunnel connection to app service...' + puts "Establishing tunnel connection to app service..." connection_stdin, connection_stdout, @connection_wait_thread = Open3.popen2e("start-app-service-ssh") port = nil @@ -75,8 +75,8 @@ def extract_msi_info endpoint = nil header = nil Net::SSH.start(HOST, username, password:, port:) do |ssh| - endpoint = ssh.exec! ["bash", "-l", "-c", "echo -n $IDENTITY_ENDPOINT"].shelljoin - header = ssh.exec! ["bash", "-l", "-c", "echo -n $IDENTITY_HEADER"].shelljoin + endpoint = ssh.exec! [ "bash", "-l", "-c", "echo -n $IDENTITY_ENDPOINT" ].shelljoin + header = ssh.exec! [ "bash", "-l", "-c", "echo -n $IDENTITY_HEADER" ].shelljoin end raise "Could not extract MSI endpoint information" unless endpoint && header @endpoint = endpoint diff --git a/test/support/azure_vm_vpn.rb b/test/support/azure_vm_vpn.rb index 6afba58..f18144b 100644 --- a/test/support/azure_vm_vpn.rb +++ b/test/support/azure_vm_vpn.rb @@ -1,8 +1,8 @@ -require 'open3' -require 'shellwords' +require "open3" +require "shellwords" class AzureVmVpn - def initialize verbose: false + def initialize(verbose: false) @verbose = verbose stdin, stdout, @wait_thread = Open3.popen2e("proxy-vps") stdout.each do |line| From 040a52d94447da1dee26d0cbdf12e2a3e4835cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 16:42:43 -0700 Subject: [PATCH 59/70] Replace echo with printf --- test/support/app_service_vpn.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/support/app_service_vpn.rb b/test/support/app_service_vpn.rb index 852ab5c..6ea2756 100644 --- a/test/support/app_service_vpn.rb +++ b/test/support/app_service_vpn.rb @@ -75,8 +75,8 @@ def extract_msi_info endpoint = nil header = nil Net::SSH.start(HOST, username, password:, port:) do |ssh| - endpoint = ssh.exec! [ "bash", "-l", "-c", "echo -n $IDENTITY_ENDPOINT" ].shelljoin - header = ssh.exec! [ "bash", "-l", "-c", "echo -n $IDENTITY_HEADER" ].shelljoin + endpoint = ssh.exec! [ "bash", "-l", "-c", %(printf "%s" "$IDENTITY_ENDPOINT") ].shelljoin + header = ssh.exec! [ "bash", "-l", "-c", %(printf "%s" "$IDENTITY_HEADER") ].shelljoin end raise "Could not extract MSI endpoint information" unless endpoint && header @endpoint = endpoint From 04da4d39a4e21ac6ff1f0d1c466a13ef4eb3fe15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 16:46:22 -0700 Subject: [PATCH 60/70] Ensure the client code was tested without Rails --- test/client/test_client.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/client/test_client.rb b/test/client/test_client.rb index 4bd7870..3d0b191 100644 --- a/test/client/test_client.rb +++ b/test/client/test_client.rb @@ -28,6 +28,10 @@ def teardown rescue AzureBlob::Http::FileNotFoundError end + def test_rails_is_not_loaded + assert_raises(NoMethodError) { 10.minutes } + end + def test_single_block_upload client.create_block_blob(key, content) From 81af788d799710a2a0223e2fccf748ce12f63ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 16:49:21 -0700 Subject: [PATCH 61/70] Fix a typo in the doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2539b9f..178ace5 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ AKS support will likely require more work. Contributions are welcome. To authenticate through managed identities instead of a shared key, omit `storage_access_key` from your `storage.yml` file. -It is recommended to add the role `principal_id` to the config. +It is recommended to add the identity `principal_id` to the config. ActiveStorage config example: From 2b4dc0348e6a3fb47117b55f955842b9cac4ce01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 16:50:14 -0700 Subject: [PATCH 62/70] Spacing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 178ace5..b0b72d6 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ AzureBlob supports managed identities on : AKS support will likely require more work. Contributions are welcome. -To authenticate through managed identities instead of a shared key, omit `storage_access_key` from your `storage.yml` file. +To authenticate through managed identities instead of a shared key, omit `storage_access_key` from your `storage.yml` file. It is recommended to add the identity `principal_id` to the config. From 9cea737e6533f5445f60691fc4da1159180412c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 16:50:42 -0700 Subject: [PATCH 63/70] Grammar --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b0b72d6..f0f8ef5 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ AKS support will likely require more work. Contributions are welcome. To authenticate through managed identities instead of a shared key, omit `storage_access_key` from your `storage.yml` file. -It is recommended to add the identity `principal_id` to the config. +It is recommended to add the identity's `principal_id` to the config. ActiveStorage config example: From 56b7561c28383ad1a9c7f7d69c31a171a695052d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 16:56:58 -0700 Subject: [PATCH 64/70] Simplify the managed id instructions --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f0f8ef5..891d92f 100644 --- a/README.md +++ b/README.md @@ -87,17 +87,15 @@ create_vm = true ``` and re-apply terraform: `terraform apply -var-file=var.tfvars`. -This will create the VPS and required roles. +This will create the VPS and required managed identities. -Use `proxy-vps` to proxy all network requests through the vps with sshuttle. sshuttle will likely ask for a sudo password. +`bin/rake test_azure_vm` and `bin/rake test_app_service` will establish a VPN connection to the VM or App service container and run the test suite. You might be prompted for a sudo password when the VPN starts (sshuttle). -Then use `bin/rake test_entra_id` to run the tests with Entra ID. - -After you are done, running terraform again without the var file (`terraform apply`) it should destroy the VPS. +After you are done, run terraform again without the var file (`terraform apply`) to destroy the VPS and App service application. #### Cleanup -Some test copied over from Rails codebase don't clean after themselves. A rake task is provided to empty your containers and keep cost low: `bin/rake flush_test_container` +Some tests copied over from Rails don't clean after themselves. A rake task is provided to empty your containers and keep cost low: `bin/rake flush_test_container` #### Run without devenv/nix From 6a9d31638099aa845e35299aa3faebc528f425a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 17:00:53 -0700 Subject: [PATCH 65/70] Lint --- Rakefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Rakefile b/Rakefile index 768227f..7536975 100644 --- a/Rakefile +++ b/Rakefile @@ -2,9 +2,9 @@ require "bundler/gem_tasks" require "minitest/test_task" -require 'azure_blob' -require_relative 'test/support/app_service_vpn' -require_relative 'test/support/azure_vm_vpn' +require "azure_blob" +require_relative "test/support/app_service_vpn" +require_relative "test/support/azure_vm_vpn" Minitest::TestTask.create(:test_rails) do self.test_globs = [ "test/rails/**/test_*.rb", @@ -49,10 +49,10 @@ task :flush_test_container do |t| account_name: ENV["AZURE_ACCOUNT_NAME"], access_key: ENV["AZURE_ACCESS_KEY"], container: ENV["AZURE_PRIVATE_CONTAINER"], - ).delete_prefix '' + ).delete_prefix "" AzureBlob::Client.new( account_name: ENV["AZURE_ACCOUNT_NAME"], access_key: ENV["AZURE_ACCESS_KEY"], container: ENV["AZURE_PUBLIC_CONTAINER"], - ).delete_prefix '' + ).delete_prefix "" end From 7a3a06c36601d4d19a172b861aeec2081ca8d953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 17:02:33 -0700 Subject: [PATCH 66/70] Not building containers yet --- devenv.lock | 75 ++--------------------------------------------------- devenv.yaml | 7 ----- 2 files changed, 2 insertions(+), 80 deletions(-) diff --git a/devenv.lock b/devenv.lock index d2b5e4e..875aa94 100644 --- a/devenv.lock +++ b/devenv.lock @@ -71,24 +71,6 @@ "inputs": { "systems": "systems_2" }, - "locked": { - "lastModified": 1710146030, - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "treeHash": "bd263f021e345cb4a39d80c126ab650bebc3c10c", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_3": { - "inputs": { - "systems": "systems_3" - }, "locked": { "lastModified": 1710146030, "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", @@ -124,42 +106,6 @@ "type": "github" } }, - "mk-shell-bin": { - "locked": { - "lastModified": 1677004959, - "owner": "rrbutani", - "repo": "nix-mk-shell-bin", - "rev": "ff5d8bd4d68a347be5042e2f16caee391cd75887", - "treeHash": "496327dabdc787353a29987f492dd4939151baad", - "type": "github" - }, - "original": { - "owner": "rrbutani", - "repo": "nix-mk-shell-bin", - "type": "github" - } - }, - "nix2container": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1724996935, - "owner": "nlewo", - "repo": "nix2container", - "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401", - "treeHash": "a934d246fadcf8b36d28f3577fad413f5ab3f7d3", - "type": "github" - }, - "original": { - "owner": "nlewo", - "repo": "nix2container", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1725001927, @@ -179,7 +125,7 @@ "nixpkgs-ruby": { "inputs": { "flake-compat": "flake-compat", - "flake-utils": "flake-utils_2", + "flake-utils": "flake-utils", "nixpkgs": "nixpkgs_2" }, "locked": { @@ -231,7 +177,7 @@ "pre-commit-hooks": { "inputs": { "flake-compat": "flake-compat_2", - "flake-utils": "flake-utils_3", + "flake-utils": "flake-utils_2", "gitignore": "gitignore", "nixpkgs": [ "nixpkgs" @@ -255,8 +201,6 @@ "root": { "inputs": { "devenv": "devenv", - "mk-shell-bin": "mk-shell-bin", - "nix2container": "nix2container", "nixpkgs": "nixpkgs", "nixpkgs-ruby": "nixpkgs-ruby", "pre-commit-hooks": "pre-commit-hooks" @@ -278,21 +222,6 @@ } }, "systems_2": { - "locked": { - "lastModified": 1681028828, - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "treeHash": "cce81f2a0f0743b2eb61bc2eb6c7adbe2f2c6beb", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "systems_3": { "locked": { "lastModified": 1681028828, "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", diff --git a/devenv.yaml b/devenv.yaml index 7e5376c..c3f54eb 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -1,12 +1,5 @@ allowUnfree: true inputs: - nix2container: - url: github:nlewo/nix2container - inputs: - nixpkgs: - follows: nixpkgs - mk-shell-bin: - url: github:rrbutani/nix-mk-shell-bin nixpkgs: url: github:NixOS/nixpkgs/nixos-24.05 nixpkgs-ruby: From 695c54482194132fd0201788af4ad608024779bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 17:03:51 -0700 Subject: [PATCH 67/70] Remove unused script --- devenv.nix | 6 ------ 1 file changed, 6 deletions(-) diff --git a/devenv.nix b/devenv.nix index 7c4b1ab..6b8d95d 100644 --- a/devenv.nix +++ b/devenv.nix @@ -20,12 +20,6 @@ languages.ruby.enable = true; languages.ruby.version = "3.1.6"; - scripts.sync-vm.exec = '' - vm_username=$(terraform output --raw "vm_username") - vm_ip=$(terraform output --raw "vm_ip") - rsync -avx --progress --exclude .devenv --exclude .terraform . $vm_username@$vm_ip:azure-blob/ - ''; - scripts.generate-env-file.exec = '' terraform output -raw devenv_local_nix > devenv.local.nix ''; From 9d9bbbad622f693a2770235e0b97ea22108f6a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 17:03:58 -0700 Subject: [PATCH 68/70] Simplify instructions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 891d92f..4596e1b 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ A dev environment is supplied through Nix with [devenv](https://devenv.sh/). 3. Log into azure CLI with `az login` 4. `terraform init` 5. `terraform apply` This will generate the necessary infrastructure on azure. -6. Generate devenv.local.nix with your private key and container information: `terraform output -raw devenv_local_nix > devenv.local.nix` +6. Generate devenv.local.nix with your private key and container information: `generate-env-file` 7. If you are using direnv, the environment will reload automatically. If not, exit the shell and reopen it by hitting and running `devenv shell` again. #### Entra ID From 705b0ceae3298f37f9373b5701d272f03c19c71c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 17:07:05 -0700 Subject: [PATCH 69/70] Spacing --- lib/azure_blob/identity_token.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb index 44d2523..0c00dae 100644 --- a/lib/azure_blob/identity_token.rb +++ b/lib/azure_blob/identity_token.rb @@ -50,7 +50,6 @@ def refresh @expiration = Time.at(response["expires_on"].to_i) end - def should_retry?(error, attempt) is_500 = error.status/500 == 1 (is_500 || [ 404, 408, 410, 429 ].include?(error.status)) && attempt < 5 From c5b28a7dd0bfad8b8e7fcc6ec4da36cd2d21e23a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A9=20Dupuis?= Date: Mon, 9 Sep 2024 17:09:00 -0700 Subject: [PATCH 70/70] Lint --- input.tf | 12 +++---- main.tf | 57 ++++++++++++++---------------- output.tf | 2 +- test/client/test_identity_token.rb | 2 -- 4 files changed, 34 insertions(+), 39 deletions(-) diff --git a/input.tf b/input.tf index 1b1500c..6bd3273 100644 --- a/input.tf +++ b/input.tf @@ -14,31 +14,31 @@ variable "storage_account_name" { } variable "create_vm" { - type = bool + type = bool default = false } variable "vm_size" { - type = string + type = string default = "Standard_B2s" } variable "vm_username" { - type = string + type = string default = "azureblob" } variable "vm_password" { - type = string + type = string default = "qwe123QWE!@#" } variable "create_app_service" { - type = bool + type = bool default = false } variable "ssh_key" { - type = string + type = string default = "" } diff --git a/main.tf b/main.tf index 002a6ce..d243219 100644 --- a/main.tf +++ b/main.tf @@ -47,9 +47,8 @@ resource "azurerm_storage_container" "public" { container_access_type = "blob" } - resource "azurerm_virtual_network" "main" { - count = var.create_vm ? 1 : 0 + count = var.create_vm ? 1 : 0 name = "${var.prefix}-network" address_space = ["10.0.0.0/16"] location = azurerm_resource_group.main.location @@ -61,7 +60,7 @@ resource "azurerm_virtual_network" "main" { } resource "azurerm_subnet" "main" { - count = var.create_vm ? 1 : 0 + count = var.create_vm ? 1 : 0 name = "${var.prefix}-main" resource_group_name = azurerm_resource_group.main.name virtual_network_name = azurerm_virtual_network.main[0].name @@ -69,7 +68,7 @@ resource "azurerm_subnet" "main" { } resource "azurerm_network_interface" "main" { - count = var.create_vm ? 1 : 0 + count = var.create_vm ? 1 : 0 name = "${var.prefix}-nic" location = azurerm_resource_group.main.location resource_group_name = azurerm_resource_group.main.name @@ -78,7 +77,7 @@ resource "azurerm_network_interface" "main" { name = "${var.prefix}-ip-config" subnet_id = azurerm_subnet.main[0].id private_ip_address_allocation = "Dynamic" - public_ip_address_id = azurerm_public_ip.main[0].id + public_ip_address_id = azurerm_public_ip.main[0].id } tags = { @@ -87,7 +86,7 @@ resource "azurerm_network_interface" "main" { } resource "azurerm_public_ip" "main" { - count = var.create_vm ? 1 : 0 + count = var.create_vm ? 1 : 0 name = "${var.prefix}-public-ip" resource_group_name = azurerm_resource_group.main.name location = azurerm_resource_group.main.location @@ -99,9 +98,9 @@ resource "azurerm_public_ip" "main" { } resource "azurerm_user_assigned_identity" "vm" { - location = azurerm_resource_group.main.location - name = "${var.prefix}-vm" - resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location + name = "${var.prefix}-vm" + resource_group_name = azurerm_resource_group.main.name } @@ -112,24 +111,24 @@ resource "azurerm_role_assignment" "vm" { } resource "azurerm_linux_virtual_machine" "main" { - count = var.create_vm ? 1 : 0 - name = "${var.prefix}-vm" - computer_name = var.prefix - resource_group_name = azurerm_resource_group.main.name - location = azurerm_resource_group.main.location - size = var.vm_size - admin_username = var.vm_username - admin_password = var.vm_password + count = var.create_vm ? 1 : 0 + name = "${var.prefix}-vm" + computer_name = var.prefix + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location + size = var.vm_size + admin_username = var.vm_username + admin_password = var.vm_password disable_password_authentication = true - network_interface_ids = [azurerm_network_interface.main[0].id] + network_interface_ids = [azurerm_network_interface.main[0].id] identity { - type = "UserAssigned" + type = "UserAssigned" identity_ids = [azurerm_user_assigned_identity.vm.id] } admin_ssh_key { - username = var.vm_username + username = var.vm_username public_key = local.public_ssh_key } @@ -150,9 +149,8 @@ resource "azurerm_linux_virtual_machine" "main" { } } - resource "azurerm_service_plan" "main" { - count = var.create_app_service ? 1 : 0 + count = var.create_app_service ? 1 : 0 name = "${var.prefix}-appserviceplan" resource_group_name = azurerm_resource_group.main.name location = azurerm_resource_group.main.location @@ -160,16 +158,15 @@ resource "azurerm_service_plan" "main" { sku_name = "B1" } - resource "azurerm_linux_web_app" "main" { - count = var.create_app_service ? 1 : 0 + count = var.create_app_service ? 1 : 0 name = "${var.prefix}-app" service_plan_id = azurerm_service_plan.main[0].id resource_group_name = azurerm_resource_group.main.name location = azurerm_resource_group.main.location identity { - type = "UserAssigned" + type = "UserAssigned" identity_ids = [azurerm_user_assigned_identity.vm.id] } @@ -181,10 +178,10 @@ resource "azurerm_linux_web_app" "main" { } resource "azurerm_app_service_source_control" "main" { - count = var.create_app_service ? 1 : 0 - app_id = azurerm_linux_web_app.main[0].id - repo_url = "https://github.com/Azure-Samples/nodejs-docs-hello-world" - branch = "master" + count = var.create_app_service ? 1 : 0 + app_id = azurerm_linux_web_app.main[0].id + repo_url = "https://github.com/Azure-Samples/nodejs-docs-hello-world" + branch = "master" use_manual_integration = true - use_mercurial = false + use_mercurial = false } diff --git a/output.tf b/output.tf index e955d08..fe570ba 100644 --- a/output.tf +++ b/output.tf @@ -1,6 +1,6 @@ output "devenv_local_nix" { sensitive = true - value = <