diff --git a/.gitignore b/.gitignore index ba232bf..d723847 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,6 @@ devenv.local.nix /.terraform terraform.tfstate -terraform.tfstate.backup \ No newline at end of file +terraform.tfstate.backup +.terraform.tfstate.lock.info +*.tfvars \ No newline at end of file diff --git a/Gemfile b/Gemfile index 89a7435..c3e24cf 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ gemspec gem "rake" gem "minitest" +gem "minitest-fail-fast" gem "rubocop-rails-omakase" @@ -19,3 +20,8 @@ gem "rails", github: "rails/rails", branch: "main" gem "propshaft", ">= 0.1.7" gem "image_processing", "~> 1.2" gem "sqlite3", ">= 1.6.6" + + +gem "net-ssh" +gem "ed25519" +gem "bcrypt_pbkdf" diff --git a/Gemfile.lock b/Gemfile.lock index 70b47e6..a8224d7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -105,6 +105,8 @@ GEM specs: ast (2.4.2) base64 (0.2.0) + bcrypt_pbkdf (1.1.1) + bcrypt_pbkdf (1.1.1-arm64-darwin) bigdecimal (3.1.8) builder (3.2.4) concurrent-ruby (1.2.3) @@ -115,6 +117,7 @@ GEM irb (~> 1.10) reline (>= 0.3.8) drb (2.2.1) + ed25519 (1.3.0) erubi (1.12.0) ffi (1.16.3) globalid (1.2.1) @@ -142,6 +145,8 @@ GEM mini_magick (4.12.0) mini_mime (1.1.5) minitest (5.23.1) + minitest-fail-fast (0.1.0) + minitest (~> 5) net-imap (0.4.11) date net-protocol @@ -151,7 +156,10 @@ GEM timeout net-smtp (0.5.0) net-protocol + net-ssh (7.2.3) nio4r (2.7.3) + nokogiri (1.16.5-arm64-darwin) + racc (~> 1.4) nokogiri (1.16.5-x86_64-linux) racc (~> 1.4) parallel (1.24.0) @@ -222,8 +230,9 @@ GEM ruby-progressbar (1.13.0) ruby-vips (2.2.1) ffi (~> 1.12) + sqlite3 (2.0.2-arm64-darwin) sqlite3 (2.0.2-x86_64-linux-gnu) - stringio (3.1.0) + stringio (3.1.1) strscan (3.1.0) thor (1.3.1) timeout (0.4.1) @@ -238,13 +247,18 @@ GEM zeitwerk (2.6.14) PLATFORMS + arm64-darwin-23 x86_64-linux DEPENDENCIES azure-blob! + bcrypt_pbkdf debug + ed25519 image_processing (~> 1.2) minitest + minitest-fail-fast + net-ssh propshaft (>= 0.1.7) rails! rake @@ -252,4 +266,4 @@ DEPENDENCIES sqlite3 (>= 1.6.6) BUNDLED WITH - 2.4.22 + 2.5.9 diff --git a/README.md b/README.md index 124ed2a..4596e1b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This gem was built to replace azure-storage-blob (deprecated) in Active Storage, ## Active Storage -## Migration +### Migration To migrate from azure-storage-blob to azure-blob: 1. Replace `azure-storage-blob` in your Gemfile with `azure-blob` @@ -12,6 +12,29 @@ To migrate from azure-storage-blob to azure-blob: 3. Change the `AzureStorage` service to `AzureBlob` in your Active Storage config (`config/storage.yml`) 4. Restart or deploy the app. +### Managed Identity (Entra ID) + +AzureBlob supports managed identities on : +- Azure VM +- App Service +- Azure Functions (Untested but should work) +- Azure Containers (Untested but should work) + +AKS support will likely require more work. Contributions are welcome. + +To authenticate through managed identities instead of a shared key, omit `storage_access_key` from your `storage.yml` file. + +It is recommended to add the identity's `principal_id` to the config. + +ActiveStorage config example: + +``` +prod: + service: AzureBlob + container: container_name + storage_account_name: account_name + principal_id: 71b34410-4c50-451d-b456-95ead1b18cce +``` ## Standalone @@ -48,10 +71,34 @@ A dev environment is supplied through Nix with [devenv](https://devenv.sh/). 2. Enter the dev environment by cd into the repo and running `devenv shell` (or `direnv allow` if you are a direnv user). 3. Log into azure CLI with `az login` 4. `terraform init` -5. `terraform apply` This will generate the necessary infrastructure on azure -6. Generate devenv.local.nix with your private key and container information: `terraform output -raw devenv_local_nix > devenv.local.nix` +5. `terraform apply` This will generate the necessary infrastructure on azure. +6. Generate devenv.local.nix with your private key and container information: `generate-env-file` 7. If you are using direnv, the environment will reload automatically. If not, exit the shell and reopen it by hitting and running `devenv shell` again. +#### Entra ID + +To test with Entra ID, the `AZURE_ACCESS_KEY` environment variable must be unset and the code must be ran or proxied through a VPS with the proper roles. + +For cost saving, the terraform variable `create_vm` is false by default. +To create the VPS, Create a var file `var.tfvars` containing: + +``` +create_vm = true +``` +and re-apply terraform: `terraform apply -var-file=var.tfvars`. + +This will create the VPS and required managed identities. + +`bin/rake test_azure_vm` and `bin/rake test_app_service` will establish a VPN connection to the VM or App service container and run the test suite. You might be prompted for a sudo password when the VPN starts (sshuttle). + +After you are done, run terraform again without the var file (`terraform apply`) to destroy the VPS and App service application. + +#### Cleanup + +Some tests copied over from Rails don't clean after themselves. A rake task is provided to empty your containers and keep cost low: `bin/rake flush_test_container` + +#### Run without devenv/nix + If you prefer not using devenv/nix: Ensure your version of Ruby fit the minimum version in `azure-blob.gemspec` @@ -63,11 +110,6 @@ and setup those Env variables: - `AZURE_PRIVATE_CONTAINER` - `AZURE_PUBLIC_CONTAINER` - -### Tests - -`bin/rake test` - ## License The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/Rakefile b/Rakefile index 56d2819..7536975 100644 --- a/Rakefile +++ b/Rakefile @@ -2,21 +2,57 @@ require "bundler/gem_tasks" require "minitest/test_task" -require 'azure_blob' +require "azure_blob" +require_relative "test/support/app_service_vpn" +require_relative "test/support/azure_vm_vpn" -Minitest::TestTask.create +Minitest::TestTask.create(:test_rails) do + self.test_globs = [ "test/rails/**/test_*.rb", + "test/rails/**/*_test.rb", ] +end + +Minitest::TestTask.create(:test_client) do + self.test_globs = [ "test/client/**/test_*.rb", + "test/client/**/*_test.rb", ] +end task default: %i[test] +task :test do + Rake::Task["test_client"].execute + Rake::Task["test_rails"].execute +end + +task :test_app_service do |t| + vpn = AppServiceVpn.new + ENV["IDENTITY_ENDPOINT"] = vpn.endpoint + ENV["IDENTITY_HEADER"] = vpn.header + Rake::Task["test_entra_id"].execute +ensure + vpn.kill +end + +task :test_azure_vm do |t| + vpn = AzureVmVpn.new + Rake::Task["test_entra_id"].execute +ensure + vpn.kill +end + +task :test_entra_id do |t| + ENV["AZURE_ACCESS_KEY"] = nil + Rake::Task["test"].execute +end + task :flush_test_container do |t| AzureBlob::Client.new( account_name: ENV["AZURE_ACCOUNT_NAME"], access_key: ENV["AZURE_ACCESS_KEY"], container: ENV["AZURE_PRIVATE_CONTAINER"], - ).delete_prefix '' + ).delete_prefix "" AzureBlob::Client.new( account_name: ENV["AZURE_ACCOUNT_NAME"], access_key: ENV["AZURE_ACCESS_KEY"], container: ENV["AZURE_PUBLIC_CONTAINER"], - ).delete_prefix '' + ).delete_prefix "" end diff --git a/azure-blob.gemspec b/azure-blob.gemspec index dee8966..30dec42 100644 --- a/azure-blob.gemspec +++ b/azure-blob.gemspec @@ -13,6 +13,7 @@ Gem::Specification.new do |spec| spec.license = "MIT" spec.required_ruby_version = ">= 3.1" + spec.metadata["rubygems_mfa_required"] = "true" spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = spec.homepage spec.metadata["changelog_uri"] = "https://github.com/testdouble/azure-blob/blob/main/CHANGELOG.md" diff --git a/devenv.lock b/devenv.lock index df9976e..875aa94 100644 --- a/devenv.lock +++ b/devenv.lock @@ -108,16 +108,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1715542476, + "lastModified": 1725001927, "owner": "NixOS", "repo": "nixpkgs", - "rev": "44072e24566c5bcc0b7aa9178a0104f4cfffab19", - "treeHash": "3f9021e4c33de6fe59b88ac8c3019fc49136dc2a", + "rev": "6e99f2a27d600612004fbd2c3282d614bfee6421", + "treeHash": "1e85443cc9f0ba302df2cf61cacb8014943e2d19", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } @@ -129,11 +129,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1713939467, + "lastModified": 1724737223, "owner": "bobvanderlinden", "repo": "nixpkgs-ruby", - "rev": "c1ba161adf31119cfdbb24489766a7bcd4dbe881", - "treeHash": "0d32620317b29f94d6718684f030dd2fc2f30cb2", + "rev": "175b5867babcbc471b94be9fd5576f2973bbdb6d", + "treeHash": "2fe3404ac0eeb7bcb7ac7b5f5f8b9b6a7e460147", "type": "github" }, "original": { @@ -160,16 +160,16 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1715542476, + "lastModified": 1725001927, "owner": "NixOS", "repo": "nixpkgs", - "rev": "44072e24566c5bcc0b7aa9178a0104f4cfffab19", - "treeHash": "3f9021e4c33de6fe59b88ac8c3019fc49136dc2a", + "rev": "6e99f2a27d600612004fbd2c3282d614bfee6421", + "treeHash": "1e85443cc9f0ba302df2cf61cacb8014943e2d19", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } diff --git a/devenv.nix b/devenv.nix index dfd810c..6b8d95d 100644 --- a/devenv.nix +++ b/devenv.nix @@ -1,14 +1,36 @@ -{ pkgs, ... }: +{ pkgs, config, ... }: { + env = { + LD_LIBRARY_PATH = "${config.devenv.profile}/lib"; + }; + packages = with pkgs; [ git libyaml terraform azure-cli + glib + vips + sshuttle + sshpass + rsync ]; - languages.ruby.enable = true; - languages.ruby.version = "3.1.5"; + languages.ruby.version = "3.1.6"; + + scripts.generate-env-file.exec = '' + terraform output -raw devenv_local_nix > devenv.local.nix + ''; + + scripts.proxy-vps.exec = '' + exec sshuttle -e "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -r "$(terraform output --raw vm_username)@$(terraform output --raw vm_ip)" 0/0 + ''; + + scripts.start-app-service-ssh.exec = '' + resource_group=$(terraform output --raw "resource_group") + app_name=$(terraform output --raw "app_service_app_name") + exec az webapp create-remote-connection --resource-group $resource_group --name $app_name + ''; } diff --git a/devenv.yaml b/devenv.yaml index fe251b1..c3f54eb 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -1,6 +1,6 @@ allowUnfree: true inputs: nixpkgs: - url: github:NixOS/nixpkgs/nixos-23.11 + url: github:NixOS/nixpkgs/nixos-24.05 nixpkgs-ruby: url: github:bobvanderlinden/nixpkgs-ruby diff --git a/input.tf b/input.tf index b1ce808..6bd3273 100644 --- a/input.tf +++ b/input.tf @@ -2,3 +2,43 @@ variable "location" { type = string default = "westus2" } + +variable "prefix" { + type = string + default = "azure-blob" +} + +variable "storage_account_name" { + type = string + default = "azureblobrubygemdev" +} + +variable "create_vm" { + type = bool + default = false +} + +variable "vm_size" { + type = string + default = "Standard_B2s" +} + +variable "vm_username" { + type = string + default = "azureblob" +} + +variable "vm_password" { + type = string + default = "qwe123QWE!@#" +} + +variable "create_app_service" { + type = bool + default = false +} + +variable "ssh_key" { + type = string + default = "" +} diff --git a/lib/active_storage/service/azure_blob_service.rb b/lib/active_storage/service/azure_blob_service.rb index 4eaf8ce..bc56c42 100644 --- a/lib/active_storage/service/azure_blob_service.rb +++ b/lib/active_storage/service/azure_blob_service.rb @@ -35,7 +35,7 @@ module ActiveStorage class Service::AzureBlobService < Service attr_reader :client, :container, :signer - def initialize(storage_account_name:, storage_access_key:, container:, public: false, **options) + def initialize(storage_account_name:, storage_access_key: nil, container:, public: false, **options) @container = container @public = public @client = AzureBlob::Client.new( diff --git a/lib/azure_blob/client.rb b/lib/azure_blob/client.rb index 18f81f3..fb8802e 100644 --- a/lib/azure_blob/client.rb +++ b/lib/azure_blob/client.rb @@ -1,10 +1,11 @@ # frozen_string_literal: true -require_relative "signer" require_relative "block_list" require_relative "blob_list" require_relative "blob" require_relative "http" +require_relative "shared_key_signer" +require_relative "entra_id_signer" require "time" require "base64" @@ -12,10 +13,13 @@ module AzureBlob # AzureBlob Client class. You interact with the Azure Blob api # through an instance of this class. class Client - def initialize(account_name:, access_key:, container:) + def initialize(account_name:, access_key:, container:, **options) @account_name = account_name @container = container - @signer = Signer.new(account_name:, access_key:) + + @signer = !access_key.nil? && !access_key.empty? ? + AzureBlob::SharedKeySigner.new(account_name:, access_key:) : + AzureBlob::EntraIdSigner.new(account_name:, **options.slice(:principal_id)) end # Create a blob of type block. Will automatically split the the blob in multiple block and send the blob in pieces (blocks) if the blob is too big. diff --git a/lib/azure_blob/entra_id_signer.rb b/lib/azure_blob/entra_id_signer.rb new file mode 100644 index 0000000..93d827d --- /dev/null +++ b/lib/azure_blob/entra_id_signer.rb @@ -0,0 +1,115 @@ +require "base64" +require "openssl" +require "net/http" +require "rexml/document" + +require_relative "canonicalized_resource" +require_relative "identity_token" + +require_relative "user_delegation_key" + +module AzureBlob + class EntraIdSigner # :nodoc: + attr_reader :token + attr_reader :account_name + + def initialize(account_name:, principal_id: nil) + @token = AzureBlob::IdentityToken.new(principal_id:) + @account_name = account_name + end + + def authorization_header(uri:, verb:, headers: {}) + "Bearer #{token}" + end + + def sas_token(uri, options = {}) + to_sign = [ + options[:permissions], + options[:start], + options[:expiry], + CanonicalizedResource.new(uri, account_name, url_safe: false, service_name: :blob), + delegation_key.signed_oid, + delegation_key.signed_tid, + delegation_key.signed_start, + delegation_key.signed_expiry, + delegation_key.signed_service, + delegation_key.signed_version, + nil, + nil, + nil, + options[:ip], + options[:protocol], + SAS::Version, + SAS::Resources::Blob, + nil, + nil, + nil, + options[:content_disposition], + nil, + nil, + options[:content_type], + ].join("\n") + + query = { + SAS::Fields::Permissions => options[:permissions], + SAS::Fields::Start => options[:start], + SAS::Fields::Expiry => options[:expiry], + + SAS::Fields::SignedObjectId => delegation_key.signed_oid, + SAS::Fields::SignedTenantId => delegation_key.signed_tid, + SAS::Fields::SignedKeyStartTime => delegation_key.signed_start, + SAS::Fields::SignedKeyExpiryTime => delegation_key.signed_expiry, + SAS::Fields::SignedKeyService => delegation_key.signed_service, + SAS::Fields::Signedkeyversion => delegation_key.signed_version, + + + SAS::Fields::SignedIp => options[:ip], + SAS::Fields::SignedProtocol => options[:protocol], + SAS::Fields::Version => SAS::Version, + SAS::Fields::Resource => SAS::Resources::Blob, + + SAS::Fields::Disposition => options[:content_disposition], + SAS::Fields::Type => options[:content_type], + SAS::Fields::Signature => sign(to_sign, key: delegation_key.to_s), + + }.reject { |_, value| value.nil? } + + URI.encode_www_form(**query) + end + + private + + def delegation_key + @delegation_key ||= UserDelegationKey.new(account_name:, signer: self) + end + + def sign(body, key:) + Base64.strict_encode64(OpenSSL::HMAC.digest("sha256", key, body)) + end + + module SAS # :nodoc: + Version = "2024-05-04" + module Fields # :nodoc: + Permissions = :sp + Version = :sv + Start = :st + Expiry = :se + Resource = :sr + Signature = :sig + Disposition = :rscd + Type = :rsct + SignedObjectId = :skoid + SignedTenantId = :sktid + SignedKeyStartTime = :skt + SignedKeyExpiryTime = :ske + SignedKeyService = :sks + Signedkeyversion = :skv + SignedIp = :sip + SignedProtocol = :spr + end + module Resources # :nodoc: + Blob = :b + end + end + end +end diff --git a/lib/azure_blob/http.rb b/lib/azure_blob/http.rb index 4f67f7f..5e52c24 100644 --- a/lib/azure_blob/http.rb +++ b/lib/azure_blob/http.rb @@ -7,7 +7,14 @@ module AzureBlob class Http # :nodoc: - class Error < AzureBlob::Error; end + class Error < AzureBlob::Error + attr_reader :body, :status + def initialize(body: nil, status: nil) + @body = body + @status = status + super(body) + end + end class FileNotFoundError < Error; end class ForbidenError < Error; end class IntegrityError < Error; end @@ -44,6 +51,15 @@ def put(content) true end + def post(content) + sign_request("POST") if signer + @response = http.start do |http| + http.post(uri, content, headers) + end + raise_error unless success? + response.body + end + def head sign_request("HEAD") if signer @response = http.start do |http| @@ -91,7 +107,7 @@ def sign_request(method) end def raise_error - raise error_from_response.new(@response.body) + raise error_from_response.new(body: @response.body, status: @response.code&.to_i) end def status diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb new file mode 100644 index 0000000..0c00dae --- /dev/null +++ b/lib/azure_blob/identity_token.rb @@ -0,0 +1,65 @@ +require "json" + +module AzureBlob + class IdentityToken + RESOURCE_URI = "https://storage.azure.com/" + EXPIRATION_BUFFER = 600 # 10 minutes + + IDENTITY_ENDPOINT = ENV["IDENTITY_ENDPOINT"] || "http://169.254.169.254/metadata/identity/oauth2/token" + API_VERSION = ENV["IDENTITY_ENDPOINT"] ? "2019-08-01" : "2018-02-01" + + def initialize(principal_id: nil) + @identity_uri = URI.parse(IDENTITY_ENDPOINT) + params = { + 'api-version': API_VERSION, + resource: RESOURCE_URI, + } + params[:principal_id] = principal_id if principal_id + @identity_uri.query = URI.encode_www_form(params) + end + + def to_s + refresh if expired? + token + end + + private + + def expired? + token.nil? || Time.now >= (expiration - EXPIRATION_BUFFER) + end + + def refresh + headers = { "Metadata" => "true" } + headers["X-IDENTITY-HEADER"] = ENV["IDENTITY_HEADER"] if ENV["IDENTITY_HEADER"] + + attempt = 0 + begin + attempt += 1 + response = JSON.parse(AzureBlob::Http.new(identity_uri, headers).get) + rescue AzureBlob::Http::Error => error + if should_retry?(error, attempt) + attempt = 1 if error.status == 410 + delay = exponential_backoff(error, attempt) + Kernel.sleep(delay) + retry + end + raise + end + @token = response["access_token"] + @expiration = Time.at(response["expires_on"].to_i) + end + + def should_retry?(error, attempt) + is_500 = error.status/500 == 1 + (is_500 || [ 404, 408, 410, 429 ].include?(error.status)) && attempt < 5 + end + + def exponential_backoff(error, attempt) + EXPONENTIAL_BACKOFF[attempt -1] || raise(AzureBlob::Error.new("Exponential backoff out of bounds!")) + end + EXPONENTIAL_BACKOFF = [ 2, 6, 14, 30 ] + + attr_reader :identity_uri, :expiration, :token + end +end diff --git a/lib/azure_blob/signer.rb b/lib/azure_blob/shared_key_signer.rb similarity index 98% rename from lib/azure_blob/signer.rb rename to lib/azure_blob/shared_key_signer.rb index 5921278..907954c 100644 --- a/lib/azure_blob/signer.rb +++ b/lib/azure_blob/shared_key_signer.rb @@ -6,7 +6,7 @@ require_relative "canonicalized_resource" module AzureBlob - class Signer # :nodoc: + class SharedKeySigner # :nodoc: def initialize(account_name:, access_key:) @account_name = account_name @access_key = Base64.decode64(access_key) @@ -71,12 +71,12 @@ def sas_token(uri, options = {}) URI.encode_www_form(**query) end + private + def sign(body) Base64.strict_encode64(OpenSSL::HMAC.digest("sha256", access_key, body)) end - private - def sanitize_headers(headers) headers = headers.dup headers[:"Content-Length"] = nil if headers[:"Content-Length"].to_i == 0 diff --git a/lib/azure_blob/user_delegation_key.rb b/lib/azure_blob/user_delegation_key.rb new file mode 100644 index 0000000..d374164 --- /dev/null +++ b/lib/azure_blob/user_delegation_key.rb @@ -0,0 +1,67 @@ +require_relative "http" + +module AzureBlob + class UserDelegationKey # :nodoc: + EXPIRATION = 25200 # 7 hours + EXPIRATION_BUFFER = 3600 # 1 hours + def initialize(account_name:, signer:) + @uri = URI.parse( + "https://#{account_name}.blob.core.windows.net/?restype=service&comp=userdelegationkey" + ) + + @signer = signer + + refresh + end + + def to_s + user_delegation_key + end + + def refresh + return unless expired? + now = Time.now.utc + + + start = now.iso8601 + @expiration = (now + EXPIRATION) + expiry = @expiration.iso8601 + + content = <<-XML.gsub!(/[[:space:]]+/, " ").strip! + + + #{start} + #{expiry} + + XML + + response = Http.new(uri, signer:).post(content) + + doc = REXML::Document.new(response) + + @signed_oid = doc.get_elements("/UserDelegationKey/SignedOid").first.get_text.to_s + @signed_tid = doc.get_elements("/UserDelegationKey/SignedTid").first.get_text.to_s + @signed_start = doc.get_elements("/UserDelegationKey/SignedStart").first.get_text.to_s + @signed_expiry = doc.get_elements("/UserDelegationKey/SignedExpiry").first.get_text.to_s + @signed_service = doc.get_elements("/UserDelegationKey/SignedService").first.get_text.to_s + @signed_version = doc.get_elements("/UserDelegationKey/SignedVersion").first.get_text.to_s + @user_delegation_key = Base64.decode64(doc.get_elements("/UserDelegationKey/Value").first.get_text.to_s) + end + + attr_reader :signed_oid, + :signed_tid, + :signed_start, + :signed_expiry, + :signed_service, + :signed_version, + :user_delegation_key + + private + + def expired? + expiration.nil? || Time.now >= (expiration - EXPIRATION_BUFFER) + end + + attr_reader :uri, :user_delegation_key, :signer, :expiration + end +end diff --git a/main.tf b/main.tf index 3118d13..d243219 100644 --- a/main.tf +++ b/main.tf @@ -11,8 +11,12 @@ provider "azurerm" { features {} } -resource "azurerm_resource_group" "rg" { - name = "azure-blob" +locals { + public_ssh_key = var.ssh_key != "" ? var.ssh_key : file("~/.ssh/id_rsa.pub") +} + +resource "azurerm_resource_group" "main" { + name = var.prefix location = var.location tags = { source = "Terraform" @@ -20,9 +24,9 @@ resource "azurerm_resource_group" "rg" { } resource "azurerm_storage_account" "main" { - name = "azureblobrubygemdev" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location + name = var.storage_account_name + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location account_tier = "Standard" account_replication_type = "LRS" @@ -43,16 +47,141 @@ resource "azurerm_storage_container" "public" { container_access_type = "blob" } -output "devenv_local_nix" { - sensitive = true - value = <(time) { slept << time } + AzureBlob::Http.stub :new, http_mock do + Kernel.stub :sleep, sleep_lambda do + @identity_token = AzureBlob::IdentityToken.new(principal_id: @principal_id) + assert_raises(AzureBlob::Http::Error) { identity_token.to_s } + end + end + + assert_equal [ 2, 6, 14, 30 ], slept + end + + + def test_410_retry + http_mock = Minitest::Mock.new + def http_mock.get; raise AzureBlob::Http::Error.new(status: 410) end + attempt = 0 + slept = [] + sleep_lambda = ->(time) do + attempt += 1 + slept << time + if attempt > 3 + def http_mock.get; raise AzureBlob::Http::Error.new(status: 404) end + end + end + AzureBlob::Http.stub :new, http_mock do + Kernel.stub :sleep, sleep_lambda do + @identity_token = AzureBlob::IdentityToken.new(principal_id: @principal_id) + assert_raises(AzureBlob::Http::Error) { identity_token.to_s } + end + end + + assert_equal [ 2, 2, 2, 2, 6, 14, 30 ], slept + end +end diff --git a/test/client/test_user_delegation_key.rb b/test/client/test_user_delegation_key.rb new file mode 100644 index 0000000..7ba0558 --- /dev/null +++ b/test/client/test_user_delegation_key.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +class TestUserDelegationKey < TestCase + attr_reader :delegation_key + def setup + skip if using_shared_key + @account_name = ENV["AZURE_ACCOUNT_NAME"] + @principal_id = ENV["AZURE_PRINCIPAL_ID"] + @signer = AzureBlob::EntraIdSigner.new(account_name: @account_name, principal_id: @principal_id) + @delegation_key = AzureBlob::UserDelegationKey.new(account_name: @account_name, signer: @signer) + end + + def test_do_not_refresh_under_expiration_buffer + now = Time.now + five_hours = 18000 + Time.stub :now, now do + @delegation_key = AzureBlob::UserDelegationKey.new(account_name: @account_name, signer: @signer) + end + initial_expiry = @delegation_key.signed_expiry + + Time.stub :now, now + five_hours do + @delegation_key.refresh + end + + assert_equal initial_expiry, @delegation_key.signed_expiry + end + + def test_refresh_when_over_expiration_buffer + now = Time.now + after_expiration_buffer = now + 21601 + Time.stub :now, now do + @delegation_key = AzureBlob::UserDelegationKey.new(account_name: @account_name, signer: @signer) + end + + initial_expiry = delegation_key.signed_expiry + Time.stub :now, after_expiration_buffer do + delegation_key.refresh + end + + refute_equal initial_expiry, delegation_key.signed_expiry + end +end diff --git a/test/rails/service/configurations.yml b/test/rails/service/configurations.yml index 23e8f70..2acc30d 100644 --- a/test/rails/service/configurations.yml +++ b/test/rails/service/configurations.yml @@ -2,6 +2,7 @@ DEFAULT: &default service: AzureBlob storage_account_name: <%= ENV["AZURE_ACCOUNT_NAME"] %> storage_access_key: <%= ENV["AZURE_ACCESS_KEY"] %> + principal_id: <%= ENV["AZURE_PRINCIPAL_ID"]%> azure: <<: *default diff --git a/test/support/app_service_vpn.rb b/test/support/app_service_vpn.rb new file mode 100644 index 0000000..6ea2756 --- /dev/null +++ b/test/support/app_service_vpn.rb @@ -0,0 +1,87 @@ +require "open3" +require "net/ssh" +require "shellwords" + +class AppServiceVpn + HOST = "127.0.0.1" + + attr_reader :header, :endpoint + + def initialize(verbose: false) + @verbose = verbose + establish_vpn_connection + end + + def kill + Process.kill("INT", tunnel_wait_thread.pid) + Process.kill("INT", connection_wait_thread.pid) + end + + private + + def establish_vpn_connection + establish_app_service_tunnel + extract_msi_info + + puts "Establishing VPN connection..." + + tunnel_stdin, tunnel_stdout, @tunnel_wait_thread = Open3.popen2e([ "sshuttle", "-e", "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null", "-r", "#{username}:#{password}@#{HOST}:#{port}", "0/0" ].shelljoin) + + connection_successful = false + tunnel_stdout.each do |line| + puts "(sshuttle) #{line}" if verbose + if line.include?("Connected to server") + connection_successful = true + puts "Connection successful!" + break + end + end + + raise "Could not establish VPN connection to app service" unless connection_successful + end + + def establish_app_service_tunnel + puts "Establishing tunnel connection to app service..." + connection_stdin, connection_stdout, @connection_wait_thread = Open3.popen2e("start-app-service-ssh") + + port = nil + username = nil + password = nil + + connection_stdout.each do |line| + puts "(start-app-service-ssh) #{line}" if verbose + if line =~ /WARNING: Opening tunnel on port: (\d+)/ + port = $1.to_i + end + + if line =~ /WARNING: SSH is available \{ username: (\w+), password: ([^\s]+) \}/ + username = $1 + password = $2 + end + + if port && username && password + break + end + end + + raise "Could not establish tunnel connection to app service" unless port && username && password + @port = port + @username = username + @password = password + end + + def extract_msi_info + puts "Extracting MSI endpoint info..." + endpoint = nil + header = nil + Net::SSH.start(HOST, username, password:, port:) do |ssh| + endpoint = ssh.exec! [ "bash", "-l", "-c", %(printf "%s" "$IDENTITY_ENDPOINT") ].shelljoin + header = ssh.exec! [ "bash", "-l", "-c", %(printf "%s" "$IDENTITY_HEADER") ].shelljoin + end + raise "Could not extract MSI endpoint information" unless endpoint && header + @endpoint = endpoint + @header = header + end + + attr_reader :port, :username, :password, :verbose, :tunnel_wait_thread, :connection_wait_thread +end diff --git a/test/support/azure_vm_vpn.rb b/test/support/azure_vm_vpn.rb new file mode 100644 index 0000000..f18144b --- /dev/null +++ b/test/support/azure_vm_vpn.rb @@ -0,0 +1,20 @@ +require "open3" +require "shellwords" + +class AzureVmVpn + def initialize(verbose: false) + @verbose = verbose + stdin, stdout, @wait_thread = Open3.popen2e("proxy-vps") + stdout.each do |line| + break if line.include?("Connected to server") + end + end + + def kill + Process.kill("INT", wait_thread.pid) + end + + private + + attr_reader :wait_thread +end