From 373211e9357c6315675770ff1e39c6bea76f8710 Mon Sep 17 00:00:00 2001 From: Stephen Moloney Date: Wed, 29 Jun 2016 16:36:42 +0100 Subject: [PATCH] v0.2.0 --- .gitignore | 14 + CHANGELOG.md | 6 + LICENCE.md | 22 + README.md | 62 ++ config/config.exs | 16 + docs/ovh/cloudstorage/getting_started.md | 157 ++++ lib/adapter/adapter.ex | 5 + lib/adapter/config/config.ex | 152 +++ lib/adapter/keystone/keystone.ex | 13 + lib/adapters/hubic/adapter.ex | 9 + lib/adapters/hubic/config/config.ex | 93 ++ lib/adapters/hubic/keystone/keystone.ex | 180 ++++ lib/adapters/hubic/keystone/utils.ex | 48 + lib/adapters/ovh/cloudstorage/adapter.ex | 9 + .../ovh/cloudstorage/config/config.ex | 184 ++++ .../ovh/cloudstorage/keystone/keystone.ex | 182 ++++ .../ovh/cloudstorage/keystone/utils.ex | 55 ++ lib/adapters/ovh/webstorage/adapter.ex | 9 + lib/adapters/ovh/webstorage/config/config.ex | 184 ++++ .../ovh/webstorage/keystone/keystone.ex | 182 ++++ lib/adapters/ovh/webstorage/keystone/utils.ex | 68 ++ lib/adapters/rackspace/cloudfiles/adapter.ex | 9 + .../rackspace/cloudfiles/config/config.ex | 232 +++++ .../rackspace/cloudfiles/keystone/keystone.ex | 176 ++++ .../rackspace/cloudfiles/keystone/utils.ex | 57 ++ .../rackspace/cloudfilesCDN/adapter.ex | 9 + .../rackspace/cloudfilesCDN/config/config.ex | 220 +++++ .../cloudfilesCDN/keystone/keystone.ex | 176 ++++ .../rackspace/cloudfilesCDN/keystone/utils.ex | 57 ++ lib/client.ex | 77 ++ lib/openstex.ex | 87 ++ lib/query.ex | 37 + lib/request.ex | 8 + lib/request/http_query/request.ex | 36 + lib/request/openstack/request.ex | 39 + lib/response.ex | 37 + lib/services/keystone/v2/helpers.ex | 185 ++++ lib/services/keystone/v2/query.ex | 114 +++ lib/services/swift/v1/helpers.ex | 882 ++++++++++++++++++ lib/services/swift/v1/query.ex | 434 +++++++++ lib/supervisor.ex | 30 + lib/transformation.ex | 8 + lib/transformation/swift/transformation.ex | 44 + lib/transformation/transformation.ex | 44 + lib/utils/utils.ex | 69 ++ mix.exs | 69 ++ 46 files changed, 4786 insertions(+) create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 LICENCE.md create mode 100644 README.md create mode 100644 config/config.exs create mode 100644 docs/ovh/cloudstorage/getting_started.md create mode 100644 lib/adapter/adapter.ex create mode 100644 lib/adapter/config/config.ex create mode 100644 lib/adapter/keystone/keystone.ex create mode 100644 lib/adapters/hubic/adapter.ex create mode 100644 lib/adapters/hubic/config/config.ex create mode 100644 lib/adapters/hubic/keystone/keystone.ex create mode 100644 lib/adapters/hubic/keystone/utils.ex create mode 100644 lib/adapters/ovh/cloudstorage/adapter.ex create mode 100644 lib/adapters/ovh/cloudstorage/config/config.ex create mode 100644 lib/adapters/ovh/cloudstorage/keystone/keystone.ex create mode 100644 lib/adapters/ovh/cloudstorage/keystone/utils.ex create mode 100644 lib/adapters/ovh/webstorage/adapter.ex create mode 100644 lib/adapters/ovh/webstorage/config/config.ex create mode 100644 lib/adapters/ovh/webstorage/keystone/keystone.ex create mode 100644 lib/adapters/ovh/webstorage/keystone/utils.ex create mode 100644 lib/adapters/rackspace/cloudfiles/adapter.ex create mode 100644 lib/adapters/rackspace/cloudfiles/config/config.ex create mode 100644 lib/adapters/rackspace/cloudfiles/keystone/keystone.ex create mode 100644 lib/adapters/rackspace/cloudfiles/keystone/utils.ex create mode 100644 lib/adapters/rackspace/cloudfilesCDN/adapter.ex create mode 100644 lib/adapters/rackspace/cloudfilesCDN/config/config.ex create mode 100644 lib/adapters/rackspace/cloudfilesCDN/keystone/keystone.ex create mode 100644 lib/adapters/rackspace/cloudfilesCDN/keystone/utils.ex create mode 100644 lib/client.ex create mode 100644 lib/openstex.ex create mode 100644 lib/query.ex create mode 100644 lib/request.ex create mode 100644 lib/request/http_query/request.ex create mode 100644 lib/request/openstack/request.ex create mode 100644 lib/response.ex create mode 100644 lib/services/keystone/v2/helpers.ex create mode 100644 lib/services/keystone/v2/query.ex create mode 100644 lib/services/swift/v1/helpers.ex create mode 100644 lib/services/swift/v1/query.ex create mode 100644 lib/supervisor.ex create mode 100644 lib/transformation.ex create mode 100644 lib/transformation/swift/transformation.ex create mode 100644 lib/transformation/transformation.ex create mode 100644 lib/utils/utils.ex create mode 100644 mix.exs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75ab626 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +/_build +/cover +/deps +/.deprecated +/doc +/.idea +/config/dev.exs + +erl_crash.dump +*.ez +mix +mix.lock +.env + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..03aab82 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,6 @@ +# Changelog + +## v0.2.0 + +- Add functions `delete_object/3`, `delete_object!/3` +- Remove some duplicated docs from the `Openstex.Services.Swift.V1.Helpers` and have them on `@callback` only. (unfinished) \ No newline at end of file diff --git a/LICENCE.md b/LICENCE.md new file mode 100644 index 0000000..b962b24 --- /dev/null +++ b/LICENCE.md @@ -0,0 +1,22 @@ +# MIT License + +Copyright (c) 2016 Stephen Moloney + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..3e604e0 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +# Openstex [![Build Status](https://travis-ci.org/stephenmoloney/openstex_test.svg)](https://travis-ci.org/stephenmoloney/openstex_test) [![Hex Version](http://img.shields.io/hexpm/v/openstex.svg?style=flat)](https://hex.pm/packages/openstex) [![Hex docs](http://img.shields.io/badge/hex.pm-docs-green.svg?style=flat)](https://hexdocs.pm/openstex) + +An elixir client for making requests to [Openstack compliant apis](http://developer.openstack.org/api-ref.html). + +#### Supported services + +| Openstack Service | Supported | +|---|---| +| Identity 2.0 (Keystone) | :heavy_check_mark: | +| Object Storage 1.0 (Swift) | :heavy_check_mark: | + + +## Features + +1. Query modules for generating query structs which can subsequently be sent to the API using a `request` function. +Example: *Creating a container:* [create_container/3](https://github.com/stephenmoloney/openstex/blob/master/lib/services/swift/v1/query.ex#L88). + +2. Helper modules for + + a. One liners for sending queries to the client API. Example: *Getting the swift public url:* [get_public_url/0](https://github.com/stephenmoloney/openstex/blob/master/lib/services/swift/v1/helpers.ex#L21) + + b. Sending more complex queries such as multi-step queries to the client API. Example: *Getting all objects in a pseudofolder:* [list_objects/3](https://github.com/stephenmoloney/openstex/blob/master/lib/services/swift/v1/helpers.ex#L247) + +3. The `Request.request/3` and `Transformation.request/3` protocols and associated implementations that send the queries and process the response. +Theoretically, the protocol can be extended so that queries are processed in a different manner is so required later during development +for particular request types. + +4. Adapter modules for [OVH Webstorage CDN](https://www.ovh.com/fr/cdn/webstorage/), [OVH Cloudstorage](https://www.ovh.ie/cloud/storage/), +[Hubic](https://hubic.com/en/), [Rackspace Cloudfiles](https://www.rackspace.com/cloud/files) +and [Rackspace Cloudfiles CDN](https://www.rackspace.com/cloud/cdn-content-delivery-network). +All of the above Adapters provide access to Swift Object Storage services which are (mostly) openstack compliant. + + +## Installation and Getting Started + +| Adapter | Getting started | +|---|---| +| [Ovh Cloudstorage Adapter](https://github.com/stephenmoloney/openstex/blob/master/lib/adapters/ovh/cloudstorage/adapter.ex) | [docs/ovh/cloudstorage/getting_started.md](https://github.com/stephenmoloney/openstex/blob/master/docs/ovh/cloudstorage/getting_started.md) | + + +# Usage + +- Examples to be added. (for now see [openstex tests](https://github.com/stephenmoloney/openstex_test/tree/master/test)) + +## Tests + +- To avoid circular dependency issues, tests are run from a separate repository [openstex_tests](https://github.com/stephenmoloney/openstex_test). + +## Available Services + +| Tables | Version | Status | +| ------------- |:-------------:| -----:| +| Identity (Keystone) , [overview](https://wiki.openstack.org/wiki/keystone), [api](http://developer.openstack.org/api-ref-identity-v2.html) | v2 | :heavy_check_mark: | +| Identity (Keystone) , [overview](https://wiki.openstack.org/wiki/keystone), [api](http://developer.openstack.org/api-ref-identity-v3.html) | v3 | :x: | +| Object Storage (Swift) , [overview](https://wiki.openstack.org/wiki/swift), [api](http://developer.openstack.org/api-ref-objectstorage-v1.html) | v1 | :heavy_check_mark: | + +[Openstack api reference](http://developer.openstack.org/api-ref.html) + + +## Licence + +[MIT Licence](LICENCE.md) diff --git a/config/config.exs b/config/config.exs new file mode 100644 index 0000000..73d6249 --- /dev/null +++ b/config/config.exs @@ -0,0 +1,16 @@ +use Mix.Config + +config :logger, + backends: [:console], + level: :debug, + format: "\n$date $time [$level] $metadata$message" + +if Mix.env == :prod do + config :logger, + backends: [:console], + compile_time_purge_level: :warn +end + +unless Mix.env == :test do + import_config "#{Mix.env}.exs" +end diff --git a/docs/ovh/cloudstorage/getting_started.md b/docs/ovh/cloudstorage/getting_started.md new file mode 100644 index 0000000..6e1771b --- /dev/null +++ b/docs/ovh/cloudstorage/getting_started.md @@ -0,0 +1,157 @@ +## Getting started - OVH Cloudstorage + +- Add `:ex_ovh` and `openstex` to your project list of dependencies. + +```elixir +defp deps() do + [ + {:ex_ovh, "~> 0.1.0"}, + {:openstex, github: "stephenmoloney/openstex", tag: "0.1"} + ] +end +``` + +- Configure an ExOvh Client + + - Create an OVH account at [OVH](https://www.ovh.com/) + + - Create an API application at the [OVH API page](https://eu.api.ovh.com/createApp/). Follow the + steps outlined by OVH there. Alternatively, there is a [mix task](https://github.com/stephenmoloney/ex_ovh/blob/master/docs/mix_task_advanced.md) which can help + generate the OVH application. + + - The mix task (if used) will generate a config file as follows: + +```elixir +config :my_app, MyApp.Cloudstorage, + ovh: [ + application_key: System.get_env("MY_APP_CLOUDSTORAGE_APPLICATION_KEY"), + application_secret: System.get_env("MY_APP_CLOUDSTORAGE_APPLICATION_SECRET"), + consumer_key: System.get_env("MY_APP_CLOUDSTORAGE_CONSUMER_KEY") + ] +``` + +- Add additional configuration as needed or as known. + +```elixir +config :my_app, MyApp.Cloudstorage, + adapter: Openstex.Adapters.Ovh.Cloudstorage.Adapter, + ovh: [ + application_key: System.get_env("MY_APP_CLOUDSTORAGE_APPLICATION_KEY"), + application_secret: System.get_env("MY_APP_CLOUDSTORAGE_APPLICATION_SECRET"), + consumer_key: System.get_env("MY_APP_CLOUDSTORAGE_CONSUMER_KEY"), + endpoint: "ovh-eu", + api_version: "1.0" + ], + keystone: [ + tenant_id: System.get_env("MY_APP_CLOUDSTORAGE_CLOUDSTORAGE_TENANT_ID"), # mandatory, corresponds to an ovh project id or ovh servicename + user_id: System.get_env("MY_APP_CLOUDSTORAGE_CLOUDSTORAGE_USER_ID"), # optional, if absent a user will be created using the ovh api. + endpoint: "https://auth.cloud.ovh.net/v2.0" + ], + swift: [ + account_temp_url_key1: System.get_env("MY_APP_CLOUDSTORAGE_CLOUDSTORAGE_TEMP_URL_KEY1"), # defaults to :nil if absent + account_temp_url_key2: System.get_env("MY_APP_CLOUDSTORAGE_CLOUDSTORAGE_TEMP_URL_KEY2"), # defaults to :nil if absent + region: :nil # defaults to "SBG1" if not set. + ], + httpoison: [ + connect_timeout: 20000, + receive_timeout: 180000 + ] +``` + +- Ensure that the following variables are available as environment variables. The `mix ovh` task generates a `.env` file +which can optionally be used for this purpose. *NOTE:* Make sure `.env` is never added to version control. + +```shell +export MY_APP_CLOUDSTORAGE_APPLICATION_KEY= +export MY_APP_CLOUDSTORAGE_APPLICATION_SECRET= +export MY_APP_CLOUDSTORAGE_CONSUMER_KEY= +export MY_APP_CLOUDSTORAGE_CLOUDSTORAGE_TENANT_ID= +export MY_APP_CLOUDSTORAGE_CLOUDSTORAGE_TEMP_URL_KEY1= +export MY_APP_CLOUDSTORAGE_CLOUDSTORAGE_TEMP_URL_KEY2= +``` + +- Add the environment variables to the enviroment. Eg run ```source .env``` + +- Add a client to your project. + + +```elixir +defmodule MyApp.Cloudstorage do + @moduledoc :false + use Openstex.Client, otp_app: :my_app, client: __MODULE__ + + defmodule SwiftHelpers do + @moduledoc :false + use Openstex.Services.Swift.V1.Helpers, otp_app: :my_app, client: MyApp.Cloudstorage + end + + defmodule Ovh do + @moduledoc :false + use ExOvh.Client, otp_app: :my_app, client: __MODULE__ + end +end +``` + +- Add the client (`Openstex.Cloudstorage`) to your project supervision tree. + + +```elixir +def start(_type, _args) do + import Supervisor.Spec, warn: false + spec1 = [supervisor(MyApp.Endpoint, [])] + spec2 = [supervisor(MyApp.Cloudstorage, [])] + opts = [strategy: :one_for_one, name: MyApp.Supervisor] + Supervisor.start_link(spec1 ++ spec2, opts) +end +``` + + +### Examples + + client = MyApp.Cloudstorage + +- An example of an API call to a swift API using the Helper functions (higher level): + + ```elixir +file_on_client = "/priv/test_file.json" +server_file = "test/nested/test_file.json" +container = "default" +client.swift().upload_file(file_on_client, server_file, container, [recv_timeout: (60000 * 60)]) + ``` + +- An example of an API call to a swift API using the Query functions (lower level): + + + ```elixir +new_container = "my_new_container" +account = client.swift().get_account() +container_metadata = [headers: [{"X-Container-Meta-Access-Control-Allow-Origin", "http://stephenmoloney.com"}]] +create_container(new_container, account, container_metadata) |> client.request() + ``` + +- An example of manually constructed query call to a swift API using the Query functions (low level): + +```elixir +%Openstex.Swift.Query{ + method: :get, + uri: account, + params: %{query_string: %{"format" => "json"}} + } +|> client.request() +``` + +- An example of manually constructed query call to a swift API using the Query functions (lowest level): + + +```elixir +%Openstex.HttpQuery{ + method: :get, + uri: account, + body: "", + headers: [{"Content-Type", "application/json; charset=utf-8"}], + options: [timeout: 10000, recv_timeout: 30000], + service: :openstack + } +|> client.prepare_request() +|> client.request() +``` diff --git a/lib/adapter/adapter.ex b/lib/adapter/adapter.ex new file mode 100644 index 0000000..59b2046 --- /dev/null +++ b/lib/adapter/adapter.ex @@ -0,0 +1,5 @@ +defmodule Openstex.Adapter do + @moduledoc :false + @callback keystone() :: atom + @callback config() :: atom +end \ No newline at end of file diff --git a/lib/adapter/config/config.ex b/lib/adapter/config/config.ex new file mode 100644 index 0000000..3f5f6c6 --- /dev/null +++ b/lib/adapter/config/config.ex @@ -0,0 +1,152 @@ +defmodule Openstex.Adapter.Config do + @moduledoc ~s""" + An adapter module to be implemented by clients that using the `Openstex` library for the storage of config info. + """ + + defmacro __using__(_) do + quote([]) do + + @doc "Starts an agent for the storage of credentials in memory" + def start_agent(_client), do: raise "Openstex.Adapter.Config.start_agent/2 has not been implemented." + + @doc "Starts an agent for the storage of credentials in memory" + def start_agent(_client, _opts), do: raise "Openstex.Adapter.Config.start_agent/2 has not been implemented." + + @doc "Gets all the config.exs environment variables" + def get_config_from_env(client, otp_app) do + case Application.get_env(otp_app, client) do + :nil -> Application.get_all_env(otp_app) + config -> config + end + end + + @doc "Gets the keystone config.exs environment variables" + def get_keystone_config_from_env(client, otp_app) do + get_config_from_env(client, otp_app) |> Keyword.fetch!(:keystone) + end + + @doc "Gets the swift config.exs environment variables" + def get_swift_config_from_env(client, otp_app) do + get_config_from_env(client, otp_app) |> Keyword.fetch!(:swift) + end + + @doc "Gets the httpoison config.exs environment variables" + def get_httpoison_config_from_env(client, otp_app) do + get_config_from_env(client, otp_app) |> Keyword.fetch!(:httpoison) + end + + @doc "Gets the openstack related config variables from a supervised Agent" + def config(client) do + Agent.get(agent_name(client), fn(config) -> config end) + end + + @doc "Gets the keystone related config variables from a supervised Agent" + def keystone_config(client) do + Agent.get(agent_name(client), fn(config) -> config[:keystone] end) + end + + @doc "Gets the swift related config variables from a supervised Agent" + def swift_config(client) do + Agent.get(agent_name(client), fn(config) -> config[:swift] end) + end + + @doc "Gets the httpoison_config related config variables from a supervised Agent" + def httpoison_config(client) do + Agent.get(agent_name(client), fn(config) -> config[:httpoison] end) + end + + @doc "Gets the tenant_id config variable from a supervised Agent" + def tenant_id(client) do + Agent.get(agent_name(client), fn(config) -> config[:keystone][:tenant_id] end) + end + + @doc "Gets the user_id config variable from a supervised Agent" + def user_id(client) do + Agent.get(agent_name(client), fn(config) -> config[:keystone][:user_id] end) + end + + @doc "Gets the keystone_endpoint config variable from a supervised Agent" + def keystone_endpoint(client) do + Agent.get(agent_name(client), fn(config) -> config[:keystone][:endpoint] end) + end + + @doc "Gets the swift_region config variable from a supervised Agent" + def swift_region(client) do + Agent.get(agent_name(client), fn(config) -> config[:swift][:region] end) + end + + @doc "Gets the account_temp_url_key1 config variablesfrom a supervised Agent" + def get_account_temp_url_key1(client) do + Agent.get(agent_name(client), fn(config) -> config[:swift][:account_temp_url_key1] end) + end + + @doc "Gets the account_temp_url_key2 config variable from a supervised Agent" + def get_account_temp_url_key2(client) do + Agent.get(agent_name(client), fn(config) -> config[:swift][:account_temp_url_key2] end) + end + + @doc "Gets the account_temp_url_key1 config variables from a supervised Agent" + def set_account_temp_url_key1(client, key) do + Agent.update(agent_name(client), + fn(config) -> put_account_key1(config, key) end + ) + end + + @doc "Gets the account_temp_url_key2 config variable from a supervised Agent" + def set_account_temp_url_key2(client, key) do + Agent.update(agent_name(client), + fn(config) -> put_account_key2(config, key) end + ) + end + + defp agent_name(client) do + Module.concat(__MODULE__, client) + end + + defp put_account_key1(config, key) do + swift_config = config[:swift] + new_swift_config = Keyword.put(swift_config, :account_temp_url_key1, key) + Keyword.put(config, :swift, new_swift_config) + end + + defp put_account_key2(config, key) do + swift_config = config[:swift] + new_swift_config = Keyword.put(swift_config, :account_temp_url_key2, key) + Keyword.put(config, :swift, new_swift_config) + end + + @doc :false + def swift_service_name(), do: "swift" + + @doc :false + def swift_service_type(), do: "object-store" + + defoverridable [start_agent: 1, start_agent: 2, swift_service_name: 0, swift_service_type: 0] + + end + end + + @callback start_agent(client :: atom) :: {:ok, pid} | {:error, :already_started} + @callback start_agent(client :: atom, opts :: Keyword.t) :: {:ok, pid} | {:error, :already_started} + @callback config(client :: atom) :: Keyword.t | no_return + @callback keystone_config(client :: atom) :: Keyword.t + @callback swift_config(client :: atom) :: Keyword.t + @callback httpoison_config(client :: atom) :: Keyword.t + @callback tenant_id(client :: atom) :: Keyword.t + @callback user_id(client :: atom) :: Keyword.t + @callback keystone_endpoint(client :: atom) :: Keyword.t + @callback swift_region(client :: atom) :: Keyword.t + @callback swift_service_name() :: String.t + @callback swift_service_type() :: String.t + @callback get_account_temp_url_key1(client :: atom) :: Keyword.t + @callback get_account_temp_url_key2(client :: atom) :: Keyword.t + @callback set_account_temp_url_key1(client :: atom, key :: String.t) :: Keyword.t + @callback set_account_temp_url_key2(client :: atom, key :: String.t) :: Keyword.t + +end + + + + + + diff --git a/lib/adapter/keystone/keystone.ex b/lib/adapter/keystone/keystone.ex new file mode 100644 index 0000000..3cb5b3a --- /dev/null +++ b/lib/adapter/keystone/keystone.ex @@ -0,0 +1,13 @@ +defmodule Openstex.Adapter.Keystone do + @moduledoc ~s""" + A behaviour adapter module to be implemented by clients that using the `Openstex` library + for handling Keystone Authentication. + """ + alias Openstex.Services.Keystone.V2.Helpers.Identity + + @callback start_link(client :: atom) :: {:ok, pid} | {:error, :already_started} + @callback start_link(client :: atom, opts :: Keyword.t) :: {:ok, pid} | {:error, :already_started} + @callback identity(client :: atom) :: Identity.t | no_return + @callback get_xauth_token(client :: atom) :: String.t | no_return + +end diff --git a/lib/adapters/hubic/adapter.ex b/lib/adapters/hubic/adapter.ex new file mode 100644 index 0000000..53a8916 --- /dev/null +++ b/lib/adapters/hubic/adapter.ex @@ -0,0 +1,9 @@ +defmodule Openstex.Adapters.Hubic.Adapter do + @moduledoc :false + @behaviour Openstex.Adapter + + def config(), do: Openstex.Adapters.Hubic.Config + def keystone(), do: Openstex.Adapters.Hubic.Keystone + +end + diff --git a/lib/adapters/hubic/config/config.ex b/lib/adapters/hubic/config/config.ex new file mode 100644 index 0000000..56e6558 --- /dev/null +++ b/lib/adapters/hubic/config/config.ex @@ -0,0 +1,93 @@ +defmodule Openstex.Adapters.Hubic.Config do + @moduledoc :false + alias Openstex.HttpQuery + use Openstex.Adapter.Config + + + # public + + def start_agent(openstex_client, opts) do + Og.context(__ENV__, :debug) + otp_app = Keyword.get(opts, :otp_app, :false) || Og.log_return(__ENV__, :error) |> raise() + + hubic_client = Module.concat(openstex_client, Hubic) + Application.ensure_all_started(:ex_hubic) + + unless supervisor_exists?(hubic_client) do + hubic_client.start_link() + delay_until_client_started(openstex_client) + end + + Agent.start_link(fn -> config({openstex_client, hubic_client}, otp_app) end, name: agent_name(openstex_client)) + end + + @doc "Gets the hubic related config variables from a supervised Agent" + def hubic_config(openstex_client) do + Agent.get(agent_name(openstex_client), fn(config) -> config[:hubic] end) + end + + @doc :false + def swift_service_name(), do: "swift" + + @doc :false + def swift_service_type(), do: "object-store" + + + # private + + defp config({openstex_client, hubic_client}, otp_app) do + swift_config = swift_config(openstex_client, otp_app) + keystone_config = keystone_config(openstex_client, otp_app) + hubic_config = hubic_client.hubic_config() + httpoison_config = httpoison_config(openstex_client, otp_app) + + [ + hubic: hubic_config, + keystone: keystone_config, + swift: swift_config, + httpoison: httpoison_config + ] + end + + defp keystone_config(_openstex_client, _otp_app) do + [ + tenant_id: :nil, + user_id: :nil, + endpoint: :nil + ] + end + + defp swift_config(_openstex_client, _otp_app) do + [ + account_temp_url_key1: :nil, + account_temp_url_key2: :nil, + region: "hubic" + ] + end + + defp httpoison_config(openstex_client, otp_app) do + + httpoison_config = get_httpoison_config_from_env(openstex_client, otp_app) + + connect_timeout = httpoison_config[:connect_timeout] || 30000 # 30 seconds + receive_timeout = httpoison_config[:receive_timeout] || (60000 * 30) # 30 minutes + + [ + timeout: connect_timeout, + recv_timeout: receive_timeout, + ] + end + + defp supervisor_exists?(hubic_client) do + Module.concat(ExHubic.Config, hubic_client) in Process.registered() + end + + defp delay_until_client_started(openstex_client) do + unless Module.concat(openstex_client, Hubic) in Process.registered() do + :timer.sleep(200) + delay_until_client_started(openstex_client) + end + end + + +end \ No newline at end of file diff --git a/lib/adapters/hubic/keystone/keystone.ex b/lib/adapters/hubic/keystone/keystone.ex new file mode 100644 index 0000000..db10dc7 --- /dev/null +++ b/lib/adapters/hubic/keystone/keystone.ex @@ -0,0 +1,180 @@ +defmodule Openstex.Adapters.Hubic.Keystone do + @moduledoc :false + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Adapters.Hubic.Keystone.Utils + import Openstex.Utils, only: [ets_tablename: 1] + @behaviour Openstex.Adapter.Keystone + @get_identity_retries 5 + @get_identity_interval 1000 + + + # Public Openstex.Adapter.Keystone callbacks + + def start_link(openstex_client) do + Og.context(__ENV__, :debug) + GenServer.start_link(__MODULE__, openstex_client, [name: openstex_client]) + end + + def start_link(openstex_client, _opts) do + start_link(openstex_client) + end + + def identity(openstex_client) do + get_identity(openstex_client) + end + + def get_xauth_token(openstex_client) do + get_identity(openstex_client) |> Map.get(:token) |> Map.get(:id) + end + + # Genserver Callbacks + + def init(openstex_client) do + Og.context(__ENV__, :debug) + :erlang.process_flag(:trap_exit, :true) + create_ets_table(openstex_client) + identity = create_identity(openstex_client) + identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, identity}) + expiry = to_seconds(identity) + Task.start_link(fn -> monitor_expiry(expiry) end) + {:ok, {openstex_client, identity}} + end + + def handle_call(:add_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :true) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:remove_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:update_identity, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + {:ok, new_identity} = Utils.create_identity(openstex_client) |> Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:stop, _from, state) do + Og.context(__ENV__, :debug) + {:stop, :shutdown, :ok, state} + end + + def terminate(:shutdown, {openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ets.delete(ets_tablename(openstex_client)) # explicilty remove + :ok + end + + def terminate(:shutdown, {:failed_to_start_child, openstex_client, {:already_started, _pid}}) do + Og.context(__ENV__, :debug) + :ets.delete(ets_tablename(openstex_client)) # explicilty remove + :ok + end + + def terminate(:normal, {_openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ok + end + + # private + + @spec create_identity(atom) :: Identity.t | no_return + defp create_identity(openstex_client) do + Og.context(__ENV__, :debug) + Utils.create_identity(openstex_client) + end + + + defp get_identity(openstex_client) do + unless supervisor_exists?(openstex_client), do: start_link(openstex_client) + get_identity(openstex_client, 0) + end + defp get_identity(openstex_client, index) do + Og.context(__ENV__, :debug) + + retry = fn(openstex_client, index) -> + if index > @get_identity_retries do + raise "Cannot retrieve openstack identity, #{__ENV__.module}, #{__ENV__.line}, client: #{openstex_client}" + else + :timer.sleep(@get_identity_interval) + get_identity(openstex_client, index + 1) + end + end + + if ets_tablename(openstex_client) in :ets.all() do + table = :ets.lookup(ets_tablename(openstex_client), :identity) + case table do + [identity: identity] -> + if identity.lock === :true do + retry.(openstex_client, index) + else + identity + end + [] -> retry.(openstex_client, index) + end + else + retry.(openstex_client, index) + end + + end + + + defp monitor_expiry(expires) do + Og.context(__ENV__, :debug) + interval = (expires - 30) * 1000 + :timer.sleep(interval) + {:reply, :ok, _identity} = GenServer.call(self(), :add_lock) + {:reply, :ok, _identity} = GenServer.call(self(), :update_identity) + {:reply, :ok, new_identity} = GenServer.call(self(), :remove_lock) + expires = to_seconds(new_identity.token.expires) + monitor_expiry(expires) + end + + + defp create_ets_table(openstex_client) do + Og.context(__ENV__, :debug) + ets_options = [ + :set, # type + :protected, # read - all, write this process only. + :named_table, + {:heir, :none}, # don't let any process inherit the table. when the ets table dies, it dies. + {:write_concurrency, :false}, + {:read_concurrency, :true} + ] + unless ets_tablename(openstex_client) in :ets.all() do + :ets.new(ets_tablename(openstex_client), ets_options) + end + end + + + defp to_seconds(identity) do + iso_time = identity.token.expires + {:ok, expiry_ndt, offset} = Calendar.NaiveDateTime.Parse.iso8601(iso_time) + offset = + case offset do + :nil -> 0 + offset -> offset + end + {:ok, expiry_dt_utc} = Calendar.NaiveDateTime.with_offset_to_datetime_utc(expiry_ndt, offset) + {:ok, now} = Calendar.DateTime.from_erl(:calendar.universal_time(), "UTC") + {:ok, seconds, _microseconds, _when} = Calendar.DateTime.diff(expiry_dt_utc, now) + if seconds > 0 do + seconds + else + 0 + end + end + + defp supervisor_exists?(client) do + Process.whereis(client) != :nil + end + +end diff --git a/lib/adapters/hubic/keystone/utils.ex b/lib/adapters/hubic/keystone/utils.ex new file mode 100644 index 0000000..8fdde08 --- /dev/null +++ b/lib/adapters/hubic/keystone/utils.ex @@ -0,0 +1,48 @@ +defmodule Openstex.Adapters.Hubic.Keystone.Utils do + @moduledoc :false + alias Openstex.Services.Keystone.V2.Helpers.Identity + + # Note: This method constructs a 'fake' identity struct so that the Swift requests can be fulfilled without changing the Adapter radically. + @doc :false + @spec create_identity(atom) :: Identity.t | no_return + def create_identity(openstex_client) do + Og.context(__ENV__, :debug) + + hubic_client = Module.concat(openstex_client, Hubic) + Application.ensure_all_started(:ex_hubic) + unless supervisor_exists?(hubic_client), do: hubic_client.start_link() + + resp = ExHubic.Services.V1.Query.openstack_credentials() |> hubic_client.request!() + + public_url = resp.body["endpoint"] || Og.log_return("Could not get endpoint", __ENV__, :error) |> raise() + xauth_token = resp.body["token"] || Og.log_return("Could not get xauth_token", __ENV__, :error) |> raise() + xauth_token_expiry = resp.body["expires"] || Og.log_return("Could not get expiry time for xauth_token", __ENV__, :error) |> raise() + + %Identity{ + token: %Identity.Token{ + id: xauth_token, + expires: xauth_token_expiry + }, + service_catalog: [ + %Identity.Service{ + name: "swift", + type: "object-store", + endpoints: [ + %Identity.Endpoint{ + region: "hubic", + public_url: public_url + } + ] + } + ], + user: %Identity.User{}, + metadata: %Identity.Metadata{}, + trust: %Identity.Trust{} + } + end + + defp supervisor_exists?(client) do + Process.whereis(client) != :nil + end + +end \ No newline at end of file diff --git a/lib/adapters/ovh/cloudstorage/adapter.ex b/lib/adapters/ovh/cloudstorage/adapter.ex new file mode 100644 index 0000000..004f13b --- /dev/null +++ b/lib/adapters/ovh/cloudstorage/adapter.ex @@ -0,0 +1,9 @@ +defmodule Openstex.Adapters.Ovh.Cloudstorage.Adapter do + @moduledoc :false + @behaviour Openstex.Adapter + + def config(), do: Openstex.Adapters.Ovh.Cloudstorage.Config + def keystone(), do: Openstex.Adapters.Ovh.Cloudstorage.Keystone + +end + diff --git a/lib/adapters/ovh/cloudstorage/config/config.ex b/lib/adapters/ovh/cloudstorage/config/config.ex new file mode 100644 index 0000000..55411fc --- /dev/null +++ b/lib/adapters/ovh/cloudstorage/config/config.ex @@ -0,0 +1,184 @@ +defmodule Openstex.Adapters.Ovh.Cloudstorage.Config do + @moduledoc :false + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + @default_options [timeout: 10000, recv_timeout: 30000] + alias Openstex.HttpQuery + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Adapters.Ovh.Cloudstorage.Keystone.Utils + use Openstex.Adapter.Config + + + # public + + + def start_agent(openstex_client, opts) do + Og.context(__ENV__, :debug) + otp_app = Keyword.get(opts, :otp_app, :false) || Og.log_return(__ENV__, :error) |> raise() + + ovh_client = Module.concat(openstex_client, Ovh) + Application.ensure_all_started(:ex_ovh) + unless supervisor_exists?(ovh_client), do: ovh_client.start_link() + identity = Utils.create_identity(openstex_client, otp_app) + + Agent.start_link(fn -> config({openstex_client, ovh_client}, otp_app, identity) end, name: agent_name(openstex_client)) + end + + + @doc "Gets the rackspace related config variables from a supervised Agent" + def ovh_config(openstex_client) do + Agent.get(agent_name(openstex_client), fn(config) -> config[:ovh] end) + end + + + @doc :false + def swift_service_name(), do: "swift" + + + @doc :false + def swift_service_type(), do: "object-store" + + + # private + + + defp config({openstex_client, ovh_client}, otp_app, identity) do + swift_config = swift_config(openstex_client, otp_app) + keystone_config = keystone_config(openstex_client, otp_app, identity) + [ + ovh: ovh_client.ovh_config(), + keystone: keystone_config, + swift: swift_config, + httpoison: httpoison_config(openstex_client, otp_app) + ] + end + + defp keystone_config(openstex_client, otp_app, identity) do + + keystone_config = get_keystone_config_from_env(openstex_client, otp_app) + + tenant_id = keystone_config[:tenant_id] || + identity.token.tenant.id || + Og.log_return("cannot retrieve the tenant_id for keystone", __ENV__, :error) |> raise() + + user_id = keystone_config[:user_id] || + identity.user.id || + Og.log_return("cannot retrieve the user_id for keystone", __ENV__, :error) |> raise() + + endpoint = keystone_config[:endpoint] || + "https://auth.cloud.ovh.net/v2.0" + + [ + tenant_id: tenant_id, + user_id: user_id, + endpoint: endpoint + ] + end + + defp swift_config(openstex_client, otp_app) do + + swift_config = get_swift_config_from_env(openstex_client, otp_app) + + account_temp_url_key1 = get_account_temp_url(openstex_client, otp_app, :key1) || + swift_config[:account_temp_url_key1] || + :nil + + if account_temp_url_key1 != :nil && swift_config[:account_temp_url_key1] != account_temp_url_key1 do + Og.log("Warning, the `account_temp_url_key1` for the elixir `config.exs` for the swift client " <> + "#{inspect openstex_client} does not match the `X-Account-Meta-Temp-Url-Key` on the server. " <> + "This issue should probably be addressed. See Openstex.Adapter.Config.set_account_temp_url_key1/2.", __ENV__, :error) + end + + account_temp_url_key2 = get_account_temp_url(openstex_client, otp_app, :key2) || + swift_config[:account_temp_url_key2] || + :nil + + if account_temp_url_key2 != :nil && swift_config[:account_temp_url_key2] != account_temp_url_key2 do + Og.log("Warning, the `account_temp_url_key2` for the elixir `config.exs` for the swift client " <> + "#{inspect openstex_client} does not match the `X-Account-Meta-Temp-Url-Key-2` on the server. " <> + "This issue should probably be addressed. See Openstex.Adapter.Config.set_account_temp_url_key2/2.", __ENV__, :error) + end + + region = swift_config[:region] || "SBG1" + + [ + account_temp_url_key1: account_temp_url_key1, + account_temp_url_key2: account_temp_url_key2, + region: region + ] + end + + + defp httpoison_config(openstex_client, otp_app) do + + httpoison_config = get_httpoison_config_from_env(openstex_client, otp_app) + + connect_timeout = httpoison_config[:connect_timeout] || 30000 # 30 seconds + receive_timeout = httpoison_config[:receive_timeout] || (60000 * 30) # 30 minutes + + [ + timeout: connect_timeout, + recv_timeout: receive_timeout, + ] + end + + + defp get_account_temp_url(openstex_client, otp_app, key_atom) do + + ovh_client = Module.concat(openstex_client, Ovh) + Application.ensure_all_started(:ex_ovh) + unless supervisor_exists?(ovh_client), do: ovh_client.start_link() + + identity = Utils.create_identity(openstex_client, otp_app) + x_auth_token = Map.get(identity, :token) |> Map.get(:id) + endpoint = get_public_url(openstex_client, otp_app, identity) + + headers = + @default_headers ++ + [ + { + "X-Auth-Token", x_auth_token + } + ] + + header = + case key_atom do + :key1 -> "X-Account-Meta-Temp-Url-Key" + :key2 -> "X-Account-Meta-Temp-Url-Key-2" + end + + query = + %HttpQuery{ + method: :get, + uri: endpoint, + body: "", + headers: headers, + options: @default_options, + service: :openstack + } + {:ok, resp} = Openstex.Request.request(query, [], :nil) + + resp + |> Map.get(:headers) + |> Map.get(header) + end + + defp get_public_url(openstex_client, otp_app, identity) do + + swift_config = get_swift_config_from_env(openstex_client, otp_app) + + region = swift_config[:region] || "SBG1" + + identity + |> Map.get(:service_catalog) + |> Enum.find(fn(%Identity.Service{} = service) -> service.name == swift_service_name() && service.type == swift_service_type() end) + |> Map.get(:endpoints) + |> Enum.find(fn(%Identity.Endpoint{} = endpoint) -> endpoint.region == region end) + |> Map.get(:public_url) + end + + defp supervisor_exists?(ovh_client) do + Module.concat(ExOvh.Config, ovh_client) in Process.registered() + end + + +end \ No newline at end of file diff --git a/lib/adapters/ovh/cloudstorage/keystone/keystone.ex b/lib/adapters/ovh/cloudstorage/keystone/keystone.ex new file mode 100644 index 0000000..0792ee5 --- /dev/null +++ b/lib/adapters/ovh/cloudstorage/keystone/keystone.ex @@ -0,0 +1,182 @@ +defmodule Openstex.Adapters.Ovh.Cloudstorage.Keystone do + @moduledoc :false + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Adapters.Ovh.Cloudstorage.Keystone.Utils + import Openstex.Utils, only: [ets_tablename: 1] + @behaviour Openstex.Adapter.Keystone + @get_identity_retries 5 + @get_identity_interval 1000 + + + # Public Openstex.Adapter.Keystone callbacks + + def start_link(openstex_client) do + Og.context(__ENV__, :debug) + GenServer.start_link(__MODULE__, openstex_client, [name: openstex_client]) + end + + def start_link(openstex_client, _opts) do + start_link(openstex_client) + end + + def identity(openstex_client) do + get_identity(openstex_client) + end + + def get_xauth_token(openstex_client) do + get_identity(openstex_client) |> Map.get(:token) |> Map.get(:id) + end + + # Genserver Callbacks + + def init(openstex_client) do + Og.context(__ENV__, :debug) + :erlang.process_flag(:trap_exit, :true) + create_ets_table(openstex_client) + identity = create_identity(openstex_client) + identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, identity}) + expiry = to_seconds(identity) + Task.start_link(fn -> monitor_expiry(expiry) end) + {:ok, {openstex_client, identity}} + end + + def handle_call(:add_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :true) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:remove_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:update_identity, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + {:ok, new_identity} = Utils.create_identity(openstex_client) |> Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:stop, _from, state) do + Og.context(__ENV__, :debug) + {:stop, :shutdown, :ok, state} + end + + def terminate(:shutdown, {openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ets.delete(ets_tablename(openstex_client)) # explicilty remove + :ok + end + + def terminate(:shutdown, {:failed_to_start_child, openstex_client, {:already_started, _pid}}) do + Og.context(__ENV__, :debug) + :ets.delete(ets_tablename(openstex_client)) # explicilty remove + :ok + end + + def terminate(:normal, {_openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ok + end + + # private + + @spec create_identity(atom) :: Identity.t | no_return + defp create_identity(openstex_client) do + Og.context(__ENV__, :debug) + Utils.create_identity(openstex_client) + end + + + defp get_identity(openstex_client) do + unless genserver_exists?(openstex_client), do: start_link(openstex_client) + get_identity(openstex_client, 0) + end + defp get_identity(openstex_client, index) do + Og.context(__ENV__, :debug) + + retry = fn(openstex_client, index) -> + if index > @get_identity_retries do + raise "Cannot retrieve openstack identity, #{__ENV__.module}, #{__ENV__.line}, client: #{openstex_client}" + else + :timer.sleep(@get_identity_interval) + get_identity(openstex_client, index + 1) + end + end + + if ets_tablename(openstex_client) in :ets.all() do + table = :ets.lookup(ets_tablename(openstex_client), :identity) + case table do + [identity: identity] -> + if identity.lock === :true do + retry.(openstex_client, index) + else + identity + end + [] -> retry.(openstex_client, index) + end + else + retry.(openstex_client, index) + end + + end + + + defp monitor_expiry(expires) do + Og.context(__ENV__, :debug) + interval = (expires - 30) * 1000 + :timer.sleep(interval) + {:reply, :ok, _identity} = GenServer.call(self(), :add_lock) + {:reply, :ok, _identity} = GenServer.call(self(), :update_identity) + {:reply, :ok, new_identity} = GenServer.call(self(), :remove_lock) + expires = to_seconds(new_identity.token.expires) + monitor_expiry(expires) + end + + + defp create_ets_table(openstex_client) do + Og.context(__ENV__, :debug) + ets_options = [ + :set, # type + :protected, # read - all, write this process only. + :named_table, + {:heir, :none}, # don't let any process inherit the table. when the ets table dies, it dies. + {:write_concurrency, :false}, + {:read_concurrency, :true} + ] + unless ets_tablename(openstex_client) in :ets.all() do + :ets.new(ets_tablename(openstex_client), ets_options) + end + end + + + defp to_seconds(identity) do + iso_time = identity.token.expires + {:ok, expiry_ndt, offset} = Calendar.NaiveDateTime.Parse.iso8601(iso_time) + offset = + case offset do + :nil -> 0 + offset -> offset + end + {:ok, expiry_dt_utc} = Calendar.NaiveDateTime.with_offset_to_datetime_utc(expiry_ndt, offset) + {:ok, now} = Calendar.DateTime.from_erl(:calendar.universal_time(), "UTC") + {:ok, seconds, _microseconds, _when} = Calendar.DateTime.diff(expiry_dt_utc, now) + if seconds > 0 do + seconds + else + 0 + end + end + + + defp genserver_exists?(client) do + Process.whereis(client) != :nil + end + + +end diff --git a/lib/adapters/ovh/cloudstorage/keystone/utils.ex b/lib/adapters/ovh/cloudstorage/keystone/utils.ex new file mode 100644 index 0000000..285e0b7 --- /dev/null +++ b/lib/adapters/ovh/cloudstorage/keystone/utils.ex @@ -0,0 +1,55 @@ +defmodule Openstex.Adapters.Ovh.Cloudstorage.Keystone.Utils do + @moduledoc :false + alias Openstex.Services.Keystone.V2.Helpers, as: Keystone + alias Openstex.Adapters.Ovh.Cloudstorage.Config + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + @default_options [timeout: 10000, recv_timeout: 30000] + + + @doc :false + @spec create_identity(atom, atom) :: Identity.t | no_return + def create_identity(openstex_client, otp_app \\ :nil) do + Og.context(__ENV__, :debug) + + ovh_client = Module.concat(openstex_client, Ovh) + Application.ensure_all_started(:ex_ovh) + unless supervisor_exists?(ovh_client), do: ovh_client.start_link() + + keystone_config = Config.get_config_from_env(openstex_client, otp_app) |> Keyword.get(:keystone, :nil) || + openstex_client.config().keystone_config(openstex_client) + tenant_id = Keyword.fetch!(keystone_config, :tenant_id) + ovh_user = ExOvh.Services.V1.Cloud.Query.get_users(tenant_id) + |> ovh_client.request!() + |> Map.get(:body) + |> Enum.find(:nil, + fn(user) -> %{"description" => "ex_ovh"} = user end + ) + + ovh_user_id = + case ovh_user do + :nil -> + # create user for "ex_ovh" description + ExOvh.Services.V1.Cloud.Query.create_user(tenant_id, "ex_ovh") + |> ovh_client.request!() + |> Map.get("id") + ovh_user -> ovh_user["id"] + end + + resp = ExOvh.Services.V1.Cloud.Query.regenerate_credentials(tenant_id, ovh_user_id) + |> ovh_client.request!() + password = resp.body["password"] || Og.log_return("Password not found", __ENV__, :error) |> raise() + username = resp.body["username"] || Og.log_return("Username not found", __ENV__, :error) |> raise() + endpoint = keystone_config[:endpoint] || "https://auth.cloud.ovh.net/v2.0" + + # make sure the regenerate credentials (in the external ovh api) had a chance to take effect + :timer.sleep(1000) + + Keystone.authenticate!(endpoint, username, password, [tenant_id: tenant_id]) + end + + defp supervisor_exists?(ovh_client) do + Module.concat(ExOvh.Config, ovh_client) in Process.registered() + end + +end + diff --git a/lib/adapters/ovh/webstorage/adapter.ex b/lib/adapters/ovh/webstorage/adapter.ex new file mode 100644 index 0000000..a77a838 --- /dev/null +++ b/lib/adapters/ovh/webstorage/adapter.ex @@ -0,0 +1,9 @@ +defmodule Openstex.Adapters.Ovh.Webstorage.Adapter do + @moduledoc :false + @behaviour Openstex.Adapter + + def config(), do: Openstex.Adapters.Ovh.Webstorage.Config + def keystone(), do: Openstex.Adapters.Ovh.Webstorage.Keystone + +end + diff --git a/lib/adapters/ovh/webstorage/config/config.ex b/lib/adapters/ovh/webstorage/config/config.ex new file mode 100644 index 0000000..53012ef --- /dev/null +++ b/lib/adapters/ovh/webstorage/config/config.ex @@ -0,0 +1,184 @@ +defmodule Openstex.Adapters.Ovh.Webstorage.Config do + @moduledoc :false + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + @default_options [timeout: 10000, recv_timeout: 30000] + alias Openstex.HttpQuery + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Adapters.Ovh.Webstorage.Keystone.Utils + use Openstex.Adapter.Config + + + # public + + + def start_agent(openstex_client, opts) do + Og.context(__ENV__, :debug) + otp_app = Keyword.get(opts, :otp_app, :false) || Og.log_return(__ENV__, :error) |> raise() + + ovh_client = Module.concat(openstex_client, Ovh) + Application.ensure_all_started(:ex_ovh) + unless supervisor_exists?(ovh_client), do: ovh_client.start_link() + identity = Utils.create_identity(openstex_client, otp_app) + + Agent.start_link(fn -> config({openstex_client, ovh_client}, otp_app, identity) end, name: agent_name(openstex_client)) + end + + + @doc "Gets the rackspace related config variables from a supervised Agent" + def ovh_config(openstex_client) do + Agent.get(agent_name(openstex_client), fn(config) -> config[:ovh] end) + end + + + @doc :false + def swift_service_name(), do: "swift" + + + @doc :false + def swift_service_type(), do: "object-store" + + + # private + + + defp config({openstex_client, ovh_client}, otp_app, identity) do + swift_config = swift_config(openstex_client, otp_app) + keystone_config = keystone_config(openstex_client, otp_app, identity) + [ + ovh: ovh_client.ovh_config(), + keystone: keystone_config, + swift: swift_config, + httpoison: httpoison_config(openstex_client, otp_app) + ] + end + + defp keystone_config(openstex_client, otp_app, identity) do + + keystone_config = get_keystone_config_from_env(openstex_client, otp_app) + + tenant_id = keystone_config[:tenant_id] || + identity.token.tenant.id || + Og.log_return("cannot retrieve the tenant_id for keystone", __ENV__, :error) |> raise() + + user_id = keystone_config[:user_id] || + identity.user.id || + Og.log_return("cannot retrieve the user_id for keystone", __ENV__, :error) |> raise() + + endpoint = keystone_config[:endpoint] || + "https://auth.cloud.ovh.net/v2.0" + + [ + tenant_id: tenant_id, + user_id: user_id, + endpoint: endpoint + ] + end + + defp swift_config(openstex_client, otp_app) do + + swift_config = get_swift_config_from_env(openstex_client, otp_app) + + account_temp_url_key1 = get_account_temp_url(openstex_client, otp_app, :key1) || + swift_config[:account_temp_url_key1] || + :nil + + if account_temp_url_key1 != :nil && swift_config[:account_temp_url_key1] != account_temp_url_key1 do + Og.log("Warning, the `account_temp_url_key1` for the elixir `config.exs` for the swift client " <> + "#{inspect openstex_client} does not match the `X-Account-Meta-Temp-Url-Key` on the server. " <> + "This issue should probably be addressed. See Openstex.Adapter.Config.set_account_temp_url_key1/2.", __ENV__, :error) + end + + account_temp_url_key2 = get_account_temp_url(openstex_client, otp_app, :key2) || + swift_config[:account_temp_url_key2] || + :nil + + if account_temp_url_key2 != :nil && swift_config[:account_temp_url_key2] != account_temp_url_key2 do + Og.log("Warning, the `account_temp_url_key2` for the elixir `config.exs` for the swift client " <> + "#{inspect openstex_client} does not match the `X-Account-Meta-Temp-Url-Key-2` on the server. " <> + "This issue should probably be addressed. See Openstex.Adapter.Config.set_account_temp_url_key2/2.", __ENV__, :error) + end + + region = swift_config[:region] || "RegionOne" + + [ + account_temp_url_key1: account_temp_url_key1, + account_temp_url_key2: account_temp_url_key2, + region: region + ] + end + + + defp httpoison_config(openstex_client, otp_app) do + + httpoison_config = get_httpoison_config_from_env(openstex_client, otp_app) + + connect_timeout = httpoison_config[:connect_timeout] || 30000 # 30 seconds + receive_timeout = httpoison_config[:receive_timeout] || (60000 * 30) # 30 minutes + + [ + timeout: connect_timeout, + recv_timeout: receive_timeout + ] + end + + + defp get_account_temp_url(openstex_client, otp_app, key_atom) do + + ovh_client = Module.concat(openstex_client, Ovh) + Application.ensure_all_started(:ex_ovh) + unless supervisor_exists?(ovh_client), do: ovh_client.start_link() + + identity = Utils.create_identity(openstex_client, otp_app) + x_auth_token = Map.get(identity, :token) |> Map.get(:id) + endpoint = get_public_url(openstex_client, otp_app, identity) + + headers = + @default_headers ++ + [ + { + "X-Auth-Token", x_auth_token + } + ] + + header = + case key_atom do + :key1 -> "X-Account-Meta-Temp-Url-Key" + :key2 -> "X-Account-Meta-Temp-Url-Key-2" + end + + query = + %HttpQuery{ + method: :get, + uri: endpoint, + body: "", + headers: headers, + options: @default_options, + service: :openstack + } + {:ok, resp} = Openstex.Request.request(query, [], :nil) + + resp + |> Map.get(:headers) + |> Map.get(header) + end + + defp get_public_url(openstex_client, otp_app, identity) do + + swift_config = get_swift_config_from_env(openstex_client, otp_app) + + region = swift_config[:region] || "RegionOne" + + identity + |> Map.get(:service_catalog) + |> Enum.find(fn(%Identity.Service{} = service) -> service.name == swift_service_name() && service.type == swift_service_type() end) + |> Map.get(:endpoints) + |> Enum.find(fn(%Identity.Endpoint{} = endpoint) -> endpoint.region == region end) + |> Map.get(:public_url) + end + + defp supervisor_exists?(ovh_client) do + Module.concat(ExOvh.Config, ovh_client) in Process.registered() + end + + +end \ No newline at end of file diff --git a/lib/adapters/ovh/webstorage/keystone/keystone.ex b/lib/adapters/ovh/webstorage/keystone/keystone.ex new file mode 100644 index 0000000..39e659b --- /dev/null +++ b/lib/adapters/ovh/webstorage/keystone/keystone.ex @@ -0,0 +1,182 @@ +defmodule Openstex.Adapters.Ovh.Webstorage.Keystone do + @moduledoc :false + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Adapters.Ovh.Webstorage.Keystone.Utils + import Openstex.Utils, only: [ets_tablename: 1] + @behaviour Openstex.Adapter.Keystone + @get_identity_retries 5 + @get_identity_interval 1000 + + + # Public Openstex.Adapter.Keystone callbacks + + def start_link(openstex_client) do + Og.context(__ENV__, :debug) + GenServer.start_link(__MODULE__, openstex_client, [name: openstex_client]) + end + + def start_link(openstex_client, _opts) do + start_link(openstex_client) + end + + def identity(openstex_client) do + get_identity(openstex_client) + end + + def get_xauth_token(openstex_client) do + get_identity(openstex_client) |> Map.get(:token) |> Map.get(:id) + end + + # Genserver Callbacks + + def init(openstex_client) do + Og.context(__ENV__, :debug) + :erlang.process_flag(:trap_exit, :true) + create_ets_table(openstex_client) + identity = create_identity(openstex_client) + identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, identity}) + expiry = to_seconds(identity) + Task.start_link(fn -> monitor_expiry(expiry) end) + {:ok, {openstex_client, identity}} + end + + def handle_call(:add_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :true) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:remove_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:update_identity, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + {:ok, new_identity} = Utils.create_identity(openstex_client) |> Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:stop, _from, state) do + Og.context(__ENV__, :debug) + {:stop, :shutdown, :ok, state} + end + + def terminate(:shutdown, {openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ets.delete(ets_tablename(openstex_client)) # explicilty remove + :ok + end + + def terminate(:shutdown, {:failed_to_start_child, openstex_client, {:already_started, _pid}}) do + Og.context(__ENV__, :debug) + :ets.delete(ets_tablename(openstex_client)) # explicilty remove + :ok + end + + def terminate(:normal, {_openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ok + end + + # private + + @spec create_identity(atom) :: Identity.t | no_return + defp create_identity(openstex_client) do + Og.context(__ENV__, :debug) + Utils.create_identity(openstex_client) + end + + + defp get_identity(openstex_client) do + unless supervisor_exists?(openstex_client), do: start_link(openstex_client) + get_identity(openstex_client, 0) + end + defp get_identity(openstex_client, index) do + Og.context(__ENV__, :debug) + + retry = fn(openstex_client, index) -> + if index > @get_identity_retries do + raise "Cannot retrieve openstack identity, #{__ENV__.module}, #{__ENV__.line}, client: #{openstex_client}" + else + :timer.sleep(@get_identity_interval) + get_identity(openstex_client, index + 1) + end + end + + if ets_tablename(openstex_client) in :ets.all() do + table = :ets.lookup(ets_tablename(openstex_client), :identity) + case table do + [identity: identity] -> + if identity.lock === :true do + retry.(openstex_client, index) + else + identity + end + [] -> retry.(openstex_client, index) + end + else + retry.(openstex_client, index) + end + + end + + + defp monitor_expiry(expires) do + Og.context(__ENV__, :debug) + interval = (expires - 30) * 1000 + :timer.sleep(interval) + {:reply, :ok, _identity} = GenServer.call(self(), :add_lock) + {:reply, :ok, _identity} = GenServer.call(self(), :update_identity) + {:reply, :ok, new_identity} = GenServer.call(self(), :remove_lock) + expires = to_seconds(new_identity.token.expires) + monitor_expiry(expires) + end + + + defp create_ets_table(openstex_client) do + Og.context(__ENV__, :debug) + ets_options = [ + :set, # type + :protected, # read - all, write this process only. + :named_table, + {:heir, :none}, # don't let any process inherit the table. when the ets table dies, it dies. + {:write_concurrency, :false}, + {:read_concurrency, :true} + ] + unless ets_tablename(openstex_client) in :ets.all() do + :ets.new(ets_tablename(openstex_client), ets_options) + end + end + + + defp to_seconds(identity) do + iso_time = identity.token.expires + {:ok, expiry_ndt, offset} = Calendar.NaiveDateTime.Parse.iso8601(iso_time) + offset = + case offset do + :nil -> 0 + offset -> offset + end + {:ok, expiry_dt_utc} = Calendar.NaiveDateTime.with_offset_to_datetime_utc(expiry_ndt, offset) + {:ok, now} = Calendar.DateTime.from_erl(:calendar.universal_time(), "UTC") + {:ok, seconds, _microseconds, _when} = Calendar.DateTime.diff(expiry_dt_utc, now) + if seconds > 0 do + seconds + else + 0 + end + end + + + defp supervisor_exists?(client) do + Process.whereis(client) != :nil + end + + +end diff --git a/lib/adapters/ovh/webstorage/keystone/utils.ex b/lib/adapters/ovh/webstorage/keystone/utils.ex new file mode 100644 index 0000000..47af875 --- /dev/null +++ b/lib/adapters/ovh/webstorage/keystone/utils.ex @@ -0,0 +1,68 @@ +defmodule Openstex.Adapters.Ovh.Webstorage.Keystone.Utils do + @moduledoc :false + alias Openstex.Services.Keystone.V2.Helpers, as: Keystone + alias Openstex.Adapters.Ovh.Webstorage.Config + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + @default_options [timeout: 10000, recv_timeout: 30000] + defstruct [ :domain, :storage_limit, :server, :endpoint, :username, :password, :tenant_name ] + @type t :: %__MODULE__{ + domain: String.t, + storage_limit: String.t, + server: String.t, + endpoint: String.t, + username: String.t, + password: String.t, + tenant_name: String.t + } + + @doc :false + @spec webstorage(atom, String.t) :: __MODULE__.t | no_return + def webstorage(ovh_client, cdn_name) do + Og.context(__ENV__, :debug) + + properties = ExOvh.Services.V1.Webstorage.Query.get_service(cdn_name) |> ovh_client.request!() |> Map.fetch!(:body) + credentials = ExOvh.Services.V1.Webstorage.Query.get_credentials(cdn_name) |> ovh_client.request!() |> Map.fetch!(:body) + + webstorage = + %{ + "domain" => _domain, + "storageLimit" => _storage_limit, + "server" => _server, + "endpoint" => _endpoint, + "login" => username, + "password" => _password, + "tenant" => tenant_name + } = Map.merge(properties, credentials) + webstorage + |> Map.delete("tenant") |> Map.delete("login") + |> Map.put("username", username) |> Map.put("tenantName", tenant_name) + |> Mapail.map_to_struct!(__MODULE__) + end + + + @doc :false + @spec create_identity(atom, atom) :: Identity.t | no_return + def create_identity(openstex_client, otp_app \\ :nil) do + Og.context(__ENV__, :debug) + + ovh_client = Module.concat(openstex_client, Ovh) + Application.ensure_all_started(:ex_ovh) + unless supervisor_exists?(ovh_client), do: ovh_client.start_link() + + ovh_config = Config.get_config_from_env(openstex_client, otp_app) |> Keyword.get(:ovh, :nil) || + openstex_client.config().ovh_config(openstex_client) || + ovh_client.config() || + Og.log_return("Cannot retrieve the ovh_config", __ENV__, :error) |> raise() + + cdn_name = ovh_config[:cdn_name] || + Og.log_return("Cannot retrieve the CDN name for the webstorage client #{openstex_client}", __ENV__, :error) |> raise() + + %{endpoint: endpoint, username: username, password: password, tenant_name: tenant_name} = webstorage(ovh_client, cdn_name) + Keystone.authenticate!(endpoint, username, password, [tenant_name: tenant_name]) + end + + defp supervisor_exists?(ovh_client) do + Module.concat(ExOvh.Config, ovh_client) in Process.registered() + end + +end \ No newline at end of file diff --git a/lib/adapters/rackspace/cloudfiles/adapter.ex b/lib/adapters/rackspace/cloudfiles/adapter.ex new file mode 100644 index 0000000..241105f --- /dev/null +++ b/lib/adapters/rackspace/cloudfiles/adapter.ex @@ -0,0 +1,9 @@ +defmodule Openstex.Adapters.Rackspace.Cloudfiles.Adapter do + @moduledoc :false + @behaviour Openstex.Adapter + + def config(), do: Openstex.Adapters.Rackspace.Cloudfiles.Config + def keystone(), do: Openstex.Adapters.Rackspace.Cloudfiles.Keystone + +end + diff --git a/lib/adapters/rackspace/cloudfiles/config/config.ex b/lib/adapters/rackspace/cloudfiles/config/config.ex new file mode 100644 index 0000000..bd0f56f --- /dev/null +++ b/lib/adapters/rackspace/cloudfiles/config/config.ex @@ -0,0 +1,232 @@ +defmodule Openstex.Adapters.Rackspace.Cloudfiles.Config do + @moduledoc :false + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + @default_options [timeout: 10000, recv_timeout: 30000] + alias Openstex.HttpQuery + alias Openstex.Services.Keystone.V2.Query + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Services.Keystone.V2.Helpers, as: Keystone + use Openstex.Adapter.Config + + + # public + + + def start_agent(client, opts) do + Og.context(__ENV__, :debug) + otp_app = Keyword.get(opts, :otp_app, :false) || Og.log_return(__ENV__, :error) |> raise() + identity = create_identity(client, otp_app) + Agent.start_link(fn -> config(client, otp_app, identity) end, name: agent_name(client)) + end + + + @doc "Gets the rackspace related config variables from a supervised Agent" + def rackspace_config(client) do + Agent.get(agent_name(client), fn(config) -> config[:rackspace] end) + end + + + @doc :false + def swift_service_name(), do: "cloudFiles" + + + @doc :false + def swift_service_type(), do: "object-store" + + + # private + + + defp config(client, otp_app, identity) do + [ + rackspace: rackspace_config(client, otp_app), + keystone: keystone_config(client, otp_app, identity), + swift: swift_config(client, otp_app, identity), + httpoison: httpoison_config(client, otp_app) + ] + end + + + defp rackspace_config(client, otp_app) do + __MODULE__.get_config_from_env(client, otp_app) |> Keyword.fetch!(:rackspace) + end + + + defp keystone_config(client, otp_app, identity) do + + keystone_config = get_keystone_config_from_env(client, otp_app) + + tenant_id = keystone_config[:tenant_id] || + identity.token.tenant.id || + Og.log_return("cannot retrieve the tenant_id for keystone", __ENV__, :error) |> raise() + + user_id = keystone_config[:user_id] || + identity.user.id || + Og.log_return("cannot retrieve the user_id for keystone", __ENV__, :error) |> raise() + + endpoint = keystone_config[:endpoint] || + "https://identity.api.rackspacecloud.com/v2.0" + + [ + tenant_id: tenant_id, + user_id: user_id, + endpoint: endpoint + ] + end + + + defp swift_config(client, otp_app, identity) do + + swift_config = get_swift_config_from_env(client, otp_app) + + account_temp_url_key1 = get_account_temp_url(client, otp_app, :key1) || + swift_config[:account_temp_url_key1] || + :nil + + if account_temp_url_key1 != :nil && swift_config[:account_temp_url_key1] != account_temp_url_key1 do + Og.log("Warning, the `account_temp_url_key1` for the elixir `config.exs` for the swift client " <> + "#{inspect client} does not match the `X-Account-Meta-Temp-Url-Key` on the server. " <> + "This issue should probably be addressed. See Openstex.Adapter.Config.set_account_temp_url_key1/2.", __ENV__, :error) + end + + account_temp_url_key2 = get_account_temp_url(client, otp_app, :key2) || + swift_config[:account_temp_url_key2] || + :nil + + if account_temp_url_key2 != :nil && swift_config[:account_temp_url_key2] != account_temp_url_key2 do + Og.log("Warning, the `account_temp_url_key2` for the elixir `config.exs` for the swift client " <> + "#{inspect client} does not match the `X-Account-Meta-Temp-Url-Key-2` on the server. " <> + "This issue should probably be addressed. See Openstex.Adapter.Config.set_account_temp_url_key2/2.", __ENV__, :error) + end + + region = swift_config[:region] || + identity.user.mapail["RAX-AUTH:defaultRegion"] || + Og.log_return("cannot retrieve the region for keystone", __ENV__, :error) |> raise() + + if swift_config[:region] != :nil && identity.user.mapail["RAX-AUTH:defaultRegion"] != swift_config[:region] do + Og.log("Warning, the `swift_config[:region]` for the elixir `config.exs` for the swift client " <> + "#{inspect client} does not match the `RAX-AUTH:defaultRegion` on the server. " <> + "This issue should probably be addressed.", __ENV__, :error) + end + + [ + account_temp_url_key1: account_temp_url_key1, + account_temp_url_key2: account_temp_url_key2, + region: region + ] + end + + + defp httpoison_config(client, otp_app) do + + httpoison_config = get_httpoison_config_from_env(client, otp_app) + + connect_timeout = httpoison_config[:connect_timeout] || 30000 # 30 seconds + receive_timeout = httpoison_config[:receive_timeout] || (60000 * 30) # 30 minutes + + [ + timeout: connect_timeout, + recv_timeout: receive_timeout, + ] + end + + + defp get_account_temp_url(client, otp_app, key_atom) do + + identity = create_identity(client, otp_app) + x_auth_token = Map.get(identity, :token) |> Map.get(:id) + endpoint = get_public_url(client, otp_app, identity) + + headers = + @default_headers ++ + [ + { + "X-Auth-Token", x_auth_token + } + ] + + header = + case key_atom do + :key1 -> "X-Account-Meta-Temp-Url-Key" + :key2 -> "X-Account-Meta-Temp-Url-Key-2" + end + + query = + %HttpQuery{ + method: :get, + uri: endpoint, + body: "", + headers: headers, + options: @default_options, + service: :openstack + } + {:ok, resp} = Openstex.Request.request(query, [], :nil) + + resp + |> Map.get(:headers) + |> Map.get(header) + end + + + defp create_identity(client, otp_app) do + Og.context(__ENV__, :debug) + + rackpace_config = rackspace_config(client, otp_app) + keystone_config = __MODULE__.get_config_from_env(client, otp_app) |> Keyword.fetch!(:keystone) + + api_key = Keyword.get(rackpace_config, :api_key, :nil) + password = Keyword.get(rackpace_config, :password, :nil) + username = Keyword.fetch!(rackpace_config, :username) + endpoint = Keyword.get(keystone_config, :endpoint, "https://identity.api.rackspacecloud.com/v2.0") + + {:ok, identity_resp} = + case api_key do + + :nil -> + {:ok, resp} = Query.get_token(endpoint, username, password) |> Openstex.Request.request([], :nil) + + api_key -> + body = + %{ + "auth" => + %{ + "RAX-KSKEY:apiKeyCredentials" => %{ + "apiKey" => api_key, + "username" => username + } + } + } + |> Poison.encode!() + query = + %HttpQuery{ + method: :post, + uri: endpoint <> "/tokens", + body: body, + headers: @default_headers, + options: @default_options, + service: :openstack + } + {:ok, resp} = Openstex.Request.request(query, [], :nil) + end + Keystone.parse_nested_map_into_identity_struct(identity_resp.body) + end + + + defp get_public_url(client, otp_app, identity) do + + swift_config = get_swift_config_from_env(client, otp_app) + + region = swift_config[:region] || + identity.user.mapail["RAX-AUTH:defaultRegion"] || + Og.log_return("cannot retrieve the region for keystone", __ENV__, :error) |> raise() + + identity + |> Map.get(:service_catalog) + |> Enum.find(fn(%Identity.Service{} = service) -> service.name == swift_service_name() && service.type == swift_service_type() end) + |> Map.get(:endpoints) + |> Enum.find(fn(%Identity.Endpoint{} = endpoint) -> endpoint.region == region end) + |> Map.get(:public_url) + end + + +end \ No newline at end of file diff --git a/lib/adapters/rackspace/cloudfiles/keystone/keystone.ex b/lib/adapters/rackspace/cloudfiles/keystone/keystone.ex new file mode 100644 index 0000000..68a0da8 --- /dev/null +++ b/lib/adapters/rackspace/cloudfiles/keystone/keystone.ex @@ -0,0 +1,176 @@ +defmodule Openstex.Adapters.Rackspace.Cloudfiles.Keystone do + @moduledoc :false + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Adapters.Rackspace.Cloudfiles.Keystone.Utils + import Openstex.Utils, only: [ets_tablename: 1] + @behaviour Openstex.Adapter.Keystone + @get_identity_retries 5 + @get_identity_interval 1000 + + + # Public Openstex.Adapter.Keystone callbacks + + def start_link(openstex_client) do + Og.context(__ENV__, :debug) + GenServer.start_link(__MODULE__, openstex_client, [name: openstex_client]) + end + + def start_link(openstex_client, _opts) do + start_link(openstex_client) + end + + def identity(openstex_client) do + get_identity(openstex_client) + end + + def get_xauth_token(openstex_client) do + get_identity(openstex_client) |> Map.get(:token) |> Map.get(:id) + end + + # Genserver Callbacks + + def init(openstex_client) do + Og.context(__ENV__, :debug) + :erlang.process_flag(:trap_exit, :true) + create_ets_table(openstex_client) + identity = create_identity(openstex_client) + identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, identity}) + expiry = to_seconds(identity) + Task.start_link(fn -> monitor_expiry(expiry) end) + {:ok, {openstex_client, identity}} + end + + def handle_call(:add_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :true) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:remove_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:update_identity, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + {:ok, new_identity} = Utils.create_identity(openstex_client) |> Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:stop, _from, state) do + Og.context(__ENV__, :debug) + {:stop, :shutdown, :ok, state} + end + + def terminate(:shutdown, {openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ets.delete(ets_tablename(openstex_client)) # explicilty remove + :ok + end + + def terminate(:normal, {_openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ok + end + + # private + + @spec create_identity(atom) :: Identity.t | no_return + defp create_identity(openstex_client) do + Og.context(__ENV__, :debug) + Utils.create_identity(openstex_client) + end + + + defp get_identity(openstex_client) do + unless supervisor_exists?(openstex_client), do: start_link(openstex_client) + get_identity(openstex_client, 0) + end + defp get_identity(openstex_client, index) do + Og.context(__ENV__, :debug) + + retry = fn(openstex_client, index) -> + if index > @get_identity_retries do + raise "Cannot retrieve openstack identity, #{__ENV__.module}, #{__ENV__.line}, client: #{openstex_client}" + else + :timer.sleep(@get_identity_interval) + get_identity(openstex_client, index + 1) + end + end + + if ets_tablename(openstex_client) in :ets.all() do + table = :ets.lookup(ets_tablename(openstex_client), :identity) + case table do + [identity: identity] -> + if identity.lock === :true do + retry.(openstex_client, index) + else + identity + end + [] -> retry.(openstex_client, index) + end + else + retry.(openstex_client, index) + end + + end + + + defp monitor_expiry(expires) do + Og.context(__ENV__, :debug) + interval = (expires - 30) * 1000 + :timer.sleep(interval) + {:reply, :ok, _identity} = GenServer.call(self(), :add_lock) + {:reply, :ok, _identity} = GenServer.call(self(), :update_identity) + {:reply, :ok, new_identity} = GenServer.call(self(), :remove_lock) + expires = to_seconds(new_identity.token.expires) + monitor_expiry(expires) + end + + + defp create_ets_table(openstex_client) do + Og.context(__ENV__, :debug) + ets_options = [ + :set, # type + :protected, # read - all, write this process only. + :named_table, + {:heir, :none}, # don't let any process inherit the table. when the ets table dies, it dies. + {:write_concurrency, :false}, + {:read_concurrency, :true} + ] + unless ets_tablename(openstex_client) in :ets.all() do + :ets.new(ets_tablename(openstex_client), ets_options) + end + end + + + defp to_seconds(identity) do + iso_time = identity.token.expires + {:ok, expiry_ndt, offset} = Calendar.NaiveDateTime.Parse.iso8601(iso_time) + offset = + case offset do + :nil -> 0 + offset -> offset + end + {:ok, expiry_dt_utc} = Calendar.NaiveDateTime.with_offset_to_datetime_utc(expiry_ndt, offset) + {:ok, now} = Calendar.DateTime.from_erl(:calendar.universal_time(), "UTC") + {:ok, seconds, _microseconds, _when} = Calendar.DateTime.diff(expiry_dt_utc, now) + if seconds > 0 do + seconds + else + 0 + end + end + + + defp supervisor_exists?(client) do + Process.whereis(client) != :nil + end + + +end diff --git a/lib/adapters/rackspace/cloudfiles/keystone/utils.ex b/lib/adapters/rackspace/cloudfiles/keystone/utils.ex new file mode 100644 index 0000000..42e6bb6 --- /dev/null +++ b/lib/adapters/rackspace/cloudfiles/keystone/utils.ex @@ -0,0 +1,57 @@ +defmodule Openstex.Adapters.Rackspace.Cloudfiles.Keystone.Utils do + @moduledoc :false + alias Openstex.HttpQuery + alias Openstex.Services.Keystone.V2.Helpers, as: Keystone + alias Openstex.Services.Keystone.V2.Query + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + @default_options [timeout: 10000, recv_timeout: 30000] + + + @doc :false + @spec create_identity(atom) :: Identity.t | no_return + def create_identity(openstex_client) do + Og.context(__ENV__, :debug) + + rackpace_config = openstex_client.config().rackspace_config(openstex_client) + keystone_config = openstex_client.config().keystone_config(openstex_client) + + api_key = rackpace_config[:api_key] + password = rackpace_config[:password] + username = rackpace_config[:username] + endpoint = keystone_config[:endpoint] + + {:ok, identity_resp} = + case api_key do + + :nil -> + {:ok, resp} = Query.get_token(endpoint, username, password) |> Openstex.Request.request([], :nil) + + api_key -> + body = + %{ + "auth" => + %{ + "RAX-KSKEY:apiKeyCredentials" => %{ + "apiKey" => api_key, + "username" => username + } + } + } + |> Poison.encode!() + query = + %HttpQuery{ + method: :post, + uri: endpoint <> "/tokens", + body: body, + headers: @default_headers, + options: @default_options, + service: :openstack + } + {:ok, resp} = Openstex.Request.request(query, [], :nil) + end + Keystone.parse_nested_map_into_identity_struct(identity_resp.body) + end + + +end + diff --git a/lib/adapters/rackspace/cloudfilesCDN/adapter.ex b/lib/adapters/rackspace/cloudfilesCDN/adapter.ex new file mode 100644 index 0000000..c909c63 --- /dev/null +++ b/lib/adapters/rackspace/cloudfilesCDN/adapter.ex @@ -0,0 +1,9 @@ +defmodule Openstex.Adapters.Rackspace.CloudfilesCDN.Adapter do + @moduledoc :false + @behaviour Openstex.Adapter + + def config(), do: Openstex.Adapters.Rackspace.CloudfilesCDN.Config + def keystone(), do: Openstex.Adapters.Rackspace.CloudfilesCDN.Keystone + +end + diff --git a/lib/adapters/rackspace/cloudfilesCDN/config/config.ex b/lib/adapters/rackspace/cloudfilesCDN/config/config.ex new file mode 100644 index 0000000..2663428 --- /dev/null +++ b/lib/adapters/rackspace/cloudfilesCDN/config/config.ex @@ -0,0 +1,220 @@ +defmodule Openstex.Adapters.Rackspace.CloudfilesCDN.Config do + @moduledoc :false + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + @default_options [timeout: 10000, recv_timeout: 30000] + alias Openstex.HttpQuery + alias Openstex.Services.Keystone.V2.Query + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Services.Keystone.V2.Helpers, as: Keystone + use Openstex.Adapter.Config + + + # public + + def start_agent(client, opts) do + Og.context(__ENV__, :debug) + otp_app = Keyword.get(opts, :otp_app, :false) || Og.log_return(__ENV__, :error) |> raise() + identity = create_identity(client, otp_app) + Agent.start_link(fn -> config(client, otp_app, identity) end, name: agent_name(client)) + end + + + @doc :false + def swift_service_name(), do: "cloudFilesCDN" + + + @doc :false + def swift_service_type(), do: "rax:object-cdn" + + + # private + + + defp config(client, otp_app, identity) do + [ + keystone: keystone_config(client, otp_app, identity), + swift: swift_config(client, otp_app, identity), + httpoison: httpoison_config(client, otp_app) + ] + end + + + defp keystone_config(client, otp_app, identity) do + + keystone_config = get_keystone_config_from_env(client, otp_app) + + tenant_id = keystone_config[:tenant_id] || + identity.token.tenant.id || + Og.log_return("cannot retrieve the tenant_id for keystone", __ENV__, :error) |> raise() + + user_id = keystone_config[:user_id] || + identity.user.id || + Og.log_return("cannot retrieve the user_id for keystone", __ENV__, :error) |> raise() + + endpoint = keystone_config[:endpoint] || + "https://identity.api.rackspacecloud.com/v2.0" + + [ + tenant_id: tenant_id, + user_id: user_id, + endpoint: endpoint + ] + + end + + + defp swift_config(client, otp_app, identity) do + + swift_config = get_swift_config_from_env(client, otp_app) + + account_temp_url_key1 = get_account_temp_url(client, otp_app, :key1) || + swift_config[:account_temp_url_key1] || + :nil + + if account_temp_url_key1 != :nil && swift_config[:account_temp_url_key1] != account_temp_url_key1 do + Og.log("Warning, the `account_temp_url_key1` for the elixir `config.exs` for the swift client " <> + "#{inspect client} does not match the `X-Account-Meta-Temp-Url-Key` on the server. " <> + "This issue should probably be addressed. See Openstex.Adapter.Config.set_account_temp_url_key1/2.", __ENV__, :error) + end + + account_temp_url_key2 = get_account_temp_url(client, otp_app, :key2) || + swift_config[:account_temp_url_key2] || + :nil + + if account_temp_url_key2 != :nil && swift_config[:account_temp_url_key2] != account_temp_url_key2 do + Og.log("Warning, the `account_temp_url_key2` for the elixir `config.exs` for the swift client " <> + "#{inspect client} does not match the `X-Account-Meta-Temp-Url-Key-2` on the server. " <> + "This issue should probably be addressed. See Openstex.Adapter.Config.set_account_temp_url_key2/2.", __ENV__, :error) + end + + region = swift_config[:region] || + identity.user.mapail["RAX-AUTH:defaultRegion"] || + Og.log_return("cannot retrieve the region for keystone", __ENV__, :error) |> raise() + + if swift_config[:region] != :nil && identity.user.mapail["RAX-AUTH:defaultRegion"] != swift_config[:region] do + Og.log("Warning, the `swift_config[:region]` for the elixir `config.exs` for the swift client " <> + "#{inspect client} does not match the `RAX-AUTH:defaultRegion` on the server. " <> + "This issue should probably be addressed.", __ENV__, :error) + end + + [ + account_temp_url_key1: account_temp_url_key1, + account_temp_url_key2: account_temp_url_key2, + region: region + ] + end + + + defp httpoison_config(client, otp_app) do + + httpoison_config = get_httpoison_config_from_env(client, otp_app) + + connect_timeout = httpoison_config[:connect_timeout] || 30000 # 30 seconds + receive_timeout = httpoison_config[:receive_timeout] || (60000 * 30) # 30 minutes + + [ + timeout: connect_timeout, + recv_timeout: receive_timeout, + ] + end + + + defp get_account_temp_url(client, otp_app, key_atom) do + + identity = create_identity(client, otp_app) + x_auth_token = Map.get(identity, :token) |> Map.get(:id) + public_url = get_public_url(client, otp_app, identity) + + headers = + @default_headers ++ + [ + { + "X-Auth-Token", x_auth_token + } + ] + + header = + case key_atom do + :key1 -> "X-Account-Meta-Temp-Url-Key" + :key2 -> "X-Account-Meta-Temp-Url-Key-2" + end + + query = + %HttpQuery{ + method: :get, + uri: public_url, + body: "", + headers: headers, + options: @default_options, + service: :openstack + } + {:ok, resp} = Openstex.Request.request(query, [], :nil) + + resp + |> Map.get(:headers) + |> Map.get(header) + end + + + defp create_identity(client, otp_app) do + Og.context(__ENV__, :debug) + + rackpace_config = __MODULE__.get_config_from_env(client, otp_app) |> Keyword.fetch!(:rackspace) + keystone_config = __MODULE__.get_config_from_env(client, otp_app) |> Keyword.fetch!(:keystone) + + api_key = Keyword.get(rackpace_config, :api_key, :nil) + password = Keyword.get(rackpace_config, :password, :nil) + username = Keyword.fetch!(rackpace_config, :username) + endpoint = Keyword.get(keystone_config, :endpoint, "https://identity.api.rackspacecloud.com/v2.0") + + {:ok, identity_resp} = + case api_key do + + :nil -> + {:ok, resp} = Query.get_token(endpoint, username, password) |> Openstex.Request.request([], :nil) + + api_key -> + body = + %{ + "auth" => + %{ + "RAX-KSKEY:apiKeyCredentials" => %{ + "apiKey" => api_key, + "username" => username + } + } + } + |> Poison.encode!() + query = + %HttpQuery{ + method: :post, + uri: endpoint <> "/tokens", + body: body, + headers: @default_headers, + options: @default_options, + service: :openstack + } + {:ok, resp} = Openstex.Request.request(query, [], :nil) + end + Keystone.parse_nested_map_into_identity_struct(identity_resp.body) + end + + + defp get_public_url(client, otp_app, identity) do + + swift_config = get_swift_config_from_env(client, otp_app) + + region = swift_config[:region] || + identity.user.mapail["RAX-AUTH:defaultRegion"] || + Og.log_return("cannot retrieve the region for keystone", __ENV__, :error) |> raise() + + identity + |> Map.get(:service_catalog) + |> Enum.find(fn(%Identity.Service{} = service) -> service.name == swift_service_name() && service.type == swift_service_type() end) + |> Map.get(:endpoints) + |> Enum.find(fn(%Identity.Endpoint{} = endpoint) -> endpoint.region == region end) + |> Map.get(:public_url) + end + + +end \ No newline at end of file diff --git a/lib/adapters/rackspace/cloudfilesCDN/keystone/keystone.ex b/lib/adapters/rackspace/cloudfilesCDN/keystone/keystone.ex new file mode 100644 index 0000000..d7af524 --- /dev/null +++ b/lib/adapters/rackspace/cloudfilesCDN/keystone/keystone.ex @@ -0,0 +1,176 @@ +defmodule Openstex.Adapters.Rackspace.CloudfilesCDN.Keystone do + @moduledoc :false + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Adapters.Rackspace.CloudfilesCDN.Keystone.Utils + import Openstex.Utils, only: [ets_tablename: 1] + @behaviour Openstex.Adapter.Keystone + @get_identity_retries 5 + @get_identity_interval 1000 + + + # Public Openstex.Adapter.Keystone callbacks + + def start_link(openstex_client) do + Og.context(__ENV__, :debug) + GenServer.start_link(__MODULE__, openstex_client, [name: openstex_client]) + end + + def start_link(openstex_client, _opts) do + start_link(openstex_client) + end + + def identity(openstex_client) do + get_identity(openstex_client) + end + + def get_xauth_token(openstex_client) do + get_identity(openstex_client) |> Map.get(:token) |> Map.get(:id) + end + + # Genserver Callbacks + + def init(openstex_client) do + Og.context(__ENV__, :debug) + :erlang.process_flag(:trap_exit, :true) + create_ets_table(openstex_client) + identity = create_identity(openstex_client) + identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, identity}) + expiry = to_seconds(identity) + Task.start_link(fn -> monitor_expiry(expiry) end) + {:ok, {openstex_client, identity}} + end + + def handle_call(:add_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :true) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:remove_lock, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + new_identity = Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:update_identity, _from, {openstex_client, identity}) do + Og.context(__ENV__, :debug) + {:ok, new_identity} = Utils.create_identity(openstex_client) |> Map.put(identity, :lock, :false) + :ets.insert(ets_tablename(openstex_client), {:identity, new_identity}) + {:reply, :ok, {openstex_client, new_identity}} + end + + def handle_call(:stop, _from, state) do + Og.context(__ENV__, :debug) + {:stop, :shutdown, :ok, state} + end + + def terminate(:shutdown, {openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ets.delete(ets_tablename(openstex_client)) # explicilty remove + :ok + end + + def terminate(:normal, {_openstex_client, _identity}) do + Og.context(__ENV__, :debug) + :ok + end + + # private + + @spec create_identity(atom) :: Identity.t | no_return + defp create_identity(openstex_client) do + Og.context(__ENV__, :debug) + Utils.create_identity(openstex_client) + end + + + defp get_identity(openstex_client) do + unless supervisor_exists?(openstex_client), do: start_link(openstex_client) + get_identity(openstex_client, 0) + end + defp get_identity(openstex_client, index) do + Og.context(__ENV__, :debug) + + retry = fn(openstex_client, index) -> + if index > @get_identity_retries do + raise "Cannot retrieve openstack identity, #{__ENV__.module}, #{__ENV__.line}, client: #{openstex_client}" + else + :timer.sleep(@get_identity_interval) + get_identity(openstex_client, index + 1) + end + end + + if ets_tablename(openstex_client) in :ets.all() do + table = :ets.lookup(ets_tablename(openstex_client), :identity) + case table do + [identity: identity] -> + if identity.lock === :true do + retry.(openstex_client, index) + else + identity + end + [] -> retry.(openstex_client, index) + end + else + retry.(openstex_client, index) + end + + end + + + defp monitor_expiry(expires) do + Og.context(__ENV__, :debug) + interval = (expires - 30) * 1000 + :timer.sleep(interval) + {:reply, :ok, _identity} = GenServer.call(self(), :add_lock) + {:reply, :ok, _identity} = GenServer.call(self(), :update_identity) + {:reply, :ok, new_identity} = GenServer.call(self(), :remove_lock) + expires = to_seconds(new_identity.token.expires) + monitor_expiry(expires) + end + + + defp create_ets_table(openstex_client) do + Og.context(__ENV__, :debug) + ets_options = [ + :set, # type + :protected, # read - all, write this process only. + :named_table, + {:heir, :none}, # don't let any process inherit the table. when the ets table dies, it dies. + {:write_concurrency, :false}, + {:read_concurrency, :true} + ] + unless ets_tablename(openstex_client) in :ets.all() do + :ets.new(ets_tablename(openstex_client), ets_options) + end + end + + + defp to_seconds(identity) do + iso_time = identity.token.expires + {:ok, expiry_ndt, offset} = Calendar.NaiveDateTime.Parse.iso8601(iso_time) + offset = + case offset do + :nil -> 0 + offset -> offset + end + {:ok, expiry_dt_utc} = Calendar.NaiveDateTime.with_offset_to_datetime_utc(expiry_ndt, offset) + {:ok, now} = Calendar.DateTime.from_erl(:calendar.universal_time(), "UTC") + {:ok, seconds, _microseconds, _when} = Calendar.DateTime.diff(expiry_dt_utc, now) + if seconds > 0 do + seconds + else + 0 + end + end + + + defp supervisor_exists?(client) do + Process.whereis(client) != :nil + end + + +end diff --git a/lib/adapters/rackspace/cloudfilesCDN/keystone/utils.ex b/lib/adapters/rackspace/cloudfilesCDN/keystone/utils.ex new file mode 100644 index 0000000..bc65bd6 --- /dev/null +++ b/lib/adapters/rackspace/cloudfilesCDN/keystone/utils.ex @@ -0,0 +1,57 @@ +defmodule Openstex.Adapters.Rackspace.CloudfilesCDN.Keystone.Utils do + @moduledoc :false + alias Openstex.HttpQuery + alias Openstex.Services.Keystone.V2.Helpers, as: Keystone + alias Openstex.Services.Keystone.V2.Query + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + @default_options [timeout: 10000, recv_timeout: 30000] + + + @doc :false + @spec create_identity(atom) :: Identity.t | no_return + def create_identity(openstex_client) do + Og.context(__ENV__, :debug) + + rackpace_config = openstex_client.config().rackspace_config(openstex_client) + keystone_config = openstex_client.config().keystone_config(openstex_client) + + api_key = rackpace_config[:api_key] + password = rackpace_config[:password] + username = rackpace_config[:username] + endpoint = keystone_config[:endpoint] + + {:ok, identity_resp} = + case api_key do + + :nil -> + {:ok, resp} = Query.get_token(endpoint, username, password) |> Openstex.Request.request([], :nil) + + api_key -> + body = + %{ + "auth" => + %{ + "RAX-KSKEY:apiKeyCredentials" => %{ + "apiKey" => api_key, + "username" => username + } + } + } + |> Poison.encode!() + query = + %HttpQuery{ + method: :post, + uri: endpoint <> "/tokens", + body: body, + headers: @default_headers, + options: @default_options, + service: :openstack + } + {:ok, resp} = Openstex.Request.request(query, [], :nil) + end + Keystone.parse_nested_map_into_identity_struct(identity_resp.body) + end + + +end + diff --git a/lib/client.ex b/lib/client.ex new file mode 100644 index 0000000..25350aa --- /dev/null +++ b/lib/client.ex @@ -0,0 +1,77 @@ +defmodule Openstex.Client do + alias Openstex.{HttpQuery, Query, Response} + + defmacro __using__(opts) do + quote bind_quoted: [opts: opts] do + alias Openstex.{Response, ResponseError, Request, Transformation} + @behaviour Openstex.Client + + # public callback functions + + @doc :false + def __adapter__() do + client = unquote(opts) |> Keyword.fetch!(:client) + otp_app = unquote(opts) |> Keyword.fetch!(:otp_app) + case Application.get_env(otp_app, client) do + :nil -> Application.get_all_env(otp_app) + config -> config + end + |> Keyword.fetch!(:adapter) + end + + @doc :false + def config(), do: __adapter__().config() + + @doc :false + def keystone(), do: __adapter__().keystone() + + @doc :false + def swift(), do: Module.concat(Keyword.fetch!(unquote(opts), :client), SwiftHelpers) + + + @doc "Starts the openstex supervision tree." + def start_link(sup_opts \\ []) do + client = unquote(opts) |> Keyword.fetch!(:client) + otp_app = unquote(opts) |> Keyword.fetch!(:otp_app) + Openstex.Supervisor.start_link(client, [otp_app: otp_app]) + end + + @doc "Prepares a request prior to sending by adding metadata such as authorization headers." + @spec prepare_request(Query.t, Keyword.t) :: {:ok, Response.t} | {:error, Response.t} + def prepare_request(query, httpoison_opts \\ []) do + client = unquote(opts) |> Keyword.fetch!(:client) + Transformation.prepare_request(query, httpoison_opts, client) + end + + @doc "Sends a request to the openstack api using [httpoison](https://hex.pm/packages/httpoison)." + @spec request(Query.t | HttpQuery.t, Keyword.t) :: {:ok, Response.t} | {:error, Response.t} + def request(query, httpoison_opts \\ []) do + client = unquote(opts) |> Keyword.fetch!(:client) + Request.request(query, httpoison_opts, client) + end + + @doc "Sends a request to the openstack api using [httpoison](https://hex.pm/packages/httpoison)." + @spec request!(Query.t | HttpQuery.t, Keyword.t) :: Response.t | no_return + def request!(query, httpoison_opts \\ []) do + case request(query) do + {:ok, resp} -> resp + {:error, resp} -> raise(ResponseError, response: resp, query: query) + end + end + + + end + end + + + @callback start_link() :: {:ok, pid} | {:error, atom} + @callback start_link(sup_opts :: list) :: {:ok, pid} | {:error, atom} + @callback prepare_request(query :: Query.t) :: Query.t | no_return + @callback prepare_request(query :: Query.t, httpoison_opts :: Keyword.t) :: Query.t | no_return + @callback request(query :: Query.t | HttpQuery.t) :: {:ok, Response.t} | {:error, Response.t} + @callback request(query :: Query.t | HttpQuery.t, httpoison_opts :: Keyword.t) :: {:ok, Response.t} | {:error, Response.t} + @callback request!(query :: Query.t | HttpQuery.t) :: {:ok, Response.t} | no_return + @callback request!(query :: Query.t | HttpQuery.t, httpoison_opts :: Keyword.t) :: {:ok, Response.t} | no_return + + +end \ No newline at end of file diff --git a/lib/openstex.ex b/lib/openstex.ex new file mode 100644 index 0000000..cfa71e6 --- /dev/null +++ b/lib/openstex.ex @@ -0,0 +1,87 @@ +defmodule Openstex do + @moduledoc ~S""" + Setting up clients for use with openstex. + + ## OVH Cloudstorage Client + + defmodule Openstex.Cloudstorage do + @moduledoc :false + use Openstex.Client, otp_app: :openstex, client: __MODULE__ + + defmodule SwiftHelpers do + @moduledoc :false + use Openstex.Services.Swift.V1.Helpers, otp_app: :openstex, client: Openstex.Cloudstorage + end + + defmodule Ovh do + @moduledoc :false + use ExOvh.Client, otp_app: :openstex, client: __MODULE__ + end + end + + + ## Ovh Webstorage CDN Client + + defmodule Openstex.Webstorage do + @moduledoc :false + use Openstex.Client, otp_app: :openstex, client: __MODULE__ + + defmodule SwiftHelpers do + @moduledoc :false + use Openstex.Services.Swift.V1.Helpers, otp_app: :openstex, client: Openstex.Webstorage + end + + defmodule Ovh do + @moduledoc :false + use ExOvh.Client, otp_app: :openstex, client: __MODULE__ + end + end + + + ## Rackspace Cloudfiles Client + + defmodule Openstex.Cloudfiles do + @moduledoc :false + use Openstex.Client, otp_app: :openstex, client: __MODULE__ + + defmodule SwiftHelpers do + @moduledoc :false + use Openstex.Services.Swift.V1.Helpers, otp_app: :openstex, client: Openstex.Cloudfiles + end + end + + + ## Rackspace CloudfilesCDN Client + + defmodule Openstex.Cloudfiles.CDN do + @moduledoc :false + use Openstex.Client, otp_app: :openstex, client: __MODULE__ + + defmodule SwiftHelpers do + @moduledoc :false + use Openstex.Services.Swift.V1.Helpers, otp_app: :openstex, client: Openstex.Cloudfiles.CDN + end + end + + + ## Hubic Client + + defmodule Openstex.ExHubic do + @moduledoc :false + use Openstex.Client, otp_app: :openstex, client: __MODULE__ + + defmodule SwiftHelpers do + @moduledoc :false + use Openstex.Services.Swift.V1.Helpers, otp_app: :openstex, client: Openstex.ExHubic + end + + defmodule Hubic do + @moduledoc :false + use ExHubic.Client, otp_app: :openstex, client: __MODULE__ + end + end + """ +end + + + diff --git a/lib/query.ex b/lib/query.ex new file mode 100644 index 0000000..ff6e134 --- /dev/null +++ b/lib/query.ex @@ -0,0 +1,37 @@ +defmodule Openstex.Query do + @moduledoc false + defstruct [:method, :uri, :params, :service, headers: []] + @type t :: %__MODULE__{ + method: atom, + uri: String.t, + headers: [{binary, binary}], + params: any, + service: atom + } +end + + +defmodule Openstex.Swift.Query do + @moduledoc false + defstruct [:method, :uri, :params, headers: [], service: :swift] + @type t :: %__MODULE__{ + method: atom, + uri: String.t, + headers: [{binary, binary}], + params: any, + service: :swift + } +end + +defmodule Openstex.HttpQuery do + @moduledoc false + defstruct [method: :nil, uri: :nil, body: "", headers: [], options: [], service: :nil] + @type t :: %__MODULE__{ + method: atom, + uri: String.t, + body: :binary | {:file, :binary}, + headers: [{binary, binary}], + options: Keyword.t, + service: atom + } +end \ No newline at end of file diff --git a/lib/request.ex b/lib/request.ex new file mode 100644 index 0000000..8f80c7b --- /dev/null +++ b/lib/request.ex @@ -0,0 +1,8 @@ +defprotocol Openstex.Request do + @moduledoc false + @fallback_to_any true + + @spec request(Openstex.Query.t, Keyword.t, atom) :: Openstex.HttpQuery.t + def request(query, httpoison_opts, client) + +end diff --git a/lib/request/http_query/request.ex b/lib/request/http_query/request.ex new file mode 100644 index 0000000..7e8392c --- /dev/null +++ b/lib/request/http_query/request.ex @@ -0,0 +1,36 @@ +defimpl Openstex.Request, for: Openstex.HttpQuery do + @moduledoc :false + alias Openstex.{HttpQuery, Response} + + @spec request(HttpQuery.t, Keyword.t, atom) :: {:ok, Response.t} | {:error, Response.t} + def request(%HttpQuery{} = q, httpoison_opts, _client) do + options = Keyword.merge(q.options, httpoison_opts) + case HTTPoison.request(q.method, q.uri, q.body, q.headers, options) do + {:ok, resp} -> + body = parse_body(resp) + resp = %Response{ body: body, headers: resp.headers |> Enum.into(%{}), status_code: resp.status_code } + if resp.status_code >= 100 and resp.status_code < 400 do + {:ok, resp} + else + {:error, resp} + end + {:error, resp} -> + {:error, %HTTPoison.Error{reason: resp.reason}} + end + end + + + # private + + + def parse_body(resp) do + try do + resp.body |> Poison.decode!() + rescue + _ -> + resp.body + end + end + + +end diff --git a/lib/request/openstack/request.ex b/lib/request/openstack/request.ex new file mode 100644 index 0000000..ece56ba --- /dev/null +++ b/lib/request/openstack/request.ex @@ -0,0 +1,39 @@ +defimpl Openstex.Request, for: Any do + @moduledoc :false + alias Openstex.{Response, Transformation} + + + @spec request(any, Keyword.t, atom) :: {:ok, Response.t} | {:error, Response.t} + def request(query, httpoison_opts, client) do + q = Transformation.prepare_request(query, httpoison_opts, client) |> Map.from_struct() + + options = Keyword.merge(q.options, httpoison_opts) + case HTTPoison.request(q.method, q.uri, q.body, q.headers, options) do + {:ok, resp} -> + body = parse_body(resp) + resp = %Response{ body: body, headers: resp.headers |> Enum.into(%{}), status_code: resp.status_code } + if resp.status_code >= 100 and resp.status_code < 400 do + {:ok, resp} + else + {:error, resp} + end + {:error, resp} -> + {:error, %HTTPoison.Error{reason: resp.reason}} + end + end + + + # private + + + def parse_body(resp) do + try do + resp.body |> Poison.decode!() + rescue + _ -> + resp.body + end + end + + +end diff --git a/lib/response.ex b/lib/response.ex new file mode 100644 index 0000000..798af5e --- /dev/null +++ b/lib/response.ex @@ -0,0 +1,37 @@ +defmodule Openstex.Response do + @moduledoc false + defstruct [:body, :headers, :status_code] + @type t :: %__MODULE__{ + body: map | binary | String.t, + headers: list, + status_code: integer + } +end + +defmodule Openstex.ResponseError do + @moduledoc :false + defexception [:response, :query] + + def message(%{query: query, response: resp}) do + ~s""" + The following http query was unsuccessful, unexpected or erroneous in some way: + + #{Kernel.inspect(query)} was unsuccessful. + """ + <> + message(%{response: resp}) + end + + def message(%{response: resp}) do + ~s""" + ** Reponse Status Code ** + #{inspect resp.status_code} + + ** Response Body ** + #{Kernel.inspect(resp.body)} + + ** Response Headers ** + #{Kernel.inspect(resp.headers)} + """ + end +end \ No newline at end of file diff --git a/lib/services/keystone/v2/helpers.ex b/lib/services/keystone/v2/helpers.ex new file mode 100644 index 0000000..386ab7b --- /dev/null +++ b/lib/services/keystone/v2/helpers.ex @@ -0,0 +1,185 @@ +defmodule Openstex.Services.Keystone.V2.Helpers do + @moduledoc ~s""" + A module that provides helper functions for executing more complex multi-step queries + for Keystone authentication. + + See the `ExOvh` library for an example usage of the helpers module. + """ + alias Openstex.Request + alias Openstex.Services.Keystone.V2.Helpers.Identity + + + @doc ~s""" + Helper function to authenticate openstack using keystone (identity) api. Returns a + `Openstex.Helpers.V2.Keystone.Identity` struct. + + ## Arguments + + - ```endpoint```: the endpoint to which the http request should be sent for accessing keystone authentication. + - ```username```: openstack username + - ```password```: openstack password + - ```tenant```: A Keyword list as follows: [tenant_id: tenant_id, tenant_name: tenant_name]. + One or the other should be present or {:error, message} is returned. + + """ + @spec authenticate(String.t, String.t, String.t, Keyword.t) :: {:ok, Identity.t} | {:error, Openstex.Response.t} | {:error, any} + def authenticate(endpoint, username, password, tenant) do + Og.context(__ENV__, :debug) + + token_query = Openstex.Services.Keystone.V2.Query.get_token(endpoint, username, password) + identity_query = fn(token, endpoint, tenant) -> Openstex.Services.Keystone.V2.Query.get_identity(token, endpoint, tenant) end + + with {:ok, resp} <- Request.request(token_query, [], :nil), + token = resp.body |> Map.get("access") |> Map.get("token") |> Map.get("id"), + {:ok, resp} <- Request.request(identity_query.(token, endpoint, tenant), [], :nil) do + {:ok, parse_nested_map_into_identity_struct(resp.body)} + end + |> case do + {:ok, identity} -> {:ok, identity} + {:error, resp} -> {:error, resp} + end + end + + @doc ~s""" + Helper function to authenticate openstack using keystone (identity) api. Returns a + `Openstex.Helpers.V2.Keystone.Identity` struct. + + ## Arguments + + - ```endpoint```: the endpoint to which the http request should be sent for accessing keystone authentication. + - ```token```: the x-auth token + - ```tenant```: A Keyword list as follows: [tenant_id: tenant_id, tenant_name: tenant_name]. + One or the other should be present or {:error, message} is returned. + + """ + @spec authenticate(String.t, String.t, Keyword.t) :: {:ok, Identity.t} | {:error, Openstex.Response.t} | {:error, any} + def authenticate(endpoint, token, tenant) do + Og.context(__ENV__, :debug) + identity_query = fn(token, endpoint, tenant) -> Openstex.Services.Keystone.V2.Query.get_identity(token, endpoint, tenant) end + case Request.request(identity_query.(token, endpoint, tenant), [], :nil) do + {:ok, resp} -> {:ok, parse_nested_map_into_identity_struct(resp.body)} + {:error, resp} -> {:error, resp} + end + |> case do + {:ok, identity} -> {:ok, identity} + {:error, resp} -> {:error, resp} + end + end + + @doc ~s""" + Defaults to authenticate(endpoint, token, []). See `authenticate/3`. + """ + @spec authenticate(String.t, String.t, Keyword.t) :: {:ok, Identity.t} | {:error, Openstex.Response.t} | {:error, any} + def authenticate(endpoint, token) do + authenticate(endpoint, token, []) + end + @doc ~s""" + Helper function to authenticate openstack using keystone (identity) api. Returns a + `Openstex.Helpers.V2.Keystone.Identity` struct or raises and error. See `authenticate/3`. + """ + @spec authenticate!(String.t, String.t) :: Identity.t | no_return + def authenticate!(endpoint, token) do + case authenticate(endpoint, token) do + {:ok, identity} -> identity + {:error, resp} -> raise(Openstex.ResponseError, response: resp) + end + end + + + + @doc ~s""" + Helper function to authenticate openstack using keystone (identity) api. Returns a + `Openstex.Helpers.V2.Keystone.Identity` struct or raises and error. See `authenticate/4`. + """ + @spec authenticate!(String.t, String.t, String.t, Keyword.t) :: Identity.t | no_return + def authenticate!(endpoint, username, password, tenant) do + case authenticate(endpoint, username, password, tenant) do + {:ok, identity} -> identity + {:error, resp} -> raise(Openstex.ResponseError, response: resp) + end + end + + + @doc :false + def parse_nested_map_into_identity_struct(identity_map) do + Og.context(__ENV__, :debug) + + identity = Map.fetch!(identity_map, "access") + tenant = Map.fetch!(identity, "token") + |> Map.fetch!("tenant") + |> Identity.Token.Tenant.build() + token = Map.fetch!(identity, "token") |> Map.delete("tenant") |> Map.put("tenant", tenant) |> Identity.Token.build() + user = Map.get(identity, "user", %{}) |> Identity.User.build() + metadata = Map.get(identity, "metadata", %{}) |> Identity.Metadata.build() + trust = Map.get(identity, "trust", %{}) |> Identity.Trust.build() + + service_catalog = + Map.fetch!(identity, "serviceCatalog") + |> Enum.map( + fn(service) -> + endpoints = Map.get(service, "endpoints", []) |> Enum.map(&Identity.Endpoint.build/1) + service = Map.delete(service, "endpoints") |> Map.put("endpoints", endpoints) + Identity.Service.build(service) + end + ) + + %{ + "token" => token, + "service_catalog" => service_catalog, + "user" => user, + "metadata" => metadata, + "trust" => trust + } + |> Identity.build() + end + + + defmodule Identity.Token.Tenant do + @moduledoc :false + defstruct [:description, :enabled, :id, :name] + def build(map), do: Mapail.map_to_struct!(map, __MODULE__, [rest: :merge]) + end + defmodule Identity.Token do + @moduledoc :false + defstruct [:audit_ids, :issued_at, :expires, :id, tenant: %Identity.Token.Tenant{}] + def build(map), do: Mapail.map_to_struct!(map, __MODULE__, [rest: :merge]) + end + defmodule Identity.Service do + @moduledoc :false + defstruct [endpoints: [], endpoints_links: [], type: "", name: ""] + def build(map), do: Mapail.map_to_struct!(map, __MODULE__, [rest: :merge]) + end + defmodule Identity.Endpoint do + @moduledoc :false + defstruct [:admin_url, :region, :internal_url, :id, :public_url] + def build(map), do: Mapail.map_to_struct!(map, __MODULE__, [rest: :merge]) + end + defmodule Identity.User do + @moduledoc :false + defstruct [:username, :roles_links, :id, :roles, :name] + def build(map), do: Mapail.map_to_struct!(map, __MODULE__, [rest: :merge]) + end + defmodule Identity.Metadata do + @moduledoc :false + defstruct [:metadata, :is_admin, :roles] + def build(map), do: Mapail.map_to_struct!(map, __MODULE__, [rest: :merge]) + end + defmodule Identity.Trust do + @moduledoc :false + defstruct [:trust, :id, :trustee_user_id, :trustor_user_id, :impersonation] + def build(map), do: Mapail.map_to_struct!(map, __MODULE__, [rest: :merge]) + end + defmodule Identity do + @moduledoc :false + defstruct [ + token: %Identity.Token{}, + service_catalog: [], + user: %Identity.User{}, + metadata: %Identity.Metadata{}, + trust: %Identity.Trust{} + ] + def build(map), do: Mapail.map_to_struct!(map, __MODULE__, [rest: :merge]) + end + + +end \ No newline at end of file diff --git a/lib/services/keystone/v2/query.ex b/lib/services/keystone/v2/query.ex new file mode 100644 index 0000000..698aaa3 --- /dev/null +++ b/lib/services/keystone/v2/query.ex @@ -0,0 +1,114 @@ +defmodule Openstex.Services.Keystone.V2.Query do + @moduledoc ~S""" + Helper functions to assist in building queries for openstack compatible keystone apis (version 2.0). + + ## Example + + Openstex.Services.Keystone.V2.Query.get_token(endpoint, username, password) |> ExHubic.request!() + """ + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + @default_options [timeout: 10000, recv_timeout: 30000] + alias Openstex.HttpQuery + + + @doc ~s""" + Generate and return a token. + + ## Api + + POST /v2.0/​{tokens} + + ## Example + + Openstex.Services.Keystone.V2.Query.get_token(endpoint, username, password) |> ExHubic.request!() + """ + @spec get_token(String.t, String.t, String.t) :: HttpQuery.t + def get_token(endpoint, username, password) do + body = + %{ + "auth" => + %{ + "passwordCredentials" => %{ + "username" => username, + "password" => password + } + } + } + |> Poison.encode!() + %HttpQuery{ + method: :post, + uri: endpoint <> "/tokens", + body: body, + headers: @default_headers, + options: @default_options, + service: :openstack + } + end + + + @doc ~s""" + Get various details about the identity access including token information, services information (service catalogue), + user information, trust information and metadata. + + ## Api + + POST /v2.0/​{tokens} + + ## Example + + Openstex.Services.Keystone.V2.Query.get_identity_info(token, endpoint, tenant) |> ExHubic.request!() + """ + @spec get_identity(String.t, String.t, Keyword.t) :: HttpQuery.t | no_return + def get_identity(token, endpoint, tenant) when tenant == [] do + body = + %{ + "auth" => + %{ + "token" => %{"id" => token} + } + } |> Poison.encode!() + + %HttpQuery{ + method: :post, + uri: endpoint <> "/tokens", + body: body, + headers: @default_headers, + options: @default_options, + service: :openstack + } + end + def get_identity(token, endpoint, tenant) do + tenant_id = Keyword.get(tenant, :tenant_id, :nil) + tenant_name = Keyword.get(tenant, :tenant_name, :nil) + body = + case tenant_name do + :nil -> + %{ + "auth" => + %{ + "tenantId" => tenant_id, + "token" => %{"id" => token} + } + } |> Poison.encode!() + _ -> + %{ + "auth" => + %{ + "tenantName" => tenant_name, + "token" => %{"id" => token} + } + } |> Poison.encode!() + end + + %HttpQuery{ + method: :post, + uri: endpoint <> "/tokens", + body: body, + headers: @default_headers, + options: @default_options, + service: :openstack + } + end + + +end \ No newline at end of file diff --git a/lib/services/swift/v1/helpers.ex b/lib/services/swift/v1/helpers.ex new file mode 100644 index 0000000..dccc0a3 --- /dev/null +++ b/lib/services/swift/v1/helpers.ex @@ -0,0 +1,882 @@ +defmodule Openstex.Services.Swift.V1.Helpers do + @moduledoc ~s""" + Helper functions for executing more complex multi-step queries for Swift Object Storage. + """ + alias Openstex.Services.Swift.V1.Query + + + defmacro __using__(opts) do + quote bind_quoted: [opts: opts] do + alias Openstex.Services.Keystone.V2.Helpers.Identity + alias Openstex.Services.Swift.V1.Query + @behaviour Openstex.Services.Swift.V1.Helpers + + @doc :false + def client(), do: Keyword.fetch!(unquote(opts), :client) + + @doc :false + def default_httpoison_opts(), do: client().config().httpoison_config(client()) + + def get_public_url() do + client().keystone().identity(client()) + |> Map.get(:service_catalog) + |> Enum.find(fn(%Identity.Service{} = service) -> service.name == client.config().swift_service_name() && service.type == client.config().swift_service_type() end) + |> Map.get(:endpoints) + |> Enum.find(fn(%Identity.Endpoint{} = endpoint) -> endpoint.region == client().config().swift_region(client) end) + |> Map.get(:public_url) + end + + def get_account() do + public_url = get_public_url() + path = URI.parse(public_url) |> Map.get(:path) + {version, account} = String.split_at(path, 4) + account + end + + def get_endpoint() do + public_url = get_public_url() + path = URI.parse(public_url) |> Map.get(:path) + {version, account} = String.split_at(path, 4) + endpoint = String.split(public_url, account) |> List.first() + endpoint + end + + def get_account_tempurl_key(key_number \\ :key1) do + + config_key = + case key_number do + :key1 -> :account_temp_url_key1 + :key2 -> :account_temp_url_key2 + end + + header = + case key_number do + :key1 -> "X-Account-Meta-Temp-Url-Key" + :key2 -> "X-Account-Meta-Temp-Url-Key-2" + end + + # first attempt to get the account key from the swift server + key = Query.account_info(get_account()) + |> client().request!() + |> Map.fetch!(:headers) + |> Map.get(header, :nil) + + # then attempt to get the get_account() key from the config file + if key == :nil do + key = client().config().get_account_temp_url_key1(client()) + # since account tempurl key is not on server, it should be set on the server. + if key != :nil do + set_account_temp_url_key(key_number, key) + end + key + else + key + end + end + + def set_account_temp_url_key(key_number \\ :key1, key \\ :nil) do + {key, header} = + case key_number do + :key1 -> + if key == :nil do + {client().config().get_account_temp_url_key1(client()), "X-Account-Meta-Temp-Url-Key"} + else + {key, "X-Account-Meta-Temp-Url-Key"} + end + :key2 -> + if key == :nil do + {client().config().account_temp_url_key2(client()), "X-Account-Meta-Temp-Url-Key-2"} + else + {key, "X-Account-Meta-Temp-Url-Key-2"} + end + end + put_temp_url_key(key, header, key_number) + end + + def delete_object(server_object, container, httpoison_opts \\ []) do + httpoison_opts = Map.merge(Enum.into(default_httpoison_opts(), %{}), Enum.into(httpoison_opts, %{})) |> Enum.into([]) + query = Query.delete_object(server_object, container, get_account()) + client().request(query, httpoison_opts) + end + + def delete_object!(server_object, container, httpoison_opts \\ []) do + case delete_object(server_object, container, httpoison_opts) do + {:ok, resp} -> + if resp.status_code == 204 and resp.body == "" do + :ok + else + query = Query.delete_object(server_object, container, get_account()) + raise(Openstex.ResponseError, query: query, response: resp) + end + {:error, resp} -> + query = Query.delete_object(server_object, container, get_account()) + raise(Openstex.ResponseError, query: query, response: resp) + end + end + + @doc ~s""" + Helper function to simplify the process of uploading a file. + + ## Arguments + + - `file`: The path of the file on the client machine. + - `server_object`: The path of the file on the openstack swift server + - `container`: The name of the container. + - `httpoison_opts`: The [httpoison options](https://hexdocs.pm/httpoison/HTTPoison.html#request/5). + - `upload_opts`: The [create_object options](https://hexdocs.pm/openstex/swift/v1/query.ex#create_object/4). + + ## Example + + file = "/priv/test_file.json" + server_object = "ex_hubic_tests/nested/test_file.json" + ExHubic.Swift.upload_file(file, server_object, "default", [recv_timeout: (60000 * 60)]) + """ + @spec upload_file(String.t, String.t, String.t, list, list) :: {:ok, Response.t} | {:error, Response.t} + def upload_file(file, server_object, container, httpoison_opts \\ [], upload_opts \\ []) do + httpoison_opts = Keyword.merge(default_httpoison_opts(), httpoison_opts) + upload_opts = upload_opts ++ [server_object: server_object] + query = Query.create_object(container, get_account(), file, upload_opts) + client().request(query, httpoison_opts) + end + + + @doc ~s""" + Helper function to simplify the process of uploading a file. See `upload_file/5`. Returns :ok + if upload suceeeded or raises and error otherwise. + """ + @spec upload_file!(String.t, String.t, String.t, list, list) :: :ok | no_return + def upload_file!(file, server_object, container, httpoison_opts \\ [], upload_opts \\ []) do + case upload_file(file, server_object, container, httpoison_opts, upload_opts) do + {:ok, resp} -> + if resp.status_code in [200, 201] and resp.body == "" do + :ok + else + upload_opts = upload_opts ++ [server_object: server_object] + query = Query.create_object(container, get_account(), file, upload_opts) + raise(Openstex.ResponseError, query: query, response: resp) + end + {:error, resp} -> + upload_opts = upload_opts ++ [server_object: server_object] + query = Query.create_object(container, get_account(), file, upload_opts) + raise(Openstex.ResponseError, query: query, response: resp) + end + end + + + @doc ~s""" + Helper function to simplify the process of downloading a file. The body of the response + contains the binary object (file). + + ## Arguments + + - `server_object`: The path of the file on the openstack swift server + - `container`: The name of the container. + - `httpoison_opts`: The [httpoison options](https://hexdocs.pm/httpoison/HTTPoison.html#request/5). + + ## Example + + server_object = "/ex_hubic_tests/nested/test_file.json" + ExHubic.Swift.download_file(server_object, "default", [recv_timeout: (60000 * 1)]) # allow 1 min for download + """ + @spec download_file(String.t, String.t, list) :: {:ok, Response.t} | {:error, Response.t} + def download_file(server_object, container, httpoison_opts \\ []) do + httpoison_opts = Keyword.merge(default_httpoison_opts, httpoison_opts) + query = Query.get_object(server_object, container, get_account()) + client().request(query, httpoison_opts) + end + + + @doc ~s""" + Helper function to simplify the process of downloading a file. Returns the binary object (file) + or raises an error. See `download_file/3`. + """ + @spec download_file!(String.t, String.t, list) :: :binary | no_return + def download_file!(server_object, container, httpoison_opts \\ []) do + case download_file(server_object, container, httpoison_opts) do + {:ok, resp} -> resp.body + {:error, resp} -> + query = Query.get_object(server_object, container, get_account()) + raise(Openstex.ResponseError, query: query, response: resp) + end + end + + + @doc ~s""" + Lists objects within a pseudofolder in a container. see `list_objects/3` + + ## Arguments + + - `container`: The name of the container. + """ + @spec list_objects(String.t) :: {:ok, list} | {:error, map} + def list_objects(container) do + list_objects("", container, [nested: :true]) + end + + + @doc ~s""" + Lists all the objects in a container or raises an error. See `list_objects/3`. + """ + @spec list_objects!(String.t) :: list | no_return + def list_objects!(container) do + list_objects!("", container, [nested: :true]) + end + + + @doc ~s""" + Lists objects within a pseudofolder in a container. Optionally include objects in nested + pseudofolders. + + ## Arguments + + - `pseudofolder`: The pseudofolder in which to list the objects. + - `container`: The container in which to find the objects. + - `nested`, defaults to :false + + [nested: true] => returns all objects in nested pseudofolders also. + + [nested: false] => returns all objects at the top level and ignores objects witihin + nested pseudofolders. + + ## Notes + + - Excludes pseudofolders from the results. + - If no pseudofolder name is entered, `pseudofolder` defaults to `""`, thereby getting all objects in + the container + - Returns the object name. + - Returns files + - Does not list pseudofolders + + ## Example + + Returns all objects in the `"test_folder/"` but not objects in nested pseudofolders + ExHubic.Swift.list_objects("test_folder/", "default", [nested: :false]) + + Returns all objects in the `"test_folder/"` and all nested pseudofolders. + ExHubic.Swift.list_objects("test_folder/", "default", [nested: :true]) + + Returns all objects in the `"default"` container + ExHubic.Swift.list_objects("default", [nested: :true]) + """ + @spec list_objects(String.t, String.t, list) :: {:ok, list} | {:error, map} + def list_objects(pseudofolder, container, opts \\ []) + + def list_objects(pseudofolder, container, [nested: :true]) do + with {:ok, pseudofolders} <- list_pseudofolders(pseudofolder, container, [nested: :true]), do: ( + pseudofolders = [ pseudofolder | pseudofolders ] + objects = Enum.reduce(pseudofolders, [], fn(pseudofolder, acc) -> + with {:ok, objects} <- get_objects_only_in_pseudofolder(pseudofolder, container, get_account()), do: ( + Enum.concat(objects, acc) + ) + end) + {:ok, objects} + ) + end + def list_objects(pseudofolder, container, [nested: :false]) do + case get_objects_only_in_pseudofolder(pseudofolder, container, get_account()) do + {:ok, objects} -> {:ok, objects} + {:error, resp} -> {:error, resp} + end + end + def list_objects(pseudofolder, container, []) do + list_objects(pseudofolder, container, [nested: :false]) + end + + + @doc ~s""" + Lists objects within a pseudofolder in a container or raises an error. Optionally include objects in nested + pseudofolders. see `list_objects/3` + """ + @spec list_objects!(String.t, String.t, list) :: {:ok, list} | no_return + def list_objects!(folder, container, opts \\ []) do + case list_objects(folder, container, opts) do + {:ok, objects} -> objects + {:error, resp} -> raise(Openstex.ResponseError, response: resp) + end + end + + + @doc ~s""" + Lists all pseudofolders including nested pseudofolders in a container. See `list_pseudofolders/3` + + ## Arguments + + - `container`: The name of the container in which to list the pseudofolders. + """ + @spec list_pseudofolders(String.t) :: {:ok, list} | {:error, Openstex.Response.t} + def list_pseudofolders(container) do + list_pseudofolders("", container, [nested: :true]) + end + + + @doc ~s""" + Lists pseudofolders at the top level in a given pseudofolder. See `list_pseudofolders/3` + + ## Arguments + + - `pseudofolder`: The pseudofolder in which to search for other top-level pseudofolders + - `container`: The name of the container in which to list the pseudofolders. + """ + @spec list_pseudofolders(String.t, String.t) :: {:ok, list} | {:error, Openstex.Response.t} + def list_pseudofolders(pseudofolder, container) do + list_pseudofolders(pseudofolder, container, []) + end + + + @doc ~s""" + Lists all pseudofolders within a pseudofolder in a container. + + ## Arguments + + - `pseudofolder`: The pseudofolder in which to search for other pseudofolders + - `container`: The name of the container in which to list the pseudofolders. + - `nested`, defaults to :false + + [nested: true] => returns all objects in nested pseudofolders also. + + [nested: false] => returns all objects at the top level and ignores objects witihin + nested pseudofolders. + + + ## Notes + + - If no pseudofolder is entered, then pseudofolder defaults to `"" ` which in turn results in all + pseudofolders being fetched one level deep at the root level. + - Returns the pseudofolders by name. + - Traverses as deep as the most nested folder beyond the passed folder to get all folders. + - Excludes non-pseudofolder objects from the results. + + ## Example + + Gets all pseudofolders in the `"test_folder/"` pseudofolder one level deep (top level) + ExHubic.Swift.list_pseudofolders("test_folder/", "default") + + Gets all pseudofolders in the `"test/"` pseudofolder one level deep (top level) + ExHubic.Swift.list_pseudofolders("test", "default") + + Gets all the pseudofolders in the container and traverse to the deepest nested pseudofolders + ExHubic.Swift.list_pseudofolders(default", [nested: :true]) + """ + @spec list_pseudofolders(String.t, String.t, list) :: {:ok, list} | {:error, map} + def list_pseudofolders(pseudofolder, container, opts) do + nested? = Keyword.get(opts, :nested, :false) + pseudofolder = Openstex.Utils.ensure_has_leading_slash(pseudofolder) + |> Openstex.Utils.remove_if_has_trailing_slash() + query = Query.get_objects_in_folder(pseudofolder, container, get_account()) + case nested? do + :true -> + process_query_to_get_pseudofolders(query, container, get_account()) + :false -> + case client().request(query, default_httpoison_opts()) do + {:ok, resp} -> + folders = filter_non_pseudofolders(resp.body) + {:ok, folders} + {:error, resp} -> + {:error, resp} + end + end + end + + + @doc ~s""" + Lists all pseudofolders including nested pseudofolders in a container. See `list_pseudofolders!/3` + + ## Arguments + + - `container`: The name of the container in which to list the pseudofolders. + """ + @spec list_pseudofolders!(String.t) :: list | no_return + def list_pseudofolders!(container) do + list_pseudofolders!("", container, [nested: :true]) + end + + + @doc ~s""" + Lists pseudofolders at the top level in a given pseudofolder. See `list_pseudofolders!/3` + + ## Arguments + + - `pseudofolder`: The pseudofolder in which to search for other top-level pseudofolders + - `container`: The name of the container in which to list the pseudofolders. + """ + @spec list_pseudofolders!(String.t, String.t) :: list | no_return + def list_pseudofolders!(pseudofolder, container) do + list_pseudofolders!(pseudofolder, container, []) + end + + + @doc ~s""" + Lists all pseudofolders within a pseudofolder in a container or raises an error. See `list_pseudofolders/3`. + """ + @spec list_pseudofolders!(String.t, String.t, list) :: list| no_return + def list_pseudofolders!(pseudofolder, container, opts) do + case list_pseudofolders(pseudofolder, container, opts) do + {:ok, objects} -> objects + {:error, resp} -> raise(Openstex.ResponseError, response: resp) + end + end + + + @doc ~s""" + Checks if a pseudofolder exists. + + ## Arguments + + - `pseudofolder`: The pseudofolder whose existence is to be checked. + - `container`: The name of the container in which to list the pseudofolders. + """ + @spec pseudofolder_exists?(String.t, String.t) :: boolean | no_return + def pseudofolder_exists?(pseudofolder, container) do + case list_objects(pseudofolder, container) do + {:ok, []} -> + :false + {:error, resp} -> + raise(Openstex.ResponseError, response: resp) + {:ok, list} -> + :true + end + end + + + @doc ~s""" + Deletes all objects in a pseudofolder effectively deleting the pseudofolder itself. + + ## Arguments + + - `pseudofolder`: The pseudofolder to be deleted. + - `container`: The container in which to delete the pseudofolder. + + ## Notes + + - Returns `:ok` anyways even if pseudofolder did not exist in the first place. + - Returns `{:error, list}` if some objects were not deleted where `list` represents + the objects for which deletion failed. + """ + @spec delete_pseudofolder(String.t, String.t) :: :ok | {:error, list} + def delete_pseudofolder(pseudofolder, container) do + if pseudofolder_exists?(pseudofolder, container) do + responses = list_objects!(pseudofolder, container, [nested: :true]) + |> Enum.map(fn(obj) -> + query = Query.delete_object(obj, container, get_account()) + case client().request(query, default_httpoison_opts()) do + {:ok, _resp} -> :ok + {:error, _resp} -> {:error, obj} + end + end) + failed_deletes = Enum.filter_map(responses, + fn(resp) -> + case resp do + :ok -> :false + :false -> :true + end + end, + fn(resp) -> + Tuple.to_list(resp) |> List.last() + end + ) + if failed_deletes == [] do + :ok + else + {:error, failed_deletes} + end + else + :ok + end + end + + + @doc """ + Generates a tempurl for an object + + ## Example + + OpenstexTest.OvhClient.Swift.Cloudstorage.generate_temp_url("test_container", "test_file.txt", []) + + ## Arguments + + - `container`: container in which the object is found. + - `server_object`: filename under which the file will be stored on the openstack object storage server. defaults to the `client_object_pathname` if none given. + - `opts`: + - `temp_url_expires_after`: Sets the length of time for which the signature to the public link will remain valid. Adds the `temp_url_expires` query string to the url. + The unix epoch time format is used. Defaults to 5 minutes from current time if `temp_url` is :true. Otherwise, it can be set by adding the time in seconds from now for which the link + should remain valid. Eg 10 days => (10 * 24 * 60 * 60) + - `temp_url_filename`: Defaults to `:false`. Swift automatically generates filenames for temp_urls but this option will allow custom names to be added. Works by adding the `filename` query string to the url. + - `temp_url_inline`: Defaults to `:false`. If set to `:true`, the file is not automatically downloaded by the browser and instead the `&inline' query string is added to the url. + - `temp_url_method: Defaults to `"GET"` but can be set to `"PUT"`. + + + ## Notes + + - The client should have a `:account_temp_url_key` option already setup in `config.exs`. + """ + @spec generate_temp_url(String.t, String.t, list) :: String.t + def generate_temp_url(container, server_object, opts \\ []) do + + temp_url_key = get_account_tempurl_key(:key1) + temp_url_expires_after = Keyword.get(opts, :temp_url_expires_after, (5 * 60)) + temp_url_filename = Keyword.get(opts, :temp_url_filename, :false) + temp_url_inline = Keyword.get(opts, :temp_url_inline, :false) + temp_url_method = Keyword.get(opts, :temp_url_method, "GET") + path = "/v1/#{get_account()}/#{container}/#{server_object}" + temp_url_expiry = :os.system_time(:seconds) + temp_url_expires_after + temp_url_sig = Openstex.Utils.gen_tempurl_signature(temp_url_method, temp_url_expiry, path, temp_url_key) + + query_string = Map.put(%{}, :temp_url_sig, temp_url_sig) + |> Map.put(:temp_url_expires, temp_url_expiry) + query_string = if temp_url_filename, do: Map.put(query_string, :temp_url_filename, temp_url_filename), else: query_string + query_string = if temp_url_inline, do: Map.put(query_string, :temp_url_inline, "inline"), else: query_string + + client().swift().get_public_url() <> "/#{container}/#{server_object}" <> "?" <> URI.encode_query(query_string) + end + + + # Private + + + defp get_objects_only_in_pseudofolder(f, container, account) do + f = Openstex.Utils.ensure_has_leading_slash(f) + |> Openstex.Utils.remove_if_has_trailing_slash() + query = Query.get_objects_in_folder(f, container, account) + case client().request(query, default_httpoison_opts()) do + {:ok, resp} -> + objects = Map.get(resp, :body) + |> Enum.filter_map( + fn(e) -> Map.has_key?(e, "subdir") == :false end, + fn(e) -> e["name"] end + ) + {:ok, objects} + {:error, resp} -> + {:error, resp} + end + + end + + + defp get_pseudofolders(f, container, account) do + f = Openstex.Utils.ensure_has_leading_slash(f) + |> Openstex.Utils.remove_if_has_trailing_slash() + Query.get_objects_in_folder(f, container, account) + |> client().request!(default_httpoison_opts()) + |> Map.get(:body) + |> Enum.filter_map( + fn(e) -> e["subdir"] != :nil end, + fn(e) -> e["subdir"] end + ) + end + + + defp filter_non_pseudofolders(objects) do + Enum.filter_map(objects, + fn(e) -> e["subdir"] != :nil end, + fn(e) -> e["subdir"] end + ) + end + + + defp process_query_to_get_pseudofolders(query, container, account) do + case client().request(query, default_httpoison_opts()) do + {:ok, resp} -> + folders = filter_non_pseudofolders(resp.body) + pseudofolders = recurse_pseudofolders(folders, container, account) + |> Enum.filter_map( + fn(e) -> e != "" end, + fn(e) -> e end + ) # filters out the very top-level root folder "/" + {:ok, pseudofolders} + {:error, resp} -> + {:error, resp} + end + end + + + defp recurse_pseudofolders(folder, container, account) when is_binary(folder) do + recurse_pseudofolders([folder], [], container, account) + end + defp recurse_pseudofolders(folders, container, account) when is_list(folders) do + recurse_pseudofolders(folders, [], container, account) + end + defp recurse_pseudofolders([ f | folders ], acc, container, account) do + deeper_nested = get_pseudofolders(f, container, account) + case deeper_nested == [] do + :true -> + acc = [ f | acc ] + recurse_pseudofolders(folders, acc, container, account) + :false -> + acc = [ f | acc ] + recurse_pseudofolders(folders, acc, container, account) + |> Enum.concat(recurse_pseudofolders(deeper_nested, [], container, account)) + end + end + defp recurse_pseudofolders([], acc, container, account) do + acc + end + + + defp put_temp_url_key(key, header, key_number) do + %Openstex.Swift.Query{method: :post, uri: get_account(), params: %{}} + |> client().prepare_request() + |> Openstex.Utils.put_http_headers(%{header => key}) + |> client().request!() + + # update config genserver with new key + case key_number do + :key1 -> + client().config().set_account_temp_url_key1(client(), key) + :key2 -> + client().config().set_account_temp_url_key2(client(), key) + end + end + + + end + end + + + @doc "Gets the public url (storage_url) for the swift endpoint" + @callback get_public_url() :: String.t + + @doc "Gets the swift account string for the swift client" + @callback get_account() :: String.t + + @doc "Gets the swift endpoint for a given swift client. Returns the publicUrl of the endpoint with the account string removed." + @callback get_endpoint() :: String.t + + @doc "Gets the tempurl key - at an account level" + @callback get_account_tempurl_key(keynumber :: atom) :: String.t + + @doc "Sets the account tempurl key - at an account level" + @callback set_account_temp_url_key(key_number :: atom, key :: String.t) :: :ok + + @doc "Deletes an object from a given container" + @callback delete_object(server_object :: String.t, container :: String.t, httpoison_opts :: list) + :: {:ok, Response.t} | {:error, Response.t} + + @doc "Deletes an object from a given container" + @callback delete_object!(server_object :: String.t, container :: String.t, httpoison_opts :: list) + :: {:ok, Response.t} | no_return + + @doc ~s""" + Helper function to simplify the process of uploading a file. + + ## Arguments + + - `file`: The path of the file on the client machine. + - `server_object`: The path of the file on the openstack swift server + - `container`: The name of the container. + - `httpoison_opts`: The [httpoison options](https://hexdocs.pm/httpoison/HTTPoison.html#request/5). + + ## Example + + file = "/priv/test_file.json" + server_object = "/ex_hubic_tests/nested/test_file.json" + ExHubic.Swift.upload_file(file, server_object, "default", [recv_timeout: (60000 * 60)]) + """ + @callback upload_file(String.t, String.t, String.t, list) :: {:ok, Response.t} | {:error, Response.t} + + + @doc ~s""" + Helper function to simplify the process of uploading a file. See `upload_file/4`. Returns :ok + if upload suceeeded or raises and error otherwise. + """ + @callback upload_file!(String.t, String.t, String.t, list) :: :ok | no_return + + + @doc ~s""" + Helper function to simplify the process of downloading a file. The body of the response + contains the binary object (file). + + ## Arguments + + - `server_object`: The path of the file on the openstack swift server + - `container`: The name of the container. + - `httpoison_opts`: The [httpoison options](https://hexdocs.pm/httpoison/HTTPoison.html#request/5). + + ## Example + + server_object = "/ex_hubic_tests/nested/test_file.json" + ExHubic.Swift.download_file(server_object, "default", [recv_timeout: (60000 * 1)]) # allow 1 min for download + """ + @callback download_file(String.t, String.t, list) :: {:ok, Response.t} | {:error, Response.t} + + + + @doc ~s""" + Helper function to simplify the process of downloading a file. Returns the binary object (file) + or raises an error. See `download_file/3`. + """ + @callback download_file!(String.t, String.t, list) :: :binary | no_return + + + @doc ~s""" + Lists objects within a pseudofolder in a container. see `list_objects/3` + + ## Arguments + + - `container`: The name of the container. + """ + @callback list_objects(String.t) :: {:ok, list} | {:error, map} + + + @doc ~s""" + Lists all the objects in a container or raises an error. See `list_objects/3`. + """ + @callback list_objects!(String.t) :: list | no_return + + + @doc ~s""" + Lists objects within a pseudofolder in a container. Optionally include objects in nested + pseudofolders. + + ## Arguments + + - `pseudofolder`: The pseudofolder in which to list the objects. + - `container`: The container in which to find the objects. + - `nested`, defaults to :false + + [nested: true] => returns all objects in nested pseudofolders also. + + [nested: false] => returns all objects at the top level and ignores objects witihin + nested pseudofolders. + + ## Notes + + - Excludes pseudofolders from the results. + - If no pseudofolder name is entered, `pseudofolder` defaults to `""`, thereby getting all objects in + the container + - Returns the object name. + - Returns files + - Does not list pseudofolders + + ## Example + + Returns all objects in the `"test_folder/"` but not objects in nested pseudofolders + ExHubic.Swift.list_objects("test_folder/", "default", [nested: :false]) + + Returns all objects in the `"test_folder/"` and all nested pseudofolders. + ExHubic.Swift.list_objects("test_folder/", "default", [nested: :true]) + + Returns all objects in the `"default"` container + ExHubic.Swift.list_objects("default", [nested: :true]) + """ + @callback list_objects(String.t, String.t, list) :: {:ok, list} | {:error, map} + + + @doc ~s""" + Lists objects within a pseudofolder in a container or raises an error. Optionally include objects in nested + pseudofolders. see `list_objects/3` + """ + @callback list_objects!(String.t, String.t, list) :: {:ok, list} | no_return + + + @doc ~s""" + Lists all pseudofolders including nested pseudofolders in a container. See `list_pseudofolders/3` + + ## Arguments + + - `container`: The name of the container in which to list the pseudofolders. + """ + @callback list_pseudofolders(String.t) :: {:ok, list} | {:error, Openstex.Response.t} + + @doc ~s""" + Lists pseudofolders at the top level in a given pseudofolder. See `list_pseudofolders/3` + + ## Arguments + + - `pseudofolder`: The pseudofolder in which to search for other top-level pseudofolders + - `container`: The name of the container in which to list the pseudofolders. + """ + @callback list_pseudofolders(String.t, String.t) :: {:ok, list} | {:error, Openstex.Response.t} + + + @doc ~s""" + Lists all pseudofolders within a pseudofolder in a container. + + ## Arguments + + - `pseudofolder`: The pseudofolder in which to search for other pseudofolders + - `container`: The name of the container in which to list the pseudofolders. + - `nested`, defaults to :false + + [nested: true] => returns all objects in nested pseudofolders also. + + [nested: false] => returns all objects at the top level and ignores objects witihin + nested pseudofolders. + + ## Notes + + - If no pseudofolder is entered, then pseudofolder defaults to `"" ` which in turn results in all + pseudofolders being fetched one level deep at the root level. + - Returns the pseudofolders by name. + - Traverses as deep as the most nested folder beyond the passed folder to get all folders. + - Excludes non-pseudofolder objects from the results. + + ## Example + + Gets all pseudofolders in the `"test_folder/"` pseudofolder one level deep (top level) + ExHubic.Swift.list_pseudofolders("test_folder/", "default") + + Gets all pseudofolders in the `"test/"` pseudofolder one level deep (top level) + ExHubic.Swift.list_pseudofolders("test", "default") + + Gets all the pseudofolders in the container and traverse to the deepest nested pseudofolders + ExHubic.Swift.list_pseudofolders(default", [nested: :true]) + """ + @callback list_pseudofolders(String.t, String.t, list) :: {:ok, list} | {:error, map} + + + @doc ~s""" + Lists all pseudofolders including nested pseudofolders in a container. See `list_pseudofolders!/3` + + ## Arguments + + - `container`: The name of the container in which to list the pseudofolders. + """ + @callback list_pseudofolders!(String.t) :: list | no_return + + + @doc ~s""" + Lists pseudofolders at the top level in a given pseudofolder. See `list_pseudofolders!/3` + + ## Arguments + + - `pseudofolder`: The pseudofolder in which to search for other top-level pseudofolders + - `container`: The name of the container in which to list the pseudofolders. + """ + @callback list_pseudofolders!(String.t, String.t) :: list | no_return + + + @doc ~s""" + Lists all pseudofolders within a pseudofolder in a container or raises an error. See `list_pseudofolders/3`. + """ + @callback list_pseudofolders!(String.t, String.t, list) :: list| no_return + + + @doc ~s""" + Checks if a pseudofolder exists. + + ## Arguments + + - `pseudofolder`: The pseudofolder whose existence is to be checked. + - `container`: The name of the container in which to list the pseudofolders. + """ + @callback pseudofolder_exists?(String.t, String.t) :: boolean | no_return + + + @doc ~s""" + Deletes all objects in a pseudofolder effectively deleting the pseudofolder itself. + + ## Arguments + + - `pseudofolder`: The pseudofolder to be deleted. + - `container`: The container in which to delete the pseudofolder. + + ## Notes + + - Returns `:ok` anyways even if pseudofolder did not exist in the first place. + - Returns `{:error, list}` if some objects were not deleted where `list` represents + the objects for which deletion failed. + """ + @callback delete_pseudofolder(String.t, String.t) :: :ok | {:error, list} + + +end diff --git a/lib/services/swift/v1/query.ex b/lib/services/swift/v1/query.ex new file mode 100644 index 0000000..f9e7be9 --- /dev/null +++ b/lib/services/swift/v1/query.ex @@ -0,0 +1,434 @@ +defmodule Openstex.Services.Swift.V1.Query do + @moduledoc ~S""" + Helper functions to assist in building queries for openstack compatible swift apis. + + Builds a query in a format that subsequently is easily modified. The query may ultimately be sent to + an openstack/swift compliant api with a library such as HTTPotion or HTTPoison. See + [ex_hubic](https://hex.pm/packages/ex_hubic) for an example implementation. + + ## Example + + client = ExHubic.Swift + account = client.swift().get_account() + Openstex.Services.Swift.V1.Query.account_info(account) |> ExHubic.request(query) + """ + alias Openstex.Swift.Query + + + # CONTAINER RELATED REQUESTS + + + @doc ~S""" + Get account details and containers for given account. + + ## Api + + GET /v1/​{account}​ + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + Openstex.Services.Swift.V1.Query.account_info(account) |> client.request() + """ + @spec account_info(String.t) :: Openstack.t + def account_info(account) do + %Query{ + method: :get, + uri: account, + params: %{query_string: %{"format" => "json"}} + } + end + + + @doc ~S""" + Create a new container. + + ## Api + + PUT /v1/​{account}/{container}​ + + + ## Arguments + + - `container`: name of the container to be created + - `account`: account of user accessing swift service + - `opts`: + + - `read_acl`: headers for the container read access control list. + - Examples: + 1. For giving public read access: `[read_acl: ".r:*" ]`, *note:* `.r:` can be any of `.ref:`, `.referer:`, or `.referrer:`. + 2. For giving a `*.some_website.com` read access: `[read_acl: ".r:.some_website.com"]` + 3. For giving a user accountread access, [read_acl: `user_account`] + 4. See [Swift Docs](https://github.com/openstack/swift/blob/master/swift/common/middleware/acl.py#L50) for more examples + 5. For giving write access and list access: `[read_acl: ".r:*,.rlistings"]` + + - `write_acl`: headers for the container write access control list. *Note:* For `X-Container-Write` referrers are not supported. + - Examples: + 1. For giving write access to a user account: `[write_acl: "user_account"]` + + - `headers`: other metadata headers to be applied to the container. + - Examples: + 1. Appying changes to the CORS restrictions for a container. + eg: + `[headers: [{"X-Container-Meta-Access-Control-Allow-Origin", "http://localhost:4000"}]]` # allowed origins to make cross-origin requests. + `[headers: [{"X-Container-Meta-Access-Control-Max-Age", "1000"}]]` # validity of preflight requests in seconds. + Other CORS headers include `X-Container-Meta-Access-Control-Allow-Headers`, `X-Container-Meta-Access-Control-Expose-Headers` + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + Openstex.Services.Swift.V1.Query.create_container("new_container", account) |> client.request() + """ + @spec create_container(String.t, String.t, Keyword.t) :: Openstack.t + def create_container(container, account, opts \\ []) do + read_acl = Keyword.get(opts, :read_acl, :nil) + write_acl = Keyword.get(opts, :write_acl, :nil) + headers = Keyword.get(opts, :headers, []) + headers = + cond do + read_acl == :nil && write_acl == :nil -> [] + read_acl != :nil && write_acl == :nil -> [{"X-Container-Read", read_acl}] + read_acl == :nil && write_acl != :nil -> [{"X-Container-Write", write_acl}] + :true -> [{"X-Container-Read", read_acl}] ++ [{"X-Container-Write", write_acl}] + end ++ headers + %Query{ + method: :put, + uri: account <> "/" <> container, + headers: headers, + params: %{query_string: %{"format" => "json"}} + } + end + + + @doc ~S""" + Modify a container. See docs for possible changes to [container metadata](http://developer.openstack.org/api-ref-objectstorage-v1.html) + which are achieved by sending changes in the request headers. + + ## Api + + POST /v1/​{account}/{container}​ + + ## Arguments + + - `container`: name of the container to be created + - `account`: account of user accessing swift service + - `opts`: + + - `read_acl`: headers for the container read access control list. + - Examples: + 1. For giving public read access: `[read_acl: ".r:*" ]` + 2. For giving a `*.some_website.com` read access: `[read_acl: ".r:.some_website.com"]` + 3. For giving a user accountread access, [read_acl: `user_account`] + 4. See [Swift Docs](https://github.com/openstack/swift/blob/master/swift/common/middleware/acl.py#L50) for more examples + 5. For giving write access and list access: `[read_acl: ".r:*,.rlistings"]` + + - `write_acl`: headers for the container write access control list. + - Example: + 1. For giving write access to a user account: `[write_acl: "user_account"]` + + - `headers`: other metadata headers to be applied to the container. + - Examples: + 1. Appying changes to the CORS restrictions for a container. + eg: + `[headers: [{"X-Container-Meta-Access-Control-Allow-Origin", "http://localhost:4000"}]]` # allowed origins to make cross-origin requests. + `[headers: [{"X-Container-Meta-Access-Control-Max-Age", "1000"}]]` # validity of preflight requests in seconds. + Other CORS headers include `X-Container-Meta-Access-Control-Allow-Headers`, `X-Container-Meta-Access-Control-Expose-Headers` + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + headers = [] + Openstex.Services.Swift.V1.Query.modify_container("new_container", account, headers) |> client.request() + """ + @spec modify_container(String.t, String.t, Keyword.t) :: Openstack.t + def modify_container(container, account, opts \\ []) do + create_container(container, account, opts) + |> Map.put(:method, :post) + end + + @doc ~S""" + Delete a container + + ## Api + + DELETE /v1/​{account}/{container}​ + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + Openstex.Services.Swift.V1.Query.delete_container("new_container", account) |> client.request(query) + """ + @spec delete_container(String.t, String.t) :: Openstack.t + def delete_container(container, account) do + %Query{ + method: :delete, + uri: account <> "/" <> container, + params: %{query_string: %{"format" => "json"}} + } + end + + + @doc ~S""" + Get information about the container + + ## Api + + DELETE /v1/​{account}/{container}​ + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + query = Openstex.Services.Swift.V1.Query.container_info("new_container", account) |> client.request() + """ + @spec container_info(String.t, String.t) :: Openstack.t + def container_info(container, account) do + %Query{ + method: :head, + uri: account <> "/" <> container, + params: %{query_string: %{"format" => "json"}} + } + end + + + # OBJECT RELATED REQUESTS + + + @doc ~S""" + List objects in a container + + ## Api + + GET /v1/​{account}​/{container} + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + Openstex.Services.Swift.V1.Query.get_objects("new_container", account) |> client.request() + """ + @spec get_objects(String.t, String.t) :: Openstack.t + def get_objects(container, account) do + %Query{ + method: :get, + uri: account <> "/" <> container, + params: %{query_string: %{"format" => "json"}} + } + end + + + + @doc ~S""" + Get/Download a specific object (file) + + ## Api + + GET /v1/​{account}​/{container}/{object} + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + server_object = "server_file.txt" + container = "new_container" + Openstex.Services.Swift.V1.Query.get_object(server_object, container, account) |> client.request(query) + + ## Arguments + + - `server_object`: The path name of the object in the server + - `container`: The container of the object in the server + - `account`: The account accessing the object + - `opts`: + - `headers`: Additional headers metadata in the request. Eg `[headers: [{"If-None-Match", ""}]`, this example would return `304` if the local file md5 was the same as the object etag on the server. + """ + @spec get_object(String.t, String.t, String.t, Keyword.t) :: Openstack.t + def get_object(server_object, container, account, opts \\ []) do + headers = Keyword.get(opts, :headers, []) + server_object = Openstex.Utils.remove_if_has_trailing_slash(server_object) + %Query{ + method: :get, + uri: account <> "/" <> container <> "/" <> server_object, + headers: headers, + params: %{} + } + end + + + @doc """ + Create or replace an object (file). + + ## Api + + PUT /v1/​{account}​/{container}/{object} + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + container = "new_container" + object_name = "client_file.txt" + client_object_pathname = Kernel.to_string(:code.priv_dir(:openstex)) <> "/" <> object_name + Openstex.Services.Swift.V1.Query.create_object(container, account, client_object_pathname, [server_object: "server_file.txt"]) + |> client.request(query) + + ## Arguments + + - `container`: container to upload the file to + - `account`: account uploading the file. + - `client_object_pathname`: path of the file being uploaded. + - `opts`: + - `server_object`: filename under which the file will be stored on the openstack object storage server. defaults to the `client_object_pathname` if none given. + - `multipart_manifest`: Defaults to `:false`. If `:true`, adds `multipart-manifest=put` to the query string. This option should be set to `:true` when uploading the manifest for a large static object. + - `x_object_manifest`: Relevant to dynamic upload of large objects. Defaults to `:false`. If set, modifies the `X-Object-Manifest` header. The format used should be `[x_object_manifest: "container/myobject/"]`. + - `chunked_transfer`: Defaults to `:false`, if `:true, set the `Transfer-Encoding` to `chunked`. + - `content_type`: Defaults to `:false`, otherwise changes the `Content-Type` header, which changes the MIME type for the object. Eg, `[content_type: "image/jpeg"]` + - `x_detect_content_type`: Defaults to `:false`, otherwise changes the `X-Detect-Content-Type` header, the `X-Detect-Content-Type` header will be ignored and the actual file MIME type will be autodetected. Eg, `[x_detect_content_type: :true]` + - `e_tag`: Defaults to `:true`, if `:true`, sets the `ETag` header of the file. Enhances upload integrity. If set to `:false`, the `ETag` header will be excluded. + - `content_disposition`: Defaults to `:false`. Otherwise the `Content-Disposition` header can be changed from the default browser behaviour `inline` to another value. Eg `[content_disposition: "attachment; my_file.pdf"]` + - `delete_after`: Defaults to `:false`. Otherwise the `X-Delete-After` header can be added so that the object is deleted after n seconds. Eg `[delete_after: (24 * 60 * 60)]` will delete the object in 1 day. + - `e_tag`: Defaults to `:true`, if `:true`, sets the `ETag` header of the file. Enhances upload integrity. If set to `:false`, the `ETag` header will be excluded. + + ## Notes + + See the openstack docs for more information relating to [object uploads](http://docs.openstack.org/developer/swift/api/object_api_v1_overview.html) and + [large object uploads](http://docs.openstack.org/developer/swift/overview_large_objects.html). + For uploading large objects, the operation typically involves multiple queries so a [Helper function](https://github.com/stephenmoloney/openstex/lib/swift/v1/helpers.ex) is planned + for large uploads. + Large objects are categorized as those over 5GB in size. + There are two ways of uploading large files - dynamic uploads and static uploads. See [here](http://docs.openstack.org/developer/swift/overview_large_objects.html#direct-api) for more information. + """ + @spec create_object(String.t, String.t, String.t, list) :: Openstack.t + def create_object(container, account, client_object_pathname, opts) do + server_object = Keyword.get(opts, :server_object, Path.basename(client_object_pathname)) + + # headers + x_object_manifest = Keyword.get(opts, :x_object_manifest, :false) + x_object_manifest = if x_object_manifest != :false, do: URI.encode(x_object_manifest), else: x_object_manifest + chunked_transfer = Keyword.get(opts, :chunked_transfer, :false) + content_type = Keyword.get(opts, :content_type, :false) + x_detect_content_type = Keyword.get(opts, :x_detect_content_type, :false) + content_disposition = Keyword.get(opts, :content_disposition, :false) + delete_after = Keyword.get(opts, :delete_after, :false) + e_tag = Keyword.get(opts, :e_tag, :true) + + # query_string + multipart_manifest = Keyword.get(opts, :multipart_manifest, :false) + + + case File.read(client_object_pathname) do + {:ok, binary_object} -> + path = account <> "/" <> container <> "/" <> server_object + query_string = + Map.merge(%{"format" => "json"}, + case multipart_manifest do + :true -> %{"multipart-manifest" => "put"} + :false -> %{} + end + ) + headers = if x_object_manifest != :false, do: [{"X-Object-Manifest", x_object_manifest}], else: [] + headers = if chunked_transfer != :false, do: headers ++ [{"Transfer-Encoding", "chunked"}], else: headers + headers = if content_type != :false, do: headers ++ [{"Content-Type", content_type}], else: headers + headers = if x_detect_content_type != :false, do: headers ++ [{"X-Detect-Content-Type", "true"}], else: headers + headers = if e_tag != :false, do: headers ++ [{"ETag", Base.encode16(:erlang.md5(binary_object), case: :lower)}], else: headers + headers = if content_disposition != :false, do: headers ++ [{"Content_Disposition", content_disposition}], else: headers + headers = if delete_after != :false, do: headers ++ [{"X-Delete-After", delete_after}], else: headers + %Query{ + + method: :put, + uri: path, + headers: headers, + params: %{ + binary: binary_object, + query_string: query_string + } + } + {:error, posix_error} -> + Og.context(__ENV__, :error) + Og.log_return(posix_error, :error) + posix_error + end + end + + + @doc """ + Delete an Object (Delete a file) + + ## Api + + DELETE /v1/​{account}​/{container}/{object} + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + container = "new_container" + server_object = "server_file.txt" + Openstex.Services.Swift.V1.Query.delete_object(server_object, container, account, server_object) |> client.request(query) + """ + @spec delete_object(String.t, String.t, String.t) :: Openstack.t + def delete_object(server_object, container, account) do + server_object = Openstex.Utils.remove_if_has_trailing_slash(server_object) + %Query{ + method: :delete, + uri: account <> "/" <> container <> "/" <> server_object, + params: %{} + } + end + + + # PSEUDOFOLDER RELATED REQUESTS + + + @doc """ + List all objects and psuedofolders in a psuedofolder for a given container. + + ## Api + + GET /v1/​{account}​/{container}?prefix=pseudofolder&delimiter=/ + + ## Notes + + - Query for only the top level objects and pseudofolders + - Query execution will *not* return nested objects and pseudofolders + - In order to view nested objects and pseudofolders, the function should be called recursively. See + `Openstex.Helpers.list_pseudofolders_recursively/2` and `Openstex.Helpers.list_all_objects/3`. + + ## Example + + as implemented by the `ExHubic` library + + client = ExHubic.Swift + account = client.swift().get_account() + Openstex.Services.Swift.V1.Query.get_objects_in_folder("test_folder/", "default", account) |> client.request(query) + """ + @spec get_objects_in_folder(String.t, String.t, String.t) :: list + def get_objects_in_folder(pseudofolder \\ "", container, account) do + q = get_objects(container, account) + Map.put(q, :params, Map.merge(q.params, %{query_string: Map.merge(q.params.query_string, %{delimiter: "/", prefix: pseudofolder})})) + end + + + +end diff --git a/lib/supervisor.ex b/lib/supervisor.ex new file mode 100644 index 0000000..14f4487 --- /dev/null +++ b/lib/supervisor.ex @@ -0,0 +1,30 @@ +defmodule Openstex.Supervisor do + @moduledoc :false + use Supervisor + + # Public + + def start_link(client, opts \\ []) do + Og.context(__ENV__, :debug) + Supervisor.start_link(__MODULE__, {client, opts}, name: supervisor_name(client)) + end + + # Callbacks + + def init({client, opts}) do + Og.context(__ENV__, :debug) + config = client.config() + sup_tree = + if config.__info__(:module) != :nil do + [{client, {config, :start_agent, [client, opts]}, :permanent, 10_000, :worker, [config]}] + else + [] + end + supervise(sup_tree, strategy: :one_for_one, max_restarts: 30) + end + + defp supervisor_name(client) do + Module.concat(Openstex.Supervisor, client) + end + +end diff --git a/lib/transformation.ex b/lib/transformation.ex new file mode 100644 index 0000000..e3c9b47 --- /dev/null +++ b/lib/transformation.ex @@ -0,0 +1,8 @@ +defprotocol Openstex.Transformation do + @moduledoc :false + @fallback_to_any true + + @spec prepare_request(Openstex.Query.t, Keyword.t, atom) :: Openstex.HttpQuery.t + def prepare_request(queryable, httpoison_opts, client) + +end diff --git a/lib/transformation/swift/transformation.ex b/lib/transformation/swift/transformation.ex new file mode 100644 index 0000000..33dee62 --- /dev/null +++ b/lib/transformation/swift/transformation.ex @@ -0,0 +1,44 @@ +defimpl Openstex.Transformation, for: Openstex.Swift.Query do + @moduledoc :false + alias Openstex.Swift.Query + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + + + # Public + + + @spec prepare_request(Query.t, Keyword.t, atom) :: Openstex.HttpQuery.t + def prepare_request(query, httpoison_opts, client) + + def prepare_request(%Query{method: method, uri: uri, headers: headers, params: params}, httpoison_opts, client) do + uri = client.swift().get_endpoint() <> uri + uri = + cond do + params == %{} -> uri + Map.get(params, :query_string, :nil) != :nil -> uri <> "?" <> (Map.fetch!(params, :query_string) |> URI.encode_query()) + :true -> uri + end + body = if Map.has_key?(params, :binary), do: Map.get(params, :binary), else: "" + headers = headers(client, headers) + default_httpoison_opts = client.config().httpoison_config(client) + options = Keyword.merge(default_httpoison_opts, httpoison_opts) + %Openstex.HttpQuery{method: method, uri: uri, body: body, headers: headers, options: options, service: :openstack} + end + + + # Private + + + defp headers(client, headers) do + @default_headers ++ + [ + { + "X-Auth-Token", client.keystone().get_xauth_token(client) + } + ] ++ + headers + |> Enum.uniq() + end + + +end diff --git a/lib/transformation/transformation.ex b/lib/transformation/transformation.ex new file mode 100644 index 0000000..6679978 --- /dev/null +++ b/lib/transformation/transformation.ex @@ -0,0 +1,44 @@ +defimpl Openstex.Transformation, for: Any do + @moduledoc :false + alias Openstex.Query + @default_headers [{"Content-Type", "application/json; charset=utf-8"}] + + + # Public + + + @spec prepare_request(Query.t, Keyword.t, atom) :: Openstex.HttpQuery.t + def prepare_request(query, httpoison_opts, client) + + def prepare_request(%Query{method: method, uri: uri, headers: headers, params: params}, httpoison_opts, client) do + uri = client.swift().get_endpoint() <> uri + uri = + cond do + params == %{} -> uri + Map.get(params, :query_string, :nil) != :nil -> uri <> "?" <> (Map.fetch!(params, :query_string) |> URI.encode_query()) + :true -> uri + end + body = if Map.has_key?(params, :binary), do: Map.get(params, :binary), else: "" + headers = headers(client, headers) + default_httpoison_opts = client.config().httpoison_config(client) + options = Keyword.merge(default_httpoison_opts, httpoison_opts) + %Openstex.HttpQuery{method: method, uri: uri, body: body, headers: headers, options: options, service: :openstack} + end + + + # Private + + + defp headers(client, headers) do + @default_headers ++ + [ + { + "X-Auth-Token", client.swift().get_xauth_token() + } + ] ++ + headers + |> Enum.uniq() + end + + +end diff --git a/lib/utils/utils.ex b/lib/utils/utils.ex new file mode 100644 index 0000000..cce79b7 --- /dev/null +++ b/lib/utils/utils.ex @@ -0,0 +1,69 @@ +defmodule Openstex.Utils do + @moduledoc "Utility functions" + + @doc """ + Put headers for a http_query. If the headers already exist, the new headers will override the old headers. + """ + @spec put_http_headers(Openstex.HttpQuery.t, map) :: Openstex.HttpQuery.t + def put_http_headers(%Openstex.HttpQuery{headers: headers} = http_query, new_headers) when is_map(new_headers) do + old_headers_map = headers |> Enum.into(%{}) + new_headers_map = Map.merge(old_headers_map, new_headers) + new_headers = Map.to_list(new_headers_map) + Map.put(http_query, :headers, new_headers) + end + + @doc :false + def ensure_has_leading_slash(folder) do + case String.last(folder) do + "/" -> + folder + :nil -> + folder + _other -> + folder <> "/" + end + end + + @doc :false + def ensure_has_trailing_slash(folder) do + case String.first(folder) do + "/" -> + folder + _other -> + "/" <> folder + end + end + + @doc :false + def remove_if_has_trailing_slash(folder) do + case String.first(folder) do + "/" -> + {"/", folder} = String.split_at(folder, 1) + folder + _other -> + folder + end + end + + @doc :false + defmacro ets_tablename(client) do + quote do + "Ets." + <> + ( + unquote(client) |> Atom.to_string() + ) + |> String.to_atom() + end + end + + + @doc "Generate tempurl Signature for Openstack Swift" + @spec gen_tempurl_signature(String.t, integer, String.t, String.t) :: String.t + def gen_tempurl_signature(method, expiry, path, temp_key) do + hmac_body = "#{method}\n#{Integer.to_string(expiry)}\n#{path}" + :crypto.hmac(:sha, temp_key, hmac_body) |> Base.encode16(case: :lower) + end + + +end diff --git a/mix.exs b/mix.exs new file mode 100644 index 0000000..3ff6d04 --- /dev/null +++ b/mix.exs @@ -0,0 +1,69 @@ +defmodule Openstex.Mixfile do + use Mix.Project + @version "0.2.0" + + def project do + [ + app: :openstex, + name: "Openstex", + version: @version, + elixir: "~> 1.2", + build_embedded: Mix.env == :prod, + start_permanent: Mix.env == :prod, + elixirc_paths: elixirc_paths(Mix.env), + source_url: "https://github.com/stephenmoloney/openstex", + description: description(), + package: package(), + deps: deps(), + docs: docs() + ] + end + + def application() do + [ + mod: [], + applications: [:calendar, :crypto, :httpoison, :logger, :mapail] + ] + end + + defp deps() do + [ + # Production deps + {:poison, "~> 2.0"}, + {:httpoison, "~> 0.8.0"}, + {:calendar, "~> 0.13.2"}, + {:mapail, github: "stephenmoloney/mapail", branch: "master"}, + + # Docs deps + {:markdown, github: "devinus/markdown", only: :dev}, + {:ex_doc, "~> 0.11", only: :dev} + ] + end + + defp description() do + ~s""" + A client in elixir for making requests to openstack compliant apis. + """ + end + + defp package do + %{ + licenses: ["MIT"], + maintainers: ["Stephen Moloney"], + links: %{ "GitHub" => "https://github.com/stephenmoloney/openstex"}, + files: ~w(lib mix.exs README* LICENCE*) + } + end + + defp docs() do + [ + main: "Openstex", + extras: [ + "docs/ovh/cloudstorage/getting_started.md": [path: "mix_task_advanced.md", title: "Getting Started (OVH Cloudstorage)"] + ] + ] + end + + defp elixirc_paths(_), do: ["lib"] + +end