From f02acc95d141f9943976ca2c9f09a9ebd43caea1 Mon Sep 17 00:00:00 2001 From: emielvdveen Date: Tue, 2 Jul 2024 17:18:35 +0200 Subject: [PATCH] #892 [Storage] CMS: export all files --- core/config/config.exs | 10 ++- core/config/dev.exs | 1 + core/config/runtime.exs | 7 -- core/frameworks/concept/context.ex | 23 +++++++ core/lib/core/apns.ex | 2 +- .../push_subscription_controller.ex | 25 ------- .../controllers/push_subscription_html.ex | 3 - .../core_web/controllers/vapid_public_key.ex | 2 - core/lib/core_web/router.ex | 6 -- core/lib/core_web/ui/timestamp.ex | 4 ++ core/mix.exs | 59 ++++++++--------- core/mix.lock | 12 ++-- .../gettext/de/LC_MESSAGES/eyra-storage.po | 4 ++ .../gettext/en/LC_MESSAGES/eyra-storage.po | 4 ++ core/priv/gettext/eyra-storage.pot | 4 ++ .../gettext/nl/LC_MESSAGES/eyra-storage.po | 4 ++ core/systems/project/_public.ex | 20 ++++++ core/systems/storage/_routes.ex | 1 + core/systems/storage/built_in/s3.ex | 10 ++- core/systems/storage/controller.ex | 65 +++++++++++++++++++ core/systems/storage/endpoint_data_view.ex | 26 +++++++- .../push_subscription_controller_test.exs | 38 ----------- 22 files changed, 206 insertions(+), 124 deletions(-) create mode 100644 core/frameworks/concept/context.ex delete mode 100644 core/lib/core_web/controllers/push_subscription_controller.ex delete mode 100644 core/lib/core_web/controllers/push_subscription_html.ex delete mode 100644 core/lib/core_web/controllers/vapid_public_key.ex create mode 100644 core/systems/storage/controller.ex delete mode 100644 core/test/core_web/controllers/push_subscription_controller_test.exs diff --git a/core/config/config.exs b/core/config/config.exs index 6c7e99855..cc5c8b8d1 100644 --- a/core/config/config.exs +++ b/core/config/config.exs @@ -40,6 +40,8 @@ config :plug, :statuses, %{ 404 => "Page not found" } +config :core, :naming, handlers: [Systems.Project.Public] + config :core, CoreWeb.FileUploader, max_file_size: 100_000_000 config :core, @@ -69,7 +71,8 @@ config :core, :rate, quotas: [ [service: :azure_blob, limit: 1000, unit: :call, window: :minute, scope: :local], [service: :azure_blob, limit: 10_000_000, unit: :byte, window: :day, scope: :local], - [service: :azure_blob, limit: 1_000_000_000, unit: :byte, window: :day, scope: :global] + [service: :azure_blob, limit: 1_000_000_000, unit: :byte, window: :day, scope: :global], + [service: :storage_export, limit: 1, unit: :call, window: :minute, scope: :local] ] config :core, ecto_repos: [Core.Repo] @@ -124,11 +127,6 @@ config :core, :ssl, config :core, :ssl_proxied, {:ok, "true"} == System.fetch_env("SSL_PROXIED") -config :web_push_encryption, :vapid_details, - subject: "mailto:administrator@example.com", - public_key: "use `mix web_push.gen.keypair`", - private_key: "" - config :core, :version, System.get_env("VERSION", "dev") config :core, :assignment, external_panels: ~w(liss ioresearch generic) diff --git a/core/config/dev.exs b/core/config/dev.exs index 28ade7bf4..aa2c4b3b7 100644 --- a/core/config/dev.exs +++ b/core/config/dev.exs @@ -61,6 +61,7 @@ config :core, Systems.Storage.BuiltIn, special: Systems.Storage.BuiltIn.LocalFS config :core, :rate, prune_interval: 5 * 60 * 1000, quotas: [ + [service: :storage_export, limit: 10, unit: :call, window: :minute, scope: :local], [service: :azure_blob, limit: 1, unit: :call, window: :second, scope: :local], [service: :azure_blob, limit: 100, unit: :byte, window: :second, scope: :local] ] diff --git a/core/config/runtime.exs b/core/config/runtime.exs index 67bbe94a5..a53e601bd 100644 --- a/core/config/runtime.exs +++ b/core/config/runtime.exs @@ -113,13 +113,6 @@ if config_env() == :prod do access_key: System.get_env("UNSPLASH_ACCESS_KEY"), app_name: System.get_env("UNSPLASH_APP_NAME") - config :web_push_encryption, :vapid_details, - subject: "mailto:admin@#{app_mail_domain}", - public_key: System.get_env("WEB_PUSH_PUBLIC_KEY"), - private_key: System.get_env("WEB_PUSH_PRIVATE_KEY") - - config :logger, level: System.get_env("LOG_LEVEL", "info") |> String.to_existing_atom() - if sentry_dsn = System.get_env("SENTRY_DSN") do config :sentry, dsn: sentry_dsn, diff --git a/core/frameworks/concept/context.ex b/core/frameworks/concept/context.ex new file mode 100644 index 000000000..a84c273c4 --- /dev/null +++ b/core/frameworks/concept/context.ex @@ -0,0 +1,23 @@ +defmodule Frameworks.Concept.Context do + defmodule Handler do + @type model :: struct() + @callback name(model) :: {:ok, binary()} | {:error, atom()} + end + + def name(model, default) when is_struct(model) and is_binary(default) do + Enum.reduce(handlers(), default, fn handler, acc -> + case handler.name(model) do + {:ok, name} -> name + {:error, _} -> acc + end + end) + end + + defp handlers() do + Access.get(settings(), :handlers, []) + end + + defp settings() do + Application.fetch_env!(:core, :naming) + end +end diff --git a/core/lib/core/apns.ex b/core/lib/core/apns.ex index 9c3ff59d0..e199b6a55 100644 --- a/core/lib/core/apns.ex +++ b/core/lib/core/apns.ex @@ -48,5 +48,5 @@ defmodule Core.APNS do Logger.error("Unexpected push error: #{inspect(response)}") end - defp backend, do: Application.get_env(:core, :apns_backend, Pigeon.APNS) + defp backend, do: Application.get_env(:core, :apns_backend) end diff --git a/core/lib/core_web/controllers/push_subscription_controller.ex b/core/lib/core_web/controllers/push_subscription_controller.ex deleted file mode 100644 index aabc5b4f3..000000000 --- a/core/lib/core_web/controllers/push_subscription_controller.ex +++ /dev/null @@ -1,25 +0,0 @@ -defmodule CoreWeb.PushSubscriptionController do - use CoreWeb, :controller - - alias Core.WebPush - - action_fallback(CoreWeb.FallbackController) - - def register(%{assigns: %{current_user: user}} = conn, %{"subscription" => subscription}) do - {:ok, _} = WebPush.register(user, subscription) - conn |> json(%{}) - end - - def register(conn, _) do - conn |> json(%{}) - end - - def vapid_public_key(conn, _) do - conn - |> put_resp_content_type("text/plain") - |> send_resp( - 200, - Application.fetch_env!(:web_push_encryption, :vapid_details) |> Keyword.get(:public_key) - ) - end -end diff --git a/core/lib/core_web/controllers/push_subscription_html.ex b/core/lib/core_web/controllers/push_subscription_html.ex deleted file mode 100644 index b17f2817f..000000000 --- a/core/lib/core_web/controllers/push_subscription_html.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule CoreWeb.PushSubscriptionHTML do - use CoreWeb, :html -end diff --git a/core/lib/core_web/controllers/vapid_public_key.ex b/core/lib/core_web/controllers/vapid_public_key.ex deleted file mode 100644 index 8509f2cba..000000000 --- a/core/lib/core_web/controllers/vapid_public_key.ex +++ /dev/null @@ -1,2 +0,0 @@ -defmodule CoreWeb.VapidPublicKeyController do -end diff --git a/core/lib/core_web/router.ex b/core/lib/core_web/router.ex index 45e92fc7e..dc6c86dd0 100644 --- a/core/lib/core_web/router.ex +++ b/core/lib/core_web/router.ex @@ -29,12 +29,6 @@ defmodule CoreWeb.Router do get("/uploads/:filename", UploadedFileController, :get) end - scope "/", CoreWeb do - pipe_through([:api, :require_authenticated_user]) - get("/web-push/vapid-public-key", PushSubscriptionController, :vapid_public_key) - post("/web-push/register", PushSubscriptionController, :register) - end - if Mix.env() == :dev do forward("/sent_emails", Bamboo.SentEmailViewerPlug) end diff --git a/core/lib/core_web/ui/timestamp.ex b/core/lib/core_web/ui/timestamp.ex index 39caebb14..d5a5af2ae 100644 --- a/core/lib/core_web/ui/timestamp.ex +++ b/core/lib/core_web/ui/timestamp.ex @@ -169,6 +169,10 @@ defmodule CoreWeb.UI.Timestamp do Timex.format!(datetime, "%Y-%m-%d", :strftime) end + def format_date_short!(datetime) do + Timex.format!(datetime, "%Y%m%d", :strftime) + end + def stamp(%DateTime{} = datetime) do weekday = Timex.format!(datetime, "%A", :strftime) month = Timex.format!(datetime, "%B", :strftime) diff --git a/core/mix.exs b/core/mix.exs index f363ef73e..149ba9d04 100644 --- a/core/mix.exs +++ b/core/mix.exs @@ -66,45 +66,46 @@ defmodule Core.MixProject do # Workaround for conflicting versions in ex_aws & ex_phone_number {:sweet_xml, "~> 0.7", override: true}, # Deps - {:esbuild, "~> 0.8", runtime: Mix.env() == :dev}, - {:tailwind, "~> 0.2.0", runtime: Mix.env() == :dev}, {:assent, "~> 0.2.3"}, + {:bamboo_phoenix, git: "https://github.com/populimited/bamboo_phoenix.git", ref: "bf3e320"}, + {:bamboo_ses, "~> 0.3.0"}, + {:bamboo, "~> 2.2"}, {:bcrypt_elixir, "~> 2.0"}, + {:csv, "~> 2.4"}, + {:currency_formatter, "~> 0.8"}, + {:ecto_commons, "~> 0.3.3"}, + {:ecto_sql, "~> 3.9.2"}, + {:esbuild, "~> 0.8", runtime: Mix.env() == :dev}, {:ex_aws_s3, "~> 2.5"}, - {:phoenix, "1.7.2"}, - {:phoenix_view, "~> 2.0"}, - {:phoenix_ecto, "~> 4.4"}, - {:phoenix_live_view, "~> 0.20.1"}, - {:phoenix_html, "~> 3.3.1"}, - {:phoenix_inline_svg, "~> 1.4"}, + {:faker, "~> 0.17"}, {:floki, ">= 0.27.0"}, - {:ecto_sql, "~> 3.9.2"}, - {:ecto_commons, "~> 0.3.3"}, - {:postgrex, ">= 0.15.13"}, {:gettext, "~> 0.19"}, + {:httpoison, "~> 2.2.1"}, {:jason, "~> 1.3"}, + {:kadabra, "~> 0.6.0"}, + {:libcluster, "~> 3.3"}, + {:logger_json, "~> 4.3"}, + {:mime, "~> 2.0"}, + {:nimble_parsec, "~> 1.2"}, + {:oban, "~> 2.13.3"}, + {:packmatic, "~> 1.2.0"}, + {:phoenix_ecto, "~> 4.4"}, + {:phoenix_html, "~> 3.3.1"}, + {:phoenix_inline_svg, "~> 1.4"}, + {:phoenix_live_view, "~> 0.20.1"}, + {:phoenix_view, "~> 2.0"}, + {:phoenix, "1.7.2"}, {:plug_cowboy, "~> 2.5"}, - {:faker, "~> 0.17"}, - {:timex, "~> 3.7"}, - {:bamboo, "~> 2.2"}, - {:bamboo_phoenix, git: "https://github.com/populimited/bamboo_phoenix.git", ref: "bf3e320"}, - {:bamboo_ses, "~> 0.3.0"}, + {:postgrex, ">= 0.15.13"}, + {:remote_ip, "~> 1.1"}, + {:sentry, "~> 8.0"}, + {:slugify, "~> 1.3"}, + {:statistics, "~> 0.6.2"}, + {:tailwind, "~> 0.2.0", runtime: Mix.env() == :dev}, {:telemetry_metrics, "~> 0.6"}, {:telemetry_poller, "~> 1.0"}, - {:currency_formatter, "~> 0.8"}, - {:web_push_encryption, "~> 0.3.1"}, - {:remote_ip, "~> 1.1"}, - {:pigeon, "~> 1.6.1"}, - {:kadabra, "~> 0.6.0"}, - {:oban, "~> 2.13.3"}, - {:nimble_parsec, "~> 1.2"}, + {:timex, "~> 3.7"}, {:typed_struct, "~> 0.2.1"}, - {:logger_json, "~> 4.3"}, - {:statistics, "~> 0.6.2"}, - {:csv, "~> 2.4"}, - {:sentry, "~> 8.0"}, - {:libcluster, "~> 3.3"}, - {:mime, "~> 2.0"}, # i18n {:ex_cldr, "~> 2.25"}, {:ex_cldr_numbers, "~> 2.23"}, diff --git a/core/mix.lock b/core/mix.lock index 96dced2e6..25a86613c 100644 --- a/core/mix.lock +++ b/core/mix.lock @@ -10,7 +10,7 @@ "burnex": {:hex, :burnex, "3.1.0", "1c1ffaab0dccd4efe80f3c3d0de61e9bb4e622fd0c52b0fccea693095e7c30b2", [:mix], [{:dns, "~> 2.2.0", [hex: :dns, repo: "hexpm", optional: false]}], "hexpm", "611af3dd131c1a5e75b367c75641c9104b0a942dfdd9767e69fbe8be883d536d"}, "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"}, - "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, + "certifi": {:hex, :certifi, "2.13.0", "e52be248590050b2dd33b0bb274b56678f9068e67805dca8aa8b1ccdb016bbf6", [:rebar3], [], "hexpm", "8f3d9533a0f06070afdfd5d596b32e21c6580667a492891851b0e2737bc507a1"}, "cldr_utils": {:hex, :cldr_utils, "2.22.0", "5df60df2603dfeeffe26e40ab1ee565df34da288a53bb2db0d0dbd243fd646ef", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.5", [hex: :certifi, repo: "hexpm", optional: true]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "ea14e8a6aa89ffd59a5d49baebe7ebf852cc024ac50dc2b3dabcd3786eeed657"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.3", "2c564dac95a35650e9b6acfe6d2952083d8a08e4a89b93a481acb552b325892e", [:mix], [], "hexpm", "3e38c9c2cb080828116597ca8807bb482618a315bfafd98c90bc22a821cc84df"}, @@ -51,11 +51,11 @@ "floki": {:hex, :floki, "0.34.2", "5fad07ef153b3b8ec110b6b155ec3780c4b2c4906297d0b4be1a7162d04a7e02", [:mix], [], "hexpm", "26b9d50f0f01796bc6be611ca815c5e0de034d2128e39cc9702eee6b66a4d1c8"}, "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, "gettext": {:hex, :gettext, "0.22.1", "e7942988383c3d9eed4bdc22fc63e712b655ae94a672a27e4900e3d4a2c43581", [:mix], [{:expo, "~> 0.4.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "ad105b8dab668ee3f90c0d3d94ba75e9aead27a62495c101d94f2657a190ac5d"}, - "hackney": {:hex, :hackney, "1.19.1", "59de4716e985dd2b5cbd4954fa1ae187e2b610a9c4520ffcb0b1653c3d6e5559", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "8aa08234bdefc269995c63c2282cf3cd0e36febe3a6bfab11b610572fdd1cad0"}, + "hackney": {:hex, :hackney, "1.17.1", "08463f93d2cc1a03817bf28d8dae6021543f773bd436c9377047224856c4422c", [:rebar3], [{:certifi, "~> 2.5", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "~> 3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "d2cba9e3c8103ad0320623e9f1c33e8d378a15eaabe2ee8ae441898f3d35a18c"}, "hpack": {:hex, :hpack_erl, "0.2.3", "17670f83ff984ae6cd74b1c456edde906d27ff013740ee4d9efaa4f1bf999633", [:rebar3], [], "hexpm", "06f580167c4b8b8a6429040df36cc93bba6d571faeaec1b28816523379cbb23a"}, "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, - "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, + "httpoison": {:hex, :httpoison, "2.2.1", "87b7ed6d95db0389f7df02779644171d7319d319178f6680438167d7b69b1f3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "51364e6d2f429d80e14fe4b5f8e39719cacd03eb3f9a9286e61e216feac2d2df"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, "joken": {:hex, :joken, "2.6.0", "b9dd9b6d52e3e6fcb6c65e151ad38bf4bc286382b5b6f97079c47ade6b1bcc6a", [:mix], [{:jose, "~> 1.11.5", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5a95b05a71cd0b54abd35378aeb1d487a23a52c324fa7efdffc512b655b5aaa7"}, @@ -70,15 +70,16 @@ "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, - "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, + "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"}, "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"}, "nimble_options": {:hex, :nimble_options, "0.4.0", "c89babbab52221a24b8d1ff9e7d838be70f0d871be823165c94dd3418eea728f", [:mix], [], "hexpm", "e6701c1af326a11eea9634a3b1c62b475339ace9456c1a23ec3bc9a847bca02d"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, "oban": {:hex, :oban, "2.13.6", "a0cb1bce3bd393770512231fb5a3695fa19fd3af10d7575bf73f837aee7abf43", [:mix], [{:ecto_sql, "~> 3.6", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c1c5eb16f377b3cbbf2ea14be24d20e3d91285af9d1ac86260b7c2af5464887"}, + "packmatic": {:hex, :packmatic, "1.2.0", "010ae3d5816e3a043de0b69c61e0e3afc76ebedabd85a9500e07c73e73fcbb93", [:mix], [{:httpoison, "~> 2.2.1", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "c56c7e998c2421a121235f511eb1a40fcaad8df36c398f1d8d336a6c29200108"}, "parallel_stream": {:hex, :parallel_stream, "1.1.0", "f52f73eb344bc22de335992377413138405796e0d0ad99d995d9977ac29f1ca9", [:mix], [], "hexpm", "684fd19191aedfaf387bbabbeb8ff3c752f0220c8112eb907d797f4592d6e871"}, "parent": {:hex, :parent, "0.12.1", "495c4386f06de0df492e0a7a7199c10323a55e9e933b27222060dd86dccd6d62", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2ab589ef1f37bfcedbfb5ecfbab93354972fb7391201b8907a866dadd20b39d1"}, - "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, + "parse_trans": {:hex, :parse_trans, "3.4.2", "c352ddc1a0d5e54f9b1654d45f9c432eef76f9cea371c55ddff769ef688fdb74", [:rebar3], [], "hexpm", "4c25347de3b7c35732d32e69ab43d1ceee0beae3f3b3ade1b59cbd3dd224d9ca"}, "phoenix": {:hex, :phoenix, "1.7.2", "c375ffb482beb4e3d20894f84dd7920442884f5f5b70b9f4528cbe0cedefec63", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.4", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "1ebca94b32b4d0e097ab2444a9742ed8ff3361acad17365e4e6b2e79b4792159"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.0", "0672ed4e4808b3fbed494dded89958e22fb882de47a97634c0b13e7b0b5f7720", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "09864e558ed31ee00bd48fcc1d4fc58ae9678c9e81649075431e69dbabb43cc1"}, "phoenix_html": {:hex, :phoenix_html, "3.3.3", "380b8fb45912b5638d2f1d925a3771b4516b9a78587249cabe394e0a5d579dc9", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "923ebe6fec6e2e3b3e569dfbdc6560de932cd54b000ada0208b5f45024bdd76c"}, @@ -98,6 +99,7 @@ "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "remote_ip": {:hex, :remote_ip, "1.1.0", "cb308841595d15df3f9073b7c39243a1dd6ca56e5020295cb012c76fbec50f2d", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "616ffdf66aaad6a72fc546dabf42eed87e2a99e97b09cbd92b10cc180d02ed74"}, "sentry": {:hex, :sentry, "8.1.0", "8d235b62fce5f8e067ea1644e30939405b71a5e1599d9529ff82899d11d03f2b", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.3", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "f9fc7641ef61e885510f5e5963c2948b9de1de597c63f781e9d3d6c9c8681ab4"}, + "slugify": {:hex, :slugify, "1.3.1", "0d3b8b7e5c1eeaa960e44dce94382bee34a39b3ea239293e457a9c5b47cc6fd3", [:mix], [], "hexpm", "cb090bbeb056b312da3125e681d98933a360a70d327820e4b7f91645c4d8be76"}, "socket": {:hex, :socket, "0.3.13", "98a2ab20ce17f95fb512c5cadddba32b57273e0d2dba2d2e5f976c5969d0c632", [:mix], [], "hexpm", "f82ea9833ef49dde272e6568ab8aac657a636acb4cf44a7de8a935acb8957c2e"}, "sourceror": {:hex, :sourceror, "0.12.2", "2ae55efd149193572e0eb723df7c7a1bda9ab33c43373c82642931dbb2f4e428", [:mix], [], "hexpm", "7ad74ade6fb079c71f29fae10c34bcf2323542d8c51ee1bcd77a546cfa89d59c"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, diff --git a/core/priv/gettext/de/LC_MESSAGES/eyra-storage.po b/core/priv/gettext/de/LC_MESSAGES/eyra-storage.po index 7306a48f8..78955ce58 100644 --- a/core/priv/gettext/de/LC_MESSAGES/eyra-storage.po +++ b/core/priv/gettext/de/LC_MESSAGES/eyra-storage.po @@ -126,3 +126,7 @@ msgstr "" #, elixir-autogen, elixir-format msgid "table.size.label" msgstr "" + +#, elixir-autogen, elixir-format +msgid "export.files.button" +msgstr "" diff --git a/core/priv/gettext/en/LC_MESSAGES/eyra-storage.po b/core/priv/gettext/en/LC_MESSAGES/eyra-storage.po index 8167115a1..9f56f2135 100644 --- a/core/priv/gettext/en/LC_MESSAGES/eyra-storage.po +++ b/core/priv/gettext/en/LC_MESSAGES/eyra-storage.po @@ -126,3 +126,7 @@ msgstr "File" #, elixir-autogen, elixir-format msgid "table.size.label" msgstr "Size" + +#, elixir-autogen, elixir-format +msgid "export.files.button" +msgstr "Export all files" diff --git a/core/priv/gettext/eyra-storage.pot b/core/priv/gettext/eyra-storage.pot index 159762c6c..dd9699274 100644 --- a/core/priv/gettext/eyra-storage.pot +++ b/core/priv/gettext/eyra-storage.pot @@ -126,3 +126,7 @@ msgstr "" #, elixir-autogen, elixir-format msgid "table.size.label" msgstr "" + +#, elixir-autogen, elixir-format +msgid "export.files.button" +msgstr "" diff --git a/core/priv/gettext/nl/LC_MESSAGES/eyra-storage.po b/core/priv/gettext/nl/LC_MESSAGES/eyra-storage.po index bfb9408ce..ce2e082fa 100644 --- a/core/priv/gettext/nl/LC_MESSAGES/eyra-storage.po +++ b/core/priv/gettext/nl/LC_MESSAGES/eyra-storage.po @@ -126,3 +126,7 @@ msgstr "" #, elixir-autogen, elixir-format msgid "table.size.label" msgstr "" + +#, elixir-autogen, elixir-format +msgid "export.files.button" +msgstr "" diff --git a/core/systems/project/_public.ex b/core/systems/project/_public.ex index 223733840..80f6e53bb 100644 --- a/core/systems/project/_public.ex +++ b/core/systems/project/_public.ex @@ -1,4 +1,6 @@ defmodule Systems.Project.Public do + @behaviour Frameworks.Concept.Context.Handler + import CoreWeb.Gettext import Ecto.Query, warn: false import Systems.Project.Queries @@ -14,6 +16,17 @@ defmodule Systems.Project.Public do alias Systems.Storage alias Systems.Workflow + @impl true + def name(%Systems.Storage.EndpointModel{} = model) do + case get_by_item_special(model) do + %{name: name} -> {:ok, name} + _ -> {:error, :not_found} + end + end + + @impl true + def name(_), do: {:error, :not_supported} + def get!(id, preload \\ []) do from(project in Project.Model, where: project.id == ^id, @@ -30,6 +43,13 @@ defmodule Systems.Project.Public do |> Repo.one() end + def get_by_item_special(special) do + special + |> get_item_by() + |> get_node_by_item!() + |> get_by_root() + end + def get_node!(id, preload \\ []) do from(node in Project.NodeModel, where: node.id == ^id, diff --git a/core/systems/storage/_routes.ex b/core/systems/storage/_routes.ex index 822124686..6370486cb 100644 --- a/core/systems/storage/_routes.ex +++ b/core/systems/storage/_routes.ex @@ -4,6 +4,7 @@ defmodule Systems.Storage.Routes do scope "/", Systems.Storage do pipe_through([:browser, :require_authenticated_user]) live("/storage/:id/content", EndpointContentPage) + get("/storage/:id/export", Controller, :export) end end end diff --git a/core/systems/storage/built_in/s3.ex b/core/systems/storage/built_in/s3.ex index a53b0e80b..82b8c4c30 100644 --- a/core/systems/storage/built_in/s3.ex +++ b/core/systems/storage/built_in/s3.ex @@ -24,6 +24,14 @@ defmodule Systems.Storage.BuiltIn.S3 do contents |> Enum.map(fn %{key: key, size: size, last_modified: last_modified} -> + {:ok, url} = + :s3 + |> ExAws.Config.new([]) + |> S3.presigned_url(:get, bucket, key, expires_in: 3600) + + %{key: key, size: size, last_modified: last_modified, url: url} + end) + |> Enum.map(fn %{key: key, size: size, last_modified: last_modified, url: url} -> path = String.replace_prefix(key, prefix, "") timestamp = @@ -32,7 +40,7 @@ defmodule Systems.Storage.BuiltIn.S3 do _ -> nil end - %{path: path, size: size, timestamp: timestamp} + %{path: path, size: String.to_integer(size), timestamp: timestamp, url: url} end) end diff --git a/core/systems/storage/controller.ex b/core/systems/storage/controller.ex new file mode 100644 index 000000000..f9005e735 --- /dev/null +++ b/core/systems/storage/controller.ex @@ -0,0 +1,65 @@ +defmodule Systems.Storage.Controller do + alias CoreWeb.UI.Timestamp + use CoreWeb, :controller + + alias Frameworks.Concept.Context + + alias Systems.Storage + alias Systems.Rate + + def export(conn, %{"id" => id}) do + if endpoint = + Storage.Public.get_endpoint!( + String.to_integer(id), + Storage.EndpointModel.preload_graph(:down) + ) do + special = Storage.EndpointModel.special(endpoint) + context_name = Context.name(endpoint, "export") + + export(conn, special, context_name) + else + service_unavailable(conn) + end + end + + def export( + %{remote_ip: remote_ip} = conn, + %Storage.BuiltIn.EndpointModel{} = builtin, + context_name + ) do + date = Timestamp.now() |> Timestamp.format_date_short!() + + export_name = + [date, context_name] + |> Enum.join(" ") + |> Slug.slugify(separator: ?_) + + builtin + |> Storage.BuiltIn.Backend.list_files() + |> request_permission(remote_ip) + |> Enum.map(fn %{url: url, path: path, timestamp: timestamp} -> + [source: {:url, url}, path: "#{export_name}/#{path}", timestamp: timestamp] + end) + |> Packmatic.build_stream() + |> Packmatic.Conn.send_chunked(conn, "#{export_name}.zip") + end + + def export(conn, _, _) do + service_unavailable(conn) + end + + defp request_permission(files, remote_ip) when is_list(files) do + size = Enum.reduce(files, 0, fn %{size: size}, acc -> acc + size end) + remote_ip = to_string(:inet_parse.ntoa(remote_ip)) + # raises error when request is denied + Rate.Public.request_permission(:storage_export, remote_ip, size) + files + end + + defp service_unavailable(conn) do + conn + |> put_status(:service_unavailable) + |> put_view(html: CoreWeb.ErrorHTML) + |> render(:"503") + end +end diff --git a/core/systems/storage/endpoint_data_view.ex b/core/systems/storage/endpoint_data_view.ex index 00d2a0cc4..ece39a83f 100644 --- a/core/systems/storage/endpoint_data_view.ex +++ b/core/systems/storage/endpoint_data_view.ex @@ -12,16 +12,40 @@ defmodule Systems.Storage.EndpointDataView do endpoint: endpoint, files: files ) + |> update_export_button() } end + def update_export_button(%{assigns: %{endpoint: %{id: id}}} = socket) do + export_button = %{ + action: %{ + type: :http_get, + to: ~p"/storage/#{id}/export", + target: "_blank" + }, + face: %{ + type: :label, + label: dgettext("eyra-storage", "export.files.button"), + icon: :export + } + } + + assign(socket, export_button: export_button) + end + @impl true def render(assigns) do ~H"""
- <%= dgettext("eyra-storage", "tabbar.item.data") %> <%= Enum.count(@files) %> +
+ <%= dgettext("eyra-storage", "tabbar.item.data") %> <%= Enum.count(@files) %> +
+
+ +
+
<%= if not Enum.empty?(@files) do %> <.spacing value="L" /> diff --git a/core/test/core_web/controllers/push_subscription_controller_test.exs b/core/test/core_web/controllers/push_subscription_controller_test.exs deleted file mode 100644 index 83d52dd8b..000000000 --- a/core/test/core_web/controllers/push_subscription_controller_test.exs +++ /dev/null @@ -1,38 +0,0 @@ -defmodule CoreWeb.PushSubscriptionControllerTest do - use CoreWeb.ConnCase - - alias Core.WebPush.PushSubscription - - @valid_attrs %{ - endpoint: "some endpoint", - expirationTime: 42, - keys: %{ - auth: "some auth", - p256dh: "some p256dh" - } - } - - setup %{conn: conn} do - {:ok, conn: put_req_header(conn, "accept", "application/json")} - end - - describe "register" do - setup [:login_as_member] - - test "register a subscription", %{conn: conn, user: user} do - conn = post(conn, ~p"/web-push/register", subscription: @valid_attrs) - - assert json_response(conn, 200) == %{} - assert Core.Repo.get_by(PushSubscription, user_id: user.id) != nil - end - end - - describe "vapid_public_key" do - setup [:login_as_member] - - test "return the public key", %{conn: conn} do - conn = get(conn, ~p"/web-push/vapid-public-key") - assert is_binary(response(conn, 200)) - end - end -end