Merge pull request #495 from eyra/logo-upload-s3

Implemented user content storage on S3 iso local FS

core/config/dev.exs

@@ -85,6 +85,15 @@ config :core,
 #   access_key_id: "my_access_key",
 #   secret_access_key: "a_super_secret"
 
+config :core, :content,
+  backend: Systems.Content.LocalFS,
+  local_fs_root_path:
+    File.cwd!()
+    |> Path.join("priv")
+    |> Path.join("static")
+    |> Path.join("content")
+    |> tap(&File.mkdir_p!/1)
+
 config :core, :feldspar,
   backend: Systems.Feldspar.LocalFS,
   local_fs_root_path:

core/config/runtime.exs

@@ -123,13 +123,19 @@ if config_env() == :prod do
     dsn: System.get_env("SENTRY_DSN"),
     environment_name: System.get_env("RELEASE_ENV") || "prod"
 
+  config :core, :content,
+    backend: Systems.Content.S3,
+    bucket: System.get_env("PUBLIC_S3_BUCKET"),
+    public_url: System.get_env("PUBLIC_S3_URL"),
+    prefix: System.get_env("CONTENT_S3_PREFIX", nil)
+
   config :core, :feldspar,
     backend: Systems.Feldspar.S3,
-    bucket: System.get_env("FELDSPAR_S3_BUCKET"),
-    prefix: System.get_env("FELDSPAR_S3_PREFIX", nil),
+    bucket: System.get_env("PUBLIC_S3_BUCKET"),
     # The public URL must point to the root's (bucket) publicly accessible URL.
     # It should have a policy that allows anonymous users to read all files.
-    public_url: System.get_env("FELDSPAR_S3_PUBLIC_URL")
+    public_url: System.get_env("PUBLIC_S3_URL"),
+    prefix: System.get_env("FELDSPAR_S3_PREFIX", nil)
 
   config :core,
          :dist_hosts,

core/config/test.exs

@@ -55,6 +55,10 @@ config :core, :bundle, :next
 
 config :core, :banking_backend, Systems.Banking.Dummy
 
+config :core, :content,
+  backend: Systems.Content.LocalFS,
+  local_fs_root_path: "/tmp"
+
 config :core, :feldspar,
   backend: Systems.Feldspar.LocalFS,
   local_fs_root_path: "/tmp"

core/lib/core_web/endpoint.ex

@@ -1,5 +1,6 @@
 defmodule CoreWeb.Endpoint do
   use Phoenix.Endpoint, otp_app: :core
+  require Systems.Content.Plug
   require Systems.Feldspar.Plug
 
   # The session will be stored in the cookie and signed,
@@ -34,6 +35,7 @@ defmodule CoreWeb.Endpoint do
     )
   end
 
+  Systems.Content.Plug.setup()
   Systems.Feldspar.Plug.setup()
 
   # Serve at "/" the static files from "priv/static" directory.

core/systems/assignment/info_form.ex

@@ -11,13 +11,19 @@ defmodule Systems.Assignment.InfoForm do
   alias CoreWeb.UI.ImageCatalogPicker
 
   alias Systems.Assignment
+  alias Systems.Content
 
   @impl true
   def process_file(
         %{assigns: %{entity: entity}} = socket,
-        {local_relative_path, _local_full_path, _remote_file}
+        {_local_relative_path, local_full_path, _remote_file}
       ) do
-    save(socket, entity, :auto_save, %{logo_url: local_relative_path})
+    logo_url =
+      Content.Public.store(local_full_path)
+      |> Content.Public.get_public_url()
+
+    socket
+    |> save(entity, :auto_save, %{logo_url: logo_url})
   end
 
   @impl true

core/systems/content/_private.ex

@@ -0,0 +1,7 @@
+defmodule Systems.Content.Private do
+  def get_backend do
+    :core
+    |> Application.fetch_env!(:content)
+    |> Access.fetch!(:backend)
+  end
+end

core/systems/content/_public.ex

@@ -1,12 +1,24 @@
 defmodule Systems.Content.Public do
   import Ecto.Query, warn: false
 
+  alias Systems.{
+    Content
+  }
+
   alias Core.Repo
   alias Ecto.Multi
 
   alias Systems.Content.TextItemModel, as: TextItem
   alias Systems.Content.TextBundleModel, as: TextBundle
 
+  def store(file) do
+    Content.Private.get_backend().store(file)
+  end
+
+  def get_public_url(id) do
+    Content.Private.get_backend().get_public_url(id)
+  end
+
   def get_text_item!(id, preload \\ []) do
     from(t in TextItem, preload: ^preload)
     |> Repo.get!(id)

core/systems/content/local_fs.ex

@@ -0,0 +1,38 @@
+defmodule Systems.Content.LocalFS do
+  alias CoreWeb.Endpoint
+
+  def store(tmp_path) do
+    uuid = Ecto.UUID.generate()
+    extname = Path.extname(tmp_path)
+    id = "#{uuid}#{extname}"
+    path = get_path(id)
+    File.cp!(tmp_path, path)
+    id
+  end
+
+  def storage_path(id) do
+    get_path(id)
+  end
+
+  def get_public_url(id) do
+    "#{Endpoint.url()}/#{public_path()}/#{id}"
+  end
+
+  def remove(id) do
+    with {:ok, _} <- File.rm_rf(get_path(id)) do
+      :ok
+    end
+  end
+
+  defp get_path(id) do
+    Path.join(get_root_path(), id)
+  end
+
+  def get_root_path do
+    :core
+    |> Application.get_env(:content, [])
+    |> Access.fetch!(:local_fs_root_path)
+  end
+
+  def public_path, do: "/content"
+end

core/systems/content/plug.ex

@@ -0,0 +1,33 @@
+defmodule Systems.Content.Plug do
+  @behaviour Plug
+
+  defmacro setup() do
+    quote do
+      plug(Systems.Content.Plug, at: Systems.Content.LocalFS.public_path())
+    end
+  end
+
+  @impl true
+  def init(opts) do
+    opts
+    # Ensure that init works, from will be set dynamically later on
+    |> Keyword.put(:from, {nil, nil})
+    |> Plug.Static.init()
+  end
+
+  @impl true
+  def call(
+        conn,
+        options
+      ) do
+    call(Systems.Content.Private.get_backend(), conn, options)
+  end
+
+  def call(Systems.Content.LocalFS, conn, options) do
+    root_path = Systems.Content.LocalFS.get_root_path()
+    options = Map.put(options, :from, root_path)
+    Plug.Static.call(conn, options)
+  end
+
+  def call(_, conn, _options), do: conn
+end

core/systems/content/s3.ex

@@ -0,0 +1,59 @@
+defmodule Systems.Content.S3 do
+  alias ExAws.S3
+
+  def store(file) do
+    bucket = Access.fetch!(s3_settings(), :bucket)
+    uuid = Ecto.UUID.generate()
+    extname = Path.extname(file)
+    id = "#{uuid}#{extname}"
+
+    upload_file(file, id, bucket)
+    id
+  end
+
+  def remove(id) do
+    bucket = Access.fetch!(s3_settings(), :bucket)
+    object_key = "#{object_key(id)}"
+
+    S3.delete_object(bucket, object_key)
+    |> backend().request!()
+  end
+
+  def get_public_url(id) do
+    settings = s3_settings()
+    public_url = Access.get(settings, :public_url)
+    "#{public_url}/#{object_key(id)}"
+  end
+
+  defp upload_file(file, id, bucket) do
+    {:ok, data} = File.read(file)
+    object_key = "#{object_key(id)}"
+
+    S3.put_object(
+      bucket,
+      object_key,
+      data,
+      content_type: content_type(file)
+    )
+    |> backend().request!()
+  end
+
+  defp content_type(name), do: MIME.from_path(name)
+
+  defp object_key(id) do
+    prefix = Access.get(s3_settings(), :prefix, nil)
+
+    [prefix, id]
+    |> Enum.filter(&(&1 != nil))
+    |> Enum.join("/")
+  end
+
+  defp s3_settings do
+    Application.fetch_env!(:core, :content)
+  end
+
+  defp backend do
+    # Allow mocking
+    Access.get(s3_settings(), :s3_backend, ExAws)
+  end
+end

core/test/systems/content/local_fs_test.exs

@@ -0,0 +1,31 @@
+defmodule Systems.Content.LocalFSTest do
+  use ExUnit.Case, async: true
+
+  alias Systems.Content.LocalFS
+
+  describe "store/1" do
+    test "extracts stores file on disk" do
+      id = LocalFS.store(Path.join(__DIR__, "hello.svg"))
+      path = LocalFS.storage_path(id)
+      assert File.exists?(path)
+    end
+  end
+
+  describe "get_public_url/1" do
+    test "returns URL" do
+      id = Ecto.UUID.generate()
+      url = LocalFS.get_public_url(id)
+      uri = URI.parse(url)
+      assert String.contains?(uri.path, id)
+    end
+  end
+
+  describe "remove/1" do
+    test "removes folder" do
+      id = LocalFS.store(Path.join(__DIR__, "hello.svg"))
+      path = LocalFS.storage_path(id)
+      assert :ok == LocalFS.remove(id)
+      refute File.exists?(path)
+    end
+  end
+end

core/test/systems/content/plug_test.exs

@@ -0,0 +1,62 @@
+defmodule Systems.Content.PlugTest do
+  use ExUnit.Case
+  use Plug.Test
+
+  require Systems.Content.Plug
+  alias Systems.Content.Plug
+
+  setup do
+    conf = Application.get_env(:core, :content, [])
+
+    on_exit(fn ->
+      Application.put_env(:core, :content, conf)
+    end)
+
+    folder_name = "temp_#{:crypto.strong_rand_bytes(16) |> Base.encode16()}"
+
+    tmp_dir =
+      System.tmp_dir()
+      |> Path.join(folder_name)
+
+    File.mkdir!(tmp_dir)
+
+    on_exit(fn ->
+      File.rm_rf!(tmp_dir)
+    end)
+
+    conf =
+      conf
+      |> Keyword.put(:backend, Systems.Content.LocalFS)
+      |> Keyword.put(:local_fs_root_path, tmp_dir)
+
+    Application.put_env(
+      :core,
+      :content,
+      conf
+    )
+
+    {:ok, tmp_dir: tmp_dir, app_conf: conf}
+  end
+
+  test "call with LocalFS backend serves static content", %{tmp_dir: tmp_dir} do
+    tmp_dir
+    |> Path.join("plug_test.txt")
+    |> File.write("hello world!")
+
+    opts = Plug.init(at: "/content")
+    conn = Plug.call(conn(:get, "/content/plug_test.txt"), opts)
+    assert "hello world!" == conn.resp_body
+  end
+
+  test "call with other backends doesn't serve static content", %{app_conf: conf} do
+    Application.put_env(
+      :core,
+      :feldspar,
+      Keyword.put(conf, :backend, Systems.Content.FakeBackend)
+    )
+
+    opts = Plug.init(at: "/txt")
+    conn = Plug.call(conn(:get, "/txt/plug_test.txt"), opts)
+    assert nil == conn.resp_body
+  end
+end