improvement: rewrite package search and storage to use sqlite

improvement: clean up a bunch of unused code

.formatter.exs

@@ -4,6 +4,7 @@
     :phoenix,
     :ash,
     :ash_postgres,
+    :ash_sqlite,
     :ash_graphql,
     :surface,
     :ash_admin,
@@ -23,7 +24,6 @@
   locals_without_parens: [
     has_name_attribute?: 1,
     name_attribute: 1,
-    library_version_attribute: 1,
     load_for_search: 1,
     doc_attribute: 1,
     render_attributes: 1,

.gitignore

@@ -35,6 +35,7 @@ npm-debug.log
 /assets/node_modules/
 
 .elixir_ls
+ash-hq.db*
 
 /indexes/*
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM hexpm/elixir:1.15.4-erlang-26.0.2-ubuntu-bionic-20230126
+FROM hexpm/elixir:1.15.4-erlang-26.0.2-ubuntu-focal-20230126
 # install build dependencies
 USER root
 RUN apt-get update
@@ -13,9 +13,13 @@ RUN apt-get install -y g++
 RUN apt-get install -y make
 RUN apt-get install -y curl
 RUN apt-get install -y build-essential
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get install -y esl-erlang
 RUN apt-get install -y apt-transport-https
 RUN apt-get install -y ca-certificates
+RUN apt-get install -y fuse3 libfuse3-dev libglib2.0-dev
+RUN apt-get install -y sqlite3
+COPY --from=flyio/litefs:0.5 /usr/local/bin/litefs /usr/local/bin/litefs
 ENV NODE_MAJOR=16
 RUN mkdir -p /etc/apt/keyrings
 RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
@@ -45,4 +49,5 @@ COPY ./config/runtime.exs config/runtime.exs
 COPY ./rel ./rel
 RUN mix release --overwrite
 RUN mkdir indexes
-CMD ["_build/prod/rel/ash_hq/bin/ash_hq", "start"]
+COPY ./litefs.yml ./litefs.yml
+ENTRYPOINT litefs mount

config/config.exs

@@ -8,7 +8,7 @@
 import Config
 
 config :ash_hq,
-  ecto_repos: [AshHq.Repo]
+  ecto_repos: [AshHq.Repo, AshHq.SqliteRepo]
 
 config :ash, allow_flow: true
 
@@ -27,9 +27,7 @@ config :appsignal, :config, revision: "test-4"
 config :ash_hq,
   ash_apis: [
     AshHq.Accounts,
-    AshHq.Ashley,
     AshHq.Blog,
-    AshHq.Discord,
     AshHq.Docs,
     AshHq.Github,
     AshHq.MailingList
@@ -88,8 +86,6 @@ config :esbuild,
     env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)}
   ]
 
-config :open_ai, :http_client_impl, AshHq.Ashley.HttpClient
-
 # Configures Elixir's Logger
 config :logger, :console,
   format: "$time $metadata[$level] $message\n",

config/dev.exs

@@ -22,6 +22,12 @@ config :git_ops,
   manage_readme_version: "README.md",
   version_tag_prefix: "v"
 
+config :ash_hq, AshHq.SqliteRepo,
+  database: Path.join(__DIR__, "../ash-hq.db"),
+  port: 5432,
+  show_sensitive_data_on_connection_error: true,
+  pool_size: 10
+
 config :ash_hq, :show_search_ranking, true
 
 secret_key_base = "FxKFwVYhDFah3bLLXXqWdpdcLf5e5T1UyVM6XQp7kCt/Reg5yuAEI3upAVDRoP5e"

config/prod.exs

@@ -20,6 +20,12 @@ config :ash_hq, :analytics?, true
 
 config :ash_hq, :download_ua_on_start, true
 
+if config_env() == :prod do
+  config :ash_hq, AshHq.SqliteRepo,
+    database: "/litefs/db",
+    pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10")
+end
+
 # ## SSL Support
 #
 # To get SSL working, you will need to add the `https` key

config/runtime.exs

@@ -12,9 +12,6 @@ if System.get_env("PHX_SERVER") && System.get_env("RELEASE_NAME") do
   config :ash_hq, AshHqWeb.Endpoint, server: true
 end
 
-config :open_ai,
-  api_key: System.get_env("OPEN_API_API_KEY")
-
 config :ash_hq, :github,
   api_key: System.get_env("GITHUB_API_KEY"),
   client_id: System.get_env("GITHUB_CLIENT_ID"),

config/test.exs

@@ -24,6 +24,10 @@ config :ash_hq, AshHqWeb.Endpoint,
   secret_key_base: secret_key_base,
   server: false
 
+config :ash_hq, AshHq.SqliteRepo,
+  database: Path.join(__DIR__, "../ash-hq#{System.get_env("MIX_TEST_PARTITION")}.db"),
+  pool_size: 10
+
 config :ash_hq, cloak_key: "J6ED3yBWjlaOW/5byrukZTEryKa++yXWblJuhP91Qq8="
 
 # In test we don't send emails.

fly.toml

@@ -6,12 +6,13 @@ kill_signal = "SIGINT"
 kill_timeout = 5
 processes = []
 
-[deploy]
-  release_command = "_build/prod/rel/ash_hq/bin/ash_hq eval 'AshHq.Release.migrate'"
-
 [env]
   RELEASE_COOKIE = "VsipafjUVIYVpiYiljPg6DNZB8XiSnEf4zLi8WOf9bAU0XK7HuHQqA=="
 
+[mounts]
+source = "litefs"
+destination = "/var/lib/litefs"
+
 [[services]]
   internal_port = 4000
   protocol = "tcp"

lib/ash_hq/accounts/resources/user/user.ex

@@ -192,10 +192,6 @@ defmodule AshHq.Accounts.User do
     attribute :shirt_size, :string
     attribute :github_info, :map
 
-    attribute :ashley_access, :boolean do
-      default false
-    end
-
     create_timestamp :created_at
     update_timestamp :updated_at
   end

lib/ash_hq/application.ex

@@ -34,6 +34,7 @@ defmodule AshHq.Application do
         AshHq.Vault,
         # Start the Ecto repository
         AshHq.Repo,
+        AshHq.SqliteRepo,
         # Start the Telemetry supervisor
         AshHqWeb.Telemetry,
         # Start the PubSub system
@@ -42,6 +43,8 @@ defmodule AshHq.Application do
         AshHqWeb.Endpoint,
         {AshHq.Docs.Library.Agent, nil},
         {Cluster.Supervisor, [topologies, [name: AshHq.ClusterSupervisor]]},
+        {Haystack.Storage.ETS, storage: AshHq.Docs.Indexer.storage()},
+        AshHq.Docs.Indexer,
         AshHq.Github.Monitor
         # Start a worker by calling: AshHq.Worker.start_link(arg)
         # {AshHq.Worker, arg}