diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1be3e1..dd3453d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,10 +7,6 @@ on: jobs: build-lint-test: runs-on: ubuntu-latest - env: - # This needs to be defined for the build to succeed, though it is not - # actually used for anything. - ZOEKT_URL: "https://www.example.com" steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 diff --git a/.gitignore b/.gitignore index 0dc1049..a2be540 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ node_modules/ .svelte-kit build/ .env +.envrc diff --git a/README.md b/README.md index bc75053..0bce923 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,27 @@ There is a [demo deployment](./demo) at https://neogrok-demo-web.fly.dev/. This deployment's configuration can serve as a guide for your own deployments of neogrok; currently there are no packaged distributions. +## Configuration + +Neogrok may be [configured](./src/lib/server/configuration.ts) using a JSON +configuration file, or, where possible, environment variables. Configuration +options defined in the environment take precedence over those defined in the +configuration file. + +The configuration file is read from `/etc/neogrok/configuration.json` by +default, but the location may be customized using the environment variable +`NEOGROK_CONFIG_FILE`. The file is entirely optional, as all of the required +configuration may be defined in the environment. If it is present, the file's +contents must be a JSON object with zero or more entires, whose keys correspond +to the option names tabulated below. + +### Configuration options + +| Option name | Environment variable name | Required Y/N | Description | +| :------------------------ | :------------------------ | :----------- | :----------------------------------------------------------------------------------------------------------------------------------- | +| `zoektUrl` | `ZOEKT_URL` | Y | The base zoekt URL at which neogrok will make API requests, at e.g. `/api/search` and `/api/list` | +| `openGrokProjectMappings` | N/A | N | An object mapping OpenGrok project names to zoekt repository names; see [below]('#renaming-opengrok-projects-to-zoekt-repositories') | + ## OpenGrok compatibility As an added bonus, neogrok can serve as a replacement for existing deployments @@ -15,15 +36,15 @@ of [OpenGrok](https://oracle.github.io/opengrok/), a much older, more intricate, slower, and generally jankier code search engine than zoekt. Neogrok strives to provide URL compatibility with OpenGrok by redirecting OpenGrok URLs to their neogrok equivalents: simply deploy neogrok at the same origin previously hosting -your OpenGrok instance, and everything will Just Work™. (Perfect compatibility -is not possible as the feature sets of each search engine do not map -one-to-one.) +your OpenGrok instance, and everything will Just Work™. To the best of our +ability, OpenGrok Lucene queries will be rewritten to the most possibly +equivalent zoekt queries. (Perfect compatibility is not possible as the feature +sets of each search engine do not map one-to-one.) -### Renaming OpenGrok projects to zoekt repository +### Renaming OpenGrok projects to zoekt repositories If your OpenGrok project names are not identical to their equivalent zoekt -repository names, you can run `neogrok` with an environment variable -`OPENGROK_PROJECT_MAPPINGS_FILE` that points at a JSON file whose contents are -an object mapping OpenGrok project names to zoekt repository names. With this -data provided, neogrok can rewrite OpenGrok queries that include project names -appropriately. +repository names, you can run `neogrok` with the appropriate +[`openGrokProjectMappings` configuration](#configuration), which maps OpenGrok +project names to zoekt repository names. With this data provided, neogrok can +rewrite OpenGrok queries that include project names appropriately. diff --git a/demo/fly.neogrok.toml b/demo/fly.neogrok.toml index 123395e..6653e30 100644 --- a/demo/fly.neogrok.toml +++ b/demo/fly.neogrok.toml @@ -1,15 +1,12 @@ -# fly.toml file generated for neogrok-demo-web on 2023-01-17T03:13:51-05:00 - app = "neogrok-demo-web" primary_region = "ewr" [build] dockerfile = "Dockerfile.neogrok" -[build.args] - ZOEKT_URL = "http://neogrok-demo-zoekt.internal:8080" [env] PORT = 8080 + ZOEKT_URL = "http://neogrok-demo-zoekt.internal:8080" [experimental] auto_rollback = true diff --git a/src/hooks.server.ts b/src/hooks.server.ts new file mode 100644 index 0000000..8f0aec2 --- /dev/null +++ b/src/hooks.server.ts @@ -0,0 +1,17 @@ +import { building } from "$app/environment"; +import { resolveConfiguration } from "$lib/server/configuration"; + +if (!building) { + // Resolve the configuration on startup, such that startup fails if the + // configuration is invalid. + // + // We don't actually use any of the features of this hooks module, other than + // that it is evaluated on startup; other modules are not. + await resolveConfiguration(); +} + +// TODO we should have prom metrics, on, among other things, HTTP requests +// handled, and the `handle` hook would be a good way to do that. + +// TODO SvelteKit logs an error every time anything requests a URL that does not +// map to a route. Bonkers. Silence those by implementing `handleError`. diff --git a/src/lib/server/configuration.ts b/src/lib/server/configuration.ts new file mode 100644 index 0000000..ab7e78d --- /dev/null +++ b/src/lib/server/configuration.ts @@ -0,0 +1,78 @@ +import * as v from "@badrap/valita"; +import fs from "node:fs"; + +const zoektUrlSchema = v.string().map((u) => new URL(u)); +const fileConfigurationSchema = v.object({ + zoektUrl: zoektUrlSchema.optional(), + openGrokProjectMappings: v + .record(v.string()) + .map((o) => new Map(Object.entries(o))) + .optional(), +}); +type FileConfiguration = v.Infer; +const defaultConfigFilePath = "/etc/neogrok/config.json"; + +const environmentConfigurationSchema = v.object({ + ZOEKT_URL: zoektUrlSchema.optional(), +}); + +type Configuration = { + readonly zoektUrl: URL; + readonly openGrokProjectMappings: ReadonlyMap; +}; + +// We have to export a not-yet-bound `configuration` at module eval time because +// either SvelteKit or Vite is _executing_ this module during the build, for +// reasons that I do not understand. We do not want to enforce required +// configuration options being defined at build time; neogrok does not prerender +// anything. +// +// So, we do not want to resolve the configuration at module eval time. +// +// So, we rely on live export bindings plus a call to `resolveConfiguration` +// from something else at server startup. Anything consuming the actual +// configuration will have to avoid dereferencing it in the module scope. +export let configuration: Configuration; +export const resolveConfiguration: () => Promise = async () => { + const configFilePath = + process.env.NEOGROK_CONFIG_FILE ?? defaultConfigFilePath; + let fileConfig: FileConfiguration | undefined; + try { + fileConfig = fileConfigurationSchema.parse( + JSON.parse(await fs.promises.readFile(configFilePath, "utf8")), + { mode: "strict" } + ); + } catch (e) { + // Swallow errors related to the default config file being missing. + if ( + !( + e && + typeof e === "object" && + "code" in e && + e.code === "ENOENT" && + configFilePath === defaultConfigFilePath + ) + ) { + throw new Error(`Configuration file at ${configFilePath} is invalid`, { + cause: e, + }); + } + } + + const environmentConfig = environmentConfigurationSchema.parse(process.env, { + mode: "strip", + }); + + const zoektUrl = environmentConfig.ZOEKT_URL ?? fileConfig?.zoektUrl; + if (zoektUrl === undefined) { + throw new Error( + `"ZOEKT_URL" must be defined in the environment, or "zoektUrl" must be defined in the configuration file at ${configFilePath}` + ); + } + + configuration = { + zoektUrl, + openGrokProjectMappings: + fileConfig?.openGrokProjectMappings ?? new Map(), + }; +}; diff --git a/src/lib/server/opengrok-compat.ts b/src/lib/server/opengrok-compat.ts deleted file mode 100644 index 63b1393..0000000 --- a/src/lib/server/opengrok-compat.ts +++ /dev/null @@ -1,25 +0,0 @@ -import * as v from "@badrap/valita"; -import fs from "node:fs"; - -// TODO we should have one environment variable pointing at a single -// configuration file, so that we can extend the configuration later without -// introducing new env vars. - -/** - * Users may provide a simple JSON file that maps from OpenGrok project names to - * Zoekt repo names, in case project names changed. - */ -export const projectToRepo = await (async function () { - if (process.env.OPENGROK_PROJECT_MAPPINGS_FILE) { - const projectMappingSchema = v.record(v.string()); - const raw = JSON.parse( - await fs.promises.readFile( - process.env.OPENGROK_PROJECT_MAPPINGS_FILE, - "utf8" - ) - ); - return new Map(Object.entries(projectMappingSchema.parse(raw))); - } else { - return new Map(); - } -})(); diff --git a/src/lib/server/search-api.ts b/src/lib/server/search-api.ts index 4d861c3..60d60dd 100644 --- a/src/lib/server/search-api.ts +++ b/src/lib/server/search-api.ts @@ -1,6 +1,6 @@ import * as v from "@badrap/valita"; import type { ReadonlyDeep } from "type-fest"; -import { ZOEKT_URL } from "$env/static/private"; +import { configuration } from "./configuration"; import { type ContentToken, parseChunkMatch, @@ -45,7 +45,7 @@ export const search = async ( }, }); - const response = await f(new URL("/api/search", ZOEKT_URL), { + const response = await f(new URL("/api/search", configuration.zoektUrl), { method: "POST", headers: { "content-type": "application/json", diff --git a/src/lib/server/zoekt-list-repositories.ts b/src/lib/server/zoekt-list-repositories.ts index 95cb17e..4e4e264 100644 --- a/src/lib/server/zoekt-list-repositories.ts +++ b/src/lib/server/zoekt-list-repositories.ts @@ -1,6 +1,6 @@ import * as v from "@badrap/valita"; import type { ReadonlyDeep } from "type-fest"; -import { ZOEKT_URL } from "$env/static/private"; +import { configuration } from "./configuration"; export type ListRepositoriesResponse = | { @@ -18,7 +18,7 @@ export async function listRepositories( ): Promise { const body = JSON.stringify({ q: query }); - const response = await f(new URL("/api/list", ZOEKT_URL), { + const response = await f(new URL("/api/list", configuration.zoektUrl), { method: "POST", headers: { "content-type": "application/json", diff --git a/src/routes/(opengrok-compat)/diff/[project]/[...file]/+page.server.ts b/src/routes/(opengrok-compat)/diff/[project]/[...file]/+page.server.ts index f519caa..c0b517b 100644 --- a/src/routes/(opengrok-compat)/diff/[project]/[...file]/+page.server.ts +++ b/src/routes/(opengrok-compat)/diff/[project]/[...file]/+page.server.ts @@ -1,5 +1,5 @@ import { escapeRegExp } from "$lib/regexp"; -import { projectToRepo } from "$lib/server/opengrok-compat"; +import { configuration } from "$lib/server/configuration"; import { listRepositories } from "$lib/server/zoekt-list-repositories"; import { error, redirect } from "@sveltejs/kit"; @@ -14,7 +14,7 @@ export const load: import("./$types").PageServerLoad = async ({ throw error(404); } const revision = url.searchParams.get("r"); - const convertedRepo = projectToRepo.get(project); + const convertedRepo = configuration.openGrokProjectMappings.get(project); const result = await listRepositories( { diff --git a/src/routes/(opengrok-compat)/download/[project]/[...file]/+page.server.ts b/src/routes/(opengrok-compat)/download/[project]/[...file]/+page.server.ts index f519caa..c0b517b 100644 --- a/src/routes/(opengrok-compat)/download/[project]/[...file]/+page.server.ts +++ b/src/routes/(opengrok-compat)/download/[project]/[...file]/+page.server.ts @@ -1,5 +1,5 @@ import { escapeRegExp } from "$lib/regexp"; -import { projectToRepo } from "$lib/server/opengrok-compat"; +import { configuration } from "$lib/server/configuration"; import { listRepositories } from "$lib/server/zoekt-list-repositories"; import { error, redirect } from "@sveltejs/kit"; @@ -14,7 +14,7 @@ export const load: import("./$types").PageServerLoad = async ({ throw error(404); } const revision = url.searchParams.get("r"); - const convertedRepo = projectToRepo.get(project); + const convertedRepo = configuration.openGrokProjectMappings.get(project); const result = await listRepositories( { diff --git a/src/routes/(opengrok-compat)/history/[project]/[...file]/+page.server.ts b/src/routes/(opengrok-compat)/history/[project]/[...file]/+page.server.ts index 5a92498..6ee97fd 100644 --- a/src/routes/(opengrok-compat)/history/[project]/[...file]/+page.server.ts +++ b/src/routes/(opengrok-compat)/history/[project]/[...file]/+page.server.ts @@ -1,5 +1,5 @@ import { escapeRegExp } from "$lib/regexp"; -import { projectToRepo } from "$lib/server/opengrok-compat"; +import { configuration } from "$lib/server/configuration"; import { listRepositories } from "$lib/server/zoekt-list-repositories"; import { redirect } from "@sveltejs/kit"; @@ -9,7 +9,7 @@ export const load: import("./$types").PageServerLoad = async ({ setHeaders, fetch, }) => { - const convertedRepo = projectToRepo.get(project); + const convertedRepo = configuration.openGrokProjectMappings.get(project); const result = await listRepositories( { diff --git a/src/routes/(opengrok-compat)/raw/[project]/[...file]/+page.server.ts b/src/routes/(opengrok-compat)/raw/[project]/[...file]/+page.server.ts index f519caa..c0b517b 100644 --- a/src/routes/(opengrok-compat)/raw/[project]/[...file]/+page.server.ts +++ b/src/routes/(opengrok-compat)/raw/[project]/[...file]/+page.server.ts @@ -1,5 +1,5 @@ import { escapeRegExp } from "$lib/regexp"; -import { projectToRepo } from "$lib/server/opengrok-compat"; +import { configuration } from "$lib/server/configuration"; import { listRepositories } from "$lib/server/zoekt-list-repositories"; import { error, redirect } from "@sveltejs/kit"; @@ -14,7 +14,7 @@ export const load: import("./$types").PageServerLoad = async ({ throw error(404); } const revision = url.searchParams.get("r"); - const convertedRepo = projectToRepo.get(project); + const convertedRepo = configuration.openGrokProjectMappings.get(project); const result = await listRepositories( { diff --git a/src/routes/(opengrok-compat)/search/+page.server.ts b/src/routes/(opengrok-compat)/search/+page.server.ts index 790d1c0..c2c4651 100644 --- a/src/routes/(opengrok-compat)/search/+page.server.ts +++ b/src/routes/(opengrok-compat)/search/+page.server.ts @@ -5,7 +5,7 @@ import { } from "./opengrok-lucene.server"; import { redirect } from "@sveltejs/kit"; import { listRepositories } from "$lib/server/zoekt-list-repositories"; -import { projectToRepo } from "$lib/server/opengrok-compat"; +import { configuration } from "$lib/server/configuration"; export const load: import("./$types").PageServerLoad = async ({ url, @@ -46,7 +46,7 @@ export const load: import("./$types").PageServerLoad = async ({ }; const { luceneQuery, zoektQuery, warnings } = await toZoekt(params, { - projectToRepo, + projectToRepo: configuration.openGrokProjectMappings, queryUnknownRepos, }); diff --git a/src/routes/(opengrok-compat)/xref/[project]/[...file]/+page.server.ts b/src/routes/(opengrok-compat)/xref/[project]/[...file]/+page.server.ts index d36ac49..56c4992 100644 --- a/src/routes/(opengrok-compat)/xref/[project]/[...file]/+page.server.ts +++ b/src/routes/(opengrok-compat)/xref/[project]/[...file]/+page.server.ts @@ -1,5 +1,5 @@ import { escapeRegExp } from "$lib/regexp"; -import { projectToRepo } from "$lib/server/opengrok-compat"; +import { configuration } from "$lib/server/configuration"; import { listRepositories } from "$lib/server/zoekt-list-repositories"; import { redirect } from "@sveltejs/kit"; @@ -11,7 +11,7 @@ export const load: import("./$types").PageServerLoad = async ({ fetch, }) => { const revision = url.searchParams.get("r"); - const convertedRepo = projectToRepo.get(project); + const convertedRepo = configuration.openGrokProjectMappings.get(project); const result = await listRepositories( {