Merge pull request #3017 from OpenNeuroOrg/feature/deno-cli-uploader

Deno command line tools

.github/workflows/deno.yml

@@ -0,0 +1,35 @@
+---
+name: Deno build
+
+on:
+  push:
+    branches: [master]
+    tags: ['*']
+  pull_request:
+    branches: [master]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    strategy:
+      matrix:
+        os: [ubuntu-22.04, macos-12, windows-2022]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: denoland/setup-deno@v1.1.2
+        with:
+          deno-version: v1.x
+      - name: Collect coverage
+        run: deno task coverage
+        if: ${{ always() }}
+      - uses: codecov/codecov-action@v4
+        if: ${{ always() }}
+        with:
+          files: coverage.lcov

.gitignore

@@ -20,3 +20,5 @@ dist
 !.yarn/versions
 .*.sw[po]
 .venv/
+coverage
+coverage.lcov

cli/README.md

@@ -0,0 +1,45 @@
+# OpenNeuro CLI for Deno
+
+Command line tools for OpenNeuro implemented in Deno. Deno eliminates the need to install the CLI and allows for more code reuse with OpenNeuro's web frontend.
+
+## Install
+
+Download deno via [any supported installation method](https://docs.deno.com/runtime/manual/getting_started/installation).
+
+## Usage
+
+OpenNeuro CLI will validate your dataset with the [bids-validator](https://github.com/bids-standard/bids-validator/) and then allow you to upload to OpenNeuro. If you wish to make changes to a dataset, the CLI can download, allow you to make local changes, and reupload only the changes to OpenNeuro.
+
+### Login
+
+To upload or download data from OpenNeuro, login with your account.
+
+```shell
+# Run login and follow the prompts
+deno run -A cli/openneuto.ts login
+```
+
+You can also create an API key on [OpenNeuro](https://openneuro.org/keygen) and specify this as an option or environment variable.
+
+```shell
+# For scripts
+export OPENNEURO_API_KEY=<api_key>
+deno run -A cli/openneuro.ts login --error-reporting true
+```
+
+### Uploading
+
+```shell
+# Path to the dataset root (directory containing dataset_description.json)
+deno run -A cli/openneuro.ts upload --affirmDefaced path/to/dataset
+```
+
+```shell
+# To debug issues - enable logging and provide this log to support or open a GitHub issue
+export OPENNEURO_LOG=INFO
+deno run -A cli/openneuro.ts upload --affirmDefaced path/to/dataset
+```
+
+## Implementation Notes
+
+This tool uses isomorphic git to download, modify, and push datasets using OpenNeuro's [git interface](https://docs.openneuro.org/git.html). Other tools that support git and git-annex repositories such as [DataLad](https://www.datalad.org/) can also be used with the local copy.

cli/openneuro.ts

@@ -0,0 +1,10 @@
+/**
+ * Entrypoint for OpenNeuro CLI
+ */
+import { commandLine } from "./src/options.ts"
+
+export async function main() {
+  await commandLine(Deno.args)
+}
+
+await main()

cli/src/bids_validator.ts

@@ -0,0 +1,5 @@
+// TODO - Switch to upstream after next release
+export { validateCommand } from "https://raw.githubusercontent.com/bids-standard/bids-validator/master/bids-validator/src/setup/options.ts"
+export { validate } from "https://deno.land/x/bids_validator@v1.14.0/main.ts"
+export { readFileTree } from "https://deno.land/x/bids_validator@v1.14.0/files/deno.ts"
+export { consoleFormat } from "https://deno.land/x/bids_validator@v1.14.0/utils/output.ts"

cli/src/commands/download.ts

@@ -0,0 +1,54 @@
+import { Command } from "../deps.ts"
+import { readConfig } from "../config.ts"
+import { logger } from "../logger.ts"
+import { getRepoAccess } from "./git-credential.ts"
+
+export const download = new Command()
+  .name("download")
+  .description("Download a dataset from OpenNeuro")
+  .arguments("<accession_number> <download_directory>")
+  .option(
+    "-d, --draft",
+    "Download a draft instead of the latest version snapshot.",
+  )
+  .option(
+    "-v, --version",
+    "Download a specific version.",
+  )
+  .action(downloadAction)
+
+export async function downloadAction(
+  options: CommandOptions,
+  accession_number: string,
+  download_directory: string,
+) {
+  const datasetId = accession_number
+  const clientConfig = readConfig()
+  const { token, endpoint } = await getRepoAccess(datasetId)
+
+  // Create the git worker
+  const worker = new Worker(new URL("../worker/git.ts", import.meta.url).href, {
+    type: "module",
+  })
+
+  // Configure worker
+  worker.postMessage({
+    "command": "setup",
+    "datasetId": datasetId,
+    "repoPath": download_directory,
+    "repoEndpoint": `${clientConfig.url}/git/${endpoint}/${datasetId}`,
+    "authorization": token,
+    "logLevel": logger.levelName,
+  })
+
+  console.log("Downloading...")
+
+  worker.postMessage({
+    "command": "clone",
+  })
+
+  // Close after all tasks are queued
+  worker.postMessage({ command: "done" })
+
+  console.log("Download complete. To download all data files, use `datalad get` or `git-annex get`.")
+}

cli/src/commands/git-credential.test.ts

@@ -0,0 +1,20 @@
+import { assertEquals } from "../deps.ts"
+import { gitCredentialAction } from "./git-credential.ts"
+
+Deno.test("git-credential parses stdin correctly", async () => {
+  const stdin = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(
+        new TextEncoder().encode(
+          "host=staging.openneuro.org\nprotocol=https\npath=/datasets/ds000001\n",
+        ),
+      )
+      controller.close()
+    },
+  })
+  const output = await gitCredentialAction(
+    stdin,
+    async () => ({ token: "token", endpoint: 2 }),
+  )
+  assertEquals(output, "username=@openneuro/cli\npassword=token\n")
+})

cli/src/commands/git-credential.ts

@@ -0,0 +1,93 @@
+import { Command, TextLineStream } from "../deps.ts"
+import { getConfig } from "../config.ts"
+
+const prepareRepoAccess = `
+  mutation prepareRepoAccess($datasetId: ID!) {
+    prepareRepoAccess(datasetId: $datasetId) {
+      token
+      endpoint
+    }
+  }
+`
+
+interface GraphQLError {
+  message: string
+  locations: { line: number, column: number }[]
+  path: string[]
+  extensions: {
+    code: string,
+    stacktrace: string[]
+  }
+}
+
+export async function getRepoAccess(datasetId?: string) {
+  const config = getConfig()
+  const req = await fetch(`${config.url}/crn/graphql`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Authorization": `Bearer ${config.token}`, // Long lived token
+    },
+    body: JSON.stringify({
+      query: prepareRepoAccess,
+      variables: {
+        datasetId,
+      },
+    }),
+  })
+  const response = await req.json()
+  if (response.errors) {
+    const errors: GraphQLError[] = response.errors
+    throw Error(errors.map(error => error.message).toString())
+  } else {
+    return {
+      token: response.data.prepareRepoAccess.token, // Short lived repo access token
+      endpoint: response.data.prepareRepoAccess.endpoint,
+    }
+  }
+}
+
+/**
+ * Provide a git-credential helper for OpenNeuro
+ */
+export async function gitCredentialAction(
+  stdinReadable: ReadableStream<Uint8Array> = Deno.stdin.readable,
+  tokenGetter = getRepoAccess,
+) {
+  let pipeOutput = ""
+  const credential: Record<string, string | undefined> = {}
+  // Create a stream of lines from stdin
+  const lineStream = stdinReadable
+    .pipeThrough(new TextDecoderStream())
+    .pipeThrough(new TextLineStream())
+  for await (const line of lineStream) {
+    const [key, value] = line.split("=", 2)
+    credential[key] = value
+  }
+  if ("path" in credential && credential.path) {
+    const datasetId = credential.path.split("/").pop()
+    const { token } = await tokenGetter(datasetId)
+    const output: Record<string, string> = {
+      username: "@openneuro/cli",
+      password: token,
+    }
+    for (const key in output) {
+      pipeOutput += `${key}=${output[key]}\n`
+    }
+  } else {
+    throw new Error(
+      "Invalid input from git, check the credential helper is configured correctly",
+    )
+  }
+  return pipeOutput
+}
+
+export const gitCredential = new Command()
+  .name("git-credential")
+  .description(
+    "A git credentials helper for easier datalad or git-annex access to datasets.",
+  )
+  .command("fill")
+  .action(async () => {
+    console.log(await gitCredentialAction())
+  })

cli/src/commands/login.test.ts

@@ -0,0 +1,33 @@
+import { assertEquals, assertSpyCalls, Select, stub } from "../deps.ts"
+import { loginAction } from "./login.ts"
+
+Deno.test("login action supports non-interactive mode if all options are provided", async () => {
+  const SelectStub = stub(Select, "prompt", () => {
+    return new Promise<void>(() => {})
+  })
+  await loginAction({
+    url: "https://example.com",
+    token: "1234",
+    errorReporting: false,
+  })
+  // Test to make sure we get here before the timeout
+  assertSpyCalls(SelectStub, 0)
+  SelectStub.restore()
+  localStorage.clear()
+})
+
+Deno.test("login action sets values in localStorage", async () => {
+  const loginOptions = {
+    url: "https://example.com",
+    token: "1234",
+    errorReporting: true,
+  }
+  await loginAction(loginOptions)
+  assertEquals(localStorage.getItem("url"), loginOptions.url)
+  assertEquals(localStorage.getItem("token"), loginOptions.token)
+  assertEquals(
+    localStorage.getItem("errorReporting"),
+    loginOptions.errorReporting.toString(),
+  )
+  localStorage.clear()
+})

cli/src/commands/login.ts

@@ -0,0 +1,54 @@
+/**
+ * Configure credentials and other persistent settings for OpenNeuro
+ */
+import { Command, Confirm, Secret, Select } from "../deps.ts"
+import type { CommandOptions } from "../deps.ts"
+
+const messages = {
+  url:
+    "URL for OpenNeuro instance to upload to (e.g. `https://openneuro.org`).",
+  token: "API key for OpenNeuro. See https://openneuro.org/keygen",
+  errorReporting:
+    "Enable error reporting. Errors and performance metrics are sent to the configured OpenNeuro instance.",
+}
+
+export async function loginAction(options: CommandOptions) {
+  const url = options.url ? options.url : await Select.prompt({
+    message: "Choose an OpenNeuro instance to use.",
+    options: [
+      "https://openneuro.org",
+      "https://staging.openneuro.org",
+      "http://localhost:9876",
+    ],
+  })
+  localStorage.setItem("url", url)
+  let token
+  // Environment variable
+  if (options.openneuroApiKey) {
+    token = options.openneuroApiKey
+  }
+  // Command line
+  if (options.token) {
+    token = options.token
+  }
+  if (!token) {
+    token = await Secret.prompt(
+      `Enter your API key for OpenNeuro (get an API key from ${url}/keygen).`,
+    )
+  }
+  localStorage.setItem("token", token)
+  const errorReporting = Object.hasOwn(options, "errorReporting")
+    ? options.errorReporting
+    : await Confirm.prompt(messages.errorReporting)
+  localStorage.setItem("errorReporting", errorReporting.toString())
+}
+
+export const login = new Command()
+  .name("login")
+  .description(
+    "Setup credentials for OpenNeuro. Set -u, -t, and -e flags to skip interactive prompts.",
+  )
+  .option("-u, --url <url>", messages.url)
+  .option("-t, --token <token>", messages.token)
+  .option("-e, --error-reporting <boolean>", messages.errorReporting)
+  .action(loginAction)