Import repo

.github/workflows/reference-converter.daily.yml

@@ -0,0 +1,53 @@
+name: Daily Reference Converter Run
+
+on:
+  schedule:
+    - cron: '0 14 * * *'  # Run daily at 2pm UTC
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: ./reference-converter
+    shell: bash
+
+jobs:
+  compare_outputs:
+    runs-on: ubuntu-latest
+    env:
+      converter-directory: ./reference-converter
+
+    steps:
+    - uses: actions/checkout@v3
+    # TODO: reuse the build steps
+    - run: make devtools-image
+    - name: let the build create files
+      run: chmod 777 .
+    - run: make build
+    - name: run converter
+      run: ./dist/reference-converter -dst ./output/reference.json
+
+    - name: diff reference.json
+      id: diff
+      run: |
+        if ! diff -q <(grep -v '"version":' ../reference-lib/src/reference.json) <(grep -v '"version":' ./output/reference.json); then
+          echo "reference_change=true" >> "$GITHUB_OUTPUT"
+          mv ./output/reference.json ../reference-lib/src/reference.json
+        fi
+
+    - uses: actions/setup-node@v3
+    - name: update npm package version
+      if: steps.diff.outputs.reference_change
+      run: npm version patch --no-git-tag-version
+      working-directory: ./reference-lib
+
+    - name: create pull request if reference.json changed
+      uses: peter-evans/create-pull-request@v3
+      if: steps.diff.outputs.reference_change
+      with:
+        commit-message: update reference.json
+        token: ${{ secrets.GITHUB_TOKEN }}
+        branch: reference-update
+        title: "Reference Json Update"
+        body: "Changes detected in reference.json"
+        delete-branch: true
+        add-paths: reference-lib

.github/workflows/reference-converter.yml

@@ -0,0 +1,60 @@
+name: CI
+
+on:
+  pull_request:
+    branches: ["main"]
+    paths:
+      [
+        "reference-converter/**",
+        ".github/workflows/reference-converter.yml",
+        ".golangci.yml",
+      ]
+
+defaults:
+  run:
+    working-directory: ./reference-converter
+    shell: bash
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      working-directory: ./reference-converter
+      testResults: ./reference-converter/results/test.xml
+    steps:
+      # TODO: re-use common/action.yml, need to refactor out some docs-specific steps
+      - uses: actions/checkout@v3
+      # TODO: publish the devtools image to a registry and use it as `jobs.build.container`
+      - run: make devtools-image
+      - name: let the build create files
+        run: chmod 777 .
+      - run: make build
+      - run: make test
+      - name: archive test artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-report
+          path: ${{ env.testResults }}
+      - name: Test Report
+        uses: dorny/test-reporter@v1
+        if: success() || failure()
+        with:
+          name: Test Report
+          path: ${{ env.testResults }}
+          reporter: jest-junit
+  lint:
+    runs-on: ubuntu-latest
+    env:
+      working-directory: ./reference-converter
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: ${{ env.working-directory }}/go.mod
+          cache: false
+      - name: use the same version of GOLANGCI_LINT_VERSION as devtools
+        run: make devtools-versions >> "$GITHUB_ENV"
+      - uses: golangci/golangci-lint-action@v3
+        with:
+          working-directory: ${{ env.working-directory }}
+          version: v${{ env.GOLANGCI_LINT_VERSION }}

.github/workflows/reference-lib.publish.yml

@@ -0,0 +1,37 @@
+name: Publish reference lib package to GitHub Packages
+on:
+  push:
+    branches: ['main']
+    paths: ['reference-lib/**']
+  pull_request:
+    branches: ['main']
+    paths: ['reference-lib/**']
+
+defaults:
+  run:
+    working-directory: ./reference-lib
+    shell: bash
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16.x'
+          registry-url: 'https://npm.pkg.github.com'
+          scope: '@nginxinc'
+      - name: Install dependencies
+        run: npm ci
+      - name: Bundle using rollup
+        run: npm run build
+      - name: publish package
+        if: github.event_name == 'push'
+        run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.golangci.yml

@@ -0,0 +1,12 @@
+linters:
+  # add non-default linters https://golangci-lint.run/usage/linters/#disabled-by-default
+  enable:
+    - bodyclose
+    - errorlint
+
+issues:
+  # Disable maximum issues count per one linter
+  max-issues-per-linter: 0
+
+  # Disable maximum count of issues with the same text
+  max-same-issues: 0

reference-converter/.gitignore

@@ -0,0 +1,2 @@
+results
+dist

reference-converter/Makefile

@@ -0,0 +1,13 @@
+ROOT_DIR:=$(shell git rev-parse --show-toplevel)
+include $(ROOT_DIR)/tools/devtools/Makefile
+
+.PHONY: build test lint
+
+build:
+	@$(MAKE) .run args="go build -v -o dist/ ."
+
+test:
+	@$(MAKE) .run args="./test.sh results/test.xml"
+
+lint:
+	@$(MAKE) .run args="golangci-lint run -v"

reference-converter/README.md

@@ -0,0 +1,42 @@
+# reference-converter
+
+This program converts the NGINX reference documentation from it's XML schema to JSON. The JSON will be checked into git, and available for static content generation, markdoc tags, monaco plugins, etc.
+
+## Design
+
+```mermaid
+flowchart
+    fetch_atom[read latest version from atom feed]
+    read_datafile[read version from data file]
+    version_matches{do the versions match?}
+    download_tarball[download tarball of all XML]
+    parse_xml[parse XML]
+    render_md[translate prose to markdown]
+    write[write JSON to disk]
+    done((done))
+
+    read_datafile --> version_matches
+    fetch_atom --> version_matches
+
+    version_matches -->|Y| done
+    version_matches -->|N| download_tarball --> parse_xml --> render_md --> write --> done
+```
+
+The NGINX docs are publicly available at <http://hg.nginx.org/nginx.org>, in XML that's a mix of data and prose (`<para>` tags contain markup). The `<para>` contents will be translated in-order to generate equivalent markdown.
+
+The atom feed at <http://hg.nginx.org/nginx.org/atom-log> will tell us if there is updated content.
+
+A scheduled github pipeline will ensure we have up-to-date reference information.
+
+```mermaid
+flowchart
+    run[./reference-converter]
+    diff{git diff shows changes?}
+    open[open a PR with the changes]
+    slack[send slack notification]
+    done((done))
+
+    run --> diff -->|N| done
+    diff -->|Y| open --> slack --> done
+    run -->|errored out| slack
+```

reference-converter/atom/atom.go

@@ -0,0 +1,72 @@
+package atom
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+type entry struct {
+	Link struct {
+		Href string `xml:"href,attr"`
+	} `xml:"link"`
+}
+
+type feed struct {
+	Entry []entry `xml:"entry"`
+}
+
+type config struct {
+	Client http.Client
+}
+type Option = func(*config)
+
+// WithHttpClient uses the provided client instead of the default for
+// downloading tarballs.
+func WithHttpClient(c http.Client) Option {
+	return func(o *config) { o.Client = c }
+}
+
+// GetVersion gets the first link of the first entry of the XML file
+func GetVersion(ctx context.Context, url string, opts ...Option) (string, error) {
+	cfg := &config{}
+	for _, opt := range opts {
+		opt(cfg)
+	}
+	body, _ := openURL(ctx, url, cfg.Client)
+	return parseXML(body)
+}
+func openURL(ctx context.Context, url string, client http.Client) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("unable to download %s: %w", url, err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unable to download %s: %s", url, res.Status)
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read response body: %w", err)
+	}
+	return body, nil
+}
+func parseXML(XMLContent []byte) (string, error) {
+	var f feed
+	err := xml.Unmarshal(XMLContent, &f)
+	if err != nil {
+		return "", fmt.Errorf("unable to parse XML: %w", err)
+	}
+	if len(f.Entry) > 0 {
+		return f.Entry[0].Link.Href, nil
+	}
+	return "", fmt.Errorf("no entry was found in the feed")
+}

reference-converter/atom/atom_test.go

@@ -0,0 +1,74 @@
+package atom_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/nginxinc/nginx-directive-reference/reference-converter/atom"
+	"github.com/stretchr/testify/require"
+)
+
+func readTestDataFile(t *testing.T) string {
+	t.Helper()
+	body, err := os.ReadFile("./testdata/test.xml")
+	require.NoError(t, err)
+	return string(body)
+}
+func TestGetVersion(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		statusCode   int
+		responseBody string
+		wantError    bool
+		want         string
+	}{
+		"BadStatus": {
+			statusCode:   http.StatusBadRequest,
+			responseBody: "",
+			wantError:    true,
+		},
+		"OKStatusWithRealisticXML": {
+			statusCode:   http.StatusOK,
+			responseBody: readTestDataFile(t),
+			wantError:    false,
+			want:         "http://hg.nginx.org/nginx.org/rev/c80a7cb452e8",
+		},
+		"OKStatusWithNoEntries": {
+			statusCode:   http.StatusOK,
+			responseBody: "<feed></feed>",
+			wantError:    true,
+		},
+		"OKStatusWithInvalidXML": {
+			statusCode:   http.StatusOK,
+			responseBody: "I'm not XML",
+			wantError:    true,
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(testCase.statusCode)
+				_, err := w.Write([]byte(testCase.responseBody))
+				if err != nil {
+					t.Errorf("Response body failed to write: %s", err)
+				}
+			}))
+			defer srv.Close()
+
+			got, err := atom.GetVersion(context.Background(), srv.URL, atom.WithHttpClient(*srv.Client()))
+
+			if testCase.wantError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, testCase.want, got)
+			}
+
+		})
+	}
+}