Merge pull request #1 from nginxinc/import-repo

Importing relevant code from a private repo.

.github/workflows/reference-converter.daily.yml

@@ -0,0 +1,50 @@
+name: Daily Reference Converter Run
+
+on:
+  schedule:
+    - cron: '0 14 * * *'  # Run daily at 2pm UTC
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: ./reference-converter
+    shell: bash
+
+jobs:
+  compare_outputs:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - run: make devtools-image
+    - name: let the build create files
+      run: chmod 777 .
+    - run: make build
+    - name: run converter
+      run: ./dist/reference-converter -dst ./output/reference.json
+
+    - name: diff reference.json
+      id: diff
+      run: |
+        if ! diff -q <(grep -v '"version":' ../reference-lib/src/reference.json) <(grep -v '"version":' ./output/reference.json); then
+          echo "reference_change=true" >> "$GITHUB_OUTPUT"
+          mv ./output/reference.json ../reference-lib/src/reference.json
+        fi
+
+    - uses: actions/setup-node@v3
+    - name: update npm package version
+      if: steps.diff.outputs.reference_change
+      run: npm version patch --no-git-tag-version
+      working-directory: ./reference-lib
+
+    - name: create pull request if reference.json changed
+      uses: peter-evans/create-pull-request@v3
+      if: steps.diff.outputs.reference_change
+      with:
+        commit-message: update reference.json
+        token: ${{ secrets.GITHUB_TOKEN }}
+        branch: reference-update
+        title: "Reference Json Update"
+        body: "Changes detected in reference.json"
+        delete-branch: true
+        add-paths: reference-lib

.github/workflows/reference-converter.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  pull_request:
+    branches: ["main"]
+    paths:
+      [
+        "reference-converter/**",
+        ".github/workflows/reference-converter.yml",
+        ".golangci.yml",
+      ]
+
+defaults:
+  run:
+    working-directory: ./reference-converter
+    shell: bash
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      testResults: ./reference-converter/results/test.xml
+    steps:
+      - uses: actions/checkout@v3
+      - run: make devtools-image
+      - name: let the build create files
+        run: chmod 777 .
+      - run: make build
+      - run: make test
+      - name: archive test artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-report
+          path: ${{ env.testResults }}
+      - name: Test Report
+        uses: dorny/test-reporter@v1
+        if: success() || failure()
+        with:
+          name: Test Report
+          path: ${{ env.testResults }}
+          reporter: jest-junit
+  lint:
+    runs-on: ubuntu-latest
+    env:
+      working-directory: ./reference-converter
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: ${{ env.working-directory }}/go.mod
+          cache: false
+      - name: use the same version of GOLANGCI_LINT_VERSION as devtools
+        run: make devtools-versions >> "$GITHUB_ENV"
+      - uses: golangci/golangci-lint-action@v3
+        with:
+          working-directory: ${{ env.working-directory }}
+          version: v${{ env.GOLANGCI_LINT_VERSION }}

.github/workflows/reference-lib.publish.yml

@@ -0,0 +1,37 @@
+name: Publish reference lib package to GitHub Packages
+on:
+  push:
+    branches: ['main']
+    paths: ['reference-lib/**']
+  pull_request:
+    branches: ['main']
+    paths: ['reference-lib/**']
+
+defaults:
+  run:
+    working-directory: ./reference-lib
+    shell: bash
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16.x'
+          registry-url: 'https://npm.pkg.github.com'
+          scope: '@nginxinc'
+      - name: Install dependencies
+        run: npm ci
+      - name: Bundle using rollup
+        run: npm run build
+      - name: publish package
+        if: github.event_name == 'push'
+        run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.golangci.yml

@@ -0,0 +1,12 @@
+linters:
+  # add non-default linters https://golangci-lint.run/usage/linters/#disabled-by-default
+  enable:
+    - bodyclose
+    - errorlint
+
+issues:
+  # Disable maximum issues count per one linter
+  max-issues-per-linter: 0
+
+  # Disable maximum count of issues with the same text
+  max-same-issues: 0

README.md

@@ -1,59 +1,28 @@
 [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
-[![Community Support](https://badgen.net/badge/support/community/cyan?icon=awesome)](https://github.com/nginxinc/nginxinc/nginx-directive-reference/blob/main/SUPPORT.md)
-<!-- [![Commercial Support](https://badgen.net/badge/support/commercial/cyan?icon=awesome)](<Insert URL>) -->
+[![License](https://img.shields.io/badge/License-Apache--2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![Community Support](https://badgen.net/badge/support/community/cyan?icon=awesome)](https://github.com/nginxinc/nginx-directive-reference/blob/main/SUPPORT.md)
 
-# NGINX Template Repository
 
-## What is included on this template?
+# NGINX Directive Reference
+This repo contains the reference-lib npm package that can be used to fetch NGINX directive reference in Markdown and HTML format. It also has the converter code that is used to generate the directive reference json.
 
-This template includes all the scaffolding you need to get started on a standards compliant NGINX repository:
 
-- Standard license for NGINX OSS projects
-- Standard `.gitignore` with minimal defaults
-- Issue and PR templates
-- Contributing guidelines
-- Support guidelines
-- Security guidelines for reporting major vulnerabilities
-- NGINX Code of Conduct
-- README placeholder. How you structure the README is up to you (although the template provides placeholder sections), but you will need to include:
-  - A [repostatus](https://www.repostatus.org/) badge
-  - A community and commercial support badge. Include the latter -- and replace the URL placeholder with the relevant support URL -- if this repository contains a commercially supported project. You can find a commented out example below the community badge in this README.
-  - An explicit link back to the [Apache License 2.0](https://github.com/nginxinc/template-repository/blob/main/LICENSE)
-  - An up to date copyright notice
-- Changelog placeholder. (Optional -- A changelog is recommended, but it is not required and can diverge in format from the placeholder here included.)
-- Codeowners placeholder. (Optional -- Codeowners is a useful GitHub feature, but not all repositories require them.)
-
-## How do I use this template?
-
-**DO NOT FORK** -- this template is meant to be used from the **[`Use this template`](https://github.com/nginxinc/template-repository/generate)** feature.
-
-1. Click on **[`Use this template`](https://github.com/nginxinc/template-repository/generate)**
-2. Give a name to your project
-3. Wait until the first run of CI finishes (GitHub Actions will process the template and commit to your new repo)
-4. Clone your new project and tweak any of the placeholders if necessary. Pay special attention to the README!
-5. Happy coding!
-
-**NOTE**: **WAIT** until the first CI run on GitHub Actions finishes before cloning your new project.
-
----
-
-<!--  DELETE THE LINES ABOVE THIS AND WRITE YOUR PROJECT README BELOW -- PLACEHOLDER SECTIONS HAVE BEEN INCLUDED FOR YOUR CONVENIENCE -->
+## Getting Started
 
-[![Project Status: Concept – Minimal or no implementation has been done yet, or the repository is only intended to be a limited example, demo, or proof-of-concept.](https://www.repostatus.org/badges/latest/concept.svg)](https://www.repostatus.org/#concept)
+Refer to the [README file](https://github.com/nginxinc/nginx-directive-reference/blob/main/reference-lib/README.md) for how to install the reference-lib package.
 
-# nginx_directive_reference
+Refer to the [README file](https://github.com/nginxinc/nginx-directive-reference/blob/main/reference-converter/README.md) for how to build and run the reference converter.
 
-## Requirements
 
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam elit turpis, varius et arcu elementum, viverra rhoncus sem. Aliquam nec sodales magna, et egestas enim. Mauris lobortis ultrices euismod. Pellentesque in arcu lacus. Mauris cursus laoreet nulla, ac vehicula est. Vestibulum eu mauris quis lorem consectetur aliquam ac nec quam. Vestibulum commodo pharetra mi, at bibendum neque faucibus ut. Mauris et tortor sed sem consectetur eleifend ut non magna. Praesent feugiat placerat nibh, varius viverra orci bibendum sed. Vestibulum dapibus ex ut pulvinar facilisis. Quisque sodales enim et augue tempor mattis. Suspendisse finibus congue felis, ac blandit ligula. Praesent condimentum ultrices odio quis semper. Nunc ultrices, nibh quis mattis pellentesque, elit nulla bibendum felis, quis dapibus erat turpis ac urna.
+## Directory Structure
+The repository is organized as follows:
 
-## Getting Started
+reference-lib/: This directory contains an npm package that allows for easy lookup of nginx directives
 
-Duis sit amet sapien vel velit ornare vulputate. Nulla rutrum euismod risus ac efficitur. Curabitur in sagittis elit, a semper leo. Suspendisse malesuada aliquam velit, eu suscipit lorem vehicula at. Proin turpis lacus, semper in placerat in, accumsan non ipsum. Cras euismod, elit eget pretium laoreet, tortor nulla finibus tortor, nec hendrerit elit turpis ut eros. Quisque congue nisi id mauris molestie, eu condimentum dolor rutrum. Nullam eleifend elit ac lobortis tristique. Pellentesque nec tellus non mauris aliquet commodo a eu elit. Ut at feugiat metus, at tristique mauris. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae;
+reference-converter/: This directory contains a program that converts the official NGINX reference documentation in XML format to JSON.
 
-## How to Use
+tools/: This directory contains development tools
 
-Maecenas at vehicula justo. Suspendisse posuere elementum elit vel posuere. Etiam quis pulvinar massa. Integer tempor semper risus, vitae maximus eros ullamcorper vitae. In egestas, ex vitae gravida sodales, ipsum dolor varius est, et cursus lorem dui a mi. Morbi faucibus ut nisi id faucibus. Sed quis ullamcorper ex. In et dolor id nunc interdum suscipit.
 
 ## Contributing
 

reference-converter/.gitignore

@@ -0,0 +1,2 @@
+results
+dist

reference-converter/Makefile

@@ -0,0 +1,13 @@
+ROOT_DIR:=$(shell git rev-parse --show-toplevel)
+include $(ROOT_DIR)/tools/devtools/Makefile
+
+.PHONY: build test lint
+
+build:
+	@$(MAKE) .run args="go build -v -o dist/ ."
+
+test:
+	@$(MAKE) .run args="./test.sh results/test.xml"
+
+lint:
+	@$(MAKE) .run args="golangci-lint run -v"

reference-converter/README.md

@@ -0,0 +1,50 @@
+# Reference Converter
+
+This program converts the NGINX reference documentation from it's XML schema to JSON. The generated JSON is available as an npm package in the reference-lib folder and can be used for static content generation, markdoc tags, monaco plugins, etc.
+
+## Design
+
+```mermaid
+flowchart
+    fetch_atom[read latest version from atom feed]
+    read_datafile[read version from data file]
+    version_matches{do the versions match?}
+    download_tarball[download tarball of all XML]
+    parse_xml[parse XML]
+    render_md[translate prose to markdown]
+    write[write JSON to disk]
+    done((done))
+
+    read_datafile --> version_matches
+    fetch_atom --> version_matches
+
+    version_matches -->|Y| done
+    version_matches -->|N| download_tarball --> parse_xml --> render_md --> write --> done
+```
+
+The NGINX docs are publicly available at <http://hg.nginx.org/nginx.org>, in XML that's a mix of data and prose (`<para>` tags contain markup). The `<para>` contents will be translated in-order to generate equivalent markdown.
+
+The atom feed at <http://hg.nginx.org/nginx.org/atom-log> will tell us if there is updated content.
+
+A scheduled github pipeline ensures that we have up-to-date reference information.
+
+```mermaid
+flowchart
+    run[./reference-converter]
+    diff{git diff shows changes?}
+    open[open a PR with the changes]
+    slack[send slack notification]
+    done((done))
+
+    run --> diff -->|N| done
+    diff -->|Y| open --> slack --> done
+    run -->|errored out| slack
+```
+
+## Usage
+
+```bash
+make devtools-image
+make build
+./dist/reference-converter --dst <output-path>
+```

reference-converter/atom/atom.go

@@ -0,0 +1,72 @@
+package atom
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+type entry struct {
+	Link struct {
+		Href string `xml:"href,attr"`
+	} `xml:"link"`
+}
+
+type feed struct {
+	Entry []entry `xml:"entry"`
+}
+
+type config struct {
+	Client http.Client
+}
+type Option = func(*config)
+
+// WithHttpClient uses the provided client instead of the default for
+// downloading tarballs.
+func WithHttpClient(c http.Client) Option {
+	return func(o *config) { o.Client = c }
+}
+
+// GetVersion gets the first link of the first entry of the XML file
+func GetVersion(ctx context.Context, url string, opts ...Option) (string, error) {
+	cfg := &config{}
+	for _, opt := range opts {
+		opt(cfg)
+	}
+	body, _ := openURL(ctx, url, cfg.Client)
+	return parseXML(body)
+}
+func openURL(ctx context.Context, url string, client http.Client) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("unable to download %s: %w", url, err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unable to download %s: %s", url, res.Status)
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read response body: %w", err)
+	}
+	return body, nil
+}
+func parseXML(XMLContent []byte) (string, error) {
+	var f feed
+	err := xml.Unmarshal(XMLContent, &f)
+	if err != nil {
+		return "", fmt.Errorf("unable to parse XML: %w", err)
+	}
+	if len(f.Entry) > 0 {
+		return f.Entry[0].Link.Href, nil
+	}
+	return "", fmt.Errorf("no entry was found in the feed")
+}