From 828c0c6bb33f2c57fc309655b175430e34741143 Mon Sep 17 00:00:00 2001 From: Miguel Liezun Date: Sat, 4 May 2024 17:27:20 -0300 Subject: [PATCH] Improve tests and docs (#12) - Removed example folders and updated readme with clearer explanation - Docker images get tagged with {ref_name}-py{version} and latest-py{version} - Added new tests folder with projects: flask, fastapi, simple, simple_async. All follow the same pattern to store, retrieve and delete items from a dict. - Fixed issues encountered thanks to new tests added. - Validating WSGI implementation with wsgiref. --- .github/workflows/docker-publish.yml | 4 +- .github/workflows/go_tests.yaml | 41 ++++ .../{tests.yaml => integration_tests.yaml} | 26 ++- README.md | 187 +++++++++--------- caddysnake.c | 16 +- caddysnake.go | 67 +++++-- examples/Caddyfile | 26 --- examples/example_fastapi.py | 8 - examples/example_flask.py | 11 -- examples/simple_app.py | 12 -- examples/simple_asgi.py | 4 - examples/simple_exception.py | 12 -- examples_test.py | 21 -- tests/fastapi/Caddyfile | 19 ++ tests/fastapi/main.py | 31 +++ tests/fastapi/main_test.py | 74 +++++++ {examples => tests/fastapi}/requirements.txt | 15 +- tests/flask/Caddyfile | 19 ++ tests/flask/main.py | 28 +++ tests/flask/main_test.py | 74 +++++++ tests/flask/requirements.txt | 13 ++ tests/simple/Caddyfile | 19 ++ tests/simple/main.py | 56 ++++++ tests/simple/main_test.py | 74 +++++++ tests/simple/requirements.txt | 5 + tests/simple_async/Caddyfile | 19 ++ tests/simple_async/main.py | 59 ++++++ tests/simple_async/main_test.py | 74 +++++++ tests/simple_async/requirements.txt | 5 + 29 files changed, 783 insertions(+), 236 deletions(-) create mode 100644 .github/workflows/go_tests.yaml rename .github/workflows/{tests.yaml => integration_tests.yaml} (69%) delete mode 100644 examples/Caddyfile delete mode 100644 examples/example_fastapi.py delete mode 100644 examples/example_flask.py delete mode 100644 examples/simple_app.py delete mode 100644 examples/simple_asgi.py delete mode 100644 examples/simple_exception.py delete mode 100644 examples_test.py create mode 100644 tests/fastapi/Caddyfile create mode 100644 tests/fastapi/main.py create mode 100644 tests/fastapi/main_test.py rename {examples => tests/fastapi}/requirements.txt (50%) create mode 100644 tests/flask/Caddyfile create mode 100644 tests/flask/main.py create mode 100644 tests/flask/main_test.py create mode 100644 tests/flask/requirements.txt create mode 100644 tests/simple/Caddyfile create mode 100644 tests/simple/main.py create mode 100644 tests/simple/main_test.py create mode 100644 tests/simple/requirements.txt create mode 100644 tests/simple_async/Caddyfile create mode 100644 tests/simple_async/main.py create mode 100644 tests/simple_async/main_test.py create mode 100644 tests/simple_async/requirements.txt diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index dca3bab..62553c8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,7 +2,7 @@ name: Docker on: push: - branches: [ "main" ] + # branches: [ "main" ] # Publish semver tags as releases. tags: [ 'v*.*.*' ] pull_request: @@ -74,7 +74,7 @@ jobs: context: . push: ${{ github.event_name != 'pull_request' }} build-args: PY_VERSION=${{ matrix.python-version }} - tags: ${{ steps.meta.outputs.tags }}-py${{ matrix.python-version }} + tags: ${{ github.ref_name }}-py${{ matrix.python-version }},latest-py${{ matrix.python-version }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/.github/workflows/go_tests.yaml b/.github/workflows/go_tests.yaml new file mode 100644 index 0000000..1aa9f0b --- /dev/null +++ b/.github/workflows/go_tests.yaml @@ -0,0 +1,41 @@ +--- + name: Go Tests + on: + pull_request: + branches: + - main + push: + branches: + - main + jobs: + tests: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + env: + GOEXPERIMENT: cgocheck2 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.21' + cache: false + - name: Install Xcaddy + run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest + - name: Set up Python ${{ matrix.python-version }} + run: | + export DEBIAN_FRONTEND=noninteractive + sudo apt-get update -yyqq + sudo apt-get install -yyqq software-properties-common + sudo add-apt-repository -y ppa:deadsnakes/ppa + sudo apt-get install -yyqq python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv + sudo mv /usr/lib/x86_64-linux-gnu/pkgconfig/python-${{ matrix.python-version }}-embed.pc /usr/lib/x86_64-linux-gnu/pkgconfig/python3-embed.pc + - name: Install global python dependencies + run: sudo pip install requests + - name: Run module tests + run: go test -race -v ./... + - name: Build the server + run: CGO_ENABLED=1 xcaddy build --with github.com/mliezun/caddy-snake=. + \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/integration_tests.yaml similarity index 69% rename from .github/workflows/tests.yaml rename to .github/workflows/integration_tests.yaml index 76b217c..851da16 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -1,5 +1,5 @@ --- -name: Tests +name: Integration Tests on: pull_request: branches: @@ -13,6 +13,7 @@ jobs: strategy: fail-fast: false matrix: + tool-name: ['flask', 'fastapi', 'simple', 'simple_async'] python-version: ['3.9', '3.10', '3.11', '3.12'] env: GOEXPERIMENT: cgocheck2 @@ -25,6 +26,7 @@ jobs: - name: Install Xcaddy run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest - name: Set up Python ${{ matrix.python-version }} + working-directory: tests/${{ matrix.tool-name }}/ run: | export DEBIAN_FRONTEND=noninteractive sudo apt-get update -yyqq @@ -32,19 +34,15 @@ jobs: sudo add-apt-repository -y ppa:deadsnakes/ppa sudo apt-get install -yyqq python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv sudo mv /usr/lib/x86_64-linux-gnu/pkgconfig/python-${{ matrix.python-version }}-embed.pc /usr/lib/x86_64-linux-gnu/pkgconfig/python3-embed.pc - python${{ matrix.python-version }} -m venv examples/venv - source examples/venv/bin/activate - pip install -r examples/requirements.txt - - name: Install global python dependencies - run: sudo pip install requests - - name: Run module tests - run: go test -race -v ./... + python${{ matrix.python-version }} -m venv venv + source venv/bin/activate + pip install -r requirements.txt - name: Build the server - working-directory: examples/ - run: CGO_ENABLED=1 xcaddy build --with github.com/mliezun/caddy-snake=.. + working-directory: tests/${{ matrix.tool-name }}/ + run: CGO_ENABLED=1 xcaddy build --with github.com/mliezun/caddy-snake=../.. - name: Run integration tests - working-directory: examples/ + working-directory: tests/${{ matrix.tool-name }}/ run: | - ./caddy run --config Caddyfile & - sleep 10 - python ../examples_test.py + ./caddy run --config Caddyfile 2>/dev/null & + sleep 2 + python main_test.py diff --git a/README.md b/README.md index 4b7a79b..2c38582 100644 --- a/README.md +++ b/README.md @@ -6,136 +6,134 @@ This plugin provides native support for Python apps. It embeds the Python interpreter inside Caddy and serves requests directly without going through a reverse proxy. -It supports both WSGI and ASGI, which means you can run all types of frameworks like Flask, Django and FastAPI. +Supports both WSGI and ASGI, which means you can run all types of frameworks like Flask, Django and FastAPI. -## Docker image +## Quickstart -There's a docker image available, it ships Python 3.12 and can be used as follows: +#### Requirements -```Dockerfile -FROM ghcr.io/mliezun/caddy-snake:main - -WORKDIR /app +- Python >= 3.9 + dev files +- C compiler and build tools +- Go >= 1.21 and [Xcaddy](https://github.com/caddyserver/xcaddy) -# Copy your project into app -COPY . /app +Install requirements on Ubuntu 24.04: -# Caddy snake is already installed and has support for Python 3.12 -CMD ["caddy", "run", "--config", "/app/Caddyfile"] +``` +$ sudo apt-get install python3-dev build-essential pkg-config golang +$ go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest ``` -## Build from source +You can also [build with Docker](#build-with-docker-or-podman). -Go 1.21 and Python 3.9 or later is required, with development files to embed the interpreter. +#### Example usage: Flask -To install in Ubuntu do: +`main.py` -```bash -sudo apt-get update -sudo apt-get install -y python3-dev -``` +```python +from flask import Flask -To install in macOS do: +app = Flask(__name__) -```bash -brew install python@3 +@app.route("/hello-world") +def hello(): + return "Hello world!" ``` -### Bundling with Caddy - -Build this module using [xcaddy](https://github.com/caddyserver/xcaddy): +`Caddyfile` -```bash -CGO_ENABLED=1 xcaddy build --with github.com/mliezun/caddy-snake@v0.0.5 +```Caddyfile +localhost:9080 { + route { + python { + module_wsgi "main:app" + } + } +} ``` -### Build with Docker (or Podman) +Run: -There's a template file in the project: [builder.Dockerfile](/builder.Dockerfile). It supports build arguments to configure which Python or Go version is desired for the build. - -```Dockerfile -FROM ubuntu:22.04 - -ARG GO_VERSION=1.22.1 -ARG PY_VERSION=3.12 - -RUN export DEBIAN_FRONTEND=noninteractive &&\ - apt-get update -yyqq &&\ - apt-get install -yyqq wget tar software-properties-common gcc pkgconf &&\ - add-apt-repository -y ppa:deadsnakes/ppa &&\ - apt-get update -yyqq &&\ - apt-get install -yyqq python${PY_VERSION}-dev &&\ - mv /usr/lib/x86_64-linux-gnu/pkgconfig/python-${PY_VERSION}-embed.pc /usr/lib/x86_64-linux-gnu/pkgconfig/python3-embed.pc &&\ - rm -rf /var/lib/apt/lists/* &&\ - wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \ - tar -C /usr/local -xzf go*.linux-amd64.tar.gz && \ - rm go*.linux-amd64.tar.gz - -ENV PATH=$PATH:/usr/local/go/bin - -RUN go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest &&\ - cd /usr/local/bin &&\ - CGO_ENABLED=1 /root/go/bin/xcaddy build --with github.com/mliezun/caddy-snake &&\ - rm -rf /build +``` +$ pip install Flask +$ CGO_ENABLED=1 xcaddy build --with github.com/mliezun/caddy-snake +$ ./caddy run --config Caddyfile +``` -CMD ["cp", "/usr/local/bin/caddy", "/output/caddy"] +``` +$ curl http://localhost:9080/hello-world +Hello world! ``` -You can copy the contents of the builder Dockerfile and execute the following commands to get your Caddy binary: +#### Example usage: FastAPI -```bash -docker build -f builder.Dockerfile --build-arg PY_VERSION=3.9 -t caddy-snake . -``` +`main.py` -```bash -docker run --rm -v $(pwd):/output caddy-snake +```python +from fastapi import FastAPI + +@app.get("/hello-world") +def hello(): + return "Hello world!" ``` -## Example Caddyfile +`Caddyfile` ```Caddyfile -{ - http_port 9080 - https_port 9443 - log { - level error - } -} localhost:9080 { route { - python "simple_app:main" + python { + module_asgi "main:app" + } } } ``` -The `python` rule is an HTTP handler that expects a WSGI app as an argument. +Run: -If you want to use an ASGI app, like FastAPI or other async frameworks you can use the following config: +``` +$ pip install fastapi +$ CGO_ENABLED=1 xcaddy build --with github.com/mliezun/caddy-snake +$ ./caddy run --config Caddyfile +``` -```Caddyfile -{ - http_port 9080 - https_port 9443 - log { - level error - } -} -localhost:9080 { - route { - python { - module_asgi "example_fastapi:app" - } - } -} ``` +$ curl http://localhost:9080/hello-world +Hello world! +``` + +## Use docker image + +There are docker images available with the following Python versions: `3.9`, `3.10`, `3.11`, `3.12` -## Examples +Example usage: -- [simple_app](/examples/simple_app.py). WSGI App that returns the standard hello world message and a UUID. -- [simple_exception](/examples/simple_exception.py). WSGI App that always raises an exception. -- [example_flask](/examples/example_flask.py). Flask application that also returns hello world message and a UUID. -- [example_fastapi](/examples/example_fastapi.py). FastAPI application that also returns hello world message and a UUID. -- [Caddyfile](/examples/Caddyfile). Caddy config that uses all of the example apps. +```Dockerfile +FROM ghcr.io/mliezun/caddy-snake:latest-py3.12 + +WORKDIR /app + +# Copy your project into app +COPY . /app + +# Caddy snake is already installed and has support for Python 3.12 +CMD ["caddy", "run", "--config", "/app/Caddyfile"] +``` + +### Build with Docker + +There's a template file in the project: [builder.Dockerfile](/builder.Dockerfile). It supports build arguments to configure which Python or Go version is desired for the build. + +Make sure to use the same Python version as you have installed in your system. + +You can copy the contents of the builder Dockerfile and execute the following commands to get your Caddy binary: + +```bash +docker build -f builder.Dockerfile --build-arg PY_VERSION=3.9 -t caddy-snake . +``` + +```bash +docker run --rm -v $(pwd):/output caddy-snake +``` **NOTE** @@ -143,8 +141,8 @@ It's also possible to provide virtual environments with the following syntax: ```Caddyfile python { - module_wsgi "simple_app:main" - venv_path "./venv" + module_wsgi "main:app" + venv "./venv" } ``` @@ -163,6 +161,7 @@ What it does behind the scenes is to append `venv/lib/python3.x/site-packages` t - [Apache mod_wsgi](https://github.com/GrahamDumpleton/mod_wsgi) - [FrankenPHP](https://github.com/dunglas/frankenphp) - [WSGI Standard PEP 3333](https://peps.python.org/pep-3333/) +- [ASGI Spec](https://asgi.readthedocs.io/en/latest/index.html) ## LICENSE diff --git a/caddysnake.c b/caddysnake.c index 3941e7f..d695fb3 100644 --- a/caddysnake.c +++ b/caddysnake.c @@ -288,6 +288,7 @@ static PyObject *response_callback(PyObject *self, PyObject *args) { if (response->response_body) { PyObject *iterator = PyObject_GetIter(response->response_body); if (iterator) { + PyObject *close_iterator = PyObject_GetAttrString(iterator, "close"); PyObject *item; while ((item = PyIter_Next(iterator))) { if (!PyBytes_Check(item)) { @@ -295,6 +296,8 @@ static PyObject *response_callback(PyObject *self, PyObject *args) { "expected response body items to be bytes"); PyErr_Print(); Py_DECREF(item); + PyObject_CallNoArgs(close_iterator); + Py_DECREF(close_iterator); Py_DECREF(iterator); if (response_body != NULL) { free(response_body); @@ -311,6 +314,8 @@ static PyObject *response_callback(PyObject *self, PyObject *args) { } Py_DECREF(item); } + PyObject_CallNoArgs(close_iterator); + Py_DECREF(close_iterator); Py_DECREF(iterator); } else { PyErr_Print(); @@ -323,6 +328,14 @@ static PyObject *response_callback(PyObject *self, PyObject *args) { goto finalize_error; } + if (PyErr_Occurred()) { + PyErr_Print(); + if (response_body != NULL) { + free(response_body); + } + goto finalize_error; + } + if (!response->response_headers) { PyErr_SetString(PyExc_RuntimeError, "expected response headers to be non-empty"); @@ -466,7 +479,8 @@ static void AsgiEvent_dealloc(AsgiEvent *self) { Py_XDECREF(self->event_ts); // Future is freed in AsgiEvent_result // Py_XDECREF(self->future); - Py_XDECREF(self->request_body); + // Request body is freed in AsgiEvent_receive_end + // Py_XDECREF(self->request_body); Py_TYPE(self)->tp_free((PyObject *)self); } diff --git a/caddysnake.go b/caddysnake.go index 5bce8fa..5c39a02 100644 --- a/caddysnake.go +++ b/caddysnake.go @@ -1,4 +1,4 @@ -// Caddy plugin that provides native support for Python WSGI apps. +// Caddy plugin to serve Python apps. package caddysnake // #cgo pkg-config: python3-embed @@ -11,6 +11,7 @@ import ( "io" "net" "net/http" + "net/textproto" "net/url" "os" "path/filepath" @@ -31,12 +32,13 @@ import ( //go:embed caddysnake.py var caddysnake_py string +// AppServer defines the interface to interacting with a WSGI or ASGI server type AppServer interface { Cleanup() HandleRequest(w http.ResponseWriter, r *http.Request) error } -// CaddySnake module that communicates with a Wsgi app to handle requests +// CaddySnake module that communicates with a Python app type CaddySnake struct { ModuleWsgi string `json:"module_wsgi,omitempty"` ModuleAsgi string `json:"module_asgi,omitempty"` @@ -145,15 +147,15 @@ func parsePythonDirective(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, return app, nil } -// WsgiRequestHandler stores the result of a request handled by a Wsgi app +// WsgiRequestHandler tracks the state of a HTTP request to a WSGI App type WsgiRequestHandler struct { status_code C.int headers *C.MapKeyVal body *C.char } -var lock sync.RWMutex = sync.RWMutex{} -var request_counter int64 = 0 +var wsgi_lock sync.RWMutex = sync.RWMutex{} +var wsgi_request_counter int64 = 0 var wsgi_handlers map[int64]chan WsgiRequestHandler = map[int64]chan WsgiRequestHandler{} func init() { @@ -217,6 +219,7 @@ type Wsgi struct { app *C.WsgiApp } +// NewWsgi imports a WSGI app func NewWsgi(wsgi_pattern string, venv_path string) (*Wsgi, error) { module_app := strings.Split(wsgi_pattern, ":") if len(module_app) != 2 { @@ -291,7 +294,17 @@ func (m *Wsgi) HandleRequest(w http.ResponseWriter, r *http.Request) error { "CONTENT_LENGTH": r.Header.Get("Content-length"), "wsgi.url_scheme": strings.ToLower(strings.Split(r.Proto, "/")[0]), } - rh := C.MapKeyVal_new(C.size_t(len(r.Header) + len(extra_headers))) + headers_length := len(r.Header) + if _, ok := r.Header[textproto.CanonicalMIMEHeaderKey("Proxy")]; ok { + headers_length -= 1 + } + if _, ok := r.Header[textproto.CanonicalMIMEHeaderKey("Content-Type")]; ok { + headers_length -= 1 + } + if _, ok := r.Header[textproto.CanonicalMIMEHeaderKey("Content-Length")]; ok { + headers_length -= 1 + } + rh := C.MapKeyVal_new(C.size_t(headers_length + len(extra_headers))) defer C.free(unsafe.Pointer(rh)) defer C.free(unsafe.Pointer(rh.keys)) defer C.free(unsafe.Pointer(rh.values)) @@ -305,6 +318,13 @@ func (m *Wsgi) HandleRequest(w http.ResponseWriter, r *http.Request) error { // golang cgi issue 16405 continue } + // Content type and length already defined in extra_headers + if key == "CONTENT_TYPE" { + continue + } + if key == "CONTENT_LENGTH" { + continue + } joinStr := ", " if k == "COOKIE" { @@ -337,11 +357,11 @@ func (m *Wsgi) HandleRequest(w http.ResponseWriter, r *http.Request) error { defer C.free(unsafe.Pointer(body_str)) ch := make(chan WsgiRequestHandler) - lock.Lock() - request_counter++ - request_id := request_counter + wsgi_lock.Lock() + wsgi_request_counter++ + request_id := wsgi_request_counter wsgi_handlers[request_id] = ch - lock.Unlock() + wsgi_lock.Unlock() runtime.LockOSThread() C.WsgiApp_handle_request(m.app, C.int64_t(request_id), rh, body_str) @@ -379,8 +399,8 @@ func (m *Wsgi) HandleRequest(w http.ResponseWriter, r *http.Request) error { //export wsgi_write_response func wsgi_write_response(request_id C.int64_t, status_code C.int, headers *C.MapKeyVal, body *C.char) { - lock.Lock() - defer lock.Unlock() + wsgi_lock.Lock() + defer wsgi_lock.Unlock() ch := wsgi_handlers[int64(request_id)] ch <- WsgiRequestHandler{ status_code: status_code, @@ -397,6 +417,7 @@ type Asgi struct { app *C.AsgiApp } +// NewAsgi imports a Python ASGI app func NewAsgi(wsgi_pattern string, venv_path string) (*Asgi, error) { module_app := strings.Split(wsgi_pattern, ":") if len(module_app) != 2 { @@ -426,7 +447,7 @@ func NewAsgi(wsgi_pattern string, venv_path string) (*Asgi, error) { return &Asgi{app}, nil } -// Cleanup deallocates CGO resources used by Wsgi app +// Cleanup deallocates CGO resources used by Asgi app func (m *Asgi) Cleanup() { if m.app != nil { runtime.LockOSThread() @@ -446,6 +467,7 @@ type AsgiRequestHandler struct { is_websocket bool } +// AsgiOperations stores operations that should be executed in the background type AsgiOperations struct { stop bool op func() @@ -463,11 +485,13 @@ func (h *AsgiRequestHandler) consume() { } } +// NewAsgiRequestHandler initializes handler and starts queue that consumes operations +// in the background. func NewAsgiRequestHandler(w http.ResponseWriter, r *http.Request) *AsgiRequestHandler { h := &AsgiRequestHandler{ w: w, r: r, - done: make(chan error), + done: make(chan error, 2), operations: make(chan AsgiOperations, 4), } @@ -610,8 +634,8 @@ func (m *Asgi) HandleRequest(w http.ResponseWriter, r *http.Request) error { //export asgi_receive_start func asgi_receive_start(request_id C.uint64_t, event *C.AsgiEvent) { asgi_lock.Lock() + defer asgi_lock.Unlock() arh := asgi_handlers[uint64(request_id)] - asgi_lock.Unlock() arh.operations <- AsgiOperations{op: func() { body, err := io.ReadAll(arh.r.Body) @@ -631,8 +655,8 @@ func asgi_receive_start(request_id C.uint64_t, event *C.AsgiEvent) { //export asgi_set_headers func asgi_set_headers(request_id C.uint64_t, status_code C.int, headers *C.MapKeyVal, event *C.AsgiEvent) { asgi_lock.Lock() + defer asgi_lock.Unlock() arh := asgi_handlers[uint64(request_id)] - asgi_lock.Unlock() arh.operations <- AsgiOperations{op: func() { if headers != nil { @@ -663,8 +687,8 @@ func asgi_set_headers(request_id C.uint64_t, status_code C.int, headers *C.MapKe //export asgi_send_response func asgi_send_response(request_id C.uint64_t, body *C.char, more_body C.uint8_t, event *C.AsgiEvent) { asgi_lock.Lock() + defer asgi_lock.Unlock() arh := asgi_handlers[uint64(request_id)] - asgi_lock.Unlock() arh.operations <- AsgiOperations{op: func() { body_bytes := []byte(C.GoString(body)) @@ -684,8 +708,9 @@ func asgi_send_response(request_id C.uint64_t, body *C.char, more_body C.uint8_t //export asgi_cancel_request func asgi_cancel_request(request_id C.uint64_t) { asgi_lock.Lock() - arh := asgi_handlers[uint64(request_id)] - asgi_lock.Unlock() - - arh.done <- errors.New("request cancelled") + defer asgi_lock.Unlock() + arh, ok := asgi_handlers[uint64(request_id)] + if ok { + arh.done <- errors.New("request cancelled") + } } diff --git a/examples/Caddyfile b/examples/Caddyfile deleted file mode 100644 index b476d43..0000000 --- a/examples/Caddyfile +++ /dev/null @@ -1,26 +0,0 @@ -{ - http_port 9080 - https_port 9443 - log { - level info - } -} -localhost:9080 { - route /app1 { - python "simple_app:main" - } - route /app2 { - python { - module_wsgi "example_flask:app" - venv "./venv" - } - } - route /app3 { - python "simple_exception:main" - } - route /app4 { - python { - module_asgi "example_fastapi:app" - } - } -} diff --git a/examples/example_fastapi.py b/examples/example_fastapi.py deleted file mode 100644 index 94a7886..0000000 --- a/examples/example_fastapi.py +++ /dev/null @@ -1,8 +0,0 @@ -from fastapi import FastAPI, Request - -app = FastAPI() - - -@app.get("/{full_path:path}") -async def root(full_path: str, request: Request): - return {"message": "Hello World", "path": full_path} diff --git a/examples/example_flask.py b/examples/example_flask.py deleted file mode 100644 index 3ec126f..0000000 --- a/examples/example_flask.py +++ /dev/null @@ -1,11 +0,0 @@ -import uuid -from flask import Flask - - -app = Flask(__name__) - - -@app.route("/", defaults={"path": ""}) -@app.route("/") -def root(path): - return f"Flask: {str(uuid.uuid4())}" diff --git a/examples/simple_app.py b/examples/simple_app.py deleted file mode 100644 index 9cd9b40..0000000 --- a/examples/simple_app.py +++ /dev/null @@ -1,12 +0,0 @@ -import uuid - - -def main(environ, start_response): - """A simple WSGI application""" - status = "200 OK" - response_headers = [ - ("Content-type", "text/plain"), - ("X-Custom-Header", "Custom-Value"), - ] - start_response(status, response_headers) - yield f"Simple app: {str(uuid.uuid4())}".encode() diff --git a/examples/simple_asgi.py b/examples/simple_asgi.py deleted file mode 100644 index b73cec9..0000000 --- a/examples/simple_asgi.py +++ /dev/null @@ -1,4 +0,0 @@ -async def main(scope, receive, send): - await receive() - await send({"type": "http.response.start", "status": 200, "headers": []}) - await send({"type": "http.response.body", "body": b"Hello from the nether"}) diff --git a/examples/simple_exception.py b/examples/simple_exception.py deleted file mode 100644 index 8cffb39..0000000 --- a/examples/simple_exception.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys - - -def main(environ, start_response): - """A simple WSGI application that passes exc_info to start_response""" - status = "200 OK" - response_headers = [("Content-type", "text/plain")] - try: - start_response(status, response_headers + 1) - except TypeError as e: - start_response(status, response_headers, sys.exc_info()) - return [b"Simple Exception"] diff --git a/examples_test.py b/examples_test.py deleted file mode 100644 index e829f11..0000000 --- a/examples_test.py +++ /dev/null @@ -1,21 +0,0 @@ -import time -from concurrent.futures import ThreadPoolExecutor -import requests - - -def do_requests(app: str, expected_status: int = 200, n: int = 1_000): - for _ in range(n): - r = requests.get(f"http://localhost:9080/{app}") - assert r.status_code == expected_status - return n - - -with ThreadPoolExecutor(max_workers=8) as t: - start = time.time() - request_count = sum( - t.map( - do_requests, ["app1", "app2", "app3", "app4"] * 3, [200, 200, 500, 200] * 3 - ) - ) - print(f"Done {request_count=}") - print(f"Elapsed: {time.time()-start}s") diff --git a/tests/fastapi/Caddyfile b/tests/fastapi/Caddyfile new file mode 100644 index 0000000..e1205c9 --- /dev/null +++ b/tests/fastapi/Caddyfile @@ -0,0 +1,19 @@ +{ + http_port 9080 + https_port 9443 + log { + level info + } +} +localhost:9080 { + route /item/* { + python { + module_asgi "main:app" + venv "./venv" + } + } + + route / { + respond 404 + } +} diff --git a/tests/fastapi/main.py b/tests/fastapi/main.py new file mode 100644 index 0000000..cad11f7 --- /dev/null +++ b/tests/fastapi/main.py @@ -0,0 +1,31 @@ +from typing import Optional + +from fastapi import FastAPI +from pydantic import BaseModel + +app = FastAPI() + +db = {} + + +class Item(BaseModel): + name: str + description: str + blob: Optional[str] + + +@app.get("/item/{id}") +async def get_item(id: str): + return db.get(id) + + +@app.post("/item/{id}") +async def store_item(id: str, item: Item): + db[id] = item + return "Stored" + + +@app.delete("/item/{id}") +async def delete_item(id: str): + del db[id] + return "Deleted" diff --git a/tests/fastapi/main_test.py b/tests/fastapi/main_test.py new file mode 100644 index 0000000..673ae5f --- /dev/null +++ b/tests/fastapi/main_test.py @@ -0,0 +1,74 @@ +import os +import base64 +import uuid +import time +from concurrent.futures import ThreadPoolExecutor +import requests + +item_count = 0 + +BASE_URL = "http://localhost:9080" + +BIG_BLOB = base64.b64encode(os.urandom(4 * 2**20)).decode("utf") + + +def get_dummy_item() -> dict: + global item_count + item_count += 1 + return { + "name": f"Item {item_count}", + "description": f"Item Description {item_count}", + "blob": BIG_BLOB if item_count % 4 == 0 else None, + } + + +def store_item(id: str, item: dict): + response = requests.post(f"{BASE_URL}/item/{id}", json=item) + return response.status_code == 200 and b"Stored" in response.content + + +def get_item(id: str, item: dict): + response = requests.get(f"{BASE_URL}/item/{id}") + return response.status_code == 200 and response.json() == item + + +def delete_item(id: str): + response = requests.delete(f"{BASE_URL}/item/{id}") + return response.status_code == 200 and b"Deleted" in response.content + + +def item_lifecycle(): + id = str(uuid.uuid4()) + item = get_dummy_item() + assert store_item(id, item), "Store item failed" + assert get_item(id, item), "Get item failed" + assert delete_item(id), "Delete item failed" + assert not delete_item(id), "Delete item should fail" + + +def make_objects(max_workers: int, count: int): + start = time.time() + failed = False + + def item_done(fut): + exc = fut.exception() + if exc: + nonlocal failed + failed = True + raise SystemExit(1) from exc + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + for _ in range(count): + future = executor.submit(item_lifecycle) + future.add_done_callback(item_done) + + if failed: + print("Tests failed") + exit(1) + + print(f"Created and destroyed {count} objects") + print(f"Elapsed: {time.time()-start}s") + + +if __name__ == "__main__": + make_objects(max_workers=4, count=2_500) diff --git a/examples/requirements.txt b/tests/fastapi/requirements.txt similarity index 50% rename from examples/requirements.txt rename to tests/fastapi/requirements.txt index 0d66021..ea36404 100644 --- a/examples/requirements.txt +++ b/tests/fastapi/requirements.txt @@ -1,19 +1,14 @@ annotated-types==0.6.0 anyio==4.3.0 -blinker==1.8.1 -click==8.1.7 exceptiongroup==1.2.1 -fastapi==0.110.2 -Flask==3.0.3 +fastapi==0.110.3 idna==3.7 -importlib_metadata==7.1.0 -itsdangerous==2.2.0 -Jinja2==3.1.3 -MarkupSafe==2.1.5 pydantic==2.7.1 pydantic_core==2.18.2 sniffio==1.3.1 starlette==0.37.2 typing_extensions==4.11.0 -Werkzeug==3.0.2 -zipp==3.18.1 +requests==2.31.0 +certifi==2024.2.2 +charset-normalizer==3.3.2 +urllib3==2.2.1 diff --git a/tests/flask/Caddyfile b/tests/flask/Caddyfile new file mode 100644 index 0000000..fed3834 --- /dev/null +++ b/tests/flask/Caddyfile @@ -0,0 +1,19 @@ +{ + http_port 9080 + https_port 9443 + log { + level info + } +} +localhost:9080 { + route /item/* { + python { + module_wsgi "main:app" + venv "./venv" + } + } + + route / { + respond 404 + } +} diff --git a/tests/flask/main.py b/tests/flask/main.py new file mode 100644 index 0000000..f9cfa43 --- /dev/null +++ b/tests/flask/main.py @@ -0,0 +1,28 @@ +import wsgiref.validate +from flask import Flask, request + + +app = Flask(__name__) + +db = {} + + +@app.route("/item/", methods=["POST"]) +def store_item(id: str): + content = request.get_json() + db[id] = content + return "Stored" + + +@app.route("/item/", methods=["GET"]) +def get_item(id: str): + return db.get(id) + + +@app.route("/item/", methods=["DELETE"]) +def delete_item(id): + del db[id] + return "Deleted" + + +app = wsgiref.validate.validator(app) diff --git a/tests/flask/main_test.py b/tests/flask/main_test.py new file mode 100644 index 0000000..673ae5f --- /dev/null +++ b/tests/flask/main_test.py @@ -0,0 +1,74 @@ +import os +import base64 +import uuid +import time +from concurrent.futures import ThreadPoolExecutor +import requests + +item_count = 0 + +BASE_URL = "http://localhost:9080" + +BIG_BLOB = base64.b64encode(os.urandom(4 * 2**20)).decode("utf") + + +def get_dummy_item() -> dict: + global item_count + item_count += 1 + return { + "name": f"Item {item_count}", + "description": f"Item Description {item_count}", + "blob": BIG_BLOB if item_count % 4 == 0 else None, + } + + +def store_item(id: str, item: dict): + response = requests.post(f"{BASE_URL}/item/{id}", json=item) + return response.status_code == 200 and b"Stored" in response.content + + +def get_item(id: str, item: dict): + response = requests.get(f"{BASE_URL}/item/{id}") + return response.status_code == 200 and response.json() == item + + +def delete_item(id: str): + response = requests.delete(f"{BASE_URL}/item/{id}") + return response.status_code == 200 and b"Deleted" in response.content + + +def item_lifecycle(): + id = str(uuid.uuid4()) + item = get_dummy_item() + assert store_item(id, item), "Store item failed" + assert get_item(id, item), "Get item failed" + assert delete_item(id), "Delete item failed" + assert not delete_item(id), "Delete item should fail" + + +def make_objects(max_workers: int, count: int): + start = time.time() + failed = False + + def item_done(fut): + exc = fut.exception() + if exc: + nonlocal failed + failed = True + raise SystemExit(1) from exc + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + for _ in range(count): + future = executor.submit(item_lifecycle) + future.add_done_callback(item_done) + + if failed: + print("Tests failed") + exit(1) + + print(f"Created and destroyed {count} objects") + print(f"Elapsed: {time.time()-start}s") + + +if __name__ == "__main__": + make_objects(max_workers=4, count=2_500) diff --git a/tests/flask/requirements.txt b/tests/flask/requirements.txt new file mode 100644 index 0000000..9024678 --- /dev/null +++ b/tests/flask/requirements.txt @@ -0,0 +1,13 @@ +blinker==1.8.1 +click==8.1.7 +Flask==3.0.3 +importlib_metadata==7.1.0 +itsdangerous==2.2.0 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +Werkzeug==3.0.2 +zipp==3.18.1 +requests==2.31.0 +certifi==2024.2.2 +charset-normalizer==3.3.2 +urllib3==2.2.1 diff --git a/tests/simple/Caddyfile b/tests/simple/Caddyfile new file mode 100644 index 0000000..fed3834 --- /dev/null +++ b/tests/simple/Caddyfile @@ -0,0 +1,19 @@ +{ + http_port 9080 + https_port 9443 + log { + level info + } +} +localhost:9080 { + route /item/* { + python { + module_wsgi "main:app" + venv "./venv" + } + } + + route / { + respond 404 + } +} diff --git a/tests/simple/main.py b/tests/simple/main.py new file mode 100644 index 0000000..990388a --- /dev/null +++ b/tests/simple/main.py @@ -0,0 +1,56 @@ +from typing import Callable +import json +import wsgiref.validate + +db = {} + +CHUNK_SIZE = 256 * 2**20 + + +def store_item(id: str, content: dict): + db[id] = content + return b"Stored" + + +def get_item(id: str): + return db.get(id) + + +def delete_item(id): + del db[id] + return b"Deleted" + + +@wsgiref.validate.validator +def app(environ: dict, start_response: Callable): + """A simple WSGI application""" + path: str = environ.get("PATH_INFO", "") + method: str = environ.get("REQUEST_METHOD", "").lower() + if path.startswith("/item/"): + item_id = path[6:] + body = b"" + status = "200 OK" + content_type = "text/plain" + if method == "get": + body = json.dumps(get_item(item_id)).encode() + content_type = "application/json" + elif method == "post": + request_body = environ["wsgi.input"] + body_content = b"" + data = request_body.read(CHUNK_SIZE) + while data: + body_content += data + data = request_body.read(CHUNK_SIZE) + content = json.loads(body_content) + body = store_item(item_id, content) + elif method == "delete": + body = delete_item(item_id) + else: + status = "405" + body = b"Method Not Allowed" + response_headers = [("Content-Type", content_type)] + start_response(status, response_headers) + yield body + else: + start_response("404 Not Found", [("Content-type", "text/plain")]) + yield b"Not found" diff --git a/tests/simple/main_test.py b/tests/simple/main_test.py new file mode 100644 index 0000000..673ae5f --- /dev/null +++ b/tests/simple/main_test.py @@ -0,0 +1,74 @@ +import os +import base64 +import uuid +import time +from concurrent.futures import ThreadPoolExecutor +import requests + +item_count = 0 + +BASE_URL = "http://localhost:9080" + +BIG_BLOB = base64.b64encode(os.urandom(4 * 2**20)).decode("utf") + + +def get_dummy_item() -> dict: + global item_count + item_count += 1 + return { + "name": f"Item {item_count}", + "description": f"Item Description {item_count}", + "blob": BIG_BLOB if item_count % 4 == 0 else None, + } + + +def store_item(id: str, item: dict): + response = requests.post(f"{BASE_URL}/item/{id}", json=item) + return response.status_code == 200 and b"Stored" in response.content + + +def get_item(id: str, item: dict): + response = requests.get(f"{BASE_URL}/item/{id}") + return response.status_code == 200 and response.json() == item + + +def delete_item(id: str): + response = requests.delete(f"{BASE_URL}/item/{id}") + return response.status_code == 200 and b"Deleted" in response.content + + +def item_lifecycle(): + id = str(uuid.uuid4()) + item = get_dummy_item() + assert store_item(id, item), "Store item failed" + assert get_item(id, item), "Get item failed" + assert delete_item(id), "Delete item failed" + assert not delete_item(id), "Delete item should fail" + + +def make_objects(max_workers: int, count: int): + start = time.time() + failed = False + + def item_done(fut): + exc = fut.exception() + if exc: + nonlocal failed + failed = True + raise SystemExit(1) from exc + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + for _ in range(count): + future = executor.submit(item_lifecycle) + future.add_done_callback(item_done) + + if failed: + print("Tests failed") + exit(1) + + print(f"Created and destroyed {count} objects") + print(f"Elapsed: {time.time()-start}s") + + +if __name__ == "__main__": + make_objects(max_workers=4, count=2_500) diff --git a/tests/simple/requirements.txt b/tests/simple/requirements.txt new file mode 100644 index 0000000..2179078 --- /dev/null +++ b/tests/simple/requirements.txt @@ -0,0 +1,5 @@ +certifi==2024.2.2 +charset-normalizer==3.3.2 +idna==3.7 +requests==2.31.0 +urllib3==2.2.1 diff --git a/tests/simple_async/Caddyfile b/tests/simple_async/Caddyfile new file mode 100644 index 0000000..e1205c9 --- /dev/null +++ b/tests/simple_async/Caddyfile @@ -0,0 +1,19 @@ +{ + http_port 9080 + https_port 9443 + log { + level info + } +} +localhost:9080 { + route /item/* { + python { + module_asgi "main:app" + venv "./venv" + } + } + + route / { + respond 404 + } +} diff --git a/tests/simple_async/main.py b/tests/simple_async/main.py new file mode 100644 index 0000000..8d371ea --- /dev/null +++ b/tests/simple_async/main.py @@ -0,0 +1,59 @@ +import json + +db = {} + + +def store_item(id: str, content: dict): + db[id] = content + return b"Stored" + + +def get_item(id: str): + return db.get(id) + + +def delete_item(id): + del db[id] + return b"Deleted" + + +async def app(scope, receive, send): + path: str = scope["path"] + method: str = scope["method"].lower() + if path.startswith("/item/"): + item_id = path[6:] + body = b"" + status = 200 + content_type = b"text/plain" + if method == "get": + body = json.dumps(get_item(item_id)).encode() + content_type = b"application/json" + elif method == "post": + request_body = await receive() + content = request_body["body"] + while request_body["more_body"]: + request_body = await receive() + content += request_body["body"] + body = store_item(item_id, json.loads(content)) + elif method == "delete": + body = delete_item(item_id) + else: + status = 405 + body = b"Method Not Allowed" + await send( + { + "type": "http.response.start", + "status": status, + "headers": [(b"Content-Type", content_type)], + } + ) + await send({"type": "http.response.body", "body": body}) + else: + await send( + { + "type": "http.response.start", + "status": 404, + "headers": [(b"Content-Type", b"text/plain")], + } + ) + await send({"type": "http.response.body", "body": b"Not Found"}) diff --git a/tests/simple_async/main_test.py b/tests/simple_async/main_test.py new file mode 100644 index 0000000..673ae5f --- /dev/null +++ b/tests/simple_async/main_test.py @@ -0,0 +1,74 @@ +import os +import base64 +import uuid +import time +from concurrent.futures import ThreadPoolExecutor +import requests + +item_count = 0 + +BASE_URL = "http://localhost:9080" + +BIG_BLOB = base64.b64encode(os.urandom(4 * 2**20)).decode("utf") + + +def get_dummy_item() -> dict: + global item_count + item_count += 1 + return { + "name": f"Item {item_count}", + "description": f"Item Description {item_count}", + "blob": BIG_BLOB if item_count % 4 == 0 else None, + } + + +def store_item(id: str, item: dict): + response = requests.post(f"{BASE_URL}/item/{id}", json=item) + return response.status_code == 200 and b"Stored" in response.content + + +def get_item(id: str, item: dict): + response = requests.get(f"{BASE_URL}/item/{id}") + return response.status_code == 200 and response.json() == item + + +def delete_item(id: str): + response = requests.delete(f"{BASE_URL}/item/{id}") + return response.status_code == 200 and b"Deleted" in response.content + + +def item_lifecycle(): + id = str(uuid.uuid4()) + item = get_dummy_item() + assert store_item(id, item), "Store item failed" + assert get_item(id, item), "Get item failed" + assert delete_item(id), "Delete item failed" + assert not delete_item(id), "Delete item should fail" + + +def make_objects(max_workers: int, count: int): + start = time.time() + failed = False + + def item_done(fut): + exc = fut.exception() + if exc: + nonlocal failed + failed = True + raise SystemExit(1) from exc + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + for _ in range(count): + future = executor.submit(item_lifecycle) + future.add_done_callback(item_done) + + if failed: + print("Tests failed") + exit(1) + + print(f"Created and destroyed {count} objects") + print(f"Elapsed: {time.time()-start}s") + + +if __name__ == "__main__": + make_objects(max_workers=4, count=2_500) diff --git a/tests/simple_async/requirements.txt b/tests/simple_async/requirements.txt new file mode 100644 index 0000000..2179078 --- /dev/null +++ b/tests/simple_async/requirements.txt @@ -0,0 +1,5 @@ +certifi==2024.2.2 +charset-normalizer==3.3.2 +idna==3.7 +requests==2.31.0 +urllib3==2.2.1