Aws security token support

Makefile

@@ -1,6 +1,9 @@
 CC=gcc
 CFLAGS=-g -I${NGX_PATH}/src/os/unix -I${NGX_PATH}/src/core -I${NGX_PATH}/src/http -I${NGX_PATH}/src/http/modules -I${NGX_PATH}/src/event -I${NGX_PATH}/objs/ -I.
 
+docker-test:
+	docker build . -f docker/tests/Dockerfile -t ngx_aws_auth_tests \
+	&& docker run --rm --name ngx_aws_auth_tests ngx_aws_auth_tests
 
 all:
 

README.md

@@ -54,6 +54,21 @@ Implements proxying of authenticated requests to S3.
   }
 ```
 
+## Security Token Usage
+
+If you are using temporary credentials through something like an IAM role, this module
+supports this by using the `aws_security_token` directive. You specify this the same as
+you do the other directives, however, **you are responsible for recycling the credentials.**
+
+This _should_ be the normal usage of this module, as even with static credentials you
+need to regenerate the signing scope and key after a date change. An example of doing
+this via python and the `envsubst` command can be found in the examples folder as a
+docker image. The original use case for the example is to pull the credentials from
+a Fargate container environment, but it can be adapted to support EC2
+instances. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
+for more details on how to retrieve temporary credentials for EC2 instances assigned an
+IAM role.
+
 ## Security considerations
 The V4 protocol does not need access to the actual secret keys that one obtains
 from the IAM service. The correct way to use the IAM key is to actually generate
@@ -103,7 +118,11 @@ L4vRLWAO92X5L3Sqk5QydUSdB0nC9+1wfqLMOKLbRp4=
 The 2.x version of the module currently only has support for GET and HEAD calls. This is because
 signing request body is complex and has not yet been implemented.
 
+## Running Tests
 
+You should be able to run all of the tests for this module via a Docker container. If you run
+`make docker-test` from the root of this project, it will build and run the tests for this project
+via container.
 
 ## Credits
 Original idea based on http://nginx.org/pipermail/nginx/2010-February/018583.html and suggestion of moving to variables rather than patching the proxy module.

aws_functions.h

@@ -55,21 +55,31 @@ struct AwsSignedRequestDetails {
 
 // mainly useful to avoid having to full instantiate request structures for
 // tests...
+#ifndef NO_AWS_AUTH_LOGS
 #define safe_ngx_log_error(req, ...)                                  \
   if (req->connection) {                                              \
-    ngx_log_error(NGX_LOG_ERR, req->connection->log, 0, __VA_ARGS__); \
+	ngx_log_error(NGX_LOG_ERR, req->connection->log, 0, __VA_ARGS__); \
   }
+#else
+#define safe_ngx_log_error(req, ...)
+#endif
 
 static const ngx_str_t EMPTY_STRING_SHA256 = ngx_string("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
 static const ngx_str_t EMPTY_STRING = ngx_null_string;
 static const ngx_str_t AMZ_HASH_HEADER = ngx_string("x-amz-content-sha256");
 static const ngx_str_t AMZ_DATE_HEADER = ngx_string("x-amz-date");
+static const ngx_str_t AMZ_SECURITY_TOKEN_HEADER = ngx_string("x-amz-security-token");
 static const ngx_str_t HOST_HEADER = ngx_string("host");
 static const ngx_str_t AUTHZ_HEADER = ngx_string("authorization");
 
 static inline char* __CHAR_PTR_U(u_char* ptr) {return (char*)ptr;}
 static inline const char* __CONST_CHAR_PTR_U(const u_char* ptr) {return (const char*)ptr;}
 
+static inline void ngx_conditional_log(const ngx_http_request_t *req, ...) {
+	#ifndef NO_AWS_AUTH_LOGS
+	#endif
+}
+
 static inline const ngx_str_t* ngx_aws_auth__compute_request_time(ngx_pool_t *pool, const time_t *timep) {
 	ngx_str_t *const retval = ngx_palloc(pool, sizeof(ngx_str_t));
 	retval->data = ngx_palloc(pool, AMZ_DATE_MAX_LEN);
@@ -183,13 +193,14 @@ static inline struct AwsCanonicalHeaderDetails ngx_aws_auth__canonize_headers(ng
 		const ngx_http_request_t *req,
 		const ngx_str_t *s3_bucket, const ngx_str_t *amz_date,
 		const ngx_str_t *content_hash,
-    const ngx_str_t *s3_endpoint) {
+    	const ngx_str_t *s3_endpoint,
+		const ngx_str_t *security_token) {
 	size_t header_names_size = 1, header_nameval_size = 1;
 	size_t i, used;
 	u_char *buf_progress;
 	struct AwsCanonicalHeaderDetails retval;
 
-	ngx_array_t *settable_header_array = ngx_array_create(pool, 3, sizeof(header_pair_t));
+	ngx_array_t *settable_header_array = ngx_array_create(pool, 4, sizeof(header_pair_t));
 	header_pair_t *header_ptr;
 
 	header_ptr = ngx_array_push(settable_header_array);
@@ -200,6 +211,12 @@ static inline struct AwsCanonicalHeaderDetails ngx_aws_auth__canonize_headers(ng
 	header_ptr->key = AMZ_DATE_HEADER;
 	header_ptr->value = *amz_date;
 
+	if (ngx_strncmp(security_token, &EMPTY_STRING, 1) != 0) {
+		header_ptr = ngx_array_push(settable_header_array);
+		header_ptr->key = AMZ_SECURITY_TOKEN_HEADER;
+		header_ptr->value = *security_token;
+	}
+
 	header_ptr = ngx_array_push(settable_header_array);
 	header_ptr->key = HOST_HEADER;
 	header_ptr->value.len = s3_bucket->len + 60;
@@ -335,7 +352,8 @@ static inline const ngx_str_t* ngx_aws_auth__canon_url(ngx_pool_t *pool, const n
 
 static inline struct AwsCanonicalRequestDetails ngx_aws_auth__make_canonical_request(ngx_pool_t *pool,
 		const ngx_http_request_t *req,
-		const ngx_str_t *s3_bucket_name, const ngx_str_t *amz_date, const ngx_str_t *s3_endpoint) {
+		const ngx_str_t *s3_bucket_name, const ngx_str_t *amz_date,
+		const ngx_str_t *s3_endpoint, const ngx_str_t *security_token) {
 	struct AwsCanonicalRequestDetails retval;
 
 	// canonize query string
@@ -345,7 +363,7 @@ static inline struct AwsCanonicalRequestDetails ngx_aws_auth__make_canonical_req
 	const ngx_str_t *request_body_hash = ngx_aws_auth__request_body_hash(pool, req);
 
 	const struct AwsCanonicalHeaderDetails canon_headers =
-		ngx_aws_auth__canonize_headers(pool, req, s3_bucket_name, amz_date, request_body_hash, s3_endpoint);
+		ngx_aws_auth__canonize_headers(pool, req, s3_bucket_name, amz_date, request_body_hash, s3_endpoint, security_token);
 	retval.signed_header_names = canon_headers.signed_header_names;
 
 	const ngx_str_t *http_method = &(req->method_name);
@@ -397,12 +415,13 @@ static inline struct AwsSignedRequestDetails ngx_aws_auth__compute_signature(ngx
 		const ngx_str_t *signing_key,
 		const ngx_str_t *key_scope,
 		const ngx_str_t *s3_bucket_name,
-    const ngx_str_t *s3_endpoint) {
+    	const ngx_str_t *s3_endpoint,
+		const ngx_str_t *security_token) {
 	struct AwsSignedRequestDetails retval;
 
 	const ngx_str_t *date = ngx_aws_auth__compute_request_time(pool, &req->start_sec);
 	const struct AwsCanonicalRequestDetails canon_request =
-		ngx_aws_auth__make_canonical_request(pool, req, s3_bucket_name, date, s3_endpoint);
+		ngx_aws_auth__make_canonical_request(pool, req, s3_bucket_name, date, s3_endpoint, security_token);
 	const ngx_str_t *canon_request_hash = ngx_aws_auth__hash_sha256(pool, canon_request.canon_request);
 
 	// get string to sign
@@ -424,8 +443,10 @@ static inline const ngx_array_t* ngx_aws_auth__sign(ngx_pool_t *pool, ngx_http_r
 		const ngx_str_t *signing_key,
 		const ngx_str_t *key_scope,
 		const ngx_str_t *s3_bucket_name,
-    const ngx_str_t *s3_endpoint) {
-	const struct AwsSignedRequestDetails signature_details = ngx_aws_auth__compute_signature(pool, req, signing_key, key_scope, s3_bucket_name, s3_endpoint);
+    	const ngx_str_t *s3_endpoint,
+		const ngx_str_t *security_token) {
+	const struct AwsSignedRequestDetails signature_details =
+		ngx_aws_auth__compute_signature(pool, req, signing_key, key_scope, s3_bucket_name, s3_endpoint, security_token);
 
 
 	const ngx_str_t *auth_header_value = ngx_aws_auth__make_auth_token(pool, signature_details.signature,

docker/tests/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:18.04 as build
+RUN apt update -y &&\
+    apt-get install -yf \
+        build-essential \
+        wget \
+        git \
+        libpcre3-dev \
+        zlib1g-dev \
+        libssl-dev \
+        cmake \
+        sudo
+RUN wget https://nginx.org/download/nginx-1.16.1.tar.gz &&\
+    tar -xzvf nginx-1.16.1.tar.gz
+RUN git clone git://git.cryptomilk.org/projects/cmocka.git /cmocka
+COPY docker/tests/run-tests.sh /run-tests.sh
+COPY . /ngx_http_aws_auth_module
+ENV NGX_PATH=/nginx-1.16.1
+ENV LD_LIBRARY_PATH=/lib:/usr/lib:/usr/local/lib
+CMD [ "./run-tests.sh" ]

docker/tests/run-tests.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd $NGX_PATH
+./configure --with-http_ssl_module --with-compat --add-module=/ngx_http_aws_auth_module && make
+cd /ngx_http_aws_auth_module
+cp -r /cmocka vendor/
+make test

example/token-auto-refresh-python/.env

@@ -0,0 +1,7 @@
+BUCKET_DOMAIN=
+AWS_ACCESS_KEY=
+AWS_SECRET_ACCESS_KEY=
+AWS_SECURITY_TOKEN=
+AWS_TOKEN_EXPIRATION=2020-01-29T05:08:57Z
+AWS_REGION=
+USE_MOCK=false

example/token-auto-refresh-python/Dockerfile

@@ -0,0 +1,32 @@
+FROM ubuntu:18.04 as build
+RUN apt update -y &&\
+    apt-get install -yf \
+        build-essential \
+        wget \
+        git \
+        libpcre3-dev \
+        zlib1g-dev \
+        libssl-dev
+RUN wget https://nginx.org/download/nginx-1.16.1.tar.gz &&\
+    tar -xzvf nginx-1.16.1.tar.gz
+COPY . /ngx_http_aws_auth_module
+WORKDIR /nginx-1.16.1
+RUN ./configure --with-compat --add-dynamic-module=../ngx_http_aws_auth_module --with-cc-opt="-D NO_AWS_AUTH_LOGS" &&\
+    make modules
+
+FROM nginx:1.16.1-alpine
+RUN apk add --update \
+    python3 \
+    curl
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py &&\
+    python3 get-pip.py &&\
+    pip install boto3 &&\
+    pip install watchtower
+COPY example/token-auto-refresh-python/nginx /etc/nginx
+COPY example/token-auto-refresh-python/nginx /tmp/nginx
+COPY example/token-auto-refresh-python/startup.sh /startup.sh
+COPY --from=build /nginx-1.16.1/objs/ngx_http_aws_auth_module.so /etc/nginx/modules
+COPY example/token-auto-refresh-python/refresh_credentials.py /refresh_credentials.py
+EXPOSE 80
+STOPSIGNAL SIGTERM
+CMD [ "./startup.sh" ]

example/token-auto-refresh-python/README.md

@@ -0,0 +1,30 @@
+# Automatically Recycling IAM Credentials via Python
+
+This example sets up the nginx module to support passing the `x-amz-security-token` header when using
+temporary IAM credentials. It's original use case is to retrieve credentials from a Fargate container,
+but it can be adapted to support any environment.
+
+## Build
+
+You can build the environment with the following **from the root of the project**:
+
+`docker build . -f example/token-auto-refresh-python/Dockerfile -t nginx-aws-auth-refresh`
+
+## Run
+
+When deploying to Fargate, the only environment variable required is the BUCKET_DOMAIN, which
+should be your full bucket domain url.
+
+For example:
+
+`docker run --rm -p 5000:80 -e BUCKET_DOMAIN=public-encrypted-s3.s3.amazonaws.com nginx-aws-auth`
+
+### Run Locally
+
+If you would like to run this locally, you will need to retrieve your temporary credentials from
+your hosting instance. If you are using Fargate, you can try deploying the image from here (https://github.com/BrutalSimplicity/fargate-ssh), to
+allow you to ssh into the instance and view the credentials from that environment. Be sure to
+read on how IAM roles are managed for details on how to access that information (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html).
+
+After you have the credentials fill in the .env file in the root directory, and you can then run
+it locally with something like `docker run --rm -p 5000:80 --env-file=example/token-auto-refresh-python/.env nginx-aws-auth-refresh:latest`

example/token-auto-refresh-python/nginx/nginx.conf

@@ -0,0 +1,27 @@
+load_module modules/ngx_http_aws_auth_module.so;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+
+    keepalive_timeout  60;
+
+    include server.conf;
+}

example/token-auto-refresh-python/nginx/server.conf

@@ -0,0 +1,26 @@
+server {
+  listen 80 default_server;
+
+  location / {
+
+    proxy_pass https://${_AWS_BUCKET_DOMAIN};
+    proxy_set_header Host ${_AWS_BUCKET_DOMAIN};
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+    aws_sign;
+    aws_access_key ${_AWS_ACCESS_KEY};
+    aws_key_scope ${_AWS_SIGNING_SCOPE};
+    aws_signing_key ${_AWS_SIGNING_KEY};
+    aws_s3_bucket ${_AWS_BUCKET};
+    aws_security_token ${_AWS_SECURITY_TOKEN};
+
+    # add the cache status as a response header for debugging
+    add_header X-Cache-Status $upstream_cache_status;
+  }
+
+  error_page   500 502 503 504  /50x.html;
+  location = /50x.html {
+    root   /usr/share/nginx/html;
+  }
+}