Add 'native' support for faster AWS requests

.github/workflows/main.yml

@@ -21,5 +21,10 @@ jobs:
           version: ${{matrix.version}}
           architecture: ${{matrix.arch}}
 
-      - name: Run the tests on PicoLisp ${{matrix.arch}} v${{matrix.version}}
+      - name: Run the tests on PicoLisp (32-bit) v${{matrix.version}}
+        run: make test-multi
+        if: ${{ matrix.arch == 'src' }}
+
+      - name: Run the tests on PicoLisp (64-bit) v${{matrix.version}}
         run: make check
+        if: ${{ matrix.arch == 'src64' }}

CHANGELOG.md

@@ -1,8 +1,9 @@
 # Changelog
 
-## 0.9.0 (2020-06-03)
+## 0.10.0 (2020-06-05)
 
-  * Optionally output data to a file with '--output' parameter
+  * Optionally generate SHA256/HMAC hashes using 'libcrypto.so' with '--native' parameter
+  * Request creation is much faster with native hashing
 
 ## 0.8.0 (2020-06-03)
 

Makefile

@@ -10,15 +10,22 @@ TEST_REF = v3.0.0
 
 .PHONY: all
 
-all: check
+all: check run-multi run-native
 
 $(TEST_DIR):
 		mkdir -p $(TEST_DIR) && \
 		git clone $(TEST_REPO) $(TEST_DIR) && \
 		cd $(TEST_DIR) && \
 		git checkout $(TEST_REF)
 
-check: $(TEST_DIR) run-tests
+check: $(TEST_DIR) test-multi test-native
 
-run-tests:
+test-multi: $(TEST_DIR) run-multi
+
+run-multi:
 		./test.l
+
+test-native: $(TEST_DIR) run-native
+
+run-native:
+		./test.l native

README.md

@@ -15,6 +15,7 @@ This command line tool can be used to sign [AWS Signature Version 4](https://doc
 # Requirements
 
   * `picolisp`: 32-bit or 64-bit `v3.1.11+`, tested up to PicoLisp `v20.5.26`, [see test runs](https://github.com/aw/picolisp-awscurl/actions/)
+  * `libcrypto.so`: for using `--native` functions with PicoLisp `v17.12+`
   * `picolisp-unit`: `v3.0.0+` for testing the library
   * `openssl`: `v1.0.0+` for signing and hashing strings
   * `curl`: for sending requests to the AWS APIs
@@ -78,6 +79,7 @@ Options:
   --endpoint <endpoint>     The API endpoint of the AWS service (default: /)
   --header <key> <value>    HTTP header data (default: None)
   --host <host>             The Host of the AWS service (default: ec2.amazonaws.com)
+  --native"                 Use faster 'native' calls for hashing data (64-bit version only, default: False)
   --output <file>           Filename where data should be output (default: STDOUT)
   --protocol http|https     Protocol for talking to AWS (default: https)
   --query <query>           The Query parameters of the AWS service (default: None)

awscurl.l

@@ -2,7 +2,7 @@
 
 [de APP_INFO
   ("name"      "awscurl")
-  ("version"   "0.9.0")
+  ("version"   "0.10.0")
   ("summary"   "PicoLisp AWS CLI tool using OpenSSL and Curl")
   ("source"    "https://github.com/aw/picolisp-awscurl")
   ("author"    "Alexander Williams")
@@ -18,6 +18,7 @@
                ("--endpoint <endpoint>"   "The API endpoint of the AWS service (default: /)")
                ("--header <key> <value>"  "HTTP header data (default: None)")
                ("--host <host>"           "The Host of the AWS service (default: ec2.amazonaws.com)")
+               ("--native"                "Use faster 'native' calls for hashing data (64-bit version only, default: False)")
                ("--output <file>"         "Filename where data should be output (default: STDOUT)")
                ("--protocol http|https"   "Protocol for talking to AWS (default: https)")
                ("--query <query>"         "The Query parameters of the AWS service (default: None)")
@@ -52,6 +53,7 @@
       (while (opt)
         (case @
           (--verbose  (on *Aws_verbose))
+          (--native   (on *Aws_native))
           (--request  (setq *Aws_method   (opt)))
           (--data     (awscurl-data       (opt)))
           (--header   (awscurl-headers    (opt) (opt)))

libawscurl.l

@@ -17,38 +17,75 @@
   *Aws_request_date     (dat$ (date T))
   *Aws_request_time     (pack (mapcar '((S) (pad 2 S)) (time (time T)))) )
 
+### native
+[de awscurl-native-sha256 (Payload)
+  (native "libcrypto.so" "SHA256" '(B . 32) Payload (length Payload) '(NIL (32) ]
+
+[de awscurl-native-hmac (Key Length Payload)
+  (let Evp (native "libcrypto.so" "EVP_sha256" 'N)
+    (native "libcrypto.so" "HMAC" '(B . 32) Evp Key Length Payload (length Payload) 0 '(NIL (32) ]
+
+# Key = hex string
+[de awscurl-native-make-hmac (Key Payload)
+  (let (*Key        (chop Key)
+        *Iterations (/ (length *Key) 2)
+        *P          (native "@" "malloc" 'N *Iterations) ) # allocate some memory to store the key
+
+    (for N *Iterations
+      [byte (+ *P (- N 1)) (hex (pack (cut 2 '*Key) ] # store the hex value in a byte
+    )
+    (byte (+ *P *Iterations 1) 0) # store a null byte
+    (let Result (awscurl-list-to-hex (awscurl-native-hmac *P *Iterations Payload))
+      (native "@" "free" NIL *P) # free the memory
+      Result ]
+
+[de awscurl-parse-native (Payload Key)
+  (if Key
+      (awscurl-native-make-hmac Key Payload)
+      (awscurl-list-to-hex (awscurl-native-sha256 Payload) ]
+
 ### main
 [de awscurl-make-curl (Cmd)
   (prin (in Cmd (till NIL T) ]
 
 [de awscurl-make-request (Auth Headers)
   (make
-    (link 'curl "--silent" "--request" *Aws_method (pack *Aws_protocol "://" *Aws_host *Aws_endpoint (when *Aws_query (pack "?" @))) "--header" Auth)
+    (link 'curl "--silent" "--request" *Aws_method (pack *Aws_protocol "://" *Aws_host *Aws_endpoint (when *Aws_query (pack "?" *Aws_query))) "--header" Auth)
     (mapc '((S) (link "--header" (pack (car S) ": " (cdr S)))) Headers)
     (when *Aws_data     (link "--data" (cdr *Aws_data)))
     (when *Aws_verbose  (link "--verbose"))
-    (when *Aws_output   (link "--output" @) ]
+    (when *Aws_output   (link "--output" *Aws_output) ]
 
 [de awscurl-make-auth-header (Headers Signature)
    (pack "Authorization: AWS4-HMAC-SHA256 Credential=" *Aws_access_key "/" *Aws_request_date "/" *Aws_region "/" *Aws_service "/aws4_request, SignedHeaders=" (glue ";"  (mapcar car Headers)) ", Signature=" Signature) ]
 
+[de awscurl-list-to-hex (List)
+  (lowc (pack (mapcar '((N) (pad 2 (hex N))) List) ]
+
+[de awscurl-string-to-hex (String)
+  (awscurl-list-to-hex (mapcar char (chop String) ]
+
 [de awscurl-make-hmac (Key Payload)
-  (awscurl-parse-openssl Payload "-mac" "HMAC" "-macopt" Key) ]
+  (if *Aws_native
+      (awscurl-parse-native Payload Key)
+      (awscurl-parse-openssl Payload "-mac" "HMAC" "-macopt" (pack "hexkey:" Key) ]
 
 [de awscurl-make-signature (Str2sign)
-  (let (A (awscurl-make-hmac (pack "key:AWS4" *Aws_secret_key) *Aws_request_date)
-        B (awscurl-make-hmac (pack "hexkey:" A) *Aws_region)
-        C (awscurl-make-hmac (pack "hexkey:" B) *Aws_service)
-        D (awscurl-make-hmac (pack "hexkey:" C) "aws4_request") )
+  (let (A (awscurl-make-hmac (awscurl-string-to-hex (pack "AWS4" *Aws_secret_key)) *Aws_request_date)
+        B (awscurl-make-hmac A *Aws_region)
+        C (awscurl-make-hmac B *Aws_service)
+        D (awscurl-make-hmac C "aws4_request") )
 
-    (awscurl-make-hmac (pack "hexkey:" D) Str2sign) ]
+    (awscurl-make-hmac D Str2sign) ]
 
 [de awscurl-make-canonical-hash (Canonical)
   (pack
     "AWS4-HMAC-SHA256" "^J"
     *Aws_request_date "T" *Aws_request_time "Z" "^J"
     *Aws_request_date "/" *Aws_region "/" *Aws_service "/aws4_request" "^J"
-    (awscurl-parse-openssl Canonical) ]
+    (if *Aws_native
+        (awscurl-parse-native Canonical)
+        (awscurl-parse-openssl Canonical) ]
 
 [de awscurl-make-canonical (Hash Headers)
   (pack
@@ -67,7 +104,7 @@
         (cons "x-amz-content-sha256" Sha256)
         (cons "x-amz-date" (pack *Aws_request_date "T" *Aws_request_time "Z")) )
       (when *Aws_data           (link (cons "content-length" (if (= "file" (car *Aws_data)) (car (info (cdr *Aws_data))) (length (cdr *Aws_data))))))
-      (when *Aws_session_token  (link (cons "x-amz-security-token" @)))
+      (when *Aws_session_token  (link (cons "x-amz-security-token" *Aws_session_token)))
       (mapc link *Aws_headers) ]
 
 [de awscurl-file-openssl (Filename)
@@ -91,7 +128,11 @@
 
 [de awscurl-start ()
   (awscurl-get-credentials)
-  (let (Sha256    (if (= "file" (car *Aws_data)) (awscurl-file-openssl (cdr *Aws_data)) (awscurl-parse-openssl (cdr *Aws_data)))
+  (let (Sha256    (if (= "file" (car *Aws_data))
+                      (awscurl-file-openssl (cdr *Aws_data))
+                      (if *Aws_native
+                          (awscurl-parse-native (cdr *Aws_data))
+                          (awscurl-parse-openssl (cdr *Aws_data)) ) )
         Headers   (awscurl-sort-headers Sha256)
         Canonical (awscurl-make-canonical Sha256 Headers)
         Str2sign  (awscurl-make-canonical-hash Canonical)
@@ -103,7 +144,7 @@
 [de awscurl-data (Value)
   (let Data (chop Value)
     (setq *Aws_data
-      (if (and (= "@" (car Data)) (info (pack (cdr Data))))
+      (if (and (= (char 64) (car Data)) (info (pack (cdr Data))))
           (cons "file" (pack (cdr Data)))
           (cons "data" Value) ]
 

test.l

@@ -9,6 +9,8 @@
   *Aws_request_date "20180713"
   *Aws_request_time "052018" )
 
+(when (= "native" (opt)) (on *Aws_native))
+
 # Default HOME to 'test' dir
 (sys "HOME" "test")
 
@@ -86,6 +88,10 @@
   '(test-headers-normalized)
   '(test-data-read)
   '(test-get-credentials)
+  (when *Aws_native
+    '(assert-equal "cf80cd8aed482d5d1527d7dc72fceff84e6326592848447d2dc0b0e87dfc9a90" (awscurl-parse-native "testing") "Ensure OpenSSL hashes a string (native)")
+    '(assert-equal "8f19bc12c7f2f3f31c2435645ff0e457d4e87f676b08d9db4ba3271bb2c3e87a" (awscurl-parse-native "testing" "cf80cd8aed482d5d1527d7dc72fceff84e6326592848447d2dc0b0e87dfc9a90") "Ensure OpenSSL hashes a string with HMAC (native)")
+  )
   '(assert-equal "cf80cd8aed482d5d1527d7dc72fceff84e6326592848447d2dc0b0e87dfc9a90" (awscurl-parse-openssl "testing") "Ensure OpenSSL hashes a string")
   '(assert-equal "8f19bc12c7f2f3f31c2435645ff0e457d4e87f676b08d9db4ba3271bb2c3e87a" (awscurl-parse-openssl "testing" "-mac" "HMAC" "-macopt" "hexkey:cf80cd8aed482d5d1527d7dc72fceff84e6326592848447d2dc0b0e87dfc9a90") "Ensure OpenSSL hashes a string with HMAC")
   '(assert-equal "9a78a98c394ec90a0bc17d994c257bfbe9aa6f30f6837efb37c1f46ad86f75c6" (awscurl-file-openssl "test/data.json") "Ensure OpenSSL hashes a file")