Add integration tests to GitHub Actions (#476)

Motivation:

Testing is good - we should check results on PRs.

Modifications:

Add integration tests to PR and scheduled GitHub Actions workflows.

Result:
Integration tests run.

.github/workflows/main.yml

@@ -25,3 +25,11 @@ jobs:
     with:
       name: "HTTP/2 spec tests"
       matrix_linux_command: "apt-get update -y -q && apt-get install -y -q wget lsof && mkdir $HOME/.tools && wget -q https://github.com/summerwind/h2spec/releases/download/v2.2.1/h2spec_linux_amd64.tar.gz -O $HOME/.tools/h2spec.tar.gz && tar xzf $HOME/.tools/h2spec.tar.gz --directory $HOME/.tools && PATH=${PATH}:$HOME/.tools && ./scripts/test_h2spec.sh"
+
+  integration-tests:
+    name: Integration Tests
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    with:
+      name: "Integration tests"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q  jq && ./scripts/integration_tests.sh"

.github/workflows/pull_request.yml

@@ -33,3 +33,11 @@ jobs:
     with:
       name: "HTTP/2 spec tests"
       matrix_linux_command: "apt-get update -y -q && apt-get install -y -q wget lsof && mkdir $HOME/.tools && wget -q https://github.com/summerwind/h2spec/releases/download/v2.2.1/h2spec_linux_amd64.tar.gz -O $HOME/.tools/h2spec.tar.gz && tar xzf $HOME/.tools/h2spec.tar.gz --directory $HOME/.tools && PATH=${PATH}:$HOME/.tools && ./scripts/test_h2spec.sh"
+
+  integration-tests:
+    name: Integration Tests
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    with:
+      name: "Integration tests"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q  jq && ./scripts/integration_tests.sh"

.github/workflows/scheduled.yml

@@ -25,3 +25,11 @@ jobs:
     with:
       name: "HTTP/2 spec tests"
       matrix_linux_command: "apt-get update -y -q && apt-get install -y -q wget lsof && mkdir $HOME/.tools && wget -q https://github.com/summerwind/h2spec/releases/download/v2.2.1/h2spec_linux_amd64.tar.gz -O $HOME/.tools/h2spec.tar.gz && tar xzf $HOME/.tools/h2spec.tar.gz --directory $HOME/.tools && PATH=${PATH}:$HOME/.tools && ./scripts/test_h2spec.sh"
+
+  integration-tests:
+    name: Integration Tests
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    with:
+      name: "Integration tests"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q  jq && ./scripts/integration_tests.sh"

IntegrationTests/tests_01_allocation_counters/Thresholds/5.10.json

@@ -0,0 +1,23 @@
+{
+    "1k_requests_inline_interleaved": 30150,
+    "1k_requests_inline_noninterleaved": 29100,
+    "1k_requests_interleaved": 36150,
+    "1k_requests_noninterleaved": 35100,
+    "client_server_h1_request_response": 276050,
+    "client_server_h1_request_response_inline": 261050,
+    "client_server_request_response": 245050,
+    "client_server_request_response_inline": 236050,
+    "client_server_request_response_many": 1190050,
+    "client_server_request_response_many_inline": 881050,
+    "create_client_stream_channel": 35050,
+    "create_client_stream_channel_inline": 35050,
+    "create_client_stream_channel_inline_no_promise_based_API": 35050,
+    "create_client_stream_channel_no_promise_based_API": 35050,
+    "get_100000_headers_canonical_form": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 300050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 200050,
+    "hpack_decoding": 5050,
+    "stream_teardown_100_concurrent": 262550,
+    "stream_teardown_100_concurrent_inline": 261650
+}

IntegrationTests/tests_01_allocation_counters/Thresholds/5.9.json

@@ -0,0 +1,23 @@
+{
+    "1k_requests_inline_interleaved": 30150,
+    "1k_requests_inline_noninterleaved": 29100,
+    "1k_requests_interleaved": 36150,
+    "1k_requests_noninterleaved": 35100,
+    "client_server_h1_request_response": 276050,
+    "client_server_h1_request_response_inline": 261050,
+    "client_server_request_response": 245050,
+    "client_server_request_response_inline": 236050,
+    "client_server_request_response_many": 1190050,
+    "client_server_request_response_many_inline": 881050,
+    "create_client_stream_channel": 35050,
+    "create_client_stream_channel_inline": 35050,
+    "create_client_stream_channel_inline_no_promise_based_API": 35050,
+    "create_client_stream_channel_no_promise_based_API": 35050,
+    "get_100000_headers_canonical_form": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 300050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 200050,
+    "hpack_decoding": 5050,
+    "stream_teardown_100_concurrent": 262550,
+    "stream_teardown_100_concurrent_inline": 261650
+}

IntegrationTests/tests_01_allocation_counters/Thresholds/6.0.json

@@ -0,0 +1,23 @@
+{
+    "1k_requests_inline_interleaved": 30150,
+    "1k_requests_inline_noninterleaved": 29100,
+    "1k_requests_interleaved": 36150,
+    "1k_requests_noninterleaved": 35100,
+    "client_server_h1_request_response": 276050,
+    "client_server_h1_request_response_inline": 261050,
+    "client_server_request_response": 245050,
+    "client_server_request_response_inline": 236050,
+    "client_server_request_response_many": 1190050,
+    "client_server_request_response_many_inline": 881050,
+    "create_client_stream_channel": 35050,
+    "create_client_stream_channel_inline": 35050,
+    "create_client_stream_channel_inline_no_promise_based_API": 35050,
+    "create_client_stream_channel_no_promise_based_API": 35050,
+    "get_100000_headers_canonical_form": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 300050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 200050,
+    "hpack_decoding": 5050,
+    "stream_teardown_100_concurrent": 262450,
+    "stream_teardown_100_concurrent_inline": 261750
+}

IntegrationTests/tests_01_allocation_counters/Thresholds/nightly-6.0.json

@@ -0,0 +1,23 @@
+{
+    "1k_requests_inline_interleaved": 30150,
+    "1k_requests_inline_noninterleaved": 29100,
+    "1k_requests_interleaved": 36150,
+    "1k_requests_noninterleaved": 35100,
+    "client_server_h1_request_response": 276050,
+    "client_server_h1_request_response_inline": 261050,
+    "client_server_request_response": 245050,
+    "client_server_request_response_inline": 236050,
+    "client_server_request_response_many": 1190050,
+    "client_server_request_response_many_inline": 881050,
+    "create_client_stream_channel": 35050,
+    "create_client_stream_channel_inline": 35050,
+    "create_client_stream_channel_inline_no_promise_based_API": 35050,
+    "create_client_stream_channel_no_promise_based_API": 35050,
+    "get_100000_headers_canonical_form": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 300050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 200050,
+    "hpack_decoding": 5050,
+    "stream_teardown_100_concurrent": 262450,
+    "stream_teardown_100_concurrent_inline": 261750
+}

IntegrationTests/tests_01_allocation_counters/Thresholds/nightly-main.json

@@ -0,0 +1,23 @@
+{
+    "1k_requests_inline_interleaved": 30150,
+    "1k_requests_inline_noninterleaved": 29100,
+    "1k_requests_interleaved": 36150,
+    "1k_requests_noninterleaved": 35100,
+    "client_server_h1_request_response": 276050,
+    "client_server_h1_request_response_inline": 261050,
+    "client_server_request_response": 245050,
+    "client_server_request_response_inline": 236050,
+    "client_server_request_response_many": 1190050,
+    "client_server_request_response_many_inline": 881050,
+    "create_client_stream_channel": 35050,
+    "create_client_stream_channel_inline": 35050,
+    "create_client_stream_channel_inline_no_promise_based_API": 35050,
+    "create_client_stream_channel_no_promise_based_API": 35050,
+    "get_100000_headers_canonical_form": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 200050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 300050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 200050,
+    "hpack_decoding": 5050,
+    "stream_teardown_100_concurrent": 262450,
+    "stream_teardown_100_concurrent_inline": 261750
+}

IntegrationTests/tests_01_allocation_counters/test_01_allocation_counts.sh

@@ -18,6 +18,8 @@ source defines.sh
 set -eu
 here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+test -n "${SWIFT_VERSION:-}" || fatal "SWIFT_VERSION unset"
+
 all_tests=()
 for file in "$here/test_01_resources/"test_*.swift; do
     test_name=$(basename "$file")
@@ -36,20 +38,24 @@ for test in "${all_tests[@]}"; do
         total_allocations=$(grep "^test_$test_case.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
         not_freed_allocations=$(grep "^test_$test_case.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
         max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test_case"
+        max_allowed=$(jq '.'\""$test_case"\" "$here/Thresholds/$SWIFT_VERSION.json")
 
         info "$test_case: allocations not freed: $not_freed_allocations"
         info "$test_case: total number of mallocs: $total_allocations"
 
         assert_less_than "$not_freed_allocations" 5     # allow some slack
         assert_greater_than "$not_freed_allocations" -5 # allow some slack
-        if [[ -z "${!max_allowed_env_name+x}" ]]; then
+        if [[ -z "${!max_allowed_env_name+x}" ]] && [ -z "${max_allowed}" ]; then
             if [[ -z "${!max_allowed_env_name+x}" ]]; then
                 warn "no reference number of allocations set (set to \$$max_allowed_env_name)"
-                warn "to set current number:"
+                warn "to set current number either:"
                 warn "    export $max_allowed_env_name=$total_allocations"
+                warn "    or set them in the Swift version specific threshold json"
             fi
         else
-            max_allowed=${!max_allowed_env_name}
+            if [ -z "${max_allowed}" ]; then
+                max_allowed=${!max_allowed_env_name}
+            fi
             assert_less_than_or_equal "$total_allocations" "$max_allowed"
             assert_greater_than "$total_allocations" "$(( max_allowed - 1000))"
         fi