diff --git a/.github/workflows/build-and-test-docker.yml b/.github/workflows/build-and-test-docker.yml index accfc1a0..89a93096 100644 --- a/.github/workflows/build-and-test-docker.yml +++ b/.github/workflows/build-and-test-docker.yml @@ -9,7 +9,7 @@ on: - main jobs: - build-and-test: + build-docker-and-test: name: Build curl-impersonate Docker images and run the tests runs-on: ubuntu-latest services: diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 8a15e688..2d4c89d1 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -14,29 +14,52 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-20.04, macos-11] + include: + - os: ubuntu-20.04 + capture_interface: eth0 + make: make + - os: macos-11 + capture_interface: en0 + make: gmake steps: + - uses: actions/setup-python@v3 + - name: Install Ubuntu dependencies if: matrix.os == 'ubuntu-20.04' run: | - sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool - # Firefox version dependencies - sudo apt install python3-pip python-is-python3 - pip install gyp-next + sudo apt-get update + sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool + # Chrome version dependencies + sudo apt-get install golang-go + # Needed to compile 'minicurl' + sudo apt-get install libcurl4-openssl-dev + # More dependencies for the tests + sudo apt-get install tcpdump nghttp2-server libnss3 + + - name: Install macOS dependencies + if: matrix.os == 'macos-11' + run: | + brew install pkg-config make cmake ninja autoconf automake libtool # Chrome version dependencies - sudo apt install golang-go + # (Go is already installed) + # brew install go # Needed to compile 'minicurl' - sudo apt install libcurl4-openssl-dev + brew install curl # More dependencies for the tests - sudo apt install tcpdump nghttp2-server libnss3 + brew install tcpdump nghttp2 nss + + - name: Install common dependencies + run: | + # Firefox version dependencies + pip3 install gyp-next - name: Check out the repo uses: actions/checkout@v2 - name: Install dependencies for the tests script run: | - # Install globally so that we can run 'pytest' with 'sudo' - sudo pip install -r tests/requirements.txt + pip3 install -r tests/requirements.txt - name: Run configure script run: | @@ -46,15 +69,15 @@ jobs: - name: Build the Chrome version of curl-impersonate run: | - make chrome-build - make chrome-checkbuild - make chrome-install + ${{ matrix.make }} chrome-build + ${{ matrix.make }} chrome-checkbuild + ${{ matrix.make }} chrome-install - name: Build the Firefox version of curl-impersonate run: | - make firefox-build - make firefox-checkbuild - make firefox-install + ${{ matrix.make }} firefox-build + ${{ matrix.make }} firefox-checkbuild + ${{ matrix.make }} firefox-install - name: Prepare the tests run: | @@ -65,4 +88,5 @@ jobs: run: | cd tests # sudo is needed for capturing packets - sudo pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp}}/install + python_bin=$(which python3) + sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp}}/install --capture-interface ${{ matrix.capture_interface }} diff --git a/Dockerfile.template b/Dockerfile.template index ed93b0c3..f4b05c09 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -82,11 +82,7 @@ ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_75_RTM/src/n RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL} RUN tar xf ${NSS_VERSION}.tar.gz && \ cd ${NSS_VERSION}/nss && \ -{{#alpine}} - # Hack to make nss compile on alpine with python3 - ln -sf python3 /usr/bin/python && \ -{{/alpine}} - ./build.sh -o --disable-tests --static + ./build.sh -o --disable-tests --static --python=python3 {{/firefox}} {{#chrome}} # BoringSSL doesn't have versions. Choose a commit that is used in a stable diff --git a/INSTALL.md b/INSTALL.md index cac475b2..59186299 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -16,7 +16,7 @@ Install dependencies for building all the components: ``` sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool # For the Firefox version only -sudo apt install python3-pip python-is-python3 +sudo apt install python3-pip libnss3 pip install gyp-next export PATH="$PATH:~/.local/bin" # Add gyp to PATH # For the Chrome version only @@ -65,11 +65,12 @@ curl-impersonate-chrome https://www.wikipedia.org ``` ### macOS -*macOS support is still a work in progress and currently supports the Chrome version only.* - Install dependencies for building all the components: ``` brew install pkg-config make cmake ninja autoconf automake libtool +# For the Firefox version only +brew install sqlite nss +pip3 install gyp-next # For the Chrome version only brew install go ``` @@ -83,6 +84,9 @@ Configure and compile: ``` mkdir build && cd build ../configure +# Build and install the Firefox version +gmake firefox-build +sudo gmake firefox-install # Build and install the Chrome version gmake chrome-build sudo gmake chrome-install @@ -93,6 +97,17 @@ cd ../ && rm -Rf build ### Static compilation To compile curl-impersonate statically with libcurl-impersonate, pass `--enable-static` to the `configure` script. +### A note about the Firefox version +The Firefox version compiles a static version of nss, Firefox's TLS library. +For NSS to have a list of root certificates, curl attempts to load at runtime `libnssckbi`, one of the NSS libraries. +If you get the error: +``` +curl: (60) Peer's Certificate issuer is not recognized +``` +Make sure that NSS is installed (see above). +If the issue persists it might be that NSS is installed in a non-standard location on your system. +Please open an issue in that case. + ## Docker build The Docker build is a bit more reproducible and serves as the reference implementation. It creates a Debian-based Docker image with the binaries. diff --git a/Makefile.in b/Makefile.in index ef4c92d3..420ecd38 100644 --- a/Makefile.in +++ b/Makefile.in @@ -126,7 +126,10 @@ $(NSS_VERSION).tar.gz: $(nss_static_libs): $(NSS_VERSION).tar.gz tar xf $(NSS_VERSION).tar.gz cd $(NSS_VERSION)/nss - ./build.sh -o --disable-tests --static + ./build.sh -o --disable-tests --static --python=python3 + # Hack for macOS: Remove dynamic libraries to force the linker to use the + # static ones when linking curl. + rm -Rf $(nss_install_dir)/lib/*.dylib boringssl.zip: diff --git a/README.md b/README.md index 7c5e7b5d..d6e1485f 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ You can call it with the target names, e.g. `chrome98`, and it will internally s Note that if you call `curl_easy_setopt()` later with one of the above it will override the options set by `curl_easy_impersonate()`. ### Using CURL_IMPERSONATE env var -*Experimental*: If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD`. You can then set the `CURL_IMPERSONATE` env var. For example: +*Experimental*: If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD` (Linux only). You can then set the `CURL_IMPERSONATE` env var. For example: ```bash LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome98 my_app ``` diff --git a/firefox/Dockerfile b/firefox/Dockerfile index ee426b2e..3ade9dab 100644 --- a/firefox/Dockerfile +++ b/firefox/Dockerfile @@ -48,7 +48,7 @@ ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_75_RTM/src/n RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL} RUN tar xf ${NSS_VERSION}.tar.gz && \ cd ${NSS_VERSION}/nss && \ - ./build.sh -o --disable-tests --static + ./build.sh -o --disable-tests --static --python=python3 ARG NGHTTP2_VERSION=nghttp2-1.46.0 ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2 diff --git a/firefox/Dockerfile.alpine b/firefox/Dockerfile.alpine index e8563044..ed9764c8 100644 --- a/firefox/Dockerfile.alpine +++ b/firefox/Dockerfile.alpine @@ -41,9 +41,7 @@ ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_75_RTM/src/n RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL} RUN tar xf ${NSS_VERSION}.tar.gz && \ cd ${NSS_VERSION}/nss && \ - # Hack to make nss compile on alpine with python3 - ln -sf python3 /usr/bin/python && \ - ./build.sh -o --disable-tests --static + ./build.sh -o --disable-tests --static --python=python3 ARG NGHTTP2_VERSION=nghttp2-1.46.0 ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2 diff --git a/firefox/patches/curl-impersonate.patch b/firefox/patches/curl-impersonate.patch index d89fc1ce..ba6b6045 100644 --- a/firefox/patches/curl-impersonate.patch +++ b/firefox/patches/curl-impersonate.patch @@ -760,7 +760,7 @@ index cc9c88870..a35a20e10 100644 killed. */ struct dynamically_allocated_data { diff --git a/lib/vtls/nss.c b/lib/vtls/nss.c -index 2b44f0512..4c60797c7 100644 +index 2b44f0512..eec2bf76f 100644 --- a/lib/vtls/nss.c +++ b/lib/vtls/nss.c @@ -143,6 +143,7 @@ static const struct cipher_s cipherlist[] = { @@ -867,32 +867,15 @@ index 2b44f0512..4c60797c7 100644 /* * Return true if at least one cipher-suite is enabled. Used to determine * if we need to call NSS_SetDomesticPolicy() to enable the default ciphers. -@@ -1320,6 +1410,24 @@ static CURLcode nss_load_module(SECMODModule **pmod, const char *library, +@@ -1320,6 +1410,7 @@ static CURLcode nss_load_module(SECMODModule **pmod, const char *library, if(module) SECMOD_DestroyModule(module); -+ -+ /* Patch for Ubuntu - add a "nss/" suffix to the library name */ -+ config_string = aprintf("library=/usr/lib/x86_64-linux-gnu/nss/%s name=%s", library, name); -+ if(!config_string) -+ return CURLE_OUT_OF_MEMORY; -+ -+ module = SECMOD_LoadUserModule(config_string, NULL, PR_FALSE); -+ free(config_string); -+ -+ if(module && module->loaded) { -+ /* loaded successfully */ -+ *pmod = module; -+ return CURLE_OK; -+ } -+ -+ if(module) -+ SECMOD_DestroyModule(module); + return CURLE_FAILED_INIT; } -@@ -1921,6 +2029,12 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, +@@ -1921,6 +2012,12 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, if(SSL_OptionSet(model, SSL_NO_CACHE, ssl_no_cache) != SECSuccess) goto error; @@ -905,7 +888,7 @@ index 2b44f0512..4c60797c7 100644 /* enable/disable the requested SSL version(s) */ if(nss_init_sslver(&sslver, data, conn) != CURLE_OK) goto error; -@@ -1960,6 +2074,14 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, +@@ -1960,6 +2057,14 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, } } @@ -920,7 +903,7 @@ index 2b44f0512..4c60797c7 100644 if(!SSL_CONN_CONFIG(verifypeer) && SSL_CONN_CONFIG(verifyhost)) infof(data, "warning: ignoring value of ssl.verifyhost"); -@@ -2113,6 +2235,10 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, +@@ -2113,6 +2218,10 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, int cur = 0; unsigned char protocols[128]; @@ -931,7 +914,7 @@ index 2b44f0512..4c60797c7 100644 #ifdef USE_HTTP2 if(data->state.httpwant >= CURL_HTTP_VERSION_2 #ifndef CURL_DISABLE_PROXY -@@ -2124,9 +2250,6 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, +@@ -2124,9 +2233,6 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, cur += ALPN_H2_LENGTH; } #endif @@ -954,19 +937,119 @@ index 8ac15d407..68d01b219 100644 Libs.private: @LIBCURL_LIBS@ Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@ diff --git a/m4/curl-nss.m4 b/m4/curl-nss.m4 -index 397ba71b1..abc09a91c 100644 +index 397ba71b1..d2a8fc1f2 100644 --- a/m4/curl-nss.m4 +++ b/m4/curl-nss.m4 -@@ -74,7 +74,7 @@ if test "x$OPT_NSS" != xno; then +@@ -74,7 +74,107 @@ if test "x$OPT_NSS" != xno; then # Without pkg-config, we'll kludge in some defaults AC_MSG_WARN([Using hard-wired libraries and compilation flags for NSS.]) addld="-L$OPT_NSS/lib" - addlib="-lssl3 -lsmime3 -lnss3 -lplds4 -lplc4 -lnspr4" -+ addlib="-Wl,-Bstatic -Wl,--start-group -lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lsmime -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lsha-x86_c_lib -lgcm-aes-x86_c_lib -lhw-acc-crypto-avx -lhw-acc-crypto-avx2 -lnssutil -lnssb -lcryptohi -l:libplc4.a -l:libplds4.a -l:libnspr4.a -lsqlite -Wl,--end-group -Wl,-Bdynamic -pthread -ldl" ++ ++ # curl-impersonate: Link NSS statically. ++ # NSS is poorly documented in this regard and a lot of trial and error ++ # was made to come up with the correct list of linking flags. The ++ # libraries have circular dependencies which makes their order extremely ++ # difficult to find out. ++ ++ # Some references: ++ # https://github.com/mozilla/application-services/blob/b2690fd2e4cc3e8e10b6868ab0de8b79c89d3a93/components/support/rc_crypto/nss/nss_build_common/src/lib.rs#L94 ++ # and ++ # https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/freebl/freebl.gyp ++ ++ # On Linux we can use special linker flags to force static linking ++ # (-l:libplc4.a etc.), otherwise the linker will prefer to use ++ # libplc4.so. On other systems the dynamic libraries would have to be ++ # removed manually from the NSS directory before building curl. ++ case $host_os in ++ linux*) ++ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -l:libplc4.a -l:libplds4.a -l:libnspr4.a -lsqlite" ++ ;; ++ darwin*) ++ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4" ++ ;; ++ *) ++ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4 -lsqlite" ++ ;; ++ esac ++ ++ case $host_cpu in ++ arm) ++ addlib="$addlib -larmv8_c_lib" ++ ;; ++ aarch64) ++ addlib="$addlib -larmv8_c_lib -lgcm-aes-aarch64_c_lib" ++ ;; ++ x86) ++ addlib="$addlib -lgcm-aes-x86_c_lib" ++ ;; ++ x86_64) ++ addlib="$addlib -lgcm-aes-x86_c_lib -lhw-acc-crypto-avx -lhw-acc-crypto-avx2 -lsha-x86_c_lib" ++ case $host_os in ++ linux*) ++ addlib="$addlib -lintel-gcm-wrap_c_lib -lintel-gcm-s_lib" ++ ;; ++ esac ++ ;; ++ esac ++ ++ # curl-impersonate: ++ # On Linux these linker flags are necessary to resolve ++ # the symbol mess and circular dependencies of NSS .a libraries ++ # to make the AC_CHECK_LIB test below pass. ++ case $host_os in ++ linux*) ++ addlib="-Wl,--start-group $addlib -Wl,--end-group" ++ ;; ++ esac ++ ++ # External dependencies for nss ++ case $host_os in ++ linux*) ++ addlib="$addlib -pthread -ldl" ++ ;; ++ darwin*) ++ addlib="$addlib -lsqlite3" ++ ;; ++ esac ++ ++ # Attempt to locate libnssckbi. ++ # This library file contains the trusted certificates and nss loads it ++ # at runtime using dlopen. If it's not in a path findable by dlopen ++ # we have to add that path explicitly using -rpath so it may find it. ++ # On Ubuntu and Mac M1 it is in a non-standard location. ++ AC_MSG_CHECKING([if libnssckbi is in a non-standard location]) ++ case $host_os in ++ linux*) ++ search_paths="/usr/lib/$host /usr/lib/$host/nss" ++ search_paths="$search_paths /usr/lib/$host_cpu-$host_os" ++ search_paths="$search_paths /usr/lib/$host_cpu-$host_os/nss" ++ search_ext="so" ++ ;; ++ darwin*) ++ search_paths="/opt/homebrew/lib" ++ search_ext="dylib" ++ ;; ++ esac ++ ++ found="no" ++ for path in $search_paths; do ++ if test -f "$path/libnssckbi.$search_ext"; then ++ AC_MSG_RESULT([$path]) ++ addld="$addld -Wl,-rpath,$path" ++ found="yes" ++ break ++ fi ++ done ++ ++ if test "$found" = "no"; then ++ AC_MSG_RESULT([no]) ++ fi ++ addcflags="-I$OPT_NSS/include" version="unknown" nssprefix=$OPT_NSS -@@ -91,7 +91,7 @@ if test "x$OPT_NSS" != xno; then +@@ -91,7 +191,7 @@ if test "x$OPT_NSS" != xno; then fi dnl The function SSL_VersionRangeSet() is needed to enable TLS > 1.0 @@ -975,6 +1058,17 @@ index 397ba71b1..abc09a91c 100644 [ AC_DEFINE(USE_NSS, 1, [if NSS is enabled]) AC_SUBST(USE_NSS, [1]) +@@ -101,9 +201,7 @@ if test "x$OPT_NSS" != xno; then + test nss != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes + ], + [ +- LDFLAGS="$CLEANLDFLAGS" +- LIBS="$CLEANLIBS" +- CPPFLAGS="$CLEANCPPFLAGS" ++ AC_MSG_ERROR([Failed linking NSS statically]) + ]) + + if test "x$USE_NSS" = "xyes"; then diff --git a/src/Makefile.am b/src/Makefile.am index c8abc93b1..fcecb10d0 100644 --- a/src/Makefile.am diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 00000000..2f4c80e3 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode = auto diff --git a/tests/requirements.txt b/tests/requirements.txt index fa2223ae..f05a4938 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,4 @@ pyyaml pytest +pytest-asyncio dpkt diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index 75c84a16..83821a5b 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -1,6 +1,8 @@ import os import io import re +import sys +import asyncio import logging import subprocess import tempfile @@ -145,7 +147,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "chrome98" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "chrome_98.0.4758.102_win10" ), ( @@ -153,7 +155,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "chrome99" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "chrome_99.0.4844.51_win10" ), ( @@ -161,7 +163,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "chrome99_android" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "chrome_99.0.4844.73_android12-pixel6" ), ( @@ -169,7 +171,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "edge98" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "edge_98.0.1108.62_win10" ), ( @@ -177,7 +179,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "edge99" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "edge_99.0.1150.30_win10" ), ( @@ -185,7 +187,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "safari15_3" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "safari_15.3_macos11.6.4" ), ( @@ -193,7 +195,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "ff91esr" }, - "libcurl-impersonate-ff.so", + "libcurl-impersonate-ff", "firefox_91.6.0esr_win10" ), ( @@ -201,7 +203,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "ff95" }, - "libcurl-impersonate-ff.so", + "libcurl-impersonate-ff", "firefox_95.0.2_win10" ), ( @@ -209,7 +211,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "ff98" }, - "libcurl-impersonate-ff.so", + "libcurl-impersonate-ff", "firefox_98.0_win10" ) ] @@ -240,20 +242,75 @@ def tcpdump(self, pytestconfig): p.terminate() p.wait(timeout=10) + async def _read_proc_output(self, proc, timeout): + """Read an async process' output until timeout is reached""" + data = bytes() + loop = asyncio.get_running_loop() + start_time = loop.time() + passed = loop.time() - start_time + while passed < timeout: + try: + data += await asyncio.wait_for( + proc.stdout.readline(), timeout=timeout - passed + ) + except asyncio.TimeoutError: + pass + passed = loop.time() - start_time + return data + + async def _wait_nghttpd(self, proc): + """Wait for nghttpd to start listening on its designated port""" + data = bytes() + while data is not None: + data = await proc.stdout.readline() + if not data: + # Process terminated + return False + + line = data.decode("utf-8").rstrip() + if "listen 0.0.0.0:8443" in line: + return True + + return False + @pytest.fixture - def nghttpd(self): - """Initiailize an HTTP/2 server""" + async def nghttpd(self): + """Initiailize an HTTP/2 server. + The returned object is an asyncio.subprocess.Process object, + so async methods must be used with it. + """ logging.debug(f"Running nghttpd on :8443") - p = subprocess.Popen([ + # Launch nghttpd and wait for it to start listening. + + proc = await asyncio.create_subprocess_exec( "nghttpd", "-v", - "8443", "ssl/server.key", "ssl/server.crt" - ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + "8443", "ssl/server.key", "ssl/server.crt", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) - yield p + try: + # Wait up to 3 seconds for nghttpd to start. + # Otherwise fail. + started = await asyncio.wait_for( + self._wait_nghttpd(proc), timeout=3 + ) + if not started: + raise Exception("nghttpd failed to start on time") + except asyncio.TimeoutError: + raise Exception("nghttpd failed to start on time") - p.terminate() - p.wait(timeout=10) + yield proc + + proc.terminate() + await proc.wait() + + def _set_ld_preload(self, env_vars, lib): + if sys.platform.startswith("linux"): + env_vars["LD_PRELOAD"] = lib + ".so" + elif sys.platform.startswith("darwin"): + env_vars["DYLD_INSERT_LIBRARIES"] = lib + ".dylib" def _run_curl(self, curl_binary, env_vars, extra_args, url, output="/dev/null"): @@ -368,9 +425,15 @@ def test_tls_client_hello(self, pytestconfig.getoption("install_dir"), "bin", curl_binary ) if ld_preload: - env_vars["LD_PRELOAD"] = os.path.join( + # Injecting libcurl-impersonate with LD_PRELOAD is supported on + # Linux only. On Mac there is DYLD_INSERT_LIBRARIES but it + # reuqires more work to be functional. + if not sys.platform.startswith("linux"): + pytest.skip() + + self._set_ld_preload(env_vars, os.path.join( pytestconfig.getoption("install_dir"), "lib", ld_preload - ) + )) ret = self._run_curl(curl_binary, env_vars=env_vars, @@ -409,41 +472,40 @@ def test_tls_client_hello(self, equals, msg = sig.equals(expected_sig, reason=True) assert equals, msg + @pytest.mark.asyncio @pytest.mark.parametrize( "curl_binary, env_vars, ld_preload, expected_signature", CURL_BINARIES_AND_SIGNATURES ) - def test_http2_headers(self, - pytestconfig, - nghttpd, - curl_binary, - env_vars, - ld_preload, - browser_signatures, - expected_signature): + async def test_http2_headers(self, + pytestconfig, + nghttpd, + curl_binary, + env_vars, + ld_preload, + browser_signatures, + expected_signature): curl_binary = os.path.join( pytestconfig.getoption("install_dir"), "bin", curl_binary ) if ld_preload: - env_vars["LD_PRELOAD"] = os.path.join( + # Injecting libcurl-impersonate with LD_PRELOAD is supported on + # Linux only. On Mac there is DYLD_INSERT_LIBRARIES but it + # reuqires more work to be functional. + if not sys.platform.startswith("linux"): + pytest.skip() + + self._set_ld_preload(env_vars, os.path.join( pytestconfig.getoption("install_dir"), "lib", ld_preload - ) + )) + ret = self._run_curl(curl_binary, env_vars=env_vars, extra_args=["-k"], url="https://localhost:8443") assert ret == 0 - try: - output, stderr = nghttpd.communicate(timeout=2) - # If nghttpd finished running before timeout, it's likely it failed - # with an error. - assert nghttpd.returncode == 0, \ - (f"nghttpd failed with error code {nghttpd.returncode}, " - f"stderr: {stderr}") - except subprocess.TimeoutExpired: - nghttpd.kill() - output, stderr = nghttpd.communicate(timeout=3) + output = await self._read_proc_output(nghttpd, timeout=2) assert len(output) > 0 pseudo_headers, headers = self._parse_nghttpd2_output(output) @@ -481,9 +543,15 @@ def test_content_encoding(self, pytestconfig.getoption("install_dir"), "bin", curl_binary ) if ld_preload: - env_vars["LD_PRELOAD"] = os.path.join( + # Injecting libcurl-impersonate with LD_PRELOAD is supported on + # Linux only. On Mac there is DYLD_INSERT_LIBRARIES but it + # reuqires more work to be functional. + if not sys.platform.startswith("linux"): + pytest.skip() + + self._set_ld_preload(env_vars, os.path.join( pytestconfig.getoption("install_dir"), "lib", ld_preload - ) + )) output = tempfile.mkstemp()[1] ret = self._run_curl(curl_binary,