Merge pull request #56 from lwthiker/macos_part2

 Add support for linking NSS statically on macOS

.github/workflows/build-and-test-docker.yml

@@ -9,7 +9,7 @@ on:
       - main
 
 jobs:
-  build-and-test:
+  build-docker-and-test:
     name: Build curl-impersonate Docker images and run the tests
     runs-on: ubuntu-latest
     services:

.github/workflows/build-and-test-make.yml

@@ -14,29 +14,52 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-20.04, macos-11]
+        include:
+          - os: ubuntu-20.04
+            capture_interface: eth0
+            make: make
+          - os: macos-11
+            capture_interface: en0
+            make: gmake
     steps:
+      - uses: actions/setup-python@v3
+
       - name: Install Ubuntu dependencies
         if: matrix.os == 'ubuntu-20.04'
         run: |
-          sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool
-          # Firefox version dependencies
-          sudo apt install python3-pip python-is-python3
-          pip install gyp-next
+          sudo apt-get update
+          sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool
+          # Chrome version dependencies
+          sudo apt-get install golang-go
+          # Needed to compile 'minicurl'
+          sudo apt-get install libcurl4-openssl-dev
+          # More dependencies for the tests
+          sudo apt-get install tcpdump nghttp2-server libnss3
+
+      - name: Install macOS dependencies
+        if: matrix.os == 'macos-11'
+        run: |
+          brew install pkg-config make cmake ninja autoconf automake libtool
           # Chrome version dependencies
-          sudo apt install golang-go
+          # (Go is already installed)
+          # brew install go
           # Needed to compile 'minicurl'
-          sudo apt install libcurl4-openssl-dev
+          brew install curl
           # More dependencies for the tests
-          sudo apt install tcpdump nghttp2-server libnss3
+          brew install tcpdump nghttp2 nss
+
+      - name: Install common dependencies
+        run: |
+          # Firefox version dependencies
+          pip3 install gyp-next
 
       - name: Check out the repo
         uses: actions/checkout@v2
 
       - name: Install dependencies for the tests script
         run: |
-          # Install globally so that we can run 'pytest' with 'sudo'
-          sudo pip install -r tests/requirements.txt
+          pip3 install -r tests/requirements.txt
 
       - name: Run configure script
         run: |
@@ -46,15 +69,15 @@ jobs:
 
       - name: Build the Chrome version of curl-impersonate
         run: |
-          make chrome-build
-          make chrome-checkbuild
-          make chrome-install
+          ${{ matrix.make }} chrome-build
+          ${{ matrix.make }} chrome-checkbuild
+          ${{ matrix.make }} chrome-install
 
       - name: Build the Firefox version of curl-impersonate
         run: |
-          make firefox-build
-          make firefox-checkbuild
-          make firefox-install
+          ${{ matrix.make }} firefox-build
+          ${{ matrix.make }} firefox-checkbuild
+          ${{ matrix.make }} firefox-install
     
       - name: Prepare the tests
         run: |
@@ -65,4 +88,5 @@ jobs:
         run: |
           cd tests
           # sudo is needed for capturing packets
-          sudo pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp}}/install
+          python_bin=$(which python3)
+          sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp}}/install --capture-interface ${{ matrix.capture_interface }}

Dockerfile.template

@@ -82,11 +82,7 @@ ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_75_RTM/src/n
 RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL}
 RUN tar xf ${NSS_VERSION}.tar.gz && \
     cd ${NSS_VERSION}/nss && \
-{{#alpine}}
-    # Hack to make nss compile on alpine with python3
-    ln -sf python3 /usr/bin/python && \
-{{/alpine}}
-    ./build.sh -o --disable-tests --static
+    ./build.sh -o --disable-tests --static --python=python3
 {{/firefox}}
 {{#chrome}}
 # BoringSSL doesn't have versions. Choose a commit that is used in a stable

INSTALL.md

@@ -16,7 +16,7 @@ Install dependencies for building all the components:
 ```
 sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool
 # For the Firefox version only
-sudo apt install python3-pip python-is-python3
+sudo apt install python3-pip libnss3
 pip install gyp-next
 export PATH="$PATH:~/.local/bin" # Add gyp to PATH
 # For the Chrome version only
@@ -65,11 +65,12 @@ curl-impersonate-chrome https://www.wikipedia.org
 ```
 
 ### macOS
-*macOS support is still a work in progress and currently supports the Chrome version only.*
-
 Install dependencies for building all the components:
 ```
 brew install pkg-config make cmake ninja autoconf automake libtool
+# For the Firefox version only
+brew install sqlite nss
+pip3 install gyp-next
 # For the Chrome version only
 brew install go
 ```
@@ -83,6 +84,9 @@ Configure and compile:
 ```
 mkdir build && cd build
 ../configure
+# Build and install the Firefox version
+gmake firefox-build
+sudo gmake firefox-install
 # Build and install the Chrome version
 gmake chrome-build
 sudo gmake chrome-install
@@ -93,6 +97,17 @@ cd ../ && rm -Rf build
 ### Static compilation
 To compile curl-impersonate statically with libcurl-impersonate, pass `--enable-static` to the `configure` script.
 
+### A note about the Firefox version
+The Firefox version compiles a static version of nss, Firefox's TLS library.
+For NSS to have a list of root certificates, curl attempts to load at runtime `libnssckbi`, one of the NSS libraries.
+If you get the error:
+```
+curl: (60) Peer's Certificate issuer is not recognized
+```
+Make sure that NSS is installed (see above).
+If the issue persists it might be that NSS is installed in a non-standard location on your system.
+Please open an issue in that case.
+
 ## Docker build
 The Docker build is a bit more reproducible and serves as the reference implementation. It creates a Debian-based Docker image with the binaries.
 

Makefile.in

@@ -126,7 +126,10 @@ $(NSS_VERSION).tar.gz:
 $(nss_static_libs): $(NSS_VERSION).tar.gz
 	tar xf $(NSS_VERSION).tar.gz
 	cd $(NSS_VERSION)/nss
-	./build.sh -o --disable-tests --static
+	./build.sh -o --disable-tests --static --python=python3
+	# Hack for macOS: Remove dynamic libraries to force the linker to use the
+	# static ones when linking curl.
+	rm -Rf $(nss_install_dir)/lib/*.dylib
 
 
 boringssl.zip:

README.md

@@ -86,7 +86,7 @@ You can call it with the target names, e.g. `chrome98`, and it will internally s
 Note that if you call `curl_easy_setopt()` later with one of the above it will override the options set by `curl_easy_impersonate()`.
 
 ### Using CURL_IMPERSONATE env var
-*Experimental*: If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD`. You can then set the `CURL_IMPERSONATE` env var. For example:
+*Experimental*: If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD` (Linux only). You can then set the `CURL_IMPERSONATE` env var. For example:
 ```bash
 LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome98 my_app
 ```

firefox/Dockerfile

@@ -48,7 +48,7 @@ ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_75_RTM/src/n
 RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL}
 RUN tar xf ${NSS_VERSION}.tar.gz && \
     cd ${NSS_VERSION}/nss && \
-    ./build.sh -o --disable-tests --static
+    ./build.sh -o --disable-tests --static --python=python3
 
 ARG NGHTTP2_VERSION=nghttp2-1.46.0
 ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2

firefox/Dockerfile.alpine

@@ -41,9 +41,7 @@ ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_75_RTM/src/n
 RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL}
 RUN tar xf ${NSS_VERSION}.tar.gz && \
     cd ${NSS_VERSION}/nss && \
-    # Hack to make nss compile on alpine with python3
-    ln -sf python3 /usr/bin/python && \
-    ./build.sh -o --disable-tests --static
+    ./build.sh -o --disable-tests --static --python=python3
 
 ARG NGHTTP2_VERSION=nghttp2-1.46.0
 ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2