From acb9f210db61d0c40bc7aaecce8b713febb1d6a2 Mon Sep 17 00:00:00 2001 From: Leon Schoorl Date: Tue, 29 Aug 2023 15:19:46 +0200 Subject: [PATCH 1/2] Enable ghc mmap crash workaround for every ghc compiled binary In PRs #2444 we added the workaround-ghc-mmap-crash flag on some of our packages. That flag added `-with-rtsopts=-xm20000000` to some of our binaries. But other binaries would occasionally still trigger the mmap bug, most importantly ghc itself. This commit undoes the previous way of applying the workaround and applies the same `-xm20000000` RTS option everywhere via the GHCRTS environment variable. But for this to work we have to link all binaries with the `-rtsopts` flag, otherwise they'll error out that the GHCRTS contains unsupported options. --- .ci/cabal.project.local | 5 ++--- .ci/gitlab/common.yml | 2 ++ .ci/gitlab/test.yml | 1 + .ci/setup.sh | 4 ---- .github/workflows/ci.yml | 2 +- .gitlab-ci.yml | 1 + .../prepare/clash-profiling-prepare.cabal | 4 ++-- benchmark/profiling/run/clash-profiling.cabal | 4 ++-- clash-cores/clash-cores.cabal | 2 +- clash-cosim/clash-cosim.cabal | 2 +- clash-ghc/clash-ghc.cabal | 14 +------------- clash-lib/clash-lib.cabal | 4 ++-- clash-prelude/clash-prelude.cabal | 16 +++------------- clash-term/clash-term.cabal | 2 +- tests/clash-testsuite.cabal | 10 +--------- tests/src/Test/Tasty/Clash.hs | 1 + 16 files changed, 22 insertions(+), 52 deletions(-) diff --git a/.ci/cabal.project.local b/.ci/cabal.project.local index cbcefd77a6..40cb9fda64 100644 --- a/.ci/cabal.project.local +++ b/.ci/cabal.project.local @@ -10,7 +10,7 @@ package * package clash-prelude ghc-options: -Werror - flags: +doctests -workaround-ghc-mmap-crash + flags: +doctests tests: True benchmarks: True @@ -24,7 +24,6 @@ package clash-lib package clash-ghc ghc-options: -Werror - flags: -workaround-ghc-mmap-crash package clash-cosim ghc-options: -Werror @@ -48,7 +47,7 @@ package clash-lib-hedgehog package clash-testsuite ghc-options: -Werror -- enable cosim - flags: +cosim -workaround-ghc-mmap-crash + flags: +cosim package clash-benchmark ghc-options: -Werror diff --git a/.ci/gitlab/common.yml b/.ci/gitlab/common.yml index 2e867da790..2c4000efbd 100644 --- a/.ci/gitlab/common.yml +++ b/.ci/gitlab/common.yml @@ -21,6 +21,8 @@ before_script: - export THREADS=$(./.ci/effective_cpus.sh) - export CABAL_JOBS=$(./.ci/effective_cpus.sh) + - export CABAL_DIR=$HOME/.cabal + - if [[ "$WORKAROUND_GHC_MMAP_CRASH" == "yes" ]]; then export GHCRTS="-xm20000000"; fi - export - tar -xf cache.tar.zst -C / || true - .ci/setup.sh diff --git a/.ci/gitlab/test.yml b/.ci/gitlab/test.yml index 9cb4ec252b..c8ebd618ee 100644 --- a/.ci/gitlab/test.yml +++ b/.ci/gitlab/test.yml @@ -37,6 +37,7 @@ stages: - export CABAL_JOBS=$(./.ci/effective_cpus.sh) - export clash_lib_datadir=$(pwd)/clash-lib/ - export clash_cosim_datadir=$(pwd)/clash-cosim/ + - if [[ "$WORKAROUND_GHC_MMAP_CRASH" == "yes" ]]; then export GHCRTS="-xm20000000"; fi - export - tar -xf cache.tar.zst -C / || true - tar -xf dist.tar.zst -C / diff --git a/.ci/setup.sh b/.ci/setup.sh index bdd41c8af8..9ef5489702 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -78,10 +78,6 @@ if [ ! -f cabal.project.local ]; then fi set +u - if [[ "$WORKAROUND_GHC_MMAP_CRASH" == "yes" ]]; then - sed -i 's/-workaround-ghc-mmap-crash/+workaround-ghc-mmap-crash/g' cabal.project.local - fi - if [[ "$GHC_HEAD" == "yes" ]]; then cat .ci/cabal.project.local.append-HEAD >> cabal.project.local fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 088d8fc7ce..f8d5d4eb14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,7 +126,7 @@ jobs: CABAL_JOBS: 2 MULTIPLE_HIDDEN: ${{ matrix.multiple_hidden }} CI_COMMIT_BRANCH: ${{ github.base_ref }} - WORKAROUND_GHC_MMAP_CRASH: ${{ matrix.workaround_ghc_mmap_crash }} + GHCRTS: ${{ fromJSON('["","-xm20000000"]')[matrix.workaround_ghc_mmap_crash == 'yes'] }} steps: - name: Checkout diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 691798ab39..0fa2d04bf9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,6 +53,7 @@ nix-build: needs: [] stage: test before_script: + - if [[ "$WORKAROUND_GHC_MMAP_CRASH" == "yes" ]]; then export GHCRTS="-xm20000000"; fi - nix-env -i gawk script: - nix-build -j$(./.ci/effective_cpus.sh) diff --git a/benchmark/profiling/prepare/clash-profiling-prepare.cabal b/benchmark/profiling/prepare/clash-profiling-prepare.cabal index 033d38e57b..331b1c4b0e 100644 --- a/benchmark/profiling/prepare/clash-profiling-prepare.cabal +++ b/benchmark/profiling/prepare/clash-profiling-prepare.cabal @@ -26,7 +26,7 @@ library executable clash-profile-normalization-prepare main-is: profile-normalization-prepare.hs default-language: Haskell2010 - ghc-options: -Wall -Wcompat + ghc-options: -Wall -Wcompat -rtsopts build-depends: base, binary, bytestring, @@ -38,7 +38,7 @@ executable clash-profile-normalization-prepare executable clash-profile-netlist-prepare main-is: profile-netlist-prepare.hs default-language: Haskell2010 - ghc-options: -Wall -Wcompat + ghc-options: -Wall -Wcompat -rtsopts build-depends: base, binary, bytestring, diff --git a/benchmark/profiling/run/clash-profiling.cabal b/benchmark/profiling/run/clash-profiling.cabal index 6730b634e2..6b8fad1fc6 100644 --- a/benchmark/profiling/run/clash-profiling.cabal +++ b/benchmark/profiling/run/clash-profiling.cabal @@ -12,7 +12,7 @@ cabal-version: >=1.10 executable clash-profile-normalization-run main-is: profile-normalization-run.hs default-language: Haskell2010 - ghc-options: -O2 -Wall -Wcompat + ghc-options: -O2 -Wall -Wcompat -rtsopts build-depends: base, binary, bytestring, @@ -30,7 +30,7 @@ executable clash-profile-normalization-run executable clash-profile-netlist-run main-is: profile-netlist-run.hs default-language: Haskell2010 - ghc-options: -O2 -Wall -Wcompat + ghc-options: -O2 -Wall -Wcompat -rtsopts build-depends: base, binary, bytestring, diff --git a/clash-cores/clash-cores.cabal b/clash-cores/clash-cores.cabal index b9e5a5ecc0..e595021993 100644 --- a/clash-cores/clash-cores.cabal +++ b/clash-cores/clash-cores.cabal @@ -99,7 +99,7 @@ test-suite unittests hs-source-dirs: test type: exitcode-stdio-1.0 main-is: unittests.hs - + ghc-options: -rtsopts if !flag(unittests) buildable: False diff --git a/clash-cosim/clash-cosim.cabal b/clash-cosim/clash-cosim.cabal index 681ceec30f..af4ceb5a49 100644 --- a/clash-cosim/clash-cosim.cabal +++ b/clash-cosim/clash-cosim.cabal @@ -116,6 +116,6 @@ test-suite test tasty-hunit, tasty-quickcheck, template-haskell - ghc-options: -Wall -Wcompat + ghc-options: -Wall -Wcompat -rtsopts if flag(pedantic) ghc-options: -Werror diff --git a/clash-ghc/clash-ghc.cabal b/clash-ghc/clash-ghc.cabal index 443eb37934..f3db38fb16 100644 --- a/clash-ghc/clash-ghc.cabal +++ b/clash-ghc/clash-ghc.cabal @@ -65,24 +65,12 @@ flag use-ghc-paths default: False manual: True -flag workaround-ghc-mmap-crash - description: - Only use this flag when hit by GHC bug #19421. See clash-compiler PR #2444. - default: False - manual: True - executable clash Main-Is: src-ghc/Batch.hs Build-Depends: base, clash-ghc - GHC-Options: -Wall -Wcompat -threaded -rtsopts + GHC-Options: -Wall -Wcompat -threaded -rtsopts -with-rtsopts=-A128m if flag(dynamic) GHC-Options: -dynamic - -- Note that multiple -with-rtsopts are not cumulative, so we can't add the - -- common RTS options in the unconditional GHC-Options - if flag(workaround-ghc-mmap-crash) - GHC-Options: "-with-rtsopts=-A128m -xm20000000" - else - GHC-Options: -with-rtsopts=-A128m extra-libraries: pthread default-language: Haskell2010 diff --git a/clash-lib/clash-lib.cabal b/clash-lib/clash-lib.cabal index 8bd2e56b02..553f53c28a 100644 --- a/clash-lib/clash-lib.cabal +++ b/clash-lib/clash-lib.cabal @@ -315,7 +315,7 @@ executable v16-upgrade-primitives directory, stringsearch, Glob - GHC-Options: -Wall -Wcompat + GHC-Options: -Wall -Wcompat -rtsopts default-language: Haskell2010 test-suite unittests @@ -323,7 +323,7 @@ test-suite unittests type: exitcode-stdio-1.0 default-language: Haskell2010 main-is: unittests.hs - ghc-options: -Wall -Wcompat -threaded -with-rtsopts=-N + ghc-options: -Wall -Wcompat -threaded -rtsopts -with-rtsopts=-N hs-source-dirs: tests if !flag(unittests) diff --git a/clash-prelude/clash-prelude.cabal b/clash-prelude/clash-prelude.cabal index 3cd17047fe..c197fd499b 100644 --- a/clash-prelude/clash-prelude.cabal +++ b/clash-prelude/clash-prelude.cabal @@ -117,12 +117,6 @@ flag benchmarks default: True manual: True -flag workaround-ghc-mmap-crash - description: - Only use this flag when hit by GHC bug #19421. See clash-compiler PR #2444. - default: False - manual: True - common common-options default-language: Haskell2010 default-extensions: BangPatterns @@ -367,7 +361,7 @@ test-suite doctests type: exitcode-stdio-1.0 default-language: Haskell2010 main-is: doctests.hs - ghc-options: -Wall -Wcompat -threaded + ghc-options: -Wall -Wcompat -threaded -rtsopts hs-source-dirs: tests if !flag(doctests) @@ -379,15 +373,11 @@ test-suite doctests doctest-parallel >= 0.2 && < 0.4, filepath - if flag(workaround-ghc-mmap-crash) - ghc-options: -with-rtsopts=-xm20000000 - - test-suite unittests import: common-options type: exitcode-stdio-1.0 main-is: unittests.hs - ghc-options: -Wall -Wcompat -threaded -with-rtsopts=-N + ghc-options: -Wall -Wcompat -threaded -rtsopts -with-rtsopts=-N hs-source-dirs: tests if !flag(unittests) @@ -451,7 +441,7 @@ benchmark benchmark-clash-prelude type: exitcode-stdio-1.0 default-language: Haskell2010 main-is: benchmark-main.hs - ghc-options: -O2 -Wall + ghc-options: -O2 -Wall -rtsopts hs-source-dirs: benchmarks ghc-options: -with-rtsopts=-T diff --git a/clash-term/clash-term.cabal b/clash-term/clash-term.cabal index 463cc718c9..5c46722769 100644 --- a/clash-term/clash-term.cabal +++ b/clash-term/clash-term.cabal @@ -30,7 +30,7 @@ executable clash-term bytestring >= 0.10.0.2 && < 0.11, rewrite-inspector == 0.1.0.11 - GHC-Options: -Wall -Wcompat -threaded + GHC-Options: -Wall -Wcompat -threaded -rtsopts extra-libraries: pthread default-language: Haskell2010 if os(windows) diff --git a/tests/clash-testsuite.cabal b/tests/clash-testsuite.cabal index 18031b4e6f..dcd80597f0 100644 --- a/tests/clash-testsuite.cabal +++ b/tests/clash-testsuite.cabal @@ -28,12 +28,6 @@ flag multiple-hidden default: False manual: True -flag workaround-ghc-mmap-crash - description: - Only use this flag when hit by GHC bug #19421. See clash-compiler PR #2444. - default: False - manual: True - common basic-config default-language: Haskell2010 ghc-options: -Wall -Wcompat @@ -82,9 +76,6 @@ common basic-config if flag(multiple-hidden) cpp-options: -DCLASH_MULTIPLE_HIDDEN - if flag(workaround-ghc-mmap-crash) - cpp-options: -DCLASH_WORKAROUND_GHC_MMAP_CRASH - library import: basic-config hs-source-dirs: @@ -127,6 +118,7 @@ library executable clash-testsuite import: basic-config main-is: Main.hs + ghc-options: -rtsopts build-tool-depends: clash-ghc:clash diff --git a/tests/src/Test/Tasty/Clash.hs b/tests/src/Test/Tasty/Clash.hs index 045996c601..c8d55a2df5 100644 --- a/tests/src/Test/Tasty/Clash.hs +++ b/tests/src/Test/Tasty/Clash.hs @@ -285,6 +285,7 @@ instance IsTest ClashBinaryTest where , "-main-is", cbModName <> ".main" <> show cbBuildTarget , "-o", oDir "out" , "-outputdir", oDir + , "-rtsopts" ] <> cbExtraBuildArgs <> [ cbSourceDirectory cbModName <.> "hs" ] From 984d5b61643152d72469a2fac47bb414640cc8dc Mon Sep 17 00:00:00 2001 From: Leon Schoorl Date: Tue, 29 Aug 2023 17:32:52 +0200 Subject: [PATCH 2/2] HACK: disable mmap crash workaround for clash-cosim They use custom setups and it is seemingly impossible to get them to link with anything other than the default -rtsopts=some Making them always throw an error when GHCRTS="-xm..." --- .ci/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.ci/build.sh b/.ci/build.sh index 93246b68cc..e396f3a98c 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -22,6 +22,11 @@ if [[ "$GHC_HEAD" != "yes" ]]; then fi set -u +cabal v2-build clash-prelude --write-ghc-environment-files=always +# Undo mmap crash workaround for clash-cosim, see PR #2572. +# It uses a custom setup, which can't be linked with -rtsopts +GHCRTS="" cabal v2-build clash-cosim --write-ghc-environment-files=always + # Build with default constraints cabal v2-build all --write-ghc-environment-files=always