feat: enforce dependency on released versions of packages

.github/workflows/dependency-check.yml

@@ -0,0 +1,92 @@
+name: Dependency Check
+
+on:
+  pull_request:
+    paths:
+      - 'go.mod'
+      - 'go.sum'
+      - '.github/workflows/dependency-check.yml'
+
+jobs:
+  dependency-check:
+    runs-on: ubuntu-latest
+    name: Dependency Check
+    env:
+      V0_PATTERN: 'v0\.0\.0-[0-9]{14}-[0-9a-f]{7,}(\s*(\/\/.*)?)?$'
+      RELEASE_PATTERN: 'v[0-9]+\.[0-9]+\.[0-9]+(\+incompatible)?(\s*(\/\/.*)?)?$'
+      IGNORE_PATTERN: 'dependency-check-ignore:\s'
+
+    steps:
+      - uses: actions/checkout@v3
+        name: Check out the repository
+        with:
+          submodules: 'recursive'
+      - uses: ./.github/actions/install-go
+
+      - id: all
+        name: Extract all dependencies from go.mod (include indirect dependencies and comments)
+        run: |
+          echo "dependencies<<EOF" >> $GITHUB_OUTPUT
+          # `go list` isn't used because:
+          # 1. it lists ALL the transitive dependencies, even those that are unused and don't make it to the go.mod file
+          # 2. It doesn't extract the inline `dependency-check-ignore` comments.
+          # Extract the lines from 'require (' to the first ')' including those lines in the go.mod file.
+          sed -n '/require (/,/)/p' go.mod |
+
+          # Remove the 'require (' line.
+          sed '/require (/d' |
+
+          # Remove the ')' line.
+          sed '/^)/d' |
+
+          # Remove leading whitespace from each line.
+          sed 's/^[[:space:]]*//' |
+
+          # Append the result to the file specified by the GITHUB_OUTPUT environment variable.
+          tee -a $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - id: unreleased
+        name: Find all dependencies that use prerelease versions (i.e., exclude vX.Y.Z and v0.0.0 versions)
+        env:
+          DEPENDENCIES: ${{ steps.all.outputs.dependencies }}
+        run: |
+          echo "dependencies<<EOF" >> $GITHUB_OUTPUT
+          grep -Pv "$V0_PATTERN|$RELEASE_PATTERN" <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - id: unexplained
+        name: Find all unreleased dependencies without a dependency-check-ignore comment
+        env:
+          DEPENDENCIES: ${{ steps.unreleased.outputs.dependencies }}
+        run: |
+          echo "dependencies<<EOF" >> $GITHUB_OUTPUT
+          grep -Pv "$IGNORE_PATTERN" <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - id: v0check
+        name: Check v0.0.0 dependencies for available tags
+        run: |
+          echo "tagged<<EOF" >> $GITHUB_OUTPUT
+          grep -P "$V0_PATTERN" go.mod | grep -Pv "$IGNORE_PATTERN" | while read -r line; do
+            dep=$(echo "$line" | cut -d' ' -f1)
+            if [ ! -z "$(go list -m -versions $dep 2>/dev/null | awk 'NF>1')" ]; then
+              echo "$dep"
+            fi
+          done | tee -a $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - if: steps.unexplained.outputs.dependencies != '' || steps.v0check.outputs.tagged != ''
+        name: Throw if any unexplained dependencies exist
+        env:
+          MESSAGE: |
+            Dependencies requiring attention found in this PR. Please follow the [dependency management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#dependency-management).
+
+            ${{ steps.unexplained.outputs.dependencies != '' && 'Unexplained unreleased dependencies:' || '' }}
+            ${{ steps.unexplained.outputs.dependencies }}
+
+            ${{ steps.v0check.outputs.tagged != '' && 'Unexplained v0.0.0 dependencies with available tags:' || '' }}
+            ${{ steps.v0check.outputs.tagged }}
+        run: |
+          echo "::error::${MESSAGE//$'\n'/%0A}"
+          exit 1

CONTRIBUTING.md

@@ -49,7 +49,7 @@ Note that this is enforced with https://github.com/filecoin-project/lotus/blob/m
 
 ## CHANGELOG Management
 
-To expedite the release process, the CHANGELOG is built-up incrementally.  
+To expedite the release process, the CHANGELOG is built-up incrementally.
 We enforce that each PR updates CHANGELOG.md or signals that the change doesn't need it.
 If the PR affects users (e.g., new feature, bug fix, system requirements change), update the CHANGELOG.md and add details to the UNRELEASED section.
 If the change does not require a CHANGELOG.md entry, do one of the following:
@@ -58,6 +58,20 @@ If the change does not require a CHANGELOG.md entry, do one of the following:
 
 Note that this is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/changelog.yml
 
+## Dependency Management
+
+We strive to use release dependencies because:
+1. Security / reliability - While there's no guarantee that a released version doesn't have bugs or issues, it seems fair to assume that non-released versions have even more. For example, https://github.com/filecoin-project/lotus/issues/12467 was triggered because of a bug in non-released library that lotus was depending on when the latest released version didn't have the bug.
+2. Faster builds
+3. Makes Lotus a better citizen when it's imported by other projects.
+
+We enforce that each dependency on an unreleased version of a package is explicitly documented in the `go.mod` file via an inline comment of the form `dependency-check-ignore: <reason>`.
+* If you are adding such a dependency, please add a suitable comment to the `go.mod` file as well.
+* This requirement applies both to direct and indirect dependencies.
+* This requirement applies to packages that have released versions (i.e., is not a `v0.0.0`).
+* This is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/dependency-check.yml
+* This enforcement was initially done per [#7131](https://github.com/filecoin-project/lotus/issues/7131).
+
 ## Markdown Conventions
 We optimize our markdown files for viewing on GitHub. That isn't to say other syntaxes can't be used, but that is the flavor we focus on and at the minimum don't want to break.
 

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/filecoin-project/go-jsonrpc v0.7.0
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.4
-	github.com/filecoin-project/go-state-types v0.16.0-rc2
+	github.com/filecoin-project/go-state-types v0.16.0-rc2 // dependency-check-ignore: unknown
 	github.com/filecoin-project/go-statemachine v1.0.3
 	github.com/filecoin-project/go-statestore v0.2.0
 	github.com/filecoin-project/go-storedcounter v0.1.0
@@ -78,7 +78,7 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/gregdhill/go-openrpc v0.0.0-20220114144539-ae6f44720487
 	github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026
-	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e
+	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // dependency-check-ignore: TODO: needs upgrading to tagged version
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/golang-lru/arc/v2 v2.0.7
 	github.com/hashicorp/golang-lru/v2 v2.0.7
@@ -134,15 +134,15 @@ require (
 	github.com/samber/lo v1.39.0
 	github.com/sirupsen/logrus v1.9.2
 	github.com/stretchr/testify v1.10.0
-	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // dependency-check-ignore: unknown
 	github.com/triplewz/poseidon v0.0.2
 	github.com/urfave/cli/v2 v2.25.5
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
 	github.com/whyrusleeping/cbor-gen v0.2.0
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	github.com/xeipuuv/gojsonschema v1.2.0
-	github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542
-	github.com/yugabyte/pgx/v5 v5.5.3-yb-2
+	github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 // dependency-check-ignore: unknown
+	github.com/yugabyte/pgx/v5 v5.5.3-yb-2 // dependency-check-ignore: unknown
 	github.com/zondax/ledger-filecoin-go v0.11.1
 	github.com/zyedidia/generic v1.2.1
 	go.opencensus.io v0.24.0
@@ -315,7 +315,7 @@ require (
 	github.com/prometheus/statsd_exporter v0.22.7 // indirect
 	github.com/quic-go/qpack v0.5.1 // indirect
 	github.com/quic-go/quic-go v0.48.2 // indirect
-	github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect
+	github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect; dependency-check-ignore: unknown
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil v2.18.12+incompatible // indirect
@@ -337,7 +337,7 @@ require (
 	github.com/zondax/ledger-go v0.14.3 // indirect
 	gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
 	gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
-	go.dedis.ch/kyber/v4 v4.0.0-pre2.0.20240924132404-4de33740016e // indirect
+	go.dedis.ch/kyber/v4 v4.0.0-pre2.0.20240924132404-4de33740016e // indirect; dependency-check-ignore: unknown
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/dig v1.18.0 // indirect
@@ -352,7 +352,7 @@ require (
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
+	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect; dependency-check-ignore: required by github.com/elastic/go-sysinfo
 	lukechampine.com/blake3 v1.3.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )