Merge pull request #8 from mrd0ll4r/rework

Lots of cleanup, dockerized build environment, unified configuration, plugins

.dockerignore

@@ -0,0 +1,5 @@
+/periodic_measurements
+/precomputed_hashes
+/eval
+/output_data_crawls
+/out

.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    ignore:
+      # These are tied to the go-libp2p-kad-dht version, so we exclude them.
+      - dependency-name: "github.com/libp2p/go-libp2p"
+      - dependency-name: "github.com/libp2p/go-libp2p-kbucket"
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/Build_release.yaml

@@ -0,0 +1,27 @@
+name: Build_Release
+on:
+  # See the documentation for more intricate event dispatch here:
+  # https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#on
+  push:
+    branches:
+    - "master"
+jobs:
+  build:
+    name: Build & Release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Build Docker Image and Extract Build Artifacts
+      run: ./build-in-docker.sh
+    - uses: "marvinpinto/action-automatic-releases@latest"
+      with: 
+        repo_token: "${{ secrets.GITHUB_TOKEN }}"
+        title: "Dev Build"
+        automatic_release_tag: "latest"
+        files: out/*
+    - name: Build and Push Docker Image
+      uses: macbre/push-to-ghcr@master
+      with:
+        image_name: ${{ github.repository_owner }}/ipfs-crawler
+        github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/Lint_Build.yaml

@@ -0,0 +1,62 @@
+name: Lint_Build_Test
+on:
+  # See the documentation for more intricate event dispatch here:
+  # https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#on
+  push:
+    branches:
+    - "!dependabot/*"
+    - "*"
+  pull_request:
+    branches:
+    - "*"
+jobs:
+  build:
+    name: Build & Lint
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Setup
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.20
+    - name: Tidy
+      run: |
+        GO_MOD_OUPUT="$(go mod tidy -v 2>&1)"
+        if [[ $GO_MOD_OUPUT == *"unused"* ]]; then
+          echo "${GO_MOD_OUPUT}"
+          exit 1
+        fi
+    - name: Format
+      run: |
+        go install mvdan.cc/gofumpt@v0.3.1
+        GOFUMPT_OUTPUT="$(gofumpt -d .)"
+        if [ -n "$GOFUMPT_OUTPUT" ]; then
+          echo "${GOFUMPT_OUTPUT}"
+          exit 1
+        fi
+    - name: Lint
+      run: |
+        go install github.com/mgechev/revive@latest
+        REVIVE_OUTPUT="$(revive -config .github/workflows/revive.toml ./...)"
+        if [ -n "$REVIVE_OUTPUT" ]; then
+          echo "${REVIVE_OUTPUT}"
+          exit 1
+        fi
+    - name: Build
+      run: go build -v -o ipfs-crawler cmd/ipfs-crawler/main.go
+    - name: Build Docker image and export binaries
+      run: ./build-in-docker.sh
+
+  unit:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Setup
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.20
+    - name: Unit Tests
+      run: go test -v -race $(go list ./...)

.github/workflows/revive.toml

@@ -0,0 +1,44 @@
+ignoreGeneratedHeader = false
+severity = "warning"
+confidence = 0.8
+errorCode = 0
+warningCode = 0
+
+# Golint rules
+[rule.blank-imports]
+[rule.context-as-argument]
+[rule.context-keys-type]
+[rule.dot-imports]
+[rule.error-return]
+[rule.error-strings]
+[rule.error-naming]
+[rule.exported]
+[rule.if-return]
+[rule.increment-decrement]
+[rule.var-naming]
+[rule.var-declaration]
+[rule.package-comments]
+[rule.range]
+[rule.receiver-naming]
+[rule.time-naming]
+[rule.unexported-return]
+[rule.indent-error-flow]
+[rule.errorf]
+[rule.empty-block]
+[rule.superfluous-else]
+[rule.unused-parameter]
+[rule.unreachable-code]
+[rule.redefines-builtin-id]
+
+# Extra rules
+[rule.unnecessary-stmt]
+[rule.modifies-value-receiver]
+[rule.constant-logical-expr]
+[rule.range-val-in-closure]
+[rule.waitgroup-by-value]
+[rule.bare-return]
+[rule.unused-receiver]
+[rule.unhandled-error]
+arguments =["fmt.Printf", "fmt.Println"]
+[rule.string-of-int]
+[rule.useless-break]

.gitignore

@@ -0,0 +1,11 @@
+# build-in-docker output directory
+/out
+
+# Crawl output data default
+/output_data_crawls
+
+# Vim undo files
+*.un~
+
+# Goland files
+/.idea

Dockerfile

@@ -0,0 +1,36 @@
+FROM golang:1.20-bullseye AS builder
+
+WORKDIR /usr/src/ipfs-crawler/
+# Download all dependencies first, this should be cached.
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+COPY . .
+RUN make build
+
+FROM debian:bullseye-slim AS runner
+
+# Create a system user to drop into.
+RUN groupadd -r ipfs \
+  && useradd --no-log-init -r -g ipfs ipfs \
+  && mkdir -p ipfs
+
+# Enter our working directory.
+WORKDIR libp2p-crawler
+
+# Copy compiled binaries from builder.
+COPY --from=builder /usr/src/ipfs-crawler/cmd/ipfs-crawler/ipfs-crawler ./libp2p-crawler
+COPY --from=builder /usr/src/ipfs-crawler/dist/docker_entrypoint.sh .
+COPY --from=builder /usr/src/ipfs-crawler/dist/config_ipfs.yaml ./config/config_ipfs.yaml
+COPY --from=builder /usr/src/ipfs-crawler/dist/config_filecoin_mainnet.yaml ./config/config_filecoin_mainnet.yaml
+
+# Set ownership.
+RUN chown -R ipfs:ipfs ./libp2p-crawler
+
+# Drop root.
+USER ipfs
+
+# Run the binary.
+ENTRYPOINT ["./docker_entrypoint.sh","--config","./config/config_ipfs.yaml"]
+

Makefile

@@ -1,19 +1,15 @@
 build:
 	go build cmd/ipfs-crawler/main.go
-	mv main cmd/ipfs-crawler/crawler
-	echo "export LIBP2P_ALLOW_WEAK_RSA_KEYS="" && export LIBP2P_SWARM_FD_LIMIT="10000" && ./cmd/ipfs-crawler/crawler \$@" > start_crawl
-	chmod u+x start_crawl
+	mv main cmd/ipfs-crawler/ipfs-crawler
 
 preimages:
 	go build cmd/hash-precomputation/main.go
-	mv main cmd/hash-precomputation/compute_preimages
-	./cmd/hash-precomputation/compute_preimages
+	mv main cmd/hash-precomputation/hash-precomputation
+	./cmd/hash-precomputation/hash-precomputation
 	mkdir -p precomputed_hashes
 	mv preimages.csv precomputed_hashes/preimages.csv
 
 clean:
 	rm cmd/ipfs-crawler/crawler
-	rm start_crawl
-# 	rm cmd/hash-precomputation/compute_preimages
 
 all: preimages build