Update #5543
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Update" | |
| on: | |
| schedule: | |
| - cron: '8 */4 * * *' # At minute 8 past every 4th hour | |
| jobs: | |
| updater: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install WHOIS client | |
| run: sudo apt install -y whois | |
| - name: Download IPs | |
| run: | | |
| set -x | |
| bash google/downloader.sh | |
| bash googlebot/downloader.sh | |
| bash amazon/downloader.sh | |
| bash microsoft/downloader.sh | |
| bash oracle/downloader.sh | |
| bash digitalocean/downloader.sh | |
| bash bing/downloader.sh | |
| bash github/downloader.sh | |
| bash facebook/downloader.sh | |
| bash twitter/downloader.sh | |
| bash linode/downloader.sh | |
| bash telegram/downloader.sh | |
| bash openai/downloader.sh || echo 'GPTBot download failed' # TODO: fixme | |
| bash cloudflare/downloader.sh | |
| bash vultr/downloader.sh | |
| - name: Create All-In-One ranges | |
| run: | | |
| cat google/ipv4.txt googlebot/ipv4.txt amazon/ipv4.txt microsoft/ipv4.txt oracle/ipv4.txt digitalocean/ipv4.txt bing/ipv4.txt github/ipv4.txt facebook/ipv4.txt twitter/ipv4.txt linode/ipv4.txt telegram/ipv4.txt openai/ipv4.txt cloudflare/ipv4.txt vultr/ipv4.txt | sort -V | uniq > all/ipv4.txt | |
| cat google/ipv6.txt googlebot/ipv6.txt amazon/ipv6.txt microsoft/ipv6.txt digitalocean/ipv6.txt github/ipv6.txt facebook/ipv6.txt twitter/ipv6.txt linode/ipv6.txt telegram/ipv6.txt cloudflare/ipv6.txt vultr/ipv6.txt | sort -V | uniq > all/ipv6.txt | |
| - name: Set up Python 3.7 | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.7' | |
| - name: Cache pip | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('utils/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| ${{ runner.os }}- | |
| - name: Install dependencies | |
| run: | | |
| pip install -r utils/requirements.txt | |
| - name: Merge Ranges | |
| run: | | |
| set -euo pipefail | |
| set -x | |
| # ipv4 | |
| python utils/merge.py --source=google/ipv4.txt | sort -V > google/ipv4_merged.txt | |
| python utils/merge.py --source=googlebot/ipv4.txt | sort -V > googlebot/ipv4_merged.txt | |
| python utils/merge.py --source=amazon/ipv4.txt | sort -V > amazon/ipv4_merged.txt | |
| python utils/merge.py --source=microsoft/ipv4.txt | sort -V > microsoft/ipv4_merged.txt | |
| python utils/merge.py --source=oracle/ipv4.txt | sort -V > oracle/ipv4_merged.txt | |
| python utils/merge.py --source=digitalocean/ipv4.txt | sort -V > digitalocean/ipv4_merged.txt | |
| python utils/merge.py --source=bing/ipv4.txt | sort -V > bing/ipv4_merged.txt | |
| python utils/merge.py --source=github/ipv4.txt | sort -V > github/ipv4_merged.txt | |
| python utils/merge.py --source=facebook/ipv4.txt | sort -V > facebook/ipv4_merged.txt | |
| python utils/merge.py --source=twitter/ipv4.txt | sort -V > twitter/ipv4_merged.txt | |
| python utils/merge.py --source=linode/ipv4.txt | sort -V > linode/ipv4_merged.txt | |
| python utils/merge.py --source=telegram/ipv4.txt | sort -V > telegram/ipv4_merged.txt | |
| python utils/merge.py --source=openai/ipv4.txt | sort -V > openai/ipv4_merged.txt | |
| python utils/merge.py --source=cloudflare/ipv4.txt | sort -V > cloudflare/ipv4_merged.txt | |
| python utils/merge.py --source=vultr/ipv4.txt | sort -V > vultr/ipv4_merged.txt | |
| python utils/merge.py --source=all/ipv4.txt | sort -V > all/ipv4_merged.txt | |
| # ipv6 | |
| python utils/merge.py --source=google/ipv6.txt | sort -V > google/ipv6_merged.txt | |
| python utils/merge.py --source=googlebot/ipv6.txt | sort -V > googlebot/ipv6_merged.txt | |
| python utils/merge.py --source=amazon/ipv6.txt | sort -V > amazon/ipv6_merged.txt | |
| python utils/merge.py --source=microsoft/ipv6.txt | sort -V > microsoft/ipv6_merged.txt | |
| # oracle not provide ipv6 | |
| python utils/merge.py --source=digitalocean/ipv6.txt | sort -V > digitalocean/ipv6_merged.txt | |
| # bing not provide ipv6 | |
| python utils/merge.py --source=github/ipv6.txt | sort -V > github/ipv6_merged.txt | |
| python utils/merge.py --source=facebook/ipv6.txt | sort -V > facebook/ipv6_merged.txt | |
| python utils/merge.py --source=twitter/ipv6.txt | sort -V > twitter/ipv6_merged.txt | |
| python utils/merge.py --source=linode/ipv6.txt | sort -V > linode/ipv6_merged.txt | |
| python utils/merge.py --source=telegram/ipv6.txt | sort -V > telegram/ipv6_merged.txt | |
| # openai not provide ipv6 | |
| python utils/merge.py --source=cloudflare/ipv6.txt | sort -V > cloudflare/ipv6_merged.txt | |
| python utils/merge.py --source=vultr/ipv6.txt | sort -V > vultr/ipv6_merged.txt | |
| python utils/merge.py --source=all/ipv6.txt | sort -V > all/ipv6_merged.txt | |
| - name: Commit files | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| git remote add github "https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| git pull github ${GITHUB_REF} --ff-only | |
| # Get name & email from 1st commit (needs `fetch-depth: 0` in step `actions/checkout@v3`) | |
| git config --local user.email "$(git log --format='%ae' --reverse | head -1)" | |
| git config --local user.name "$(git log --format='%an' --reverse | head -1)" | |
| # try commit | |
| git add . | |
| if [ -z "$(git status --porcelain)" ]; then | |
| echo 'No changes' | |
| exit 0 | |
| fi | |
| git commit -m "Auto-update ip ranges" | |
| # push changes | |
| git push github HEAD:${GITHUB_REF} |