fetch-new-data.yaml

name: Update Open-Data
on:
  schedule:
    - cron: "0 0 * * *"

jobs:
  update:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Open Data
        uses: actions/checkout@v2

      - name: Fetch and update
        run: |
          ## Fetching
          # Disposable
          echo "Downloading disposable"
          curl https://raw.githubusercontent.com/disposable/disposable-email-domains/master/domains.txt > new-disposable-email-domains.txt

          echo "Downloading free providers 1/2"
          # Free
          curl https://raw.githubusercontent.com/Kikobeats/free-email-domains/master/domains.json | jq -r '.[]' > new-free-email-domains.txt
          echo "2/2"
          curl https://gist.githubusercontent.com/okutbay/5b4974b70673dfdcc21c517632c1f984/raw/free_email_provider_domains.txt >> new-free-email-domains.txt
          echo "Sorting"
          sort -u new-free-email-domains.txt > tmp ; mv tmp new-free-email-domains.txt

          # ISP
          echo "Downloading ISP domains"
          curl https://raw.githubusercontent.com/Immortalin/email-domain-db/master/leadstage-email-domain-db-v1-20150907.csv |
            sed 's/"[^"]*"//g' |
            awk -F, '$2 != "Fake" && NR > 1 {print $1}' |
            sort -u > ISP-domains.txt
          cat ISP-domains.txt >> new-free-email-domains.txt
          sort -u new-free-email-domains.txt > tmp ; mv tmp new-free-email-domains.txt

          ## Update
          # Disposable
          comm -23 <(sort -u new-disposable-email-domains.txt) disposable-email-domains.txt > tmp; cat tmp >> disposable-email-domains.txt
          sort -u disposable-email-domains.txt > tmp ; mv tmp disposable-email-domains.txt

          # Free
          comm -23 new-free-email-domains.txt disposable-email-domains.txt > free-email-domains-1.txt # remove disposable
          comm -23 free-email-domains-1.txt public-email-providers.txt > free-email-domains.txt # remove public

          # Banned IP list
          curl https://raw.githubusercontent.com/SecOps-Institute/SpamhausIPLists/master/drop.txt > banned_tmp.txt
          curl https://raw.githubusercontent.com/SecOps-Institute/SpamhausIPLists/master/drop_ipv6.txt >> banned_tmp.txt
          curl https://raw.githubusercontent.com/SecOps-Institute/SpamhausIPLists/master/edrop.txt >> banned_tmp.txt
          wget www.stopforumspam.com/downloads/listed_ip_365_ipv46_all.zip
          unzip listed_ip_365_ipv46_all.zip
          awk -F \" '{print $2}' listed_ip_365_ipv46_all.txt >> banned_tmp.txt
          curl https://www.stopforumspam.com/downloads/toxic_ip_cidr.txt >> banned_tmp.txt
          sort -u banned_tmp.txt > mailserver-banned-ips.txt

          ## Generate Json
          jq -Rn '[inputs]' disposable-email-domains.txt > disposable-email-domains.json
          jq -Rn '[inputs]' ISP-domains.txt > ISP-domains.json
          jq -Rn '[inputs]' free-email-domains.txt > free-email-domains.json
          jq -Rn '[inputs]' mailserver-banned-ips.txt > mailserver-banned-ips.json

          # Commit to repo
          git add disposable-email-domains.*
          git add ISP-domains.*
          git add free-email-domains.*
          git add mailserver-banned-ips.*

      - name: Commit and push changes
        run: |
          git config --global user.name "GitHub Actions"
          git config --global user.email "actions@github.com"
          git commit -m "Update Open-Data on $(date)"
          git push