From 922ff19415148f92e62f38193f64f0dc4e6a62bf Mon Sep 17 00:00:00 2001 From: Jiovanni-bot Date: Sat, 3 Aug 2024 18:20:11 +0200 Subject: [PATCH] dumpyara: rebrand for personal usage * Setup a python bot with venv * Setup a github workflow * Setup rootless podman for the bot Change-Id: I746c928ccd13e80b44175448a731ba4905328afc --- .github/workflows/dump.yml | 51 ++++++++++++ .gitignore | 2 + README.md | 17 ++-- docker-compose.yml | 8 ++ dumpyara.sh | 13 +-- example_config.py | 8 ++ main.py | 162 +++++++++++++++++++++++++++++++++++++ 7 files changed, 246 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/dump.yml create mode 100644 docker-compose.yml create mode 100644 example_config.py create mode 100755 main.py diff --git a/.github/workflows/dump.yml b/.github/workflows/dump.yml new file mode 100644 index 0000000..85a101a --- /dev/null +++ b/.github/workflows/dump.yml @@ -0,0 +1,51 @@ +name: Dump + +on: + workflow_dispatch: + inputs: + urls: + description: 'URLs' + required: true + +jobs: + dump: + runs-on: ubuntu-latest + + steps: + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + overprovision-lvm: true + remove-dotnet: true + remove-android: true + remove-codeql: true + remove-haskell: true + remove-docker-images: true + + - name: Set up Git + uses: actions/checkout@v3 + + - name: Create .githubtoken file + run: echo "${{ secrets.GIT_TOKEN }}" > .githubtoken + + #- name: Create .githubssh file + # run: echo "${{ secrets.GIT_SSH }}" > .githubssh + + - name: Create .tgtoken file + run: echo "${{ secrets.TELEGRAM_TOKEN }}" > .tgtoken + + - name: Set up environment + run: bash setup.sh + + # protobuf-compiler is required for this + #- name: Install otadump + # uses: actions-rs/install@v0.1 + # with: + # crate: otadump + # version: latest + + - name: Run dumpyara + run: | + bash dumpyara.sh ${{ inputs.urls }} diff --git a/.gitignore b/.gitignore index a8e2009..9d61a11 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ mkbootimg_tools/ TWRP-device-tree-generator/ vmlinux-to-elf/ working/ +config.py +__pycache__/ diff --git a/README.md b/README.md index 103899b..8c2187e 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ # dumpyara -**[Telegram channel (feed)](https://t.me/android_dumps)** -**[Telegram group for dump requests](https://t.me/dumprequests)** +[![Dump](https://github.com/Jiovanni-dump/dumpyara/actions/workflows/dump.yml/badge.svg)](https://github.com/Jiovanni-dump/dumpyara/actions/workflows/dump.yml) + +**[Telegram channel (feed)](https://t.me/jiovanni_dumps)** **Script requirements**: Linux or Mac - + ***For setting up requirements***: bash setup.sh @@ -15,10 +16,8 @@ bash dumpyara.sh " OR " yourGithubToken -You can also place your github oauth token in a file called `.githubtoken` and telegram bot token in a file called `.tgtoken` in the root of this repository, if you wish it is ignored by git. -Before you start, make sure that dumpyara scripts are mapped to your own org or account, otherwise you'll only dump, not push. - -**Supported image types**: -check [here](https://github.com/AndroidDumps/Firmware_extractor/blob/master/extractor.sh#L3) +You can also place your github oauth token in a file called `.githubtoken` and telegram bot token in a file called `.tgtoken` in the root of this repository, if you wish it is ignored by git. +Before you start, make sure that dumpyara scripts are mapped to your own org or account, otherwise you'll only dump, not push. -If you're a member of AndroidDumps org, use a token with following permissions: `admin:org, public_repo` +**Supported image types**: +check [here](https://github.com/AndroidDumps/Firmware_extractor/blob/master/extractor.sh#L3) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9d006d8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,8 @@ +services: + app: + image: python:3.11-alpine + working_dir: /app + volumes: + - .:/app + command: sh -c "pip install --no-cache-dir --disable-pip-version-check pyTelegramBotAPI requests && python main.py" + restart: unless-stopped diff --git a/dumpyara.sh b/dumpyara.sh index fd293db..168f674 100755 --- a/dumpyara.sh +++ b/dumpyara.sh @@ -42,7 +42,7 @@ if echo "$1" | grep -e '^\(https\?\|ftp\)://.*$' > /dev/null; then URL=$1 fi cd "$PROJECT_DIR"/input || exit - { type -p aria2c > /dev/null 2>&1 && printf "Downloading File...\n" && aria2c -x16 -j"$(nproc)" "${URL}"; } || { printf "Downloading File...\n" && wget -q --content-disposition --show-progress --progress=bar:force "${URL}" || exit 1; } + { type -p aria2c > /dev/null 2>&1 && printf "Downloading File...\n" && aria2c -x16 --check-certificate=false -j"$(nproc)" "${URL}"; } || { printf "Downloading File...\n" && wget -q --content-disposition --show-progress --progress=bar:force "${URL}" || exit 1; } if [[ ! -f "$(echo ${URL##*/} | inline-detox)" ]]; then URL=$(wget --server-response --spider "${URL}" 2>&1 | awk -F"filename=" '{print $2}') fi @@ -52,7 +52,7 @@ else [[ -e "$URL" ]] || { echo "Invalid Input" && exit 1; } fi -ORG=AndroidDumps #your GitHub org name +ORG=Jiovanni-dump #your GitHub org name FILE=$(echo ${URL##*/} | inline-detox) EXTENSION=$(echo ${URL##*.} | inline-detox) UNZIP_DIR=${FILE/.$EXTENSION/} @@ -70,7 +70,7 @@ fi if [[ -d "$PROJECT_DIR/Firmware_extractor" ]]; then git -C "$PROJECT_DIR"/Firmware_extractor pull --recurse-submodules else - git clone -q --recurse-submodules https://github.com/AndroidDumps/Firmware_extractor "$PROJECT_DIR"/Firmware_extractor + git clone -q --recurse-submodules https://github.com/Jiovanni-dump/Firmware_extractor "$PROJECT_DIR"/Firmware_extractor fi if [[ -d "$PROJECT_DIR/mkbootimg_tools" ]]; then git -C "$PROJECT_DIR"/mkbootimg_tools pull --recurse-submodules @@ -248,11 +248,12 @@ if [[ -n $GIT_OAUTH_TOKEN ]]; then curl --silent --fail "https://raw.githubusercontent.com/$ORG/$repo/$branch/all_files.txt" 2> /dev/null && echo "Firmware already dumped!" && exit 1 git init if [[ -z "$(git config --get user.email)" ]]; then - git config user.email AndroidDumps@github.com + git config user.email giovanniricca@duck.com fi if [[ -z "$(git config --get user.name)" ]]; then - git config user.name AndroidDumps + git config user.name Jiovanni-bot fi + git config http.postBuffer 157286400 curl -s -X POST -H "Authorization: token ${GIT_OAUTH_TOKEN}" -d '{ "name": "'"$repo"'" }' "https://api.github.com/orgs/${ORG}/repos" #create new repo curl -s -X PUT -H "Authorization: token ${GIT_OAUTH_TOKEN}" -H "Accept: application/vnd.github.mercy-preview+json" -d '{ "names": ["'"$manufacturer"'","'"$platform"'","'"$top_codename"'"]}' "https://api.github.com/repos/${ORG}/${repo}/topics" git remote add origin https://github.com/$ORG/"${repo,,}".git @@ -321,7 +322,7 @@ fi # Telegram channel TG_TOKEN=$(< "$PROJECT_DIR"/.tgtoken) if [[ -n "$TG_TOKEN" ]]; then - CHAT_ID="@android_dumps" + CHAT_ID="@jiovanni_dumps" commit_head=$(git log --format=format:%H | head -n 1) commit_link="https://github.com/$ORG/$repo/commit/$commit_head" echo -e "Sending telegram notification" diff --git a/example_config.py b/example_config.py new file mode 100644 index 0000000..52aa5f2 --- /dev/null +++ b/example_config.py @@ -0,0 +1,8 @@ +allowed_user_ids = [ + telegram_user_id, + another_telegram_user_id, +] +github_repo = 'Jiovanni-dump/dumpyara' +workflow_id = 'dump.yml' +tg_token = '' +github_token = '' diff --git a/main.py b/main.py new file mode 100755 index 0000000..e7f8a41 --- /dev/null +++ b/main.py @@ -0,0 +1,162 @@ +#!/usr/bin/python3 + +import os +import re +import time + +import telebot +from requests import get, post +from requests.exceptions import ReadTimeout + +from config import ( + allowed_user_ids, + github_repo, + github_token, + tg_token, + workflow_id, +) + +bot = telebot.TeleBot(tg_token) + + +def is_valid_url(url): + regex = re.compile( + r'^(https?:\/\/)?(www\.)?([^\s.]+\.\S{2,}|localhost[\:?\d]*)\S*$' + ) + return re.match(regex, url) + + +@bot.message_handler(commands=['alive']) +def check_alive(message): + chat_id = message.chat.id + bot.send_message(chat_id, "I'm alive and ready to go!") + + +@bot.message_handler(commands=['cancel']) +def cancel_run(message): + chat_id = message.chat.id + user_id = message.from_user.id + + if user_id not in allowed_user_ids: + bot.send_message(chat_id, 'You are not authorized to use this command.') + return + + # Extract the run ID from the message + run_id = ( + message.text.split(' ', 1)[1].strip() + if len(message.text.split(' ', 1)) > 1 + else '' + ) + + if not run_id.isdigit(): + bot.send_message(chat_id, 'Invalid run ID provided.') + return + + # Prepare the request to cancel the workflow run + url = f'https://api.github.com/repos/{github_repo}/actions/runs/{run_id}/cancel' + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'token {github_token}', + } + + response = post(url, headers=headers) + + if response.status_code == 202: + bot.send_message( + chat_id, + f'Run `{run_id}` cancellation initiated successfully.', + parse_mode='Markdown', + disable_web_page_preview=True, + ) + else: + bot.send_message( + chat_id, + f'Failed to cancel the run {run_id}: {response.text}', + parse_mode='Markdown', + disable_web_page_preview=True, + ) + + +@bot.message_handler(commands=['dump']) +def dump(message): + chat_id = message.chat.id + user_id = message.from_user.id + + if user_id not in allowed_user_ids: + bot.send_message(chat_id, 'You are not authorized to use this command.') + return + + # Extract the URL from the message + dump_url = ( + message.text.split(' ', 1)[1].strip() + if len(message.text.split(' ', 1)) > 1 + else '' + ) + + if not is_valid_url(dump_url): + bot.send_message(chat_id, 'Invalid URL provided.') + return + + # Prepare the request to trigger the workflow_dispatch event + url = f'https://api.github.com/repos/{github_repo}/actions/workflows/{workflow_id}/dispatches' + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'token {github_token}', + } + payload = { + 'ref': 'master', + 'inputs': {'urls': dump_url}, + } + + response = post(url, headers=headers, json=payload) + + if response.status_code == 204: + time.sleep(3) # hack + bot.send_message(chat_id, 'Dump started successfully!') + + # Fetch the latest workflow run to get the run_id + runs_url = f'https://api.github.com/repos/{github_repo}/actions/runs' + runs_response = get(runs_url, headers=headers) + if runs_response.status_code == 200: + runs_data = runs_response.json() + if ( + 'workflow_runs' in runs_data + and len(runs_data['workflow_runs']) > 0 + ): + run_id = runs_data['workflow_runs'][0]['id'] + run_url = ( + f'https://github.com/{github_repo}/actions/runs/{run_id}' + ) + bot.send_message( + chat_id, + f'Check the dump progress [here]({run_url})\nTo cancel this dump run: `/cancel {run_id}`', + parse_mode='Markdown', + disable_web_page_preview=True, + ) + else: + bot.send_message(chat_id, 'Failed to retrieve the run ID.') + else: + bot.send_message( + chat_id, + f'Failed to retrieve workflow runs: {runs_response.text}', + disable_web_page_preview=True, + ) + else: + bot.send_message( + chat_id, + f'Failed to start the dump: {response.text}', + disable_web_page_preview=True, + ) + + +if __name__ == '__main__': + while True: + try: + bot.polling(none_stop=True, interval=0, timeout=35) + break + except ReadTimeout as e: + print(f'ReadTimeout occurred: {e}') + time.sleep(1) + except Exception as e: + print(f'An unexpected error occurred: {e}') + time.sleep(1)