From 993b6ac9cfd02ace188c78c65b1d8d85ec741732 Mon Sep 17 00:00:00 2001 From: Candace Savonen Date: Thu, 29 Feb 2024 09:38:42 -0500 Subject: [PATCH] Have URLs periodically checked (#673) * Testing git on terminal * Revert changes * Initial pass at URL checker * Test URL checker by running every 2 mint * Fix syntax * Syntax * chatgpt? * Fix spacing * Remove "shell:bash" * Please work * Add spacing and indents * Fix indentation * Tryna find this indentation * Add config option * Add workflow dispatch button too * temporarily have a pull_request trigger * Add URL config * Add a delete step * Fix spec * Update for branch problem * Add issue creating infrastructure * Add issue template * Add steps: * Declare docker * get rid of dash * Move if * Add a needs argument * Add an option for it branch exists * missing paranthesis * Make sure its checked out * Try mapfile? * Add a file * Didn't fix template issue * Put file link * Fix link * Fix less than or greater than * Add in a spelling error to check if this works * Add spelling errors to test part 2 * Try to fix * Reverse order or < and = * Change branch name * Make sure branch is checked out * Look for an issue * Trying a different issue finder * Gonna make my own issue finder * Add find issue bits * Add CRAN mirror * Fix syntax * Delete spelling errors * Add a missing githubr:: * Install from github * Fix GH_PAT * Install from CRAN again * Install from github but use devtools * dependencies = TRUE * Install from CRAN instead * Test variables * Retrigger a run for GHA * Print out session info * Actually what I did won't work! * use cat() instead * unlist * I need cat() too * Why is GHA not printing this out? Try message() * Save session info to a file and archive upload it * Shut off the other things so this step completes * Comment out one more step * Order dictionary.txt * Add broken URL * Make it so blank is the logic * Just retrigger the check * Try again * fix true * drop quotes * Let's try this * repo_name -> repo * change per_page * mysterious error * Print out clues * Update script * Fix logic * fix so runs once a month --------- Co-authored-by: howardbaek Co-authored-by: Howard Baek <50791792+howardbaek@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/url-error.md | 8 +++ .github/workflows/check-url.yml | 107 ++++++++++++++++++++++++++++ .github/workflows/pull_request.yml | 2 +- 01-intro.Rmd | 1 + OTTR_template.Rproj | 17 +++++ README.md | 1 - config_automation.yml | 2 + resources/dictionary.txt | 42 +++++++++-- scripts/find_issue.R | 55 ++++++++++++++ 9 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/url-error.md create mode 100644 .github/workflows/check-url.yml create mode 100644 OTTR_template.Rproj create mode 100644 scripts/find_issue.R diff --git a/.github/ISSUE_TEMPLATE/url-error.md b/.github/ISSUE_TEMPLATE/url-error.md new file mode 100644 index 00000000..fe131966 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/url-error.md @@ -0,0 +1,8 @@ +--- +title: Broken URLs found in the course! +labels: url-error +--- +URL's in this course were just checked and some broken URLs were found. + +**Number of errors:** {{ env.ERROR_NUM }} +**File where errors are:** [url checks file here]({{ env.FILE_URL }}) diff --git a/.github/workflows/check-url.yml b/.github/workflows/check-url.yml new file mode 100644 index 00000000..493ea2be --- /dev/null +++ b/.github/workflows/check-url.yml @@ -0,0 +1,107 @@ +name: Periodic URL Check + +on: + workflow_dispatch: + schedule: + - cron: '0 0 1 * *' + +jobs: + set-up: + name: Load user automation choices + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # Use the yaml-env-action action. + - name: Load environment from YAML + uses: doughepi/yaml-env-action@v1.0.0 + with: + files: config_automation.yml # Pass a space-separated list of configuration files. Rightmost files take precedence. + outputs: + toggle_url_check_periodically: "${{ env.URL_CHECK_PERIODICALLY }}" + + url-check: + name: Check URLs + needs: set-up + if: ${{needs.set-up.outputs.toggle_url_check_periodically == 'yes'}} + runs-on: ubuntu-latest + container: + image: jhudsl/base_ottr:main + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # Delete the branch if this has been run before + - name: Delete branch locally and remotely + run: git push origin --delete preview-spell-error || echo "No branch to delete" + + # Make the branch fresh + - name: Make the branch fresh + run: | + git config --global --add safe.directory $GITHUB_WORKSPACE + git config --global user.email "itcrtrainingnetwork@gmail.com" + git config --global user.name "jhudsl-robot" + + echo branch doesnt exist + git checkout -b preview-spell-error || echo branch exists + git push --set-upstream origin preview-spell-error || echo echo branch exists remotely + shell: bash + + - name: Run the check + uses: jhudsl/ottr-reports@main + id: check_results + continue-on-error: true + with: + check_type: urls + error_min: 1 + + - name: Declare file path and time + id: check-report + run: | + error_num=$(cat check_reports/url_checks.tsv | wc -l) + error_num="$((error_num-1))" + echo "error_num=$error_num" >> $GITHUB_OUTPUT + echo "error_url=https://github.com/${GITHUB_REPOSITORY}/blob/preview-spell-error/check_reports/url_checks.tsv" >> $GITHUB_OUTPUT + shell: bash + + - name: Stop if failure + if: steps.check_results.outcome == 'failure' + run: exit 1 + + - name: Print out error variables + run: | + echo ${{ steps.check-report.outputs.error_url }} + echo ${{ steps.check-report.outputs.error_num }} + + - name: Find issues + id: find-issue + env: + GH_PAT: ${{ secrets.GH_PAT }} + run: | + echo "$GH_PAT" + echo "$GITHUB_REPOSITORY" + issue_exists=$(Rscript --vanilla scripts/find_issue.R --repo $GITHUB_REPOSITORY --git_pat $GH_PAT) + echo URL issue exists: $issue_exists + echo "issue_existence=$issue_exists" >> $GITHUB_OUTPUT + + - name: If too many URL errors, then make an issue + if: ${{ steps.check-report.outputs.error_num >= 1 && steps.find-issue.outputs.issue_existence == 0}} + uses: JasonEtco/create-an-issue@v2 + with: + filename: .github/ISSUE_TEMPLATE/url-error.md + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FILE_URL: ${{ steps.check-report.outputs.error_url }} + ERROR_NUM: ${{ steps.check-report.outputs.error_num }} + + - name: If no URL errors than delete the branch we made + if: ${{ steps.check-report.outputs.error_num < 1 }} + run: | + git config --system --add safe.directory "$GITHUB_WORKSPACE" + git push origin --delete preview-spell-error || echo "No branch to delete" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c58b39a7..781c439c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -39,7 +39,7 @@ jobs: branch_name='preview-${{ github.event.pull_request.number }}' echo branch doesnt exist git checkout -b $branch_name || echo branch exists - git push --set-upstream origin $branch_name + git push --set-upstream origin $branch_name || echo echo branch exists remotely shell: bash diff --git a/01-intro.Rmd b/01-intro.Rmd index 886eb528..622874d6 100644 --- a/01-intro.Rmd +++ b/01-intro.Rmd @@ -4,6 +4,7 @@ ottrpal::set_knitr_image_path() ``` # Introduction +https://ahuidshiushiufhs.com ## Motivation diff --git a/OTTR_template.Rproj b/OTTR_template.Rproj new file mode 100644 index 00000000..628359e1 --- /dev/null +++ b/OTTR_template.Rproj @@ -0,0 +1,17 @@ +Version: 1.0 + +RestoreWorkspace: Default +SaveWorkspace: Default +AlwaysSaveHistory: Default + +EnableCodeIndexing: Yes +UseSpacesForTab: Yes +NumSpacesForTab: 2 +Encoding: UTF-8 + +RnwWeave: knitr +LaTeX: pdfLaTeX + +AutoAppendNewline: Yes + +BuildType: Website diff --git a/README.md b/README.md index 04db84a6..44ebd89d 100644 --- a/README.md +++ b/README.md @@ -30,4 +30,3 @@ _This template and guide helps you_: - Check out our [course on using Leanpub](https://hutchdatascience.org/Using_Leanpub/index.html) for help publishing on this platform. - Please take a look at the [code of conduct](./code_of_conduct.md). - If you encounter any problems or have ideas for improvements to this template repository or this getting started guide, please [file an issue here](https://github.com/jhudsl/OTTR_Template/issues/new/choose)! Your feedback is very much appreciated. - diff --git a/config_automation.yml b/config_automation.yml index 1c9d38a5..1a1a5776 100644 --- a/config_automation.yml +++ b/config_automation.yml @@ -12,6 +12,8 @@ spell-check: yes style-code: yes # Test build the docker image if any docker-relevant files have been changed docker-test: no +# Should URLs be tested periodically? +url-check-periodically: yes ##### Renderings run upon merge to main branch ##### # Rendering each platform's content diff --git a/resources/dictionary.txt b/resources/dictionary.txt index e3afe6f1..1520d003 100644 --- a/resources/dictionary.txt +++ b/resources/dictionary.txt @@ -1,33 +1,67 @@ AnVIL +Audiographer +Audiography +automagic BIPOC Bloomberg Bookdown +chapt Coursera +creativecommons css Datatrail DataTrail +DISCOVERABLE Dockerfile Dockerhub dropdown +enforceability +faq favicon +FirstName +Funder +fyi GDSCN -GitHub -Github +Generis GH +Github +GitHub +https +ies +iframe +immunities impactful ITCR itcrtraining ITN -fyi +LastName Leanpub +Licensor +LICENSOR Markua mentorship +MERCHANTABILITY +Muschelli NCI NHGRI +OTTR ottrpal +ottrproject Pandoc +png +pre +reproducibility +rmarkdown +RMarkdown +Savonen +sexualized +socio +synched +tada UE UE5 -reproducibility underserved +Videographer +Videography +WIPO www diff --git a/scripts/find_issue.R b/scripts/find_issue.R new file mode 100644 index 00000000..9661a569 --- /dev/null +++ b/scripts/find_issue.R @@ -0,0 +1,55 @@ +#!/usr/bin/env Rscript + +# Written by Candace Savonen Sept 2021 + +if (!("optparse" %in% installed.packages())){ + install.packages("optparse") +} + +library(optparse) + +option_list <- list( + optparse::make_option( + c("--repo"), + type = "character", + default = NULL, + help = "GitHub repository name, e.g. jhudsl/OTTR_Template", + ), + optparse::make_option( + c("--git_pat"), + type = "character", + default = NULL, + help = "GitHub personal access token", + ) +) + +# Read the arguments passed +opt_parser <- optparse::OptionParser(option_list = option_list) +opt <- optparse::parse_args(opt_parser) + +repo <- opt$repo +git_pat <- opt$git_pat + +if (!is.character(repo)) { + repo <- as.character(repo) +} + +# Github api get +result <- httr::GET( + paste0("https://api.github.com/repos/", repo, "/issues"), + httr::add_headers(Authorization = paste0("Bearer ", git_pat)), + httr::accept_json() +) + +if (httr::status_code(result) != 200) { + httr::stop_for_status(result) +} + +# Process and return results +result_content <- httr::content(result, "text") +result_list <- jsonlite::fromJSON(result_content) + +issue_exists <- length(grep('Broken URLs found in the course!', result_list$title)) + +# Print out the result +write(issue_exists, stdout())