diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8e9d58776747f0..96cc207e8f5421 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -67,6 +67,7 @@ jobs: export PATH=/home/runner/.local/bin:$PATH make annotate ACTIVATE_ENV=pwd curl -L https://docs.google.com/spreadsheets/d/1NfZhi5Jav7kl9zFCkeb7rIC2F8xW1isruv1TeO4WpNI/export\?format\=tsv | ruby bin/prepare_feedback.rb + curl -L https://hexylena.github.io/toolshed-version-database/tool-meta.json > metadata/tool-meta.json npm install make rebuild-search-index ACTIVATE_ENV=pwd cat metadata/swagger.yaml | python bin/yaml2json.py > api/swagger.json diff --git a/.github/workflows/monthly-release-backfill.yml b/.github/workflows/monthly-release-backfill.yml index 001662dbefa256..6a3173ba65fdd4 100644 --- a/.github/workflows/monthly-release-backfill.yml +++ b/.github/workflows/monthly-release-backfill.yml @@ -66,6 +66,7 @@ jobs: sed -i s"|^baseurl: .*|baseurl: '/archive/${SOURCE_TAG}'|g" _config.yml sed -i s"|^github_repository_branch: .*|github_repository_branch: '${SOURCE_TAG}'|g" _config.yml sed -i s"|^title: .*|title: 'GTN Archive ${SOURCE_TAG}'|g" _config.yml + curl -L https://hexylena.github.io/toolshed-version-database/tool-meta.json > metadata/tool-meta.json npm install make rebuild-search-index ACTIVATE_ENV=pwd cat metadata/swagger.yaml | python bin/yaml2json.py > api/swagger.json diff --git a/.github/workflows/monthly-release.yml b/.github/workflows/monthly-release.yml index 54176d0a4e633e..bd983e23bc57dc 100644 --- a/.github/workflows/monthly-release.yml +++ b/.github/workflows/monthly-release.yml @@ -68,6 +68,7 @@ jobs: sed -i s"|^baseurl: .*|baseurl: '/archive/${SOURCE_TAG}'|g" _config.yml sed -i s"|^github_repository_branch: .*|github_repository_branch: '${SOURCE_TAG}'|g" _config.yml sed -i s"|^title: .*|title: 'GTN Archive ${SOURCE_TAG}'|g" _config.yml + curl -L https://hexylena.github.io/toolshed-version-database/tool-meta.json > metadata/tool-meta.json npm install make rebuild-search-index ACTIVATE_ENV=pwd cat metadata/swagger.yaml | ruby bin/yaml2json.rb > api/swagger.json diff --git a/.github/workflows/workflowhub.yml b/.github/workflows/workflowhub.yml index 93e792c34a5aa1..d304bb248c65ea 100644 --- a/.github/workflows/workflowhub.yml +++ b/.github/workflows/workflowhub.yml @@ -1,10 +1,12 @@ name: "[Cron] Upload workflows to (DEV)WorkflowHub" +concurrency: + group: wfh-${{ github.head_ref }} + cancel-in-progress: false # Not sure about this. + on: schedule: - # * is a special character in YAML so you have to quote this string - # We'll run this daily at noon. - - cron: '59 12 * * *' + - cron: '55 12 * * 1' workflow_dispatch: jobs: @@ -53,7 +55,11 @@ jobs: run: | python bin/wfh-upload.py env: - DEV_WFH_TOKEN: ${{ secrets.DEV_WFH_TOKEN }} + WFH_TOKEN: ${{ secrets.DEV_WFH_TOKEN }} + + - name: Any changes? + run: | + git diff - name: Create Pull Request uses: peter-evans/create-pull-request@v3 diff --git a/_layouts/by_tool.html b/_layouts/by_tool.html index a27f06731a52e2..3c6837296ea004 100644 --- a/_layouts/by_tool.html +++ b/_layouts/by_tool.html @@ -2,24 +2,64 @@ layout: base --- -

Tutorials using {{ page.short_tool }}

+{% assign tool_id_path = page.observed_tool_ids[0][0] %} + +
+

+ {{ site.data['tool-meta'][tool_id_path].name }}: + {{ site.data['tool-meta'][tool_id_path].desc }} +

+
{{ page.observed_tool_ids[0] }}
+
+
-

Servers Offering this Tool

+

Metadata

+

Servers Offering this Tool

UseGalaxy.eu ({{ page.observed_tool_ids[0][1] }})

This is a guess, we are not currently checking if that is actually present.

-

Observed Tool Versions

+
+

Observed Tool Versions

+

Within GTN tutorials

+
    {% for tool in page.observed_tool_ids %}
  • {{ tool[1] }}
  • {% endfor %}
+ + {% if site.data['tool-meta'][tool_id_path].edam_operations %} +

EDAM Operations

+
    + {% for topic in site.data['tool-meta'][tool_id_path].edam_operations %} +
  • + + {{ topic }} + +
  • + {% endfor %} +
+ {% endif %} + + {% if site.data['tool-meta'][tool_id_path].edam_topics%} +

EDAM Topics

+
    + {% for topic in site.data['tool-meta'][tool_id_path].edam_topics %} +
  • + + {{ topic }} + +
  • + {% endfor %} +
+ {% endif %}
+

Relevant Tutorials

    {% for tutorial in page.tutorial_list %}
  • diff --git a/_layouts/workflow.html b/_layouts/workflow.html index 11ead822106762..dce3e843eab3fa 100644 --- a/_layouts/workflow.html +++ b/_layouts/workflow.html @@ -116,8 +116,9 @@

    {{ workflow.title }}


    {% if jekyll.environment == "production" %} - RO-Crate logo with flask Download Workflow RO-Crate + RO-Crate logo with flask Download Workflow RO-Crate {% endif %} + {% if workflow.workflowhub_id %} Workflowhub cloud with gears logo View on WorkflowHub {% endif %} @@ -148,6 +149,70 @@

    {{ workflow.title }}

    Download
    {{ workflow.mermaid }}
    +

    Inputs

    + + + + + + + + + {% for input in workflow.inputs %} + + + + + {% endfor %} + +
    InputLabel
    {{ input.name }}{{ input.label }}
    + +

    Outputs

    + + + + + + + + + + {% for output in workflow.outputs %} + + + + + + {% endfor %} + +
    FromOutputLabel
    {{ output.tool_id }}{{ output.name }}{{ output.label }}
    + + +

    Tools

    + + + + + + + + + {% for tool in workflow.workflow_tools %} + + + + + {% endfor %} + +
    ToolLinks
    {{ tool }} + {% if site.data['toolshed-revisions'][tool] %} + {% assign tsdata = site.data['toolshed-revisions'][tool] %} + + View in ToolShed + + {% endif %} +
    +

    To use these workflows in Galaxy you can either click the links to download the workflows, or you can right-click and copy the link to the workflow which can be used in the Galaxy form to import workflows.

    diff --git a/_plugins/gtn/ro-crate.rb b/_plugins/gtn/ro-crate.rb index 8d38bb8769e7ae..0f6fd4b7475d8c 100644 --- a/_plugins/gtn/ro-crate.rb +++ b/_plugins/gtn/ro-crate.rb @@ -79,9 +79,9 @@ def self.write(site, dir, associated_material, workflow, url, baseurl, time_base } mat_contribs = [ + ['Workflow Author(s)', workflow['creators'].map { |c| c['name'] }], ['Tutorial Author(s)', Gtn::Contributors.get_authors(associated_material).map { |n| name2md(site, n) }], ['Tutorial Contributor(s)', Gtn::Contributors.get_non_authors(associated_material).map { |n| name2md(site, n) }], - ['Workflow Author(s)', workflow['creators'].map { |c| c['name'] }], ['Funder(s)', Gtn::Contributors.get_funders(associated_material).map { |n| name2md(site, n) }], ].reject { |_, v| v.empty? } @@ -90,7 +90,7 @@ def self.write(site, dir, associated_material, workflow, url, baseurl, time_base ## Associated Tutorial -This workflows is part of the tutorial [#{workflow['name']}](#{url}#{baseurl}/topics/#{workflow['topic_id']}/tutorials/#{workflow['tutorial_id']}/tutorial.html), available in the [GTN](https://training.galaxyproject.org) +This workflows is part of the tutorial [#{associated_material['title']}](#{url}#{baseurl}/topics/#{workflow['topic_id']}/tutorials/#{workflow['tutorial_id']}/tutorial.html), available in the [GTN](https://training.galaxyproject.org) #{"## Features" if features.values.any?} @@ -100,9 +100,8 @@ def self.write(site, dir, associated_material, workflow, url, baseurl, time_base #{mat_contribs.map { |k, v| "**#{k}**: #{v.join(', ')}" }.join("\n\n")} -[![gtn star logo followed by the word workflows](http://galaxy-training.s3-website.us-east-1.amazonaws.com/misc/gtn-workflows.png)](https://training.galaxyproject.org/training-material/) +[![gtn star logo followed by the word workflows](https://training.galaxyproject.org/training-material/assets/branding/gtn-workflows.png)](https://training.galaxyproject.org/training-material/) ).strip - # ![gtn star logo followed by the word workflows](https://training.galaxyproject.org/training-material/assets/branding/gtn-workflows.png) crate = { '@context' => ['https://w3id.org/ro/crate/1.1/context'], diff --git a/_plugins/jekyll-topic-filter.rb b/_plugins/jekyll-topic-filter.rb index d6cc1e0bf368b7..c5b502d6c42ea3 100644 --- a/_plugins/jekyll-topic-filter.rb +++ b/_plugins/jekyll-topic-filter.rb @@ -699,7 +699,9 @@ def self.resolve_material(site, material) 'modified' => File.mtime(wf_path), 'mermaid' => mermaid(wf_json), 'graph_dot' => graph_dot(wf_json), - 'workflow_tools' => extract_workflow_tool_list(wf_json).uniq, + 'workflow_tools' => extract_workflow_tool_list(wf_json).flatten.uniq.sort, + 'inputs' => wf_json['steps'].select { |_k, v| ['data_input', 'data_collection_input', 'parameter_input'].include? v['type'] }.map{|_, v| v}, + 'outputs' => wf_json['steps'].select { |_k, v| v['workflow_outputs'] && v['workflow_outputs'].length.positive? }.map{|_, v| v}, } end end diff --git a/assets/images/ro-crate.png b/assets/images/ro-crate.png new file mode 100644 index 00000000000000..64b00a9a09df11 Binary files /dev/null and b/assets/images/ro-crate.png differ diff --git a/assets/images/workflowhub-icon.svg b/assets/images/workflowhub-icon.svg index 1c90ac0872b2f9..fc95a3e705a40f 100644 --- a/assets/images/workflowhub-icon.svg +++ b/assets/images/workflowhub-icon.svg @@ -2,23 +2,23 @@ + inkscape:export-ydpi="159.83502" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:dc="http://purl.org/dc/elements/1.1/"> WorkflowHub logo + inkscape:snap-bbox-edge-midpoints="true" + inkscape:showpageshadow="2" + inkscape:pagecheckerboard="0" + inkscape:deskcolor="#d1d1d1" /> + transform="translate(-27.218581,-35.645213)" /> + transform="translate(-27.218581,-35.645213)" /> + transform="translate(-27.218581,75.604784)" /> - - - - - - - - - - - - - WorkflowHub + transform="translate(-27.218581,-4.3952111)"> diff --git a/assets/images/workflowhub.svg b/assets/images/workflowhub.svg new file mode 100644 index 00000000000000..1c90ac0872b2f9 --- /dev/null +++ b/assets/images/workflowhub.svg @@ -0,0 +1,212 @@ + + + + + WorkflowHub logo + + + + image/svg+xml + + WorkflowHub logo + 2020-09-23 + + + Finn Bacall, Stian Soiland-Reyes + + + + + The University of Manchester + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + WorkflowHub + + + + + diff --git a/bin/wfh-upload.py b/bin/wfh-upload.py index bac496101d3c07..e1d212b95333bd 100755 --- a/bin/wfh-upload.py +++ b/bin/wfh-upload.py @@ -7,11 +7,24 @@ import glob import time import multiprocessing +import argparse -GTN_PROJECT_ID = 63 +# GTN_BOT_ID = https://workflowhub.eu/people/731 -if len(sys.argv) == 2: - crates = [sys.argv[1]] +argparser = argparse.ArgumentParser() +argparser.add_argument("--prod", action="store_true", help="Upload to production") +argparser.add_argument("--crate", type=str, help="Upload a single crate (defaults to all)") +args = argparser.parse_args() + +if args.prod: + WORKFLOWHUB = "https://workflowhub.eu" + GTN_PROJECT_ID = 12 +else: + WORKFLOWHUB = "https://dev.workflowhub.eu" + GTN_PROJECT_ID = 63 + +if args.crate: + crates = [args.crate] else: crates = glob.glob( "_site/training-material/api/workflows/**/rocrate.zip", recursive=True @@ -34,12 +47,14 @@ def doUpload(crate_path): "workflow[project_ids][]": (None, GTN_PROJECT_ID), # GTN's ID. } headers = { - "authorization": "Token " + os.environ["DEV_WFH_TOKEN"], + "authorization": "Token " + os.environ["WFH_TOKEN"], 'User-Agent': 'GTN (github.com/galaxyproject/training-material@1.0)', + # 'Content-type': 'application/json', + 'Accept': 'application/json', } response = requests.post( - "https://dev.workflowhub.eu/workflows/submit", files=payload, headers=headers + f"{WORKFLOWHUB}/workflows/submit", files=payload, headers=headers ) code = response.status_code if code != 200: @@ -56,25 +71,57 @@ def doUpload(crate_path): "id": wfid, "type": "workflows", "attributes": { - "policy": { - "access": "download", - "permissions": [ - { - "resource": { - "id": str(GTN_PROJECT_ID), - "type": "projects" - }, - "access": "manage" - } - ] - } } } } + current_policy = response.json()['data']['attributes']['policy'] + + updated_policy = {} + if current_policy['access'] != 'download': + updated_policy['access'] = 'download' - if response.json()['data']['attributes']['discussion_links'] and not any( + gtn_permission = [x for x in current_policy['permissions'] if x['resource']['id'] == str(GTN_PROJECT_ID)] + if len(gtn_permission) != 1: + updated_policy['permissions'] = current_policy['permissions'] + [ + { + "resource": { + "id": str(GTN_PROJECT_ID), + "type": "projects" + }, + "access": "manage" + } + ] + else: + if gtn_permission[0]['access'] != 'manage': + gtn_permission[0]['access'] = 'manage' + updated_policy['permissions'] = current_policy['permissions'] + + # "policy": { + # "access": "download", + # "permissions": [ + # { + # "resource": { + # "id": str(GTN_PROJECT_ID), + # "type": "projects" + # }, + # "access": "manage" + # } + # ] + # } + + # {'access': 'download', + # 'permissions': [{'access': 'manage', + # 'resource': {'id': '63', 'type': 'projects'}}]} + # + if updated_policy: + push = True + permissions_update['data']['attributes']['policy'] = updated_policy + + dls = response.json()['data']['attributes']['discussion_links'] + if dls is None or len(dls) == 0 or (not any( x['label'] == 'GTN Matrix' for x in response.json()['data']['attributes']['discussion_links'] - ): + )): + push = True permissions_update['data']['attributes']['discussion_links'] = [ { "label": "GTN Matrix", @@ -82,14 +129,16 @@ def doUpload(crate_path): } ] - - headers.update({ - 'Content-type': 'application/json', - 'Accept': 'application/json', - }) - response2 = requests.put(f"https://dev.workflowhub.eu/workflows/{wfid}", headers=headers, json=permissions_update) - if response2.status_code != 200: - print(f"Error {response2.status_code} updating permissions for {wfid}: {response2.text}") + # https://github.com/seek4science/seek/issues/1957 + if push: + print(f"Permissions update required for {wfid}: {permissions_update['data']['attributes']}") + headers.update({ + 'Content-type': 'application/json', + 'Accept': 'application/json', + }) + response2 = requests.put(f"{WORKFLOWHUB}/workflows/{wfid}", headers=headers, json=permissions_update) + if response2.status_code != 200: + print(f"Error {response2.status_code} updating permissions for {wfid}: {response2.text}") p = crate_path.split("/") windex = p.index("workflows")