config.yaml

---
work_dir: ./work
log_level: INFO
pool: 8

scratchpad:
  chembl_version: '34'
  efo_version: '3.70.0'
  ensembl_version: '113'

steps:
  biosample:
    - name: download cell ontology
      source: https://github.com/obophenotype/cell-ontology/releases/latest/download/cl.json
      destination: biosample/cl.json
    - name: download uberon
      source: https://github.com/obophenotype/uberon/releases/latest/download/uberon.json
      destination: biosample/uberon.json
    - name: download efo
      source: https://github.com/EBISPOT/efo/releases/download/v${efo_version}/efo.json
      destination: biosample/efo.json

  disease:
    - name: download efo otar_slim
      source: https://github.com/EBISPOT/efo/releases/download/v${efo_version}/efo_otar_slim.json
      destination: disease/efo_otar_slim.json
    - name: download hpo
      source: https://raw.githubusercontent.com/obophenotype/human-phenotype-ontology/v2021-10-10/hp.json
      destination: disease/hp.json
    - name: download hpo phenotypes
      source: http://purl.obolibrary.org/obo/hp/hpoa/phenotype.hpoa
      destination: disease/phenotype.hpoa
    - name: download mondo
      source: https://github.com/monarch-initiative/mondo/releases/download/v2021-12-30/mondo.json
      destination: disease/mondo.json

  drug:
    - name: download_latest chemical probes
      source: gs://otar001-core/ChemicalProbes/annotation
      destination: drug/chemicalprobes.json.gz
    - name: download_latest drugbank annotation
      source: gs://otar001-core/DrugBank/annotation
      destination: drug/drugbank.csv.gz
    - name: elasticsearch chembl drug indication
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: drug/chembl_drug_indication.jsonl
      index: chembl_${chembl_version}_drug_indication
      fields:
        - _metadata
        - molecule_chembl_id
        - efo_id
        - max_phase_for_ind
        - indication_refs
    - name: elasticsearch chembl drug warning
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: drug/chembl_drug_warning.jsonl
      index: chembl_${chembl_version}_drug_warning
      fields:
        - _metadata
        - molecule_chembl_id
        - parent_molecule_chembl_id
        - efo_id
        - efo_term
        - efo_id_for_warning_class
        - warning_class
        - warning_country
        - warning_description
        - warning_id
        - warning_refs
        - warning_type
        - warning_year
    - name: elasticsearch chembl mechanism of action
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: drug/chembl_mechanism.jsonl
      index: chembl_${chembl_version}_mechanism
      fields:
        - _metadata
        - molecule_chembl_id
        - target_chembl_id
        - action_type
        - mechanism_of_action
        - mechanism_refs
        - record_id
        - parent_molecule_chembl_id
    - name: elasticsearch chembl molecule
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: drug/chembl_molecule.jsonl
      index: chembl_${chembl_version}_molecule
      fields:
        - molecule_chembl_id
        - molecule_hierarchy
        - molecule_type
        - pref_name
        - first_approval
        - max_phase
        - withdrawn_flag
        - black_box_warning
        - molecule_synonyms
        - cross_references
        - chebi_par_id
        - molecule_structures
    - name: elasticsearch chembl target
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: drug/chembl_target.jsonl
      index: chembl_${chembl_version}_target
      fields:
        - _metadata
        - target_chembl_id
        - target_components
        - target_type
        - pref_name

  evidence:
    - name: download_latest cancerbiomarkers
      source: gs://otar000-evidence_input/CancerBiomarkers/json
      destination: evidence/cancerbiomarkers.json.gz
    - name: download_latest chembl
      source: gs://otar000-evidence_input/ChEMBL/json
      destination: evidence/chembl.json.gz
    - name: download_latest clingen
      source: gs://otar000-evidence_input/ClinGen/json
      destination: evidence/clingen.json.gz
    - name: download_latest crispr
      source: gs://otar000-evidence_input/CRISPR/json
      destination: evidence/crispr.json.gz
    - name: download_latest crispr_screen
      source: gs://otar000-evidence_input/Crispr_screens/json
      destination: evidence/crispr_screen.json.gz
    - name: download_latest gene2phenotype
      source: gs://otar000-evidence_input/Gene2Phenotype/json
      destination: evidence/gene2phenotype.json.gz
    - name: download_latest gene_burden
      source: gs://otar000-evidence_input/GeneBurden/json
      destination: evidence/gene_burden.json.gz
    - name: download_latest evidences
      source: gs://otar000-evidence_input/Genetics_portal/json
      destination: evidence/genetics-portal-evidences.json.gz
    - name: download_latest genomics_england
      source: gs://otar000-evidence_input/GenomicsEngland/json
      destination: evidence/genomics_england.json.gz
    - name: download_latest impc
      source: gs://otar000-evidence_input/IMPC/json
      destination: evidence/impc.json.gz
    - name: download_latest intogen
      source: gs://otar000-evidence_input/IntOgen/json
      destination: evidence/intogen.json.gz
    - name: download_latest orphanet
      source: gs://otar000-evidence_input/Orphanet/json
      destination: evidence/orphanet.json.gz
    - name: download_latest progeny
      source: gs://otar000-evidence_input/PROGENy/json
      destination: evidence/progeny.json.gz
    - name: download_latest slapenrich
      source: gs://otar000-evidence_input/SLAPEnrich/json
      destination: evidence/slapenrich.json.gz
    - name: download_latest sysbio
      source: gs://otar000-evidence_input/SysBio/json
      destination: evidence/sysbio.json.gz
    - name: download_latest reactome
      source: gs://otar006-reactome
      destination: evidence/reactome.json.gz
    - name: download_latest cosmic
      source: gs://otar007-cosmic
      pattern: '!hallmarks'
      destination: evidence/cosmic.json.gz
    - name: download_latest atlas
      source: gs://otar010-atlas
      destination: evidence/atlas.json.bz2
    - name: download_latest uniprot
      source: gs://otar011-uniprot
      destination: evidence/uniprot.json.gz
    - name: download_latest eva
      source: gs://otar012-eva/disease-target-evidence
      destination: evidence/eva.json.gz

  evidence_ppp:
    - name: download_latest validation_lab
      source: gs://otar013-ppp/validation_lab
      destination: evidence/validation_lab.json.gz
    - name: download_latest encore
      source: gs://otar013-ppp/encore
      destination: evidence/encore.json.gz
    - name: download_latest ot_crispr
      source: gs://otar013-ppp/ot_crispr
      destination: evidence/ot_crispr.json.gz

  expression:
    - name: download expression hierarchy
      source: https://raw.githubusercontent.com/opentargets/expression_hierarchy/master/process/curation.tsv
      destination: expression/expression_hierarchy_curation.tsv
    - name: download tissue translation map
      source: https://raw.githubusercontent.com/opentargets/expression_hierarchy/master/process/map_with_efos.json
      destination: expression/map_with_efos.json
    - name: download normal tissues
      source: https://www.proteinatlas.org/download/tsv/normal_tissue.tsv.zip
      destination: expression/normal_tissue.tsv.zip
    - name: download_latest baseline expression binned
      source: gs://atlas_baseline_expression/baseline_expression_binned
      destination: expression/baseline_expression_binned.tsv
    - name: download_latest baseline expression counts
      source: gs://atlas_baseline_expression/baseline_expression_counts
      destination: expression/baseline_expression_counts.tsv
    - name: download_latest baseline expression zscore binned
      source: gs://atlas_baseline_expression/baseline_expression_zscore_binned
      destination: expression/baseline_expression_zscore_binned.tsv

  go:
    - name: download gene-ontology
      source: http://purl.obolibrary.org/obo/go.obo
      destination: go/go.obo

  interaction:
    - name: download ensembl interactions grch38
      source: https://ftp.ensembl.org/pub/release-${ensembl_version}/gtf/homo_sapiens/Homo_sapiens.GRCh38.${ensembl_version}.chr.gtf.gz
      destination: interaction/Homo_sapiens.GRCh38.chr.gtf.gz
    - name: download human 9606 idmapping
      source: https://ftp.ebi.ac.uk/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/HUMAN_9606_idmapping.dat.gz
      destination: interaction/HUMAN_9606_idmapping.dat.gz
    - name: download intact interactors
      source: https://ftp.ebi.ac.uk/pub/databases/intact/various/ot_graphdb/current/data/interactor_pair_interactions.json
      destination: interaction/intact-interactors.json
    - name: download rna central
      source: https://ftp.ebi.ac.uk/pub/databases/RNAcentral/current_release/id_mapping/database_mappings/ensembl.tsv
      destination: interaction/rna_central_ensembl.tsv
    - name: download_latest string interactions
      source: gs://otar001-core/stringInteractions
      destination: interaction/string-interactions.txt.gz

  l2g_prediction:
    - name: download model
      source: gs://ot_orchestration/benchmarks/l2g/fm0/v5.1_best_cv/locus_to_gene_model/classifier.skops
      destination: l2g_prediction/classifier.skops

  literature:
    - name: download literature
      source: https://ftp.ebi.ac.uk/pub/databases/pmc/DOI/PMID_PMCID_DOI.csv.gz
      destination: literature/PMID_PMCID_DOI.csv.gz

  openfda:
    - name: download blacklisted events
      source: https://raw.githubusercontent.com/opentargets/platform-etl-backend/master/src/main/resources/blacklisted_events.txt
      destination: openfda/blacklisted_events.txt
    - name: explode fda events
      foreach_function: urls_from_json
      foreach_function_args:
        source: https://api.fda.gov/download.json
        destination: openfda/fda_events.json
        json_path: .results.drug.event.partitions[].file
        prefix: https://download.open.fda.gov/drug/event/
      do:
        - name: download fda events ${destination}
          source: ${source}
          destination: openfda/${destination}

  otar:
    - name: download otar meta spreadsheet
      source: https://docs.google.com/spreadsheets/d/1CV_shXJy1ACM09HZBB_-3Nl6l_dfkrA26elMAF0ttHs/export?format=csv&gid=1179867447
      destination: otar/otar_meta.csv
    - name: download otar project to efo spreadsheet
      source: https://docs.google.com/spreadsheets/d/1CV_shXJy1ACM09HZBB_-3Nl6l_dfkrA26elMAF0ttHs/export?format=csv&gid=72910172
      destination: otar/otar_project_to_efo.csv

  pharmacogenomics:
    - name: download_latest pharmacogenomics
      source: gs://otar001-core/Pharmacogenetics/json
      destination: pharmacogenomics/pharmacogenomics.json.gz

  reactome:
    - name: download reactome pathways
      source: https://reactome.org/download/current/ReactomePathways.txt
      destination: reactome/ReactomePathways.txt
    - name: download reactome pathways relation
      source: https://reactome.org/download/current/ReactomePathwaysRelation.txt
      destination: reactome/ReactomePathwaysRelation.txt

  so:
    - name: download so
      source: https://raw.githubusercontent.com/The-Sequence-Ontology/SO-Ontologies/master/Ontology_Files/so.json
      destination: so/so.json

  target:
    - name: explode homologies
      foreach:
        - species: caenorhabditis_elegans
        - species: canis_lupus_familiaris
        - species: cavia_porcellus
        - species: danio_rerio
        - species: drosophila_melanogaster
        - species: macaca_mulatta
        - species: mus_musculus
        - species: oryctolagus_cuniculus
        - species: pan_troglodytes
        - species: rattus_norvegicus
        - species: sus_scrofa
        - species: xenopus_tropicalis
        - species: homo_sapiens
      do:
        - name: download ${species} gene dictionary
          source: https://ftp.ensembl.org/pub/release-${ensembl_version}/json/${species}/${species}.json
          destination: target/homologue/gene_dictionary/${species}.json
        - name: download ${species} protein homologies
          source: https://ftp.ensembl.org/pub/release-${ensembl_version}/tsv/ensembl-compara/homologies/${species}/Compara.${ensembl_version}.protein_default.homologies.tsv
          destination: target/homologue/homologies/protein-${species}.tsv
        - name: download ${species} ncrna homologies
          source: https://ftp.ensembl.org/pub/release-${ensembl_version}/tsv/ensembl-compara/homologies/${species}/Compara.${ensembl_version}.ncrna_default.homologies.tsv
          destination: target/homologue/homologies/ncrna-${species}.tsv
    - name: elasticsearch chembl targets
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: target/chembl/chembl_target.jsonl
      index: chembl_${chembl_version}_target
      fields:
        - _metadata
        - target_chembl_id
        - target_components
        - target_type
        - pref_name
    - name: download hgnc complete set
      source: https://storage.googleapis.com/public-download-files/hgnc/json/json/hgnc_complete_set.json
      destination: target/genenames/hgnc_complete_set.json
    - name: download Ensembl2Reactome
      source: https://reactome.org/download/current/Ensembl2Reactome.txt
      destination: target/reactome/Ensembl2Reactome.txt
    - name: download human_all_hcop_sixteen_column
      source: https://ftp.ebi.ac.uk/pub/databases/genenames/hcop/human_all_hcop_sixteen_column.txt.gz
      destination: target/genenames/human_all_hcop_sixteen_column.txt.gz
    - name: download gene identifiers
      source: https://cog.sanger.ac.uk/cmp/download/gene_identifiers_latest.csv.gz
      destination: target/project-scores/gene_identifiers_latest.csv.gz
    - name: download uniprot
      source: https://rest.uniprot.org/uniprotkb/stream?compressed=true&format=txt&query=%28%28reviewed%3Atrue%29%29%20AND%20%28model_organism%3A9606%29
      destination: target/uniprot/uniprot.txt.gz
    - name: download uniprot-ssl
      source: https://rest.uniprot.org/locations/stream?compressed=true&fields=id%2Cname%2Ccategory&format=tsv&query=%28%2A%29
      destination: target/uniprot/uniprot-ssl.tsv.gz
    - name: download gencode
      source: https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_40/gencode.v40.annotation.gff3.gz
      destination: target/gencode/gencode.gff3.gz
    - name: download ensembl
      source: https://ftp.ebi.ac.uk/pub/databases/RNAcentral/current_release/id_mapping/database_mappings/ensembl.tsv
      destination: target/go/ensembl.tsv
    - name: download goa_human gaf
      source: https://ftp.ebi.ac.uk/pub/databases/GO/goa/HUMAN/goa_human.gaf.gz
      destination: target/go/goa_human.gaf.gz
    - name: download goa_human_rna gaf
      source: https://ftp.ebi.ac.uk/pub/databases/GO/goa/HUMAN/goa_human_rna.gaf.gz
      destination: target/go/goa_human_rna.gaf.gz
    - name: download goa_human gpa
      source: https://ftp.ebi.ac.uk/pub/databases/GO/goa/HUMAN/goa_human.gpa.gz
      destination: target/go/goa_human_eco.gpa.gz
    - name: download ensembl vertebrates
      source: https://ftp.ensembl.org/pub/release-${ensembl_version}/species_EnsemblVertebrates.txt
      destination: target/homologue/species_EnsemblVertebrates.txt
    - name: download homosapiens gene_info
      source: https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Homo_sapiens.gene_info.gz
      destination: target/ncbi/Homo_sapiens.gene_info.gz
    - name: download gnomad
      source: https://storage.googleapis.com/gcp-public-data--gnomad/release/2.1.1/constraint/gnomad.v2.1.1.lof_metrics.by_gene.txt.bgz
      destination: target/gnomad/gnomad.v2.1.1.lof_metrics.by_gene.txt.bgz
    - name: download protein atlas subcellular location
      source: https://www.proteinatlas.org/download/tsv/subcellular_location.tsv.zip
      destination: target/hpa/subcellular_location.tsv.zip
    - name: download essentiality matrices
      source: https://cog.sanger.ac.uk/cmp/download/essentiality_matrices.zip
      destination: target/project-scores/essentiality_matrices.zip
    - name: download ensembl
      source: https://ftp.ensembl.org/pub/release-${ensembl_version}/json/homo_sapiens/homo_sapiens.json
      destination: target/ensembl/homo_sapiens.json
    - name: download_latest essentiality
      source: gs://otar000-evidence_input/Essentiality/json
      destination: target/gene-essentiality/essentiality.json.gz
    - name: download_latest subcellular locations
      source: gs://otar001-core/subcellularLocations
      destination: target/hpa/subcellular_locations_ssl.tsv
    - name: download_latest teps
      source: gs://otar001-core/TEPs
      destination: target/tep/tep.json.gz
    - name: download_latest chemical probes
      source: gs://otar001-core/ChemicalProbes/annotation
      destination: target/chemicalprobes/chemicalprobes.json.gz
    - name: download_latest target safety
      source: gs://otar001-core/TargetSafety/json
      destination: target/safety/safetyLiabilities.json.gz
    - name: download_latest tractability
      source: gs://otar001-core/Tractability
      destination: target/tractability/tractability.tsv
    - name: download_latest cosmic
      source: gs://otar007-cosmic
      pattern: 'hallmarks'
      destination: target/hallmarks/cosmic-hallmarks.tsv.gz

  target_engine:
    - name: download protein atlas
      source: https://www.proteinatlas.org/download/proteinatlas.json.gz
      destination: target_engine/proteinatlas.json.gz
    - name: download uniprot locations
      source: https://rest.uniprot.org/locations/stream?compressed=true&fields=id%2Cname%2Ccategory%2Cgene_ontologies%2Cpart_of%2Cis_a&format=tsv&query=%28%2A%29
      destination: target_engine/uniprot_locations.tsv.gz
    - name: download mouse phenotype scores
      source: https://raw.githubusercontent.com/opentargets/target_engine/main/src/data_flow/phenotypeScores/20230825_mousePheScores.csv
      destination: target_engine/mouse_pheno_scores.csv

  target_validation:
    - name: download_latest mouse phenotypes
      source: gs://otar001-core/MousePhenotypes
      destination: target_validation/mouse_phenotypes.json.gz