Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixes and tests for data fetch models. #18834

Merged
merged 1 commit into from
Oct 1, 2024
Merged

Fixes and tests for data fetch models. #18834

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions client/src/api/schema/schema.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6330,6 +6330,12 @@ export interface components {
CompositeDataElement: {
/** Md5 */
MD5?: string | null;
/** Sha-1 */
"SHA-1"?: string | null;
/** Sha-256 */
"SHA-256"?: string | null;
/** Sha-512 */
"SHA-512"?: string | null;
/**
* Auto Decompress
* @description Decompress compressed data before sniffing?
Expand Down Expand Up @@ -6360,6 +6366,8 @@ export interface components {
*/
ext: string;
extra_files?: components["schemas"]["ExtraFiles"] | null;
/** Hashes */
hashes?: components["schemas"]["FetchDatasetHash"][] | null;
/** Info */
info?: string | null;
/** Name */
Expand Down Expand Up @@ -8668,10 +8676,26 @@ export interface components {
} & {
[key: string]: unknown;
};
/** FetchDatasetHash */
FetchDatasetHash: {
/**
* Hash Function
* @enum {string}
*/
hash_function: "MD5" | "SHA-1" | "SHA-256" | "SHA-512";
/** Hash Value */
hash_value: string;
};
/** FileDataElement */
FileDataElement: {
/** Md5 */
MD5?: string | null;
/** Sha-1 */
"SHA-1"?: string | null;
/** Sha-256 */
"SHA-256"?: string | null;
/** Sha-512 */
"SHA-512"?: string | null;
/**
* Auto Decompress
* @description Decompress compressed data before sniffing?
Expand Down Expand Up @@ -8701,6 +8725,8 @@ export interface components {
*/
ext: string;
extra_files?: components["schemas"]["ExtraFiles"] | null;
/** Hashes */
hashes?: components["schemas"]["FetchDatasetHash"][] | null;
/** Info */
info?: string | null;
/** Name */
Expand Down Expand Up @@ -8966,6 +8992,12 @@ export interface components {
FtpImportElement: {
/** Md5 */
MD5?: string | null;
/** Sha-1 */
"SHA-1"?: string | null;
/** Sha-256 */
"SHA-256"?: string | null;
/** Sha-512 */
"SHA-512"?: string | null;
/**
* Auto Decompress
* @description Decompress compressed data before sniffing?
Expand Down Expand Up @@ -8997,6 +9029,8 @@ export interface components {
extra_files?: components["schemas"]["ExtraFiles"] | null;
/** Ftp Path */
ftp_path: string;
/** Hashes */
hashes?: components["schemas"]["FetchDatasetHash"][] | null;
/** Info */
info?: string | null;
/** Name */
Expand Down Expand Up @@ -13374,6 +13408,12 @@ export interface components {
NestedElement: {
/** Md5 */
MD5?: string | null;
/** Sha-1 */
"SHA-1"?: string | null;
/** Sha-256 */
"SHA-256"?: string | null;
/** Sha-512 */
"SHA-512"?: string | null;
/**
* Auto Decompress
* @description Decompress compressed data before sniffing?
Expand Down Expand Up @@ -13416,6 +13456,8 @@ export interface components {
*/
ext: string;
extra_files?: components["schemas"]["ExtraFiles"] | null;
/** Hashes */
hashes?: components["schemas"]["FetchDatasetHash"][] | null;
/** Info */
info?: string | null;
/** Name */
Expand Down Expand Up @@ -14018,6 +14060,12 @@ export interface components {
PastedDataElement: {
/** Md5 */
MD5?: string | null;
/** Sha-1 */
"SHA-1"?: string | null;
/** Sha-256 */
"SHA-256"?: string | null;
/** Sha-512 */
"SHA-512"?: string | null;
/**
* Auto Decompress
* @description Decompress compressed data before sniffing?
Expand Down Expand Up @@ -14047,6 +14095,8 @@ export interface components {
*/
ext: string;
extra_files?: components["schemas"]["ExtraFiles"] | null;
/** Hashes */
hashes?: components["schemas"]["FetchDatasetHash"][] | null;
/** Info */
info?: string | null;
/** Name */
Expand Down Expand Up @@ -14078,6 +14128,12 @@ export interface components {
PathDataElement: {
/** Md5 */
MD5?: string | null;
/** Sha-1 */
"SHA-1"?: string | null;
/** Sha-256 */
"SHA-256"?: string | null;
/** Sha-512 */
"SHA-512"?: string | null;
/**
* Auto Decompress
* @description Decompress compressed data before sniffing?
Expand Down Expand Up @@ -14107,6 +14163,8 @@ export interface components {
*/
ext: string;
extra_files?: components["schemas"]["ExtraFiles"] | null;
/** Hashes */
hashes?: components["schemas"]["FetchDatasetHash"][] | null;
/** Info */
info?: string | null;
/** Link Data Only */
Expand Down Expand Up @@ -14799,6 +14857,12 @@ export interface components {
ServerDirElement: {
/** Md5 */
MD5?: string | null;
/** Sha-1 */
"SHA-1"?: string | null;
/** Sha-256 */
"SHA-256"?: string | null;
/** Sha-512 */
"SHA-512"?: string | null;
/**
* Auto Decompress
* @description Decompress compressed data before sniffing?
Expand Down Expand Up @@ -14828,6 +14892,8 @@ export interface components {
*/
ext: string;
extra_files?: components["schemas"]["ExtraFiles"] | null;
/** Hashes */
hashes?: components["schemas"]["FetchDatasetHash"][] | null;
/** Info */
info?: string | null;
/** Link Data Only */
Expand Down Expand Up @@ -16443,6 +16509,12 @@ export interface components {
UrlDataElement: {
/** Md5 */
MD5?: string | null;
/** Sha-1 */
"SHA-1"?: string | null;
/** Sha-256 */
"SHA-256"?: string | null;
/** Sha-512 */
"SHA-512"?: string | null;
/**
* Auto Decompress
* @description Decompress compressed data before sniffing?
Expand Down Expand Up @@ -16472,6 +16544,8 @@ export interface components {
*/
ext: string;
extra_files?: components["schemas"]["ExtraFiles"] | null;
/** Hashes */
hashes?: components["schemas"]["FetchDatasetHash"][] | null;
/** Info */
info?: string | null;
/** Name */
Expand Down
11 changes: 11 additions & 0 deletions lib/galaxy/schema/fetch_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,13 @@ class ExtraFiles(FetchBaseModel):
)


class FetchDatasetHash(Model):
hash_function: Literal["MD5", "SHA-1", "SHA-256", "SHA-512"]
hash_value: str

model_config = ConfigDict(extra="forbid")


class BaseDataElement(FetchBaseModel):
name: Optional[CoercedStringType] = None
dbkey: str = Field("?")
Expand All @@ -116,6 +123,10 @@ class BaseDataElement(FetchBaseModel):
items_from: Optional[ElementsFromType] = Field(None, alias="elements_from")
collection_type: Optional[str] = None
MD5: Optional[str] = None
SHA1: Optional[str] = Field(None, alias="SHA-1")
SHA256: Optional[str] = Field(None, alias="SHA-256")
SHA512: Optional[str] = Field(None, alias="SHA-512")
hashes: Optional[List[FetchDatasetHash]] = None
description: Optional[str] = None
model_config = ConfigDict(extra="forbid")

Expand Down
4 changes: 4 additions & 0 deletions lib/galaxy/tools/data_fetch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,10 @@ def _resolve_item_with_primary(item):
if url:
sources.append(source_dict)
hashes = item.get("hashes", [])
for hash_function in HASH_NAMES:
hash_value = item.get(hash_function)
if hash_value:
hashes.append({"hash_function": hash_function, "hash_value": hash_value})
for hash_dict in hashes:
hash_function = hash_dict.get("hash_function")
hash_value = hash_dict.get("hash_value")
Expand Down
61 changes: 61 additions & 0 deletions lib/galaxy_test/api/test_tools_upload.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import json
import os
import urllib.parse
from base64 import b64encode

import pytest
from tusclient import client
Expand All @@ -25,6 +26,9 @@
)
from ._framework import ApiTestCase

B64_FOR_1_2_3 = b64encode(b"1 2 3").decode("utf-8")
URI_FOR_1_2_3 = f"base64://{B64_FOR_1_2_3}"


class TestToolsUpload(ApiTestCase):
dataset_populator: DatasetPopulator
Expand Down Expand Up @@ -927,6 +931,63 @@ def test_upload_and_validate_valid(self):
terminal_validated_state = self.dataset_populator.validate_dataset_and_wait(history_id, dataset_id)
assert terminal_validated_state == "ok", terminal_validated_state

def test_upload_and_validate_hash_valid(self):
with self.dataset_populator.test_history() as history_id:
destination = {"type": "hdas"}
targets = [
{
"destination": destination,
"items": [
{
"src": "url",
"url": URI_FOR_1_2_3,
"hashes": [
{"hash_function": "SHA-1", "hash_value": "65e9d53484d28eef5447bc06fe2d754d1090975a"}
],
},
],
}
]
payload = {
"history_id": history_id,
"targets": targets,
"validate_hashes": True,
}
fetch_response = self.dataset_populator.fetch(payload)
self._assert_status_code_is(fetch_response, 200)
# history ok implies the dataset upload work
self.dataset_populator.wait_for_history(history_id, assert_ok=True)

def test_upload_and_validate_hash_invalid(self):
with self.dataset_populator.test_history() as history_id:
destination = {"type": "hdas"}
targets = [
{
"destination": destination,
"items": [
{
"src": "url",
"url": URI_FOR_1_2_3,
"hashes": [{"hash_function": "SHA-1", "hash_value": "invalidhash"}],
},
],
}
]
payload = {
"history_id": history_id,
"targets": targets,
"validate_hashes": True,
}
fetch_response = self.dataset_populator.fetch(payload, assert_ok=True, wait=False)
self._assert_status_code_is(fetch_response, 200)
outputs = fetch_response.json()["outputs"]
new_dataset = outputs[0]
self.dataset_populator.wait_for_history(history_id, assert_ok=False)
dataset_details = self.dataset_populator.get_history_dataset_details(
history_id, dataset=new_dataset, assert_ok=False
)
assert dataset_details["state"] == "error"

def _velvet_upload(self, history_id, extra_inputs):
payload = self.dataset_populator.upload_payload(
history_id,
Expand Down
Loading
Loading