diff --git a/docs/changelog.md b/docs/changelog.md index efe355a0..51715b78 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +## [0.5.3] - 2019-10-29 + +### Changed +- `genome_server` config key to `genome_servers` +- enable multiple refgenieservers for `pull_asset` method +- `_chk_digest_if_avail` and `_chk_digest_update_child` require server URL argument to get the asset information from + ## [0.5.2] - 2019-10-22 ### Changed diff --git a/refgenconf/_version.py b/refgenconf/_version.py index 72251527..43a1e95b 100644 --- a/refgenconf/_version.py +++ b/refgenconf/_version.py @@ -1 +1 @@ -__version__ = "0.5.2" +__version__ = "0.5.3" diff --git a/refgenconf/const.py b/refgenconf/const.py index 72388771..51fb20f6 100644 --- a/refgenconf/const.py +++ b/refgenconf/const.py @@ -36,6 +36,7 @@ "API_ID_RECIPE", "API_ID_LOG"] CFG_FOLDER_KEY = "genome_folder" +CFG_SERVERS_KEY = "genome_servers" CFG_SERVER_KEY = "genome_server" CFG_ARCHIVE_KEY = "genome_archive" CFG_VERSION_KEY = "config_version" @@ -59,7 +60,7 @@ CFG_TAG_DESC_KEY = "tag_description" CFG_TOP_LEVEL_KEYS = [ - CFG_FOLDER_KEY, CFG_SERVER_KEY, CFG_ARCHIVE_KEY, CFG_GENOMES_KEY, CFG_VERSION_KEY] + CFG_FOLDER_KEY, CFG_SERVER_KEY, CFG_SERVERS_KEY, CFG_ARCHIVE_KEY, CFG_GENOMES_KEY, CFG_VERSION_KEY] CFG_GENOME_KEYS = [ CFG_GENOME_DESC_KEY, CFG_ASSETS_KEY, CFG_CHECKSUM_KEY] CFG_GENOME_ATTRS_KEYS = [CFG_GENOME_DESC_KEY, CFG_CHECKSUM_KEY] @@ -67,7 +68,7 @@ CFG_ARCHIVE_CHECKSUM_KEY, CFG_SEEK_KEYS_KEY] CFG_KEY_NAMES = [ - "CFG_FOLDER_KEY", "CFG_SERVER_KEY", "CFG_GENOMES_KEY", + "CFG_FOLDER_KEY", "CFG_SERVER_KEY", "CFG_SERVERS_KEY", "CFG_GENOMES_KEY", "CFG_ASSET_PATH_KEY", "CFG_ASSET_DESC_KEY", "CFG_ARCHIVE_KEY", "CFG_ARCHIVE_SIZE_KEY", "CFG_SEEK_KEYS_KEY", "CFG_ASSET_SIZE_KEY", "CFG_CHECKSUM_KEY", "CFG_ARCHIVE_CHECKSUM_KEY", "CFG_VERSION_KEY", "CFG_ASSET_PARENTS_KEY", "CFG_ASSET_CHILDREN_KEY", "CFG_TAG_DESC_KEY", "CFG_ASSET_CHECKSUM_KEY", "CFG_ASSET_TAGS_KEY"] @@ -113,7 +114,7 @@ fasta: hg38.fa.gz fai: hg38.fa.fai chrom_sizes: sizes.txt -""".format(folder=CFG_FOLDER_KEY, server=CFG_SERVER_KEY, version=CFG_VERSION_KEY, assets=CFG_ASSETS_KEY, +""".format(folder=CFG_FOLDER_KEY, server=CFG_SERVERS_KEY, version=CFG_VERSION_KEY, assets=CFG_ASSETS_KEY, archive=CFG_ARCHIVE_KEY, digest=CFG_CHECKSUM_KEY, genomes=CFG_GENOMES_KEY, desc_genome=CFG_GENOME_DESC_KEY, asset_path=CFG_ASSET_PATH_KEY, desc_asset=CFG_ASSET_DESC_KEY, archive_digest=CFG_ARCHIVE_CHECKSUM_KEY, asset_size=CFG_ASSET_SIZE_KEY, archive_size=CFG_ARCHIVE_SIZE_KEY, diff --git a/refgenconf/refgenconf.py b/refgenconf/refgenconf.py index 4930a697..215fd517 100755 --- a/refgenconf/refgenconf.py +++ b/refgenconf/refgenconf.py @@ -96,13 +96,23 @@ def __init__(self, filepath=None, entries=None, writable=False, wait_max=10): raise ConfigNotCompliantError(msg) else: _LOGGER.debug("Config version is compliant: {}".format(version)) + if CFG_SERVERS_KEY not in self and CFG_SERVER_KEY in self: + # backwards compatibility after server config key change + self[CFG_SERVERS_KEY] = self[CFG_SERVER_KEY] + del self[CFG_SERVER_KEY] + _LOGGER.debug("Moved servers list from '{}' to '{}'".format(CFG_SERVER_KEY, CFG_SERVERS_KEY)) try: - self[CFG_SERVER_KEY] = self[CFG_SERVER_KEY].rstrip("/") + if isinstance(self[CFG_SERVERS_KEY], list): + tmp_list = [server_url.rstrip("/") for server_url in self[CFG_SERVERS_KEY]] + self[CFG_SERVERS_KEY] = tmp_list + else: # Logic in pull_asset expects a list, even for a single server + self[CFG_SERVERS_KEY] = self[CFG_SERVERS_KEY].rstrip("/") + self[CFG_SERVERS_KEY] = [self[CFG_SERVERS_KEY]] except KeyError: raise MissingConfigDataError(CFG_SERVER_KEY) def __bool__(self): - minkeys = set(self.keys()) == {CFG_SERVER_KEY, CFG_FOLDER_KEY, CFG_GENOMES_KEY} + minkeys = set(self.keys()) == {CFG_SERVERS_KEY, CFG_FOLDER_KEY, CFG_GENOMES_KEY} return not minkeys or bool(self[CFG_GENOMES_KEY]) __nonzero__ = __bool__ @@ -322,8 +332,17 @@ def list_local(self, genome=None, order=None): names for sort :return str, str: text reps of locally available genomes and assets """ + exceptions = [] if genome is not None: - _assert_gat_exists(self[CFG_GENOMES_KEY], gname=genome) + if isinstance(genome, str): + genome = [genome] + for g in genome: + try: + _assert_gat_exists(self[CFG_GENOMES_KEY], g) + except MissingGenomeError as e: + exceptions.append(e) + if exceptions: + raise MissingGenomeError(", ".join(map(str, exceptions))) genomes_str = self.genomes_str(order=order) if genome is None \ else ", ".join(_select_genomes(sorted(self[CFG_GENOMES_KEY].keys(), key=order), genome)) return genomes_str, self.assets_str(genome=genome, order=order) @@ -339,7 +358,7 @@ def list_remote(self, genome=None, order=None, get_url=lambda server, id: constr names for sort :return str, str: text reps of remotely available genomes and assets """ - url = get_url(self.genome_server, API_ID_ASSETS) + url = get_url(self[CFG_SERVERS_KEY], API_ID_ASSETS) _LOGGER.info("Querying available assets from server: {}".format(url)) genomes, assets = _list_remote(url, genome, order) return genomes, assets @@ -458,104 +477,133 @@ def pull_asset(self, genome, asset, tag, unpack=True, force=None, :raise refgenconf.UnboundEnvironmentVariablesError: if genome folder path contains any env. var. that's unbound """ - missing_vars = unbound_env_vars(self.genome_folder) + missing_vars = unbound_env_vars(self[CFG_FOLDER_KEY]) if missing_vars: raise UnboundEnvironmentVariablesError(", ".join(missing_vars)) def raise_unpack_error(): raise NotImplementedError("Option to not extract tarballs is not yet supported.") - tag = _download_json(get_json_url(self.genome_server, API_ID_DEFAULT_TAG).format(genome=genome, asset=asset)) \ - if tag is None else tag - _LOGGER.debug("Determined tag: '{}'".format(tag)) - unpack or raise_unpack_error() - - url_attrs = get_json_url(self.genome_server, API_ID_ASSET_ATTRS).format(genome=genome, asset=asset) - url_archive = get_json_url(self.genome_server, API_ID_ARCHIVE).format(genome=genome, asset=asset) - archive_data = _download_json(url_attrs, params={"tag": tag}) - - if sys.version_info[0] == 2: - archive_data = asciify_json_dict(archive_data) - gat = [genome, asset, tag] - # local directory that the asset data will be stored in - tag_dir = os.path.dirname(self.filepath(*gat)) - # local directory the downloaded archive will be temporarily saved in - genome_dir_path = os.path.join(self[CFG_FOLDER_KEY], genome) - # local path to the temporarily saved archive - filepath = os.path.join(genome_dir_path, asset + "__" + tag + ".tgz") - # check if the genome/asset:tag exists and get request user decision - if os.path.exists(tag_dir): - def preserve(): - _LOGGER.debug("Preserving existing: {}".format(tag_dir)) - return gat, None - - def msg_overwrite(): - _LOGGER.debug("Overwriting: {}".format(tag_dir)) - shutil.rmtree(tag_dir) - if force is False: - return preserve() - elif force is None: - if not query_yes_no("Replace existing ({})?".format(tag_dir), "no"): + num_servers = 0 + bad_servers = [] + no_asset_json = [] + for server_url in self[CFG_SERVERS_KEY]: + num_servers += 1 + try: + determined_tag = _download_json(get_json_url(server_url, API_ID_DEFAULT_TAG).format(genome=genome, asset=asset)) \ + if tag is None else tag + except DownloadJsonError: + _LOGGER.warning("Could not retrieve JSON from: {}".format(server_url)) + bad_servers.append(server_url) + continue + else: + _LOGGER.debug("Determined tag: {}".format(determined_tag)) + unpack or raise_unpack_error() + gat = [genome, asset, determined_tag] + url_attrs = get_json_url(server_url, API_ID_ASSET_ATTRS).format(genome=genome, asset=asset) + url_archive = get_json_url(server_url, API_ID_ARCHIVE).format(genome=genome, asset=asset) + + try: + archive_data = _download_json(url_attrs, params={"tag": determined_tag}) + _LOGGER.debug("Determined server URL: {}".format(server_url)) + except DownloadJsonError: + no_asset_json.append(server_url) + if num_servers == len(self[CFG_SERVERS_KEY]): + _LOGGER.error("Asset '{}/{}:{}' not available on any of the following servers: {}". + format(genome, asset, determined_tag, ", ".join(no_asset_json))) + return gat, None, None + continue + + if sys.version_info[0] == 2: + archive_data = asciify_json_dict(archive_data) + + # local directory that the asset data will be stored in + tag_dir = os.path.dirname(self.filepath(*gat)) + # local directory the downloaded archive will be temporarily saved in + genome_dir_path = os.path.join(self[CFG_FOLDER_KEY], genome) + # local path to the temporarily saved archive + filepath = os.path.join(genome_dir_path, asset + "__" + determined_tag + ".tgz") + # check if the genome/asset:tag exists and get request user decision + if os.path.exists(tag_dir): + def preserve(): + _LOGGER.debug("Preserving existing: {}".format(tag_dir)) + return gat, None, None + + def msg_overwrite(): + _LOGGER.debug("Overwriting: {}".format(tag_dir)) + shutil.rmtree(tag_dir) + if force is False: return preserve() + elif force is None: + if not query_yes_no("Replace existing ({})?".format(tag_dir), "no"): + return preserve() + else: + msg_overwrite() else: msg_overwrite() - else: - msg_overwrite() - # check asset digests local-server match for each parent - [self._chk_digest_if_avail(genome, x) for x in archive_data[CFG_ASSET_PARENTS_KEY] if CFG_ASSET_PARENTS_KEY in archive_data] + # check asset digests local-server match for each parent + [self._chk_digest_if_avail(genome, x, server_url) + for x in archive_data[CFG_ASSET_PARENTS_KEY] if CFG_ASSET_PARENTS_KEY in archive_data] - bundle_name = '{}/{}:{}'.format(*gat) - archsize = archive_data[CFG_ARCHIVE_SIZE_KEY] - _LOGGER.debug("'{}' archive size: {}".format(bundle_name, archsize)) - if _is_large_archive(archsize) and not query_yes_no("Are you sure you want to download this large archive?"): - _LOGGER.info("pull action aborted by user") - return gat, None + bundle_name = '{}/{}:{}'.format(*gat) + archsize = archive_data[CFG_ARCHIVE_SIZE_KEY] + _LOGGER.debug("'{}' archive size: {}".format(bundle_name, archsize)) + if _is_large_archive(archsize) and not query_yes_no("Are you sure you want to download this large archive?"): + _LOGGER.info("pull action aborted by user") + return gat, None, None - if not os.path.exists(genome_dir_path): - _LOGGER.debug("Creating directory: {}".format(genome_dir_path)) - os.makedirs(genome_dir_path) + if not os.path.exists(genome_dir_path): + _LOGGER.debug("Creating directory: {}".format(genome_dir_path)) + os.makedirs(genome_dir_path) - # Download the file from `url` and save it locally under `filepath`: - _LOGGER.info("Downloading URL: {}".format(url_archive)) - try: - signal.signal(signal.SIGINT, build_signal_handler(filepath)) - _download_url_progress(url_archive, filepath, bundle_name, params={"tag": tag}) - except HTTPError as e: - _LOGGER.error("File not found on server: {}".format(e)) - return gat, None - except ConnectionRefusedError as e: - _LOGGER.error(str(e)) - _LOGGER.error("Server {}/{} refused download. Check your internet settings".format(self.genome_server, - API_VERSION)) - return gat, None - except ContentTooShortError as e: - _LOGGER.error(str(e)) - _LOGGER.error("'{}' download incomplete".format(bundle_name)) - return gat, None - else: - _LOGGER.info("Download complete: {}".format(filepath)) + # Download the file from `url` and save it locally under `filepath`: + _LOGGER.info("Downloading URL: {}".format(url_archive)) + try: + signal.signal(signal.SIGINT, build_signal_handler(filepath)) + _download_url_progress(url_archive, filepath, bundle_name, params={"tag": determined_tag}) + except HTTPError: + _LOGGER.error("Asset archive '{}/{}:{}' is missing on the server: {s}".format(*gat, s=server_url)) + if server_url == self[CFG_SERVERS_KEY][-1]: + # it this was the last server on the list, return + return gat, None, None + else: + _LOGGER.info("Trying next server") + # set the tag value back to what user requested + determined_tag = tag + continue + except ConnectionRefusedError as e: + _LOGGER.error(str(e)) + _LOGGER.error("Server {}/{} refused download. Check your internet settings".format(server_url, + API_VERSION)) + return gat, None, None + except ContentTooShortError as e: + _LOGGER.error(str(e)) + _LOGGER.error("'{}' download incomplete".format(bundle_name)) + return gat, None, None + else: + _LOGGER.info("Download complete: {}".format(filepath)) - new_checksum = checksum(filepath) - old_checksum = archive_data and archive_data.get(CFG_ARCHIVE_CHECKSUM_KEY) - if old_checksum and new_checksum != old_checksum: - _LOGGER.error("Checksum mismatch: ({}, {})".format(new_checksum, old_checksum)) - return gat, None - else: - _LOGGER.debug("Matched checksum: '{}'".format(old_checksum)) - import tempfile - # successfully downloaded and moved tarball; untar it - if unpack and filepath.endswith(".tgz"): - _LOGGER.info("Extracting asset tarball and saving to: {}".format(tag_dir)) - tmpdir = tempfile.mkdtemp(dir=genome_dir_path) # TODO: use context manager here when we drop support for py2 - untar(filepath, tmpdir) - # here we suspect the unarchived asset to be an asset-named directory with the asset data inside - # and we transfer it to the tag-named subdirectory - shutil.move(os.path.join(tmpdir, asset), tag_dir) - shutil.rmtree(tmpdir) - if os.path.isfile(filepath): - os.remove(filepath) - return gat, archive_data + new_checksum = checksum(filepath) + old_checksum = archive_data and archive_data.get(CFG_ARCHIVE_CHECKSUM_KEY) + if old_checksum and new_checksum != old_checksum: + _LOGGER.error("Checksum mismatch: ({}, {})".format(new_checksum, old_checksum)) + return gat, None, None + else: + _LOGGER.debug("Matched checksum: '{}'".format(old_checksum)) + import tempfile + # successfully downloaded and moved tarball; untar it + if unpack and filepath.endswith(".tgz"): + _LOGGER.info("Extracting asset tarball and saving to: {}".format(tag_dir)) + tmpdir = tempfile.mkdtemp(dir=genome_dir_path) # TODO: use context manager here when we drop support for py2 + untar(filepath, tmpdir) + # here we suspect the unarchived asset to be an asset-named directory with the asset data inside + # and we transfer it to the tag-named subdirectory + shutil.move(os.path.join(tmpdir, asset), tag_dir) + shutil.rmtree(tmpdir) + if os.path.isfile(filepath): + os.remove(filepath) + return gat, archive_data, server_url def update_relatives_assets(self, genome, asset, tag=None, data=None, children=False): """ @@ -762,7 +810,7 @@ def _invert_genomes(self, order=None): assets = sorted(genomes.keys(), key=order) return OrderedDict([(a, sorted(genomes[a], key=order)) for a in assets]) - def _chk_digest_if_avail(self, genome, remote_asset_name): + def _chk_digest_if_avail(self, genome, remote_asset_name, server_url): """ Check local asset digest against the remote one and populate children of the asset with the provided asset:tag. @@ -770,12 +818,13 @@ def _chk_digest_if_avail(self, genome, remote_asset_name): :param str genome: name of the genome to check the asset digests for :param str remote_asset_name: asset and tag names, formatted like: asset:tag + :param str server_url: addres of the server to query for the digests :raise RefgenconfError: if the local digest does not match its remote counterpart """ remote_asset_data = prp(remote_asset_name) asset = remote_asset_data["item"] tag = remote_asset_data["tag"] - asset_digest_url = construct_request_url(self.genome_server, API_ID_DIGEST).\ + asset_digest_url = construct_request_url(server_url, API_ID_DIGEST).\ format(genome=genome, asset=asset, tag=tag) try: remote_digest = _download_json(asset_digest_url) @@ -793,7 +842,7 @@ def _chk_digest_if_avail(self, genome, remote_asset_name): "with the server one after the pull".format(genome, asset, tag)) return - def chk_digest_update_child(self, genome, remote_asset_name, child_name): + def chk_digest_update_child(self, genome, remote_asset_name, child_name, server_url): """ Check local asset digest against the remote one and populate children of the asset with the provided asset:tag. @@ -802,12 +851,13 @@ def chk_digest_update_child(self, genome, remote_asset_name, child_name): :param str genome: name of the genome to check the asset digests for :param str remote_asset_name: asset and tag names, formatted like: asset:tag :param str child_name: name to be appended to the children of the parent + :param str server_url: addres of the server to query for the digests :raise RefgenconfError: if the local digest does not match its remote counterpart """ remote_asset_data = prp(remote_asset_name) asset = remote_asset_data["item"] tag = remote_asset_data["tag"] - asset_digest_url = construct_request_url(self.genome_server, API_ID_DIGEST).\ + asset_digest_url = construct_request_url(server_url, API_ID_DIGEST).\ format(genome=genome, asset=asset, tag=tag) try: remote_digest = _download_json(asset_digest_url) @@ -822,7 +872,7 @@ def chk_digest_update_child(self, genome, remote_asset_name, child_name): self.update_tags(genome, asset, tag, {CFG_ASSET_CHECKSUM_KEY: remote_digest}) _LOGGER.info("Could not find '{}/{}:{}' digest. Populating with server data".format(genome, asset, tag)) else: - local_digest = self[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset][CFG_ASSET_TAGS_KEY]\ + local_digest = self[CFG_GENOMES_KEY][genome][CFG_ASSETS_KEY][asset][CFG_ASSET_TAGS_KEY] \ [tag][CFG_ASSET_CHECKSUM_KEY] if remote_digest != local_digest: raise RemoteDigestMismatchError(asset, local_digest, remote_digest) @@ -860,6 +910,8 @@ def _download_json(url, params=None): resp = requests.get(url, params=params) if resp.ok: return resp.json() + elif resp.status_code == 404: + resp = None raise DownloadJsonError(resp) @@ -942,7 +994,7 @@ def _assert_gat_exists(genomes, gname, aname=None, tname=None, allow_incomplete= try: genome = genomes[gname] except KeyError: - raise MissingGenomeError("Your genomes do not include {}".format(gname)) + raise MissingGenomeError("Your genomes do not include '{}'".format(gname)) if aname is not None: try: asset_data = genome[CFG_ASSETS_KEY][aname] @@ -983,7 +1035,9 @@ def _list_remote(url, genome, order=None): :return str, str: text reps of remotely available genomes and assets """ genomes_data = _read_remote_data(url) - refgens = _select_genomes(sorted(genomes_data.keys(), key=order), genome) + refgens = _select_genomes(sorted(genomes_data.keys(), key=order), genome, strict=True) + if not refgens: + return None, None filtered_genomes_data = {refgen: genomes_data[refgen] for refgen in refgens} asset_texts = ["{}/ {}".format(g.rjust(20), ", ".join(a)) for g, a in filtered_genomes_data.items()] return ", ".join(refgens), "\n".join(asset_texts) @@ -1053,11 +1107,13 @@ def _check_insert_data(obj, datatype, name): return True -def _select_genomes(genomes, genome=None): +def _select_genomes(genomes, genome=None, strict=False): """ Safely select a subset of genomes :param list[str] | str genome: genomes that the assets should be found for + :param bool strict: whether a non-existent genome should lead to a warning. + Specific genome request is disregarded otherwise :raise TypeError: if genome argument type is not a list or str :return list: selected subset of genomes """ @@ -1066,7 +1122,20 @@ def _select_genomes(genomes, genome=None): genome = [genome] elif not isinstance(genome, list) or not all(isinstance(i, str) for i in genome): raise TypeError("genome has to be a list[str] or a str, got '{}'".format(genome.__class__.__name__)) - return genomes if (genome is None or not all(x in genomes for x in genome)) else genome + else: + return genomes + if strict: + missing = [] + filtered = [] + for g in genome: + if g in genomes: + filtered.append(g) + else: + missing.append(g) + if missing: + _LOGGER.warning("Genomes do not include: {}".format(", ".join(missing))) + return None if not filtered else filtered + return genomes if not all(x in genomes for x in genome) else genome def get_asset_tags(asset): diff --git a/tests/conftest.py b/tests/conftest.py index 2e3f8ca7..ea5601c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -120,7 +120,7 @@ def made_genome_config_file(temp_genome_config_file): """ Make the test session's genome config file. """ genome_folder = os.path.dirname(temp_genome_config_file) extra_kv_lines = ["{}: {}".format(CFG_FOLDER_KEY, genome_folder), - "{}: {}".format(CFG_SERVER_KEY, "http://staging.refgenomes.databio.org/"), + "{}: {}".format(CFG_SERVERS_KEY, "http://staging.refgenomes.databio.org/"), "{}: {}".format(CFG_VERSION_KEY, package_version), "{}:".format(CFG_GENOMES_KEY)] gen_data_lines = PathExAttMap(CONF_DATA).get_yaml_lines() diff --git a/tests/data/genomes.yaml b/tests/data/genomes.yaml index 9cf92145..5e6cb98c 100644 --- a/tests/data/genomes.yaml +++ b/tests/data/genomes.yaml @@ -1,6 +1,6 @@ config_version: 0.3 genome_folder: /tmp -genome_server: http://staging.refgenomes.databio.org +genome_servers: http://staging.refgenomes.databio.org genomes: rCRSd: assets: diff --git a/tests/test_1pull_asset.py b/tests/test_1pull_asset.py index 22e8f92e..469c5b72 100644 --- a/tests/test_1pull_asset.py +++ b/tests/test_1pull_asset.py @@ -28,7 +28,7 @@ def test_no_unpack(rgc, genome, asset, tag): rgc.pull_asset(genome, asset, tag, unpack=False) -@pytest.mark.parametrize(["gname", "aname"],[("human_repeats", 1), ("mouse_chrM2x", None)]) +@pytest.mark.parametrize(["gname", "aname"], [("human_repeats", 1), ("mouse_chrM2x", None)]) def test_pull_asset_illegal_asset_name(rgc, gname, aname): """ TypeError occurs if asset argument is not iterable. """ with pytest.raises(TypeError): @@ -40,7 +40,7 @@ def test_negative_response_to_large_download_prompt(rgc, gname, aname, tname): """ Test responsiveness to user abortion of pull request. """ with mock.patch("refgenconf.refgenconf._is_large_archive", return_value=True), \ mock.patch("refgenconf.refgenconf.query_yes_no", return_value=False): - gat, archive_dict = rgc.pull_asset(gname, aname, tname) + gat, archive_dict, server_url = rgc.pull_asset(gname, aname, tname) assert gat == [gname, aname, tname] @@ -50,6 +50,7 @@ def test_download_interruption(my_rgc, gname, aname, tname, caplog): """ Download interruption provides appropriate warning message and halts. """ import signal print("filepath: " + my_rgc._file_path) + def kill_download(*args, **kwargs): os.kill(os.getpid(), signal.SIGINT) diff --git a/tests/test_config_constructor.py b/tests/test_config_constructor.py index bdeeba8a..a73cf3c9 100644 --- a/tests/test_config_constructor.py +++ b/tests/test_config_constructor.py @@ -4,7 +4,7 @@ import pytest from attmap import PathExAttMap from refgenconf import RefGenConf, MissingConfigDataError, ConfigNotCompliantError -from refgenconf.const import CFG_FOLDER_KEY, CFG_GENOMES_KEY, CFG_SERVER_KEY, \ +from refgenconf.const import CFG_FOLDER_KEY, CFG_GENOMES_KEY, CFG_SERVERS_KEY, \ DEFAULT_SERVER __author__ = "Vince Reuter" @@ -37,16 +37,16 @@ def test_genome_folder_is_value_from_config_file_if_key_present(self, tmpdir_fac fout.write("{}: {}\n".format(CFG_FOLDER_KEY, expected)) else: fout.write(l) - if l.startswith(CFG_SERVER_KEY): + if l.startswith(CFG_SERVERS_KEY): found = True if not found: - fout.write("{}: {}".format(CFG_SERVER_KEY, DEFAULT_SERVER)) + fout.write("{}: {}".format(CFG_SERVERS_KEY, DEFAULT_SERVER)) rgc = RefGenConf(filepath=conf_file) assert expected != os.path.dirname(conf_file) assert expected == rgc[CFG_FOLDER_KEY] def test_empty_rgc_is_false(self): - assert bool(RefGenConf(entries={CFG_SERVER_KEY: DEFAULT_SERVER})) is False + assert bool(RefGenConf(entries={CFG_SERVERS_KEY: DEFAULT_SERVER})) is False def test_nonempty_rgc_is_true(self, rgc): assert bool(rgc) is True @@ -56,7 +56,7 @@ def test_illegal_genomes_mapping_type_gets_converted_to_empty_mapping(self, geno rgc = RefGenConf(entries={ CFG_FOLDER_KEY: tmpdir.strpath, CFG_GENOMES_KEY: genomes, - CFG_SERVER_KEY: DEFAULT_SERVER + CFG_SERVERS_KEY: DEFAULT_SERVER }) res = rgc[CFG_GENOMES_KEY] assert isinstance(res, PathExAttMap) diff --git a/tests/test_list_remote.py b/tests/test_list_remote.py index 2078ad2b..41be2e53 100644 --- a/tests/test_list_remote.py +++ b/tests/test_list_remote.py @@ -2,7 +2,7 @@ import mock from refgenconf import RefGenConf, CFG_FOLDER_KEY, CFG_GENOMES_KEY, \ - CFG_SERVER_KEY, DEFAULT_SERVER + CFG_SERVERS_KEY, DEFAULT_SERVER __author__ = "Vince Reuter" __email__ = "vreuter@virginia.edu" @@ -11,9 +11,9 @@ def test_list_remote(rgc, tmpdir): """ Verify expected behavior of remote genome/asset listing. """ new_rgc = RefGenConf(entries={CFG_FOLDER_KEY: tmpdir.strpath, - CFG_SERVER_KEY: DEFAULT_SERVER, + CFG_SERVERS_KEY: DEFAULT_SERVER, CFG_GENOMES_KEY: rgc[CFG_GENOMES_KEY]}) - new_rgc.genome_server = "http://staging.refgenomes.databio.org" + new_rgc[CFG_SERVERS_KEY] = "http://staging.refgenomes.databio.org" print("NEW RGC KEYS: {}".format(list(new_rgc.keys()))) with mock.patch("refgenconf.refgenconf._read_remote_data", return_value=rgc.genomes):