Merge pull request #68 from databio/dev

v0.5.3

docs/changelog.md

@@ -2,6 +2,13 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. 
 
+## [0.5.3] - 2019-10-29
+
+### Changed
+- `genome_server` config key to `genome_servers`
+- enable multiple refgenieservers for `pull_asset` method
+- `_chk_digest_if_avail` and `_chk_digest_update_child` require server URL argument to get the asset information from
+
 ## [0.5.2] - 2019-10-22
 
 ### Changed

refgenconf/_version.py

@@ -1 +1 @@
-__version__ = "0.5.2"
+__version__ = "0.5.3"

refgenconf/const.py

@@ -36,6 +36,7 @@
     "API_ID_RECIPE", "API_ID_LOG"]
 
 CFG_FOLDER_KEY = "genome_folder"
+CFG_SERVERS_KEY = "genome_servers"
 CFG_SERVER_KEY = "genome_server"
 CFG_ARCHIVE_KEY = "genome_archive"
 CFG_VERSION_KEY = "config_version"
@@ -59,15 +60,15 @@
 CFG_TAG_DESC_KEY = "tag_description"
 
 CFG_TOP_LEVEL_KEYS = [
-    CFG_FOLDER_KEY, CFG_SERVER_KEY, CFG_ARCHIVE_KEY, CFG_GENOMES_KEY, CFG_VERSION_KEY]
+    CFG_FOLDER_KEY, CFG_SERVER_KEY, CFG_SERVERS_KEY, CFG_ARCHIVE_KEY, CFG_GENOMES_KEY, CFG_VERSION_KEY]
 CFG_GENOME_KEYS = [
     CFG_GENOME_DESC_KEY, CFG_ASSETS_KEY, CFG_CHECKSUM_KEY]
 CFG_GENOME_ATTRS_KEYS = [CFG_GENOME_DESC_KEY, CFG_CHECKSUM_KEY]
 CFG_SINGLE_ASSET_SECTION_KEYS = [CFG_ASSET_PATH_KEY, CFG_ASSET_DESC_KEY, CFG_ASSET_SIZE_KEY, CFG_ARCHIVE_SIZE_KEY,
                                  CFG_ARCHIVE_CHECKSUM_KEY, CFG_SEEK_KEYS_KEY]
 
 CFG_KEY_NAMES = [
-    "CFG_FOLDER_KEY", "CFG_SERVER_KEY", "CFG_GENOMES_KEY",
+    "CFG_FOLDER_KEY", "CFG_SERVER_KEY", "CFG_SERVERS_KEY", "CFG_GENOMES_KEY",
     "CFG_ASSET_PATH_KEY", "CFG_ASSET_DESC_KEY", "CFG_ARCHIVE_KEY", "CFG_ARCHIVE_SIZE_KEY", "CFG_SEEK_KEYS_KEY",
     "CFG_ASSET_SIZE_KEY", "CFG_CHECKSUM_KEY", "CFG_ARCHIVE_CHECKSUM_KEY", "CFG_VERSION_KEY", "CFG_ASSET_PARENTS_KEY",
     "CFG_ASSET_CHILDREN_KEY", "CFG_TAG_DESC_KEY", "CFG_ASSET_CHECKSUM_KEY", "CFG_ASSET_TAGS_KEY"]
@@ -113,7 +114,7 @@
                             fasta: hg38.fa.gz
                             fai: hg38.fa.fai
                             chrom_sizes: sizes.txt
-""".format(folder=CFG_FOLDER_KEY, server=CFG_SERVER_KEY, version=CFG_VERSION_KEY, assets=CFG_ASSETS_KEY,
+""".format(folder=CFG_FOLDER_KEY, server=CFG_SERVERS_KEY, version=CFG_VERSION_KEY, assets=CFG_ASSETS_KEY,
            archive=CFG_ARCHIVE_KEY, digest=CFG_CHECKSUM_KEY, genomes=CFG_GENOMES_KEY,
            desc_genome=CFG_GENOME_DESC_KEY, asset_path=CFG_ASSET_PATH_KEY, desc_asset=CFG_ASSET_DESC_KEY,
            archive_digest=CFG_ARCHIVE_CHECKSUM_KEY, asset_size=CFG_ASSET_SIZE_KEY, archive_size=CFG_ARCHIVE_SIZE_KEY,

tests/conftest.py

@@ -120,7 +120,7 @@ def made_genome_config_file(temp_genome_config_file):
     """ Make the test session's genome config file. """
     genome_folder = os.path.dirname(temp_genome_config_file)
     extra_kv_lines = ["{}: {}".format(CFG_FOLDER_KEY, genome_folder),
-                      "{}: {}".format(CFG_SERVER_KEY, "http://staging.refgenomes.databio.org/"),
+                      "{}: {}".format(CFG_SERVERS_KEY, "http://staging.refgenomes.databio.org/"),
                       "{}: {}".format(CFG_VERSION_KEY, package_version),
                       "{}:".format(CFG_GENOMES_KEY)]
     gen_data_lines = PathExAttMap(CONF_DATA).get_yaml_lines()

tests/data/genomes.yaml

@@ -1,6 +1,6 @@
 config_version: 0.3
 genome_folder: /tmp
-genome_server: http://staging.refgenomes.databio.org
+genome_servers: http://staging.refgenomes.databio.org
 genomes:
   rCRSd:
     assets:

tests/test_1pull_asset.py

@@ -28,7 +28,7 @@ def test_no_unpack(rgc, genome, asset, tag):
         rgc.pull_asset(genome, asset, tag, unpack=False)
 
 
-@pytest.mark.parametrize(["gname", "aname"],[("human_repeats", 1), ("mouse_chrM2x", None)])
+@pytest.mark.parametrize(["gname", "aname"], [("human_repeats", 1), ("mouse_chrM2x", None)])
 def test_pull_asset_illegal_asset_name(rgc, gname, aname):
     """ TypeError occurs if asset argument is not iterable. """
     with pytest.raises(TypeError):
@@ -40,7 +40,7 @@ def test_negative_response_to_large_download_prompt(rgc, gname, aname, tname):
     """ Test responsiveness to user abortion of pull request. """
     with mock.patch("refgenconf.refgenconf._is_large_archive", return_value=True), \
          mock.patch("refgenconf.refgenconf.query_yes_no", return_value=False):
-        gat, archive_dict = rgc.pull_asset(gname, aname, tname)
+        gat, archive_dict, server_url = rgc.pull_asset(gname, aname, tname)
     assert gat == [gname, aname, tname]
 
 
@@ -50,6 +50,7 @@ def test_download_interruption(my_rgc, gname, aname, tname, caplog):
     """ Download interruption provides appropriate warning message and halts. """
     import signal
     print("filepath: " + my_rgc._file_path)
+
     def kill_download(*args, **kwargs):
         os.kill(os.getpid(), signal.SIGINT)
 

tests/test_config_constructor.py

@@ -4,7 +4,7 @@
 import pytest
 from attmap import PathExAttMap
 from refgenconf import RefGenConf, MissingConfigDataError, ConfigNotCompliantError
-from refgenconf.const import CFG_FOLDER_KEY, CFG_GENOMES_KEY, CFG_SERVER_KEY, \
+from refgenconf.const import CFG_FOLDER_KEY, CFG_GENOMES_KEY, CFG_SERVERS_KEY, \
     DEFAULT_SERVER
 
 __author__ = "Vince Reuter"
@@ -37,16 +37,16 @@ def test_genome_folder_is_value_from_config_file_if_key_present(self, tmpdir_fac
                     fout.write("{}: {}\n".format(CFG_FOLDER_KEY, expected))
                 else:
                     fout.write(l)
-                    if l.startswith(CFG_SERVER_KEY):
+                    if l.startswith(CFG_SERVERS_KEY):
                         found = True
             if not found:
-                fout.write("{}: {}".format(CFG_SERVER_KEY, DEFAULT_SERVER))
+                fout.write("{}: {}".format(CFG_SERVERS_KEY, DEFAULT_SERVER))
         rgc = RefGenConf(filepath=conf_file)
         assert expected != os.path.dirname(conf_file)
         assert expected == rgc[CFG_FOLDER_KEY]
 
     def test_empty_rgc_is_false(self):
-        assert bool(RefGenConf(entries={CFG_SERVER_KEY: DEFAULT_SERVER})) is False
+        assert bool(RefGenConf(entries={CFG_SERVERS_KEY: DEFAULT_SERVER})) is False
 
     def test_nonempty_rgc_is_true(self, rgc):
         assert bool(rgc) is True
@@ -56,7 +56,7 @@ def test_illegal_genomes_mapping_type_gets_converted_to_empty_mapping(self, geno
         rgc = RefGenConf(entries={
             CFG_FOLDER_KEY: tmpdir.strpath,
             CFG_GENOMES_KEY: genomes,
-            CFG_SERVER_KEY: DEFAULT_SERVER
+            CFG_SERVERS_KEY: DEFAULT_SERVER
         })
         res = rgc[CFG_GENOMES_KEY]
         assert isinstance(res, PathExAttMap)

tests/test_list_remote.py

@@ -2,7 +2,7 @@
 
 import mock
 from refgenconf import RefGenConf, CFG_FOLDER_KEY, CFG_GENOMES_KEY, \
-    CFG_SERVER_KEY, DEFAULT_SERVER
+    CFG_SERVERS_KEY, DEFAULT_SERVER
 
 __author__ = "Vince Reuter"
 __email__ = "vreuter@virginia.edu"
@@ -11,9 +11,9 @@
 def test_list_remote(rgc, tmpdir):
     """ Verify expected behavior of remote genome/asset listing. """
     new_rgc = RefGenConf(entries={CFG_FOLDER_KEY: tmpdir.strpath,
-                          CFG_SERVER_KEY: DEFAULT_SERVER,
+                          CFG_SERVERS_KEY: DEFAULT_SERVER,
                           CFG_GENOMES_KEY: rgc[CFG_GENOMES_KEY]})
-    new_rgc.genome_server = "http://staging.refgenomes.databio.org"
+    new_rgc[CFG_SERVERS_KEY] = "http://staging.refgenomes.databio.org"
     print("NEW RGC KEYS: {}".format(list(new_rgc.keys())))
     with mock.patch("refgenconf.refgenconf._read_remote_data",
                     return_value=rgc.genomes):