From 086f73a1ee9a29ee307414000be68a922dca1c59 Mon Sep 17 00:00:00 2001 From: b97pla Date: Wed, 8 Feb 2023 15:17:16 +0100 Subject: [PATCH 1/3] switched to pyproject.toml, update rq and dependencies --- .python-version | 2 +- .travis.yml | 9 +- Pipfile | 25 -- Pipfile.lock | 495 ------------------------------------- README.md | 61 +++-- archive-verify-ws | 11 - archive_verify/__init__.py | 1 - archive_verify/app.py | 9 +- pyproject.toml | 42 ++++ requirements.txt | 28 --- tests/mock_redis_client.py | 1 + tests/test_handlers.py | 7 +- tests/test_pdc_client.py | 2 +- tests/test_workers.py | 2 +- 14 files changed, 98 insertions(+), 597 deletions(-) delete mode 100644 Pipfile delete mode 100644 Pipfile.lock delete mode 100755 archive-verify-ws create mode 100644 pyproject.toml delete mode 100644 requirements.txt diff --git a/.python-version b/.python-version index 8b7b0b5..0a59033 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.6.12 +3.9.10 diff --git a/.travis.yml b/.travis.yml index e210a65..95b882e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,17 @@ language: python python: - - "3.6.12" + - "3.9.10" before_install: - - sudo python -m pip install pipenv + - python -m venv --upgrade-deps .venv install: - - pipenv install --dev + - source .venv/bin/activate + - pip install -r requirements/dev . script: - - pipenv run nosetests tests/ + - nosetests tests/ notifications: email: false diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 077d124..0000000 --- a/Pipfile +++ /dev/null @@ -1,25 +0,0 @@ -[[source]] - -url = "https://pypi.python.org/simple" -verify_ssl = true -name = "pypi" - -[requires] -python_version = "3.6" - -[packages] - -aiohttp = "==3.7.4" -rq = "==0.10.0" -redis = "==2.10.6" -aiodns = "==1.1.1" -cchardet = "==2.1.1" -pyyaml = "==5.4" -yarl = "==1.6.3" - - -[dev-packages] - -mock = "*" -nose = "*" -fakeredis = "*" diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 3837fca..0000000 --- a/Pipfile.lock +++ /dev/null @@ -1,495 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "3e017a5247eebcb52ab3472665f90f1a60d5d86be67f664d2d790ff3b49b45c4" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.6" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.python.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "aiodns": { - "hashes": [ - "sha256:99d0652f2c02f73bfa646bf44af82705260a523014576647d7959e664830b26b", - "sha256:d8677adc679ce8d0ef706c14d9c3d2f27a0e0cc11d59730cdbaf218ad52dd9ea" - ], - "index": "pypi", - "version": "==1.1.1" - }, - "aiohttp": { - "hashes": [ - "sha256:119feb2bd551e58d83d1b38bfa4cb921af8ddedec9fad7183132db334c3133e0", - "sha256:16d0683ef8a6d803207f02b899c928223eb219111bd52420ef3d7a8aa76227b6", - "sha256:2eb3efe243e0f4ecbb654b08444ae6ffab37ac0ef8f69d3a2ffb958905379daf", - "sha256:2ffea7904e70350da429568113ae422c88d2234ae776519549513c8f217f58a9", - "sha256:40bd1b101b71a18a528ffce812cc14ff77d4a2a1272dfb8b11b200967489ef3e", - "sha256:418597633b5cd9639e514b1d748f358832c08cd5d9ef0870026535bd5eaefdd0", - "sha256:481d4b96969fbfdcc3ff35eea5305d8565a8300410d3d269ccac69e7256b1329", - "sha256:4c1bdbfdd231a20eee3e56bd0ac1cd88c4ff41b64ab679ed65b75c9c74b6c5c2", - "sha256:5563ad7fde451b1986d42b9bb9140e2599ecf4f8e42241f6da0d3d624b776f40", - "sha256:58c62152c4c8731a3152e7e650b29ace18304d086cb5552d317a54ff2749d32a", - "sha256:5b50e0b9460100fe05d7472264d1975f21ac007b35dcd6fd50279b72925a27f4", - "sha256:5d84ecc73141d0a0d61ece0742bb7ff5751b0657dab8405f899d3ceb104cc7de", - "sha256:5dde6d24bacac480be03f4f864e9a67faac5032e28841b00533cd168ab39cad9", - "sha256:5e91e927003d1ed9283dee9abcb989334fc8e72cf89ebe94dc3e07e3ff0b11e9", - "sha256:62bc216eafac3204877241569209d9ba6226185aa6d561c19159f2e1cbb6abfb", - "sha256:6c8200abc9dc5f27203986100579fc19ccad7a832c07d2bc151ce4ff17190076", - "sha256:6ca56bdfaf825f4439e9e3673775e1032d8b6ea63b8953d3812c71bd6a8b81de", - "sha256:71680321a8a7176a58dfbc230789790639db78dad61a6e120b39f314f43f1907", - "sha256:7c7820099e8b3171e54e7eedc33e9450afe7cd08172632d32128bd527f8cb77d", - "sha256:7dbd087ff2f4046b9b37ba28ed73f15fd0bc9f4fdc8ef6781913da7f808d9536", - "sha256:822bd4fd21abaa7b28d65fc9871ecabaddc42767884a626317ef5b75c20e8a2d", - "sha256:8ec1a38074f68d66ccb467ed9a673a726bb397142c273f90d4ba954666e87d54", - "sha256:950b7ef08b2afdab2488ee2edaff92a03ca500a48f1e1aaa5900e73d6cf992bc", - "sha256:99c5a5bf7135607959441b7d720d96c8e5c46a1f96e9d6d4c9498be8d5f24212", - "sha256:b84ad94868e1e6a5e30d30ec419956042815dfaea1b1df1cef623e4564c374d9", - "sha256:bc3d14bf71a3fb94e5acf5bbf67331ab335467129af6416a437bd6024e4f743d", - "sha256:c2a80fd9a8d7e41b4e38ea9fe149deed0d6aaede255c497e66b8213274d6d61b", - "sha256:c44d3c82a933c6cbc21039326767e778eface44fca55c65719921c4b9661a3f7", - "sha256:cc31e906be1cc121ee201adbdf844522ea3349600dd0a40366611ca18cd40e81", - "sha256:d5d102e945ecca93bcd9801a7bb2fa703e37ad188a2f81b1e65e4abe4b51b00c", - "sha256:dd7936f2a6daa861143e376b3a1fb56e9b802f4980923594edd9ca5670974895", - "sha256:dee68ec462ff10c1d836c0ea2642116aba6151c6880b688e56b4c0246770f297", - "sha256:e76e78863a4eaec3aee5722d85d04dcbd9844bc6cd3bfa6aa880ff46ad16bfcb", - "sha256:eab51036cac2da8a50d7ff0ea30be47750547c9aa1aa2cf1a1b710a1827e7dbe", - "sha256:f4496d8d04da2e98cc9133e238ccebf6a13ef39a93da2e87146c8c8ac9768242", - "sha256:fbd3b5e18d34683decc00d9a360179ac1e7a320a5fee10ab8053ffd6deab76e0", - "sha256:feb24ff1226beeb056e247cf2e24bba5232519efb5645121c4aea5b6ad74c1f2" - ], - "index": "pypi", - "version": "==3.7.4" - }, - "async-timeout": { - "hashes": [ - "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", - "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" - ], - "markers": "python_version >= '3.5.3'", - "version": "==3.0.1" - }, - "attrs": { - "hashes": [ - "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1", - "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb" - ], - "markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.3.*' and python_version != '3.0.*' and python_version != '3.4.*'", - "version": "==21.2.0" - }, - "cchardet": { - "hashes": [ - "sha256:07dace80abce108d42a82be5a598797c0c07575741d81e698819bd42d367cdde", - "sha256:36d58862c158de32ace6497e7bafc7f85049b35a3abbd65118baffbe2a1ec1e5", - "sha256:3f70d1c41f0694d1411b47868fdb7c3147fd1bf09c22e6565a765eedfb888989", - "sha256:69311e20183056b45313475cc05c3e968faa2b14a466a6b0c23780645a462afe", - "sha256:6dfc76b71f66e002a99efa68efe4366143e8845b54cf5623eb05b5fa8fb030d6", - "sha256:6e001eb2ff93c4c31a9952cf01c71f5f95c758314032094df5cf086168678b23", - "sha256:7187a01130b838cea449904f3aa5c0bee0609fcc0f5f667f4ce08ea99d102ddc", - "sha256:823a981ba75fe8c12a0a0259eb80ec3a657273559f6d7445ba6fe2d2b061c8f9", - "sha256:9c9269208b9f8d7446dbd970f6544ce48104096efab0f769ee5918066ba1ee7e", - "sha256:a62b29c8c5a41f5ae95f620746d6db03b86fb259340fd991c9a608aabc60a275", - "sha256:b94a65d3a8cc900058e6aaedc0dde9c99ffe436d8670d156784d7b561b874cf5", - "sha256:d12b3f1913068975f9b9431f3cdc44488786523cc6d5467ffcb5bd43d3210157", - "sha256:d6c8eb90a9aa77f94e040a75d563f65849ab3b0c8f675b27928a91583648f8f8", - "sha256:e1c3addf0c7408f76b98bd5f55f3abe844716d47dd6ab0d32eea8caa11a8fa41", - "sha256:e32c4a420c6f7c6ea8d8a1fe36c60c70316a4ca1779dba2e00044b61d8ee2017", - "sha256:e47d90a8484cc425ca4c13a204901e24e2d0b3e206deef7cf391c10639d33d6b", - "sha256:eb8ee148e9fc13101e0e19ac98552d24b82731fcfddc915eed216c13ebbebec0", - "sha256:f4e3d0d9a0113cdfbc2fafa995674c1c49ed4166543b454945ca44d6e2148935", - "sha256:feda07443d732d86c9821671a898107b96ceb00462f405ec1dc08a353a9ddab0" - ], - "index": "pypi", - "version": "==2.1.1" - }, - "cffi": { - "hashes": [ - "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d", - "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771", - "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872", - "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c", - "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc", - "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762", - "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202", - "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5", - "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548", - "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a", - "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f", - "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20", - "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218", - "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c", - "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e", - "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56", - "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224", - "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a", - "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2", - "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a", - "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819", - "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346", - "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b", - "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e", - "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534", - "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb", - "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0", - "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156", - "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd", - "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87", - "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc", - "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195", - "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33", - "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f", - "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d", - "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd", - "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728", - "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7", - "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca", - "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99", - "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf", - "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e", - "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c", - "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5", - "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69" - ], - "markers": "python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.2.*'", - "version": "==1.14.6" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a", - "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6" - ], - "markers": "python_version >= '3.6'", - "version": "==8.0.1" - }, - "idna": { - "hashes": [ - "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", - "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" - ], - "markers": "python_version >= '3.5'", - "version": "==3.2" - }, - "idna-ssl": { - "hashes": [ - "sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c" - ], - "markers": "python_version >= '3.5'", - "version": "==1.1.0" - }, - "importlib-metadata": { - "hashes": [ - "sha256:b618b6d2d5ffa2f16add5697cf57a46c76a56229b0ed1c438322e4e95645bd15", - "sha256:f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1" - ], - "markers": "python_version < '3.8'", - "version": "==4.8.1" - }, - "multidict": { - "hashes": [ - "sha256:06560fbdcf22c9387100979e65b26fba0816c162b888cb65b845d3def7a54c9b", - "sha256:067150fad08e6f2dd91a650c7a49ba65085303fcc3decbd64a57dc13a2733031", - "sha256:0a2cbcfbea6dc776782a444db819c8b78afe4db597211298dd8b2222f73e9cd0", - "sha256:0dd1c93edb444b33ba2274b66f63def8a327d607c6c790772f448a53b6ea59ce", - "sha256:0fed465af2e0eb6357ba95795d003ac0bdb546305cc2366b1fc8f0ad67cc3fda", - "sha256:116347c63ba049c1ea56e157fa8aa6edaf5e92925c9b64f3da7769bdfa012858", - "sha256:1b4ac3ba7a97b35a5ccf34f41b5a8642a01d1e55454b699e5e8e7a99b5a3acf5", - "sha256:1c7976cd1c157fa7ba5456ae5d31ccdf1479680dc9b8d8aa28afabc370df42b8", - "sha256:246145bff76cc4b19310f0ad28bd0769b940c2a49fc601b86bfd150cbd72bb22", - "sha256:25cbd39a9029b409167aa0a20d8a17f502d43f2efebfe9e3ac019fe6796c59ac", - "sha256:28e6d883acd8674887d7edc896b91751dc2d8e87fbdca8359591a13872799e4e", - "sha256:2d1d55cdf706ddc62822d394d1df53573d32a7a07d4f099470d3cb9323b721b6", - "sha256:2e77282fd1d677c313ffcaddfec236bf23f273c4fba7cdf198108f5940ae10f5", - "sha256:32fdba7333eb2351fee2596b756d730d62b5827d5e1ab2f84e6cbb287cc67fe0", - "sha256:35591729668a303a02b06e8dba0eb8140c4a1bfd4c4b3209a436a02a5ac1de11", - "sha256:380b868f55f63d048a25931a1632818f90e4be71d2081c2338fcf656d299949a", - "sha256:3822c5894c72e3b35aae9909bef66ec83e44522faf767c0ad39e0e2de11d3b55", - "sha256:38ba256ee9b310da6a1a0f013ef4e422fca30a685bcbec86a969bd520504e341", - "sha256:3bc3b1621b979621cee9f7b09f024ec76ec03cc365e638126a056317470bde1b", - "sha256:3d2d7d1fff8e09d99354c04c3fd5b560fb04639fd45926b34e27cfdec678a704", - "sha256:517d75522b7b18a3385726b54a081afd425d4f41144a5399e5abd97ccafdf36b", - "sha256:5f79c19c6420962eb17c7e48878a03053b7ccd7b69f389d5831c0a4a7f1ac0a1", - "sha256:5f841c4f14331fd1e36cbf3336ed7be2cb2a8f110ce40ea253e5573387db7621", - "sha256:637c1896497ff19e1ee27c1c2c2ddaa9f2d134bbb5e0c52254361ea20486418d", - "sha256:6ee908c070020d682e9b42c8f621e8bb10c767d04416e2ebe44e37d0f44d9ad5", - "sha256:77f0fb7200cc7dedda7a60912f2059086e29ff67cefbc58d2506638c1a9132d7", - "sha256:7878b61c867fb2df7a95e44b316f88d5a3742390c99dfba6c557a21b30180cac", - "sha256:78c106b2b506b4d895ddc801ff509f941119394b89c9115580014127414e6c2d", - "sha256:8b911d74acdc1fe2941e59b4f1a278a330e9c34c6c8ca1ee21264c51ec9b67ef", - "sha256:93de39267c4c676c9ebb2057e98a8138bade0d806aad4d864322eee0803140a0", - "sha256:9416cf11bcd73c861267e88aea71e9fcc35302b3943e45e1dbb4317f91a4b34f", - "sha256:94b117e27efd8e08b4046c57461d5a114d26b40824995a2eb58372b94f9fca02", - "sha256:9815765f9dcda04921ba467957be543423e5ec6a1136135d84f2ae092c50d87b", - "sha256:98ec9aea6223adf46999f22e2c0ab6cf33f5914be604a404f658386a8f1fba37", - "sha256:a37e9a68349f6abe24130846e2f1d2e38f7ddab30b81b754e5a1fde32f782b23", - "sha256:a43616aec0f0d53c411582c451f5d3e1123a68cc7b3475d6f7d97a626f8ff90d", - "sha256:a4771d0d0ac9d9fe9e24e33bed482a13dfc1256d008d101485fe460359476065", - "sha256:a5635bcf1b75f0f6ef3c8a1ad07b500104a971e38d3683167b9454cb6465ac86", - "sha256:a9acb76d5f3dd9421874923da2ed1e76041cb51b9337fd7f507edde1d86535d6", - "sha256:ac42181292099d91217a82e3fa3ce0e0ddf3a74fd891b7c2b347a7f5aa0edded", - "sha256:b227345e4186809d31f22087d0265655114af7cda442ecaf72246275865bebe4", - "sha256:b61f85101ef08cbbc37846ac0e43f027f7844f3fade9b7f6dd087178caedeee7", - "sha256:b70913cbf2e14275013be98a06ef4b412329fe7b4f83d64eb70dce8269ed1e1a", - "sha256:b9aad49466b8d828b96b9e3630006234879c8d3e2b0a9d99219b3121bc5cdb17", - "sha256:baf1856fab8212bf35230c019cde7c641887e3fc08cadd39d32a421a30151ea3", - "sha256:bd6c9c50bf2ad3f0448edaa1a3b55b2e6866ef8feca5d8dbec10ec7c94371d21", - "sha256:c1ff762e2ee126e6f1258650ac641e2b8e1f3d927a925aafcfde943b77a36d24", - "sha256:c30ac9f562106cd9e8071c23949a067b10211917fdcb75b4718cf5775356a940", - "sha256:c9631c642e08b9fff1c6255487e62971d8b8e821808ddd013d8ac058087591ac", - "sha256:cdd68778f96216596218b4e8882944d24a634d984ee1a5a049b300377878fa7c", - "sha256:ce8cacda0b679ebc25624d5de66c705bc53dcc7c6f02a7fb0f3ca5e227d80422", - "sha256:cfde464ca4af42a629648c0b0d79b8f295cf5b695412451716531d6916461628", - "sha256:d3def943bfd5f1c47d51fd324df1e806d8da1f8e105cc7f1c76a1daf0f7e17b0", - "sha256:d9b668c065968c5979fe6b6fa6760bb6ab9aeb94b75b73c0a9c1acf6393ac3bf", - "sha256:da7d57ea65744d249427793c042094c4016789eb2562576fb831870f9c878d9e", - "sha256:dc3a866cf6c13d59a01878cd806f219340f3e82eed514485e094321f24900677", - "sha256:df23c83398715b26ab09574217ca21e14694917a0c857e356fd39e1c64f8283f", - "sha256:dfc924a7e946dd3c6360e50e8f750d51e3ef5395c95dc054bc9eab0f70df4f9c", - "sha256:e4a67f1080123de76e4e97a18d10350df6a7182e243312426d508712e99988d4", - "sha256:e5283c0a00f48e8cafcecadebfa0ed1dac8b39e295c7248c44c665c16dc1138b", - "sha256:e58a9b5cc96e014ddf93c2227cbdeca94b56a7eb77300205d6e4001805391747", - "sha256:e6453f3cbeb78440747096f239d282cc57a2997a16b5197c9bc839099e1633d0", - "sha256:e6c4fa1ec16e01e292315ba76eb1d012c025b99d22896bd14a66628b245e3e01", - "sha256:e7d81ce5744757d2f05fc41896e3b2ae0458464b14b5a2c1e87a6a9d69aefaa8", - "sha256:ea21d4d5104b4f840b91d9dc8cbc832aba9612121eaba503e54eaab1ad140eb9", - "sha256:ecc99bce8ee42dcad15848c7885197d26841cb24fa2ee6e89d23b8993c871c64", - "sha256:f0bb0973f42ffcb5e3537548e0767079420aefd94ba990b61cf7bb8d47f4916d", - "sha256:f19001e790013ed580abfde2a4465388950728861b52f0da73e8e8a9418533c0", - "sha256:f76440e480c3b2ca7f843ff8a48dc82446b86ed4930552d736c0bac507498a52", - "sha256:f9bef5cff994ca3026fcc90680e326d1a19df9841c5e3d224076407cc21471a1", - "sha256:fc66d4016f6e50ed36fb39cd287a3878ffcebfa90008535c62e0e90a7ab713ae", - "sha256:fd77c8f3cba815aa69cb97ee2b2ef385c7c12ada9c734b0f3b32e26bb88bbf1d" - ], - "markers": "python_version >= '3.6'", - "version": "==5.2.0" - }, - "pycares": { - "hashes": [ - "sha256:09b28fc7bc2cc05f7f69bf1636ddf46086e0a1837b62961e2092fcb40477320d", - "sha256:0aa97f900a7ffb259be77d640006585e2a907b0cd4edeee0e85cf16605995d5a", - "sha256:103353577a6266a53e71bfee4cf83825f1401fefa60f0fb8bdec35f13be6a5f2", - "sha256:1b959dd5921d207d759d421eece1b60416df33a7f862465739d5f2c363c2f523", - "sha256:26e67e4f81c80a5955dcf6193f3d9bee3c491fc0056299b383b84d792252fba4", - "sha256:38e54037f36c149146ff15f17a4a963fbdd0f9871d4a21cd94ff9f368140f57e", - "sha256:3c7fb8d34ee11971c39acfaf98d0fac66725385ccef3bfe1b174c92b210e1aa4", - "sha256:3d5e50c95849f6905d2a9dbf02ed03f82580173e3c5604a39e2ad054185631f1", - "sha256:44896d6e191a6b5a914dbe3aa7c748481bf6ad19a9df33c1e76f8f2dc33fc8f0", - "sha256:4876fc790ae32832ae270c4a010a1a77e12ddf8d8e6ad70ad0b0a9d506c985f7", - "sha256:53bc4f181b19576499b02cea4b45391e8dcbe30abd4cd01492f66bfc15615a13", - "sha256:57315b8eb8fdbc56b3ad4932bc4b17132bb7c7fd2bd590f7fb84b6b522098aa9", - "sha256:615406013cdcd1b445e5d1a551d276c6200b3abe77e534f8a7f7e1551208d14f", - "sha256:6580aef5d1b29a88c3d72fe73c691eacfd454f86e74d3fdd18f4bad8e8def98b", - "sha256:6f258c1b74c048a9501a25f732f11b401564005e5e3c18f1ca6cad0c3dc0fb19", - "sha256:7661d6bbd51a337e7373cb356efa8be9b4655fda484e068f9455e939aec8d54e", - "sha256:82b3259cb590ddd107a6d2dc52da2a2e9a986bf242e893d58c786af2f8191047", - "sha256:8ebb3ba0485f66cae8eed7ce3e9ed6f2c0bfd5e7319d5d0fbbb511064f17e1d4", - "sha256:a34b0e3e693dceb60b8a1169668d606c75cb100ceba0a2df53c234a0eb067fbc", - "sha256:ad6caf580ee69806fc6534be93ddbb6e99bf94296d79ab351c37b2992b17abfd", - "sha256:b17ef48729786e62b574c6431f675f4cb02b27691b49e7428a605a50cd59c072", - "sha256:c5362b7690ca481440f6b98395ac6df06aa50518ccb183c560464d1e5e2ab5d4", - "sha256:c95c964d5dd307e104b44b193095c67bb6b10c9eda1ffe7d44ab7a9e84c476d9", - "sha256:cd3011ffd5e1ad55880f7256791dbab9c43ebeda260474a968f19cd0319e1aef", - "sha256:d0154fc5753b088758fbec9bc137e1b24bb84fc0c6a09725c8bac25a342311cd", - "sha256:d4a5081e232c1d181883dcac4675807f3a6cf33911c4173fbea00c0523687ed4", - "sha256:d52f9c725d2a826d5ffa37681eb07ffb996bfe21788590ef257664a3898fc0b5", - "sha256:db5a533111a3cfd481e7e4fb2bf8bef69f4fa100339803e0504dd5aecafb96a5", - "sha256:dca9dc58845a9d083f302732a3130c68ded845ad5d463865d464e53c75a3dd45", - "sha256:e9773e07684a55f54657df05237267611a77b294ec3bacb5f851c4ffca38a465", - "sha256:eb60be66accc9a9ea1018b591a1f5800cba83491d07e9acc8c56bc6e6607ab54", - "sha256:f60c04c5561b1ddf85ca4e626943cc09d7fb684e1adb22abb632095415a40fd7", - "sha256:fdff88393c25016f417770d82678423fc7a56995abb2df3d2a1e55725db6977d" - ], - "markers": "python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.2.*'", - "version": "==4.0.0" - }, - "pycparser": { - "hashes": [ - "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", - "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" - ], - "markers": "python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.2.*'", - "version": "==2.20" - }, - "pyyaml": { - "hashes": [ - "sha256:02c78d77281d8f8d07a255e57abdbf43b02257f59f50cc6b636937d68efa5dd0", - "sha256:0dc9f2eb2e3c97640928dec63fd8dc1dd91e6b6ed236bd5ac00332b99b5c2ff9", - "sha256:124fd7c7bc1e95b1eafc60825f2daf67c73ce7b33f1194731240d24b0d1bf628", - "sha256:26fcb33776857f4072601502d93e1a619f166c9c00befb52826e7b774efaa9db", - "sha256:31ba07c54ef4a897758563e3a0fcc60077698df10180abe4b8165d9895c00ebf", - "sha256:3c49e39ac034fd64fd576d63bb4db53cda89b362768a67f07749d55f128ac18a", - "sha256:52bf0930903818e600ae6c2901f748bc4869c0c406056f679ab9614e5d21a166", - "sha256:5a3f345acff76cad4aa9cb171ee76c590f37394186325d53d1aa25318b0d4a09", - "sha256:5e7ac4e0e79a53451dc2814f6876c2fa6f71452de1498bbe29c0b54b69a986f4", - "sha256:7242790ab6c20316b8e7bb545be48d7ed36e26bbe279fd56f2c4a12510e60b4b", - "sha256:737bd70e454a284d456aa1fa71a0b429dd527bcbf52c5c33f7c8eee81ac16b89", - "sha256:8635d53223b1f561b081ff4adecb828fd484b8efffe542edcfdff471997f7c39", - "sha256:8b818b6c5a920cbe4203b5a6b14256f0e5244338244560da89b7b0f1313ea4b6", - "sha256:8bf38641b4713d77da19e91f8b5296b832e4db87338d6aeffe422d42f1ca896d", - "sha256:a36a48a51e5471513a5aea920cdad84cbd56d70a5057cca3499a637496ea379c", - "sha256:b2243dd033fd02c01212ad5c601dafb44fbb293065f430b0d3dbf03f3254d615", - "sha256:cc547d3ead3754712223abb7b403f0a184e4c3eae18c9bb7fd15adef1597cc4b", - "sha256:cc552b6434b90d9dbed6a4f13339625dc466fd82597119897e9489c953acbc22", - "sha256:f3790156c606299ff499ec44db422f66f05a7363b39eb9d5b064f17bd7d7c47b", - "sha256:f7a21e3d99aa3095ef0553e7ceba36fb693998fbb1226f1392ce33681047465f", - "sha256:fdc6b2cb4b19e431994f25a9160695cc59a4e861710cc6fc97161c5e845fc579" - ], - "index": "pypi", - "version": "==5.4" - }, - "redis": { - "hashes": [ - "sha256:8a1900a9f2a0a44ecf6e8b5eb3e967a9909dfed219ad66df094f27f7d6f330fb", - "sha256:a22ca993cea2962dbb588f9f30d0015ac4afcc45bee27d3978c0dbe9e97c6c0f" - ], - "index": "pypi", - "version": "==2.10.6" - }, - "rq": { - "hashes": [ - "sha256:31a5f04d1410111617ae78756b86fc6b0cf300fe7445843ea3758b86d9f67bc5", - "sha256:c1711bc43f298061166805763e6fa2353f03142e057e83f338d6e197a1be3157" - ], - "index": "pypi", - "version": "==0.10.0" - }, - "typing-extensions": { - "hashes": [ - "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e", - "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7", - "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34" - ], - "markers": "python_version < '3.8'", - "version": "==3.10.0.2" - }, - "yarl": { - "hashes": [ - "sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e", - "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434", - "sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366", - "sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3", - "sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec", - "sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959", - "sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e", - "sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c", - "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6", - "sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a", - "sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6", - "sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424", - "sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e", - "sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f", - "sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50", - "sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2", - "sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc", - "sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4", - "sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970", - "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10", - "sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0", - "sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406", - "sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896", - "sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643", - "sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721", - "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478", - "sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724", - "sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e", - "sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8", - "sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96", - "sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25", - "sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76", - "sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2", - "sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2", - "sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c", - "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a", - "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71" - ], - "index": "pypi", - "version": "==1.6.3" - }, - "zipp": { - "hashes": [ - "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832", - "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc" - ], - "markers": "python_version >= '3.6'", - "version": "==3.6.0" - } - }, - "develop": { - "fakeredis": { - "hashes": [ - "sha256:0d06a9384fb79da9f2164ce96e34eb9d4e2ea46215070805ea6fd3c174590b47", - "sha256:5eb1516f1fe1813e9da8f6c482178fc067af09f53de587ae03887ef5d9d13024" - ], - "index": "pypi", - "version": "==1.6.1" - }, - "mock": { - "hashes": [ - "sha256:122fcb64ee37cfad5b3f48d7a7d51875d7031aaf3d8be7c42e2bee25044eee62", - "sha256:7d3fbbde18228f4ff2f1f119a45cdffa458b4c0dee32eb4d2bb2f82554bac7bc" - ], - "index": "pypi", - "version": "==4.0.3" - }, - "nose": { - "hashes": [ - "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", - "sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a", - "sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98" - ], - "index": "pypi", - "version": "==1.3.7" - }, - "packaging": { - "hashes": [ - "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7", - "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14" - ], - "markers": "python_version != '3.1.*' and python_version >= '2.6' and python_version != '3.0.*' and python_version != '3.2.*'", - "version": "==21.0" - }, - "pyparsing": { - "hashes": [ - "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", - "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" - ], - "markers": "python_version != '3.1.*' and python_version >= '2.6' and python_version != '3.0.*' and python_version != '3.2.*'", - "version": "==2.4.7" - }, - "redis": { - "hashes": [ - "sha256:8a1900a9f2a0a44ecf6e8b5eb3e967a9909dfed219ad66df094f27f7d6f330fb", - "sha256:a22ca993cea2962dbb588f9f30d0015ac4afcc45bee27d3978c0dbe9e97c6c0f" - ], - "index": "pypi", - "version": "==2.10.6" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version != '3.1.*' and python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.0.*'", - "version": "==1.16.0" - }, - "sortedcontainers": { - "hashes": [ - "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", - "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" - ], - "version": "==2.4.0" - } - } -} diff --git a/README.md b/README.md index 5faf709..ddf73f9 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,63 @@ SNPSEQ Archive Verify ================== -A self contained (aiohttp) REST service that helps verify uploaded SNP&SEQ archives by first downloading the archive from PDC, and then compare the MD5 sums for all associated files. +A self contained (aiohttp) REST service that helps verify uploaded SNP&SEQ archives by first downloading the archive +PDC, and then compare the MD5 sums for all associated files. The downloaded files are deleted on successful +verification, and retained if any error occurs. -The web service enqueues certain job functions in the RQ/Redis queue, where they get picked up by the separate RQ worker process. +The service is composed of 3 components which must all be running and the system must be set up to allow the services +to communicate over the configured protocols and ports (refer to the redis documentation for details): -The downloaded files are deleted on successful verification, and retained if any error occurs. +- archive-verify-ws REST service +- Redis queue server +- RQ worker + +The web service enqueues certain job functions in the RQ/Redis queue, where they get picked up by the separate RQ +worker process. Pre-requisites -------------- -You will need python 3.6 and redis. +You will need python >=3.9 and redis. -Install redis with: +[Download and install redis](https://redis.io/docs/getting-started/installation/install-redis-on-linux/) - apt-get install redis-server -And start it with: +Install +------- +It is recommended to set up the service in a virtual environment. [venv](https://docs.python.org/3/library/venv.html) +is used below with bash on a Linux system. - redis-server + python3 -m venv --upgrade-deps .venv + source .venv/bin/activate + pip install . +Running the service +------------------- -Trying it out -------------- +Start the Redis server and RQ worker: - python3 -m pip install pipenv - pipenv install --deploy + redis-server + rq worker -Try running it: +Start the REST service - pipenv run ./archive-verify-ws -c=config/ - pipenv run rq worker + archive-verify-ws -c=config/ Mock Downloading ---------------- -If you are running this service locally and don't have IBM's dsmc client installed, you can skip the downloading step and verify an archive that is already on your machine. +If you are running this service locally and don't have IBM's dsmc client installed, you can skip the downloading step +and verify an archive that is already on your machine. To use this method: -- copy* an archive that has been pre-downloaded from PDC into the verify_root_dir set in app.yaml +- copy an archive that has been pre-downloaded from PDC into the verify_root_dir set in app.yaml - Delete or edit some files from the archive if you wish to trigger a validation error. - in app.yml, set: - pdc_client: "MockPdcClient" -\*Note that the archive will be deleted from verify_root_dir on successful verification +*Note that the archive will be deleted from verify_root_dir on successful verification* #### Naming Conventions for Mock Download #### Note that when an archive is downloaded from PDC using snpseq-archive-verify, the downloaded directory is formatted with the name of the archive plus the RQ job id, like so: @@ -58,8 +70,9 @@ When mocking downloading, we search verify_root_dir for archive_name and use the Running tests ------------- - pipenv install --dev - pipenv run nosetests tests/ + source .venv/bin/activate + pip install -e .[test] + nosetests tests/ REST endpoints -------------- @@ -68,8 +81,10 @@ Enqueue a verification job of a specific archive: curl -i -X "POST" -d '{"host": "my-host", "description": "my-descr", "archive": "my_001XBC_archive"}' http://localhost:8989/api/1.0/verify -Check the current status of an enqueued job: +Enqueue a download job of a specific archive: - curl -i -X "GET" http://localhost:8989/api/1.0/status/ + curl -i -X "POST" -d '{"host": "my-host", "description": "my-descr", "archive": "my_001XBC_archive"}' http://localhost:8989/api/1.0/download +Check the current status of an enqueued job: + curl -i -X "GET" http://localhost:8989/api/1.0/status/ diff --git a/archive-verify-ws b/archive-verify-ws deleted file mode 100755 index 27ab478..0000000 --- a/archive-verify-ws +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -# -*- coding: utf-8 -*- -import re -import sys - -from archive_verify.app import start - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(start()) diff --git a/archive_verify/__init__.py b/archive_verify/__init__.py index 68cdeee..e69de29 100644 --- a/archive_verify/__init__.py +++ b/archive_verify/__init__.py @@ -1 +0,0 @@ -__version__ = "1.0.5" diff --git a/archive_verify/app.py b/archive_verify/app.py index 54e4492..68c6a38 100644 --- a/archive_verify/app.py +++ b/archive_verify/app.py @@ -11,10 +11,12 @@ log = logging.getLogger(__name__) + def setup_routes(app): app.router.add_post(app["config"]["base_url"] + "/verify", handlers.verify) app.router.add_get(app["config"]["base_url"] + "/status/{job_id}", handlers.status) + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("-c", "--configroot", help="Path to config root dir", type=str, default="/etc/arteria/archive-verify/") @@ -27,25 +29,28 @@ def parse_args(): return args + def load_config(args): config_file = os.path.join(args.configroot, "app.yaml") logger_file = os.path.join(args.configroot, "logger.yaml") try: with open(logger_file) as logger: - logger_conf = yaml.load(logger) + logger_conf = yaml.safe_load(logger) logging.config.dictConfig(logger_conf) with open(config_file) as config: - return yaml.load(config) + return yaml.safe_load(config) except Exception as e: log.error("Could not parse config file {}".format(e)) sys.exit(1) + def init_config(): args = parse_args() return load_config(args) + def start(): conf = init_config() log.info("Starting archive-verify-ws on {}...".format(conf["port"])) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c30b1fe --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["archive_verify"] + +[project] +name = "archive-verify" +version = "1.1.0" +authors = [ + {name = "SNP&SEQ Technology Platform, Uppsala University", email = "seq@medsci.uu.se" }, +] +keywords = ["bioinformatics"] +description = "Micro-service for downloading and verifying archives with DSMC" +readme = "README.md" +license = {file = "LICENSE"} +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: POSIX :: Linux", + "Topic :: Scientific/Engineering :: Bio-Informatics", + "Topic :: System :: Archiving" +] +dependencies = [ + "aiohttp", + "pyyaml", + "redis", + "rq" +] + +[project.optional-dependencies] +test = [ + "nose", + "fakeredis"] + +[project.scripts] +archive-verify-ws = "archive_verify.app:start" + +[project.urls] +homepage = "https://github.com/Molmed/snpseq-archive-verify" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index eb1612e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,28 +0,0 @@ -aiodns==1.1.1 -aiohttp==3.7.4 -async-timeout==3.0.1 -attrs==21.2.0 -cchardet==2.1.1 -cffi==1.14.6 -chardet==3.0.4 -click==8.0.1 -fakeredis==1.6.1 -idna==3.2 -idna-ssl==1.1.0 -importlib-metadata==4.8.1 -mock==4.0.3 -multidict==5.2.0 -nose==1.3.7 -packaging==21.0 -pbr==5.1.1 -pycares==4.0.0 -pycparser==2.20 -pyparsing==2.4.7 -PyYAML==5.4 -redis==2.10.6 -rq==0.10.0 -six==1.16.0 -sortedcontainers==2.4.0 -typing-extensions==3.10.0.2 -yarl==1.6.3 -zipp==3.6.0 diff --git a/tests/mock_redis_client.py b/tests/mock_redis_client.py index 50b3bbb..601c79a 100644 --- a/tests/mock_redis_client.py +++ b/tests/mock_redis_client.py @@ -1,4 +1,5 @@ import fakeredis + def get_redis_instance(): return fakeredis.FakeStrictRedis() diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 432e874..9a19625 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -1,6 +1,6 @@ import yaml -from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop +from aiohttp.test_utils import AioHTTPTestCase from aiohttp import web import archive_verify.app as app_setup @@ -23,14 +23,12 @@ async def get_application(self): app_setup.setup_routes(app) return app - @unittest_run_loop - async def test_root(self): + async def test_root(self): request = await self.client.request("GET", "/") assert request.status == 404 text = await request.text() assert "not found" in text.lower() - @unittest_run_loop async def test_basic_verify(self): url = self.BASE_URL + "/verify" payload = {"host": "testbox", "archive": "test_archive", "description": "test-description"} @@ -40,7 +38,6 @@ async def test_basic_verify(self): assert resp["status"] == "pending" assert resp["job_id"] != "" - @unittest_run_loop async def test_basic_status_wrong_id(self): url = self.BASE_URL + "/status/foobar" request = await self.client.request("GET", url) diff --git a/tests/test_pdc_client.py b/tests/test_pdc_client.py index ba14137..31ec3bf 100644 --- a/tests/test_pdc_client.py +++ b/tests/test_pdc_client.py @@ -1,5 +1,5 @@ -import mock import unittest +import unittest.mock as mock import yaml from archive_verify.pdc_client import PdcClient diff --git a/tests/test_workers.py b/tests/test_workers.py index a739948..7f7961b 100644 --- a/tests/test_workers.py +++ b/tests/test_workers.py @@ -1,6 +1,6 @@ import copy -import mock import unittest +import unittest.mock as mock import yaml from archive_verify.workers import compare_md5sum, pdc_client_factory, verify_archive From 52fc02c497d95841eb17e6e6ef5881ca942ca942 Mon Sep 17 00:00:00 2001 From: b97pla Date: Thu, 16 Feb 2023 11:40:11 +0100 Subject: [PATCH 2/3] fix typo in readme --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ddf73f9..c64e2e0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ SNPSEQ Archive Verify ================== A self contained (aiohttp) REST service that helps verify uploaded SNP&SEQ archives by first downloading the archive -PDC, and then compare the MD5 sums for all associated files. The downloaded files are deleted on successful +from PDC, and then compare the MD5 sums for all associated files. The downloaded files are deleted on successful verification, and retained if any error occurs. The service is composed of 3 components which must all be running and the system must be set up to allow the services @@ -60,11 +60,13 @@ To use this method: *Note that the archive will be deleted from verify_root_dir on successful verification* #### Naming Conventions for Mock Download #### -Note that when an archive is downloaded from PDC using snpseq-archive-verify, the downloaded directory is formatted with the name of the archive plus the RQ job id, like so: +Note that when an archive is downloaded from PDC using snpseq-archive-verify, the downloaded directory is formatted +with the name of the archive plus the RQ job id, like so: {verify_root_dir}/{archive_name}_{rq_job_id} -When mocking downloading, we search verify_root_dir for archive_name and use the first directory found, ignoring the rq_job_id. +When mocking downloading, we search verify_root_dir for archive_name and use the first directory found, ignoring the +rq_job_id. Running tests From 725aa7656afbad13d3ef80f8cdd19abffb071fd7 Mon Sep 17 00:00:00 2001 From: b97pla Date: Thu, 16 Feb 2023 16:54:29 +0100 Subject: [PATCH 3/3] correct python version in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c30b1fe..402fd6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ keywords = ["bioinformatics"] description = "Micro-service for downloading and verifying archives with DSMC" readme = "README.md" license = {file = "LICENSE"} -requires-python = ">=3.7" +requires-python = ">=3.9" classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License",