From 0c1251acc2310f4e9bd14d75b991b3e58b88212f Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Tue, 24 Aug 2021 14:22:41 +0800 Subject: [PATCH 01/48] Support deploying artifact by checksum --- README.md | 25 ++++++ artifactory.py | 49 ++++++++++++ .../test_integration_artifactory_path.py | 77 +++++++++++++++++++ 3 files changed, 151 insertions(+) diff --git a/README.md b/README.md index 7bda380e..0807e933 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,31 @@ path.deploy_file( ) ``` +[Deploy artifact by checksum](https://www.jfrog.com/confluence/display/RTF6X/Artifactory+REST+API#ArtifactoryRESTAPI-DeployArtifactbyChecksum): deploy an artifact to the specified destination by checking if the artifact +content already exists in Artifactory. If Artifactory already contains a user +readable artifact with the same checksum the artifact content is copied over to +the new location without requiring content transfer. + +```python +from artifactory import ArtifactoryPath +path = ArtifactoryPath("http://my-artifactory/artifactory/my_repo/foo") +sha1="1be5d2dbe52ddee96ef2d17d354e2be0a155a951" +sha256="00bbf80ccca376893d60183e1a714e707fd929aea3e458f9ffda60f7ae75cc51" + +# Each of the following 4 methods works fine if the artifact content already +# exists in Artifactory. +path.deploy_by_checksum(sha1=sha1) + +# deploy by sha1 via checksum parameter +path.deploy_by_checksum(checksum=sha1) + +# deploy by sha256 via sha256 parameter +path.deploy_by_checksum(sha256=sha256) + +# deploy by sha256 via checksum parameter +path.deploy_by_checksum(checksum=sha256) +``` + Deploy a debian package ```myapp-1.0.deb``` to an ```existent``` folder ```python diff --git a/artifactory.py b/artifactory.py index 95e037ae..71079185 100755 --- a/artifactory.py +++ b/artifactory.py @@ -1030,11 +1030,30 @@ def deploy( parameters=None, explode_archive=None, explode_archive_atomic=None, + checksum=None, + by_checksum=None, ): """ Uploads a given file-like object HTTP chunked encoding will be attempted + + If by_checksum is True, fobj should be None + + :param pathobj: ArtifactoryPath object + :param fobj: file object to be deployed + :param md5: + :param sha1: + :param sha256: + :param parameters: Artifact properties + :param explode_archive(bool): True: archive will be exploded upon deployment + :param explode_archive_atomic(bool): True: archive will be exploded in an atomic operation upon deployment + :param checksum: sha1Value or sha256Value + :param by_checksum(bool): True: deploy artifact by checksum """ + + if fobj and by_checksum: + raise RuntimeError("Either fobj or by_checksum, but not both") + if isinstance(fobj, urllib3.response.HTTPResponse): fobj = HTTPResponseWrapper(fobj) @@ -1055,6 +1074,10 @@ def deploy( headers["X-Explode-Archive"] = "true" if explode_archive_atomic: headers["X-Explode-Archive-Atomic"] = "true" + if by_checksum: + headers["X-Checksum-Deploy"] = "true" + if checksum: + headers["X-Checksum"] = checksum text, code = self.rest_put_stream( url, @@ -1764,6 +1787,32 @@ def deploy_file( explode_archive_atomic=explode_archive_atomic, ) + def deploy_by_checksum( + self, + sha1=None, + sha256=None, + checksum=None, + parameters={}, + ): + """ + Deploy an artifact to the specified destination by checking if the + artifact content already exists in Artifactory. + + :param pathobj: ArtifactoryPath object + :param sha1: sha1Value + :param sha256: sha256Value + :param checksum: sha1Value or sha256Value + """ + return self._accessor.deploy( + self, + fobj=None, + sha1=sha1, + sha256=sha256, + checksum=checksum, + by_checksum=True, + parameters=parameters, + ) + def deploy_deb( self, file_name, distribution, component, architecture, parameters={} ): diff --git a/tests/integration/test_integration_artifactory_path.py b/tests/integration/test_integration_artifactory_path.py index 89ee9aec..146ca366 100644 --- a/tests/integration/test_integration_artifactory_path.py +++ b/tests/integration/test_integration_artifactory_path.py @@ -6,6 +6,8 @@ import pytest import artifactory +from artifactory import sha1sum +from artifactory import sha256sum if sys.version_info[0] < 3: import StringIO as io @@ -173,6 +175,81 @@ def test_deploy_file(path): p.unlink() +def test_deploy_file_by_checksum(path): + p = path("/integration-artifactory-path-repo/foo") + p1 = path("/integration-artifactory-path-repo/bar") + + if p.exists(): + p.unlink() + if p1.exists(): + p1.unlink() + + tf = tempfile.NamedTemporaryFile() + tf.write(b"Some test string") + tf.flush() + + sha1 = sha1sum(tf.name) + sha256 = sha256sum(tf.name) + + sha1_non_existent = "1111111111111111111111111111111111111111" + sha256_non_existent = ( + "1111111111111111111111111111111111111111111111111111111111111111" + ) + + p.deploy_file(tf.name) + tf.close() + with p.open() as fd: + result = fd.read() + assert result == b"Some test string" + + # The matrix is a full list of all possible combinations + # 1st row is for 1st parameter of deploy_by_checksum + # 2nd row is for 2nd parameter and 3rd is for 3rd parameter + # matrix = [ + # [sha1, sha1_non_existent, None], + # [sha256, sha256_non_existent, None], + # [sha1, sha1_non_existent, sha256, sha256_non_existent, None] + # ] + + p1.deploy_by_checksum(sha1=sha1) + with p1.open() as fd: + result = fd.read() + p1.unlink() + assert result == b"Some test string" + + with pytest.raises(RuntimeError) as excinfo: + p.deploy_by_checksum(sha1=sha1_non_existent) + + p1.deploy_by_checksum(sha256=sha256) + with p1.open() as fd: + result = fd.read() + p1.unlink() + assert result == b"Some test string" + + with pytest.raises(RuntimeError) as excinfo: + p.deploy_by_checksum(sha256=sha256_non_existent) + + p1.deploy_by_checksum(checksum=sha1) + with p1.open() as fd: + result = fd.read() + p1.unlink() + assert result == b"Some test string" + + with pytest.raises(RuntimeError) as excinfo: + p.deploy_by_checksum(checksum=sha1_non_existent) + + p1.deploy_by_checksum(checksum=sha256) + with p1.open() as fd: + result = fd.read() + p1.unlink() + assert result == b"Some test string" + + with pytest.raises(RuntimeError) as excinfo: + p.deploy_by_checksum(checksum=sha256_non_existent) + + p.unlink() + + def test_open(path): p = path("/integration-artifactory-path-repo/foo") From ad69506700d258eaa9380508ced9c7fd38bc259b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Sep 2021 14:16:25 +0000 Subject: [PATCH 02/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0807e933..6e64aa9e 100644 --- a/README.md +++ b/README.md @@ -305,9 +305,10 @@ the new location without requiring content transfer. ```python from artifactory import ArtifactoryPath + path = ArtifactoryPath("http://my-artifactory/artifactory/my_repo/foo") -sha1="1be5d2dbe52ddee96ef2d17d354e2be0a155a951" -sha256="00bbf80ccca376893d60183e1a714e707fd929aea3e458f9ffda60f7ae75cc51" +sha1 = "1be5d2dbe52ddee96ef2d17d354e2be0a155a951" +sha256 = "00bbf80ccca376893d60183e1a714e707fd929aea3e458f9ffda60f7ae75cc51" # Each of the following 4 methods works fine if the artifact content already # exists in Artifactory. From 323159f36ed4674b5a6e63c711b6e093faf08c83 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Wed, 8 Sep 2021 13:07:08 +0800 Subject: [PATCH 03/48] Update readme --- README.md | 5 +++++ artifactory.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e64aa9e..40fd87c5 100644 --- a/README.md +++ b/README.md @@ -310,6 +310,11 @@ path = ArtifactoryPath("http://my-artifactory/artifactory/my_repo/foo") sha1 = "1be5d2dbe52ddee96ef2d17d354e2be0a155a951" sha256 = "00bbf80ccca376893d60183e1a714e707fd929aea3e458f9ffda60f7ae75cc51" +# If you don't know sha value, you can calculate it via +# sha1 = artifactory.sha1sum("local_path_of_your_file") +# or +# sha256 = artifactory.sha256sum("local_path_of_your_file") + # Each of the following 4 methods works fine if the artifact content already # exists in Artifactory. path.deploy_by_checksum(sha1=sha1) diff --git a/artifactory.py b/artifactory.py index 71079185..8d741128 100755 --- a/artifactory.py +++ b/artifactory.py @@ -1031,7 +1031,7 @@ def deploy( explode_archive=None, explode_archive_atomic=None, checksum=None, - by_checksum=None, + by_checksum=False, ): """ Uploads a given file-like object @@ -1048,7 +1048,7 @@ def deploy( :param explode_archive(bool): True: archive will be exploded upon deployment :param explode_archive_atomic(bool): True: archive will be exploded in an atomic operation upon deployment :param checksum: sha1Value or sha256Value - :param by_checksum(bool): True: deploy artifact by checksum + :param by_checksum(bool): if True, deploy artifact by checksum, default False """ if fobj and by_checksum: From 2b7926c1b4676f1d05f76380b0e77fc3defe41e0 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Wed, 8 Sep 2021 15:13:57 +0800 Subject: [PATCH 04/48] Add unit test --- tests/unit/test_artifactory_path.py | 149 ++++++++++++++++++++++++++-- 1 file changed, 140 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_artifactory_path.py b/tests/unit/test_artifactory_path.py index 9b65c5a3..9c365eb9 100644 --- a/tests/unit/test_artifactory_path.py +++ b/tests/unit/test_artifactory_path.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import json import os import pathlib import tempfile @@ -395,30 +396,43 @@ def test_join_with_multiple_folder_and_artifactory_substr_in_it(self): class ClassSetup(unittest.TestCase): def setUp(self): - self.file_stat = { + self.artifact_url = "http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz" + self.path = ArtifactoryPath(self.artifact_url) + self.sha1 = "fc6c9e8ba6eaca4fa97868ac900570282133c095" + self.sha256 = "fc6c9e8ba6eaca4fa97868ac900570282133c095fc6c9e8ba6eaca4fa97868ac900570282133c095" + # Response for deploying artifact by checksum + self.file_stat_without_modification_date = { "repo": "ext-release-local", "path": "/org/company/tool/1.0/tool-1.0.tar.gz", "created": "2014-02-24T21:20:59.999+04:00", "createdBy": "someuser", - "lastModified": "2014-02-24T21:20:36.000+04:00", - "modifiedBy": "anotheruser", - "lastUpdated": "2014-02-24T21:20:36.000+04:00", "downloadUri": "http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", "mimeType": "application/octet-stream", "size": "26776462", "checksums": { - "sha1": "fc6c9e8ba6eaca4fa97868ac900570282133c095", - "sha256": "fc6c9e8ba6eaca4fa97868ac900570282133c095fc6c9e8ba6eaca4fa97868ac900570282133c095", + "sha1": self.sha1, + "sha256": self.sha256, "md5": "2af7d54a09e9c36d704cb3a2de28aff3", }, "originalChecksums": { - "sha1": "fc6c9e8ba6eaca4fa97868ac900570282133c095", - "sha256": "fc6c9e8ba6eaca4fa97868ac900570282133c095fc6c9e8ba6eaca4fa97868ac900570282133c095", + "sha1": self.sha1, + "sha256": self.sha256, "md5": "2af7d54a09e9c36d704cb3a2de28aff3", }, - "uri": "http://artifactory.local/artifactory/api/storage/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", + "uri": "http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", } + # Response for file info api + self.file_stat = self.file_stat_without_modification_date.copy() + self.file_stat.update( + { + "lastModified": "2014-02-24T21:20:36.000+04:00", + "modifiedBy": "anotheruser", + "lastUpdated": "2014-02-24T21:20:36.000+04:00", + "uri": "http://artifactory.local/artifactory/api/storage/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", + } + ) + self.dir_stat = { "repo": "libs-release-local", "path": "/", @@ -441,6 +455,15 @@ def setUp(self): "uri" : "http://artifactory.local/artifactory/api/storage/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz" }""" + self.deploy_by_checksum_error = { + "errors": [ + { + "status": 400, + "message": "Checksum values not provided" + } + ] + } + class ArtifactoryAccessorTest(ClassSetup): """Test the real artifactory integration""" @@ -896,6 +919,114 @@ def test_deploy_file(self): self.assertIn("X-Explode-Archive-Atomic", headers) self.assertEqual(headers["X-Explode-Archive-Atomic"], "true") + def test_deploy_by_checksum_sha1(self): + """ + Test that file is deployed by sha1 + :return: + """ + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.file_stat_without_modification_date, + status=200 + ) + self.path.deploy_by_checksum(sha1=self.sha1) + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum-Sha1"], self.sha1) + self.assertNotIn("X-Checksum-Sha256", headers) + self.assertNotIn("X-Checksum", headers) + + def test_deploy_by_checksum_sha256(self): + """ + Test that file is deployed by sha256 + :return: + """ + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.file_stat_without_modification_date, + status=200 + ) + self.path.deploy_by_checksum(sha256=self.sha256) + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum-Sha256"], self.sha256) + self.assertNotIn("X-Checksum-Sha1", headers) + self.assertNotIn("X-Checksum", headers) + + def test_deploy_by_checksum_sha1_or_sha256(self): + """ + Test that file is deployed by sha1 or sha256 + :return: + """ + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.file_stat_without_modification_date, + status=200 + ) + self.path.deploy_by_checksum(checksum=self.sha1) + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum"], self.sha1) + self.assertNotIn("X-Checksum-Sha1", headers) + self.assertNotIn("X-Checksum-Sha256", headers) + + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.file_stat_without_modification_date, + status=200 + ) + self.path.deploy_by_checksum(checksum=self.sha256) + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum"], self.sha256) + self.assertNotIn("X-Checksum-Sha1", headers) + self.assertNotIn("X-Checksum-Sha256", headers) + + def test_deploy_by_checksum_error(self): + """ + Test that file is deployed by checksum, which raises error + :return: + """ + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.deploy_by_checksum_error, + status=400 + ) + with self.assertRaises(RuntimeError) as context: + self.path.deploy_by_checksum(sha1=f"{self.sha1}invalid") + + self.assertEqual(str(context.exception), json.dumps(self.deploy_by_checksum_error)) + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum-Sha1"], f"{self.sha1}invalid") + self.assertNotIn("X-Checksum-Sha256", headers) + self.assertNotIn("X-Checksum", headers) + @responses.activate def test_deploy_deb(self): """ From b7a55392594855ff03fc82035119ad2080f80f55 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 Sep 2021 07:14:16 +0000 Subject: [PATCH 05/48] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_artifactory_path.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_artifactory_path.py b/tests/unit/test_artifactory_path.py index 9c365eb9..48fbeede 100644 --- a/tests/unit/test_artifactory_path.py +++ b/tests/unit/test_artifactory_path.py @@ -456,12 +456,7 @@ def setUp(self): }""" self.deploy_by_checksum_error = { - "errors": [ - { - "status": 400, - "message": "Checksum values not provided" - } - ] + "errors": [{"status": 400, "message": "Checksum values not provided"}] } @@ -929,7 +924,7 @@ def test_deploy_by_checksum_sha1(self): responses.PUT, self.artifact_url, json=self.file_stat_without_modification_date, - status=200 + status=200, ) self.path.deploy_by_checksum(sha1=self.sha1) @@ -951,7 +946,7 @@ def test_deploy_by_checksum_sha256(self): responses.PUT, self.artifact_url, json=self.file_stat_without_modification_date, - status=200 + status=200, ) self.path.deploy_by_checksum(sha256=self.sha256) @@ -973,7 +968,7 @@ def test_deploy_by_checksum_sha1_or_sha256(self): responses.PUT, self.artifact_url, json=self.file_stat_without_modification_date, - status=200 + status=200, ) self.path.deploy_by_checksum(checksum=self.sha1) @@ -990,7 +985,7 @@ def test_deploy_by_checksum_sha1_or_sha256(self): responses.PUT, self.artifact_url, json=self.file_stat_without_modification_date, - status=200 + status=200, ) self.path.deploy_by_checksum(checksum=self.sha256) @@ -1012,12 +1007,14 @@ def test_deploy_by_checksum_error(self): responses.PUT, self.artifact_url, json=self.deploy_by_checksum_error, - status=400 + status=400, ) with self.assertRaises(RuntimeError) as context: self.path.deploy_by_checksum(sha1=f"{self.sha1}invalid") - self.assertEqual(str(context.exception), json.dumps(self.deploy_by_checksum_error)) + self.assertEqual( + str(context.exception), json.dumps(self.deploy_by_checksum_error) + ) self.assertEqual(len(rsps.calls), 1) self.assertEqual(rsps.calls[0].request.url, self.artifact_url) From a92a46a3dc58f6eb4dca46930ea634312a7ced75 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Wed, 8 Sep 2021 16:04:01 +0800 Subject: [PATCH 06/48] Move upload code into pytest.fixture --- .../test_integration_artifactory_path.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_integration_artifactory_path.py b/tests/integration/test_integration_artifactory_path.py index 146ca366..5442e99f 100644 --- a/tests/integration/test_integration_artifactory_path.py +++ b/tests/integration/test_integration_artifactory_path.py @@ -175,10 +175,10 @@ def test_deploy_file(path): p.unlink() -def test_deploy_file_by_checksum(path): +@pytest.fixture() +def deploy_file(path): p = path("/integration-artifactory-path-repo/foo") p1 = path("/integration-artifactory-path-repo/bar") - if p.exists(): p.unlink() if p1.exists(): @@ -191,17 +191,19 @@ def test_deploy_file_by_checksum(path): sha1 = sha1sum(tf.name) sha256 = sha256sum(tf.name) - sha1_non_existent = "1111111111111111111111111111111111111111" - sha256_non_existent = ( - "1111111111111111111111111111111111111111111111111111111111111111" - ) - p.deploy_file(tf.name) tf.close() with p.open() as fd: result = fd.read() assert result == b"Some test string" + return (sha1, sha256) + + +def test_deploy_file_by_checksum(path, deploy_file): + p = path("/integration-artifactory-path-repo/foo") + p1 = path("/integration-artifactory-path-repo/bar") + # The matrix is a full list of all possible combinations # 1st row is for 1st parameter of deploy_by_checksum # 2nd row is for 2nd parameter and 3rd is for 3rd parameter @@ -211,7 +213,11 @@ def test_deploy_file_by_checksum(path): # [sha1, sha1_non_existent, sha256, sha256_non_existent, None] # ] - p1.deploy_by_checksum(sha1=sha1) + sha1_non_existent = "1111111111111111111111111111111111111111" + sha256_non_existent = ( + "1111111111111111111111111111111111111111111111111111111111111111" + ) + p1.deploy_by_checksum(sha1=deploy_file[0]) with p1.open() as fd: result = fd.read() p1.unlink() @@ -220,7 +226,7 @@ def test_deploy_file_by_checksum(path): with pytest.raises(RuntimeError) as excinfo: p.deploy_by_checksum(sha1=sha1_non_existent) - p1.deploy_by_checksum(sha256=sha256) + p1.deploy_by_checksum(sha256=deploy_file[1]) with p1.open() as fd: result = fd.read() p1.unlink() @@ -229,7 +235,7 @@ def test_deploy_file_by_checksum(path): with pytest.raises(RuntimeError) as excinfo: p.deploy_by_checksum(sha256=sha256_non_existent) - p1.deploy_by_checksum(checksum=sha1) + p1.deploy_by_checksum(checksum=deploy_file[0]) with p1.open() as fd: result = fd.read() p1.unlink() @@ -238,7 +244,7 @@ def test_deploy_file_by_checksum(path): with pytest.raises(RuntimeError) as excinfo: p.deploy_by_checksum(checksum=sha1_non_existent) - p1.deploy_by_checksum(checksum=sha256) + p1.deploy_by_checksum(checksum=deploy_file[1]) with p1.open() as fd: result = fd.read() p1.unlink() From 146deebc117de32903722c23b8d1797b6b1bb46a Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Wed, 8 Sep 2021 07:52:17 +0200 Subject: [PATCH 07/48] log to our own artifactory logger. Add example how to configure it fix logging and aql debug for #131 and #235 --- README.md | 34 +++++++++++++++++++++++++--------- artifactory.py | 16 ++++++++++++---- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7bda380e..ac463b12 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ package. * [FileStat](#filestat) * [Promote Docker image](#promote-docker-image) * [Builds](#builds) + * [Logging](#logging) - [Admin area](#admin-area) * [User](#user) + [API Keys](#api-keys) @@ -85,7 +86,6 @@ pip install dohq-artifactory==0.5.dev243 ``` # Usage - ## Authentication ## `dohq-artifactory` supports these ways of authentication: @@ -474,19 +474,19 @@ You can use [Artifactory Query Language](https://www.jfrog.com/confluence/displa ```python from artifactory import ArtifactoryPath -aql = ArtifactoryPath( +arti_path = ArtifactoryPath( "http://my-artifactory/artifactory" ) # path to artifactory, NO repo # dict support # Send query: # items.find({"repo": "myrepo"}) -artifacts = aql.aql("items.find", {"repo": "myrepo"}) +artifacts = arti_path.aql("items.find", {"repo": "myrepo"}) # list support. # Send query: # items.find().include("name", "repo") -artifacts = aql.aql("items.find()", ".include", ["name", "repo"]) +artifacts = arti_path.aql("items.find()", ".include", ["name", "repo"]) # support complex query # Example 1 @@ -498,7 +498,7 @@ artifacts = aql.aql("items.find()", ".include", ["name", "repo"]) # ] # } # ) -args = [ +aqlargs = [ "items.find", { "$and": [ @@ -515,7 +515,7 @@ args = [ # artifacts_list contains raw data (list of dict) # Send query -artifacts_list = aql.aql(*args) +artifacts_list = arti_path.aql(*aqlargs) # Example 2 # The query will find all items in repo docker-prod that are of type file and were created after timecode. The @@ -541,11 +541,11 @@ aqlargs = [ ".sort", {"$asc": ["repo", "path", "name"]}, ] -artifacts_list = aql.aql(*args) +artifacts_list = arti_path.aql(*aqlargs) # You can convert to pathlib object: -artifact_pathlib = map(aql.from_aql, artifacts_list) -artifact_pathlib_list = list(map(aql.from_aql, artifacts_list)) +artifact_pathlib = map(arti_path.from_aql, artifacts_list) +artifact_pathlib_list = list(map(arti_path.from_aql, artifacts_list)) ``` @@ -620,6 +620,22 @@ build_number1.promote(ci_user="admin", properties={ }) ~~~ +## Logging +The library can be configured to emit logging that will give you better insight into what it's doing. +Just configure `logging` module in your python script. Simplest example to add debug messages to a console: +~~~python +import logging +from artifactory import ArtifactoryPath + +logging.basicConfig() +# set level only for artifactory module, if omitted, then global log level is used, eg from basicConfig +logging.getLogger('artifactory').setLevel(logging.DEBUG) + +path = ArtifactoryPath( + "http://my-artifactory/artifactory/myrepo/restricted-path", apikey="MY_API_KEY" +) +~~~ + # Admin area You can manipulate with user\group\repository and permission. First, create `ArtifactoryPath` object without a repository ```python diff --git a/artifactory.py b/artifactory.py index 95e037ae..32906661 100755 --- a/artifactory.py +++ b/artifactory.py @@ -64,6 +64,11 @@ default_config_path = "~/.artifactory_python.cfg" global_config = None +# set logger to be configurable from external +logger = logging.getLogger(__name__) +# Set default logging handler to avoid "No handler found" warnings. +logger.addHandler(logging.NullHandler()) + def read_config(config_path=default_config_path): """ @@ -265,7 +270,7 @@ def log_download_progress(bytes_now, total_size): else: msg = "Downloaded {}MB".format(int(bytes_now / 1024 / 1024)) - logging.debug(msg) + logger.debug(msg) class HTTPResponseWrapper(object): @@ -385,6 +390,7 @@ def quote_url(url): :param url: (str) URL that should be quoted :return: (str) quoted URL """ + logger.debug(f"Raw URL passed for encoding: {url}") parsed_url = urllib3.util.parse_url(url) if parsed_url.port: quoted_path = requests.utils.quote( @@ -1309,10 +1315,10 @@ def __new__(cls, *args, **kwargs): auth_type = kwargs.get("auth_type") if apikey: - logging.debug("Use XJFrogApiAuth apikey") + logger.debug("Use XJFrogApiAuth apikey") obj.auth = XJFrogArtApiAuth(apikey=apikey) elif token: - logging.debug("Use XJFrogArtBearerAuth token") + logger.debug("Use XJFrogArtBearerAuth token") obj.auth = XJFrogArtBearerAuth(token=token) else: auth = kwargs.get("auth") @@ -1912,6 +1918,7 @@ def aql(self, *args): """ aql_query_url = "{}/api/search/aql".format(self.drive.rstrip("/")) aql_query_text = self.create_aql_text(*args) + logger.debug(f"AQL query request text: {aql_query_text}") r = self.session.post(aql_query_url, data=aql_query_text) r.raise_for_status() content = r.json() @@ -1920,7 +1927,7 @@ def aql(self, *args): @staticmethod def create_aql_text(*args): """ - Create AQL querty from string or list or dict arguments + Create AQL query from string or list or dict arguments """ aql_query_text = "" for arg in args: @@ -1929,6 +1936,7 @@ def create_aql_text(*args): elif isinstance(arg, list): arg = "({})".format(json.dumps(arg)).replace("[", "").replace("]", "") aql_query_text += arg + return aql_query_text def from_aql(self, result): From f54056ceea314e50b29d18a5cce95b01fabad0fe Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Mon, 13 Sep 2021 23:21:28 +0200 Subject: [PATCH 08/48] added dry run to copy and move --- README.md | 6 ++++ artifactory.py | 80 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index cde3ae98..06836869 100644 --- a/README.md +++ b/README.md @@ -418,6 +418,9 @@ http://example.com/artifactory/published/production/foo-0.0.1.pom http://example.com/artifactory/published/production/product-1.0.0.tar.gz http://example.com/artifactory/published/production/product-1.0.0.tar.pom """ + +# you can use dry run just to check if command will succeed without real change, adds debug message +source.copy(dest, dry_run=True) ``` ## Move Artifacts @@ -436,6 +439,9 @@ source = ArtifactoryPath("http://example.com/artifactory/builds/product/product/ dest = ArtifactoryPath("http://example.com/artifactory/published/production/") source.move(dest) + +# you can use dry run just to check if command will succeed without real change, adds debug message +source.move(dest, dry_run=True) ``` ## Remove Artifacts diff --git a/artifactory.py b/artifactory.py index 767aec22..1ec1f527 100755 --- a/artifactory.py +++ b/artifactory.py @@ -1099,24 +1099,35 @@ def deploy( if code not in (200, 201): raise RuntimeError(text) - def copy(self, src, dst, suppress_layouts=False): + def copy(self, src, dst, suppress_layouts=False, fail_fast=False, dry_run=False): """ Copy artifact from src to dst + Args: + src: from + dst: to + suppress_layouts: suppress cross-layout module path translation during copy + fail_fast: parameter will fail and abort the operation upon receiving an error. + dry_run: If true, distribution is only simulated. + + Returns: + if dry_run==True (dict) response.json() else None """ url = "/".join( [ src.drive.rstrip("/"), "api/copy", - str(src.relative_to(src.drive)).rstrip("/"), + str(src.relative_to(src.drive)).strip("/"), ] ) params = { "to": str(dst.relative_to(dst.drive)).rstrip("/"), "suppressLayouts": int(suppress_layouts), + "failFast": int(fail_fast), + "dry": int(dry_run), } - self.rest_post( + response = self.rest_post( url, params=params, session=src.session, @@ -1124,10 +1135,22 @@ def copy(self, src, dst, suppress_layouts=False): cert=src.cert, timeout=src.timeout, ) + if dry_run: + logger.debug(response.text) + return response.json() - def move(self, src, dst, suppress_layouts=False): + def move(self, src, dst, suppress_layouts=False, fail_fast=False, dry_run=False): """ Move artifact from src to dst + Args: + src: from + dst: to + suppress_layouts: suppress cross-layout module path translation during copy + fail_fast: parameter will fail and abort the operation upon receiving an error. + dry_run: If true, distribution is only simulated. + + Returns: + if dry_run==True (dict) response.json() else None """ url = "/".join( [ @@ -1140,9 +1163,11 @@ def move(self, src, dst, suppress_layouts=False): params = { "to": str(dst.relative_to(dst.drive)).rstrip("/"), "suppressLayouts": int(suppress_layouts), + "failFast": int(fail_fast), + "dry": int(dry_run), } - self.rest_post( + response = self.rest_post( url, params=params, session=src.session, @@ -1150,6 +1175,9 @@ def move(self, src, dst, suppress_layouts=False): cert=src.cert, timeout=src.timeout, ) + if dry_run: + logger.debug(response.text) + return response.json() def get_properties(self, pathobj): """ @@ -1841,7 +1869,7 @@ def deploy_deb( self.deploy_file(file_name, parameters=params) - def copy(self, dst, suppress_layouts=False): + def copy(self, dst, suppress_layouts=False, fail_fast=False, dry_run=False): """ Copy artifact from this path to destination. If files are on the same instance of artifactory, lightweight (local) @@ -1853,6 +1881,9 @@ def copy(self, dst, suppress_layouts=False): root, but remap the [org], [module], [baseVer], etc. structure to the target repository. + fail_fast: parameter will fail and abort the operation upon receiving an error. + dry_run: If true, distribution is only simulated. + For example, if we have a builds repository using the default maven2 repository where we publish our builds. We also have a published repository where a directory for production and a directory for @@ -1887,27 +1918,56 @@ def copy(self, dst, suppress_layouts=False): http://example.com/artifactory/published/production/foo-0.0.1.pom http://example.com/artifactory/published/production/product-1.0.0.tar.gz http://example.com/artifactory/published/production/product-1.0.0.tar.pom + + Returns: + if dry_run==True (dict) response.json() else None """ if self.drive.rstrip("/") == dst.drive.rstrip("/"): - self._accessor.copy(self, dst, suppress_layouts=suppress_layouts) + output = self._accessor.copy( + self, + dst, + suppress_layouts=suppress_layouts, + fail_fast=fail_fast, + dry_run=dry_run, + ) + return output else: + if dry_run: + logger.debug( + "Artifactory drive is different. Will do a standard upload" + ) + return + with self.open() as fobj: dst.deploy(fobj) - def move(self, dst, suppress_layouts=False): + def move(self, dst, suppress_layouts=False, fail_fast=False, dry_run=False): """ - Move artifact from this path to destinaiton. + Move artifact from this path to destination. The suppress_layouts parameter, when set to True, will allow artifacts from one path to be moved directly into another path without enforcing repository layouts. The default behaviour is to move the repository root, but remap the [org], [module], [baseVer], etc. structure to the target repository. + + fail_fast: parameter will fail and abort the operation upon receiving an error. + dry_run: If true, distribution is only simulated. + + Returns: + if dry_run==True (dict) response.json() else None """ if self.drive.rstrip("/") != dst.drive.rstrip("/"): raise NotImplementedError("Moving between instances is not implemented yet") - self._accessor.move(self, dst) + output = self._accessor.move( + self, + dst, + suppress_layouts=suppress_layouts, + fail_fast=fail_fast, + dry_run=dry_run, + ) + return output @property def properties(self): From c824c4ffc253328d5ccda9beec6a899565b02c7c Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Mon, 20 Sep 2021 13:18:34 +0200 Subject: [PATCH 09/48] refactor exception handling --- README.md | 22 ++++++ artifactory.py | 113 ++++++++++++---------------- dohq_artifactory/admin.py | 24 ++---- dohq_artifactory/exception.py | 33 ++++++++ tests/unit/test_artifactory_path.py | 7 +- tests/unit/test_exceptions.py | 20 +++-- 6 files changed, 126 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index 06836869..4511b1b9 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ package. * [Promote Docker image](#promote-docker-image) * [Builds](#builds) * [Logging](#logging) + * [Exception handling](#exception-handling) - [Admin area](#admin-area) * [User](#user) + [API Keys](#api-keys) @@ -673,6 +674,27 @@ path = ArtifactoryPath( ) ~~~ +## Exception handling +Exceptions in this library are represented by `dohq_artifactory.exception.ArtifactoryException` or by `OSError` +If exception was caused by HTTPError you can always drill down the root cause by using following example: +~~~python +from artifactory import ArtifactoryPath +from dohq_artifactory.exception import ArtifactoryException + +path = ArtifactoryPath( + "http://my_arti:8080/artifactory/installer/", + auth=("wrong_user", "wrong_pass") +) + +try: + path.stat() +except ArtifactoryException as exc: + print(exc) # clean artifactory error message + # >>> Bad credentials + print(exc.__cause__) # HTTP error that triggered exception, you can use this object for more info + # >>> 401 Client Error: Unauthorized for url: http://my_arti:8080/artifactory/installer/ +~~~ + # Admin area You can manipulate with user\group\repository and permission. First, create `ArtifactoryPath` object without a repository ```python diff --git a/artifactory.py b/artifactory.py index 1ec1f527..3b640932 100755 --- a/artifactory.py +++ b/artifactory.py @@ -51,6 +51,7 @@ from dohq_artifactory.auth import XJFrogArtApiAuth from dohq_artifactory.auth import XJFrogArtBearerAuth from dohq_artifactory.exception import ArtifactoryException +from dohq_artifactory.exception import raise_http_errors try: import requests.packages.urllib3 as urllib3 @@ -91,7 +92,7 @@ def read_config(config_path=default_config_path): config_path = os.path.expanduser(config_path) if not os.path.isfile(config_path): raise OSError( - errno.ENOENT, "Artifactory configuration file not found: '%s'" % config_path + errno.ENOENT, f"Artifactory configuration file not found: '{config_path}'" ) p = configparser.ConfigParser() @@ -675,7 +676,7 @@ def rest_put( Perform a PUT request to url with requests.session """ url = quote_url(url) - res = session.put( + response = session.put( url, params=params, headers=headers, @@ -683,7 +684,7 @@ def rest_put( cert=cert, timeout=timeout, ) - return res.text, res.status_code + return response @staticmethod def rest_post( @@ -707,7 +708,7 @@ def rest_post( cert=cert, timeout=timeout, ) - response.raise_for_status() + raise_http_errors(response) return response @@ -727,7 +728,7 @@ def rest_del(url, params=None, session=None, verify=True, cert=None, timeout=Non response = session.delete( url, params=params, verify=verify, cert=cert, timeout=timeout ) - response.raise_for_status() + raise_http_errors(response) return response @staticmethod @@ -751,10 +752,11 @@ def rest_put_stream( # added later, otherwise ; and = are converted url += matrix_parameters - res = session.put( + response = session.put( url, headers=headers, data=stream, verify=verify, cert=cert, timeout=timeout ) - return res.text, res.status_code + raise_http_errors(response) + return response @staticmethod def rest_get_stream( @@ -768,7 +770,7 @@ def rest_get_stream( response = session.get( url, params=params, stream=True, verify=verify, cert=cert, timeout=timeout ) - response.raise_for_status() + raise_http_errors(response) return response def get_stat_json(self, pathobj): @@ -796,9 +798,9 @@ def get_stat_json(self, pathobj): if code == 404 and ("Unable to find item" in text or "Not Found" in text): raise OSError(2, f"No such file or directory: {url}") - response.raise_for_status() + raise_http_errors(response) - return json.loads(text) + return response.json() def stat(self, pathobj): """ @@ -866,7 +868,7 @@ def listdir(self, pathobj): stat = self.stat(pathobj) if not stat.is_dir: - raise OSError(20, "Not a directory: %s" % str(pathobj)) + raise OSError(20, f"Not a directory: {pathobj}") return stat.children @@ -876,13 +878,13 @@ def mkdir(self, pathobj, _): Note that this operation is not recursive """ if not pathobj.drive or not pathobj.root: - raise RuntimeError("Full path required: '%s'" % str(pathobj)) + raise ArtifactoryException(f"Full path required: '{pathobj}'") if pathobj.exists(): - raise OSError(17, "File exists: '%s'" % str(pathobj)) + raise OSError(17, f"File exists: '{pathobj}'") url = str(pathobj) + "/" - text, code = self.rest_put( + response = self.rest_put( url, session=pathobj.session, verify=pathobj.verify, @@ -890,8 +892,7 @@ def mkdir(self, pathobj, _): timeout=pathobj.timeout, ) - if code != 201: - raise RuntimeError("%s %d" % (text, code)) + raise_http_errors(response) def rmdir(self, pathobj): """ @@ -900,7 +901,7 @@ def rmdir(self, pathobj): stat = self.stat(pathobj) if not stat.is_dir: - raise OSError(20, "Not a directory: '%s'" % str(pathobj)) + raise OSError(20, f"Not a directory: '{pathobj}'") url = str(pathobj) + "/" @@ -931,8 +932,8 @@ def unlink(self, pathobj): cert=pathobj.cert, timeout=pathobj.timeout, ) - except requests.exceptions.HTTPError as err: - if err.response.status_code == 404: + except ArtifactoryException as err: + if err.__cause__.response.status_code == 404: # since we performed existence check we can say it is permissions issue # see https://github.com/devopshq/artifactory/issues/36 docs_url = ( @@ -951,13 +952,13 @@ def touch(self, pathobj): Create an empty file """ if not pathobj.drive or not pathobj.root: - raise RuntimeError("Full path required") + raise ArtifactoryException("Full path required") if pathobj.exists(): return url = str(pathobj) - text, code = self.rest_put( + response = self.rest_put( url, session=pathobj.session, verify=pathobj.verify, @@ -965,8 +966,7 @@ def touch(self, pathobj): timeout=pathobj.timeout, ) - if code != 201: - raise RuntimeError("%s %d" % (text, code)) + raise_http_errors(response) def owner(self, pathobj): """ @@ -1051,14 +1051,14 @@ def deploy( :param sha1: :param sha256: :param parameters: Artifact properties - :param explode_archive(bool): True: archive will be exploded upon deployment - :param explode_archive_atomic(bool): True: archive will be exploded in an atomic operation upon deployment + :param explode_archive: (bool) if True, archive will be exploded upon deployment + :param explode_archive_atomic: (bool) if True, archive will be exploded in an atomic operation upon deployment :param checksum: sha1Value or sha256Value - :param by_checksum(bool): if True, deploy artifact by checksum, default False + :param by_checksum: (bool) if True, deploy artifact by checksum, default False """ if fobj and by_checksum: - raise RuntimeError("Either fobj or by_checksum, but not both") + raise ArtifactoryException("Either fobj or by_checksum, but not both") if isinstance(fobj, urllib3.response.HTTPResponse): fobj = HTTPResponseWrapper(fobj) @@ -1085,7 +1085,7 @@ def deploy( if checksum: headers["X-Checksum"] = checksum - text, code = self.rest_put_stream( + self.rest_put_stream( url, fobj, headers=headers, @@ -1096,9 +1096,6 @@ def deploy( matrix_parameters=matrix_parameters, ) - if code not in (200, 201): - raise RuntimeError(text) - def copy(self, src, dst, suppress_layouts=False, fail_fast=False, dry_run=False): """ Copy artifact from src to dst @@ -1204,13 +1201,13 @@ def get_properties(self, pathobj): code = response.status_code text = response.text if code == 404 and ("Unable to find item" in text or "Not Found" in text): - raise OSError(2, "No such file or directory: '%s'" % url) + raise OSError(2, f"No such file or directory: '{url}'") if code == 404 and "No properties could be found" in text: return {} - response.raise_for_status() + raise_http_errors(response) - return json.loads(text)["properties"] + return response.json()["properties"] def set_properties(self, pathobj, props, recursive): """ @@ -1229,7 +1226,7 @@ def set_properties(self, pathobj, props, recursive): if not recursive: params["recursive"] = "0" - text, code = self.rest_put( + response = self.rest_put( url, params=params, session=pathobj.session, @@ -1238,10 +1235,12 @@ def set_properties(self, pathobj, props, recursive): timeout=pathobj.timeout, ) + code = response.status_code + text = response.text if code == 404 and ("Unable to find item" in text or "Not Found" in text): - raise OSError(2, "No such file or directory: '%s'" % url) - if code != 204: - raise RuntimeError(text) + raise OSError(2, f"No such file or directory: '{url}'") + + raise_http_errors(response) def del_properties(self, pathobj, props, recursive): """ @@ -1502,7 +1501,7 @@ def archive(self, archive_type="zip", check_sum=False): :return: raw object for download """ if self.is_file(): - raise OSError("Only folders could be archived") + raise ArtifactoryException("Only folders could be archived") if archive_type not in ["zip", "tar", "tar.gz", "tgz"]: raise NotImplementedError(archive_type + " is not support by current API") @@ -2028,9 +2027,9 @@ def aql(self, *args): aql_query_url = "{}/api/search/aql".format(self.drive.rstrip("/")) aql_query_text = self.create_aql_text(*args) logger.debug(f"AQL query request text: {aql_query_text}") - r = self.session.post(aql_query_url, data=aql_query_text) - r.raise_for_status() - content = r.json() + response = self.session.post(aql_query_url, data=aql_query_text) + raise_http_errors(response) + content = response.json() return content["results"] @staticmethod @@ -2056,10 +2055,8 @@ def from_aql(self, result): """ result_type = result.get("type") if result_type not in ("file", "folder"): - raise RuntimeError( - "Path object with type '{}' doesn't support. File or folder only".format( - result_type - ) + raise ArtifactoryException( + f"Path object with type '{result_type}' doesn't support. File or folder only" ) result_path = "{}/{repo}/{path}/{name}".format(self.drive.rstrip("/"), **result) @@ -2082,7 +2079,7 @@ def promote_docker_image( :param target_repo: target repository :param docker_repo: Docker repository to promote :param tag: Docker tag to promote - :param copy (bool): whether to move the image or copy it + :param copy: (bool) whether to move the image or copy it :return: """ promote_url = "{}/api/docker/{}/v2/promote".format( @@ -2094,18 +2091,8 @@ def promote_docker_image( "tag": tag, "copy": copy, } - r = self.session.post(promote_url, json=promote_data) - if ( - r.status_code == 400 - and "Unsupported docker" in r.text - or r.status_code == 403 - and "No permission" in r.text - or r.status_code == 404 - and "Unable to find" in r.text - ): - raise ArtifactoryException(r.text) - else: - r.raise_for_status() + response = self.session.post(promote_url, json=promote_data) + raise_http_errors(response) @property def repo(self): @@ -2208,11 +2195,11 @@ def _get_all(self, lazy: bool, url=None, key="name", cls=None): request_url = self.drive.rstrip("/artifactory") + url else: request_url = self.drive + url - r = self.session.get(request_url, auth=self.auth) - r.raise_for_status() - response = r.json() + response = self.session.get(request_url, auth=self.auth) + raise_http_errors(response) + response_json = response.json() results = [] - for i in response: + for i in response_json: if cls is Repository: item = Repository.create_by_type(i["type"], self, i[key]) else: diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index 5ee7a4dc..ed000f33 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -7,26 +7,16 @@ import time import jwt -import requests from dateutil.parser import isoparse from dohq_artifactory.exception import ArtifactoryException +from dohq_artifactory.exception import raise_http_errors def rest_delay(): time.sleep(0.5) -def raise_errors(r): - try: - r.raise_for_status() - except requests.HTTPError as e: - if e.response.status_code >= 400: - raise ArtifactoryException(e.response.text) - else: - raise e - - def _old_function_for_secret(pw_len=16): alphabet_lower = "abcdefghijklmnopqrstuvwxyz" alphabet_upper = alphabet_lower.upper() @@ -118,7 +108,7 @@ def _create_and_update(self, method): headers={"Content-Type": "application/json"}, auth=self._auth, ) - raise_errors(r) + raise_http_errors(r) rest_delay() self.read() @@ -151,7 +141,7 @@ def read(self): logging.debug( f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] exist" ) - raise_errors(r) + raise_http_errors(r) response = r.json() self.raw = response self._read_response(response) @@ -200,7 +190,7 @@ def delete(self): request_url, auth=self._auth, ) - raise_errors(r) + raise_http_errors(r) rest_delay() @@ -301,7 +291,7 @@ def _authenticated_user_request(self, api_url, request_type): request_url, auth=(self.name, self.password), ) - raise_errors(r) + raise_http_errors(r) return r.text @property @@ -1363,7 +1353,7 @@ def create(self): headers={"Content-Type": "application/json"}, auth=self._auth, ) - raise_errors(r) + raise_http_errors(r) rest_delay() self.read() @@ -1384,7 +1374,7 @@ def update(self): headers={"Content-Type": "application/json"}, auth=self._auth, ) - raise_errors(r) + raise_http_errors(r) rest_delay() self.read() diff --git a/dohq_artifactory/exception.py b/dohq_artifactory/exception.py index 62f348c3..0f1eaecd 100644 --- a/dohq_artifactory/exception.py +++ b/dohq_artifactory/exception.py @@ -1,2 +1,35 @@ +from json import JSONDecodeError + +import requests + + class ArtifactoryException(Exception): pass + + +def raise_http_errors(response): + """ + Custom raise_for_status method. + Raises ArtifactoryException with clear message and keeps cause + Args: + response: HTTP response object + + Returns: + None + """ + + try: + response.raise_for_status() + except requests.HTTPError as err: + try: + error_list = err.response.json().setdefault( + "errors", [{}] + ) # prepare a container + if isinstance(error_list[0], dict): + err_msg = error_list[0].setdefault("message", str(err)) + else: + err_msg = str(error_list[0]) + except JSONDecodeError: + err_msg = str(err) + + raise ArtifactoryException(err_msg) from err diff --git a/tests/unit/test_artifactory_path.py b/tests/unit/test_artifactory_path.py index 48fbeede..9adc71cf 100644 --- a/tests/unit/test_artifactory_path.py +++ b/tests/unit/test_artifactory_path.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import json import os import pathlib import tempfile @@ -1009,12 +1008,10 @@ def test_deploy_by_checksum_error(self): json=self.deploy_by_checksum_error, status=400, ) - with self.assertRaises(RuntimeError) as context: + with self.assertRaises(ArtifactoryException) as context: self.path.deploy_by_checksum(sha1=f"{self.sha1}invalid") - self.assertEqual( - str(context.exception), json.dumps(self.deploy_by_checksum_error) - ) + self.assertEqual(str(context.exception), "Checksum values not provided") self.assertEqual(len(rsps.calls), 1) self.assertEqual(rsps.calls[0].request.url, self.artifact_url) diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 3bef6892..e6744d8f 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -1,21 +1,25 @@ #!/usr/bin/env python import unittest -import mock import requests +import responses -from dohq_artifactory.admin import raise_errors from dohq_artifactory.exception import ArtifactoryException +from dohq_artifactory.exception import raise_http_errors class UtilTest(unittest.TestCase): def test_raise_errors(self): - r = requests.Response() - r.status_code = 400 - type(r).text = mock.PropertyMock(return_value="asd") - with self.assertRaises(ArtifactoryException) as cm: - raise_errors(r) - self.assertEqual("asd", str(cm.exception)) + with responses.RequestsMock() as mock: + url = "http://b.com/artifactory/" + mock.add(responses.GET, url, status=403) + resp = requests.get("http://b.com/artifactory/") + + with self.assertRaises(ArtifactoryException) as cm: + raise_http_errors(resp) + self.assertEqual( + f"403 Client Error: Forbidden for url: {url}", str(cm.exception) + ) if __name__ == "__main__": From 83b7332bbb3857db8f20dde3b5549916ffe36b21 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Mon, 20 Sep 2021 13:54:07 +0200 Subject: [PATCH 10/48] updated test --- tests/unit/test_exceptions.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index e6744d8f..1ab09ae2 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -10,6 +10,7 @@ class UtilTest(unittest.TestCase): def test_raise_errors(self): + # no JSON body, just HTTP response message with responses.RequestsMock() as mock: url = "http://b.com/artifactory/" mock.add(responses.GET, url, status=403) @@ -21,6 +22,21 @@ def test_raise_errors(self): f"403 Client Error: Forbidden for url: {url}", str(cm.exception) ) + # real JSON body, can parse for clean message + with responses.RequestsMock() as mock: + url = "http://b.com/artifactory/" + mock.add( + responses.GET, + url, + status=403, + json={"errors": [{"status": 401, "message": "Bad credentials"}]}, + ) + resp = requests.get("http://b.com/artifactory/") + + with self.assertRaises(ArtifactoryException) as cm: + raise_http_errors(resp) + self.assertEqual("Bad credentials", str(cm.exception)) + if __name__ == "__main__": unittest.main() From 9f95e630387b5cb287348255801bc8c71ada9f43 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Mon, 20 Sep 2021 14:17:17 +0200 Subject: [PATCH 11/48] added copying with checksum --- artifactory.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/artifactory.py b/artifactory.py index 767aec22..50b1a224 100755 --- a/artifactory.py +++ b/artifactory.py @@ -824,9 +824,9 @@ def stat(self, pathobj): modified_by=jsn.get("modifiedBy"), mime_type=jsn.get("mimeType"), size=int(jsn.get("size", "0")), - sha1=checksums.get("sha1"), - sha256=checksums.get("sha256"), - md5=checksums.get("md5"), + sha1=checksums.get("sha1", None), + sha256=checksums.get("sha256", None), + md5=checksums.get("md5", None), is_dir=is_dir, children=children, ) @@ -1047,9 +1047,9 @@ def deploy( :param pathobj: ArtifactoryPath object :param fobj: file object to be deployed - :param md5: - :param sha1: - :param sha256: + :param md5: (str) MD5 checksum value + :param sha1: (str) SHA1 checksum value + :param sha256: (str) SHA256 checksum value :param parameters: Artifact properties :param explode_archive(bool): True: archive will be exploded upon deployment :param explode_archive_atomic(bool): True: archive will be exploded in an atomic operation upon deployment @@ -1891,8 +1891,14 @@ def copy(self, dst, suppress_layouts=False): if self.drive.rstrip("/") == dst.drive.rstrip("/"): self._accessor.copy(self, dst, suppress_layouts=suppress_layouts) else: + stat = self.stat() + if stat.is_dir: + raise ArtifactoryException( + "Only files could be copied across different instances" + ) + with self.open() as fobj: - dst.deploy(fobj) + dst.deploy(fobj, md5=stat.md5, sha1=stat.sha1, sha256=stat.sha256) def move(self, dst, suppress_layouts=False): """ From 78c7fe90bda02bd9844f8f065b8042403916098c Mon Sep 17 00:00:00 2001 From: Tibor Dohany <48722396+tdohany@users.noreply.github.com> Date: Thu, 23 Sep 2021 14:13:02 +0200 Subject: [PATCH 12/48] without that it throws a 405 Method Not Allowed for url requests.exceptions.HTTPError: 405 Client Error: Method Not Allowed for url --- dohq_artifactory/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index 5ee7a4dc..2c412ef1 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -501,7 +501,7 @@ def create(self): ) data_json = self._create_json() data_json.update(self.additional_params) - request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}" + request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}/{getattr(self, self.resource_name)}" r = self._session.post( request_url, json=data_json, From 31553bba25acb77c7d869104111217d53114f2dc Mon Sep 17 00:00:00 2001 From: Tibor Dohany <48722396+tdohany@users.noreply.github.com> Date: Thu, 23 Sep 2021 17:49:53 +0200 Subject: [PATCH 13/48] Update admin.py --- dohq_artifactory/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index 2c412ef1..ae62bc24 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -502,7 +502,7 @@ def create(self): data_json = self._create_json() data_json.update(self.additional_params) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}/{getattr(self, self.resource_name)}" - r = self._session.post( + r = self._session.put( request_url, json=data_json, headers={"Content-Type": "application/json"}, From ef90c5e7fdfe6c4465a96c599f6891be20e77a59 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Mon, 20 Sep 2021 17:32:04 +0200 Subject: [PATCH 14/48] added logger to admin module --- artifactory.py | 7 +------ dohq_artifactory/admin.py | 34 +++++++++++++++++----------------- dohq_artifactory/logger.py | 6 ++++++ 3 files changed, 24 insertions(+), 23 deletions(-) create mode 100644 dohq_artifactory/logger.py diff --git a/artifactory.py b/artifactory.py index 50b1a224..4a8dece0 100755 --- a/artifactory.py +++ b/artifactory.py @@ -29,7 +29,6 @@ import hashlib import io import json -import logging import os import pathlib import re @@ -51,6 +50,7 @@ from dohq_artifactory.auth import XJFrogArtApiAuth from dohq_artifactory.auth import XJFrogArtBearerAuth from dohq_artifactory.exception import ArtifactoryException +from dohq_artifactory.logger import logger try: import requests.packages.urllib3 as urllib3 @@ -64,11 +64,6 @@ default_config_path = "~/.artifactory_python.cfg" global_config = None -# set logger to be configurable from external -logger = logging.getLogger(__name__) -# Set default logging handler to avoid "No handler found" warnings. -logger.addHandler(logging.NullHandler()) - def read_config(config_path=default_config_path): """ diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index 5ee7a4dc..77a712c6 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -1,5 +1,4 @@ import json -import logging import random import re import string @@ -11,6 +10,7 @@ from dateutil.parser import isoparse from dohq_artifactory.exception import ArtifactoryException +from dohq_artifactory.logger import logger def rest_delay(): @@ -99,7 +99,7 @@ def create(self): Create object :return: None """ - logging.debug( + logger.debug( f"Create {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) self._create_and_update(self._session.put) @@ -137,18 +137,18 @@ def read(self): True if object exist, False else """ - logging.debug( + logger.debug( f"Read {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}/{getattr(self, self.resource_name)}" r = self._session.get(request_url, auth=self._auth) if 404 == r.status_code or 400 == r.status_code: - logging.debug( + logger.debug( f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] does not exist" ) return False else: - logging.debug( + logger.debug( f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] exist" ) raise_errors(r) @@ -163,18 +163,18 @@ def list(self): :return: List of objects """ - # logging.debug('List {x.__class__.__name__} [{x.name}]'.format(x=self)) + logger.debug(f"List {self.__class__.__name__} [{self.name}]") request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}" response = self._session.get( request_url, auth=self._auth, ) if response.status_code == 200: - # logging.debug('{x.__class__.__name__} [{x.name}] does not exist'.format(x=self)) + logger.debug(f"{self.__class__.__name__} [{self.name}] does not exist") json_response = response.json() return [item.get(self.resource_name) for item in json_response] else: - # logging.debug('{x.__class__.__name__} [{x.name}] exist'.format(x=self)) + logger.debug(f"{self.__class__.__name__} [{self.name}] exist") return "failed" def update(self): @@ -182,7 +182,7 @@ def update(self): Update object :return: None """ - logging.debug( + logger.debug( f"Create {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) self._create_and_update(self._session.post) @@ -192,7 +192,7 @@ def delete(self): Remove object :return: None """ - logging.debug( + logger.debug( f"Remove {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}/{getattr(self, self.resource_name)}" @@ -483,7 +483,7 @@ def delete(self): TODO: New entrypoint would go like /api/groups/delete and consumes ["list", "of", "groupnames"] """ - logging.debug( + logger.debug( f"Remove {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri_deletion}/{getattr(self, self.resource_name)}" @@ -496,7 +496,7 @@ def create(self): Create object :return: None """ - logging.debug( + logger.debug( f"Create {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) data_json = self._create_json() @@ -1270,19 +1270,19 @@ def read(self): True if object exist, False else """ - logging.debug( + logger.debug( f"Read {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}" r = self._session.get(request_url, auth=self._auth) if 404 == r.status_code or 400 == r.status_code: - logging.debug( + logger.debug( f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] does not exist" ) return False else: - logging.debug( - "{self.__class__.__name__} [{getattr(self, self.resource_name)}] exist" + logger.debug( + f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] exist" ) r.raise_for_status() response = r.json() @@ -1299,7 +1299,7 @@ def delete(self): POST security/token/revoke revoke (calling it deletion to be consistent with other classes) a token """ - logging.debug( + logger.debug( f"Delete {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}/revoke" diff --git a/dohq_artifactory/logger.py b/dohq_artifactory/logger.py new file mode 100644 index 00000000..1bd70d89 --- /dev/null +++ b/dohq_artifactory/logger.py @@ -0,0 +1,6 @@ +import logging + +# set logger to be configurable from external +logger = logging.getLogger("artifactory") +# Set default logging handler to avoid "No handler found" warnings. +logger.addHandler(logging.NullHandler()) From 84fab5c1438e1d99921af400f1c474685d9eddae Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Mon, 20 Sep 2021 18:11:48 +0200 Subject: [PATCH 15/48] read config file with CA_BUNDLE property for #281 --- README.md | 21 +++++++++++++++------ artifactory.py | 34 +++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index cde3ae98..a0e5b2b1 100644 --- a/README.md +++ b/README.md @@ -1368,28 +1368,37 @@ path.touch() ## Global Configuration File ## -Artifactory Python module also can specify all connection-related settings in a central file, ```~/.artifactory_python.cfg``` that is read upon the creation of first ```ArtifactoryPath``` object and is stored globally. For instance, you can specify per-instance settings of authentication tokens, so that you won't need to explicitly pass ```auth``` parameter to ```ArtifactoryPath```. +Artifactory Python module also can specify all connection-related settings in a central file, +```~/.artifactory_python.cfg``` that is read upon the creation of first ```ArtifactoryPath``` object and is stored +globally. For instance, you can specify per-instance settings of authentication tokens, so that you won't need to +explicitly pass ```auth``` parameter to ```ArtifactoryPath```. Example: ```ini +[DEFAULT] +username = nameforallinstances + [http://artifactory-instance.com/artifactory] -username = deployer password = ilikerandompasswords verify = false [another-artifactory-instance.com/artifactory] -username = foo password = @dmin cert = ~/mycert ``` -Whether or not you specify ```http://``` or ```https://```, the prefix is not essential. The module will first try to locate the best match and then try to match URLs without prefixes. So in the config, if you specify ```https://my-instance.local``` and call ```ArtifactoryPath``` with ```http://my-instance.local```, it will still do the right thing. +Whether or not you specify ```http://``` or ```https://```, the prefix is not essential. The module will first try to +locate the best match and then try to match URLs without prefixes. So in the config, if you specify +```https://my-instance.local``` and call ```ArtifactoryPath``` with ```http://my-instance.local```, it will still do +the right thing. # Contribute [About contributing and testing](docs/CONTRIBUTE.md) # Advertising -- [artifactory-du](https://github.com/devopshq/artifactory-du) - estimate file space usage. Summarize disk usage in JFrog Artifactory of the set of FILEs, recursively for directories. -- [artifactory-cleanup-rules](https://github.com/devopshq/artifactory-du/issues/2) - python-script for Artifactory intelligence cleanup rules with config. +- [artifactory-du](https://github.com/devopshq/artifactory-du) - estimate file space usage. Summarize disk usage in + JFrog Artifactory of the set of FILEs, recursively for directories. +- [artifactory-cleanup-rules](https://github.com/devopshq/artifactory-du/issues/2) - python-script for Artifactory + intelligence cleanup rules with config. diff --git a/artifactory.py b/artifactory.py index 50b1a224..4c808703 100755 --- a/artifactory.py +++ b/artifactory.py @@ -75,7 +75,7 @@ def read_config(config_path=default_config_path): Read configuration file and produce a dictionary of the following structure: {'': {'username': '', 'password': '', - 'verify': , 'cert': ''} + 'verify': , 'cert': ''} '': {...}, ...} @@ -91,7 +91,7 @@ def read_config(config_path=default_config_path): config_path = os.path.expanduser(config_path) if not os.path.isfile(config_path): raise OSError( - errno.ENOENT, "Artifactory configuration file not found: '%s'" % config_path + errno.ENOENT, f"Artifactory configuration file not found: '{config_path}'" ) p = configparser.ConfigParser() @@ -100,16 +100,22 @@ def read_config(config_path=default_config_path): result = {} for section in p.sections(): - username = ( - p.get(section, "username") if p.has_option(section, "username") else None - ) - password = ( - p.get(section, "password") if p.has_option(section, "password") else None - ) - verify = ( - p.getboolean(section, "verify") if p.has_option(section, "verify") else True - ) - cert = p.get(section, "cert") if p.has_option(section, "cert") else None + username = p.get(section, "username", fallback=None) + password = p.get(section, "password", fallback=None) + + try: + verify = p.getboolean(section, "verify", fallback=True) + except ValueError: + # the path to a CA_BUNDLE file or directory with certificates of trusted CAs + # see https://github.com/devopshq/artifactory/issues/281 + verify = p.get(section, "verify", fallback=True) + # path may contain '~', and we'd better expand it properly + verify = os.path.expanduser(verify) + + cert = p.get(section, "cert", fallback=None) + if cert: + # certificate path may contain '~', and we'd better expand it properly + cert = os.path.expanduser(cert) result[section] = { "username": username, @@ -117,9 +123,7 @@ def read_config(config_path=default_config_path): "verify": verify, "cert": cert, } - # certificate path may contain '~', and we'd better expand it properly - if result[section]["cert"]: - result[section]["cert"] = os.path.expanduser(result[section]["cert"]) + return result From 8388e2d1b97eb3fc63c28650a6b837a97dfb8fe2 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Mon, 27 Sep 2021 12:16:52 +0200 Subject: [PATCH 16/48] added comments --- artifactory.py | 26 +++++++++++++------------- dohq_artifactory/admin.py | 14 +++++++------- dohq_artifactory/exception.py | 5 ++++- tests/unit/test_exceptions.py | 6 +++--- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/artifactory.py b/artifactory.py index 3b640932..aea2f2b8 100755 --- a/artifactory.py +++ b/artifactory.py @@ -51,7 +51,7 @@ from dohq_artifactory.auth import XJFrogArtApiAuth from dohq_artifactory.auth import XJFrogArtBearerAuth from dohq_artifactory.exception import ArtifactoryException -from dohq_artifactory.exception import raise_http_errors +from dohq_artifactory.exception import raise_for_status try: import requests.packages.urllib3 as urllib3 @@ -708,7 +708,7 @@ def rest_post( cert=cert, timeout=timeout, ) - raise_http_errors(response) + raise_for_status(response) return response @@ -728,7 +728,7 @@ def rest_del(url, params=None, session=None, verify=True, cert=None, timeout=Non response = session.delete( url, params=params, verify=verify, cert=cert, timeout=timeout ) - raise_http_errors(response) + raise_for_status(response) return response @staticmethod @@ -755,7 +755,7 @@ def rest_put_stream( response = session.put( url, headers=headers, data=stream, verify=verify, cert=cert, timeout=timeout ) - raise_http_errors(response) + raise_for_status(response) return response @staticmethod @@ -770,7 +770,7 @@ def rest_get_stream( response = session.get( url, params=params, stream=True, verify=verify, cert=cert, timeout=timeout ) - raise_http_errors(response) + raise_for_status(response) return response def get_stat_json(self, pathobj): @@ -798,7 +798,7 @@ def get_stat_json(self, pathobj): if code == 404 and ("Unable to find item" in text or "Not Found" in text): raise OSError(2, f"No such file or directory: {url}") - raise_http_errors(response) + raise_for_status(response) return response.json() @@ -892,7 +892,7 @@ def mkdir(self, pathobj, _): timeout=pathobj.timeout, ) - raise_http_errors(response) + raise_for_status(response) def rmdir(self, pathobj): """ @@ -966,7 +966,7 @@ def touch(self, pathobj): timeout=pathobj.timeout, ) - raise_http_errors(response) + raise_for_status(response) def owner(self, pathobj): """ @@ -1205,7 +1205,7 @@ def get_properties(self, pathobj): if code == 404 and "No properties could be found" in text: return {} - raise_http_errors(response) + raise_for_status(response) return response.json()["properties"] @@ -1240,7 +1240,7 @@ def set_properties(self, pathobj, props, recursive): if code == 404 and ("Unable to find item" in text or "Not Found" in text): raise OSError(2, f"No such file or directory: '{url}'") - raise_http_errors(response) + raise_for_status(response) def del_properties(self, pathobj, props, recursive): """ @@ -2028,7 +2028,7 @@ def aql(self, *args): aql_query_text = self.create_aql_text(*args) logger.debug(f"AQL query request text: {aql_query_text}") response = self.session.post(aql_query_url, data=aql_query_text) - raise_http_errors(response) + raise_for_status(response) content = response.json() return content["results"] @@ -2092,7 +2092,7 @@ def promote_docker_image( "copy": copy, } response = self.session.post(promote_url, json=promote_data) - raise_http_errors(response) + raise_for_status(response) @property def repo(self): @@ -2196,7 +2196,7 @@ def _get_all(self, lazy: bool, url=None, key="name", cls=None): else: request_url = self.drive + url response = self.session.get(request_url, auth=self.auth) - raise_http_errors(response) + raise_for_status(response) response_json = response.json() results = [] for i in response_json: diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index ed000f33..9c6c1764 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -10,7 +10,7 @@ from dateutil.parser import isoparse from dohq_artifactory.exception import ArtifactoryException -from dohq_artifactory.exception import raise_http_errors +from dohq_artifactory.exception import raise_for_status def rest_delay(): @@ -108,7 +108,7 @@ def _create_and_update(self, method): headers={"Content-Type": "application/json"}, auth=self._auth, ) - raise_http_errors(r) + raise_for_status(r) rest_delay() self.read() @@ -141,7 +141,7 @@ def read(self): logging.debug( f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] exist" ) - raise_http_errors(r) + raise_for_status(r) response = r.json() self.raw = response self._read_response(response) @@ -190,7 +190,7 @@ def delete(self): request_url, auth=self._auth, ) - raise_http_errors(r) + raise_for_status(r) rest_delay() @@ -291,7 +291,7 @@ def _authenticated_user_request(self, api_url, request_type): request_url, auth=(self.name, self.password), ) - raise_http_errors(r) + raise_for_status(r) return r.text @property @@ -1353,7 +1353,7 @@ def create(self): headers={"Content-Type": "application/json"}, auth=self._auth, ) - raise_http_errors(r) + raise_for_status(r) rest_delay() self.read() @@ -1374,7 +1374,7 @@ def update(self): headers={"Content-Type": "application/json"}, auth=self._auth, ) - raise_http_errors(r) + raise_for_status(r) rest_delay() self.read() diff --git a/dohq_artifactory/exception.py b/dohq_artifactory/exception.py index 0f1eaecd..0cdd09e3 100644 --- a/dohq_artifactory/exception.py +++ b/dohq_artifactory/exception.py @@ -7,7 +7,7 @@ class ArtifactoryException(Exception): pass -def raise_http_errors(response): +def raise_for_status(response): """ Custom raise_for_status method. Raises ArtifactoryException with clear message and keeps cause @@ -21,13 +21,16 @@ def raise_http_errors(response): try: response.raise_for_status() except requests.HTTPError as err: + # start processing HTTP error and try to extract meaningful data from it try: error_list = err.response.json().setdefault( "errors", [{}] ) # prepare a container if isinstance(error_list[0], dict): + # get message from HTTP errors message err_msg = error_list[0].setdefault("message", str(err)) else: + # if for some reason we don't receive standard HTTP errors dict, we need to raise the whole object err_msg = str(error_list[0]) except JSONDecodeError: err_msg = str(err) diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 1ab09ae2..44062474 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -5,7 +5,7 @@ import responses from dohq_artifactory.exception import ArtifactoryException -from dohq_artifactory.exception import raise_http_errors +from dohq_artifactory.exception import raise_for_status class UtilTest(unittest.TestCase): @@ -17,7 +17,7 @@ def test_raise_errors(self): resp = requests.get("http://b.com/artifactory/") with self.assertRaises(ArtifactoryException) as cm: - raise_http_errors(resp) + raise_for_status(resp) self.assertEqual( f"403 Client Error: Forbidden for url: {url}", str(cm.exception) ) @@ -34,7 +34,7 @@ def test_raise_errors(self): resp = requests.get("http://b.com/artifactory/") with self.assertRaises(ArtifactoryException) as cm: - raise_http_errors(resp) + raise_for_status(resp) self.assertEqual("Bad credentials", str(cm.exception)) From 6f5141c71204e25628b03f0204adbafc6100ef06 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 28 Sep 2021 14:31:53 +0200 Subject: [PATCH 17/48] refactor error raising --- dohq_artifactory/exception.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/dohq_artifactory/exception.py b/dohq_artifactory/exception.py index 0cdd09e3..c37529f8 100644 --- a/dohq_artifactory/exception.py +++ b/dohq_artifactory/exception.py @@ -20,19 +20,22 @@ def raise_for_status(response): try: response.raise_for_status() - except requests.HTTPError as err: + except requests.HTTPError as exception: # start processing HTTP error and try to extract meaningful data from it try: - error_list = err.response.json().setdefault( - "errors", [{}] - ) # prepare a container - if isinstance(error_list[0], dict): - # get message from HTTP errors message - err_msg = error_list[0].setdefault("message", str(err)) - else: - # if for some reason we don't receive standard HTTP errors dict, we need to raise the whole object - err_msg = str(error_list[0]) + response_json = exception.response.json() + error_list = response_json.pop("errors", None) except JSONDecodeError: - err_msg = str(err) + # not a JSON response + raise ArtifactoryException(str(exception)) from exception - raise ArtifactoryException(err_msg) from err + if not isinstance(error_list, list) or not error_list: + # no standard error list in the exception + raise ArtifactoryException(str(exception)) from exception + + error_info_dict = error_list[0] + if not isinstance(error_info_dict, dict) or "message" not in error_info_dict: + # if for some reason we don't receive standard HTTP errors dict, we need to raise the whole object + raise ArtifactoryException(str(error_info_dict)) from exception + + raise ArtifactoryException(error_info_dict["message"]) from exception From 426590f8bad7d13aee8633a79008336266db6f47 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 5 Oct 2021 12:03:36 +0200 Subject: [PATCH 18/48] move logging docs --- README.md | 45 +++++++++++++-------------------------------- 1 file changed, 13 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 70eb2557..9860ed7c 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,6 @@ package. * [FileStat](#filestat) * [Promote Docker image](#promote-docker-image) * [Builds](#builds) - * [Logging](#logging) * [Exception handling](#exception-handling) - [Admin area](#admin-area) * [User](#user) @@ -65,7 +64,7 @@ package. * [Session](#session) * [SSL Cert Verification Options](#ssl-cert-verification-options) * [Timeout on requests](#timeout-on-requests) - * [Troubleshooting](#troubleshooting) + * [Logging](#logging) * [Global Configuration File](#global-configuration-file) - [Contribute](#contribute) - [Advertising](#advertising) @@ -658,22 +657,6 @@ build_number1.promote(ci_user="admin", properties={ }) ~~~ -## Logging -The library can be configured to emit logging that will give you better insight into what it's doing. -Just configure `logging` module in your python script. Simplest example to add debug messages to a console: -~~~python -import logging -from artifactory import ArtifactoryPath - -logging.basicConfig() -# set level only for artifactory module, if omitted, then global log level is used, eg from basicConfig -logging.getLogger('artifactory').setLevel(logging.DEBUG) - -path = ArtifactoryPath( - "http://my-artifactory/artifactory/myrepo/restricted-path", apikey="MY_API_KEY" -) -~~~ - ## Exception handling Exceptions in this library are represented by `dohq_artifactory.exception.ArtifactoryException` or by `OSError` If exception was caused by HTTPError you can always drill down the root cause by using following example: @@ -1374,24 +1357,22 @@ path = ArtifactoryPath( ) ``` -## Troubleshooting ## -Use [logging](https://docs.python.org/3/library/logging.html) for debug: -```python -def init_logging(): - logger_format_string = "%(thread)5s %(module)-20s %(levelname)-8s %(message)s" - logging.basicConfig( - level=logging.DEBUG, format=logger_format_string, stream=sys.stdout - ) +## Logging ## +The library can be configured to emit logging that will give you better insight into what it's doing. +Just configure [logging](https://docs.python.org/3/library/logging.html) module in your python script. +Simplest example to add debug messages to a console: +~~~python +import logging +from artifactory import ArtifactoryPath +logging.basicConfig() +# set level only for artifactory module, if omitted, then global log level is used, eg from basicConfig +logging.getLogger('artifactory').setLevel(logging.DEBUG) -init_logging() path = ArtifactoryPath( - "http://my-artifactory/artifactory/myrepo/restricted-path", - auth=("USERNAME", "PASSWORD or API_KEY"), + "http://my-artifactory/artifactory/myrepo/restricted-path", apikey="MY_API_KEY" ) - -path.touch() -``` +~~~ ## Global Configuration File ## From 5e9ee2dc3babd6e48d13fcb9d49dec69bb004767 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 5 Oct 2021 11:45:37 +0200 Subject: [PATCH 19/48] refactor module to follow python conventions and preserve backwards compatibility --- dohq_artifactory/admin.py | 184 +++++++++++++++++++++++--------- tests/integration/test_admin.py | 2 +- 2 files changed, 134 insertions(+), 52 deletions(-) diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index 7d6139a2..b66fd375 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -4,6 +4,7 @@ import string import sys import time +from warnings import warn import jwt from dateutil.parser import isoparse @@ -219,7 +220,7 @@ def __init__( self.internal_password_disabled = False self._groups = [] - self._lastLoggedIn = None + self._last_logged_in = None self._realm = None def _create_json(self): @@ -250,7 +251,7 @@ def _read_response(self, response): self.disable_ui_access = response.get("disableUIAccess") self.internal_password_disabled = response.get("internalPasswordDisabled") self._groups = response.get("groups", []) - self._lastLoggedIn = ( + self._last_logged_in = ( isoparse(response["lastLoggedIn"]) if response.get("lastLoggedIn") else None ) self._realm = response.get("realm") @@ -261,6 +262,11 @@ def encryptedPassword(self): Method for backwards compatibility, see property encrypted_password :return: """ + # todo remove around April 2022 + warn( + "encryptedPassword is deprecated, use encrypted_password", + DeprecationWarning, + ) return self.encrypted_password @property @@ -296,7 +302,17 @@ def _authenticated_user_request(self, api_url, request_type): @property def lastLoggedIn(self): - return self._lastLoggedIn + """ + Method for backwards compatibility, see property last_logged_in + :return: + """ + # todo remove around April 2022 + warn("lastLoggedIn is deprecated, use last_logged_in", DeprecationWarning) + return self.last_logged_in + + @property + def last_logged_in(self): + return self._last_logged_in @property def realm(self): @@ -526,7 +542,7 @@ def path(self): return self._artifactory.joinpath(self.name) def _generate_query(self, package): - if self.packageType == Repository.DOCKER: + if self.package_type == Repository.DOCKER: parts = package.split(":") name = parts[0] @@ -536,7 +552,7 @@ def _generate_query(self, package): return {"name": "manifest.json", "path": {"$match": package}} - if self.packageType == Repository.PYPI and "/" not in package: + if self.package_type == Repository.PYPI and "/" not in package: operators = { "<=": "$lte", "<": "$lt", @@ -557,7 +573,7 @@ def _generate_query(self, package): return {"@pypi.name": {"$match": package}} - if self.packageType == Repository.MAVEN and "/" not in package: + if self.package_type == Repository.MAVEN and "/" not in package: package = package.replace("#", ":") parts = list(package.split(":")) @@ -577,7 +593,7 @@ def _generate_query(self, package): "$or": [ {"name": {"$match": package}}, {"path": {"$match": package}}, - {"@{}.name".format(self.packageType): {"$match": package}}, + {"@{}.name".format(self.package_type): {"$match": package}}, {"@build.name": {"$match": package}}, {"artifact.module.build.name": {"$match": package}}, ] @@ -636,7 +652,7 @@ def __rtruediv__(self, key): class Repository(GenericRepository): - # List packageType from wiki: + # List package_type from wiki: # https://www.jfrog.com/confluence/display/RTF/Repository+Configuration+JSON#RepositoryConfigurationJSON-application/vnd.org.jfrog.artifactory.repositories.LocalRepositoryConfiguration+json ALPINE = "alpine" BOWER = "bower" @@ -654,11 +670,6 @@ class Repository(GenericRepository): HELM = "helm" IVY = "ivy" MAVEN = "maven" - SBT = "sbt" - HELM = "helm" - RPM = "rpm" - NUGET = "nuget" - GEMS = "gems" NPM = "npm" NUGET = "nuget" PUPPET = "puppet" @@ -667,21 +678,50 @@ class Repository(GenericRepository): SBT = "sbt" YUM = "yum" - # List dockerApiVersion from wiki: + # List docker_api_version from wiki: V1 = "V1" V2 = "V2" @staticmethod - def create_by_type(type: str, artifactory, name): - if type == "LOCAL": + def create_by_type(repo_type: str, artifactory, name): + if repo_type == "LOCAL": return RepositoryLocal(artifactory, name) - elif type == "REMOTE": + elif repo_type == "REMOTE": return RepositoryRemote(artifactory, name) - elif type == "VIRTUAL": + elif repo_type == "VIRTUAL": return RepositoryVirtual(artifactory, name) else: return None + @property + def packageType(self): + # todo remove around April 2022 + warn("packageType is deprecated, use package_type", DeprecationWarning) + return self.package_type + + @property + def repoLayoutRef(self): + # todo remove around April 2022 + warn("repoLayoutRef is deprecated, use repo_layout_ref", DeprecationWarning) + return self.repo_layout_ref + + @property + def dockerApiVersion(self): + # todo remove around April 2022 + warn( + "dockerApiVersion is deprecated, use docker_api_version", DeprecationWarning + ) + return self.docker_api_version + + @property + def archiveBrowsingEnabled(self): + # todo remove around April 2022 + warn( + "archiveBrowsingEnabled is deprecated, use archive_browsing_enabled", + DeprecationWarning, + ) + return self.archive_browsing_enabled + class RepositoryLocal(Repository): _uri = "repositories" @@ -694,20 +734,35 @@ def __init__( self, artifactory, name, - packageType=Repository.GENERIC, - dockerApiVersion=Repository.V1, - repoLayoutRef="maven-2-default", + package_type=Repository.GENERIC, + docker_api_version=Repository.V1, + repo_layout_ref="maven-2-default", max_unique_tags=0, + *, + packageType=None, # todo remove around April 2022 + dockerApiVersion=None, # todo remove around April 2022 + repoLayoutRef=None, # todo remove around April 2022 ): super(RepositoryLocal, self).__init__(artifactory) self.name = name self.description = "" - self.packageType = packageType - self.repoLayoutRef = repoLayoutRef - self.archiveBrowsingEnabled = True - self.dockerApiVersion = dockerApiVersion + self.package_type = packageType or package_type # todo remove around April 2022 + self.repo_layout_ref = ( + repoLayoutRef or repo_layout_ref + ) # todo remove around April 2022 + self.archive_browsing_enabled = True + self.docker_api_version = ( + dockerApiVersion or docker_api_version + ) # todo remove around April 2022 self.max_unique_tags = max_unique_tags + if any([packageType, dockerApiVersion, repoLayoutRef]): + msg = ( + "packageType, dockerApiVersion, repoLayoutRef are deprecated, " + "use package_type, docker_api_version, repo_layout_ref" + ) + warn(msg, DeprecationWarning) + def _create_json(self): """ JSON Documentation: https://www.jfrog.com/confluence/display/RTF/Repository+Configuration+JSON @@ -716,12 +771,12 @@ def _create_json(self): "rclass": "local", "key": self.name, "description": self.description, - "packageType": self.packageType, + "packageType": self.package_type, "notes": "", "includesPattern": "**/*", "excludesPattern": "", - "repoLayoutRef": self.repoLayoutRef, - "dockerApiVersion": self.dockerApiVersion, + "repoLayoutRef": self.repo_layout_ref, + "dockerApiVersion": self.docker_api_version, "checksumPolicyType": "client-checksums", "handleReleases": True, "handleSnapshots": True, @@ -730,13 +785,13 @@ def _create_json(self): "suppressPomConsistencyChecks": True, "blackedOut": False, "propertySets": [], - "archiveBrowsingEnabled": self.archiveBrowsingEnabled, + "archiveBrowsingEnabled": self.archive_browsing_enabled, "yumRootDepth": 0, } """ Docker V2 API specific fields """ - if self.dockerApiVersion == Repository.V2: + if self.docker_api_version == Repository.V2: data_json["maxUniqueTags"] = self.max_unique_tags return data_json @@ -755,9 +810,9 @@ def _read_response(self, response): self.name = response["key"] self.description = response.get("description") - self.packageType = response.get("packageType") - self.repoLayoutRef = response.get("repoLayoutRef") - self.archiveBrowsingEnabled = response.get("archiveBrowsingEnabled") + self.package_type = response.get("packageType") + self.repo_layout_ref = response.get("repoLayoutRef") + self.archive_browsing_enabled = response.get("archiveBrowsingEnabled") class RepositoryVirtual(GenericRepository): @@ -790,15 +845,27 @@ def __init__( artifactory, name, repositories=None, - packageType=Repository.GENERIC, + package_type=Repository.GENERIC, + *, + packageType=None, # todo remove around April 2022 ): super(RepositoryVirtual, self).__init__(artifactory) self.name = name self.description = "" self.notes = "" - self.packageType = packageType + self.package_type = packageType or package_type # todo remove around April 2022 self.repositories = repositories or [] + if packageType: + msg = "packageType is deprecated, use package_type" + warn(msg, DeprecationWarning) + + @property + def packageType(self): + # todo remove around April 2022 + warn("packageType is deprecated, use package_type", DeprecationWarning) + return self.package_type + def _create_json(self): """ JSON Documentation: https://www.jfrog.com/confluence/display/RTF/Repository+Configuration+JSON @@ -807,7 +874,7 @@ def _create_json(self): "rclass": "virtual", "key": self.name, "description": self.description, - "packageType": self.packageType, + "packageType": self.package_type, "repositories": self._repositories, "notes": self.notes, } @@ -828,7 +895,7 @@ def _read_response(self, response): self.name = response["key"] self.description = response.get("description") - self.packageType = response.get("packageType") + self.package_type = response.get("packageType") self._repositories = response.get("repositories") def add_repository(self, *repos): @@ -871,19 +938,34 @@ def __init__( artifactory, name, url=None, - packageType=Repository.GENERIC, - dockerApiVersion=Repository.V1, - repoLayoutRef="maven-2-default", + package_type=Repository.GENERIC, + docker_api_version=Repository.V1, + repo_layout_ref="maven-2-default", + *, + packageType=None, # todo remove around April 2022 + dockerApiVersion=None, # todo remove around April 2022 + repoLayoutRef=None, # todo remove around April 2022 ): super(RepositoryRemote, self).__init__(artifactory) self.name = name self.description = "" - self.packageType = packageType - self.repoLayoutRef = repoLayoutRef - self.archiveBrowsingEnabled = True - self.dockerApiVersion = dockerApiVersion + self.package_type = packageType or package_type # todo remove around April 2022 + self.repo_layout_ref = ( + repoLayoutRef or repo_layout_ref + ) # todo remove around April 2022 + self.archive_browsing_enabled = True + self.docker_api_version = ( + dockerApiVersion or docker_api_version + ) # todo remove around April 2022 self.url = url + if any([packageType, dockerApiVersion, repoLayoutRef]): + msg = ( + "packageType, dockerApiVersion, repoLayoutRef are deprecated, " + "use package_type, docker_api_version, repo_layout_ref" + ) + warn(msg, DeprecationWarning) + def _create_json(self): """ JSON Documentation: https://www.jfrog.com/confluence/display/RTF/Repository+Configuration+JSON @@ -892,12 +974,12 @@ def _create_json(self): "rclass": "remote", "key": self.name, "description": self.description, - "packageType": self.packageType, + "packageType": self.package_type, "notes": "", "includesPattern": "**/*", "excludesPattern": "", - "repoLayoutRef": self.repoLayoutRef, - "dockerApiVersion": self.dockerApiVersion, + "repoLayoutRef": self.repo_layout_ref, + "dockerApiVersion": self.docker_api_version, "checksumPolicyType": "client-checksums", "handleReleases": True, "handleSnapshots": True, @@ -906,7 +988,7 @@ def _create_json(self): "suppressPomConsistencyChecks": True, "blackedOut": False, "propertySets": [], - "archiveBrowsingEnabled": self.archiveBrowsingEnabled, + "archiveBrowsingEnabled": self.archive_browsing_enabled, "yumRootDepth": 0, "url": self.url, "debianTrivialLayout": False, @@ -933,9 +1015,9 @@ def _read_response(self, response): self.name = response["key"] self.description = response.get("description") - self.packageType = response.get("packageType") - self.repoLayoutRef = response.get("repoLayoutRef") - self.archiveBrowsingEnabled = response.get("archiveBrowsingEnabled") + self.package_type = response.get("packageType") + self.repo_layout_ref = response.get("repoLayoutRef") + self.archive_browsing_enabled = response.get("archiveBrowsingEnabled") self.url = response.get("url") diff --git a/tests/integration/test_admin.py b/tests/integration/test_admin.py index 42c484a3..2f0641ff 100644 --- a/tests/integration/test_admin.py +++ b/tests/integration/test_admin.py @@ -127,7 +127,7 @@ def test_create_delete(self, artifactory): test_repo.delete() test_repo = RepositoryLocal( - artifactory=artifactory, name=name, packageType=RepositoryLocal.DEBIAN + artifactory=artifactory, name=name, package_type=RepositoryLocal.DEBIAN ) # CREATE test_repo.create() From 9a72a1491839b5e25b39b0ef1d0a4820f9237a2a Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Wed, 6 Oct 2021 14:19:15 +0200 Subject: [PATCH 20/48] remove todos move warn to new method --- dohq_artifactory/admin.py | 77 +++++++++++++++------------------------ 1 file changed, 30 insertions(+), 47 deletions(-) diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index b66fd375..5976a7c8 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -4,7 +4,7 @@ import string import sys import time -from warnings import warn +import warnings import jwt from dateutil.parser import isoparse @@ -57,6 +57,10 @@ def _new_function_with_secret_module(pw_len=16): generate_password = _new_function_with_secret_module +def deprecation(message): + warnings.warn(message, DeprecationWarning, stacklevel=2) + + class AdminObject(object): prefix_uri = "api" _uri = None @@ -262,11 +266,7 @@ def encryptedPassword(self): Method for backwards compatibility, see property encrypted_password :return: """ - # todo remove around April 2022 - warn( - "encryptedPassword is deprecated, use encrypted_password", - DeprecationWarning, - ) + deprecation("encryptedPassword is deprecated, use encrypted_password") return self.encrypted_password @property @@ -306,8 +306,7 @@ def lastLoggedIn(self): Method for backwards compatibility, see property last_logged_in :return: """ - # todo remove around April 2022 - warn("lastLoggedIn is deprecated, use last_logged_in", DeprecationWarning) + deprecation("lastLoggedIn is deprecated, use last_logged_in") return self.last_logged_in @property @@ -695,30 +694,23 @@ def create_by_type(repo_type: str, artifactory, name): @property def packageType(self): - # todo remove around April 2022 - warn("packageType is deprecated, use package_type", DeprecationWarning) + deprecation("packageType is deprecated, use package_type") return self.package_type @property def repoLayoutRef(self): - # todo remove around April 2022 - warn("repoLayoutRef is deprecated, use repo_layout_ref", DeprecationWarning) + deprecation("repoLayoutRef is deprecated, use repo_layout_ref") return self.repo_layout_ref @property def dockerApiVersion(self): - # todo remove around April 2022 - warn( - "dockerApiVersion is deprecated, use docker_api_version", DeprecationWarning - ) + deprecation("dockerApiVersion is deprecated, use docker_api_version") return self.docker_api_version @property def archiveBrowsingEnabled(self): - # todo remove around April 2022 - warn( - "archiveBrowsingEnabled is deprecated, use archive_browsing_enabled", - DeprecationWarning, + deprecation( + "archiveBrowsingEnabled is deprecated, use archive_browsing_enabled" ) return self.archive_browsing_enabled @@ -739,21 +731,17 @@ def __init__( repo_layout_ref="maven-2-default", max_unique_tags=0, *, - packageType=None, # todo remove around April 2022 - dockerApiVersion=None, # todo remove around April 2022 - repoLayoutRef=None, # todo remove around April 2022 + packageType=None, + dockerApiVersion=None, + repoLayoutRef=None, ): super(RepositoryLocal, self).__init__(artifactory) self.name = name self.description = "" - self.package_type = packageType or package_type # todo remove around April 2022 - self.repo_layout_ref = ( - repoLayoutRef or repo_layout_ref - ) # todo remove around April 2022 + self.package_type = packageType or package_type + self.repo_layout_ref = repoLayoutRef or repo_layout_ref self.archive_browsing_enabled = True - self.docker_api_version = ( - dockerApiVersion or docker_api_version - ) # todo remove around April 2022 + self.docker_api_version = dockerApiVersion or docker_api_version self.max_unique_tags = max_unique_tags if any([packageType, dockerApiVersion, repoLayoutRef]): @@ -761,7 +749,7 @@ def __init__( "packageType, dockerApiVersion, repoLayoutRef are deprecated, " "use package_type, docker_api_version, repo_layout_ref" ) - warn(msg, DeprecationWarning) + deprecation(msg) def _create_json(self): """ @@ -847,23 +835,22 @@ def __init__( repositories=None, package_type=Repository.GENERIC, *, - packageType=None, # todo remove around April 2022 + packageType=None, ): super(RepositoryVirtual, self).__init__(artifactory) self.name = name self.description = "" self.notes = "" - self.package_type = packageType or package_type # todo remove around April 2022 + self.package_type = packageType or package_type self.repositories = repositories or [] if packageType: msg = "packageType is deprecated, use package_type" - warn(msg, DeprecationWarning) + deprecation(msg) @property def packageType(self): - # todo remove around April 2022 - warn("packageType is deprecated, use package_type", DeprecationWarning) + deprecation("packageType is deprecated, use package_type") return self.package_type def _create_json(self): @@ -942,21 +929,17 @@ def __init__( docker_api_version=Repository.V1, repo_layout_ref="maven-2-default", *, - packageType=None, # todo remove around April 2022 - dockerApiVersion=None, # todo remove around April 2022 - repoLayoutRef=None, # todo remove around April 2022 + packageType=None, + dockerApiVersion=None, + repoLayoutRef=None, ): super(RepositoryRemote, self).__init__(artifactory) self.name = name self.description = "" - self.package_type = packageType or package_type # todo remove around April 2022 - self.repo_layout_ref = ( - repoLayoutRef or repo_layout_ref - ) # todo remove around April 2022 + self.package_type = packageType or package_type + self.repo_layout_ref = repoLayoutRef or repo_layout_ref self.archive_browsing_enabled = True - self.docker_api_version = ( - dockerApiVersion or docker_api_version - ) # todo remove around April 2022 + self.docker_api_version = dockerApiVersion or docker_api_version self.url = url if any([packageType, dockerApiVersion, repoLayoutRef]): @@ -964,7 +947,7 @@ def __init__( "packageType, dockerApiVersion, repoLayoutRef are deprecated, " "use package_type, docker_api_version, repo_layout_ref" ) - warn(msg, DeprecationWarning) + deprecation(msg) def _create_json(self): """ From fe912b4dac3a52c6f277d6ac637d3835365defbe Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Thu, 7 Oct 2021 07:27:15 +0200 Subject: [PATCH 21/48] py 3.10 compatibility --- artifactory.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/artifactory.py b/artifactory.py index 077e2ce6..ab7e8bbe 100755 --- a/artifactory.py +++ b/artifactory.py @@ -1346,6 +1346,11 @@ class ArtifactoryPath(pathlib.Path, PureArtifactoryPath): # so authentication information has to be added via __slots__ __slots__ = ("auth", "verify", "cert", "session", "timeout") + if sys.version_info.major == 3 and sys.version_info.minor >= 10: + # see changes in pathlib.Path, slots are no more applied + # https://github.com/python/cpython/blob/ce121fd8755d4db9511ce4aab39d0577165e118e/Lib/pathlib.py#L952 + _accessor = _artifactory_accessor + def __new__(cls, *args, **kwargs): """ pathlib.Path overrides __new__ in order to create objects From 7603a10c2f4f8d4ff2882fd215aa8693f711c95e Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Thu, 7 Oct 2021 12:39:04 +0200 Subject: [PATCH 22/48] added py 3.9 and 3.10 to CI --- .travis.yml | 2 ++ setup.py | 1 + tox.ini | 2 ++ 3 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 56b5f735..b19839d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ python: - "3.6" - "3.7" - "3.8" + - "3.9" + - "3.10" install: pip install tox-travis script: tox deploy: diff --git a/setup.py b/setup.py index 6803995e..6fd4240a 100755 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Topic :: Software Development :: Libraries", "Topic :: System :: Filesystems", ], diff --git a/tox.ini b/tox.ini index d7fba88d..3767f8ba 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,8 @@ python = 3.6: py36 3.7: py37, pre-commit 3.8: py38 + 3.9: py39 + 3.10: py310 [testenv] deps = -rrequirements-dev.txt From 4f0e444f4645436711985faf3f0555a3587e2aeb Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Thu, 7 Oct 2021 13:00:55 +0200 Subject: [PATCH 23/48] CI --- .travis.yml | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b19839d5..cd1bbadf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: - "3.7" - "3.8" - "3.9" - - "3.10" + - "3.10-dev" install: pip install tox-travis script: tox deploy: diff --git a/setup.py b/setup.py index 6fd4240a..b8c21ca7 100755 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Libraries", "Topic :: System :: Filesystems", ], From c405bdbc76e1c3fdf9496d7e9cacd5b44de21438 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Thu, 7 Oct 2021 13:04:56 +0200 Subject: [PATCH 24/48] slots --- artifactory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/artifactory.py b/artifactory.py index ab7e8bbe..300514a8 100755 --- a/artifactory.py +++ b/artifactory.py @@ -1342,14 +1342,14 @@ class ArtifactoryPath(pathlib.Path, PureArtifactoryPath): on regular constructors, but rather on templates. """ - # Pathlib limits what members can be present in 'Path' class, - # so authentication information has to be added via __slots__ - __slots__ = ("auth", "verify", "cert", "session", "timeout") - if sys.version_info.major == 3 and sys.version_info.minor >= 10: # see changes in pathlib.Path, slots are no more applied # https://github.com/python/cpython/blob/ce121fd8755d4db9511ce4aab39d0577165e118e/Lib/pathlib.py#L952 _accessor = _artifactory_accessor + else: + # Pathlib limits what members can be present in 'Path' class, + # so authentication information has to be added via __slots__ + __slots__ = ("auth", "verify", "cert", "session", "timeout") def __new__(cls, *args, **kwargs): """ From f2340e83682ad7bb7c151538e2166768da0dc7f4 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Thu, 7 Oct 2021 13:14:19 +0200 Subject: [PATCH 25/48] slots --- artifactory.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/artifactory.py b/artifactory.py index 300514a8..814594e1 100755 --- a/artifactory.py +++ b/artifactory.py @@ -1347,8 +1347,7 @@ class ArtifactoryPath(pathlib.Path, PureArtifactoryPath): # https://github.com/python/cpython/blob/ce121fd8755d4db9511ce4aab39d0577165e118e/Lib/pathlib.py#L952 _accessor = _artifactory_accessor else: - # Pathlib limits what members can be present in 'Path' class, - # so authentication information has to be added via __slots__ + # in 3.9 and below Pathlib limits what members can be present in 'Path' class __slots__ = ("auth", "verify", "cert", "session", "timeout") def __new__(cls, *args, **kwargs): From b7b86f9f20d117dc930a44dfe10a1add94662f81 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Fri, 8 Oct 2021 10:38:46 +0200 Subject: [PATCH 26/48] create_by_type fix --- dohq_artifactory/admin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index 5976a7c8..a82c94eb 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -682,7 +682,11 @@ class Repository(GenericRepository): V2 = "V2" @staticmethod - def create_by_type(repo_type: str, artifactory, name): + def create_by_type(repo_type: str, artifactory, name, *, type=None): + if type is not None: + deprecation("'type' argument is deprecated, use 'repo_type'") + repo_type = type + if repo_type == "LOCAL": return RepositoryLocal(artifactory, name) elif repo_type == "REMOTE": From cac3041e3faf7d3ce4702cda361daf944127739d Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Fri, 1 Oct 2021 16:02:43 +0200 Subject: [PATCH 27/48] use update properties via patch to eliminate URL limit, see #65 --- artifactory.py | 102 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 12 deletions(-) diff --git a/artifactory.py b/artifactory.py index 077e2ce6..fb2b4902 100755 --- a/artifactory.py +++ b/artifactory.py @@ -730,6 +730,38 @@ def rest_del(url, params=None, session=None, verify=True, cert=None, timeout=Non raise_for_status(response) return response + @staticmethod + def rest_patch( + url, + json_data=None, + params=None, + session=None, + verify=True, + cert=None, + timeout=None, + ): + """ + Perform a PATCH request to url with requests.session + :param url: url + :param json_data: (dict) JSON data to attach to patch request + :param params: request parameters + :param session: + :param verify: + :param cert: + :param timeout: + :return: request response object + """ + url = quote_url(url) + response = session.patch( + url=url, + json=json_data, + params=params, + verify=verify, + cert=cert, + timeout=timeout, + ) + return response + @staticmethod def rest_put_stream( url, @@ -1270,6 +1302,41 @@ def del_properties(self, pathobj, props, recursive): timeout=pathobj.timeout, ) + def update_properties(self, pathobj, properties, recursive=False): + """ + Update item properties + + Args: + pathobj: (ArtifactoryPath) object + properties: (dict) properties + recursive: (bool) apply recursively or not. For folders + + Returns: None + """ + url = "/".join( + [ + pathobj.drive.rstrip("/"), + "api/metadata", + str(pathobj.relative_to(pathobj.drive)).strip("/"), + ] + ) + + # construct data according to Artifactory format + json_data = {"props": properties} + + params = {"recursive": int(recursive)} + + response = self.rest_patch( + url, + json_data=json_data, + params=params, + session=pathobj.session, + verify=pathobj.verify, + cert=pathobj.cert, + timeout=pathobj.timeout, + ) + raise_for_status(response) + def scandir(self, pathobj): return _ScandirIter((pathobj.joinpath(x) for x in self.listdir(pathobj))) @@ -1983,12 +2050,15 @@ def properties(self): @properties.setter def properties(self, properties): properties_to_remove = set(self.properties) - set(properties) - if properties_to_remove: - self.del_properties(properties_to_remove, recursive=False) - self.set_properties(properties, recursive=False) + for prop in properties_to_remove: + properties[prop] = None + self.update_properties(properties=properties, recursive=False) @properties.deleter def properties(self): + """ + Delete properties + """ self.del_properties(self.properties, recursive=False) def set_properties(self, properties, recursive=True): @@ -2004,15 +2074,10 @@ def set_properties(self, properties, recursive=True): if not properties: return - # If URL > 13KB, nginx default raise error '414 Request-URI Too Large' - MAX_SIZE = 50 - if len(properties) > MAX_SIZE: - for chunk in chunks(properties, MAX_SIZE): - self._accessor.set_properties(self, chunk, recursive) - else: - self._accessor.set_properties(self, properties, recursive) + # Uses update properties since it can consume JSON as input and removes URL limit + self.update_properties(properties, recursive=recursive) - def del_properties(self, properties, recursive=None): + def del_properties(self, properties, recursive=False): """ Delete properties listed in properties @@ -2021,7 +2086,20 @@ def del_properties(self, properties, recursive=None): recursive - on folders property attachment is recursive by default. It is possible to force recursive behavior. """ - return self._accessor.del_properties(self, properties, recursive) + properties_to_remove = dict.fromkeys(properties, None) + # Uses update properties since it can consume JSON as input and removes URL limit + self.update_properties(properties_to_remove, recursive=recursive) + + def update_properties(self, properties, recursive=False): + """ + Update properties, set/update/remove item or folder properties + Args: + properties: (dict) data to be set + recursive: (bool) recursive on folder + + Returns: None + """ + return self._accessor.update_properties(self, properties, recursive) def aql(self, *args): """ From 3380b09433365b4e2681b2db04f1c2d837192763 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Fri, 1 Oct 2021 16:22:53 +0200 Subject: [PATCH 28/48] update tests --- requirements-dev.txt | 2 +- tests/unit/test_artifactory_path.py | 48 ++++++++++++++--------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0dd35f7a..f9ea1e10 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,4 +9,4 @@ flake8 pre-commit tox PyJWT -responses +responses>=0.14.0 diff --git a/tests/unit/test_artifactory_path.py b/tests/unit/test_artifactory_path.py index 9adc71cf..2d2eb76d 100644 --- a/tests/unit/test_artifactory_path.py +++ b/tests/unit/test_artifactory_path.py @@ -6,6 +6,8 @@ import dateutil import responses +from responses.matchers import json_params_matcher +from responses.matchers import query_param_matcher import artifactory from artifactory import ArtifactoryPath @@ -667,17 +669,22 @@ def test_set_properties(self): } path = self._mock_properties_response() - path.properties = properties - # Must delete only removed property - # rest delete is a second call, use index 1 - self.assertEqual(responses.calls[1].request.params["properties"], "removethis") + resp_props = properties.copy() + resp_props["removethis"] = None + self.assertNotEqual( + properties, resp_props + ) # ensure not update original properties - # Must put all property - self.assertEqual( - responses.calls[2].request.params["properties"], - "addthis=addthis;test=test_property;time=2018-01-16 12:17:44.135143", + responses.add( + responses.PATCH, + url="http://artifactory.local/artifactory/api/metadata/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", + match=[ + json_params_matcher({"props": resp_props}), + query_param_matcher({"recursive": "0"}), + ], ) + path.properties = properties @responses.activate def test_set_properties_without_remove(self): @@ -695,12 +702,17 @@ def test_set_properties_without_remove(self): } path = self._mock_properties_response() - path.properties = properties - self.assertEqual( - responses.calls[1].request.params["properties"], - "addthis=addthis;removethis=removethis_property;test=test_property;time=2018-01-16 12:17:44.135143", + responses.add( + responses.PATCH, + url="http://artifactory.local/artifactory/api/metadata/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", + match=[ + json_params_matcher({"props": properties}), + query_param_matcher({"recursive": "0"}), + ], ) + path.properties = properties + @staticmethod def _mock_properties_response(): """ @@ -730,18 +742,6 @@ def _mock_properties_response(): "uri": constructed_url, }, ) - responses.add( - responses.DELETE, - constructed_url, - status=204, - body="", - ) - responses.add( - responses.PUT, - constructed_url, - status=204, - body="", - ) return path @responses.activate From 3cc80bfbefd7b5de32eca1d9cb25429c77dda0d7 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Fri, 15 Oct 2021 15:19:08 +0200 Subject: [PATCH 29/48] fix repo type --- dohq_artifactory/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index a82c94eb..92bbed4a 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -682,7 +682,7 @@ class Repository(GenericRepository): V2 = "V2" @staticmethod - def create_by_type(repo_type: str, artifactory, name, *, type=None): + def create_by_type(repo_type="LOCAL", artifactory=None, name=None, *, type=None): if type is not None: deprecation("'type' argument is deprecated, use 'repo_type'") repo_type = type From 885f6cfab760af16430fe29eda1513484ed719b6 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 26 Oct 2021 08:45:03 +0200 Subject: [PATCH 30/48] Create ci.yml --- .github/workflows/ci.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..df7cf873 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + branches: + - '*' + +jobs: + tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.6', '3.7', '3.8'] + + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 1 + + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements-dev.txt + + - name: Unittests + run: | + python -m unittest discover tests/unit -v From 9d5d1eb5c3c12c4764c744998b81d451cfe7beb8 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Sat, 30 Oct 2021 21:30:53 +0200 Subject: [PATCH 31/48] delete travis configs --- .travis.yml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 56b5f735..00000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -sudo: false -language: python -python: - - "3.6" - - "3.7" - - "3.8" -install: pip install tox-travis -script: tox -deploy: - provider: pypi - user: devopshq - password: - secure: "H3lHdZhgPw9q2IiM1ck/6AAXSpg6GZW+ug50/rKP0UVpKdpBDedhtoylWK480bmmGGGgUjDnFNwssCoq8AdBjE/LJ3hcOts6c8XpcUejL/DxGy+i0Ef3CtsdspvsisUn3g5rpLGEH9wr3m5fWi6M2231Xj85LDaAwdDTXHuFN3ZrIAPEkGFKcUVpivSczonAqLY8BE/l6HNSHaluGh6HB1JiHZStebfTTUvXAmMMcBojQxo5LHPK04KgChQ+0uJdmbDA91luKrjYX+cqKDzYaa7T0GieDd1E6jCQtam1sQxH2qDffbWRZsA+bfMoZjFgBcvwPp1hHaKoiJpJTAltd5IsLYzPxpbc079zABq0WSQ632k8Kt+9CPvJ5PbYf50AOGguZViie/1bZ3akj6rb1ycpBPDcWOAJuDS7Tifm1M4QCRd+QuDVOsj5xu+NkLVs037p/afPfNneeYWJYyyOkdVMfiC1AB9L5OppakS4q2DSwY49sTkySXY2gOZCjFlcXZkTW4PWkD9Ie/ulimRtWKyw/CvCZqa6goYWAhlrJE0+xM+/4xAr2b6KRutD4Jd3UzdM3uRRkvGSCfdU79uNTc3IQbECsPTWBrlQ34yL1et/5UnqNxSVm1XXFE+pPoaEFezih7Ek1AZ4ewTxigJLmXvcbA2dz9r6svQbaNY5G3c=" - distributions: sdist bdist_wheel - skip_cleanup: true - on: - python: '3.7' - branch: - - master - - develop From 4735144501058133e47a2d4b0715217f4ffd3f41 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Sat, 30 Oct 2021 21:32:38 +0200 Subject: [PATCH 32/48] remove pre-commit hook --- .pre-commit-config.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 924f3258..17719744 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,8 +33,4 @@ repos: files: FAQ.md$ - id: markdown-toc name: CONTRIBUTE.md - files: CONTRIBUTE.md$ - - repo: https://github.com/alphagov/verify-travis-pre-commit-hook - rev: 2d69b57aa5e58dd31c01002ae4abcdf9b54549a1 - hooks: - - id: travis-yml-lint + files: CONTRIBUTE.md$ \ No newline at end of file From d5b028263fba6aa11fc19151ad41dc24b54be5f8 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 9 Nov 2021 12:45:31 +0100 Subject: [PATCH 33/48] update ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df7cf873..be1c40e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v1 From 3a66de0b1957e576c513ea86c4ee66be6497cedc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Nov 2021 21:33:38 +0000 Subject: [PATCH 34/48] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 21.8b0 → 21.11b1](https://github.com/psf/black/compare/21.8b0...21.11b1) - [github.com/asottile/blacken-docs: v1.11.0 → v1.12.0](https://github.com/asottile/blacken-docs/compare/v1.11.0...v1.12.0) - [github.com/pycqa/flake8: 3.9.2 → 4.0.1](https://github.com/pycqa/flake8/compare/3.9.2...4.0.1) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 924f3258..576e1b39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,12 +5,12 @@ repos: - id: autoflake args: ['-i', '--remove-all-unused-imports'] - repo: https://github.com/psf/black - rev: 21.8b0 + rev: 21.11b1 hooks: - id: black language_version: python3 - repo: https://github.com/asottile/blacken-docs - rev: v1.11.0 + rev: v1.12.0 hooks: - id: blacken-docs additional_dependencies: [black] @@ -19,7 +19,7 @@ repos: hooks: - id: reorder-python-imports - repo: https://github.com/pycqa/flake8 - rev: 3.9.2 + rev: 4.0.1 hooks: - id: flake8 - repo: https://github.com/Lucas-C/pre-commit-hooks-nodejs From ea405493d6e707c33770f98e2ed1dd92b3d9ebf3 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 23 Nov 2021 11:15:02 +0100 Subject: [PATCH 35/48] add workflow to publish on PyPi when new version is released --- .github/workflows/deploy.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..3211e1cf --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,32 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + + +name: Upload Python Package + +on: + release: + types: [published] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} From 255f0095dee3621d7c2a2db9399998b0aeb968d0 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 23 Nov 2021 12:08:07 +0100 Subject: [PATCH 36/48] update yml --- .github/workflows/deploy.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3211e1cf..25a3997b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,12 +1,12 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - +# This workflow will be triggered when new TAG is pushed. It will create version.txt file with tag name +# Then workflow will publish a package to PyPi with version from version.txt name: Upload Python Package on: - release: - types: [published] + push: + tags: + - '*' jobs: deploy: @@ -23,6 +23,15 @@ jobs: run: | python -m pip install --upgrade pip pip install build + + - name: Set version + run: | + echo ${{ github.ref_name }} > version.txt + + - name: Test version.txt file + run: | + python -m unittest tests.test_version + - name: Build package run: python -m build - name: Publish package From 52124da55f9d5515a15f743d148f120c742c1d51 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 23 Nov 2021 12:08:55 +0100 Subject: [PATCH 37/48] add version test --- tests/test_version.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/test_version.py diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 00000000..cb1c8c51 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,11 @@ +import os +import unittest + + +class TestVersion(unittest.TestCase): + def test_file(self): + self.assertTrue(os.path.isfile("version.txt")) + with open("version.txt") as file: + print() + print() + print("Version is", file.readlines()[0]) From cc33a557d31417790bcd6728efd28c7a14fce0ce Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 23 Nov 2021 12:22:27 +0100 Subject: [PATCH 38/48] add test to check regex update setup.py --- setup.py | 35 +++++++---------------------------- tests/test_version.py | 10 +++++++--- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/setup.py b/setup.py index b8c21ca7..49d13692 100755 --- a/setup.py +++ b/setup.py @@ -6,37 +6,16 @@ from setuptools import setup except ImportError: from distutils.core import setup -import os -__version__ = "0.7" -devStatus = "4 - Beta" # default build status, see: https://pypi.python.org/pypi?%3Aaction=list_classifiers - -if "TRAVIS_BUILD_NUMBER" in os.environ and "TRAVIS_BRANCH" in os.environ: - print("This is TRAVIS-CI build") - print("TRAVIS_BUILD_NUMBER = {}".format(os.environ["TRAVIS_BUILD_NUMBER"])) - print("TRAVIS_BRANCH = {}".format(os.environ["TRAVIS_BRANCH"])) - - __version__ += ".{}{}".format( - "" - if "release" in os.environ["TRAVIS_BRANCH"] - or os.environ["TRAVIS_BRANCH"] == "master" - else "dev", - os.environ["TRAVIS_BUILD_NUMBER"], - ) - - if ( - "release" in os.environ["TRAVIS_BRANCH"] - or os.environ["TRAVIS_BRANCH"] == "master" - ): - devStatus = "5 - Production/Stable" - - else: - devStatus = devStatus +with open("version.txt") as file: + __version__ = next(file).strip() +# default build status, see: https://pypi.python.org/pypi?%3Aaction=list_classifiers +if "dev" in __version__: + dev_status = "4 - Beta" else: - print("This is local build") - __version__ += ".dev0" # set version as major.minor.localbuild if local build: python setup.py install + dev_status = "5 - Production/Stable" setup( @@ -49,7 +28,7 @@ author="Alexey Burov", author_email="aburov@ptsecurity.com", classifiers=[ - "Development Status :: {}".format(devStatus), + "Development Status :: {}".format(dev_status), "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", diff --git a/tests/test_version.py b/tests/test_version.py index cb1c8c51..bb71e0f7 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -5,7 +5,11 @@ class TestVersion(unittest.TestCase): def test_file(self): self.assertTrue(os.path.isfile("version.txt")) + + def test_version_regex(self): with open("version.txt") as file: - print() - print() - print("Version is", file.readlines()[0]) + version = next(file).strip() + print("\n\nVersion is", version) + + # check that version matches vX.X.X or vX.X.X.devXXX + self.assertRegex(version, r"\d\.\d\.\d$|\d\.\d\.\d\.dev\d+$") From a5c514f9b8a7d69782285559537561f79ceef396 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 23 Nov 2021 12:44:34 +0100 Subject: [PATCH 39/48] refactor --- setup.py | 5 ++++- tests/test_version.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 49d13692..1b861cb2 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - +import re try: from setuptools import setup @@ -11,6 +11,9 @@ with open("version.txt") as file: __version__ = next(file).strip() +# just double check that version is correct +assert re.match(r"^v\d\.\d\.\d$|^v\d\.\d\.\d\.dev\d+$", __version__) + # default build status, see: https://pypi.python.org/pypi?%3Aaction=list_classifiers if "dev" in __version__: dev_status = "4 - Beta" diff --git a/tests/test_version.py b/tests/test_version.py index bb71e0f7..73ba92c6 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -1,4 +1,5 @@ import os +import re import unittest @@ -12,4 +13,4 @@ def test_version_regex(self): print("\n\nVersion is", version) # check that version matches vX.X.X or vX.X.X.devXXX - self.assertRegex(version, r"\d\.\d\.\d$|\d\.\d\.\d\.dev\d+$") + assert re.match(r"^v\d\.\d\.\d$|^v\d\.\d\.\d\.dev\d+$", version) From 196fb42dfb3613f2ce48bf533a823e4b1c091808 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 23 Nov 2021 12:45:35 +0100 Subject: [PATCH 40/48] remove pathlib from deps --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 1b861cb2..6a3af0ae 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,6 @@ url="https://devopshq.github.io/artifactory/", download_url="https://github.com/devopshq/artifactory", install_requires=[ - 'pathlib ; python_version<"3.4"', "requests", "python-dateutil", "PyJWT", From f88726bee37f60c021eeaf5d9327188b6fda004f Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 23 Nov 2021 12:54:10 +0100 Subject: [PATCH 41/48] update MANIFEST.in --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index bb3ec5f0..ee1e33b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include README.md +include version.txt From b196ed03195faeb611e6a39d6238dca974d83ec8 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 23 Nov 2021 13:19:44 +0100 Subject: [PATCH 42/48] update requirements-dev.txt --- requirements-dev.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f9ea1e10..07b24519 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,9 @@ --e . - -pathlib; python_version<"3.4" requests python-dateutil +PyJWT pytest mock flake8 pre-commit tox -PyJWT responses>=0.14.0 From 0556d996b8d768789d526b1f6c60863d49c8e141 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Mon, 29 Nov 2021 11:01:56 +0100 Subject: [PATCH 43/48] remove unittests of version add bash check that file exists add default version for pip install git+ --- .github/workflows/deploy.yml | 5 ++++- setup.py | 15 ++++++++++----- tests/test_version.py | 16 ---------------- 3 files changed, 14 insertions(+), 22 deletions(-) delete mode 100644 tests/test_version.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 25a3997b..90da41df 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -30,7 +30,10 @@ jobs: - name: Test version.txt file run: | - python -m unittest tests.test_version + if [ ! -f version.txt ]; then + echo "version.txt does not exist" + exit 1 + fi - name: Build package run: python -m build diff --git a/setup.py b/setup.py index 6a3af0ae..92c441e5 100755 --- a/setup.py +++ b/setup.py @@ -8,14 +8,19 @@ from distutils.core import setup -with open("version.txt") as file: - __version__ = next(file).strip() +try: + with open("version.txt") as file: + __version__ = file.readline().strip() -# just double check that version is correct -assert re.match(r"^v\d\.\d\.\d$|^v\d\.\d\.\d\.dev\d+$", __version__) + # check that version is correct (vX.X.X or vX.X.X.devXXX), eg v0.8.0.dev0 + assert re.match(r"^v\d\.\d\.\d$|^v\d\.\d\.\d\.dev\d+$", __version__) +except FileNotFoundError: + # when user installs via pip install git+https://github.com/devopshq/artifactory.git + # this should be discouraged, publish dev package instead + __version__ = "0.8.0.alpha0" # default build status, see: https://pypi.python.org/pypi?%3Aaction=list_classifiers -if "dev" in __version__: +if "dev" in __version__ or "alpha" in __version__: dev_status = "4 - Beta" else: dev_status = "5 - Production/Stable" diff --git a/tests/test_version.py b/tests/test_version.py deleted file mode 100644 index 73ba92c6..00000000 --- a/tests/test_version.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import re -import unittest - - -class TestVersion(unittest.TestCase): - def test_file(self): - self.assertTrue(os.path.isfile("version.txt")) - - def test_version_regex(self): - with open("version.txt") as file: - version = next(file).strip() - print("\n\nVersion is", version) - - # check that version matches vX.X.X or vX.X.X.devXXX - assert re.match(r"^v\d\.\d\.\d$|^v\d\.\d\.\d\.dev\d+$", version) From afe8b773cab19c597d58b1f9e31ece3229951718 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 30 Nov 2021 10:27:10 +0100 Subject: [PATCH 44/48] remove v from version --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 92c441e5..14ae686b 100755 --- a/setup.py +++ b/setup.py @@ -12,11 +12,10 @@ with open("version.txt") as file: __version__ = file.readline().strip() - # check that version is correct (vX.X.X or vX.X.X.devXXX), eg v0.8.0.dev0 - assert re.match(r"^v\d\.\d\.\d$|^v\d\.\d\.\d\.dev\d+$", __version__) + # check that version is correct (X.X.X or X.X.X.devXXX), eg 0.8.0.dev0 + assert re.match(r"^\d\.\d\.\d$|^\d\.\d\.\d\.dev\d+$", __version__) except FileNotFoundError: # when user installs via pip install git+https://github.com/devopshq/artifactory.git - # this should be discouraged, publish dev package instead __version__ = "0.8.0.alpha0" # default build status, see: https://pypi.python.org/pypi?%3Aaction=list_classifiers From 682d65fe0f84a599362f93699dab7d80cf2c8aaa Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 30 Nov 2021 12:28:47 +0100 Subject: [PATCH 45/48] add default version.txt --- .github/workflows/deploy.yml | 1 + setup.py | 19 ++++++++++--------- version.txt | 1 + 3 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 version.txt diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 90da41df..634a9b8d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,6 +26,7 @@ jobs: - name: Set version run: | + rm -f version.txt echo ${{ github.ref_name }} > version.txt - name: Test version.txt file diff --git a/setup.py b/setup.py index 14ae686b..86545a66 100755 --- a/setup.py +++ b/setup.py @@ -8,18 +8,19 @@ from distutils.core import setup -try: - with open("version.txt") as file: - __version__ = file.readline().strip() +with open("version.txt") as file: + __version__ = file.readline().strip() + +# check that version is correct (X.X.X or X.X.X.devXXX or X.X.X.alphaX), eg 0.8.0.dev0 +assert re.match( + r"^\d\.\d\.\d$|^\d\.\d\.\d\.dev\d+$|^\d\.\d\.\d\.alpha\d+$", __version__ +) - # check that version is correct (X.X.X or X.X.X.devXXX), eg 0.8.0.dev0 - assert re.match(r"^\d\.\d\.\d$|^\d\.\d\.\d\.dev\d+$", __version__) -except FileNotFoundError: - # when user installs via pip install git+https://github.com/devopshq/artifactory.git - __version__ = "0.8.0.alpha0" # default build status, see: https://pypi.python.org/pypi?%3Aaction=list_classifiers -if "dev" in __version__ or "alpha" in __version__: +if "alpha" in __version__: + dev_status = "3 - Alpha" +elif "dev" in __version__: dev_status = "4 - Beta" else: dev_status = "5 - Production/Stable" diff --git a/version.txt b/version.txt new file mode 100644 index 00000000..42ff8542 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +0.8.0.alpha0 \ No newline at end of file From c66c8c69d21b043b6a0a67bb9cf9cf22617ef094 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 30 Nov 2021 17:55:08 +0100 Subject: [PATCH 46/48] added repo name to stats, may be useful to see if it is cached or not --- artifactory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/artifactory.py b/artifactory.py index 28195497..f6c3dfd7 100755 --- a/artifactory.py +++ b/artifactory.py @@ -599,6 +599,7 @@ def _get_base_url(self, url): "md5", "is_dir", "children", + "repo", ], ) @@ -862,6 +863,7 @@ def stat(self, pathobj): md5=checksums.get("md5", None), is_dir=is_dir, children=children, + repo=jsn.get("repo", None), ) return stat From 3d6fdb9fad1b720661482ba157133d53b0f60e8b Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Tue, 30 Nov 2021 18:31:13 +0100 Subject: [PATCH 47/48] retrieve download count --- artifactory.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/artifactory.py b/artifactory.py index f6c3dfd7..a8e01780 100755 --- a/artifactory.py +++ b/artifactory.py @@ -603,6 +603,18 @@ def _get_base_url(self, url): ], ) +ArtifactoryDownloadStat = collections.namedtuple( + "ArtifactoryDownloadStat", + [ + "last_downloaded", + "download_count", + "last_downloaded_by", + "remote_download_count", + "remote_last_downloaded", + "uri", + ], +) + class _ScandirIter: """ @@ -805,11 +817,18 @@ def rest_get_stream( raise_for_status(response) return response - def get_stat_json(self, pathobj): + def get_stat_json(self, pathobj, key=None): """ Request remote file/directory status info Returns a json object as specified by Artifactory REST API + Args: + pathobj: ArtifactoryPath for which we request data + key: (str) (optional) additional key to specify query, eg 'stats', 'lastModified' + + Returns: + (dict) stat dictionary """ + url = "/".join( [ pathobj.drive.rstrip("/"), @@ -824,6 +843,7 @@ def get_stat_json(self, pathobj): verify=pathobj.verify, cert=pathobj.cert, timeout=pathobj.timeout, + params=key, ) code = response.status_code text = response.text @@ -868,6 +888,24 @@ def stat(self, pathobj): return stat + def download_stats(self, pathobj): + jsn = self.get_stat_json(pathobj, key="stats") + + # divide timestamp by 1000 since it is provided in ms + download_time = datetime.datetime.fromtimestamp( + jsn.get("lastDownloaded", 0) / 1000 + ) + stat = ArtifactoryDownloadStat( + last_downloaded=download_time, + last_downloaded_by=jsn.get("lastDownloadedBy", None), + download_count=jsn.get("downloadCount", None), + remote_download_count=jsn.get("remoteDownloadCount", None), + remote_last_downloaded=jsn.get("remoteLastDownloaded", None), + uri=jsn.get("uri", None), + ) + + return stat + def is_dir(self, pathobj): """ Returns True if given path is a directory @@ -1540,6 +1578,18 @@ def stat(self, pathobj=None): pathobj = pathobj or self return self._accessor.stat(pathobj=pathobj) + def download_stats(self, pathobj=None): + """ + Item statistics record the number of times an item was downloaded, last download date and last downloader. + Args: + pathobj: (optional) path object for which to retrieve stats + + Returns: + + """ + pathobj = pathobj or self + return self._accessor.download_stats(pathobj=pathobj) + def with_name(self, name): """ Return a new path with the file name changed. From 4d81328dd70bd30b8f3537b70231368f4cb88198 Mon Sep 17 00:00:00 2001 From: Maksim Beliaev Date: Thu, 9 Dec 2021 10:04:01 +0100 Subject: [PATCH 48/48] fixed examples for stats replace ~~~ by ``` for autotooling --- README.md | 67 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 9860ed7c..a16d1b66 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,9 @@ package. * [Artifact properties](#artifact-properties) * [Repository Scheduled Replication Status](#repository-scheduled-replication-status) * [Artifactory Query Language](#artifactory-query-language) - * [FileStat](#filestat) + * [Artifact Stat](#artifact-stat) + + [File/Folder Statistics](#filefolder-statistics) + + [Get Download Statistics](#get-download-statistics) * [Promote Docker image](#promote-docker-image) * [Builds](#builds) * [Exception handling](#exception-handling) @@ -586,8 +588,9 @@ artifact_pathlib_list = list(map(arti_path.from_aql, artifacts_list)) ``` -## FileStat -You can get hash (`md5`, `sha1`, `sha256`), create date, and change date: +## Artifact Stat +### File/Folder Statistics +You can get hash (`md5`, `sha1`, `sha256`), creator, create date, and change date: ```python from artifactory import ArtifactoryPath @@ -605,6 +608,27 @@ print(stat.sha256) print(stat.ctime) print(stat.is_dir) print(stat.size) +print(stat.created_by) +``` + +### Get Download Statistics +Information about number of downloads, user that last downloaded and date of last download +```python +from artifactory import ArtifactoryPath + +path = ArtifactoryPath( + "http://repo.jfrog.org/artifactory/distributions/org/apache/tomcat/apache-tomcat-7.0.11.tar.gz" +) + +# Get FileStat +download_stat = path.download_stats() +print(download_stat) +print(download_stat.last_downloaded) +print(download_stat.last_downloaded_by) +print(download_stat.download_count) +print(download_stat.remote_download_count) +print(download_stat.remote_last_downloaded) +print(download_stat.uri) ``` ## Promote Docker image @@ -618,10 +642,12 @@ path.promote_docker_image("docker-staging", "docker-prod", "my-application", "0. ``` ## Builds -~~~python +```python from artifactory import ArtifactoryBuildManager -arti_build = ArtifactoryBuildManager("https://repo.jfrog.org/artifactory", project="proj_name", auth=("admin", "admin")) +arti_build = ArtifactoryBuildManager( + "https://repo.jfrog.org/artifactory", project="proj_name", auth=("admin", "admin") +) # Get all builds all_builds = arti_build.builds @@ -651,22 +677,21 @@ print(build_number1.diff(3)) All artifacts from all scopes are included by default while dependencies are not. Scopes are additive (or) """ -build_number1.promote(ci_user="admin", properties={ - "components": ["c1","c3","c14"], - "release-name": ["fb3-ga"] - }) -~~~ +build_number1.promote( + ci_user="admin", + properties={"components": ["c1", "c3", "c14"], "release-name": ["fb3-ga"]}, +) +``` ## Exception handling Exceptions in this library are represented by `dohq_artifactory.exception.ArtifactoryException` or by `OSError` If exception was caused by HTTPError you can always drill down the root cause by using following example: -~~~python +```python from artifactory import ArtifactoryPath from dohq_artifactory.exception import ArtifactoryException path = ArtifactoryPath( - "http://my_arti:8080/artifactory/installer/", - auth=("wrong_user", "wrong_pass") + "http://my_arti:8080/artifactory/installer/", auth=("wrong_user", "wrong_pass") ) try: @@ -674,9 +699,11 @@ try: except ArtifactoryException as exc: print(exc) # clean artifactory error message # >>> Bad credentials - print(exc.__cause__) # HTTP error that triggered exception, you can use this object for more info + print( + exc.__cause__ + ) # HTTP error that triggered exception, you can use this object for more info # >>> 401 Client Error: Unauthorized for url: http://my_arti:8080/artifactory/installer/ -~~~ +``` # Admin area You can manipulate with user\group\repository and permission. First, create `ArtifactoryPath` object without a repository @@ -728,7 +755,7 @@ user.delete() ``` ### API Keys -~~~python +```python from dohq_artifactory import User user = User(artifactory_, "username") @@ -751,7 +778,7 @@ user.api_key.revoke() # remove all API keys in system, only if user has admin rights user.api_key.revoke_for_all_users() -~~~ +``` ## Group ### Internal @@ -1361,18 +1388,18 @@ path = ArtifactoryPath( The library can be configured to emit logging that will give you better insight into what it's doing. Just configure [logging](https://docs.python.org/3/library/logging.html) module in your python script. Simplest example to add debug messages to a console: -~~~python +```python import logging from artifactory import ArtifactoryPath logging.basicConfig() # set level only for artifactory module, if omitted, then global log level is used, eg from basicConfig -logging.getLogger('artifactory').setLevel(logging.DEBUG) +logging.getLogger("artifactory").setLevel(logging.DEBUG) path = ArtifactoryPath( "http://my-artifactory/artifactory/myrepo/restricted-path", apikey="MY_API_KEY" ) -~~~ +``` ## Global Configuration File ##