diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..be1c40e7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + branches: + - '*' + +jobs: + tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 1 + + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements-dev.txt + + - name: Unittests + run: | + python -m unittest discover tests/unit -v diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..634a9b8d --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,45 @@ +# This workflow will be triggered when new TAG is pushed. It will create version.txt file with tag name +# Then workflow will publish a package to PyPi with version from version.txt + +name: Upload Python Package + +on: + push: + tags: + - '*' + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + + - name: Set version + run: | + rm -f version.txt + echo ${{ github.ref_name }} > version.txt + + - name: Test version.txt file + run: | + if [ ! -f version.txt ]; then + echo "version.txt does not exist" + exit 1 + fi + + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 924f3258..d2a41c5f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,12 +5,12 @@ repos: - id: autoflake args: ['-i', '--remove-all-unused-imports'] - repo: https://github.com/psf/black - rev: 21.8b0 + rev: 21.11b1 hooks: - id: black language_version: python3 - repo: https://github.com/asottile/blacken-docs - rev: v1.11.0 + rev: v1.12.0 hooks: - id: blacken-docs additional_dependencies: [black] @@ -19,7 +19,7 @@ repos: hooks: - id: reorder-python-imports - repo: https://github.com/pycqa/flake8 - rev: 3.9.2 + rev: 4.0.1 hooks: - id: flake8 - repo: https://github.com/Lucas-C/pre-commit-hooks-nodejs @@ -33,8 +33,4 @@ repos: files: FAQ.md$ - id: markdown-toc name: CONTRIBUTE.md - files: CONTRIBUTE.md$ - - repo: https://github.com/alphagov/verify-travis-pre-commit-hook - rev: 2d69b57aa5e58dd31c01002ae4abcdf9b54549a1 - hooks: - - id: travis-yml-lint + files: CONTRIBUTE.md$ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 56b5f735..00000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -sudo: false -language: python -python: - - "3.6" - - "3.7" - - "3.8" -install: pip install tox-travis -script: tox -deploy: - provider: pypi - user: devopshq - password: - secure: "H3lHdZhgPw9q2IiM1ck/6AAXSpg6GZW+ug50/rKP0UVpKdpBDedhtoylWK480bmmGGGgUjDnFNwssCoq8AdBjE/LJ3hcOts6c8XpcUejL/DxGy+i0Ef3CtsdspvsisUn3g5rpLGEH9wr3m5fWi6M2231Xj85LDaAwdDTXHuFN3ZrIAPEkGFKcUVpivSczonAqLY8BE/l6HNSHaluGh6HB1JiHZStebfTTUvXAmMMcBojQxo5LHPK04KgChQ+0uJdmbDA91luKrjYX+cqKDzYaa7T0GieDd1E6jCQtam1sQxH2qDffbWRZsA+bfMoZjFgBcvwPp1hHaKoiJpJTAltd5IsLYzPxpbc079zABq0WSQ632k8Kt+9CPvJ5PbYf50AOGguZViie/1bZ3akj6rb1ycpBPDcWOAJuDS7Tifm1M4QCRd+QuDVOsj5xu+NkLVs037p/afPfNneeYWJYyyOkdVMfiC1AB9L5OppakS4q2DSwY49sTkySXY2gOZCjFlcXZkTW4PWkD9Ie/ulimRtWKyw/CvCZqa6goYWAhlrJE0+xM+/4xAr2b6KRutD4Jd3UzdM3uRRkvGSCfdU79uNTc3IQbECsPTWBrlQ34yL1et/5UnqNxSVm1XXFE+pPoaEFezih7Ek1AZ4ewTxigJLmXvcbA2dz9r6svQbaNY5G3c=" - distributions: sdist bdist_wheel - skip_cleanup: true - on: - python: '3.7' - branch: - - master - - develop diff --git a/MANIFEST.in b/MANIFEST.in index bb3ec5f0..ee1e33b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include README.md +include version.txt diff --git a/README.md b/README.md index 7bda380e..a16d1b66 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,12 @@ package. * [Artifact properties](#artifact-properties) * [Repository Scheduled Replication Status](#repository-scheduled-replication-status) * [Artifactory Query Language](#artifactory-query-language) - * [FileStat](#filestat) + * [Artifact Stat](#artifact-stat) + + [File/Folder Statistics](#filefolder-statistics) + + [Get Download Statistics](#get-download-statistics) * [Promote Docker image](#promote-docker-image) * [Builds](#builds) + * [Exception handling](#exception-handling) - [Admin area](#admin-area) * [User](#user) + [API Keys](#api-keys) @@ -63,7 +66,7 @@ package. * [Session](#session) * [SSL Cert Verification Options](#ssl-cert-verification-options) * [Timeout on requests](#timeout-on-requests) - * [Troubleshooting](#troubleshooting) + * [Logging](#logging) * [Global Configuration File](#global-configuration-file) - [Contribute](#contribute) - [Advertising](#advertising) @@ -85,7 +88,6 @@ pip install dohq-artifactory==0.5.dev243 ``` # Usage - ## Authentication ## `dohq-artifactory` supports these ways of authentication: @@ -298,6 +300,37 @@ path.deploy_file( ) ``` +[Deploy artifact by checksum](https://www.jfrog.com/confluence/display/RTF6X/Artifactory+REST+API#ArtifactoryRESTAPI-DeployArtifactbyChecksum): deploy an artifact to the specified destination by checking if the artifact +content already exists in Artifactory. If Artifactory already contains a user +readable artifact with the same checksum the artifact content is copied over to +the new location without requiring content transfer. + +```python +from artifactory import ArtifactoryPath + +path = ArtifactoryPath("http://my-artifactory/artifactory/my_repo/foo") +sha1 = "1be5d2dbe52ddee96ef2d17d354e2be0a155a951" +sha256 = "00bbf80ccca376893d60183e1a714e707fd929aea3e458f9ffda60f7ae75cc51" + +# If you don't know sha value, you can calculate it via +# sha1 = artifactory.sha1sum("local_path_of_your_file") +# or +# sha256 = artifactory.sha256sum("local_path_of_your_file") + +# Each of the following 4 methods works fine if the artifact content already +# exists in Artifactory. +path.deploy_by_checksum(sha1=sha1) + +# deploy by sha1 via checksum parameter +path.deploy_by_checksum(checksum=sha1) + +# deploy by sha256 via sha256 parameter +path.deploy_by_checksum(sha256=sha256) + +# deploy by sha256 via checksum parameter +path.deploy_by_checksum(checksum=sha256) +``` + Deploy a debian package ```myapp-1.0.deb``` to an ```existent``` folder ```python @@ -387,6 +420,9 @@ http://example.com/artifactory/published/production/foo-0.0.1.pom http://example.com/artifactory/published/production/product-1.0.0.tar.gz http://example.com/artifactory/published/production/product-1.0.0.tar.pom """ + +# you can use dry run just to check if command will succeed without real change, adds debug message +source.copy(dest, dry_run=True) ``` ## Move Artifacts @@ -405,6 +441,9 @@ source = ArtifactoryPath("http://example.com/artifactory/builds/product/product/ dest = ArtifactoryPath("http://example.com/artifactory/published/production/") source.move(dest) + +# you can use dry run just to check if command will succeed without real change, adds debug message +source.move(dest, dry_run=True) ``` ## Remove Artifacts @@ -474,19 +513,19 @@ You can use [Artifactory Query Language](https://www.jfrog.com/confluence/displa ```python from artifactory import ArtifactoryPath -aql = ArtifactoryPath( +arti_path = ArtifactoryPath( "http://my-artifactory/artifactory" ) # path to artifactory, NO repo # dict support # Send query: # items.find({"repo": "myrepo"}) -artifacts = aql.aql("items.find", {"repo": "myrepo"}) +artifacts = arti_path.aql("items.find", {"repo": "myrepo"}) # list support. # Send query: # items.find().include("name", "repo") -artifacts = aql.aql("items.find()", ".include", ["name", "repo"]) +artifacts = arti_path.aql("items.find()", ".include", ["name", "repo"]) # support complex query # Example 1 @@ -498,7 +537,7 @@ artifacts = aql.aql("items.find()", ".include", ["name", "repo"]) # ] # } # ) -args = [ +aqlargs = [ "items.find", { "$and": [ @@ -515,7 +554,7 @@ args = [ # artifacts_list contains raw data (list of dict) # Send query -artifacts_list = aql.aql(*args) +artifacts_list = arti_path.aql(*aqlargs) # Example 2 # The query will find all items in repo docker-prod that are of type file and were created after timecode. The @@ -541,16 +580,17 @@ aqlargs = [ ".sort", {"$asc": ["repo", "path", "name"]}, ] -artifacts_list = aql.aql(*args) +artifacts_list = arti_path.aql(*aqlargs) # You can convert to pathlib object: -artifact_pathlib = map(aql.from_aql, artifacts_list) -artifact_pathlib_list = list(map(aql.from_aql, artifacts_list)) +artifact_pathlib = map(arti_path.from_aql, artifacts_list) +artifact_pathlib_list = list(map(arti_path.from_aql, artifacts_list)) ``` -## FileStat -You can get hash (`md5`, `sha1`, `sha256`), create date, and change date: +## Artifact Stat +### File/Folder Statistics +You can get hash (`md5`, `sha1`, `sha256`), creator, create date, and change date: ```python from artifactory import ArtifactoryPath @@ -568,6 +608,27 @@ print(stat.sha256) print(stat.ctime) print(stat.is_dir) print(stat.size) +print(stat.created_by) +``` + +### Get Download Statistics +Information about number of downloads, user that last downloaded and date of last download +```python +from artifactory import ArtifactoryPath + +path = ArtifactoryPath( + "http://repo.jfrog.org/artifactory/distributions/org/apache/tomcat/apache-tomcat-7.0.11.tar.gz" +) + +# Get FileStat +download_stat = path.download_stats() +print(download_stat) +print(download_stat.last_downloaded) +print(download_stat.last_downloaded_by) +print(download_stat.download_count) +print(download_stat.remote_download_count) +print(download_stat.remote_last_downloaded) +print(download_stat.uri) ``` ## Promote Docker image @@ -581,10 +642,12 @@ path.promote_docker_image("docker-staging", "docker-prod", "my-application", "0. ``` ## Builds -~~~python +```python from artifactory import ArtifactoryBuildManager -arti_build = ArtifactoryBuildManager("https://repo.jfrog.org/artifactory", project="proj_name", auth=("admin", "admin")) +arti_build = ArtifactoryBuildManager( + "https://repo.jfrog.org/artifactory", project="proj_name", auth=("admin", "admin") +) # Get all builds all_builds = arti_build.builds @@ -614,11 +677,33 @@ print(build_number1.diff(3)) All artifacts from all scopes are included by default while dependencies are not. Scopes are additive (or) """ -build_number1.promote(ci_user="admin", properties={ - "components": ["c1","c3","c14"], - "release-name": ["fb3-ga"] - }) -~~~ +build_number1.promote( + ci_user="admin", + properties={"components": ["c1", "c3", "c14"], "release-name": ["fb3-ga"]}, +) +``` + +## Exception handling +Exceptions in this library are represented by `dohq_artifactory.exception.ArtifactoryException` or by `OSError` +If exception was caused by HTTPError you can always drill down the root cause by using following example: +```python +from artifactory import ArtifactoryPath +from dohq_artifactory.exception import ArtifactoryException + +path = ArtifactoryPath( + "http://my_arti:8080/artifactory/installer/", auth=("wrong_user", "wrong_pass") +) + +try: + path.stat() +except ArtifactoryException as exc: + print(exc) # clean artifactory error message + # >>> Bad credentials + print( + exc.__cause__ + ) # HTTP error that triggered exception, you can use this object for more info + # >>> 401 Client Error: Unauthorized for url: http://my_arti:8080/artifactory/installer/ +``` # Admin area You can manipulate with user\group\repository and permission. First, create `ArtifactoryPath` object without a repository @@ -670,7 +755,7 @@ user.delete() ``` ### API Keys -~~~python +```python from dohq_artifactory import User user = User(artifactory_, "username") @@ -693,7 +778,7 @@ user.api_key.revoke() # remove all API keys in system, only if user has admin rights user.api_key.revoke_for_all_users() -~~~ +``` ## Group ### Internal @@ -1299,50 +1384,57 @@ path = ArtifactoryPath( ) ``` -## Troubleshooting ## -Use [logging](https://docs.python.org/3/library/logging.html) for debug: +## Logging ## +The library can be configured to emit logging that will give you better insight into what it's doing. +Just configure [logging](https://docs.python.org/3/library/logging.html) module in your python script. +Simplest example to add debug messages to a console: ```python -def init_logging(): - logger_format_string = "%(thread)5s %(module)-20s %(levelname)-8s %(message)s" - logging.basicConfig( - level=logging.DEBUG, format=logger_format_string, stream=sys.stdout - ) +import logging +from artifactory import ArtifactoryPath +logging.basicConfig() +# set level only for artifactory module, if omitted, then global log level is used, eg from basicConfig +logging.getLogger("artifactory").setLevel(logging.DEBUG) -init_logging() path = ArtifactoryPath( - "http://my-artifactory/artifactory/myrepo/restricted-path", - auth=("USERNAME", "PASSWORD or API_KEY"), + "http://my-artifactory/artifactory/myrepo/restricted-path", apikey="MY_API_KEY" ) - -path.touch() ``` ## Global Configuration File ## -Artifactory Python module also can specify all connection-related settings in a central file, ```~/.artifactory_python.cfg``` that is read upon the creation of first ```ArtifactoryPath``` object and is stored globally. For instance, you can specify per-instance settings of authentication tokens, so that you won't need to explicitly pass ```auth``` parameter to ```ArtifactoryPath```. +Artifactory Python module also can specify all connection-related settings in a central file, +```~/.artifactory_python.cfg``` that is read upon the creation of first ```ArtifactoryPath``` object and is stored +globally. For instance, you can specify per-instance settings of authentication tokens, so that you won't need to +explicitly pass ```auth``` parameter to ```ArtifactoryPath```. Example: ```ini +[DEFAULT] +username = nameforallinstances + [http://artifactory-instance.com/artifactory] -username = deployer password = ilikerandompasswords verify = false [another-artifactory-instance.com/artifactory] -username = foo password = @dmin cert = ~/mycert ``` -Whether or not you specify ```http://``` or ```https://```, the prefix is not essential. The module will first try to locate the best match and then try to match URLs without prefixes. So in the config, if you specify ```https://my-instance.local``` and call ```ArtifactoryPath``` with ```http://my-instance.local```, it will still do the right thing. +Whether or not you specify ```http://``` or ```https://```, the prefix is not essential. The module will first try to +locate the best match and then try to match URLs without prefixes. So in the config, if you specify +```https://my-instance.local``` and call ```ArtifactoryPath``` with ```http://my-instance.local```, it will still do +the right thing. # Contribute [About contributing and testing](docs/CONTRIBUTE.md) # Advertising -- [artifactory-du](https://github.com/devopshq/artifactory-du) - estimate file space usage. Summarize disk usage in JFrog Artifactory of the set of FILEs, recursively for directories. -- [artifactory-cleanup-rules](https://github.com/devopshq/artifactory-du/issues/2) - python-script for Artifactory intelligence cleanup rules with config. +- [artifactory-du](https://github.com/devopshq/artifactory-du) - estimate file space usage. Summarize disk usage in + JFrog Artifactory of the set of FILEs, recursively for directories. +- [artifactory-cleanup-rules](https://github.com/devopshq/artifactory-du/issues/2) - python-script for Artifactory + intelligence cleanup rules with config. diff --git a/artifactory.py b/artifactory.py index 95e037ae..a8e01780 100755 --- a/artifactory.py +++ b/artifactory.py @@ -29,7 +29,6 @@ import hashlib import io import json -import logging import os import pathlib import re @@ -51,6 +50,8 @@ from dohq_artifactory.auth import XJFrogArtApiAuth from dohq_artifactory.auth import XJFrogArtBearerAuth from dohq_artifactory.exception import ArtifactoryException +from dohq_artifactory.exception import raise_for_status +from dohq_artifactory.logger import logger try: import requests.packages.urllib3 as urllib3 @@ -70,7 +71,7 @@ def read_config(config_path=default_config_path): Read configuration file and produce a dictionary of the following structure: {'': {'username': '', 'password': '', - 'verify': , 'cert': ''} + 'verify': , 'cert': ''} '': {...}, ...} @@ -86,7 +87,7 @@ def read_config(config_path=default_config_path): config_path = os.path.expanduser(config_path) if not os.path.isfile(config_path): raise OSError( - errno.ENOENT, "Artifactory configuration file not found: '%s'" % config_path + errno.ENOENT, f"Artifactory configuration file not found: '{config_path}'" ) p = configparser.ConfigParser() @@ -95,16 +96,22 @@ def read_config(config_path=default_config_path): result = {} for section in p.sections(): - username = ( - p.get(section, "username") if p.has_option(section, "username") else None - ) - password = ( - p.get(section, "password") if p.has_option(section, "password") else None - ) - verify = ( - p.getboolean(section, "verify") if p.has_option(section, "verify") else True - ) - cert = p.get(section, "cert") if p.has_option(section, "cert") else None + username = p.get(section, "username", fallback=None) + password = p.get(section, "password", fallback=None) + + try: + verify = p.getboolean(section, "verify", fallback=True) + except ValueError: + # the path to a CA_BUNDLE file or directory with certificates of trusted CAs + # see https://github.com/devopshq/artifactory/issues/281 + verify = p.get(section, "verify", fallback=True) + # path may contain '~', and we'd better expand it properly + verify = os.path.expanduser(verify) + + cert = p.get(section, "cert", fallback=None) + if cert: + # certificate path may contain '~', and we'd better expand it properly + cert = os.path.expanduser(cert) result[section] = { "username": username, @@ -112,9 +119,7 @@ def read_config(config_path=default_config_path): "verify": verify, "cert": cert, } - # certificate path may contain '~', and we'd better expand it properly - if result[section]["cert"]: - result[section]["cert"] = os.path.expanduser(result[section]["cert"]) + return result @@ -265,7 +270,7 @@ def log_download_progress(bytes_now, total_size): else: msg = "Downloaded {}MB".format(int(bytes_now / 1024 / 1024)) - logging.debug(msg) + logger.debug(msg) class HTTPResponseWrapper(object): @@ -385,6 +390,7 @@ def quote_url(url): :param url: (str) URL that should be quoted :return: (str) quoted URL """ + logger.debug(f"Raw URL passed for encoding: {url}") parsed_url = urllib3.util.parse_url(url) if parsed_url.port: quoted_path = requests.utils.quote( @@ -593,6 +599,19 @@ def _get_base_url(self, url): "md5", "is_dir", "children", + "repo", + ], +) + +ArtifactoryDownloadStat = collections.namedtuple( + "ArtifactoryDownloadStat", + [ + "last_downloaded", + "download_count", + "last_downloaded_by", + "remote_download_count", + "remote_last_downloaded", + "uri", ], ) @@ -669,7 +688,7 @@ def rest_put( Perform a PUT request to url with requests.session """ url = quote_url(url) - res = session.put( + response = session.put( url, params=params, headers=headers, @@ -677,7 +696,7 @@ def rest_put( cert=cert, timeout=timeout, ) - return res.text, res.status_code + return response @staticmethod def rest_post( @@ -701,7 +720,7 @@ def rest_post( cert=cert, timeout=timeout, ) - response.raise_for_status() + raise_for_status(response) return response @@ -721,7 +740,39 @@ def rest_del(url, params=None, session=None, verify=True, cert=None, timeout=Non response = session.delete( url, params=params, verify=verify, cert=cert, timeout=timeout ) - response.raise_for_status() + raise_for_status(response) + return response + + @staticmethod + def rest_patch( + url, + json_data=None, + params=None, + session=None, + verify=True, + cert=None, + timeout=None, + ): + """ + Perform a PATCH request to url with requests.session + :param url: url + :param json_data: (dict) JSON data to attach to patch request + :param params: request parameters + :param session: + :param verify: + :param cert: + :param timeout: + :return: request response object + """ + url = quote_url(url) + response = session.patch( + url=url, + json=json_data, + params=params, + verify=verify, + cert=cert, + timeout=timeout, + ) return response @staticmethod @@ -745,10 +796,11 @@ def rest_put_stream( # added later, otherwise ; and = are converted url += matrix_parameters - res = session.put( + response = session.put( url, headers=headers, data=stream, verify=verify, cert=cert, timeout=timeout ) - return res.text, res.status_code + raise_for_status(response) + return response @staticmethod def rest_get_stream( @@ -762,14 +814,21 @@ def rest_get_stream( response = session.get( url, params=params, stream=True, verify=verify, cert=cert, timeout=timeout ) - response.raise_for_status() + raise_for_status(response) return response - def get_stat_json(self, pathobj): + def get_stat_json(self, pathobj, key=None): """ Request remote file/directory status info Returns a json object as specified by Artifactory REST API + Args: + pathobj: ArtifactoryPath for which we request data + key: (str) (optional) additional key to specify query, eg 'stats', 'lastModified' + + Returns: + (dict) stat dictionary """ + url = "/".join( [ pathobj.drive.rstrip("/"), @@ -784,15 +843,16 @@ def get_stat_json(self, pathobj): verify=pathobj.verify, cert=pathobj.cert, timeout=pathobj.timeout, + params=key, ) code = response.status_code text = response.text if code == 404 and ("Unable to find item" in text or "Not Found" in text): raise OSError(2, f"No such file or directory: {url}") - response.raise_for_status() + raise_for_status(response) - return json.loads(text) + return response.json() def stat(self, pathobj): """ @@ -818,11 +878,30 @@ def stat(self, pathobj): modified_by=jsn.get("modifiedBy"), mime_type=jsn.get("mimeType"), size=int(jsn.get("size", "0")), - sha1=checksums.get("sha1"), - sha256=checksums.get("sha256"), - md5=checksums.get("md5"), + sha1=checksums.get("sha1", None), + sha256=checksums.get("sha256", None), + md5=checksums.get("md5", None), is_dir=is_dir, children=children, + repo=jsn.get("repo", None), + ) + + return stat + + def download_stats(self, pathobj): + jsn = self.get_stat_json(pathobj, key="stats") + + # divide timestamp by 1000 since it is provided in ms + download_time = datetime.datetime.fromtimestamp( + jsn.get("lastDownloaded", 0) / 1000 + ) + stat = ArtifactoryDownloadStat( + last_downloaded=download_time, + last_downloaded_by=jsn.get("lastDownloadedBy", None), + download_count=jsn.get("downloadCount", None), + remote_download_count=jsn.get("remoteDownloadCount", None), + remote_last_downloaded=jsn.get("remoteLastDownloaded", None), + uri=jsn.get("uri", None), ) return stat @@ -860,7 +939,7 @@ def listdir(self, pathobj): stat = self.stat(pathobj) if not stat.is_dir: - raise OSError(20, "Not a directory: %s" % str(pathobj)) + raise OSError(20, f"Not a directory: {pathobj}") return stat.children @@ -870,13 +949,13 @@ def mkdir(self, pathobj, _): Note that this operation is not recursive """ if not pathobj.drive or not pathobj.root: - raise RuntimeError("Full path required: '%s'" % str(pathobj)) + raise ArtifactoryException(f"Full path required: '{pathobj}'") if pathobj.exists(): - raise OSError(17, "File exists: '%s'" % str(pathobj)) + raise OSError(17, f"File exists: '{pathobj}'") url = str(pathobj) + "/" - text, code = self.rest_put( + response = self.rest_put( url, session=pathobj.session, verify=pathobj.verify, @@ -884,8 +963,7 @@ def mkdir(self, pathobj, _): timeout=pathobj.timeout, ) - if code != 201: - raise RuntimeError("%s %d" % (text, code)) + raise_for_status(response) def rmdir(self, pathobj): """ @@ -894,7 +972,7 @@ def rmdir(self, pathobj): stat = self.stat(pathobj) if not stat.is_dir: - raise OSError(20, "Not a directory: '%s'" % str(pathobj)) + raise OSError(20, f"Not a directory: '{pathobj}'") url = str(pathobj) + "/" @@ -925,8 +1003,8 @@ def unlink(self, pathobj): cert=pathobj.cert, timeout=pathobj.timeout, ) - except requests.exceptions.HTTPError as err: - if err.response.status_code == 404: + except ArtifactoryException as err: + if err.__cause__.response.status_code == 404: # since we performed existence check we can say it is permissions issue # see https://github.com/devopshq/artifactory/issues/36 docs_url = ( @@ -945,13 +1023,13 @@ def touch(self, pathobj): Create an empty file """ if not pathobj.drive or not pathobj.root: - raise RuntimeError("Full path required") + raise ArtifactoryException("Full path required") if pathobj.exists(): return url = str(pathobj) - text, code = self.rest_put( + response = self.rest_put( url, session=pathobj.session, verify=pathobj.verify, @@ -959,8 +1037,7 @@ def touch(self, pathobj): timeout=pathobj.timeout, ) - if code != 201: - raise RuntimeError("%s %d" % (text, code)) + raise_for_status(response) def owner(self, pathobj): """ @@ -1030,11 +1107,30 @@ def deploy( parameters=None, explode_archive=None, explode_archive_atomic=None, + checksum=None, + by_checksum=False, ): """ Uploads a given file-like object HTTP chunked encoding will be attempted + + If by_checksum is True, fobj should be None + + :param pathobj: ArtifactoryPath object + :param fobj: file object to be deployed + :param md5: (str) MD5 checksum value + :param sha1: (str) SHA1 checksum value + :param sha256: (str) SHA256 checksum value + :param parameters: Artifact properties + :param explode_archive: (bool) if True, archive will be exploded upon deployment + :param explode_archive_atomic: (bool) if True, archive will be exploded in an atomic operation upon deployment + :param checksum: sha1Value or sha256Value + :param by_checksum: (bool) if True, deploy artifact by checksum, default False """ + + if fobj and by_checksum: + raise ArtifactoryException("Either fobj or by_checksum, but not both") + if isinstance(fobj, urllib3.response.HTTPResponse): fobj = HTTPResponseWrapper(fobj) @@ -1055,8 +1151,12 @@ def deploy( headers["X-Explode-Archive"] = "true" if explode_archive_atomic: headers["X-Explode-Archive-Atomic"] = "true" + if by_checksum: + headers["X-Checksum-Deploy"] = "true" + if checksum: + headers["X-Checksum"] = checksum - text, code = self.rest_put_stream( + self.rest_put_stream( url, fobj, headers=headers, @@ -1067,27 +1167,35 @@ def deploy( matrix_parameters=matrix_parameters, ) - if code not in (200, 201): - raise RuntimeError(text) - - def copy(self, src, dst, suppress_layouts=False): + def copy(self, src, dst, suppress_layouts=False, fail_fast=False, dry_run=False): """ Copy artifact from src to dst + Args: + src: from + dst: to + suppress_layouts: suppress cross-layout module path translation during copy + fail_fast: parameter will fail and abort the operation upon receiving an error. + dry_run: If true, distribution is only simulated. + + Returns: + if dry_run==True (dict) response.json() else None """ url = "/".join( [ src.drive.rstrip("/"), "api/copy", - str(src.relative_to(src.drive)).rstrip("/"), + str(src.relative_to(src.drive)).strip("/"), ] ) params = { "to": str(dst.relative_to(dst.drive)).rstrip("/"), "suppressLayouts": int(suppress_layouts), + "failFast": int(fail_fast), + "dry": int(dry_run), } - self.rest_post( + response = self.rest_post( url, params=params, session=src.session, @@ -1095,10 +1203,22 @@ def copy(self, src, dst, suppress_layouts=False): cert=src.cert, timeout=src.timeout, ) + if dry_run: + logger.debug(response.text) + return response.json() - def move(self, src, dst, suppress_layouts=False): + def move(self, src, dst, suppress_layouts=False, fail_fast=False, dry_run=False): """ Move artifact from src to dst + Args: + src: from + dst: to + suppress_layouts: suppress cross-layout module path translation during copy + fail_fast: parameter will fail and abort the operation upon receiving an error. + dry_run: If true, distribution is only simulated. + + Returns: + if dry_run==True (dict) response.json() else None """ url = "/".join( [ @@ -1111,9 +1231,11 @@ def move(self, src, dst, suppress_layouts=False): params = { "to": str(dst.relative_to(dst.drive)).rstrip("/"), "suppressLayouts": int(suppress_layouts), + "failFast": int(fail_fast), + "dry": int(dry_run), } - self.rest_post( + response = self.rest_post( url, params=params, session=src.session, @@ -1121,6 +1243,9 @@ def move(self, src, dst, suppress_layouts=False): cert=src.cert, timeout=src.timeout, ) + if dry_run: + logger.debug(response.text) + return response.json() def get_properties(self, pathobj): """ @@ -1147,13 +1272,13 @@ def get_properties(self, pathobj): code = response.status_code text = response.text if code == 404 and ("Unable to find item" in text or "Not Found" in text): - raise OSError(2, "No such file or directory: '%s'" % url) + raise OSError(2, f"No such file or directory: '{url}'") if code == 404 and "No properties could be found" in text: return {} - response.raise_for_status() + raise_for_status(response) - return json.loads(text)["properties"] + return response.json()["properties"] def set_properties(self, pathobj, props, recursive): """ @@ -1172,7 +1297,7 @@ def set_properties(self, pathobj, props, recursive): if not recursive: params["recursive"] = "0" - text, code = self.rest_put( + response = self.rest_put( url, params=params, session=pathobj.session, @@ -1181,10 +1306,12 @@ def set_properties(self, pathobj, props, recursive): timeout=pathobj.timeout, ) + code = response.status_code + text = response.text if code == 404 and ("Unable to find item" in text or "Not Found" in text): - raise OSError(2, "No such file or directory: '%s'" % url) - if code != 204: - raise RuntimeError(text) + raise OSError(2, f"No such file or directory: '{url}'") + + raise_for_status(response) def del_properties(self, pathobj, props, recursive): """ @@ -1215,6 +1342,41 @@ def del_properties(self, pathobj, props, recursive): timeout=pathobj.timeout, ) + def update_properties(self, pathobj, properties, recursive=False): + """ + Update item properties + + Args: + pathobj: (ArtifactoryPath) object + properties: (dict) properties + recursive: (bool) apply recursively or not. For folders + + Returns: None + """ + url = "/".join( + [ + pathobj.drive.rstrip("/"), + "api/metadata", + str(pathobj.relative_to(pathobj.drive)).strip("/"), + ] + ) + + # construct data according to Artifactory format + json_data = {"props": properties} + + params = {"recursive": int(recursive)} + + response = self.rest_patch( + url, + json_data=json_data, + params=params, + session=pathobj.session, + verify=pathobj.verify, + cert=pathobj.cert, + timeout=pathobj.timeout, + ) + raise_for_status(response) + def scandir(self, pathobj): return _ScandirIter((pathobj.joinpath(x) for x in self.listdir(pathobj))) @@ -1287,9 +1449,13 @@ class ArtifactoryPath(pathlib.Path, PureArtifactoryPath): on regular constructors, but rather on templates. """ - # Pathlib limits what members can be present in 'Path' class, - # so authentication information has to be added via __slots__ - __slots__ = ("auth", "verify", "cert", "session", "timeout") + if sys.version_info.major == 3 and sys.version_info.minor >= 10: + # see changes in pathlib.Path, slots are no more applied + # https://github.com/python/cpython/blob/ce121fd8755d4db9511ce4aab39d0577165e118e/Lib/pathlib.py#L952 + _accessor = _artifactory_accessor + else: + # in 3.9 and below Pathlib limits what members can be present in 'Path' class + __slots__ = ("auth", "verify", "cert", "session", "timeout") def __new__(cls, *args, **kwargs): """ @@ -1309,10 +1475,10 @@ def __new__(cls, *args, **kwargs): auth_type = kwargs.get("auth_type") if apikey: - logging.debug("Use XJFrogApiAuth apikey") + logger.debug("Use XJFrogApiAuth apikey") obj.auth = XJFrogArtApiAuth(apikey=apikey) elif token: - logging.debug("Use XJFrogArtBearerAuth token") + logger.debug("Use XJFrogArtBearerAuth token") obj.auth = XJFrogArtBearerAuth(token=token) else: auth = kwargs.get("auth") @@ -1412,6 +1578,18 @@ def stat(self, pathobj=None): pathobj = pathobj or self return self._accessor.stat(pathobj=pathobj) + def download_stats(self, pathobj=None): + """ + Item statistics record the number of times an item was downloaded, last download date and last downloader. + Args: + pathobj: (optional) path object for which to retrieve stats + + Returns: + + """ + pathobj = pathobj or self + return self._accessor.download_stats(pathobj=pathobj) + def with_name(self, name): """ Return a new path with the file name changed. @@ -1445,7 +1623,7 @@ def archive(self, archive_type="zip", check_sum=False): :return: raw object for download """ if self.is_file(): - raise OSError("Only folders could be archived") + raise ArtifactoryException("Only folders could be archived") if archive_type not in ["zip", "tar", "tar.gz", "tgz"]: raise NotImplementedError(archive_type + " is not support by current API") @@ -1764,6 +1942,32 @@ def deploy_file( explode_archive_atomic=explode_archive_atomic, ) + def deploy_by_checksum( + self, + sha1=None, + sha256=None, + checksum=None, + parameters={}, + ): + """ + Deploy an artifact to the specified destination by checking if the + artifact content already exists in Artifactory. + + :param pathobj: ArtifactoryPath object + :param sha1: sha1Value + :param sha256: sha256Value + :param checksum: sha1Value or sha256Value + """ + return self._accessor.deploy( + self, + fobj=None, + sha1=sha1, + sha256=sha256, + checksum=checksum, + by_checksum=True, + parameters=parameters, + ) + def deploy_deb( self, file_name, distribution, component, architecture, parameters={} ): @@ -1786,7 +1990,7 @@ def deploy_deb( self.deploy_file(file_name, parameters=params) - def copy(self, dst, suppress_layouts=False): + def copy(self, dst, suppress_layouts=False, fail_fast=False, dry_run=False): """ Copy artifact from this path to destination. If files are on the same instance of artifactory, lightweight (local) @@ -1798,6 +2002,9 @@ def copy(self, dst, suppress_layouts=False): root, but remap the [org], [module], [baseVer], etc. structure to the target repository. + fail_fast: parameter will fail and abort the operation upon receiving an error. + dry_run: If true, distribution is only simulated. + For example, if we have a builds repository using the default maven2 repository where we publish our builds. We also have a published repository where a directory for production and a directory for @@ -1832,27 +2039,62 @@ def copy(self, dst, suppress_layouts=False): http://example.com/artifactory/published/production/foo-0.0.1.pom http://example.com/artifactory/published/production/product-1.0.0.tar.gz http://example.com/artifactory/published/production/product-1.0.0.tar.pom + + Returns: + if dry_run==True (dict) response.json() else None """ if self.drive.rstrip("/") == dst.drive.rstrip("/"): - self._accessor.copy(self, dst, suppress_layouts=suppress_layouts) + output = self._accessor.copy( + self, + dst, + suppress_layouts=suppress_layouts, + fail_fast=fail_fast, + dry_run=dry_run, + ) + return output else: + stat = self.stat() + if stat.is_dir: + raise ArtifactoryException( + "Only files could be copied across different instances" + ) + + if dry_run: + logger.debug( + "Artifactory drive is different. Will do a standard upload" + ) + return + with self.open() as fobj: - dst.deploy(fobj) + dst.deploy(fobj, md5=stat.md5, sha1=stat.sha1, sha256=stat.sha256) - def move(self, dst, suppress_layouts=False): + def move(self, dst, suppress_layouts=False, fail_fast=False, dry_run=False): """ - Move artifact from this path to destinaiton. + Move artifact from this path to destination. The suppress_layouts parameter, when set to True, will allow artifacts from one path to be moved directly into another path without enforcing repository layouts. The default behaviour is to move the repository root, but remap the [org], [module], [baseVer], etc. structure to the target repository. + + fail_fast: parameter will fail and abort the operation upon receiving an error. + dry_run: If true, distribution is only simulated. + + Returns: + if dry_run==True (dict) response.json() else None """ if self.drive.rstrip("/") != dst.drive.rstrip("/"): raise NotImplementedError("Moving between instances is not implemented yet") - self._accessor.move(self, dst) + output = self._accessor.move( + self, + dst, + suppress_layouts=suppress_layouts, + fail_fast=fail_fast, + dry_run=dry_run, + ) + return output @property def properties(self): @@ -1864,12 +2106,15 @@ def properties(self): @properties.setter def properties(self, properties): properties_to_remove = set(self.properties) - set(properties) - if properties_to_remove: - self.del_properties(properties_to_remove, recursive=False) - self.set_properties(properties, recursive=False) + for prop in properties_to_remove: + properties[prop] = None + self.update_properties(properties=properties, recursive=False) @properties.deleter def properties(self): + """ + Delete properties + """ self.del_properties(self.properties, recursive=False) def set_properties(self, properties, recursive=True): @@ -1885,15 +2130,10 @@ def set_properties(self, properties, recursive=True): if not properties: return - # If URL > 13KB, nginx default raise error '414 Request-URI Too Large' - MAX_SIZE = 50 - if len(properties) > MAX_SIZE: - for chunk in chunks(properties, MAX_SIZE): - self._accessor.set_properties(self, chunk, recursive) - else: - self._accessor.set_properties(self, properties, recursive) + # Uses update properties since it can consume JSON as input and removes URL limit + self.update_properties(properties, recursive=recursive) - def del_properties(self, properties, recursive=None): + def del_properties(self, properties, recursive=False): """ Delete properties listed in properties @@ -1902,7 +2142,20 @@ def del_properties(self, properties, recursive=None): recursive - on folders property attachment is recursive by default. It is possible to force recursive behavior. """ - return self._accessor.del_properties(self, properties, recursive) + properties_to_remove = dict.fromkeys(properties, None) + # Uses update properties since it can consume JSON as input and removes URL limit + self.update_properties(properties_to_remove, recursive=recursive) + + def update_properties(self, properties, recursive=False): + """ + Update properties, set/update/remove item or folder properties + Args: + properties: (dict) data to be set + recursive: (bool) recursive on folder + + Returns: None + """ + return self._accessor.update_properties(self, properties, recursive) def aql(self, *args): """ @@ -1912,15 +2165,16 @@ def aql(self, *args): """ aql_query_url = "{}/api/search/aql".format(self.drive.rstrip("/")) aql_query_text = self.create_aql_text(*args) - r = self.session.post(aql_query_url, data=aql_query_text) - r.raise_for_status() - content = r.json() + logger.debug(f"AQL query request text: {aql_query_text}") + response = self.session.post(aql_query_url, data=aql_query_text) + raise_for_status(response) + content = response.json() return content["results"] @staticmethod def create_aql_text(*args): """ - Create AQL querty from string or list or dict arguments + Create AQL query from string or list or dict arguments """ aql_query_text = "" for arg in args: @@ -1929,6 +2183,7 @@ def create_aql_text(*args): elif isinstance(arg, list): arg = "({})".format(json.dumps(arg)).replace("[", "").replace("]", "") aql_query_text += arg + return aql_query_text def from_aql(self, result): @@ -1939,10 +2194,8 @@ def from_aql(self, result): """ result_type = result.get("type") if result_type not in ("file", "folder"): - raise RuntimeError( - "Path object with type '{}' doesn't support. File or folder only".format( - result_type - ) + raise ArtifactoryException( + f"Path object with type '{result_type}' doesn't support. File or folder only" ) result_path = "{}/{repo}/{path}/{name}".format(self.drive.rstrip("/"), **result) @@ -1965,7 +2218,7 @@ def promote_docker_image( :param target_repo: target repository :param docker_repo: Docker repository to promote :param tag: Docker tag to promote - :param copy (bool): whether to move the image or copy it + :param copy: (bool) whether to move the image or copy it :return: """ promote_url = "{}/api/docker/{}/v2/promote".format( @@ -1977,18 +2230,8 @@ def promote_docker_image( "tag": tag, "copy": copy, } - r = self.session.post(promote_url, json=promote_data) - if ( - r.status_code == 400 - and "Unsupported docker" in r.text - or r.status_code == 403 - and "No permission" in r.text - or r.status_code == 404 - and "Unable to find" in r.text - ): - raise ArtifactoryException(r.text) - else: - r.raise_for_status() + response = self.session.post(promote_url, json=promote_data) + raise_for_status(response) @property def repo(self): @@ -2091,11 +2334,11 @@ def _get_all(self, lazy: bool, url=None, key="name", cls=None): request_url = self.drive.rstrip("/artifactory") + url else: request_url = self.drive + url - r = self.session.get(request_url, auth=self.auth) - r.raise_for_status() - response = r.json() + response = self.session.get(request_url, auth=self.auth) + raise_for_status(response) + response_json = response.json() results = [] - for i in response: + for i in response_json: if cls is Repository: item = Repository.create_by_type(i["type"], self, i[key]) else: diff --git a/dohq_artifactory/admin.py b/dohq_artifactory/admin.py index 5ee7a4dc..92bbed4a 100644 --- a/dohq_artifactory/admin.py +++ b/dohq_artifactory/admin.py @@ -1,32 +1,23 @@ import json -import logging import random import re import string import sys import time +import warnings import jwt -import requests from dateutil.parser import isoparse from dohq_artifactory.exception import ArtifactoryException +from dohq_artifactory.exception import raise_for_status +from dohq_artifactory.logger import logger def rest_delay(): time.sleep(0.5) -def raise_errors(r): - try: - r.raise_for_status() - except requests.HTTPError as e: - if e.response.status_code >= 400: - raise ArtifactoryException(e.response.text) - else: - raise e - - def _old_function_for_secret(pw_len=16): alphabet_lower = "abcdefghijklmnopqrstuvwxyz" alphabet_upper = alphabet_lower.upper() @@ -66,6 +57,10 @@ def _new_function_with_secret_module(pw_len=16): generate_password = _new_function_with_secret_module +def deprecation(message): + warnings.warn(message, DeprecationWarning, stacklevel=2) + + class AdminObject(object): prefix_uri = "api" _uri = None @@ -99,7 +94,7 @@ def create(self): Create object :return: None """ - logging.debug( + logger.debug( f"Create {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) self._create_and_update(self._session.put) @@ -118,7 +113,7 @@ def _create_and_update(self, method): headers={"Content-Type": "application/json"}, auth=self._auth, ) - raise_errors(r) + raise_for_status(r) rest_delay() self.read() @@ -137,21 +132,21 @@ def read(self): True if object exist, False else """ - logging.debug( + logger.debug( f"Read {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}/{getattr(self, self.resource_name)}" r = self._session.get(request_url, auth=self._auth) if 404 == r.status_code or 400 == r.status_code: - logging.debug( + logger.debug( f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] does not exist" ) return False else: - logging.debug( + logger.debug( f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] exist" ) - raise_errors(r) + raise_for_status(r) response = r.json() self.raw = response self._read_response(response) @@ -163,18 +158,18 @@ def list(self): :return: List of objects """ - # logging.debug('List {x.__class__.__name__} [{x.name}]'.format(x=self)) + logger.debug(f"List {self.__class__.__name__} [{self.name}]") request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}" response = self._session.get( request_url, auth=self._auth, ) if response.status_code == 200: - # logging.debug('{x.__class__.__name__} [{x.name}] does not exist'.format(x=self)) + logger.debug(f"{self.__class__.__name__} [{self.name}] does not exist") json_response = response.json() return [item.get(self.resource_name) for item in json_response] else: - # logging.debug('{x.__class__.__name__} [{x.name}] exist'.format(x=self)) + logger.debug(f"{self.__class__.__name__} [{self.name}] exist") return "failed" def update(self): @@ -182,7 +177,7 @@ def update(self): Update object :return: None """ - logging.debug( + logger.debug( f"Create {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) self._create_and_update(self._session.post) @@ -192,7 +187,7 @@ def delete(self): Remove object :return: None """ - logging.debug( + logger.debug( f"Remove {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}/{getattr(self, self.resource_name)}" @@ -200,7 +195,7 @@ def delete(self): request_url, auth=self._auth, ) - raise_errors(r) + raise_for_status(r) rest_delay() @@ -229,7 +224,7 @@ def __init__( self.internal_password_disabled = False self._groups = [] - self._lastLoggedIn = None + self._last_logged_in = None self._realm = None def _create_json(self): @@ -260,7 +255,7 @@ def _read_response(self, response): self.disable_ui_access = response.get("disableUIAccess") self.internal_password_disabled = response.get("internalPasswordDisabled") self._groups = response.get("groups", []) - self._lastLoggedIn = ( + self._last_logged_in = ( isoparse(response["lastLoggedIn"]) if response.get("lastLoggedIn") else None ) self._realm = response.get("realm") @@ -271,6 +266,7 @@ def encryptedPassword(self): Method for backwards compatibility, see property encrypted_password :return: """ + deprecation("encryptedPassword is deprecated, use encrypted_password") return self.encrypted_password @property @@ -301,12 +297,21 @@ def _authenticated_user_request(self, api_url, request_type): request_url, auth=(self.name, self.password), ) - raise_errors(r) + raise_for_status(r) return r.text @property def lastLoggedIn(self): - return self._lastLoggedIn + """ + Method for backwards compatibility, see property last_logged_in + :return: + """ + deprecation("lastLoggedIn is deprecated, use last_logged_in") + return self.last_logged_in + + @property + def last_logged_in(self): + return self._last_logged_in @property def realm(self): @@ -483,7 +488,7 @@ def delete(self): TODO: New entrypoint would go like /api/groups/delete and consumes ["list", "of", "groupnames"] """ - logging.debug( + logger.debug( f"Remove {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri_deletion}/{getattr(self, self.resource_name)}" @@ -496,13 +501,13 @@ def create(self): Create object :return: None """ - logging.debug( + logger.debug( f"Create {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) data_json = self._create_json() data_json.update(self.additional_params) - request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}" - r = self._session.post( + request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}/{getattr(self, self.resource_name)}" + r = self._session.put( request_url, json=data_json, headers={"Content-Type": "application/json"}, @@ -536,7 +541,7 @@ def path(self): return self._artifactory.joinpath(self.name) def _generate_query(self, package): - if self.packageType == Repository.DOCKER: + if self.package_type == Repository.DOCKER: parts = package.split(":") name = parts[0] @@ -546,7 +551,7 @@ def _generate_query(self, package): return {"name": "manifest.json", "path": {"$match": package}} - if self.packageType == Repository.PYPI and "/" not in package: + if self.package_type == Repository.PYPI and "/" not in package: operators = { "<=": "$lte", "<": "$lt", @@ -567,7 +572,7 @@ def _generate_query(self, package): return {"@pypi.name": {"$match": package}} - if self.packageType == Repository.MAVEN and "/" not in package: + if self.package_type == Repository.MAVEN and "/" not in package: package = package.replace("#", ":") parts = list(package.split(":")) @@ -587,7 +592,7 @@ def _generate_query(self, package): "$or": [ {"name": {"$match": package}}, {"path": {"$match": package}}, - {"@{}.name".format(self.packageType): {"$match": package}}, + {"@{}.name".format(self.package_type): {"$match": package}}, {"@build.name": {"$match": package}}, {"artifact.module.build.name": {"$match": package}}, ] @@ -646,7 +651,7 @@ def __rtruediv__(self, key): class Repository(GenericRepository): - # List packageType from wiki: + # List package_type from wiki: # https://www.jfrog.com/confluence/display/RTF/Repository+Configuration+JSON#RepositoryConfigurationJSON-application/vnd.org.jfrog.artifactory.repositories.LocalRepositoryConfiguration+json ALPINE = "alpine" BOWER = "bower" @@ -664,11 +669,6 @@ class Repository(GenericRepository): HELM = "helm" IVY = "ivy" MAVEN = "maven" - SBT = "sbt" - HELM = "helm" - RPM = "rpm" - NUGET = "nuget" - GEMS = "gems" NPM = "npm" NUGET = "nuget" PUPPET = "puppet" @@ -677,21 +677,47 @@ class Repository(GenericRepository): SBT = "sbt" YUM = "yum" - # List dockerApiVersion from wiki: + # List docker_api_version from wiki: V1 = "V1" V2 = "V2" @staticmethod - def create_by_type(type: str, artifactory, name): - if type == "LOCAL": + def create_by_type(repo_type="LOCAL", artifactory=None, name=None, *, type=None): + if type is not None: + deprecation("'type' argument is deprecated, use 'repo_type'") + repo_type = type + + if repo_type == "LOCAL": return RepositoryLocal(artifactory, name) - elif type == "REMOTE": + elif repo_type == "REMOTE": return RepositoryRemote(artifactory, name) - elif type == "VIRTUAL": + elif repo_type == "VIRTUAL": return RepositoryVirtual(artifactory, name) else: return None + @property + def packageType(self): + deprecation("packageType is deprecated, use package_type") + return self.package_type + + @property + def repoLayoutRef(self): + deprecation("repoLayoutRef is deprecated, use repo_layout_ref") + return self.repo_layout_ref + + @property + def dockerApiVersion(self): + deprecation("dockerApiVersion is deprecated, use docker_api_version") + return self.docker_api_version + + @property + def archiveBrowsingEnabled(self): + deprecation( + "archiveBrowsingEnabled is deprecated, use archive_browsing_enabled" + ) + return self.archive_browsing_enabled + class RepositoryLocal(Repository): _uri = "repositories" @@ -704,20 +730,31 @@ def __init__( self, artifactory, name, - packageType=Repository.GENERIC, - dockerApiVersion=Repository.V1, - repoLayoutRef="maven-2-default", + package_type=Repository.GENERIC, + docker_api_version=Repository.V1, + repo_layout_ref="maven-2-default", max_unique_tags=0, + *, + packageType=None, + dockerApiVersion=None, + repoLayoutRef=None, ): super(RepositoryLocal, self).__init__(artifactory) self.name = name self.description = "" - self.packageType = packageType - self.repoLayoutRef = repoLayoutRef - self.archiveBrowsingEnabled = True - self.dockerApiVersion = dockerApiVersion + self.package_type = packageType or package_type + self.repo_layout_ref = repoLayoutRef or repo_layout_ref + self.archive_browsing_enabled = True + self.docker_api_version = dockerApiVersion or docker_api_version self.max_unique_tags = max_unique_tags + if any([packageType, dockerApiVersion, repoLayoutRef]): + msg = ( + "packageType, dockerApiVersion, repoLayoutRef are deprecated, " + "use package_type, docker_api_version, repo_layout_ref" + ) + deprecation(msg) + def _create_json(self): """ JSON Documentation: https://www.jfrog.com/confluence/display/RTF/Repository+Configuration+JSON @@ -726,12 +763,12 @@ def _create_json(self): "rclass": "local", "key": self.name, "description": self.description, - "packageType": self.packageType, + "packageType": self.package_type, "notes": "", "includesPattern": "**/*", "excludesPattern": "", - "repoLayoutRef": self.repoLayoutRef, - "dockerApiVersion": self.dockerApiVersion, + "repoLayoutRef": self.repo_layout_ref, + "dockerApiVersion": self.docker_api_version, "checksumPolicyType": "client-checksums", "handleReleases": True, "handleSnapshots": True, @@ -740,13 +777,13 @@ def _create_json(self): "suppressPomConsistencyChecks": True, "blackedOut": False, "propertySets": [], - "archiveBrowsingEnabled": self.archiveBrowsingEnabled, + "archiveBrowsingEnabled": self.archive_browsing_enabled, "yumRootDepth": 0, } """ Docker V2 API specific fields """ - if self.dockerApiVersion == Repository.V2: + if self.docker_api_version == Repository.V2: data_json["maxUniqueTags"] = self.max_unique_tags return data_json @@ -765,9 +802,9 @@ def _read_response(self, response): self.name = response["key"] self.description = response.get("description") - self.packageType = response.get("packageType") - self.repoLayoutRef = response.get("repoLayoutRef") - self.archiveBrowsingEnabled = response.get("archiveBrowsingEnabled") + self.package_type = response.get("packageType") + self.repo_layout_ref = response.get("repoLayoutRef") + self.archive_browsing_enabled = response.get("archiveBrowsingEnabled") class RepositoryVirtual(GenericRepository): @@ -800,15 +837,26 @@ def __init__( artifactory, name, repositories=None, - packageType=Repository.GENERIC, + package_type=Repository.GENERIC, + *, + packageType=None, ): super(RepositoryVirtual, self).__init__(artifactory) self.name = name self.description = "" self.notes = "" - self.packageType = packageType + self.package_type = packageType or package_type self.repositories = repositories or [] + if packageType: + msg = "packageType is deprecated, use package_type" + deprecation(msg) + + @property + def packageType(self): + deprecation("packageType is deprecated, use package_type") + return self.package_type + def _create_json(self): """ JSON Documentation: https://www.jfrog.com/confluence/display/RTF/Repository+Configuration+JSON @@ -817,7 +865,7 @@ def _create_json(self): "rclass": "virtual", "key": self.name, "description": self.description, - "packageType": self.packageType, + "packageType": self.package_type, "repositories": self._repositories, "notes": self.notes, } @@ -838,7 +886,7 @@ def _read_response(self, response): self.name = response["key"] self.description = response.get("description") - self.packageType = response.get("packageType") + self.package_type = response.get("packageType") self._repositories = response.get("repositories") def add_repository(self, *repos): @@ -881,19 +929,30 @@ def __init__( artifactory, name, url=None, - packageType=Repository.GENERIC, - dockerApiVersion=Repository.V1, - repoLayoutRef="maven-2-default", + package_type=Repository.GENERIC, + docker_api_version=Repository.V1, + repo_layout_ref="maven-2-default", + *, + packageType=None, + dockerApiVersion=None, + repoLayoutRef=None, ): super(RepositoryRemote, self).__init__(artifactory) self.name = name self.description = "" - self.packageType = packageType - self.repoLayoutRef = repoLayoutRef - self.archiveBrowsingEnabled = True - self.dockerApiVersion = dockerApiVersion + self.package_type = packageType or package_type + self.repo_layout_ref = repoLayoutRef or repo_layout_ref + self.archive_browsing_enabled = True + self.docker_api_version = dockerApiVersion or docker_api_version self.url = url + if any([packageType, dockerApiVersion, repoLayoutRef]): + msg = ( + "packageType, dockerApiVersion, repoLayoutRef are deprecated, " + "use package_type, docker_api_version, repo_layout_ref" + ) + deprecation(msg) + def _create_json(self): """ JSON Documentation: https://www.jfrog.com/confluence/display/RTF/Repository+Configuration+JSON @@ -902,12 +961,12 @@ def _create_json(self): "rclass": "remote", "key": self.name, "description": self.description, - "packageType": self.packageType, + "packageType": self.package_type, "notes": "", "includesPattern": "**/*", "excludesPattern": "", - "repoLayoutRef": self.repoLayoutRef, - "dockerApiVersion": self.dockerApiVersion, + "repoLayoutRef": self.repo_layout_ref, + "dockerApiVersion": self.docker_api_version, "checksumPolicyType": "client-checksums", "handleReleases": True, "handleSnapshots": True, @@ -916,7 +975,7 @@ def _create_json(self): "suppressPomConsistencyChecks": True, "blackedOut": False, "propertySets": [], - "archiveBrowsingEnabled": self.archiveBrowsingEnabled, + "archiveBrowsingEnabled": self.archive_browsing_enabled, "yumRootDepth": 0, "url": self.url, "debianTrivialLayout": False, @@ -943,9 +1002,9 @@ def _read_response(self, response): self.name = response["key"] self.description = response.get("description") - self.packageType = response.get("packageType") - self.repoLayoutRef = response.get("repoLayoutRef") - self.archiveBrowsingEnabled = response.get("archiveBrowsingEnabled") + self.package_type = response.get("packageType") + self.repo_layout_ref = response.get("repoLayoutRef") + self.archive_browsing_enabled = response.get("archiveBrowsingEnabled") self.url = response.get("url") @@ -1270,19 +1329,19 @@ def read(self): True if object exist, False else """ - logging.debug( + logger.debug( f"Read {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}" r = self._session.get(request_url, auth=self._auth) if 404 == r.status_code or 400 == r.status_code: - logging.debug( + logger.debug( f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] does not exist" ) return False else: - logging.debug( - "{self.__class__.__name__} [{getattr(self, self.resource_name)}] exist" + logger.debug( + f"{self.__class__.__name__} [{getattr(self, self.resource_name)}] exist" ) r.raise_for_status() response = r.json() @@ -1299,7 +1358,7 @@ def delete(self): POST security/token/revoke revoke (calling it deletion to be consistent with other classes) a token """ - logging.debug( + logger.debug( f"Delete {self.__class__.__name__} [{getattr(self, self.resource_name)}]" ) request_url = f"{self.base_url}/{self.prefix_uri}/{self._uri}/revoke" @@ -1363,7 +1422,7 @@ def create(self): headers={"Content-Type": "application/json"}, auth=self._auth, ) - raise_errors(r) + raise_for_status(r) rest_delay() self.read() @@ -1384,7 +1443,7 @@ def update(self): headers={"Content-Type": "application/json"}, auth=self._auth, ) - raise_errors(r) + raise_for_status(r) rest_delay() self.read() diff --git a/dohq_artifactory/exception.py b/dohq_artifactory/exception.py index 62f348c3..c37529f8 100644 --- a/dohq_artifactory/exception.py +++ b/dohq_artifactory/exception.py @@ -1,2 +1,41 @@ +from json import JSONDecodeError + +import requests + + class ArtifactoryException(Exception): pass + + +def raise_for_status(response): + """ + Custom raise_for_status method. + Raises ArtifactoryException with clear message and keeps cause + Args: + response: HTTP response object + + Returns: + None + """ + + try: + response.raise_for_status() + except requests.HTTPError as exception: + # start processing HTTP error and try to extract meaningful data from it + try: + response_json = exception.response.json() + error_list = response_json.pop("errors", None) + except JSONDecodeError: + # not a JSON response + raise ArtifactoryException(str(exception)) from exception + + if not isinstance(error_list, list) or not error_list: + # no standard error list in the exception + raise ArtifactoryException(str(exception)) from exception + + error_info_dict = error_list[0] + if not isinstance(error_info_dict, dict) or "message" not in error_info_dict: + # if for some reason we don't receive standard HTTP errors dict, we need to raise the whole object + raise ArtifactoryException(str(error_info_dict)) from exception + + raise ArtifactoryException(error_info_dict["message"]) from exception diff --git a/dohq_artifactory/logger.py b/dohq_artifactory/logger.py new file mode 100644 index 00000000..1bd70d89 --- /dev/null +++ b/dohq_artifactory/logger.py @@ -0,0 +1,6 @@ +import logging + +# set logger to be configurable from external +logger = logging.getLogger("artifactory") +# Set default logging handler to avoid "No handler found" warnings. +logger.addHandler(logging.NullHandler()) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0dd35f7a..07b24519 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,9 @@ --e . - -pathlib; python_version<"3.4" requests python-dateutil +PyJWT pytest mock flake8 pre-commit tox -PyJWT -responses +responses>=0.14.0 diff --git a/setup.py b/setup.py index 6803995e..86545a66 100755 --- a/setup.py +++ b/setup.py @@ -1,42 +1,29 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - +import re try: from setuptools import setup except ImportError: from distutils.core import setup -import os - -__version__ = "0.7" -devStatus = "4 - Beta" # default build status, see: https://pypi.python.org/pypi?%3Aaction=list_classifiers -if "TRAVIS_BUILD_NUMBER" in os.environ and "TRAVIS_BRANCH" in os.environ: - print("This is TRAVIS-CI build") - print("TRAVIS_BUILD_NUMBER = {}".format(os.environ["TRAVIS_BUILD_NUMBER"])) - print("TRAVIS_BRANCH = {}".format(os.environ["TRAVIS_BRANCH"])) +with open("version.txt") as file: + __version__ = file.readline().strip() - __version__ += ".{}{}".format( - "" - if "release" in os.environ["TRAVIS_BRANCH"] - or os.environ["TRAVIS_BRANCH"] == "master" - else "dev", - os.environ["TRAVIS_BUILD_NUMBER"], - ) - - if ( - "release" in os.environ["TRAVIS_BRANCH"] - or os.environ["TRAVIS_BRANCH"] == "master" - ): - devStatus = "5 - Production/Stable" +# check that version is correct (X.X.X or X.X.X.devXXX or X.X.X.alphaX), eg 0.8.0.dev0 +assert re.match( + r"^\d\.\d\.\d$|^\d\.\d\.\d\.dev\d+$|^\d\.\d\.\d\.alpha\d+$", __version__ +) - else: - devStatus = devStatus +# default build status, see: https://pypi.python.org/pypi?%3Aaction=list_classifiers +if "alpha" in __version__: + dev_status = "3 - Alpha" +elif "dev" in __version__: + dev_status = "4 - Beta" else: - print("This is local build") - __version__ += ".dev0" # set version as major.minor.localbuild if local build: python setup.py install + dev_status = "5 - Production/Stable" setup( @@ -49,7 +36,7 @@ author="Alexey Burov", author_email="aburov@ptsecurity.com", classifiers=[ - "Development Status :: {}".format(devStatus), + "Development Status :: {}".format(dev_status), "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", @@ -57,13 +44,14 @@ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Libraries", "Topic :: System :: Filesystems", ], url="https://devopshq.github.io/artifactory/", download_url="https://github.com/devopshq/artifactory", install_requires=[ - 'pathlib ; python_version<"3.4"', "requests", "python-dateutil", "PyJWT", diff --git a/tests/integration/test_admin.py b/tests/integration/test_admin.py index 42c484a3..2f0641ff 100644 --- a/tests/integration/test_admin.py +++ b/tests/integration/test_admin.py @@ -127,7 +127,7 @@ def test_create_delete(self, artifactory): test_repo.delete() test_repo = RepositoryLocal( - artifactory=artifactory, name=name, packageType=RepositoryLocal.DEBIAN + artifactory=artifactory, name=name, package_type=RepositoryLocal.DEBIAN ) # CREATE test_repo.create() diff --git a/tests/integration/test_integration_artifactory_path.py b/tests/integration/test_integration_artifactory_path.py index 89ee9aec..5442e99f 100644 --- a/tests/integration/test_integration_artifactory_path.py +++ b/tests/integration/test_integration_artifactory_path.py @@ -6,6 +6,8 @@ import pytest import artifactory +from artifactory import sha1sum +from artifactory import sha256sum if sys.version_info[0] < 3: import StringIO as io @@ -173,6 +175,87 @@ def test_deploy_file(path): p.unlink() +@pytest.fixture() +def deploy_file(path): + p = path("/integration-artifactory-path-repo/foo") + p1 = path("/integration-artifactory-path-repo/bar") + if p.exists(): + p.unlink() + if p1.exists(): + p1.unlink() + + tf = tempfile.NamedTemporaryFile() + tf.write(b"Some test string") + tf.flush() + + sha1 = sha1sum(tf.name) + sha256 = sha256sum(tf.name) + + p.deploy_file(tf.name) + tf.close() + with p.open() as fd: + result = fd.read() + assert result == b"Some test string" + + return (sha1, sha256) + + +def test_deploy_file_by_checksum(path, deploy_file): + p = path("/integration-artifactory-path-repo/foo") + p1 = path("/integration-artifactory-path-repo/bar") + + # The matrix is a full list of all possible combinations + # 1st row is for 1st parameter of deploy_by_checksum + # 2nd row is for 2nd parameter and 3rd is for 3rd parameter + # matrix = [ + # [sha1, sha1_non_existent, None], + # [sha256, sha256_non_existent, None], + # [sha1, sha1_non_existent, sha256, sha256_non_existent, None] + # ] + + sha1_non_existent = "1111111111111111111111111111111111111111" + sha256_non_existent = ( + "1111111111111111111111111111111111111111111111111111111111111111" + ) + p1.deploy_by_checksum(sha1=deploy_file[0]) + with p1.open() as fd: + result = fd.read() + p1.unlink() + assert result == b"Some test string" + + with pytest.raises(RuntimeError) as excinfo: + p.deploy_by_checksum(sha1=sha1_non_existent) + + p1.deploy_by_checksum(sha256=deploy_file[1]) + with p1.open() as fd: + result = fd.read() + p1.unlink() + assert result == b"Some test string" + + with pytest.raises(RuntimeError) as excinfo: + p.deploy_by_checksum(sha256=sha256_non_existent) + + p1.deploy_by_checksum(checksum=deploy_file[0]) + with p1.open() as fd: + result = fd.read() + p1.unlink() + assert result == b"Some test string" + + with pytest.raises(RuntimeError) as excinfo: + p.deploy_by_checksum(checksum=sha1_non_existent) + + p1.deploy_by_checksum(checksum=deploy_file[1]) + with p1.open() as fd: + result = fd.read() + p1.unlink() + assert result == b"Some test string" + + with pytest.raises(RuntimeError) as excinfo: + p.deploy_by_checksum(checksum=sha256_non_existent) + + p.unlink() + + def test_open(path): p = path("/integration-artifactory-path-repo/foo") diff --git a/tests/unit/test_artifactory_path.py b/tests/unit/test_artifactory_path.py index 9b65c5a3..2d2eb76d 100644 --- a/tests/unit/test_artifactory_path.py +++ b/tests/unit/test_artifactory_path.py @@ -6,6 +6,8 @@ import dateutil import responses +from responses.matchers import json_params_matcher +from responses.matchers import query_param_matcher import artifactory from artifactory import ArtifactoryPath @@ -395,30 +397,43 @@ def test_join_with_multiple_folder_and_artifactory_substr_in_it(self): class ClassSetup(unittest.TestCase): def setUp(self): - self.file_stat = { + self.artifact_url = "http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz" + self.path = ArtifactoryPath(self.artifact_url) + self.sha1 = "fc6c9e8ba6eaca4fa97868ac900570282133c095" + self.sha256 = "fc6c9e8ba6eaca4fa97868ac900570282133c095fc6c9e8ba6eaca4fa97868ac900570282133c095" + # Response for deploying artifact by checksum + self.file_stat_without_modification_date = { "repo": "ext-release-local", "path": "/org/company/tool/1.0/tool-1.0.tar.gz", "created": "2014-02-24T21:20:59.999+04:00", "createdBy": "someuser", - "lastModified": "2014-02-24T21:20:36.000+04:00", - "modifiedBy": "anotheruser", - "lastUpdated": "2014-02-24T21:20:36.000+04:00", "downloadUri": "http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", "mimeType": "application/octet-stream", "size": "26776462", "checksums": { - "sha1": "fc6c9e8ba6eaca4fa97868ac900570282133c095", - "sha256": "fc6c9e8ba6eaca4fa97868ac900570282133c095fc6c9e8ba6eaca4fa97868ac900570282133c095", + "sha1": self.sha1, + "sha256": self.sha256, "md5": "2af7d54a09e9c36d704cb3a2de28aff3", }, "originalChecksums": { - "sha1": "fc6c9e8ba6eaca4fa97868ac900570282133c095", - "sha256": "fc6c9e8ba6eaca4fa97868ac900570282133c095fc6c9e8ba6eaca4fa97868ac900570282133c095", + "sha1": self.sha1, + "sha256": self.sha256, "md5": "2af7d54a09e9c36d704cb3a2de28aff3", }, - "uri": "http://artifactory.local/artifactory/api/storage/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", + "uri": "http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", } + # Response for file info api + self.file_stat = self.file_stat_without_modification_date.copy() + self.file_stat.update( + { + "lastModified": "2014-02-24T21:20:36.000+04:00", + "modifiedBy": "anotheruser", + "lastUpdated": "2014-02-24T21:20:36.000+04:00", + "uri": "http://artifactory.local/artifactory/api/storage/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", + } + ) + self.dir_stat = { "repo": "libs-release-local", "path": "/", @@ -441,6 +456,10 @@ def setUp(self): "uri" : "http://artifactory.local/artifactory/api/storage/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz" }""" + self.deploy_by_checksum_error = { + "errors": [{"status": 400, "message": "Checksum values not provided"}] + } + class ArtifactoryAccessorTest(ClassSetup): """Test the real artifactory integration""" @@ -650,17 +669,22 @@ def test_set_properties(self): } path = self._mock_properties_response() - path.properties = properties - # Must delete only removed property - # rest delete is a second call, use index 1 - self.assertEqual(responses.calls[1].request.params["properties"], "removethis") + resp_props = properties.copy() + resp_props["removethis"] = None + self.assertNotEqual( + properties, resp_props + ) # ensure not update original properties - # Must put all property - self.assertEqual( - responses.calls[2].request.params["properties"], - "addthis=addthis;test=test_property;time=2018-01-16 12:17:44.135143", + responses.add( + responses.PATCH, + url="http://artifactory.local/artifactory/api/metadata/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", + match=[ + json_params_matcher({"props": resp_props}), + query_param_matcher({"recursive": "0"}), + ], ) + path.properties = properties @responses.activate def test_set_properties_without_remove(self): @@ -678,12 +702,17 @@ def test_set_properties_without_remove(self): } path = self._mock_properties_response() - path.properties = properties - self.assertEqual( - responses.calls[1].request.params["properties"], - "addthis=addthis;removethis=removethis_property;test=test_property;time=2018-01-16 12:17:44.135143", + responses.add( + responses.PATCH, + url="http://artifactory.local/artifactory/api/metadata/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz", + match=[ + json_params_matcher({"props": properties}), + query_param_matcher({"recursive": "0"}), + ], ) + path.properties = properties + @staticmethod def _mock_properties_response(): """ @@ -713,18 +742,6 @@ def _mock_properties_response(): "uri": constructed_url, }, ) - responses.add( - responses.DELETE, - constructed_url, - status=204, - body="", - ) - responses.add( - responses.PUT, - constructed_url, - status=204, - body="", - ) return path @responses.activate @@ -896,6 +913,114 @@ def test_deploy_file(self): self.assertIn("X-Explode-Archive-Atomic", headers) self.assertEqual(headers["X-Explode-Archive-Atomic"], "true") + def test_deploy_by_checksum_sha1(self): + """ + Test that file is deployed by sha1 + :return: + """ + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.file_stat_without_modification_date, + status=200, + ) + self.path.deploy_by_checksum(sha1=self.sha1) + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum-Sha1"], self.sha1) + self.assertNotIn("X-Checksum-Sha256", headers) + self.assertNotIn("X-Checksum", headers) + + def test_deploy_by_checksum_sha256(self): + """ + Test that file is deployed by sha256 + :return: + """ + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.file_stat_without_modification_date, + status=200, + ) + self.path.deploy_by_checksum(sha256=self.sha256) + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum-Sha256"], self.sha256) + self.assertNotIn("X-Checksum-Sha1", headers) + self.assertNotIn("X-Checksum", headers) + + def test_deploy_by_checksum_sha1_or_sha256(self): + """ + Test that file is deployed by sha1 or sha256 + :return: + """ + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.file_stat_without_modification_date, + status=200, + ) + self.path.deploy_by_checksum(checksum=self.sha1) + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum"], self.sha1) + self.assertNotIn("X-Checksum-Sha1", headers) + self.assertNotIn("X-Checksum-Sha256", headers) + + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.file_stat_without_modification_date, + status=200, + ) + self.path.deploy_by_checksum(checksum=self.sha256) + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum"], self.sha256) + self.assertNotIn("X-Checksum-Sha1", headers) + self.assertNotIn("X-Checksum-Sha256", headers) + + def test_deploy_by_checksum_error(self): + """ + Test that file is deployed by checksum, which raises error + :return: + """ + with responses.RequestsMock() as rsps: + rsps.add( + responses.PUT, + self.artifact_url, + json=self.deploy_by_checksum_error, + status=400, + ) + with self.assertRaises(ArtifactoryException) as context: + self.path.deploy_by_checksum(sha1=f"{self.sha1}invalid") + + self.assertEqual(str(context.exception), "Checksum values not provided") + + self.assertEqual(len(rsps.calls), 1) + self.assertEqual(rsps.calls[0].request.url, self.artifact_url) + headers = rsps.calls[0].request.headers + self.assertEqual(headers["X-Checksum-Deploy"], "true") + self.assertEqual(headers["X-Checksum-Sha1"], f"{self.sha1}invalid") + self.assertNotIn("X-Checksum-Sha256", headers) + self.assertNotIn("X-Checksum", headers) + @responses.activate def test_deploy_deb(self): """ diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 3bef6892..44062474 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -1,21 +1,41 @@ #!/usr/bin/env python import unittest -import mock import requests +import responses -from dohq_artifactory.admin import raise_errors from dohq_artifactory.exception import ArtifactoryException +from dohq_artifactory.exception import raise_for_status class UtilTest(unittest.TestCase): def test_raise_errors(self): - r = requests.Response() - r.status_code = 400 - type(r).text = mock.PropertyMock(return_value="asd") - with self.assertRaises(ArtifactoryException) as cm: - raise_errors(r) - self.assertEqual("asd", str(cm.exception)) + # no JSON body, just HTTP response message + with responses.RequestsMock() as mock: + url = "http://b.com/artifactory/" + mock.add(responses.GET, url, status=403) + resp = requests.get("http://b.com/artifactory/") + + with self.assertRaises(ArtifactoryException) as cm: + raise_for_status(resp) + self.assertEqual( + f"403 Client Error: Forbidden for url: {url}", str(cm.exception) + ) + + # real JSON body, can parse for clean message + with responses.RequestsMock() as mock: + url = "http://b.com/artifactory/" + mock.add( + responses.GET, + url, + status=403, + json={"errors": [{"status": 401, "message": "Bad credentials"}]}, + ) + resp = requests.get("http://b.com/artifactory/") + + with self.assertRaises(ArtifactoryException) as cm: + raise_for_status(resp) + self.assertEqual("Bad credentials", str(cm.exception)) if __name__ == "__main__": diff --git a/tox.ini b/tox.ini index d7fba88d..3767f8ba 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,8 @@ python = 3.6: py36 3.7: py37, pre-commit 3.8: py38 + 3.9: py39 + 3.10: py310 [testenv] deps = -rrequirements-dev.txt diff --git a/version.txt b/version.txt new file mode 100644 index 00000000..42ff8542 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +0.8.0.alpha0 \ No newline at end of file