From 7c271595f33c778ae11098757704eb7e293db314 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sun, 21 Jan 2024 21:36:08 +0100 Subject: [PATCH] Add docker_image_export module. --- README.md | 1 + docs/docsite/rst/scenario_guide.rst | 3 + meta/runtime.yml | 1 + plugins/module_utils/image_archive.py | 123 +++++--- plugins/modules/docker_image_export.py | 279 ++++++++++++++++++ .../docker_image_load/tasks/tests/basic.yml | 4 +- 6 files changed, 364 insertions(+), 47 deletions(-) create mode 100644 plugins/modules/docker_image_export.py diff --git a/README.md b/README.md index a86735a39..edf443b9b 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ If you use the Ansible package and do not update collections independently, use - community.docker.docker_host_info: retrieve information on the Docker daemon - community.docker.docker_image: manage Docker images - community.docker.docker_image_build: build Docker images using Docker buildx + - community.docker.docker_image_export: export (archive) Docker images - community.docker.docker_image_info: retrieve information on Docker images - community.docker.docker_image_load: load Docker images from archives - community.docker.docker_image_pull: pull Docker images from registries diff --git a/docs/docsite/rst/scenario_guide.rst b/docs/docsite/rst/scenario_guide.rst index 29e37b932..203aa57d2 100644 --- a/docs/docsite/rst/scenario_guide.rst +++ b/docs/docsite/rst/scenario_guide.rst @@ -203,6 +203,9 @@ For working with a plain Docker daemon, that is without Swarm, there are connect docker_image_build The :ansplugin:`community.docker.docker_image_build module ` allows you to build a Docker image using Docker buildx. + docker_image_export module + The :ansplugin:`community.docker.docker_image_export module ` allows you to export (archive) images. + docker_image_info module The :ansplugin:`community.docker.docker_image_info module ` allows you to list and inspect images. diff --git a/meta/runtime.yml b/meta/runtime.yml index 688a6853d..22f4947db 100644 --- a/meta/runtime.yml +++ b/meta/runtime.yml @@ -16,6 +16,7 @@ action_groups: - docker_host_info - docker_image - docker_image_build + - docker_image_export - docker_image_info - docker_image_load - docker_image_pull diff --git a/plugins/module_utils/image_archive.py b/plugins/module_utils/image_archive.py index 11e615f2b..46b5abc14 100644 --- a/plugins/module_utils/image_archive.py +++ b/plugins/module_utils/image_archive.py @@ -23,7 +23,7 @@ def __init__(self, image_id, repo_tags): :param image_id: File name portion of Config entry, e.g. abcde12345 from abcde12345.json :type image_id: str :param repo_tags Docker image names, e.g. ["hello-world:latest"] - :type repo_tags: list + :type repo_tags: list[str] ''' self.image_id = image_id @@ -60,13 +60,13 @@ def api_image_id(archive_image_id): return 'sha256:%s' % archive_image_id -def archived_image_manifest(archive_path): +def load_archived_image_manifest(archive_path): ''' - Attempts to get Image.Id and image name from metadata stored in the image + Attempts to get image IDs and image names from metadata stored in the image archive tar file. - The tar should contain a file "manifest.json" with an array with a single entry, - and the entry should have a Config field with the image ID in its file name, as + The tar should contain a file "manifest.json" with an array with one or more entries, + and every entry should have a Config field with the image ID in its file name, as well as a RepoTags list, which typically has only one entry. :raises: @@ -75,7 +75,7 @@ def archived_image_manifest(archive_path): :param archive_path: Tar file to read :type archive_path: str - :return: None, if no file at archive_path, or the extracted image ID, which will not have a sha256: prefix. + :return: None, if no file at archive_path, or a list of ImageArchiveManifestSummary objects. :rtype: ImageArchiveManifestSummary ''' @@ -100,50 +100,51 @@ def archived_image_manifest(archive_path): # In Python 2.6, this does not have __exit__ ef.close() - if len(manifest) != 1: + if len(manifest) == 0: raise ImageArchiveInvalidException( - "Expected to have one entry in manifest.json but found %s" % len(manifest), + "Expected to have at least one entry in manifest.json but found none", None ) - m0 = manifest[0] - - try: - config_file = m0['Config'] - except KeyError as exc: - raise ImageArchiveInvalidException( - "Failed to get Config entry from manifest.json: %s" % to_native(exc), - exc - ) - - # Extracts hash without 'sha256:' prefix - try: - # Strip off .json filename extension, leaving just the hash. - image_id = os.path.splitext(config_file)[0] - except Exception as exc: - raise ImageArchiveInvalidException( - "Failed to extract image id from config file name %s: %s" % (config_file, to_native(exc)), - exc - ) - - for prefix in ( - 'blobs/sha256/', # Moby 25.0.0, Docker API 1.44 - ): - if image_id.startswith(prefix): - image_id = image_id[len(prefix):] - - try: - repo_tags = m0['RepoTags'] - except KeyError as exc: - raise ImageArchiveInvalidException( - "Failed to get RepoTags entry from manifest.json: %s" % to_native(exc), - exc - ) - - return ImageArchiveManifestSummary( - image_id=image_id, - repo_tags=repo_tags - ) + result = [] + for index, meta in enumerate(manifest): + try: + config_file = meta['Config'] + except KeyError as exc: + raise ImageArchiveInvalidException( + "Failed to get Config entry from {0}th manifest in manifest.json: {1}".format(index + 1, to_native(exc)), + exc + ) + + # Extracts hash without 'sha256:' prefix + try: + # Strip off .json filename extension, leaving just the hash. + image_id = os.path.splitext(config_file)[0] + except Exception as exc: + raise ImageArchiveInvalidException( + "Failed to extract image id from config file name %s: %s" % (config_file, to_native(exc)), + exc + ) + + for prefix in ( + 'blobs/sha256/', # Moby 25.0.0, Docker API 1.44 + ): + if image_id.startswith(prefix): + image_id = image_id[len(prefix):] + + try: + repo_tags = meta['RepoTags'] + except KeyError as exc: + raise ImageArchiveInvalidException( + "Failed to get RepoTags entry from {0}th manifest in manifest.json: {1}".format(index + 1, to_native(exc)), + exc + ) + + result.append(ImageArchiveManifestSummary( + image_id=image_id, + repo_tags=repo_tags + )) + return result except ImageArchiveInvalidException: raise @@ -161,3 +162,33 @@ def archived_image_manifest(archive_path): raise except Exception as exc: raise ImageArchiveInvalidException("Failed to open tar file %s: %s" % (archive_path, to_native(exc)), exc) + + +def archived_image_manifest(archive_path): + ''' + Attempts to get Image.Id and image name from metadata stored in the image + archive tar file. + + The tar should contain a file "manifest.json" with an array with a single entry, + and the entry should have a Config field with the image ID in its file name, as + well as a RepoTags list, which typically has only one entry. + + :raises: + ImageArchiveInvalidException: A file already exists at archive_path, but could not extract an image ID from it. + + :param archive_path: Tar file to read + :type archive_path: str + + :return: None, if no file at archive_path, or the extracted image ID, which will not have a sha256: prefix. + :rtype: ImageArchiveManifestSummary + ''' + + results = load_archived_image_manifest(archive_path) + if results is None: + return None + if len(results) == 1: + return results[0] + raise ImageArchiveInvalidException( + "Expected to have one entry in manifest.json but found %s" % len(results), + None + ) diff --git a/plugins/modules/docker_image_export.py b/plugins/modules/docker_image_export.py new file mode 100644 index 000000000..96e4fb4bb --- /dev/null +++ b/plugins/modules/docker_image_export.py @@ -0,0 +1,279 @@ +#!/usr/bin/python +# +# Copyright (c) 2023, Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +DOCUMENTATION = ''' +--- +module: docker_image_export + +short_description: Export (archive) Docker images + +version_added: 3.7.0 + +description: + - Creates an archive (tarball) from one or more Docker images. + - This can be copied to another machine and loaded with M(community.docker.docker_image_load). + +extends_documentation_fragment: + - community.docker.docker.api_documentation + - community.docker.attributes + - community.docker.attributes.actiongroup_docker + +attributes: + check_mode: + support: full + diff_mode: + support: none + +options: + names: + description: + - "One or more image names. Name format will be one of: C(name), C(repository/name), C(registry_server:port/name). + When pushing or pulling an image the name can optionally include the tag by appending C(:tag_name)." + - Note that image IDs (hashes) can also be used. + type: list + elements: str + required: true + aliases: + - name + tag: + description: + - Tag for the image name O(name) that is to be tagged. + - If O(name)'s format is C(name:tag), then the tag value from O(name) will take precedence. + type: str + default: latest + path: + description: + - The C(.tar) file the image should be exported to. + type: path + force: + description: + - Export the image even if the C(.tar) file already exists and seems to contain the right image. + type: bool + default: false + +requirements: + - "Docker API >= 1.25" + +author: + - Felix Fontein (@felixfontein) + +seealso: + - module: community.docker.docker_image + - module: community.docker.docker_image_info + - module: community.docker.docker_image_load +''' + +EXAMPLES = ''' +- name: Export an image + community.docker.docker_image_export: + name: pacur/centos-7 + path: /tmp/centos-7.tar + +- name: Export multiple images + community.docker.docker_image_export: + names: + - hello-world:latest + - pacur/centos-7:latest + path: /tmp/various.tar +''' + +RETURN = ''' +images: + description: Image inspection results for the affected images. + returned: success + type: list + elements: dict + sample: [] +''' + +import traceback + +from ansible.module_utils.common.text.converters import to_native + +from ansible_collections.community.docker.plugins.module_utils.common_api import ( + AnsibleDockerClient, + RequestException, +) + +from ansible_collections.community.docker.plugins.module_utils.image_archive import ( + load_archived_image_manifest, + api_image_id, + ImageArchiveInvalidException, +) + +from ansible_collections.community.docker.plugins.module_utils.util import ( + DockerBaseClass, + is_image_name_id, + is_valid_tag, +) +from ansible_collections.community.docker.plugins.module_utils._api.constants import ( + DEFAULT_DATA_CHUNK_SIZE, +) +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException +from ansible_collections.community.docker.plugins.module_utils._api.utils.utils import ( + parse_repository_tag, +) + + +class ImageExportManager(DockerBaseClass): + def __init__(self, client): + super(ImageExportManager, self).__init__() + + self.client = client + parameters = self.client.module.params + self.check_mode = self.client.check_mode + + self.path = parameters['path'] + self.force = parameters['force'] + self.tag = parameters['tag'] + + if not is_valid_tag(self.tag, allow_empty=True): + self.fail('"{0}" is not a valid docker tag'.format(self.tag)) + + # If name contains a tag, it takes precedence over tag parameter. + self.names = [] + for name in parameters['names']: + if is_image_name_id(name): + self.names.append({'id': name, 'joined': name}) + else: + repo, repo_tag = parse_repository_tag(name) + if not repo_tag: + repo_tag = self.tag + self.names.append({'name': repo, 'tag': repo_tag, 'joined': '%s:%s' % (repo, repo_tag)}) + + if not self.names: + self.fail('At least one image name must be specified') + + def fail(self, msg): + self.client.fail(msg) + + def get_export_reason(self): + if self.force: + return 'Exporting since force=true' + + try: + archived_images = load_archived_image_manifest(self.path) + if archived_images is None: + return 'Overwriting since no image is present in archive' + except ImageArchiveInvalidException as exc: + self.log('Unable to extract manifest summary from archive: %s' % to_native(exc)) + return 'Overwriting an unreadable archive file' + + left_names = list(self.names) + for archived_image in archived_images: + found = False + for i, name in enumerate(left_names): + if name['id'] == api_image_id(archived_image.image_id) and [name['joined']] == archived_image.repo_tags: + del left_names[i] + found = True + break + if not found: + return 'Overwriting archive since it contains unexpected image %s named %s' % ( + archived_image.image_id, ', '.join(archived_image.repo_tags) + ) + if left_names: + return 'Overwriting archive since it is missing image(s) %s' % (', '.join([name['joined'] for name in left_names])) + + return None + + def write_chunks(self, chunks): + try: + with open(self.path, 'wb') as fd: + for chunk in chunks: + fd.write(chunk) + except Exception as exc: + self.fail("Error writing image archive %s - %s" % (self.path, to_native(exc))) + + def export_images(self): + image_names = [name['joined'] for name in self.names] + image_names_str = ', '.join(image_names) + if len(image_names) == 1: + self.log("Getting archive of image %s" % image_names[0]) + try: + chunks = self.client._stream_raw_result( + self.client._get(self.client._url('/images/{0}/get', image_names[0]), stream=True), + DEFAULT_DATA_CHUNK_SIZE, + False, + ) + except Exception as exc: + self.fail("Error getting image %s - %s" % (image_names[0], to_native(exc))) + else: + self.log("Getting archive of images %s" % image_names_str) + try: + chunks = self.client._stream_raw_result( + self.client._get(self.client._url('/images/get', names=image_names), stream=True), + DEFAULT_DATA_CHUNK_SIZE, + False, + ) + except Exception as exc: + self.fail("Error getting images %s - %s" % (image_names_str, to_native(exc))) + + self.write_chunks(chunks) + + def run(self): + tag = self.tag + if not tag: + tag = "latest" + + images = [] + for name in self.names: + if 'id' in name: + image = self.client.find_image_by_id(name['id'], accept_missing_image=True) + else: + image = self.client.find_image(name=name['name'], tag=name['tag']) + if not image: + self.fail("Image %s not found" % name['joined']) + images.append(image) + + # Will have a 'sha256:' prefix + name['id'] = image['Id'] + + results = { + 'changed': False, + 'images': images, + } + + reason = self.get_export_reason() + if reason is not None: + results['msg'] = reason + results['changed'] = True + + if not self.check_mode: + self.export_images() + + return results + + +def main(): + argument_spec = dict( + path=dict(type='path'), + force=dict(type='bool', default=False), + names=dict(type='list', elements='str', required=True, aliases=['name']), + tag=dict(type='str', default='latest'), + ) + + client = AnsibleDockerClient( + argument_spec=argument_spec, + supports_check_mode=True, + ) + + try: + results = ImageExportManager(client).run() + client.module.exit_json(**results) + except DockerException as e: + client.fail('An unexpected Docker error occurred: {0}'.format(to_native(e)), exception=traceback.format_exc()) + except RequestException as e: + client.fail( + 'An unexpected requests error occurred when trying to talk to the Docker daemon: {0}'.format(to_native(e)), + exception=traceback.format_exc()) + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/docker_image_load/tasks/tests/basic.yml b/tests/integration/targets/docker_image_load/tasks/tests/basic.yml index 42808770e..21a27d246 100644 --- a/tests/integration/targets/docker_image_load/tasks/tests/basic.yml +++ b/tests/integration/targets/docker_image_load/tasks/tests/basic.yml @@ -21,7 +21,9 @@ all_images: "{{ image_names + (images.results | map(attribute='image') | map(attribute='Id') | list) }}" - name: Create archives - command: docker save {{ item.images | join(' ') }} -o {{ remote_tmp_dir }}/{{ item.file }} + docker_image_export: + names: "{{ item.images }}" + path: "{{ item.file }}" loop: - file: archive-1.tar images: "{{ image_names }}"