diff --git a/.github/workflows/buf-pull-request.yml b/.github/workflows/buf-pull-request.yml new file mode 100644 index 000000000..8139a48a4 --- /dev/null +++ b/.github/workflows/buf-pull-request.yml @@ -0,0 +1,15 @@ +name: Buf (pull request) +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: bufbuild/buf-setup-action@v1 + - uses: bufbuild/buf-lint-action@v1 + with: + input: proto + # - uses: bufbuild/buf-breaking-action@v1 + # with: + # input: proto + # against: 'https://github.com/${GITHUB_REPOSITORY}.git#branch=main' diff --git a/.github/workflows/buf-push.yml b/.github/workflows/buf-push.yml new file mode 100644 index 000000000..abb7521d7 --- /dev/null +++ b/.github/workflows/buf-push.yml @@ -0,0 +1,22 @@ +name: Buf (push) +on: + push: + branches: + - main +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: bufbuild/buf-setup-action@v1 + - uses: bufbuild/buf-lint-action@v1 + with: + input: proto + # - uses: bufbuild/buf-breaking-action@v1 + # with: + # input: proto + # against: 'https://github.com/${GITHUB_REPOSITORY}.git#branch=main' + - uses: bufbuild/buf-push-action@v1 + with: + input: proto + buf_token: ${{ secrets.BUF_TOKEN }} diff --git a/Makefile b/Makefile index 0f7027e83..2536000dc 100644 --- a/Makefile +++ b/Makefile @@ -134,16 +134,13 @@ amflow: ## Display the workflow in amflow. artefactual/amflow:latest \ edit --file=/workflow.json -.PHONY: protoc -protoc: ## Generate gRPC code. - $(call compose_run, \ - --entrypoint python \ - a3m \ - -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. a3m/server/rpc/proto/a3m.proto) - $(call compose_run, \ - --entrypoint black \ - a3m \ - a3m/server/rpc/proto) +.PHONY: buf +buf: + docker run \ + --volume "$(CURDIR)/proto:/workspace" \ + --workdir /workspace \ + bufbuild/buf:1.4.0 \ + $(ARG) .PHONY: help help: ## Print this help message. diff --git a/a3m/api/transferservice/v1beta1/__init__.py b/a3m/api/transferservice/v1beta1/__init__.py new file mode 100644 index 000000000..9d6299b4d --- /dev/null +++ b/a3m/api/transferservice/v1beta1/__init__.py @@ -0,0 +1,12 @@ +from . import request_response_pb2 +from . import request_response_pb2_grpc +from . import service_pb2 +from . import service_pb2_grpc + + +__all__ = [ + "request_response_pb2_grpc", + "request_response_pb2", + "service_pb2_grpc", + "service_pb2", +] diff --git a/a3m/api/transferservice/v1beta1/request_response_pb2.py b/a3m/api/transferservice/v1beta1/request_response_pb2.py new file mode 100644 index 000000000..52d42f7a6 --- /dev/null +++ b/a3m/api/transferservice/v1beta1/request_response_pb2.py @@ -0,0 +1,53 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: a3m/api/transferservice/v1beta1/request_response.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n6a3m/api/transferservice/v1beta1/request_response.proto\x12\x1f\x61\x33m.api.transferservice.v1beta1\x1a\x1fgoogle/protobuf/timestamp.proto"\x80\x01\n\rSubmitRequest\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x10\n\x03url\x18\x02 \x01(\tR\x03url\x12I\n\x06\x63onfig\x18\x03 \x01(\x0b\x32\x31.a3m.api.transferservice.v1beta1.ProcessingConfigR\x06\x63onfig" \n\x0eSubmitResponse\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id"\x1d\n\x0bReadRequest\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id"\xa2\x01\n\x0cReadResponse\x12\x46\n\x06status\x18\x01 \x01(\x0e\x32..a3m.api.transferservice.v1beta1.PackageStatusR\x06status\x12\x10\n\x03job\x18\x02 \x01(\tR\x03job\x12\x38\n\x04jobs\x18\x03 \x03(\x0b\x32$.a3m.api.transferservice.v1beta1.JobR\x04jobs")\n\x10ListTasksRequest\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId"P\n\x11ListTasksResponse\x12;\n\x05tasks\x18\x01 \x03(\x0b\x32%.a3m.api.transferservice.v1beta1.TaskR\x05tasks"\xb9\x02\n\x03Job\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x14\n\x05group\x18\x03 \x01(\tR\x05group\x12\x17\n\x07link_id\x18\x04 \x01(\tR\x06linkId\x12\x43\n\x06status\x18\x05 \x01(\x0e\x32+.a3m.api.transferservice.v1beta1.Job.StatusR\x06status\x12\x39\n\nstart_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime"_\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x13\n\x0fSTATUS_COMPLETE\x10\x01\x12\x15\n\x11STATUS_PROCESSING\x10\x02\x12\x11\n\rSTATUS_FAILED\x10\x03"\xc6\x02\n\x04Task\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x17\n\x07\x66ile_id\x18\x02 \x01(\tR\x06\x66ileId\x12\x1b\n\texit_code\x18\x03 \x01(\x05R\x08\x65xitCode\x12\x1a\n\x08\x66ilename\x18\x04 \x01(\tR\x08\x66ilename\x12\x1c\n\texecution\x18\x05 \x01(\tR\texecution\x12\x1c\n\targuments\x18\x06 \x01(\tR\targuments\x12\x16\n\x06stdout\x18\x07 \x01(\tR\x06stdout\x12\x16\n\x06stderr\x18\x08 \x01(\tR\x06stderr\x12\x39\n\nstart_time\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12\x35\n\x08\x65nd_time\x18\n \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x07\x65ndTime"\xcc\n\n\x10ProcessingConfig\x12=\n\x1b\x61ssign_uuids_to_directories\x18\x01 \x01(\x08R\x18\x61ssignUuidsToDirectories\x12)\n\x10\x65xamine_contents\x18\x02 \x01(\x08R\x0f\x65xamineContents\x12K\n"generate_transfer_structure_report\x18\x03 \x01(\x08R\x1fgenerateTransferStructureReport\x12<\n\x1a\x64ocument_empty_directories\x18\x04 \x01(\x08R\x18\x64ocumentEmptyDirectories\x12)\n\x10\x65xtract_packages\x18\x05 \x01(\x08R\x0f\x65xtractPackages\x12G\n delete_packages_after_extraction\x18\x06 \x01(\x08R\x1d\x64\x65letePackagesAfterExtraction\x12+\n\x11identify_transfer\x18\x07 \x01(\x08R\x10identifyTransfer\x12G\n identify_submission_and_metadata\x18\x08 \x01(\x08R\x1didentifySubmissionAndMetadata\x12\x42\n\x1didentify_before_normalization\x18\t \x01(\x08R\x1bidentifyBeforeNormalization\x12\x1c\n\tnormalize\x18\n \x01(\x08R\tnormalize\x12)\n\x10transcribe_files\x18\x0b \x01(\x08R\x0ftranscribeFiles\x12J\n"perform_policy_checks_on_originals\x18\x0c \x01(\x08R\x1eperformPolicyChecksOnOriginals\x12g\n1perform_policy_checks_on_preservation_derivatives\x18\r \x01(\x08R,performPolicyChecksOnPreservationDerivatives\x12\x32\n\x15\x61ip_compression_level\x18\x0e \x01(\x05R\x13\x61ipCompressionLevel\x12\x85\x01\n\x19\x61ip_compression_algorithm\x18\x0f \x01(\x0e\x32I.a3m.api.transferservice.v1beta1.ProcessingConfig.AIPCompressionAlgorithmR\x17\x61ipCompressionAlgorithm"\xda\x02\n\x17\x41IPCompressionAlgorithm\x12)\n%AIP_COMPRESSION_ALGORITHM_UNSPECIFIED\x10\x00\x12*\n&AIP_COMPRESSION_ALGORITHM_UNCOMPRESSED\x10\x01\x12!\n\x1d\x41IP_COMPRESSION_ALGORITHM_TAR\x10\x02\x12\'\n#AIP_COMPRESSION_ALGORITHM_TAR_BZIP2\x10\x03\x12&\n"AIP_COMPRESSION_ALGORITHM_TAR_GZIP\x10\x04\x12%\n!AIP_COMPRESSION_ALGORITHM_S7_COPY\x10\x05\x12&\n"AIP_COMPRESSION_ALGORITHM_S7_BZIP2\x10\x06\x12%\n!AIP_COMPRESSION_ALGORITHM_S7_LZMA\x10\x07*\xa3\x01\n\rPackageStatus\x12\x1e\n\x1aPACKAGE_STATUS_UNSPECIFIED\x10\x00\x12\x19\n\x15PACKAGE_STATUS_FAILED\x10\x01\x12\x1b\n\x17PACKAGE_STATUS_REJECTED\x10\x02\x12\x1b\n\x17PACKAGE_STATUS_COMPLETE\x10\x03\x12\x1d\n\x19PACKAGE_STATUS_PROCESSING\x10\x04\x42\xb1\x02\n#com.a3m.api.transferservice.v1beta1B\x14RequestResponseProtoP\x01ZUgithub.com/artefactual-labs/a3m/proto/a3m/api/transferservice/v1beta1;transferservice\xa2\x02\x03\x41\x41T\xaa\x02\x1f\x41\x33m.Api.Transferservice.V1beta1\xca\x02\x1f\x41\x33m\\Api\\Transferservice\\V1beta1\xe2\x02+A3m\\Api\\Transferservice\\V1beta1\\GPBMetadata\xea\x02"A3m::Api::Transferservice::V1beta1b\x06proto3' +) + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages( + DESCRIPTOR, "a3m.api.transferservice.v1beta1.request_response_pb2", globals() +) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n#com.a3m.api.transferservice.v1beta1B\024RequestResponseProtoP\001ZUgithub.com/artefactual-labs/a3m/proto/a3m/api/transferservice/v1beta1;transferservice\242\002\003AAT\252\002\037A3m.Api.Transferservice.V1beta1\312\002\037A3m\\Api\\Transferservice\\V1beta1\342\002+A3m\\Api\\Transferservice\\V1beta1\\GPBMetadata\352\002"A3m::Api::Transferservice::V1beta1' + _PACKAGESTATUS._serialized_start = 2615 + _PACKAGESTATUS._serialized_end = 2778 + _SUBMITREQUEST._serialized_start = 125 + _SUBMITREQUEST._serialized_end = 253 + _SUBMITRESPONSE._serialized_start = 255 + _SUBMITRESPONSE._serialized_end = 287 + _READREQUEST._serialized_start = 289 + _READREQUEST._serialized_end = 318 + _READRESPONSE._serialized_start = 321 + _READRESPONSE._serialized_end = 483 + _LISTTASKSREQUEST._serialized_start = 485 + _LISTTASKSREQUEST._serialized_end = 526 + _LISTTASKSRESPONSE._serialized_start = 528 + _LISTTASKSRESPONSE._serialized_end = 608 + _JOB._serialized_start = 611 + _JOB._serialized_end = 924 + _JOB_STATUS._serialized_start = 829 + _JOB_STATUS._serialized_end = 924 + _TASK._serialized_start = 927 + _TASK._serialized_end = 1253 + _PROCESSINGCONFIG._serialized_start = 1256 + _PROCESSINGCONFIG._serialized_end = 2612 + _PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM._serialized_start = 2266 + _PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM._serialized_end = 2612 +# @@protoc_insertion_point(module_scope) diff --git a/a3m/api/transferservice/v1beta1/request_response_pb2.pyi b/a3m/api/transferservice/v1beta1/request_response_pb2.pyi new file mode 100644 index 000000000..e82aea2ab --- /dev/null +++ b/a3m/api/transferservice/v1beta1/request_response_pb2.pyi @@ -0,0 +1,442 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import google.protobuf.timestamp_pb2 +import typing +import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _PackageStatus: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _PackageStatusEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + _PackageStatus.ValueType + ], + builtins.type, +): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + PACKAGE_STATUS_UNSPECIFIED: _PackageStatus.ValueType # 0 + PACKAGE_STATUS_FAILED: _PackageStatus.ValueType # 1 + PACKAGE_STATUS_REJECTED: _PackageStatus.ValueType # 2 + PACKAGE_STATUS_COMPLETE: _PackageStatus.ValueType # 3 + PACKAGE_STATUS_PROCESSING: _PackageStatus.ValueType # 4 + +class PackageStatus(_PackageStatus, metaclass=_PackageStatusEnumTypeWrapper): + pass + +PACKAGE_STATUS_UNSPECIFIED: PackageStatus.ValueType # 0 +PACKAGE_STATUS_FAILED: PackageStatus.ValueType # 1 +PACKAGE_STATUS_REJECTED: PackageStatus.ValueType # 2 +PACKAGE_STATUS_COMPLETE: PackageStatus.ValueType # 3 +PACKAGE_STATUS_PROCESSING: PackageStatus.ValueType # 4 +global___PackageStatus = PackageStatus + +class SubmitRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + NAME_FIELD_NUMBER: builtins.int + URL_FIELD_NUMBER: builtins.int + CONFIG_FIELD_NUMBER: builtins.int + name: typing.Text + url: typing.Text + @property + def config(self) -> global___ProcessingConfig: ... + def __init__( + self, + *, + name: typing.Text = ..., + url: typing.Text = ..., + config: typing.Optional[global___ProcessingConfig] = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["config", b"config"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "config", b"config", "name", b"name", "url", b"url" + ], + ) -> None: ... + +global___SubmitRequest = SubmitRequest + +class SubmitResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + ID_FIELD_NUMBER: builtins.int + id: typing.Text + def __init__( + self, + *, + id: typing.Text = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["id", b"id"] + ) -> None: ... + +global___SubmitResponse = SubmitResponse + +class ReadRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + ID_FIELD_NUMBER: builtins.int + id: typing.Text + def __init__( + self, + *, + id: typing.Text = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["id", b"id"] + ) -> None: ... + +global___ReadRequest = ReadRequest + +class ReadResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + STATUS_FIELD_NUMBER: builtins.int + JOB_FIELD_NUMBER: builtins.int + JOBS_FIELD_NUMBER: builtins.int + status: global___PackageStatus.ValueType + job: typing.Text + @property + def jobs( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___Job + ]: ... + def __init__( + self, + *, + status: global___PackageStatus.ValueType = ..., + job: typing.Text = ..., + jobs: typing.Optional[typing.Iterable[global___Job]] = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "job", b"job", "jobs", b"jobs", "status", b"status" + ], + ) -> None: ... + +global___ReadResponse = ReadResponse + +class ListTasksRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + JOB_ID_FIELD_NUMBER: builtins.int + job_id: typing.Text + def __init__( + self, + *, + job_id: typing.Text = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["job_id", b"job_id"] + ) -> None: ... + +global___ListTasksRequest = ListTasksRequest + +class ListTasksResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + TASKS_FIELD_NUMBER: builtins.int + @property + def tasks( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___Task + ]: ... + def __init__( + self, + *, + tasks: typing.Optional[typing.Iterable[global___Task]] = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["tasks", b"tasks"] + ) -> None: ... + +global___ListTasksResponse = ListTasksResponse + +class Job(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class _Status: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + + class _StatusEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + Job._Status.ValueType + ], + builtins.type, + ): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + STATUS_UNSPECIFIED: Job._Status.ValueType # 0 + STATUS_COMPLETE: Job._Status.ValueType # 1 + STATUS_PROCESSING: Job._Status.ValueType # 2 + STATUS_FAILED: Job._Status.ValueType # 3 + + class Status(_Status, metaclass=_StatusEnumTypeWrapper): + pass + STATUS_UNSPECIFIED: Job.Status.ValueType # 0 + STATUS_COMPLETE: Job.Status.ValueType # 1 + STATUS_PROCESSING: Job.Status.ValueType # 2 + STATUS_FAILED: Job.Status.ValueType # 3 + + ID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + GROUP_FIELD_NUMBER: builtins.int + LINK_ID_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + START_TIME_FIELD_NUMBER: builtins.int + id: typing.Text + name: typing.Text + group: typing.Text + link_id: typing.Text + status: global___Job.Status.ValueType + @property + def start_time(self) -> google.protobuf.timestamp_pb2.Timestamp: ... + def __init__( + self, + *, + id: typing.Text = ..., + name: typing.Text = ..., + group: typing.Text = ..., + link_id: typing.Text = ..., + status: global___Job.Status.ValueType = ..., + start_time: typing.Optional[google.protobuf.timestamp_pb2.Timestamp] = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["start_time", b"start_time"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "group", + b"group", + "id", + b"id", + "link_id", + b"link_id", + "name", + b"name", + "start_time", + b"start_time", + "status", + b"status", + ], + ) -> None: ... + +global___Job = Job + +class Task(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + ID_FIELD_NUMBER: builtins.int + FILE_ID_FIELD_NUMBER: builtins.int + EXIT_CODE_FIELD_NUMBER: builtins.int + FILENAME_FIELD_NUMBER: builtins.int + EXECUTION_FIELD_NUMBER: builtins.int + ARGUMENTS_FIELD_NUMBER: builtins.int + STDOUT_FIELD_NUMBER: builtins.int + STDERR_FIELD_NUMBER: builtins.int + START_TIME_FIELD_NUMBER: builtins.int + END_TIME_FIELD_NUMBER: builtins.int + id: typing.Text + file_id: typing.Text + exit_code: builtins.int + filename: typing.Text + execution: typing.Text + arguments: typing.Text + stdout: typing.Text + stderr: typing.Text + @property + def start_time(self) -> google.protobuf.timestamp_pb2.Timestamp: ... + @property + def end_time(self) -> google.protobuf.timestamp_pb2.Timestamp: ... + def __init__( + self, + *, + id: typing.Text = ..., + file_id: typing.Text = ..., + exit_code: builtins.int = ..., + filename: typing.Text = ..., + execution: typing.Text = ..., + arguments: typing.Text = ..., + stdout: typing.Text = ..., + stderr: typing.Text = ..., + start_time: typing.Optional[google.protobuf.timestamp_pb2.Timestamp] = ..., + end_time: typing.Optional[google.protobuf.timestamp_pb2.Timestamp] = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "end_time", b"end_time", "start_time", b"start_time" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "arguments", + b"arguments", + "end_time", + b"end_time", + "execution", + b"execution", + "exit_code", + b"exit_code", + "file_id", + b"file_id", + "filename", + b"filename", + "id", + b"id", + "start_time", + b"start_time", + "stderr", + b"stderr", + "stdout", + b"stdout", + ], + ) -> None: ... + +global___Task = Task + +class ProcessingConfig(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class _AIPCompressionAlgorithm: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + + class _AIPCompressionAlgorithmEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + ProcessingConfig._AIPCompressionAlgorithm.ValueType + ], + builtins.type, + ): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + AIP_COMPRESSION_ALGORITHM_UNSPECIFIED: ProcessingConfig._AIPCompressionAlgorithm.ValueType # 0 + AIP_COMPRESSION_ALGORITHM_UNCOMPRESSED: ProcessingConfig._AIPCompressionAlgorithm.ValueType # 1 + """It breaks in verify_aip.""" + + AIP_COMPRESSION_ALGORITHM_TAR: ProcessingConfig._AIPCompressionAlgorithm.ValueType # 2 + """Not supported yet!""" + + AIP_COMPRESSION_ALGORITHM_TAR_BZIP2: ProcessingConfig._AIPCompressionAlgorithm.ValueType # 3 + AIP_COMPRESSION_ALGORITHM_TAR_GZIP: ProcessingConfig._AIPCompressionAlgorithm.ValueType # 4 + AIP_COMPRESSION_ALGORITHM_S7_COPY: ProcessingConfig._AIPCompressionAlgorithm.ValueType # 5 + AIP_COMPRESSION_ALGORITHM_S7_BZIP2: ProcessingConfig._AIPCompressionAlgorithm.ValueType # 6 + AIP_COMPRESSION_ALGORITHM_S7_LZMA: ProcessingConfig._AIPCompressionAlgorithm.ValueType # 7 + + class AIPCompressionAlgorithm( + _AIPCompressionAlgorithm, metaclass=_AIPCompressionAlgorithmEnumTypeWrapper + ): + pass + AIP_COMPRESSION_ALGORITHM_UNSPECIFIED: ProcessingConfig.AIPCompressionAlgorithm.ValueType # 0 + AIP_COMPRESSION_ALGORITHM_UNCOMPRESSED: ProcessingConfig.AIPCompressionAlgorithm.ValueType # 1 + """It breaks in verify_aip.""" + + AIP_COMPRESSION_ALGORITHM_TAR: ProcessingConfig.AIPCompressionAlgorithm.ValueType # 2 + """Not supported yet!""" + + AIP_COMPRESSION_ALGORITHM_TAR_BZIP2: ProcessingConfig.AIPCompressionAlgorithm.ValueType # 3 + AIP_COMPRESSION_ALGORITHM_TAR_GZIP: ProcessingConfig.AIPCompressionAlgorithm.ValueType # 4 + AIP_COMPRESSION_ALGORITHM_S7_COPY: ProcessingConfig.AIPCompressionAlgorithm.ValueType # 5 + AIP_COMPRESSION_ALGORITHM_S7_BZIP2: ProcessingConfig.AIPCompressionAlgorithm.ValueType # 6 + AIP_COMPRESSION_ALGORITHM_S7_LZMA: ProcessingConfig.AIPCompressionAlgorithm.ValueType # 7 + + ASSIGN_UUIDS_TO_DIRECTORIES_FIELD_NUMBER: builtins.int + EXAMINE_CONTENTS_FIELD_NUMBER: builtins.int + GENERATE_TRANSFER_STRUCTURE_REPORT_FIELD_NUMBER: builtins.int + DOCUMENT_EMPTY_DIRECTORIES_FIELD_NUMBER: builtins.int + EXTRACT_PACKAGES_FIELD_NUMBER: builtins.int + DELETE_PACKAGES_AFTER_EXTRACTION_FIELD_NUMBER: builtins.int + IDENTIFY_TRANSFER_FIELD_NUMBER: builtins.int + IDENTIFY_SUBMISSION_AND_METADATA_FIELD_NUMBER: builtins.int + IDENTIFY_BEFORE_NORMALIZATION_FIELD_NUMBER: builtins.int + NORMALIZE_FIELD_NUMBER: builtins.int + TRANSCRIBE_FILES_FIELD_NUMBER: builtins.int + PERFORM_POLICY_CHECKS_ON_ORIGINALS_FIELD_NUMBER: builtins.int + PERFORM_POLICY_CHECKS_ON_PRESERVATION_DERIVATIVES_FIELD_NUMBER: builtins.int + AIP_COMPRESSION_LEVEL_FIELD_NUMBER: builtins.int + AIP_COMPRESSION_ALGORITHM_FIELD_NUMBER: builtins.int + assign_uuids_to_directories: builtins.bool + examine_contents: builtins.bool + generate_transfer_structure_report: builtins.bool + document_empty_directories: builtins.bool + extract_packages: builtins.bool + delete_packages_after_extraction: builtins.bool + identify_transfer: builtins.bool + identify_submission_and_metadata: builtins.bool + identify_before_normalization: builtins.bool + normalize: builtins.bool + transcribe_files: builtins.bool + perform_policy_checks_on_originals: builtins.bool + perform_policy_checks_on_preservation_derivatives: builtins.bool + aip_compression_level: builtins.int + """AIP compression level (1 is the fastest, 9 is the smallest).""" + + aip_compression_algorithm: global___ProcessingConfig.AIPCompressionAlgorithm.ValueType + """AIP compression algorithm""" + + def __init__( + self, + *, + assign_uuids_to_directories: builtins.bool = ..., + examine_contents: builtins.bool = ..., + generate_transfer_structure_report: builtins.bool = ..., + document_empty_directories: builtins.bool = ..., + extract_packages: builtins.bool = ..., + delete_packages_after_extraction: builtins.bool = ..., + identify_transfer: builtins.bool = ..., + identify_submission_and_metadata: builtins.bool = ..., + identify_before_normalization: builtins.bool = ..., + normalize: builtins.bool = ..., + transcribe_files: builtins.bool = ..., + perform_policy_checks_on_originals: builtins.bool = ..., + perform_policy_checks_on_preservation_derivatives: builtins.bool = ..., + aip_compression_level: builtins.int = ..., + aip_compression_algorithm: global___ProcessingConfig.AIPCompressionAlgorithm.ValueType = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "aip_compression_algorithm", + b"aip_compression_algorithm", + "aip_compression_level", + b"aip_compression_level", + "assign_uuids_to_directories", + b"assign_uuids_to_directories", + "delete_packages_after_extraction", + b"delete_packages_after_extraction", + "document_empty_directories", + b"document_empty_directories", + "examine_contents", + b"examine_contents", + "extract_packages", + b"extract_packages", + "generate_transfer_structure_report", + b"generate_transfer_structure_report", + "identify_before_normalization", + b"identify_before_normalization", + "identify_submission_and_metadata", + b"identify_submission_and_metadata", + "identify_transfer", + b"identify_transfer", + "normalize", + b"normalize", + "perform_policy_checks_on_originals", + b"perform_policy_checks_on_originals", + "perform_policy_checks_on_preservation_derivatives", + b"perform_policy_checks_on_preservation_derivatives", + "transcribe_files", + b"transcribe_files", + ], + ) -> None: ... + +global___ProcessingConfig = ProcessingConfig diff --git a/a3m/api/transferservice/v1beta1/request_response_pb2_grpc.py b/a3m/api/transferservice/v1beta1/request_response_pb2_grpc.py new file mode 100644 index 000000000..8a9393943 --- /dev/null +++ b/a3m/api/transferservice/v1beta1/request_response_pb2_grpc.py @@ -0,0 +1,3 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc diff --git a/a3m/api/transferservice/v1beta1/service_pb2.py b/a3m/api/transferservice/v1beta1/service_pb2.py new file mode 100644 index 000000000..45d28cce6 --- /dev/null +++ b/a3m/api/transferservice/v1beta1/service_pb2.py @@ -0,0 +1,33 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: a3m/api/transferservice/v1beta1/service.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from a3m.api.transferservice.v1beta1 import ( + request_response_pb2 as a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2, +) + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n-a3m/api/transferservice/v1beta1/service.proto\x12\x1f\x61\x33m.api.transferservice.v1beta1\x1a\x36\x61\x33m/api/transferservice/v1beta1/request_response.proto2\xdb\x02\n\x0fTransferService\x12k\n\x06Submit\x12..a3m.api.transferservice.v1beta1.SubmitRequest\x1a/.a3m.api.transferservice.v1beta1.SubmitResponse"\x00\x12\x65\n\x04Read\x12,.a3m.api.transferservice.v1beta1.ReadRequest\x1a-.a3m.api.transferservice.v1beta1.ReadResponse"\x00\x12t\n\tListTasks\x12\x31.a3m.api.transferservice.v1beta1.ListTasksRequest\x1a\x32.a3m.api.transferservice.v1beta1.ListTasksResponse"\x00\x42\xa9\x02\n#com.a3m.api.transferservice.v1beta1B\x0cServiceProtoP\x01ZUgithub.com/artefactual-labs/a3m/proto/a3m/api/transferservice/v1beta1;transferservice\xa2\x02\x03\x41\x41T\xaa\x02\x1f\x41\x33m.Api.Transferservice.V1beta1\xca\x02\x1f\x41\x33m\\Api\\Transferservice\\V1beta1\xe2\x02+A3m\\Api\\Transferservice\\V1beta1\\GPBMetadata\xea\x02"A3m::Api::Transferservice::V1beta1b\x06proto3' +) + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages( + DESCRIPTOR, "a3m.api.transferservice.v1beta1.service_pb2", globals() +) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n#com.a3m.api.transferservice.v1beta1B\014ServiceProtoP\001ZUgithub.com/artefactual-labs/a3m/proto/a3m/api/transferservice/v1beta1;transferservice\242\002\003AAT\252\002\037A3m.Api.Transferservice.V1beta1\312\002\037A3m\\Api\\Transferservice\\V1beta1\342\002+A3m\\Api\\Transferservice\\V1beta1\\GPBMetadata\352\002"A3m::Api::Transferservice::V1beta1' + _TRANSFERSERVICE._serialized_start = 139 + _TRANSFERSERVICE._serialized_end = 486 +# @@protoc_insertion_point(module_scope) diff --git a/a3m/api/transferservice/v1beta1/service_pb2.pyi b/a3m/api/transferservice/v1beta1/service_pb2.pyi new file mode 100644 index 000000000..e08fa11c2 --- /dev/null +++ b/a3m/api/transferservice/v1beta1/service_pb2.pyi @@ -0,0 +1,7 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import google.protobuf.descriptor + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor diff --git a/a3m/server/rpc/proto/a3m_pb2_grpc.py b/a3m/api/transferservice/v1beta1/service_pb2_grpc.py similarity index 54% rename from a3m/server/rpc/proto/a3m_pb2_grpc.py rename to a3m/api/transferservice/v1beta1/service_pb2_grpc.py index ef625d431..d54a57edb 100644 --- a/a3m/server/rpc/proto/a3m_pb2_grpc.py +++ b/a3m/api/transferservice/v1beta1/service_pb2_grpc.py @@ -2,12 +2,12 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -from a3m.server.rpc.proto import ( - a3m_pb2 as a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2, +from a3m.api.transferservice.v1beta1 import ( + request_response_pb2 as a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2, ) -class TransferStub: +class TransferServiceStub: """Missing associated documentation comment in .proto file.""" def __init__(self, channel): @@ -17,23 +17,23 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Submit = channel.unary_unary( - "/a3m.Transfer/Submit", - request_serializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.SubmitRequest.SerializeToString, - response_deserializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.SubmitReply.FromString, + "/a3m.api.transferservice.v1beta1.TransferService/Submit", + request_serializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.SubmitRequest.SerializeToString, + response_deserializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.SubmitResponse.FromString, ) self.Read = channel.unary_unary( - "/a3m.Transfer/Read", - request_serializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ReadRequest.SerializeToString, - response_deserializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ReadReply.FromString, + "/a3m.api.transferservice.v1beta1.TransferService/Read", + request_serializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ReadRequest.SerializeToString, + response_deserializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ReadResponse.FromString, ) self.ListTasks = channel.unary_unary( - "/a3m.Transfer/ListTasks", - request_serializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ListTasksRequest.SerializeToString, - response_deserializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ListTasksReply.FromString, + "/a3m.api.transferservice.v1beta1.TransferService/ListTasks", + request_serializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ListTasksRequest.SerializeToString, + response_deserializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ListTasksResponse.FromString, ) -class TransferServicer: +class TransferServiceServicer: """Missing associated documentation comment in .proto file.""" def Submit(self, request, context): @@ -55,32 +55,32 @@ def ListTasks(self, request, context): raise NotImplementedError("Method not implemented!") -def add_TransferServicer_to_server(servicer, server): +def add_TransferServiceServicer_to_server(servicer, server): rpc_method_handlers = { "Submit": grpc.unary_unary_rpc_method_handler( servicer.Submit, - request_deserializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.SubmitRequest.FromString, - response_serializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.SubmitReply.SerializeToString, + request_deserializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.SubmitRequest.FromString, + response_serializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.SubmitResponse.SerializeToString, ), "Read": grpc.unary_unary_rpc_method_handler( servicer.Read, - request_deserializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ReadRequest.FromString, - response_serializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ReadReply.SerializeToString, + request_deserializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ReadRequest.FromString, + response_serializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ReadResponse.SerializeToString, ), "ListTasks": grpc.unary_unary_rpc_method_handler( servicer.ListTasks, - request_deserializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ListTasksRequest.FromString, - response_serializer=a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ListTasksReply.SerializeToString, + request_deserializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ListTasksRequest.FromString, + response_serializer=a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ListTasksResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( - "a3m.Transfer", rpc_method_handlers + "a3m.api.transferservice.v1beta1.TransferService", rpc_method_handlers ) server.add_generic_rpc_handlers((generic_handler,)) # This class is part of an EXPERIMENTAL API. -class Transfer: +class TransferService: """Missing associated documentation comment in .proto file.""" @staticmethod @@ -99,9 +99,9 @@ def Submit( return grpc.experimental.unary_unary( request, target, - "/a3m.Transfer/Submit", - a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.SubmitRequest.SerializeToString, - a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.SubmitReply.FromString, + "/a3m.api.transferservice.v1beta1.TransferService/Submit", + a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.SubmitRequest.SerializeToString, + a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.SubmitResponse.FromString, options, channel_credentials, insecure, @@ -128,9 +128,9 @@ def Read( return grpc.experimental.unary_unary( request, target, - "/a3m.Transfer/Read", - a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ReadRequest.SerializeToString, - a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ReadReply.FromString, + "/a3m.api.transferservice.v1beta1.TransferService/Read", + a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ReadRequest.SerializeToString, + a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ReadResponse.FromString, options, channel_credentials, insecure, @@ -157,9 +157,9 @@ def ListTasks( return grpc.experimental.unary_unary( request, target, - "/a3m.Transfer/ListTasks", - a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ListTasksRequest.SerializeToString, - a3m_dot_server_dot_rpc_dot_proto_dot_a3m__pb2.ListTasksReply.FromString, + "/a3m.api.transferservice.v1beta1.TransferService/ListTasks", + a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ListTasksRequest.SerializeToString, + a3m_dot_api_dot_transferservice_dot_v1beta1_dot_request__response__pb2.ListTasksResponse.FromString, options, channel_credentials, insecure, diff --git a/a3m/cli/client/__main__.py b/a3m/cli/client/__main__.py index 4ac76b790..5696c4714 100644 --- a/a3m/cli/client/__main__.py +++ b/a3m/cli/client/__main__.py @@ -11,11 +11,11 @@ from rich.panel import Panel from rich.table import Table +from a3m.api.transferservice import v1beta1 as transfer_service_api from a3m.cli.client.wrapper import ClientWrapper from a3m.cli.common import init_django from a3m.cli.common import suppress_warnings from a3m.server.rpc.client import Client -from a3m.server.rpc.proto import a3m_pb2 @click.command() @@ -74,11 +74,11 @@ def main(ctx, uri, name, address, processing_config, wait_for_ready, no_input): resp = cw.client.wait_until_complete(resp.id) if (status := resp.status) in ( - a3m_pb2.FAILED, - a3m_pb2.REJECTED, + transfer_service_api.request_response_pb2.PACKAGE_STATUS_FAILED, + transfer_service_api.request_response_pb2.PACKAGE_STATUS_REJECTED, ): click.secho( - f"Error processing package ({a3m_pb2.PackageStatus.Name(status)})!", + f"Error processing package ({transfer_service_api.request_response_pb2.PackageStatus.Name(status)})!", fg="red", ) _print_failed_jobs(cw.client, resp.jobs) @@ -95,7 +95,7 @@ def _to_int(value: str) -> Optional[int]: def _prepare_config(user_pairs): - """Consolidate ``a3m_pb2.ProcessingConfig`` defaults and user-provided. + """Consolidate ``ProcessingConfig`` defaults and user-provided. A3M-TODO: defaults should be set on the server! @@ -111,7 +111,7 @@ def _prepare_config(user_pairs): A comprehensive list can be found in the definition of the ``ProcessingConfig`` message in the proto file. """ - config = a3m_pb2.ProcessingConfig( + config = transfer_service_api.request_response_pb2.ProcessingConfig( assign_uuids_to_directories=True, examine_contents=False, generate_transfer_structure_report=True, @@ -126,13 +126,15 @@ def _prepare_config(user_pairs): perform_policy_checks_on_originals=True, perform_policy_checks_on_preservation_derivatives=True, aip_compression_level=1, - aip_compression_algorithm=a3m_pb2.ProcessingConfig.S7_COPY, + aip_compression_algorithm=transfer_service_api.request_response_pb2.ProcessingConfig.AIP_COMPRESSION_ALGORITHM_S7_COPY, ) for item in user_pairs: head, sep, tail = item.partition("=") if head and sep: try: - field = a3m_pb2.ProcessingConfig.DESCRIPTOR.fields_by_name[head] + field = transfer_service_api.request_response_pb2.ProcessingConfig.DESCRIPTOR.fields_by_name[ + head + ] except KeyError: continue if field.type == FieldDescriptor.TYPE_BOOL: diff --git a/a3m/client/clientScripts/compress_aip.py b/a3m/client/clientScripts/compress_aip.py index 8226f8b6f..d517033f8 100644 --- a/a3m/client/clientScripts/compress_aip.py +++ b/a3m/client/clientScripts/compress_aip.py @@ -5,9 +5,9 @@ from django.db import transaction from a3m import databaseFunctions +from a3m.api.transferservice.v1beta1.request_response_pb2 import ProcessingConfig from a3m.executeOrRunSubProcess import executeOrRun from a3m.main.models import SIP -from a3m.server.rpc.proto import a3m_pb2 def update_unit(sip_uuid, compressed_location): @@ -40,19 +40,22 @@ def compress_aip( # Default is uncompressed. compression = int(compression) - a3m_pb2.ProcessingConfig.AIPCompressionAlgorithm.Name(compression) - if compression == a3m_pb2.ProcessingConfig.UNSPECIFIED: - compression = a3m_pb2.ProcessingConfig.UNCOMPRESSED + ProcessingConfig.AIPCompressionAlgorithm.Name(compression) + if compression == ProcessingConfig.AIP_COMPRESSION_ALGORITHM_UNSPECIFIED: + compression = ProcessingConfig.AIP_COMPRESSION_ALGORITHM_UNCOMPRESSED # Translation to make compress_aip happy. mapping = { - a3m_pb2.ProcessingConfig.UNCOMPRESSED: ("None", ""), - a3m_pb2.ProcessingConfig.TAR: ("gzip", "tar.gzip"), # A3M-TODO: support - a3m_pb2.ProcessingConfig.TAR_BZIP2: ("pbzip2", "pbzip2"), - a3m_pb2.ProcessingConfig.TAR_GZIP: ("gzip", "tar.gzip"), - a3m_pb2.ProcessingConfig.S7_COPY: ("7z", "copy"), - a3m_pb2.ProcessingConfig.S7_BZIP2: ("7z", "bzip2"), - a3m_pb2.ProcessingConfig.S7_LZMA: ("7z", "lzma"), + ProcessingConfig.AIP_COMPRESSION_ALGORITHM_UNCOMPRESSED: ("None", ""), + ProcessingConfig.AIP_COMPRESSION_ALGORITHM_TAR: ( + "gzip", + "tar.gzip", + ), # A3M-TODO: support + ProcessingConfig.AIP_COMPRESSION_ALGORITHM_TAR_BZIP2: ("pbzip2", "pbzip2"), + ProcessingConfig.AIP_COMPRESSION_ALGORITHM_TAR_GZIP: ("gzip", "tar.gzip"), + ProcessingConfig.AIP_COMPRESSION_ALGORITHM_S7_COPY: ("7z", "copy"), + ProcessingConfig.AIP_COMPRESSION_ALGORITHM_S7_BZIP2: ("7z", "bzip2"), + ProcessingConfig.AIP_COMPRESSION_ALGORITHM_S7_LZMA: ("7z", "lzma"), } try: diff --git a/a3m/server/packages.py b/a3m/server/packages.py index 2bbd298c5..05ab2cf51 100644 --- a/a3m/server/packages.py +++ b/a3m/server/packages.py @@ -12,12 +12,13 @@ from uuid import uuid4 from django.conf import settings +from google.protobuf import timestamp_pb2 +from a3m.api.transferservice import v1beta1 as transfer_service_api from a3m.archivematicaFunctions import strToUnicode from a3m.main import models from a3m.server.db import auto_close_old_connections from a3m.server.jobs import JobChain -from a3m.server.rpc.proto import a3m_pb2 logger = logging.getLogger(__name__) @@ -255,7 +256,7 @@ def get_replacement_mapping(self): rf"%config:{config_attr.name}%": str( getattr(self.config, config_attr.name) ) - for config_attr in a3m_pb2.ProcessingConfig.DESCRIPTOR.fields + for config_attr in transfer_service_api.request_response_pb2.ProcessingConfig.DESCRIPTOR.fields } ) @@ -411,7 +412,8 @@ def get_latest_job(unit_id): # Reminder: package.subid can be in Transfer or Ingest. job = get_latest_job(package.subid) return PackageStatus( - status=a3m_pb2.PROCESSING, job=job.jobtype if job else None + status=transfer_service_api.request_response_pb2.PACKAGE_STATUS_PROCESSING, + job=job.jobtype if job else None, ) # A3M-TODO: persist package-workflow status! @@ -430,14 +432,16 @@ def get_latest_job(unit_id): ) job = get_latest_job(sip.transfer_id) if job is None: - return PackageStatus(status=a3m_pb2.PROCESSING) + return PackageStatus( + status=transfer_service_api.request_response_pb2.PACKAGE_STATUS_PROCESSING + ) if "failed" in job.microservicegroup.lower(): - status = a3m_pb2.FAILED + status = transfer_service_api.request_response_pb2.PACKAGE_STATUS_FAILED elif "reject" in job.microservicegroup.lower(): - status = a3m_pb2.REJECTED + status = transfer_service_api.request_response_pb2.PACKAGE_STATUS_REJECTED elif job.jobtype == "a3m - Store AIP": - status = a3m_pb2.COMPLETE + status = transfer_service_api.request_response_pb2.PACKAGE_STATUS_COMPLETE else: raise Exception( f"Package status cannot be determined (job.currentstep={job.currentstep}, job.type={job.jobtype}, job.microservicegroup={job.microservicegroup})" @@ -455,15 +459,20 @@ def get_latest_job(unit_id): "microservicegroup", "microservicechainlink", "currentstep", + "createdtime", ) ): + start_time = timestamp_pb2.Timestamp() + start_time.FromDatetime(item["createdtime"]) + package_status.jobs.append( - a3m_pb2.Job( + transfer_service_api.request_response_pb2.Job( id=str(item["jobuuid"]), name=item["jobtype"], group=item["microservicegroup"], link_id=str(item["microservicechainlink"]), status=item["currentstep"], + start_time=start_time, ) ) diff --git a/a3m/server/rpc/client.py b/a3m/server/rpc/client.py index d28aaefaa..e98cc3d04 100644 --- a/a3m/server/rpc/client.py +++ b/a3m/server/rpc/client.py @@ -7,8 +7,7 @@ from grpc import RpcError from a3m import __version__ -from a3m.server.rpc.proto import a3m_pb2 -from a3m.server.rpc.proto import a3m_pb2_grpc +from a3m.api.transferservice import v1beta1 as transfer_service_api logger = logging.getLogger(__name__) @@ -30,7 +29,10 @@ def __init__( rpc_timeout: Optional[int] = _GRPC_DEFAULT_TIMEOUT_SECS, wait_for_ready: bool = False, ): - self.transfer_stub = a3m_pb2_grpc.TransferStub(channel) + + self.transfer_stub = transfer_service_api.service_pb2_grpc.TransferServiceStub( + channel + ) self.rpc_timeout = rpc_timeout self.wait_for_ready = wait_for_ready @@ -52,19 +54,31 @@ def _unary_call(self, api_method, request): def version_metadata(): return ((_VERSION_METADATA_KEY, __version__),) - def submit(self, url: str, name: str, config: a3m_pb2.ProcessingConfig = None): - request = a3m_pb2.SubmitRequest(name=name, url=url, config=config) + def submit( + self, + url: str, + name: str, + config: transfer_service_api.request_response_pb2.ProcessingConfig = None, + ): + request = transfer_service_api.request_response_pb2.SubmitRequest( + name=name, url=url, config=config + ) return self._unary_call(self.transfer_stub.Submit, request) def read(self, package_id: str): - request = a3m_pb2.ReadRequest(id=package_id) + request = transfer_service_api.request_response_pb2.ReadRequest(id=package_id) return self._unary_call(self.transfer_stub.Read, request) def wait_until_complete(self, package_id: str, spin_cb: Callable = None): """Blocks until processing of a package has completed.""" - def _should_continue(resp): - if resp.status == a3m_pb2.PROCESSING: + def _should_continue( + resp: transfer_service_api.request_response_pb2.ReadResponse, + ): + if ( + resp.status + == transfer_service_api.request_response_pb2.PACKAGE_STATUS_PROCESSING + ): return True return False @@ -84,5 +98,7 @@ def _poll(): return _poll() def list_tasks(self, job_id: str): - request = a3m_pb2.ListTasksRequest(job_id=job_id) + request = transfer_service_api.request_response_pb2.ListTasksRequest( + job_id=job_id + ) return self._unary_call(self.transfer_stub.ListTasks, request) diff --git a/a3m/server/rpc/proto/__init__.py b/a3m/server/rpc/proto/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/a3m/server/rpc/proto/a3m_pb2.py b/a3m/server/rpc/proto/a3m_pb2.py deleted file mode 100644 index 619af8826..000000000 --- a/a3m/server/rpc/proto/a3m_pb2.py +++ /dev/null @@ -1,1296 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: a3m/server/rpc/proto/a3m.proto -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import enum_type_wrapper - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="a3m/server/rpc/proto/a3m.proto", - package="a3m", - syntax="proto3", - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x1e\x61\x33m/server/rpc/proto/a3m.proto\x12\x03\x61\x33m"Q\n\rSubmitRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12%\n\x06\x63onfig\x18\x03 \x01(\x0b\x32\x15.a3m.ProcessingConfig"\x19\n\x0bSubmitReply\x12\n\n\x02id\x18\x01 \x01(\t"\x19\n\x0bReadRequest\x12\n\n\x02id\x18\x01 \x01(\t"T\n\tReadReply\x12"\n\x06status\x18\x01 \x01(\x0e\x32\x12.a3m.PackageStatus\x12\x0b\n\x03job\x18\x02 \x01(\t\x12\x16\n\x04jobs\x18\x03 \x03(\x0b\x32\x08.a3m.Job""\n\x10ListTasksRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t"*\n\x0eListTasksReply\x12\x18\n\x05tasks\x18\x01 \x03(\x0b\x32\t.a3m.Task"\xa1\x01\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05group\x18\x03 \x01(\t\x12\x0f\n\x07link_id\x18\x04 \x01(\t\x12\x1f\n\x06status\x18\x05 \x01(\x0e\x32\x0f.a3m.Job.Status"?\n\x06Status\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x43OMPLETE\x10\x01\x12\x0e\n\nPROCESSING\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03"\x8e\x01\n\x04Task\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07\x66ile_id\x18\x02 \x01(\t\x12\x11\n\texit_code\x18\x03 \x01(\x05\x12\x10\n\x08\x66ilename\x18\x04 \x01(\t\x12\x11\n\texecution\x18\x05 \x01(\t\x12\x11\n\targuments\x18\x06 \x01(\t\x12\x0e\n\x06stdout\x18\x07 \x01(\t\x12\x0e\n\x06stderr\x18\x08 \x01(\t"\xe3\x05\n\x10ProcessingConfig\x12#\n\x1b\x61ssign_uuids_to_directories\x18\x01 \x01(\x08\x12\x18\n\x10\x65xamine_contents\x18\x02 \x01(\x08\x12*\n"generate_transfer_structure_report\x18\x03 \x01(\x08\x12"\n\x1a\x64ocument_empty_directories\x18\x04 \x01(\x08\x12\x18\n\x10\x65xtract_packages\x18\x05 \x01(\x08\x12(\n delete_packages_after_extraction\x18\x06 \x01(\x08\x12\x19\n\x11identify_transfer\x18\x07 \x01(\x08\x12(\n identify_submission_and_metadata\x18\x08 \x01(\x08\x12%\n\x1didentify_before_normalization\x18\t \x01(\x08\x12\x11\n\tnormalize\x18\n \x01(\x08\x12\x18\n\x10transcribe_files\x18\x0b \x01(\x08\x12*\n"perform_policy_checks_on_originals\x18\x0c \x01(\x08\x12\x39\n1perform_policy_checks_on_preservation_derivatives\x18\r \x01(\x08\x12\x1d\n\x15\x61ip_compression_level\x18\x0e \x01(\x05\x12P\n\x19\x61ip_compression_algorithm\x18\x0f \x01(\x0e\x32-.a3m.ProcessingConfig.AIPCompressionAlgorithm"\x8a\x01\n\x17\x41IPCompressionAlgorithm\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x10\n\x0cUNCOMPRESSED\x10\x01\x12\x07\n\x03TAR\x10\x02\x12\r\n\tTAR_BZIP2\x10\x03\x12\x0c\n\x08TAR_GZIP\x10\x04\x12\x0b\n\x07S7_COPY\x10\x05\x12\x0c\n\x08S7_BZIP2\x10\x06\x12\x0b\n\x07S7_LZMA\x10\x07*G\n\rPackageStatus\x12\n\n\x06\x46\x41ILED\x10\x00\x12\x0c\n\x08REJECTED\x10\x01\x12\x0c\n\x08\x43OMPLETE\x10\x02\x12\x0e\n\nPROCESSING\x10\x03\x32\xa3\x01\n\x08Transfer\x12\x30\n\x06Submit\x12\x12.a3m.SubmitRequest\x1a\x10.a3m.SubmitReply"\x00\x12*\n\x04Read\x12\x10.a3m.ReadRequest\x1a\x0e.a3m.ReadReply"\x00\x12\x39\n\tListTasks\x12\x15.a3m.ListTasksRequest\x1a\x13.a3m.ListTasksReply"\x00\x62\x06proto3', -) - -_PACKAGESTATUS = _descriptor.EnumDescriptor( - name="PackageStatus", - full_name="a3m.PackageStatus", - filename=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - values=[ - _descriptor.EnumValueDescriptor( - name="FAILED", - index=0, - number=0, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="REJECTED", - index=1, - number=1, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="COMPLETE", - index=2, - number=2, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="PROCESSING", - index=3, - number=3, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - ], - containing_type=None, - serialized_options=None, - serialized_start=1393, - serialized_end=1464, -) -_sym_db.RegisterEnumDescriptor(_PACKAGESTATUS) - -PackageStatus = enum_type_wrapper.EnumTypeWrapper(_PACKAGESTATUS) -FAILED = 0 -REJECTED = 1 -COMPLETE = 2 -PROCESSING = 3 - - -_JOB_STATUS = _descriptor.EnumDescriptor( - name="Status", - full_name="a3m.Job.Status", - filename=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - values=[ - _descriptor.EnumValueDescriptor( - name="UNKNOWN", - index=0, - number=0, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="COMPLETE", - index=1, - number=1, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="PROCESSING", - index=2, - number=2, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="FAILED", - index=3, - number=3, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - ], - containing_type=None, - serialized_options=None, - serialized_start=441, - serialized_end=504, -) -_sym_db.RegisterEnumDescriptor(_JOB_STATUS) - -_PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM = _descriptor.EnumDescriptor( - name="AIPCompressionAlgorithm", - full_name="a3m.ProcessingConfig.AIPCompressionAlgorithm", - filename=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - values=[ - _descriptor.EnumValueDescriptor( - name="UNSPECIFIED", - index=0, - number=0, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="UNCOMPRESSED", - index=1, - number=1, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="TAR", - index=2, - number=2, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="TAR_BZIP2", - index=3, - number=3, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="TAR_GZIP", - index=4, - number=4, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="S7_COPY", - index=5, - number=5, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="S7_BZIP2", - index=6, - number=6, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.EnumValueDescriptor( - name="S7_LZMA", - index=7, - number=7, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key, - ), - ], - containing_type=None, - serialized_options=None, - serialized_start=1253, - serialized_end=1391, -) -_sym_db.RegisterEnumDescriptor(_PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM) - - -_SUBMITREQUEST = _descriptor.Descriptor( - name="SubmitRequest", - full_name="a3m.SubmitRequest", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="name", - full_name="a3m.SubmitRequest.name", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="url", - full_name="a3m.SubmitRequest.url", - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="config", - full_name="a3m.SubmitRequest.config", - index=2, - number=3, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=39, - serialized_end=120, -) - - -_SUBMITREPLY = _descriptor.Descriptor( - name="SubmitReply", - full_name="a3m.SubmitReply", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="id", - full_name="a3m.SubmitReply.id", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=122, - serialized_end=147, -) - - -_READREQUEST = _descriptor.Descriptor( - name="ReadRequest", - full_name="a3m.ReadRequest", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="id", - full_name="a3m.ReadRequest.id", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=149, - serialized_end=174, -) - - -_READREPLY = _descriptor.Descriptor( - name="ReadReply", - full_name="a3m.ReadReply", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="status", - full_name="a3m.ReadReply.status", - index=0, - number=1, - type=14, - cpp_type=8, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="job", - full_name="a3m.ReadReply.job", - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="jobs", - full_name="a3m.ReadReply.jobs", - index=2, - number=3, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=176, - serialized_end=260, -) - - -_LISTTASKSREQUEST = _descriptor.Descriptor( - name="ListTasksRequest", - full_name="a3m.ListTasksRequest", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="job_id", - full_name="a3m.ListTasksRequest.job_id", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=262, - serialized_end=296, -) - - -_LISTTASKSREPLY = _descriptor.Descriptor( - name="ListTasksReply", - full_name="a3m.ListTasksReply", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="tasks", - full_name="a3m.ListTasksReply.tasks", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=298, - serialized_end=340, -) - - -_JOB = _descriptor.Descriptor( - name="Job", - full_name="a3m.Job", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="id", - full_name="a3m.Job.id", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="name", - full_name="a3m.Job.name", - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="group", - full_name="a3m.Job.group", - index=2, - number=3, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="link_id", - full_name="a3m.Job.link_id", - index=3, - number=4, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="status", - full_name="a3m.Job.status", - index=4, - number=5, - type=14, - cpp_type=8, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[ - _JOB_STATUS, - ], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=343, - serialized_end=504, -) - - -_TASK = _descriptor.Descriptor( - name="Task", - full_name="a3m.Task", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="id", - full_name="a3m.Task.id", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="file_id", - full_name="a3m.Task.file_id", - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="exit_code", - full_name="a3m.Task.exit_code", - index=2, - number=3, - type=5, - cpp_type=1, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="filename", - full_name="a3m.Task.filename", - index=3, - number=4, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="execution", - full_name="a3m.Task.execution", - index=4, - number=5, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="arguments", - full_name="a3m.Task.arguments", - index=5, - number=6, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="stdout", - full_name="a3m.Task.stdout", - index=6, - number=7, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="stderr", - full_name="a3m.Task.stderr", - index=7, - number=8, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=507, - serialized_end=649, -) - - -_PROCESSINGCONFIG = _descriptor.Descriptor( - name="ProcessingConfig", - full_name="a3m.ProcessingConfig", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="assign_uuids_to_directories", - full_name="a3m.ProcessingConfig.assign_uuids_to_directories", - index=0, - number=1, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="examine_contents", - full_name="a3m.ProcessingConfig.examine_contents", - index=1, - number=2, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="generate_transfer_structure_report", - full_name="a3m.ProcessingConfig.generate_transfer_structure_report", - index=2, - number=3, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="document_empty_directories", - full_name="a3m.ProcessingConfig.document_empty_directories", - index=3, - number=4, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="extract_packages", - full_name="a3m.ProcessingConfig.extract_packages", - index=4, - number=5, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="delete_packages_after_extraction", - full_name="a3m.ProcessingConfig.delete_packages_after_extraction", - index=5, - number=6, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="identify_transfer", - full_name="a3m.ProcessingConfig.identify_transfer", - index=6, - number=7, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="identify_submission_and_metadata", - full_name="a3m.ProcessingConfig.identify_submission_and_metadata", - index=7, - number=8, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="identify_before_normalization", - full_name="a3m.ProcessingConfig.identify_before_normalization", - index=8, - number=9, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="normalize", - full_name="a3m.ProcessingConfig.normalize", - index=9, - number=10, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="transcribe_files", - full_name="a3m.ProcessingConfig.transcribe_files", - index=10, - number=11, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="perform_policy_checks_on_originals", - full_name="a3m.ProcessingConfig.perform_policy_checks_on_originals", - index=11, - number=12, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="perform_policy_checks_on_preservation_derivatives", - full_name="a3m.ProcessingConfig.perform_policy_checks_on_preservation_derivatives", - index=12, - number=13, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="aip_compression_level", - full_name="a3m.ProcessingConfig.aip_compression_level", - index=13, - number=14, - type=5, - cpp_type=1, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="aip_compression_algorithm", - full_name="a3m.ProcessingConfig.aip_compression_algorithm", - index=14, - number=15, - type=14, - cpp_type=8, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[ - _PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM, - ], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=652, - serialized_end=1391, -) - -_SUBMITREQUEST.fields_by_name["config"].message_type = _PROCESSINGCONFIG -_READREPLY.fields_by_name["status"].enum_type = _PACKAGESTATUS -_READREPLY.fields_by_name["jobs"].message_type = _JOB -_LISTTASKSREPLY.fields_by_name["tasks"].message_type = _TASK -_JOB.fields_by_name["status"].enum_type = _JOB_STATUS -_JOB_STATUS.containing_type = _JOB -_PROCESSINGCONFIG.fields_by_name[ - "aip_compression_algorithm" -].enum_type = _PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM -_PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM.containing_type = _PROCESSINGCONFIG -DESCRIPTOR.message_types_by_name["SubmitRequest"] = _SUBMITREQUEST -DESCRIPTOR.message_types_by_name["SubmitReply"] = _SUBMITREPLY -DESCRIPTOR.message_types_by_name["ReadRequest"] = _READREQUEST -DESCRIPTOR.message_types_by_name["ReadReply"] = _READREPLY -DESCRIPTOR.message_types_by_name["ListTasksRequest"] = _LISTTASKSREQUEST -DESCRIPTOR.message_types_by_name["ListTasksReply"] = _LISTTASKSREPLY -DESCRIPTOR.message_types_by_name["Job"] = _JOB -DESCRIPTOR.message_types_by_name["Task"] = _TASK -DESCRIPTOR.message_types_by_name["ProcessingConfig"] = _PROCESSINGCONFIG -DESCRIPTOR.enum_types_by_name["PackageStatus"] = _PACKAGESTATUS -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -SubmitRequest = _reflection.GeneratedProtocolMessageType( - "SubmitRequest", - (_message.Message,), - { - "DESCRIPTOR": _SUBMITREQUEST, - "__module__": "a3m.server.rpc.proto.a3m_pb2" - # @@protoc_insertion_point(class_scope:a3m.SubmitRequest) - }, -) -_sym_db.RegisterMessage(SubmitRequest) - -SubmitReply = _reflection.GeneratedProtocolMessageType( - "SubmitReply", - (_message.Message,), - { - "DESCRIPTOR": _SUBMITREPLY, - "__module__": "a3m.server.rpc.proto.a3m_pb2" - # @@protoc_insertion_point(class_scope:a3m.SubmitReply) - }, -) -_sym_db.RegisterMessage(SubmitReply) - -ReadRequest = _reflection.GeneratedProtocolMessageType( - "ReadRequest", - (_message.Message,), - { - "DESCRIPTOR": _READREQUEST, - "__module__": "a3m.server.rpc.proto.a3m_pb2" - # @@protoc_insertion_point(class_scope:a3m.ReadRequest) - }, -) -_sym_db.RegisterMessage(ReadRequest) - -ReadReply = _reflection.GeneratedProtocolMessageType( - "ReadReply", - (_message.Message,), - { - "DESCRIPTOR": _READREPLY, - "__module__": "a3m.server.rpc.proto.a3m_pb2" - # @@protoc_insertion_point(class_scope:a3m.ReadReply) - }, -) -_sym_db.RegisterMessage(ReadReply) - -ListTasksRequest = _reflection.GeneratedProtocolMessageType( - "ListTasksRequest", - (_message.Message,), - { - "DESCRIPTOR": _LISTTASKSREQUEST, - "__module__": "a3m.server.rpc.proto.a3m_pb2" - # @@protoc_insertion_point(class_scope:a3m.ListTasksRequest) - }, -) -_sym_db.RegisterMessage(ListTasksRequest) - -ListTasksReply = _reflection.GeneratedProtocolMessageType( - "ListTasksReply", - (_message.Message,), - { - "DESCRIPTOR": _LISTTASKSREPLY, - "__module__": "a3m.server.rpc.proto.a3m_pb2" - # @@protoc_insertion_point(class_scope:a3m.ListTasksReply) - }, -) -_sym_db.RegisterMessage(ListTasksReply) - -Job = _reflection.GeneratedProtocolMessageType( - "Job", - (_message.Message,), - { - "DESCRIPTOR": _JOB, - "__module__": "a3m.server.rpc.proto.a3m_pb2" - # @@protoc_insertion_point(class_scope:a3m.Job) - }, -) -_sym_db.RegisterMessage(Job) - -Task = _reflection.GeneratedProtocolMessageType( - "Task", - (_message.Message,), - { - "DESCRIPTOR": _TASK, - "__module__": "a3m.server.rpc.proto.a3m_pb2" - # @@protoc_insertion_point(class_scope:a3m.Task) - }, -) -_sym_db.RegisterMessage(Task) - -ProcessingConfig = _reflection.GeneratedProtocolMessageType( - "ProcessingConfig", - (_message.Message,), - { - "DESCRIPTOR": _PROCESSINGCONFIG, - "__module__": "a3m.server.rpc.proto.a3m_pb2" - # @@protoc_insertion_point(class_scope:a3m.ProcessingConfig) - }, -) -_sym_db.RegisterMessage(ProcessingConfig) - - -_TRANSFER = _descriptor.ServiceDescriptor( - name="Transfer", - full_name="a3m.Transfer", - file=DESCRIPTOR, - index=0, - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_start=1467, - serialized_end=1630, - methods=[ - _descriptor.MethodDescriptor( - name="Submit", - full_name="a3m.Transfer.Submit", - index=0, - containing_service=None, - input_type=_SUBMITREQUEST, - output_type=_SUBMITREPLY, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name="Read", - full_name="a3m.Transfer.Read", - index=1, - containing_service=None, - input_type=_READREQUEST, - output_type=_READREPLY, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name="ListTasks", - full_name="a3m.Transfer.ListTasks", - index=2, - containing_service=None, - input_type=_LISTTASKSREQUEST, - output_type=_LISTTASKSREPLY, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - ], -) -_sym_db.RegisterServiceDescriptor(_TRANSFER) - -DESCRIPTOR.services_by_name["Transfer"] = _TRANSFER - -# @@protoc_insertion_point(module_scope) diff --git a/a3m/server/runner.py b/a3m/server/runner.py index b040ca8d4..8abd9cd33 100644 --- a/a3m/server/runner.py +++ b/a3m/server/runner.py @@ -28,13 +28,12 @@ import grpc from grpc_reflection.v1alpha import reflection +from a3m.api.transferservice import v1beta1 as transfer_service_api from a3m.server import metrics from a3m.server import shared_dirs from a3m.server.db import migrate from a3m.server.jobs import Job from a3m.server.queues import PackageQueue -from a3m.server.rpc.proto import a3m_pb2 -from a3m.server.rpc.proto import a3m_pb2_grpc from a3m.server.tasks import Task from a3m.server.tasks.backends import get_task_backend from a3m.server.tasks.backends import TaskBackend @@ -98,11 +97,13 @@ def _mount_services(self): transfer_service = TransferService( self.workflow, self.queue, self.queue_executor ) - a3m_pb2_grpc.add_TransferServicer_to_server(transfer_service, self.grpc_server) + transfer_service_api.service_pb2_grpc.add_TransferServiceServicer_to_server( + transfer_service, self.grpc_server + ) services = tuple( service.full_name - for service in a3m_pb2.DESCRIPTOR.services_by_name.values() + for service in transfer_service_api.service_pb2_grpc.DESCRIPTOR.services_by_name.values() ) + (reflection.SERVICE_NAME,) reflection.enable_server_reflection(services, self.grpc_server) diff --git a/a3m/server/transfer_service.py b/a3m/server/transfer_service.py index 9f71b0451..d20c63cb8 100644 --- a/a3m/server/transfer_service.py +++ b/a3m/server/transfer_service.py @@ -1,18 +1,18 @@ import logging +from google.protobuf import timestamp_pb2 from google.rpc import code_pb2 +from a3m.api.transferservice import v1beta1 as transfer_service_api from a3m.main.models import Task from a3m.server.packages import get_package_status from a3m.server.packages import Package from a3m.server.packages import PackageNotFoundError -from a3m.server.rpc.proto import a3m_pb2 -from a3m.server.rpc.proto import a3m_pb2_grpc logger = logging.getLogger(__name__) -class TransferService(a3m_pb2_grpc.TransferServicer): +class TransferService(transfer_service_api.service_pb2_grpc.TransferServiceServicer): def __init__(self, workflow, package_queue, executor): self.workflow = workflow self.package_queue = package_queue @@ -31,7 +31,9 @@ def Submit(self, request, context): except Exception as err: logger.warning("TransferService.Submit handler error: %s", err) context.abort(code_pb2.INTERNAL, "Unknown error") - return a3m_pb2.SubmitReply(id=str(package.uuid)) + return transfer_service_api.request_response_pb2.SubmitResponse( + id=str(package.uuid) + ) def Read(self, request, context): try: @@ -41,20 +43,26 @@ def Read(self, request, context): except Exception as err: logger.warning("TransferService.Status handler error: %s", err) context.abort(code_pb2.INTERNAL, "Unknown error") - reply = a3m_pb2.ReadReply(status=package_status.status) + resp = transfer_service_api.request_response_pb2.ReadResponse( + status=package_status.status + ) if package_status.job: - reply.job = package_status.job + resp.job = package_status.job if package_status.jobs: - reply.jobs.extend(package_status.jobs) - return reply + resp.jobs.extend(package_status.jobs) + return resp def ListTasks(self, request, context): if not request.job_id: context.abort(code_pb2.INVALID_ARGUMENT, "job_id is mandatory") - reply = a3m_pb2.ListTasksReply() + resp = transfer_service_api.request_response_pb2.ListTasksResponse() for item in Task.objects.filter(job_id=request.job_id): - reply.tasks.append( - a3m_pb2.Task( + start_time = timestamp_pb2.Timestamp() + start_time.FromDatetime(item.starttime) + end_time = timestamp_pb2.Timestamp() + end_time.FromDatetime(item.endtime) + resp.tasks.append( + transfer_service_api.request_response_pb2.Task( id=item.pk, file_id=item.fileuuid, exit_code=item.exitcode, @@ -63,6 +71,8 @@ def ListTasks(self, request, context): arguments=item.arguments, stdout=item.stdout, stderr=item.stderror, + start_time=timestamp_pb2.Timestamp.FromDatetime(start_time), + end_time=timestamp_pb2.Timestamp.FromDatetime(end_time), ) ) - return reply + return resp diff --git a/docs/development.rst b/docs/development.rst index 8c7b8ae5d..324bf059b 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -48,9 +48,7 @@ our service schema. .. _idl: -.. literalinclude:: ../a3m/server/rpc/proto/a3m.proto - :language: protobuf - +Find the generated documentation of the a3m API at `buf.build/artefactual/a3m`_. Reference --------- @@ -64,3 +62,6 @@ Reference :undoc-members: .. autoclass:: a3m.cli.client.wrapper.ClientWrapper + + +.. _`buf.build/artefactual/a3m`: https://buf.build/artefactual/a3m diff --git a/docs/usage.rst b/docs/usage.rst index 26d750892..1b83ce7f0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -68,10 +68,12 @@ the desired settings:: The Python client can do similarly:: + from a3m.api.transferservice.v1beta1.request_response_pb2 import ProcessingConfig + c = Client(...) c.submit( url="URL...", name="Name...", - config=a3m_pb2.ProcessingConfig(normalize=False)) + config=ProcessingConfig(normalize=False)) The full list of settings or their defaults are not described yet but it can be found in our :ref:`service definition file `, under the diff --git a/a3m/server/rpc/proto/a3m.proto b/proto/a3m/api/transferservice/v1beta1/request_response.proto similarity index 57% rename from a3m/server/rpc/proto/a3m.proto rename to proto/a3m/api/transferservice/v1beta1/request_response.proto index 3fa5f67c0..e1e2c0a7f 100644 --- a/a3m/server/rpc/proto/a3m.proto +++ b/proto/a3m/api/transferservice/v1beta1/request_response.proto @@ -1,19 +1,10 @@ syntax = "proto3"; -package a3m; +package a3m.api.transferservice.v1beta1; -service Transfer { +option go_package = "github.com/artefactual-labs/a3m/proto/a3m/api/transferservice/v1beta1;transferservice"; - // Submits a new transfer. - rpc Submit (SubmitRequest) returns (SubmitReply) {} - - // Reads the status of a given transfer. - rpc Read (ReadRequest) returns (ReadReply) {} - - // Lists all tasks in a given transfer. - rpc ListTasks (ListTasksRequest) returns (ListTasksReply) {} - -} +import "google/protobuf/timestamp.proto"; message SubmitRequest { string name = 1; @@ -21,7 +12,7 @@ message SubmitRequest { ProcessingConfig config = 3; } -message SubmitReply { +message SubmitResponse { string id = 1; } @@ -29,7 +20,7 @@ message ReadRequest { string id = 1; } -message ReadReply { +message ReadResponse { PackageStatus status = 1; string job = 2; repeated Job jobs = 3; @@ -39,15 +30,16 @@ message ListTasksRequest { string job_id = 1; } -message ListTasksReply { +message ListTasksResponse { repeated Task tasks = 1; } enum PackageStatus { - FAILED = 0; - REJECTED = 1; - COMPLETE = 2; - PROCESSING = 3; + PACKAGE_STATUS_UNSPECIFIED = 0; + PACKAGE_STATUS_FAILED = 1; + PACKAGE_STATUS_REJECTED = 2; + PACKAGE_STATUS_COMPLETE = 3; + PACKAGE_STATUS_PROCESSING = 4; } message Job { @@ -57,13 +49,14 @@ message Job { string link_id = 4; enum Status { - UNKNOWN = 0; - COMPLETE = 1; - PROCESSING = 2; - FAILED = 3; + STATUS_UNSPECIFIED = 0; + STATUS_COMPLETE = 1; + STATUS_PROCESSING = 2; + STATUS_FAILED = 3; } Status status = 5; + google.protobuf.Timestamp start_time = 6; } message Task { @@ -75,6 +68,8 @@ message Task { string arguments = 6; string stdout = 7; string stderr = 8; + google.protobuf.Timestamp start_time = 9; + google.protobuf.Timestamp end_time = 10; } message ProcessingConfig { @@ -99,13 +94,13 @@ message ProcessingConfig { AIPCompressionAlgorithm aip_compression_algorithm = 15; enum AIPCompressionAlgorithm { - UNSPECIFIED = 0; - UNCOMPRESSED = 1; // It breaks in verify_aip. - TAR = 2; // Not supported yet! - TAR_BZIP2 = 3; - TAR_GZIP = 4; - S7_COPY = 5; - S7_BZIP2 = 6; - S7_LZMA = 7; + AIP_COMPRESSION_ALGORITHM_UNSPECIFIED = 0; + AIP_COMPRESSION_ALGORITHM_UNCOMPRESSED = 1; // It breaks in verify_aip. + AIP_COMPRESSION_ALGORITHM_TAR = 2; // Not supported yet! + AIP_COMPRESSION_ALGORITHM_TAR_BZIP2 = 3; + AIP_COMPRESSION_ALGORITHM_TAR_GZIP = 4; + AIP_COMPRESSION_ALGORITHM_S7_COPY = 5; + AIP_COMPRESSION_ALGORITHM_S7_BZIP2 = 6; + AIP_COMPRESSION_ALGORITHM_S7_LZMA = 7; } } diff --git a/proto/a3m/api/transferservice/v1beta1/service.proto b/proto/a3m/api/transferservice/v1beta1/service.proto new file mode 100644 index 000000000..c67720756 --- /dev/null +++ b/proto/a3m/api/transferservice/v1beta1/service.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package a3m.api.transferservice.v1beta1; + +option go_package = "github.com/artefactual-labs/a3m/proto/a3m/api/transferservice/v1beta1;transferservice"; + +import "a3m/api/transferservice/v1beta1/request_response.proto"; + +service TransferService { + + // Submits a new transfer. + rpc Submit (SubmitRequest) returns (SubmitResponse) {} + + // Reads the status of a given transfer. + rpc Read (ReadRequest) returns (ReadResponse) {} + + // Lists all tasks in a given transfer. + rpc ListTasks (ListTasksRequest) returns (ListTasksResponse) {} + +} diff --git a/proto/buf.gen.yaml b/proto/buf.gen.yaml new file mode 100644 index 000000000..2d011bee7 --- /dev/null +++ b/proto/buf.gen.yaml @@ -0,0 +1,10 @@ +version: v1 +managed: + enabled: true +plugins: + - remote: buf.build/protocolbuffers/plugins/python:v3.20.1-1 + out: ../ + - remote: buf.build/grpc/plugins/python:v1.46.3-1 + out: ../ + - remote: buf.build/adriansahlman/plugins/mypy-protobuf:v3.2.0-1 + out: ../ diff --git a/proto/buf.lock b/proto/buf.lock new file mode 100644 index 000000000..c91b5810c --- /dev/null +++ b/proto/buf.lock @@ -0,0 +1,2 @@ +# Generated by buf. DO NOT EDIT. +version: v1 diff --git a/proto/buf.yaml b/proto/buf.yaml new file mode 100644 index 000000000..f400d78ab --- /dev/null +++ b/proto/buf.yaml @@ -0,0 +1,8 @@ +version: v1 +name: buf.build/artefactual/a3m +breaking: + use: + - FILE +lint: + use: + - DEFAULT diff --git a/setup.cfg b/setup.cfg index 9a404a946..911664807 100644 --- a/setup.cfg +++ b/setup.cfg @@ -97,19 +97,18 @@ addopts = -Werror testpaths = tests -[tool:flake8] -exclude = .tox, .git, __pycache__, .cache, build, dist, *.pyc, *.egg-info, .eggs +[flake8] +exclude = .venv .tox .git __pycache__ .cache build dist *.pyc *.egg-info .eggs a3m/api/**/*.py application-import-names = flake8 select = C, E, F, W, B, B950 ignore = E203, E402, E501, E722, W503, W605 -per-file-ignores = - *.pyi: F401 [coverage:run] source = a3m/ branch = True omit = + a3m/api/ a3m/externals/ a3m/server/rpc/ **/migrations/* diff --git a/tests/server/test_integration.py b/tests/server/test_integration.py index dd2de3318..a925b62c0 100644 --- a/tests/server/test_integration.py +++ b/tests/server/test_integration.py @@ -6,6 +6,7 @@ import pytest from django.utils import timezone +from a3m.api.transferservice.v1beta1.request_response_pb2 import ProcessingConfig from a3m.main import models from a3m.server.jobs import DirectoryClientScriptJob from a3m.server.jobs import FilesClientScriptJob @@ -13,7 +14,6 @@ from a3m.server.jobs import NextLinkDecisionJob from a3m.server.packages import Package from a3m.server.queues import PackageQueue -from a3m.server.rpc.proto.a3m_pb2 import ProcessingConfig from a3m.server.tasks import TaskBackend from a3m.server.workflow import load as load_workflow diff --git a/tests/server/test_package.py b/tests/server/test_package.py index 498573675..e07f8cdf5 100644 --- a/tests/server/test_package.py +++ b/tests/server/test_package.py @@ -6,10 +6,10 @@ import pytest +from a3m.api.transferservice.v1beta1.request_response_pb2 import ProcessingConfig from a3m.main import models from a3m.server.packages import Package from a3m.server.queues import PackageQueue -from a3m.server.rpc.proto.a3m_pb2 import ProcessingConfig from a3m.server.workflow import load as load_workflow FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") diff --git a/tests/server/test_queues.py b/tests/server/test_queues.py index 77070176a..e9382f4d8 100644 --- a/tests/server/test_queues.py +++ b/tests/server/test_queues.py @@ -6,10 +6,10 @@ import pytest +from a3m.api.transferservice.v1beta1.request_response_pb2 import ProcessingConfig from a3m.server.jobs import Job from a3m.server.packages import Package from a3m.server.queues import PackageQueue -from a3m.server.rpc.proto.a3m_pb2 import ProcessingConfig from a3m.server.workflow import Link diff --git a/tox.ini b/tox.ini index ee4981f92..a30328cc4 100644 --- a/tox.ini +++ b/tox.ini @@ -33,9 +33,3 @@ commands = [testenv:vulture] skip_install = True commands = - vulture --exclude="a3m/settings,a3m/main/models.py,a3m/main/migrations,a3m/fpr/models.py,a3m/fpr/migrations,a3m/server/rpc/proto" a3m - -[flake8] -exclude = .tox, .git, __pycache__, .cache, build, dist, *.pyc, *.egg-info, .eggs -application-import-names = flake8 -select = C, E, F, W, B, B950 -ignore = E203, E402, E501, E722, W503, W605