From a1dd2b283f304be77760f3841d79d63a983cf008 Mon Sep 17 00:00:00 2001 From: Dhanshree Arora Date: Fri, 26 Jul 2024 17:09:28 +0300 Subject: [PATCH] Make MODEL_SIZE_FILE always go in the model dest directory (#1212) * Make size.json always go in the model dest directory * Write to model source file when the model is actually fetched and the dest directory is created in eos * Fix modifying model size for dockerized models in information.json file --- ersilia/hub/content/information.py | 3 +- ersilia/hub/fetch/fetch.py | 11 +++---- ersilia/hub/fetch/lazy_fetchers/dockerhub.py | 34 ++++++-------------- ersilia/hub/fetch/register/register.py | 2 +- ersilia/hub/pull/pull.py | 11 +++---- 5 files changed, 21 insertions(+), 40 deletions(-) diff --git a/ersilia/hub/content/information.py b/ersilia/hub/content/information.py index 4a453142..d7421a5f 100644 --- a/ersilia/hub/content/information.py +++ b/ersilia/hub/content/information.py @@ -16,7 +16,6 @@ CARD_FILE, SERVICE_CLASS_FILE, APIS_LIST_FILE, - EOS, MODEL_SOURCE_FILE, ) @@ -47,7 +46,7 @@ def _get_service_class(self): return None def _get_model_source(self): - model_source_file = os.path.join(EOS, MODEL_SOURCE_FILE) + model_source_file = os.path.join((self.dest_folder), MODEL_SOURCE_FILE) if os.path.exists(model_source_file): with open(model_source_file) as f: return f.read().rstrip() diff --git a/ersilia/hub/fetch/fetch.py b/ersilia/hub/fetch/fetch.py index fa6f2c94..24dcb3bb 100644 --- a/ersilia/hub/fetch/fetch.py +++ b/ersilia/hub/fetch/fetch.py @@ -81,7 +81,7 @@ def __init__( self.model_source = next( (source for condition, source in sources.items() if condition), "DockerHub" ) - self.logger.debug("Model was fetched from {0}".format(self.model_source)) + self.logger.debug("Model getting fetched from {0}".format(self.model_source)) @throw_ersilia_exception def _decide_fetcher(self, model_id): @@ -228,8 +228,7 @@ def _fetch(self, model_id): return do_dockerhub = self._decide_if_use_dockerhub(model_id=model_id) if do_dockerhub: - print("Fetching from DockerHub") - self.logger.debug("Fetching from DockerHub") + self.logger.debug("Decided to fetch from DockerHub") self._fetch_from_dockerhub(model_id=model_id) return if self.overwrite is None: @@ -239,9 +238,9 @@ def _fetch(self, model_id): self._fetch_not_from_dockerhub(model_id=model_id) def fetch(self, model_id): + self._fetch(model_id) + self._standard_csv_example(model_id) self.logger.debug("Writing model source to file") - model_source_file = os.path.join(EOS, MODEL_SOURCE_FILE) + model_source_file = os.path.join(self._model_path(model_id), MODEL_SOURCE_FILE) with open(model_source_file, "w") as f: f.write(self.model_source) - self._fetch(model_id) - self._standard_csv_example(model_id) diff --git a/ersilia/hub/fetch/lazy_fetchers/dockerhub.py b/ersilia/hub/fetch/lazy_fetchers/dockerhub.py index 420463e3..5307ae91 100644 --- a/ersilia/hub/fetch/lazy_fetchers/dockerhub.py +++ b/ersilia/hub/fetch/lazy_fetchers/dockerhub.py @@ -10,8 +10,6 @@ PREDEFINED_EXAMPLE_FILES, INFORMATION_FILE, API_SCHEMA_FILE, - SERVICE_CLASS_FILE, - MODEL_SIZE_FILE, ) from ...pull.pull import ModelPuller @@ -108,34 +106,19 @@ def modify_information(self, model_id): :param service_class_file: File containing the model service class. :size_file: File containing the size of the pulled docker image. """ - file = "{0}/dest/{1}/{2}".format(EOS, model_id, INFORMATION_FILE) - service_class_file = os.path.join( - self._get_bundle_location(model_id), SERVICE_CLASS_FILE - ) - size_file = os.path.join(EOS, MODEL_SIZE_FILE) - - try: - with open(service_class_file, "r") as f: - service_class = f.read().strip() - except FileNotFoundError: - return None - - try: - with open(size_file, "r") as m: - size = json.load(m) - except FileNotFoundError: - return None - + information_file = "{0}/dest/{1}/{2}".format(EOS, model_id, INFORMATION_FILE) + mp = ModelPuller(model_id=model_id, config_json=self.config_json) try: - with open(file, "r") as infile: + with open(information_file, "r") as infile: data = json.load(infile) except FileNotFoundError: + self.logger.error("Information file not found, not modifying anything") return None - data["service_class"] = service_class - data["size"] = size - - with open(file, "w") as outfile: + data["service_class"] = "pulled_docker" # Using this literal here to prevent a file read + # from service class file for a model fetched through DockerHub since we already know the service class. + data["size"] = mp._get_size_of_local_docker_image_in_mb() + with open(information_file, "w") as outfile: json.dump(data, outfile, indent=4) @throw_ersilia_exception @@ -143,6 +126,7 @@ def fetch(self, model_id): if not DockerRequirement().is_active(): raise DockerNotActiveError() mp = ModelPuller(model_id=model_id, config_json=self.config_json) + self.logger.debug("Pulling model image from DockerHub") mp.pull() mr = ModelRegisterer(model_id=model_id, config_json=self.config_json) mr.register(is_from_dockerhub=True) diff --git a/ersilia/hub/fetch/register/register.py b/ersilia/hub/fetch/register/register.py index d151d9b4..23c826a7 100644 --- a/ersilia/hub/fetch/register/register.py +++ b/ersilia/hub/fetch/register/register.py @@ -125,7 +125,7 @@ def register(self, is_from_dockerhub=False, is_from_hosted=False): raise Exception if is_from_dockerhub and not is_from_hosted: self.register_from_dockerhub() - self.register_not_from_dockerhub() + self.register_not_from_hosted() if not is_from_dockerhub and is_from_hosted: self.register_from_hosted() self.register_not_from_dockerhub() diff --git a/ersilia/hub/pull/pull.py b/ersilia/hub/pull/pull.py index 2e60e07d..e809f4e7 100644 --- a/ersilia/hub/pull/pull.py +++ b/ersilia/hub/pull/pull.py @@ -141,14 +141,13 @@ def pull(self): size = self._get_size_of_local_docker_image_in_mb() if size: self.logger.debug("Size of image {0} MB".format(size)) - path = os.path.join(EOS, MODEL_SIZE_FILE) - with open(path, "w") as f: - json.dump({"size": size, "units": "MB"}, f, indent=4) - self.logger.debug("Size written to {}".format(path)) + # path = os.path.join(self._model_path(self.model_id), MODEL_SIZE_FILE) + # with open(path, "w") as f: + # json.dump({"size": size, "units": "MB"}, f, indent=4) + # self.logger.debug("Size written to {}".format(path)) else: self.logger.warning("Could not obtain size of image") - # except: #TODO add better error - # raise DockerImageArchitectureNotAvailableError(model=self.model_id) + return size else: self.logger.info("Image {0} is not available".format(self.image_name)) raise DockerImageNotAvailableError(model=self.model_id)