From 466c08c893455fed01c282e91d13e3736a55f6a6 Mon Sep 17 00:00:00 2001 From: kamangir Date: Sun, 5 Jan 2025 19:25:31 -0800 Subject: [PATCH] refactors - kamangir/bolt#746 --- README.md | 4 +- vancouver_watching/__init__.py | 2 +- vancouver_watching/help/detect.py | 1 - vancouver_watching/target/__main__.py | 38 +++--- vancouver_watching/target/classes.py | 164 +----------------------- vancouver_watching/target/detection.py | 116 +++++++++++++++++ vancouver_watching/target/ingest.py | 36 ++++++ vancouver_watching/tests/test_target.py | 12 +- 8 files changed, 181 insertions(+), 192 deletions(-) create mode 100644 vancouver_watching/target/detection.py create mode 100644 vancouver_watching/target/ingest.py diff --git a/README.md b/README.md index 7a5ad6a..ac49de8 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,12 @@ vanwatch ingest \ | | | | | --- | --- | --- | -| [ingest -> detect](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true) [![image](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true)](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true) | [time-series](https://kamangir-public.s3.ca-central-1.amazonaws.com/vanwatch-cache-2024-02-28-21-04-19-26236.tar.gz) [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/2024-01-06-20-39-46-73614/2024-01-06-20-39-46-73614-2X.gif?raw=true&random=yGTPdNGrgIpp9Er9)](https://kamangir-public.s3.ca-central-1.amazonaws.com/vanwatch-cache-2024-02-28-21-04-19-26236.tar.gz) | [last build](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=T7ABUNFjyjjE08AR) [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=iEVgV3d8gZlub9EP)](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=T7ABUNFjyjjE08AR) | +| [ingest -> detect](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true) [![image](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true)](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true) | [time-series](https://kamangir-public.s3.ca-central-1.amazonaws.com/vanwatch-cache-2024-02-28-21-04-19-26236.tar.gz) [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/2024-01-06-20-39-46-73614/2024-01-06-20-39-46-73614-2X.gif?raw=true&random=BhL84PjciIFpIfIX)](https://kamangir-public.s3.ca-central-1.amazonaws.com/vanwatch-cache-2024-02-28-21-04-19-26236.tar.gz) | [last build](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=cD5zC8m01fCFFK7o) [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=RPZzhW30mSSFEvHU)](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=cD5zC8m01fCFFK7o) | --- [![pylint](https://github.com/kamangir/vancouver-watching/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/vancouver-watching/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/vancouver-watching/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/vancouver-watching/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/vancouver-watching/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/vancouver-watching/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/vancouver-watching.svg)](https://pypi.org/project/vancouver-watching/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/vancouver-watching)](https://pypistats.org/packages/vancouver-watching) -built by 🌀 [`blue_options-4.175.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌈 [`vancouver_watching-3.467.1`](https://github.com/kamangir/vancouver-watching). +built by 🌀 [`blue_options-4.175.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌈 [`vancouver_watching-3.468.1`](https://github.com/kamangir/vancouver-watching). diff --git a/vancouver_watching/__init__.py b/vancouver_watching/__init__.py index d94df26..b2cea53 100644 --- a/vancouver_watching/__init__.py +++ b/vancouver_watching/__init__.py @@ -8,7 +8,7 @@ DESCRIPTION = f"{ICON} Vancouver Watching with AI." -VERSION = "3.467.1" +VERSION = "3.468.1" REPO_NAME = "vancouver-watching" diff --git a/vancouver_watching/help/detect.py b/vancouver_watching/help/detect.py index f35f2eb..93f7218 100644 --- a/vancouver_watching/help/detect.py +++ b/vancouver_watching/help/detect.py @@ -4,7 +4,6 @@ args = [ - "[--detect 0]", "[--overwrite 1]", "[--verbose 1]", ] diff --git a/vancouver_watching/target/__main__.py b/vancouver_watching/target/__main__.py index 6c2ec13..68160a7 100644 --- a/vancouver_watching/target/__main__.py +++ b/vancouver_watching/target/__main__.py @@ -5,15 +5,19 @@ from vancouver_watching import NAME from vancouver_watching.target import Target +from vancouver_watching.target.detection import detect_in_target +from vancouver_watching.target.ingest import ingest_target from vancouver_watching.logger import logger NAME = module.name(__file__, NAME) +list_of_tasks = ["detect", "ingest"] + parser = argparse.ArgumentParser(NAME) parser.add_argument( "task", type=str, - help="detect|ingest", + help=" | ".join(list_of_tasks), ) parser.add_argument( "--geojson", @@ -50,12 +54,6 @@ default=0, help="0|1", ) -parser.add_argument( - "--detect", - type=int, - default=1, - help="0|1", -) parser.add_argument( "--model_id", type=str, @@ -63,8 +61,9 @@ ) args = parser.parse_args() -success = False -if args.task == "detect": +success = args.task in list_of_tasks + +if success: target = Target( map_filename=args.geojson, do_dryrun=args.do_dryrun, @@ -72,26 +71,21 @@ ) success = target.valid - if success and args.detect: - success = target.detect( +if args.task == "detect": + if success: + success = detect_in_target( + target=target, model_id=args.model_id, animated_gif=args.animated_gif, count=args.count, overwrite=args.overwrite, ) - - if success: - success = target.summarize() elif args.task == "ingest": - target = Target( - map_filename=args.geojson, - do_dryrun=args.do_dryrun, - verbose=args.verbose, - ) - success = target.valid - if success: - success = target.ingest(count=args.count) + success = ingest_target( + target=target, + count=args.count, + ) else: success = None diff --git a/vancouver_watching/target/classes.py b/vancouver_watching/target/classes.py index af85d37..5d95a75 100644 --- a/vancouver_watching/target/classes.py +++ b/vancouver_watching/target/classes.py @@ -1,14 +1,5 @@ -import os -import os.path -from typing import List -import re -from collections import Counter -from tqdm import tqdm - from blue_objects import file, path -from blue_objects.graphics.gif import generate_animated_gif -from vancouver_watching.ai import Ultralytics_API from vancouver_watching.logger import logger @@ -52,116 +43,10 @@ def __init__( ) success, self.metadata = file.load_json( - self.metadata_filename, ignore_error=True - ) - if not success: - logger.info("generating metadata: {}".format(self.metadata_filename)) - - p = re.compile( - r"https?:\/\/([0-9.]+).\:([0-9.]+)\/webcapture.jpg.*.channel=([0-9.]+).*" - ) - for _, row in tqdm(self.gdf.iterrows()): - cameras = {} - for url in row["cameras"].split(","): - matches = p.match(url) - if matches: - filename = ( - f"{matches[1]}-{matches[2]}-{matches[3]}".replace(".", "-") - + ".jpg" - ) - else: - filename = file.name_and_extension(url) - - if file.extension(filename) not in "jpg,jpeg,png".split(","): - logger.error("bad url: {}.".format(url)) - continue - - cameras[filename] = {"url": url} - - self.metadata[row["mapid"]] = {"cameras": cameras} - - self.save_metadata() - - def detect( - self, - model_id: str, - animated_gif: bool = False, - count: int = -1, - overwrite: bool = False, - ) -> bool: - logger.info( - "{}.detect({},model_id{},count={}{})".format( - self.__class__.__name__, - model_id, - ",animated_gif" if animated_gif else "", - count, - ",overwrite" if overwrite else "", - ) - ) - - ultralytics_api = Ultralytics_API( - model_id, - self.do_dryrun, - self.verbose, - ) - - list_of_images: List[str] = [] - counter: int = 0 - for mapid in tqdm(self.metadata): - for filename, metadata in self.metadata[mapid]["cameras"].items(): - full_filename = os.path.join(self.object_path, filename) - if not file.exists(full_filename): - continue - - if self.do_dryrun: - logger.info(full_filename) - continue - - if not overwrite and "inference" in metadata: - continue - - success, inference = ultralytics_api.infer(full_filename) - if success: - metadata["inference"] = inference - list_of_images += [inference.get("render_filename", "")] - - counter += 1 - if count != -1 and counter >= count: - break - if count != -1 and counter >= count: - break - - if not self.save_metadata(): - return False - - return not animated_gif or generate_animated_gif( - [filename for filename in list_of_images if filename], - os.path.join(self.object_path, f"{self.object_name}.gif"), - frame_duration=500, + self.metadata_filename, + ignore_error=True, ) - - def ingest( - self, - count: int, - ) -> bool: - logger.info("{}.ingest({})".format(self.__class__.__name__, count)) - counter = 0 - for mapid in tqdm(self.metadata): - for filename, metadata in self.metadata[mapid]["cameras"].items(): - url = metadata["url"] - - if self.do_dryrun: - logger.info(url) - elif not file.download(url, os.path.join(self.object_path, filename)): - logger.error("bad url: {}.".format(url)) - - counter += 1 - if counter >= count and count != -1: - break - if counter >= count and count != -1: - break - - return True + assert success def on_map( self, @@ -193,46 +78,3 @@ def save_metadata(self) -> bool: self.metadata, log=True, ) - - def summarize(self) -> bool: - all_things = {} - for mapid in tqdm(self.metadata): - detections = {} - for metadata in self.metadata[mapid]["cameras"].values(): - if not "inference" in metadata: - continue - - for thing, count in Counter( - [ - thing["name"] - for thing in metadata["inference"]["images"][0]["results"] - ] - ).items(): - detections[thing] = detections.get(thing, 0) + count - - if detections: - logger.info( - "{}: {}".format( - mapid, - " + ".join( - [f"{count}*{thing}" for thing, count in detections.items()] - ), - ) - ) - - for thing, count in detections.items(): - all_things[thing] = all_things.get(thing, 0) + count - - if thing not in self.gdf.columns: - self.gdf[thing] = 0 - logger.info("+= {}".format(thing)) - - self.gdf.loc[self.gdf["mapid"] == mapid, thing] += count - - logger.info( - "total: {}".format( - " + ".join([f"{count}*{thing}" for thing, count in all_things.items()]) - ) - ) - - return self.save_gdf() diff --git a/vancouver_watching/target/detection.py b/vancouver_watching/target/detection.py new file mode 100644 index 0000000..7a36546 --- /dev/null +++ b/vancouver_watching/target/detection.py @@ -0,0 +1,116 @@ +import os +import os.path +from typing import List +import re +from collections import Counter +from tqdm import tqdm + +from blueness import module +from blue_objects import file +from blue_objects.graphics.gif import generate_animated_gif + +from vancouver_watching import NAME +from vancouver_watching.ai import Ultralytics_API +from vancouver_watching.target.classes import Target +from vancouver_watching.logger import logger + + +NAME = module.name(__file__, NAME) + + +def detect_in_target( + target: Target, + model_id: str, + animated_gif: bool = False, + count: int = -1, + overwrite: bool = False, +) -> bool: + logger.info( + "{}.detect_in_target(model_id={},count={}){}{}".format( + NAME, + model_id, + count, + " +animated_gif" if animated_gif else "", + " +overwrite" if overwrite else "", + ) + ) + + ultralytics_api = Ultralytics_API( + model_id, + target.do_dryrun, + target.verbose, + ) + + list_of_images: List[str] = [] + counter: int = 0 + all_detections = {} + for mapid in tqdm(target.metadata): + detections = {} + for filename, metadata in target.metadata[mapid]["cameras"].items(): + full_filename = os.path.join(target.object_path, filename) + if not file.exists(full_filename): + continue + + if target.do_dryrun: + logger.info(f"dryrun: skipping {full_filename} ...") + continue + + if overwrite or "inference" not in metadata: + success, inference = ultralytics_api.infer(full_filename) + if success: + metadata["inference"] = inference + list_of_images += [inference.get("render_filename", "")] + + if "inference" not in metadata: + continue + + for thing, count in Counter( + [ + thing["name"] + for thing in metadata["inference"]["images"][0]["results"] + ] + ).items(): + detections[thing] = detections.get(thing, 0) + count + + if detections: + logger.info( + "{}: {}".format( + mapid, + " + ".join( + [f"{count}*{thing}" for thing, count in detections.items()] + ), + ) + ) + + for thing, count in detections.items(): + all_detections[thing] = all_detections.get(thing, 0) + count + + if thing not in target.gdf.columns: + target.gdf[thing] = 0 + logger.info("+= {}".format(thing)) + + target.gdf.loc[target.gdf["mapid"] == mapid, thing] += count + + counter += 1 + if count != -1 and counter >= count: + break + if count != -1 and counter >= count: + break + + logger.info( + "all: {}".format( + " + ".join([f"{count}*{thing}" for thing, count in all_detections.items()]) + ) + ) + + if not target.save_metadata(): + return False + + if not target.save_gdf(): + return False + + return not animated_gif or generate_animated_gif( + [filename for filename in list_of_images if filename], + os.path.join(target.object_path, f"{target.object_name}.gif"), + frame_duration=500, + ) diff --git a/vancouver_watching/target/ingest.py b/vancouver_watching/target/ingest.py new file mode 100644 index 0000000..609c857 --- /dev/null +++ b/vancouver_watching/target/ingest.py @@ -0,0 +1,36 @@ +import os +import os.path +from tqdm import tqdm + +from blueness import module +from blue_objects import file + +from vancouver_watching import NAME +from vancouver_watching.target.classes import Target +from vancouver_watching.logger import logger + +NAME = module.name(__file__, NAME) + + +def ingest_target( + target: Target, + count: int = -1, +) -> bool: + logger.info("{}.ingest({})".format(NAME, count)) + counter = 0 + for mapid in tqdm(target.metadata): + for filename, metadata in target.metadata[mapid]["cameras"].items(): + url = metadata["url"] + + if target.do_dryrun: + logger.info(url) + elif not file.download(url, os.path.join(target.object_path, filename)): + logger.error("bad url: {}.".format(url)) + + counter += 1 + if counter >= count and count != -1: + break + if counter >= count and count != -1: + break + + return True diff --git a/vancouver_watching/tests/test_target.py b/vancouver_watching/tests/test_target.py index 0fa2b56..4250c05 100644 --- a/vancouver_watching/tests/test_target.py +++ b/vancouver_watching/tests/test_target.py @@ -4,6 +4,8 @@ from vancouver_watching import env from vancouver_watching.target import Target +from vancouver_watching.target.detection import detect_in_target +from vancouver_watching.target.ingest import ingest_target @pytest.mark.parametrize( @@ -18,7 +20,7 @@ ), ], ) -def test_Area( +def test_target( object_name, model_id, ): @@ -52,13 +54,13 @@ def test_Area( target = Target(geojson_filename) assert target.valid - assert target.ingest( + assert ingest_target( + target=target, count=1, ) - assert target.detect( + assert detect_in_target( + target=target, model_id=model_id, count=1, ) - - assert target.summarize()