refactors - kamangir/bolt#746

kamangir · Jan 6, 2025 · 466c08c · 466c08c
1 parent b958f1f
commit 466c08c
Show file tree

Hide file tree

Showing 8 changed files with 181 additions and 192 deletions.
diff --git a/README.md b/README.md
@@ -17,12 +17,12 @@ vanwatch ingest \
 
 |   |   |   |
 | --- | --- | --- |
-| [ingest -> detect](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true) [![image](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true)](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true) | [time-series](https://kamangir-public.s3.ca-central-1.amazonaws.com/vanwatch-cache-2024-02-28-21-04-19-26236.tar.gz) [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/2024-01-06-20-39-46-73614/2024-01-06-20-39-46-73614-2X.gif?raw=true&random=yGTPdNGrgIpp9Er9)](https://kamangir-public.s3.ca-central-1.amazonaws.com/vanwatch-cache-2024-02-28-21-04-19-26236.tar.gz) | [last build](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=T7ABUNFjyjjE08AR) [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=iEVgV3d8gZlub9EP)](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=T7ABUNFjyjjE08AR) |
+| [ingest -> detect](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true) [![image](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true)](https://github.com/kamangir/assets/raw/main/vanwatch/2023-11-25-openai-vision/QGIS.png?raw=true) | [time-series](https://kamangir-public.s3.ca-central-1.amazonaws.com/vanwatch-cache-2024-02-28-21-04-19-26236.tar.gz) [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/2024-01-06-20-39-46-73614/2024-01-06-20-39-46-73614-2X.gif?raw=true&random=BhL84PjciIFpIfIX)](https://kamangir-public.s3.ca-central-1.amazonaws.com/vanwatch-cache-2024-02-28-21-04-19-26236.tar.gz) | [last build](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=cD5zC8m01fCFFK7o) [![image](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=RPZzhW30mSSFEvHU)](https://kamangir-public.s3.ca-central-1.amazonaws.com/test_vancouver_watching_ingest/animation.gif?raw=true&random=cD5zC8m01fCFFK7o) |
 
 ---
 
 
 [![pylint](https://github.com/kamangir/vancouver-watching/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/vancouver-watching/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/vancouver-watching/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/vancouver-watching/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/vancouver-watching/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/vancouver-watching/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/vancouver-watching.svg)](https://pypi.org/project/vancouver-watching/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/vancouver-watching)](https://pypistats.org/packages/vancouver-watching)
 
-built by 🌀 [`blue_options-4.175.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌈 [`vancouver_watching-3.467.1`](https://github.com/kamangir/vancouver-watching).
+built by 🌀 [`blue_options-4.175.1`](https://github.com/kamangir/awesome-bash-cli), based on 🌈 [`vancouver_watching-3.468.1`](https://github.com/kamangir/vancouver-watching).
 
diff --git a/vancouver_watching/__init__.py b/vancouver_watching/__init__.py
@@ -8,7 +8,7 @@
 
 DESCRIPTION = f"{ICON} Vancouver Watching with AI."
 
-VERSION = "3.467.1"
+VERSION = "3.468.1"
 
 REPO_NAME = "vancouver-watching"
 

diff --git a/vancouver_watching/help/detect.py b/vancouver_watching/help/detect.py
@@ -4,7 +4,6 @@
 
 
 args = [
-    "[--detect 0]",
     "[--overwrite 1]",
     "[--verbose 1]",
 ]

diff --git a/vancouver_watching/target/__main__.py b/vancouver_watching/target/__main__.py
@@ -5,15 +5,19 @@
 
 from vancouver_watching import NAME
 from vancouver_watching.target import Target
+from vancouver_watching.target.detection import detect_in_target
+from vancouver_watching.target.ingest import ingest_target
 from vancouver_watching.logger import logger
 
 NAME = module.name(__file__, NAME)
 
+list_of_tasks = ["detect", "ingest"]
+
 parser = argparse.ArgumentParser(NAME)
 parser.add_argument(
     "task",
     type=str,
-    help="detect|ingest",
+    help=" | ".join(list_of_tasks),
 )
 parser.add_argument(
     "--geojson",
@@ -50,48 +54,38 @@
     default=0,
     help="0|1",
 )
-parser.add_argument(
-    "--detect",
-    type=int,
-    default=1,
-    help="0|1",
-)
 parser.add_argument(
     "--model_id",
     type=str,
     default="",
 )
 args = parser.parse_args()
 
-success = False
-if args.task == "detect":
+success = args.task in list_of_tasks
+
+if success:
     target = Target(
         map_filename=args.geojson,
         do_dryrun=args.do_dryrun,
         verbose=args.verbose,
     )
     success = target.valid
 
-    if success and args.detect:
-        success = target.detect(
+if args.task == "detect":
+    if success:
+        success = detect_in_target(
+            target=target,
             model_id=args.model_id,
             animated_gif=args.animated_gif,
             count=args.count,
             overwrite=args.overwrite,
         )
-
-    if success:
-        success = target.summarize()
 elif args.task == "ingest":
-    target = Target(
-        map_filename=args.geojson,
-        do_dryrun=args.do_dryrun,
-        verbose=args.verbose,
-    )
-    success = target.valid
-
     if success:
-        success = target.ingest(count=args.count)
+        success = ingest_target(
+            target=target,
+            count=args.count,
+        )
 else:
     success = None
 

diff --git a/vancouver_watching/target/classes.py b/vancouver_watching/target/classes.py
@@ -1,14 +1,5 @@
-import os
-import os.path
-from typing import List
-import re
-from collections import Counter
-from tqdm import tqdm
-
 from blue_objects import file, path
-from blue_objects.graphics.gif import generate_animated_gif
 
-from vancouver_watching.ai import Ultralytics_API
 from vancouver_watching.logger import logger
 
 
@@ -52,116 +43,10 @@ def __init__(
         )
 
         success, self.metadata = file.load_json(
-            self.metadata_filename, ignore_error=True
-        )
-        if not success:
-            logger.info("generating metadata: {}".format(self.metadata_filename))
-
-            p = re.compile(
-                r"https?:\/\/([0-9.]+).\:([0-9.]+)\/webcapture.jpg.*.channel=([0-9.]+).*"
-            )
-            for _, row in tqdm(self.gdf.iterrows()):
-                cameras = {}
-                for url in row["cameras"].split(","):
-                    matches = p.match(url)
-                    if matches:
-                        filename = (
-                            f"{matches[1]}-{matches[2]}-{matches[3]}".replace(".", "-")
-                            + ".jpg"
-                        )
-                    else:
-                        filename = file.name_and_extension(url)
-
-                    if file.extension(filename) not in "jpg,jpeg,png".split(","):
-                        logger.error("bad url: {}.".format(url))
-                        continue
-
-                    cameras[filename] = {"url": url}
-
-                self.metadata[row["mapid"]] = {"cameras": cameras}
-
-            self.save_metadata()
-
-    def detect(
-        self,
-        model_id: str,
-        animated_gif: bool = False,
-        count: int = -1,
-        overwrite: bool = False,
-    ) -> bool:
-        logger.info(
-            "{}.detect({},model_id{},count={}{})".format(
-                self.__class__.__name__,
-                model_id,
-                ",animated_gif" if animated_gif else "",
-                count,
-                ",overwrite" if overwrite else "",
-            )
-        )
-
-        ultralytics_api = Ultralytics_API(
-            model_id,
-            self.do_dryrun,
-            self.verbose,
-        )
-
-        list_of_images: List[str] = []
-        counter: int = 0
-        for mapid in tqdm(self.metadata):
-            for filename, metadata in self.metadata[mapid]["cameras"].items():
-                full_filename = os.path.join(self.object_path, filename)
-                if not file.exists(full_filename):
-                    continue
-
-                if self.do_dryrun:
-                    logger.info(full_filename)
-                    continue
-
-                if not overwrite and "inference" in metadata:
-                    continue
-
-                success, inference = ultralytics_api.infer(full_filename)
-                if success:
-                    metadata["inference"] = inference
-                    list_of_images += [inference.get("render_filename", "")]
-
-                    counter += 1
-                    if count != -1 and counter >= count:
-                        break
-            if count != -1 and counter >= count:
-                break
-
-        if not self.save_metadata():
-            return False
-
-        return not animated_gif or generate_animated_gif(
-            [filename for filename in list_of_images if filename],
-            os.path.join(self.object_path, f"{self.object_name}.gif"),
-            frame_duration=500,
+            self.metadata_filename,
+            ignore_error=True,
         )
-
-    def ingest(
-        self,
-        count: int,
-    ) -> bool:
-        logger.info("{}.ingest({})".format(self.__class__.__name__, count))
-        counter = 0
-        for mapid in tqdm(self.metadata):
-            for filename, metadata in self.metadata[mapid]["cameras"].items():
-                url = metadata["url"]
-
-                if self.do_dryrun:
-                    logger.info(url)
-                elif not file.download(url, os.path.join(self.object_path, filename)):
-                    logger.error("bad url: {}.".format(url))
-
-                counter += 1
-                if counter >= count and count != -1:
-                    break
-            if counter >= count and count != -1:
-                break
-
-        return True
+        assert success
 
     def on_map(
         self,
@@ -193,46 +78,3 @@ def save_metadata(self) -> bool:
             self.metadata,
             log=True,
         )
-
-    def summarize(self) -> bool:
-        all_things = {}
-        for mapid in tqdm(self.metadata):
-            detections = {}
-            for metadata in self.metadata[mapid]["cameras"].values():
-                if not "inference" in metadata:
-                    continue
-
-                for thing, count in Counter(
-                    [
-                        thing["name"]
-                        for thing in metadata["inference"]["images"][0]["results"]
-                    ]
-                ).items():
-                    detections[thing] = detections.get(thing, 0) + count
-
-            if detections:
-                logger.info(
-                    "{}: {}".format(
-                        mapid,
-                        " + ".join(
-                            [f"{count}*{thing}" for thing, count in detections.items()]
-                        ),
-                    )
-                )
-
-            for thing, count in detections.items():
-                all_things[thing] = all_things.get(thing, 0) + count
-
-                if thing not in self.gdf.columns:
-                    self.gdf[thing] = 0
-                    logger.info("+= {}".format(thing))
-
-                self.gdf.loc[self.gdf["mapid"] == mapid, thing] += count
-
-        logger.info(
-            "total: {}".format(
-                " + ".join([f"{count}*{thing}" for thing, count in all_things.items()])
-            )
-        )
-
-        return self.save_gdf()