From bbd1de6a4e1dade16cb0090237ef9fd6470a96c5 Mon Sep 17 00:00:00 2001 From: seanavery Date: Mon, 30 Sep 2024 16:19:11 -0400 Subject: [PATCH 01/15] Improve module desc --- meta.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.json b/meta.json index 49f931c..94b7f55 100644 --- a/meta.json +++ b/meta.json @@ -2,7 +2,7 @@ "module_id": "viam:video-store", "visibility": "private", "url": "https://github.com/viam-modules/video-store", - "description": "module for video storage and retrieval", + "description": "Golang module for video storage and retrieval from a camera", "models": [ { "api": "rdk:component:camera", From 85ad19d7de88c00c4380b82a99d9f974d8bf4956 Mon Sep 17 00:00:00 2001 From: seanavery Date: Thu, 3 Oct 2024 17:53:06 -0400 Subject: [PATCH 02/15] Implement simple keyframe force interval --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1ce0066..a136143 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( go.viam.com/rdk v0.40.0 go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 go.viam.com/utils v0.1.98 + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc golang.org/x/mobile v0.0.0-20240112133503-c713f31d574b golang.org/x/tools v0.22.0 gotest.tools/gotestsum v1.10.0 @@ -306,7 +307,6 @@ require ( go.viam.com/api v0.1.336 // indirect goji.io v2.0.2+incompatible // indirect golang.org/x/crypto v0.24.0 // indirect - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/exp/typeparams v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/image v0.19.0 // indirect golang.org/x/mod v0.18.0 // indirect From dd7dc93826fca19809928347e0f50d93694b341a Mon Sep 17 00:00:00 2001 From: seanavery Date: Mon, 7 Oct 2024 11:38:52 -0400 Subject: [PATCH 03/15] Add tables for attributes and do command calls --- README.md | 123 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index d8d5405..8715bd0 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,67 @@ -# Video Storage +# [`video-store` module](https://app.viam.com/module/viam/video-store) The `video-store` module brings security camera functionality to your smart machine! The module consumes a source [Camera](https://docs.viam.com/components/camera/) and saves the output as video files on disk. You can then upload video slices to the cloud using the [save](#save) command, or request the video bytes directly using the [fetch](#fetch) command. ## Configure your `video-store` component -Fill in the attributes as applicable to the component, according to the template below. +> [!NOTE] +> For more information, see [Configure a Machine](https://docs.viam.com/manage/configuration/). + + +### Attributes + +| Attribute | Sub-Attribute | Type | Inclusion | Description | +|-----------------|-------------------|---------|-----------|---------------------------------------------------------------------------------------------------| +| `camera` | | string | required | Name of the source camera to read images from. | +| `sync` | | string | required | Name of the dependency datamanager service. | +| `storage` | | object | required | | +| | `segment_seconds` | integer | optional | Length in seconds of the individual segment video files. | +| | `size_gb` | integer | required | Total amount of allocated storage in gigabytes. | +| | `storage_path` | string | optional | Custom path to use for video storage. | +| | `upload_path` | string | optional | Custom path to use for uploading files. If not under `~/.viam/capture`, you will need to add to `additional_sync_paths` in datamanager service configuration. | +| `video` | | object | optional | | +| | `format` | string | optional | Name of video format to use (e.g., mp4). | +| | `codec` | string | optional | Name of video codec to use (e.g., h264). | +| | `bitrate` | integer | optional | Throughput of encoder in bits per second. Higher for better quality video, and lower for better storage efficiency. | +| | `preset` | string | optional | Name of codec video preset to use. See [here](https://trac.ffmpeg.org/wiki/Encode/H.264#a2.Chooseapresetandtune) for preset options. | +| `cam_props` | | object | required | | +| | `width` | integer | required | Width of the source camera frames in pixels. | +| | `height` | integer | required | Height of the source camera frames in pixels. | +| | `framerate` | integer | required | Number of frames per second provided by the source camera. | + +### Example Configuration: ```json { - "name": , + "name": "video-store", "namespace": "rdk", "type": "camera", "model": "viam:video:storage", "attributes": { - "camera": , [required] - "sync": , [required] - "storage": { [required] - "segment_seconds": , [optional] - "size_gb": , [required] - "storage_path": , [optional] - "upload_path": , [optional] - }, - "video": { [optional] - "format": , [optional] - "codec": , [optional] - "bitrate": , [optional] - "preset": , [optional] - }, - "cam_props": { [required] - "width": , [required] - "height": , [required] - "framerate": , [required] + "camera": "wc-cam" + "sync": "data-manager", + "storage": { + "segment_seconds": 10, + "size_gb": 50, }, + "cam_props": { + "width": 640, + "height": 480, + "framerate": 25 + } }, "depends_on": [ - , [required] - [required] + "wc-cam", + "data-manager" ] } ``` -Make sure to configure a [Data Manager Service](https://docs.viam.com/services/data/cloud-sync/) to uplaod video files to the cloud. +### Configure a Data Manager Service + +Make sure to configure a [Data Manager Service](https://docs.viam.com/services/data/cloud-sync/) to uplaod video files to the cloud when saving video slices. + +> [!NOTE] +> The `additional_sync_paths` attribute must include the custom path specified in the `upload_path` attribute of the `video-store` component if it is not under `~/.viam/capture`. ```json { @@ -58,7 +80,7 @@ Make sure to configure a [Data Manager Service](https://docs.viam.com/services/d } ``` -## DoCommands API +## DoCommand API ### From/To @@ -81,18 +103,35 @@ The datetime format used is: `YYYY-MM-DD_HH-MM-SS` ### `save` -The save command retreives video from local storage and, uploads the clip to the cloud. +The save command retreives video from local storage, concatenates and trims underlying storage segments based on time range, and uploads the clip to the cloud. + +| Attribute | Type | Required/Optional | Description | +|-------------|---------------------|-------------------|----------------------------------| +| `command` | string | required | Command to be executed. | +| `from` | timestamp | required | Start timestamp. | +| `to` | timestamp | required | End timestamp. | +| `metadata` | string | optional | Arbitrary metadata string. | +| `async` | boolean | optional | Whether the operation is async. | #### Save Request ```json { "command": "save", - "from": , [required] - "to": , [required] - "metadata": [optional] + "from": , + "to": , + "metadata": +} +``` + +#### Save Response +```json +{ + "command": "save", + "filename": } ``` + #### Async Save Request The async save command performs the same operation as the save command, but does not wait for the operation to complete. Use this command when you want to save video slices that include the current in-progress video storage segment. It will wait for the current segment to finish recording before saving the video slice. @@ -100,31 +139,39 @@ The async save command performs the same operation as the save command, but does ```json { "command": "save", - "from": , [required] - "to": , [required] - "metadata": , [optional] - "async": true [optional] + "from": , + "to": , + "metadata": , + "async": true } ``` -#### Save Response +#### Async Save Response ```json { "command": "save", - "filename": + "filename": , + "status": "async" } ``` + ### `fetch` -The fetch command retrieves video from local storage, and sends the bytes back to the client. +The fetch command retrieves video from local storage, and sends the bytes directly back to the client. + +| Attribute | Type | Required/Optional | Description | +|-----------|------------|-------------------|----------------------| +| `command` | string | required | Command to be executed. | +| `from` | timestamp | required | Start timestamp. | +| `to` | timestamp | required | End timestamp. | #### Fetch Request ```json { "command": "fetch", - "from": , [required] - "to": [required] + "from": , + "to": } ``` From 68b11381cce260594e8709e444a72c4800222425 Mon Sep 17 00:00:00 2001 From: seanavery Date: Mon, 7 Oct 2024 11:45:01 -0400 Subject: [PATCH 04/15] Add local development section --- README.md | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8715bd0..d7f93e8 100644 --- a/README.md +++ b/README.md @@ -181,4 +181,37 @@ The fetch command retrieves video from local storage, and sends the bytes direct "command": "fetch", "video": } -``` \ No newline at end of file +``` + +## Local Development + +### Building + +| Platform | Architecture | +|----------------|--------------| +| `linux` | `arm64` | +| `linux` | `amd64` | +| `darwin` | `arm64` | + +``` +canon -arch arm64 +make +``` + +### Testing + +``` +make test +``` + +### Linting + +``` +make lint +``` + +### Package module + +``` +make module +``` From 19bbbdaaf91bb0ff9038a9399e299498a59dd4ec Mon Sep 17 00:00:00 2001 From: seanavery Date: Mon, 7 Oct 2024 15:00:07 -0400 Subject: [PATCH 05/15] Mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a136143..1ce0066 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( go.viam.com/rdk v0.40.0 go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 go.viam.com/utils v0.1.98 - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc golang.org/x/mobile v0.0.0-20240112133503-c713f31d574b golang.org/x/tools v0.22.0 gotest.tools/gotestsum v1.10.0 @@ -307,6 +306,7 @@ require ( go.viam.com/api v0.1.336 // indirect goji.io v2.0.2+incompatible // indirect golang.org/x/crypto v0.24.0 // indirect + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/exp/typeparams v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/image v0.19.0 // indirect golang.org/x/mod v0.18.0 // indirect From f7bbcbe50eee2c108b8ec60876865759e42e12ff Mon Sep 17 00:00:00 2001 From: seanavery Date: Mon, 7 Oct 2024 15:03:14 -0400 Subject: [PATCH 06/15] Make module public --- meta.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.json b/meta.json index 94b7f55..11aa865 100644 --- a/meta.json +++ b/meta.json @@ -1,6 +1,6 @@ { "module_id": "viam:video-store", - "visibility": "private", + "visibility": "public", "url": "https://github.com/viam-modules/video-store", "description": "Golang module for video storage and retrieval from a camera", "models": [ From 6c9890a53605720bbbd5bfae3255d427b2a7ab2c Mon Sep 17 00:00:00 2001 From: seanavery Date: Mon, 7 Oct 2024 15:10:23 -0400 Subject: [PATCH 07/15] Add more detail local dev instructions --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index d7f93e8..aa79258 100644 --- a/README.md +++ b/README.md @@ -187,19 +187,35 @@ The fetch command retrieves video from local storage, and sends the bytes direct ### Building +The [Makefile](./Makefile) supports building for the following platforms: + | Platform | Architecture | |----------------|--------------| | `linux` | `arm64` | | `linux` | `amd64` | | `darwin` | `arm64` | +To build for linux/arm64, run the following commands: ``` canon -arch arm64 make ``` +To build for linux/amd64, run the following commands: +``` +canon -arch amd64 +make +``` + +To build for darwin/arm64, run the following command on a Mac with an M series chip: +``` +make +``` + ### Testing +Run [tests](./tests) to ensure the module is functioning as expected with the following command: + ``` make test ``` From 0a42047d4f6d356448ad7a9d78316421b864cdd3 Mon Sep 17 00:00:00 2001 From: seanavery Date: Mon, 7 Oct 2024 16:06:58 -0400 Subject: [PATCH 08/15] Make not on async save --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index aa79258..b64810f 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,10 @@ The save command retreives video from local storage, concatenates and trims unde The async save command performs the same operation as the save command, but does not wait for the operation to complete. Use this command when you want to save video slices that include the current in-progress video storage segment. It will wait for the current segment to finish recording before saving the video slice. +> [!NOTE] +> The async save command does not support future timestamps. The `from` timestamp must be in the past. +> and the `to` timestamp must be the current time or in the past. + ```json { "command": "save", From 11f4b21faec8638380fb89034560159474955916 Mon Sep 17 00:00:00 2001 From: seanavery Date: Mon, 7 Oct 2024 16:59:45 -0400 Subject: [PATCH 09/15] Fix spacing --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index b64810f..1943dc3 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,6 @@ The save command retreives video from local storage, concatenates and trims unde } ``` - #### Async Save Request The async save command performs the same operation as the save command, but does not wait for the operation to complete. Use this command when you want to save video slices that include the current in-progress video storage segment. It will wait for the current segment to finish recording before saving the video slice. @@ -159,7 +158,6 @@ The async save command performs the same operation as the save command, but does } ``` - ### `fetch` The fetch command retrieves video from local storage, and sends the bytes directly back to the client. From 1afcfd46c3a48eeda611db7c22587bb97d3cce84 Mon Sep 17 00:00:00 2001 From: seanavery Date: Mon, 7 Oct 2024 17:00:52 -0400 Subject: [PATCH 10/15] Format --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 1943dc3..fb306bb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ The `video-store` module brings security camera functionality to your smart mach > [!NOTE] > For more information, see [Configure a Machine](https://docs.viam.com/manage/configuration/). - ### Attributes | Attribute | Sub-Attribute | Type | Inclusion | Description | @@ -28,7 +27,7 @@ The `video-store` module brings security camera functionality to your smart mach | | `height` | integer | required | Height of the source camera frames in pixels. | | | `framerate` | integer | required | Number of frames per second provided by the source camera. | -### Example Configuration: +### Example Configuration ```json { From 16bd424ae1deeb151704b3de3ba5b4832ec1dd42 Mon Sep 17 00:00:00 2001 From: Sean Pollock Date: Tue, 8 Oct 2024 12:33:29 -0400 Subject: [PATCH 11/15] Update README.md Co-authored-by: Sierra Guequierre --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fb306bb..b6e39c3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# [`video-store` module](https://app.viam.com/module/viam/video-store) +# [`video-store` camera](https://app.viam.com/module/viam/video-store) + The `video-store` module brings security camera functionality to your smart machine! The module consumes a source [Camera](https://docs.viam.com/components/camera/) and saves the output as video files on disk. You can then upload video slices to the cloud using the [save](#save) command, or request the video bytes directly using the [fetch](#fetch) command. ## Configure your `video-store` component From 44d353942ad439fa7bb9720167501915c2e45d54 Mon Sep 17 00:00:00 2001 From: Sean Pollock Date: Tue, 8 Oct 2024 12:33:39 -0400 Subject: [PATCH 12/15] Update README.md Co-authored-by: Sierra Guequierre --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b6e39c3..9ffb755 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ The `video-store` module brings security camera functionality to your smart mach ### Configure a Data Manager Service -Make sure to configure a [Data Manager Service](https://docs.viam.com/services/data/cloud-sync/) to uplaod video files to the cloud when saving video slices. +Make sure to configure a [Data Manager Service](https://docs.viam.com/services/data/cloud-sync/) to upload video files to the cloud when saving video slices. > [!NOTE] > The `additional_sync_paths` attribute must include the custom path specified in the `upload_path` attribute of the `video-store` component if it is not under `~/.viam/capture`. From d0faa27b152cba4c4df48cb863e026d2e66fed0c Mon Sep 17 00:00:00 2001 From: Sean Pollock Date: Tue, 8 Oct 2024 12:34:28 -0400 Subject: [PATCH 13/15] Update README.md Co-authored-by: Sierra Guequierre --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9ffb755..274f721 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,25 @@ The `video-store` module brings security camera functionality to your smart mach ## Configure your `video-store` component -> [!NOTE] +Navigate to the [**CONFIGURE** tab](https://docs.viam.com/configure/) of your [machine](https://docs.viam.com/fleet/machines/) in [the Viam app](https://app.viam.com/). +[Add camera / video-store to your machine](https://docs.viam.com/configure/#components). + +On the new component panel, copy and paste the following attribute template into your camera’s attributes field: + +```json +{ + "camera": "" + "sync": "", + "storage": { + "segment_seconds": , + "size_gb": , + }, + "cam_props": { + "width": , + "height": , + "framerate": + } +} > For more information, see [Configure a Machine](https://docs.viam.com/manage/configuration/). ### Attributes From 1d8b0237715863a645aa700a45fa202bd77eb43a Mon Sep 17 00:00:00 2001 From: Sean Pollock Date: Tue, 8 Oct 2024 12:34:41 -0400 Subject: [PATCH 14/15] Update README.md Co-authored-by: Sierra Guequierre --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 274f721..2f7cc3f 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ The async save command performs the same operation as the save command, but does > [!NOTE] > The async save command does not support future timestamps. The `from` timestamp must be in the past. -> and the `to` timestamp must be the current time or in the past. +> The `to` timestamp must be the current time or in the past. ```json { From c6e8a0930216af1893e8093fc27c5cd4efa02dbb Mon Sep 17 00:00:00 2001 From: seanavery Date: Tue, 8 Oct 2024 12:43:08 -0400 Subject: [PATCH 15/15] Add requirements section --- README.md | 123 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 64 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 2f7cc3f..54e4487 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,34 @@ The `video-store` module brings security camera functionality to your smart machine! The module consumes a source [Camera](https://docs.viam.com/components/camera/) and saves the output as video files on disk. You can then upload video slices to the cloud using the [save](#save) command, or request the video bytes directly using the [fetch](#fetch) command. +## Requirements + +### Configure a Data Manager Service + +Make sure to configure a [Data Manager Service](https://docs.viam.com/services/data/cloud-sync/) to upload video files to the cloud when saving video slices. + +Navigate to the [**CONFIGURE** tab](https://docs.viam.com/configure/) of your [machine](https://docs.viam.com/fleet/machines/) in [the Viam app](https://app.viam.com/). [Add data-management to your machine](https://docs.viam.com/configure/#services). + +> [!NOTE] +> The `additional_sync_paths` attribute must include the custom path specified in the `upload_path` attribute of the `video-store` component if it is not under `~/.viam/capture`. + +```json + { + "name": , + "namespace": "rdk", + "type": "data_manager", + "attributes": { + "tags": [], + "additional_sync_paths": [ + + ], + "capture_disabled": true, + "sync_interval_mins": , + "capture_dir": "" + } + } +``` + ## Configure your `video-store` component Navigate to the [**CONFIGURE** tab](https://docs.viam.com/configure/) of your [machine](https://docs.viam.com/fleet/machines/) in [the Viam app](https://app.viam.com/). @@ -11,18 +39,19 @@ On the new component panel, copy and paste the following attribute template into ```json { - "camera": "" - "sync": "", - "storage": { - "segment_seconds": , - "size_gb": , - }, - "cam_props": { - "width": , - "height": , - "framerate": - } + "camera": "", + "sync": "", + "storage": { + "segment_seconds": , + "size_gb": , + }, + "cam_props": { + "width": , + "height": , + "framerate": + } } +``` > For more information, see [Configure a Machine](https://docs.viam.com/manage/configuration/). ### Attributes @@ -49,53 +78,29 @@ On the new component panel, copy and paste the following attribute template into ### Example Configuration ```json - { - "name": "video-store", - "namespace": "rdk", - "type": "camera", - "model": "viam:video:storage", - "attributes": { - "camera": "wc-cam" - "sync": "data-manager", - "storage": { - "segment_seconds": 10, - "size_gb": 50, - }, - "cam_props": { - "width": 640, - "height": 480, - "framerate": 25 - } - }, - "depends_on": [ - "wc-cam", - "data-manager" - ] - } -``` - -### Configure a Data Manager Service - -Make sure to configure a [Data Manager Service](https://docs.viam.com/services/data/cloud-sync/) to upload video files to the cloud when saving video slices. - -> [!NOTE] -> The `additional_sync_paths` attribute must include the custom path specified in the `upload_path` attribute of the `video-store` component if it is not under `~/.viam/capture`. - -```json - { - "name": , - "namespace": "rdk", - "type": "data_manager", - "attributes": { - "tags": [], - "additional_sync_paths": [ - - ], - "capture_disabled": true, - "sync_interval_mins": , - "capture_dir": "" - } +{ + "name": "video-store", + "namespace": "rdk", + "type": "camera", + "model": "viam:video:storage", + "attributes": { + "camera": "wc-cam" + "sync": "data-manager", + "storage": { + "segment_seconds": 10, + "size_gb": 50, + }, + "cam_props": { + "width": 640, + "height": 480, + "framerate": 25 } + }, + "depends_on": [ + "wc-cam", + "data-manager" + ] +} ``` ## DoCommand API @@ -119,7 +124,7 @@ The datetime format used is: `YYYY-MM-DD_HH-MM-SS` - `2024-01-15_14-30-45` represents January 15, 2024, at 2:30:45 PM. -### `save` +### `Save` The save command retreives video from local storage, concatenates and trims underlying storage segments based on time range, and uploads the clip to the cloud. @@ -176,7 +181,7 @@ The async save command performs the same operation as the save command, but does } ``` -### `fetch` +### `Fetch` The fetch command retrieves video from local storage, and sends the bytes directly back to the client.