[init] initalize main branch

.github/workflows/test.yml

@@ -0,0 +1,38 @@
+name: Core Tests
+
+on:
+  push:
+    branches: [ main, init_dev ]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.8" ]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          git submodule update --init
+          python -m pip install --upgrade pip
+          pip install --upgrade -e .[dev]
+      - name: Run Black
+        run: |
+          black dex_retargeting/ tests/ --check
+      #      - name: Run Pyright
+      #        run: |
+      #          pyright
+      - name: Test with pytest
+        run: |
+          pytest

.gitignore

@@ -0,0 +1,17 @@
+build
+.vscode
+.pyc
+*.blend
+cmake-build-debug
+/cmake-build-debug/
+/.ccls-cache/
+/.dir-locals.el
+__pycache__
+.ini
+*.convex.*
+imgui.ini
+.mypy_cache
+.DS_Store
+/.idea
+/log
+/example/data/output_*.mp4

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "assets"]
+	path = assets
+	url = https://github.com/dexsuite/dex-urdf.git
+	branch = init_dev

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Yuzhe Qin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,70 @@
+Dex Retargeting
+---
+<p align="center">
+    <!-- code check badges -->
+    <a href='https://github.com/dexsuite/dex-retargeting/blob/main/.github/workflows/test.yml'>
+        <img src='https://github.com/dexsuite/dex-retargeting/actions/workflows/test.yml/badge.svg' alt='Test Status' />
+    </a>
+    <!-- license badge -->
+    <a href="https://github.com/dexsuite/dex-retargeting/blob/main/LICENSE">
+        <img alt="License" src="https://img.shields.io/badge/license-MIT-blue">
+    </a>
+</p>
+
+## Installation
+
+```shell
+pip3 install -e ".[example]"
+# If you do not need to run the examples:
+# pip install -e .
+
+```
+
+## Examples
+
+### Retargeting from human video
+
+1. **Generate the robot joint pose trajectory from our pre-recorded video.**
+
+```shell
+export PYTHONPATH=$PYTHONPATH:`pwd`
+python3 example/detect_from_video.py \
+  --robot-name allegro \
+  --video-path example/data/human_hand_video.mp4 \
+  --retargeting-type vector \
+  --hand-type right \
+  --output-path example/data/allegro_joints.pkl 
+```
+
+This command will output the joint trajectory as a pickle file at the `output_path`.
+
+The pickle file is a python dictionary with two keys: `meta_data` and `data`. `meta_data`, a dictionary, includes
+details about the robot, while `data`, a list, contains the robotic joint positions for each frame. For additional
+options, refer to the help information. Note that the time cost here includes both the hand pose detection from video,
+and the hand pose retargeting in single process mode.
+
+```shell
+python3 example/detect_from_video.py --help
+```
+
+2. **Utilize the pickle file to produce a video of the robot**
+
+```shell
+export PYTHONPATH=$PYTHONPATH:`pwd`
+python3 example/render_robot_hand.py \
+  --pickle-path example/data/allegro_joints.pkl \
+  --output-video-path example/data/retargeted_allegro.mp4 \
+  --headless
+```
+
+This command uses the data saved from the previous step to create a rendered video.
+
+3. **Record a video of your own hand**
+
+```bash
+export PYTHONPATH=$PYTHONPATH:`pwd`
+python3 example/capture_webcam.py --video-path example/data/my_human_hand_video.mp4
+```
+
+This command will access your webcam (which should be connected to your computer) and record the video stream in mp4
+format. To end video recording, press `q` on the keyboard.

dex_retargeting/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

dex_retargeting/configs/offline/allegro_hand_right.yml

@@ -0,0 +1,12 @@
+retargeting:
+  type: position
+  urdf_path: allegro_hand/allegro_hand_right.urdf
+  use_camera_frame_retargeting: False
+
+  target_joint_names: null
+  target_link_names: [ "link_15.0_tip", "link_3.0_tip", "link_7.0_tip", "link_11.0_tip" ]
+
+  target_link_human_indices: [ 4, 8, 12, 16 ]
+
+  # A smaller alpha means stronger filtering, i.e. more smooth but also larger latency
+  low_pass_alpha: 0.2

dex_retargeting/configs/teleop/allegro_hand_left.yml

@@ -0,0 +1,17 @@
+retargeting:
+  type: vector
+  urdf_path: allegro_hand/allegro_hand_left.urdf
+  use_camera_frame_retargeting: False
+
+  # Target refers to the retargeting target, which is the robot hand
+  target_joint_names: null
+  target_origin_link_names: [ "wrist", "wrist", "wrist", "wrist" ]
+  target_task_link_names: [ "link_15.0_tip", "link_3.0_tip", "link_7.0_tip", "link_11.0_tip" ]
+  scaling_factor: 1.6
+
+  # Source refers to the retargeting input, which usually corresponds to the human hand
+  # The joint indices of human hand joint which corresponds to each link in the target_link_names
+  target_link_human_indices: [ [ 0, 0, 0, 0 ], [ 4, 8, 12, 16 ] ]
+
+  # A smaller alpha means stronger filtering, i.e. more smooth but also larger latency
+  low_pass_alpha: 0.2

dex_retargeting/configs/teleop/allegro_hand_left_dexpilot.yml

@@ -0,0 +1,13 @@
+retargeting:
+  type: DexPilot
+  urdf_path: allegro_hand/allegro_hand_left.urdf
+  use_camera_frame_retargeting: False
+
+  # Target refers to the retargeting target, which is the robot hand
+  target_joint_names: null
+  wrist_link_name: "wrist"
+  finger_tip_link_names: [ "link_15.0_tip", "link_3.0_tip", "link_7.0_tip", "link_11.0_tip" ]
+  scaling_factor: 1.6
+
+  # A smaller alpha means stronger filtering, i.e. more smooth but also larger latency
+  low_pass_alpha: 0.2

dex_retargeting/configs/teleop/allegro_hand_right.yml

@@ -0,0 +1,17 @@
+retargeting:
+  type: vector
+  urdf_path: allegro_hand/allegro_hand_right.urdf
+  use_camera_frame_retargeting: False
+
+  # Target refers to the retargeting target, which is the robot hand
+  target_joint_names: null
+  target_origin_link_names: [ "wrist", "wrist", "wrist", "wrist" ]
+  target_task_link_names: [ "link_15.0_tip", "link_3.0_tip", "link_7.0_tip", "link_11.0_tip" ]
+  scaling_factor: 1.6
+
+  # Source refers to the retargeting input, which usually corresponds to the human hand
+  # The joint indices of human hand joint which corresponds to each link in the target_link_names
+  target_link_human_indices: [ [ 0, 0, 0, 0 ], [ 4, 8, 12, 16 ] ]
+
+  # A smaller alpha means stronger filtering, i.e. more smooth but also larger latency
+  low_pass_alpha: 0.2

dex_retargeting/configs/teleop/allegro_hand_right_dexpilot.yml

@@ -0,0 +1,13 @@
+retargeting:
+  type: DexPilot
+  urdf_path: allegro_hand/allegro_hand_right.urdf
+  use_camera_frame_retargeting: False
+
+  # Target refers to the retargeting target, which is the robot hand
+  target_joint_names: null
+  wrist_link_name: "wrist"
+  finger_tip_link_names: [ "link_15.0_tip", "link_3.0_tip", "link_7.0_tip", "link_11.0_tip" ]
+  scaling_factor: 1.6
+
+  # A smaller alpha means stronger filtering, i.e. more smooth but also larger latency
+  low_pass_alpha: 0.2

dex_retargeting/configs/teleop/schunk_svh_hand_right.yml

@@ -0,0 +1,17 @@
+retargeting:
+  type: vector
+  urdf_path: schunk_hand/schunk_svh_hand_right.urdf
+  use_camera_frame_retargeting: False
+
+  # Target refers to the retargeting target, which is the robot hand
+  target_joint_names: null
+  target_origin_link_names: [ "right_hand_base_link","right_hand_base_link", "right_hand_base_link", "right_hand_base_link", "right_hand_base_link", ]
+  target_task_link_names: [ "right_hand_c", "right_hand_t", "right_hand_s", "right_hand_r", "right_hand_q" ]
+  scaling_factor: 1.2
+
+  # Source refers to the retargeting input, which usually corresponds to the human hand
+  # The joint indices of human hand joint which corresponds to each link in the target_link_names
+  target_link_human_indices: [ [ 0, 0, 0, 0, 0 ], [ 4, 8, 12, 16, 20, ] ]
+
+  # A smaller alpha means stronger filtering, i.e. more smooth but also larger latency
+  low_pass_alpha: 0.2

dex_retargeting/configs/teleop/schunk_svh_hand_right_dexpilot.yml

@@ -0,0 +1,17 @@
+retargeting:
+  type: DexPilot
+  urdf_path: schunk_hand/schunk_svh_hand_right.urdf
+  use_camera_frame_retargeting: False
+
+  # Target refers to the retargeting target, which is the robot hand
+  target_joint_names: null
+  wrist_link_name: "right_hand_base_link"
+  finger_tip_link_names: [ "right_hand_c", "right_hand_t", "right_hand_s", "right_hand_r", "right_hand_q" ]
+  scaling_factor: 1.2
+
+  # Source refers to the retargeting input, which usually corresponds to the human hand
+  # The joint indices of human hand joint which corresponds to each link in the target_link_names
+  target_link_human_indices: [ [ 0, 0, 0, 0, 0 ], [ 4, 8, 12, 16, 20, ] ]
+
+  # A smaller alpha means stronger filtering, i.e. more smooth but also larger latency
+  low_pass_alpha: 0.2

dex_retargeting/configs/teleop/shadow_hand_right.yml

@@ -0,0 +1,17 @@
+retargeting:
+  type: vector
+  urdf_path: shadow_hand/shadow_hand_right.urdf
+  use_camera_frame_retargeting: False
+
+  # Target refers to the retargeting target, which is the robot hand
+  target_joint_names: null
+  target_origin_link_names: [ "palm", "palm", "palm", "palm", "palm", "palm", "palm", "palm", "palm", "palm" ]
+  target_task_link_names: [ "thtip", "fftip", "mftip", "rftip", "lftip",  "thmiddle", "ffmiddle", "mfmiddle", "rfmiddle", "lfmiddle" ]
+  scaling_factor: 1.2
+
+  # Source refers to the retargeting input, which usually corresponds to the human hand
+  # The joint indices of human hand joint which corresponds to each link in the target_link_names
+  target_link_human_indices: [ [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 4, 8, 12, 16, 20, 2, 6, 10, 14, 18 ] ]
+
+  # A smaller alpha means stronger filtering, i.e. more smooth but also larger latency
+  low_pass_alpha: 0.2

dex_retargeting/configs/teleop/shadow_hand_right_dexpilot.yml

@@ -0,0 +1,13 @@
+retargeting:
+  type: DexPilot
+  urdf_path: shadow_hand/shadow_hand_right.urdf
+  use_camera_frame_retargeting: False
+
+  # Target refers to the retargeting target, which is the robot hand
+  target_joint_names: null
+  wrist_link_name: "palm"
+  finger_tip_link_names: [ "thtip", "fftip", "mftip", "rftip", "lftip" ]
+  scaling_factor: 1.2
+
+  # A smaller alpha means stronger filtering, i.e. more smooth but also larger latency
+  low_pass_alpha: 0.2

dex_retargeting/constants.py

@@ -0,0 +1,44 @@
+import enum
+from pathlib import Path
+
+
+class RobotName(enum.Enum):
+    allegro = enum.auto()
+    shadow = enum.auto()
+    svh = enum.auto()
+
+
+class RetargetingType(enum.Enum):
+    vector = enum.auto()  # For teleoperation, no finger closing prior
+    position = enum.auto()  # For offline data processing, especially hand-object interaction data
+    dexpilot = enum.auto()  # For teleoperation, with finger closing prior
+
+
+class HandType(enum.Enum):
+    right = enum.auto()
+    left = enum.auto()
+
+
+ROBOT_NAME_MAP = {
+    RobotName.allegro: "allegro_hand",
+    RobotName.shadow: "shadow_hand",
+    RobotName.svh: "schunk_svh_hand",
+}
+
+ROBOT_NAMES = list(ROBOT_NAME_MAP.keys())
+
+
+def get_config_path(robot_name: RobotName, retargeting_type: RetargetingType, hand_type: HandType) -> Path:
+    config_path = Path(__file__).parent / "configs"
+    if retargeting_type is RetargetingType.position:
+        config_path = config_path / "offline"
+    else:
+        config_path = config_path / "teleop"
+
+    robot_name_str = ROBOT_NAME_MAP[robot_name]
+    hand_type_str = hand_type.name
+    if retargeting_type == RetargetingType.dexpilot:
+        config_name = f"{robot_name_str}_{hand_type_str}_dexpilot.yml"
+    else:
+        config_name = f"{robot_name_str}_{hand_type_str}.yml"
+    return config_path / config_name