Test TRELLIS models, still waiting for spconv to fix its error for py…

…thon3.12 cu124

traveller59/spconv#731

.gitignore

@@ -157,6 +157,7 @@ cython_debug/
 *.pth
 *.safetensors
 *.pkl
+*.pt
 .huggingface
 
 # Ignore test outputs

Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/README.md

@@ -0,0 +1,16 @@
+---
+library_name: trellis
+pipeline_tag: image-to-3d
+license: mit
+language:
+- en
+---
+# TRELLIS Image Large
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+The image conditioned version of TRELLIS, a large 3D genetive model. It was introduced in the paper [Structured 3D Latents for Scalable and Versatile 3D Generation](https://huggingface.co/papers/2412.01506).
+
+Project page: https://trellis3d.github.io/
+
+Code: https://github.com/Microsoft/TRELLIS

Checkpoints/Diffusers/tencent/Hunyuan3D-1/README.md

@@ -6,6 +6,10 @@ license_link: https://huggingface.co/tencent/Hunyuan3D-1/blob/main/LICENSE.txt
 language:
 - en
 - zh
+tags:
+- image-to-3d
+- text-to-3d
+pipeline_tag: text-to-3d
 ---
 <!-- ## **Hunyuan3D-1.0** -->
 
@@ -237,4 +241,4 @@ If you found this repository helpful, please cite our report:
     archivePrefix={arXiv},
     primaryClass={cs.CV}
 }
-```
+```

Gen_3D_Modules/CharacterGen/Stage_3D/lrm/utils/ops.py

@@ -6,7 +6,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 from torch.autograd import Function
-from torch.cuda.amp import custom_bwd, custom_fwd
+from torch.amp import custom_bwd, custom_fwd
 
 from CharacterGen.Stage_3D import lrm
 from .typing import *
@@ -41,13 +41,13 @@ class _TruncExp(Function):  # pylint: disable=abstract-method
     # Implementation from torch-ngp:
     # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py
     @staticmethod
-    @custom_fwd(cast_inputs=torch.float32)
+    @custom_fwd(cast_inputs=torch.float32, device_type="cuda")
     def forward(ctx, x):  # pylint: disable=arguments-differ
         ctx.save_for_backward(x)
         return torch.exp(x)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, g):  # pylint: disable=arguments-differ
         x = ctx.saved_tensors[0]
         return g * torch.exp(torch.clamp(x, max=15))

Gen_3D_Modules/Hunyuan3D_V1/svrm/ldm/utils/ops.py

@@ -7,7 +7,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 from torch.autograd import Function
-from torch.cuda.amp import custom_bwd, custom_fwd
+from torch.amp import custom_bwd, custom_fwd
 from igl import fast_winding_number_for_meshes, point_mesh_squared_distance, read_obj
 
 from .typing import *
@@ -52,13 +52,13 @@ class _TruncExp(Function):  # pylint: disable=abstract-method
     # Implementation from torch-ngp:
     # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py
     @staticmethod
-    @custom_fwd(cast_inputs=torch.float32)
+    @custom_fwd(cast_inputs=torch.float32, device_type="cuda")
     def forward(ctx, x):  # pylint: disable=arguments-differ
         ctx.save_for_backward(x)
         return torch.exp(x)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, g):  # pylint: disable=arguments-differ
         x = ctx.saved_tensors[0]
         return g * torch.exp(torch.clamp(x, max=15))
@@ -68,14 +68,14 @@ class SpecifyGradient(Function):
     # Implementation from stable-dreamfusion
     # https://github.com/ashawkey/stable-dreamfusion
     @staticmethod
-    @custom_fwd
+    @custom_fwd(device_type="cuda")
     def forward(ctx, input_tensor, gt_grad):
         ctx.save_for_backward(gt_grad)
         # we return a dummy value 1, which will be scaled by amp's scaler so we get the scale in backward.
         return torch.ones([1], device=input_tensor.device, dtype=input_tensor.dtype)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, grad_scale):
         (gt_grad,) = ctx.saved_tensors
         gt_grad = gt_grad * grad_scale

Gen_3D_Modules/StableFast3D/sf3d/models/network.py

@@ -8,7 +8,7 @@
 from jaxtyping import Float
 from torch import Tensor
 from torch.autograd import Function
-from torch.cuda.amp import custom_bwd, custom_fwd
+from torch.amp import custom_bwd, custom_fwd
 
 from StableFast3D.sf3d.models.utils import BaseModule, normalize
 
@@ -65,13 +65,13 @@ class _TruncExp(Function):  # pylint: disable=abstract-method
     # Implementation from torch-ngp:
     # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py
     @staticmethod
-    @custom_fwd(cast_inputs=torch.float32)
+    @custom_fwd(cast_inputs=torch.float32, device_type="cuda")
     def forward(ctx, x):  # pylint: disable=arguments-differ
         ctx.save_for_backward(x)
         return torch.exp(x)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, g):  # pylint: disable=arguments-differ
         x = ctx.saved_tensors[0]
         return g * torch.exp(torch.clamp(x, max=15))

Gen_3D_Modules/TriplaneGaussian/utils/ops.py

@@ -4,7 +4,7 @@
 import torch
 import torch.nn.functional as F
 from torch.autograd import Function
-from torch.cuda.amp import custom_bwd, custom_fwd
+from torch.amp import custom_bwd, custom_fwd
 from pytorch3d import io
 from pytorch3d.renderer import (
     PointsRasterizationSettings, 
@@ -35,13 +35,13 @@ class _TruncExp(Function):  # pylint: disable=abstract-method
     # Implementation from torch-ngp:
     # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py
     @staticmethod
-    @custom_fwd(cast_inputs=torch.float32)
+    @custom_fwd(cast_inputs=torch.float32, device_type="cuda")
     def forward(ctx, x):  # pylint: disable=arguments-differ
         ctx.save_for_backward(x)
         return torch.exp(x)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, g):  # pylint: disable=arguments-differ
         x = ctx.saved_tensors[0]
         return g * torch.exp(torch.clamp(x, max=15))

Gen_3D_Modules/craftsman/utils/checkpoint.py

@@ -36,7 +36,7 @@ def checkpoint(
 
 class CheckpointFunction(torch.autograd.Function):
     @staticmethod
-    @torch.cuda.amp.custom_fwd
+    @torch.amp.custom_fwd(device_type="cuda")
     def forward(ctx, run_function, length, *args):
         ctx.run_function = run_function
         ctx.input_tensors = list(args[:length])
@@ -47,7 +47,7 @@ def forward(ctx, run_function, length, *args):
         return output_tensors
 
     @staticmethod
-    @torch.cuda.amp.custom_bwd
+    @torch.amp.custom_bwd(device_type="cuda")
     def backward(ctx, *output_grads):
         ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
         with torch.enable_grad():

README.md

@@ -252,6 +252,6 @@ This is an extensive node suite that enables ComfyUI to process 3D inputs (Mesh
   ```
 * If you encounter OpenGL errors (e.g., `[F glutil.cpp:338] eglInitialize() failed`), then set `force_cuda_rasterize` to true on corresponding node
 * If after the installation, your ComfyUI gets stuck at starting or running, you can follow the instructions in the following link to solve the problem: [Code Hangs Indefinitely When Evaluating Neuron Models on GPU](https://github.com/lava-nc/lava-dl/discussions/211)
-
+* [Fix "nvcc fatal : Could not set up the environment for Microsoft Visual Studio using 'C:/Program Files (x86)/Microsoft V isual Studio/.../vcvars64.bat'](https://github.com/NVlabs/instant-ngp/issues/709#issuecomment-1378155274)
 ## Supporters
 - [MrNeRF](https://twitter.com/janusch_patas)

_Pre_Builds/_Build_Scripts/build_config.yaml

@@ -13,7 +13,7 @@ libs_dir_name: "_Libs"
 # Folder used to store built wheels
 wheels_dir_name: "_Build_Wheels"
 
-build_base_packages: ["torch", "torchvision"]
+build_base_packages: ["torch", "torchvision", "xformers", "spconv-${cuda_version}"]
 # All packages will be automatically installed at beginning of install.py follwing orders from top to bottom 
 remote_packages:
   torch:

_Pre_Builds/_Build_Scripts/build_utils.py

@@ -140,12 +140,17 @@ def git_folder_parallel(repo_id: str, folder: str, recursive: bool=True, root_ou
 
 def install_remote_packages(package_names):
     for package_name in package_names:
-        package_attr = build_config.remote_packages[package_name]
-        if hasattr(package_attr, "version"):
-            package_name += f"=={package_attr.version}"
-        url_option = package_attr.url_option if hasattr(package_attr, "url_option") else "--index-url"
-
-        subprocess.run([
-            PYTHON_PATH, "-s", "-m", "pip", "install", 
-            package_name, url_option, package_attr.url
-        ])
+        if package_name in build_config.remote_packages:
+            package_attr = build_config.remote_packages[package_name]
+            if hasattr(package_attr, "version"):
+                package_name += f"=={package_attr.version}"
+            if hasattr(package_attr, "url"):
+                url_option = package_attr.url_option if hasattr(package_attr, "url_option") else "--index-url"
+
+                subprocess.run([
+                    PYTHON_PATH, "-s", "-m", "pip", "install", 
+                    package_name, url_option, package_attr.url
+                ])
+                continue
+
+        subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", package_name])

_Pre_Builds/_Build_Scripts/dependencies.txt

@@ -2,4 +2,5 @@ https://github.com/ashawkey/diff-gaussian-rasterization.git
 https://github.com/NVlabs/nvdiffrast.git
 https://github.com/ashawkey/kiuikit.git
 https://github.com/facebookresearch/pytorch3d.git
-https://github.com/rusty1s/pytorch_scatter.git
+https://github.com/rusty1s/pytorch_scatter.git
+https://github.com/EasternJournalist/utils3d.git

install.py

@@ -47,7 +47,7 @@ def install_local_wheels(builds_dir):
             cstr(f"pip install {wheel_path} to {PYTHON_PATH}").msg.print()
 
     # Install packages that needs specify remote url
-    install_remote_packages(build_config.remote_packages.keys())
+    install_remote_packages(build_config.build_base_packages)
 
     # Get the target remote pre-built wheels directory name and path
     platform_config_name = get_platform_config_name()