diff --git a/analyze_results.py b/analyze_results.py
new file mode 100644
index 0000000000..ba8e9121f2
--- /dev/null
+++ b/analyze_results.py
@@ -0,0 +1,65 @@
+
+
+textfile_name = "log.log"
+file1 = open(textfile_name, 'r')
+lines = file1.readlines()
+baseline_technique_number = 3
+
+results = {}
+models = []
+technique="err"
+cur_model_name = "err"
+prev_line = "err"
+for line in lines:
+    if "start" in line and line[:6] == "start ":
+        start = line.find("start ")
+        technique = line[start+6:-1]
+        results[technique]={}
+        cur_model_name = "err"
+    elif "cuda eval" in line:
+        assert technique!="err"
+        end = line[11:].find(" ")
+        cur_model_name = line[11:11+end]
+        if cur_model_name not in models:
+            models.append(cur_model_name)
+    elif "running benchmark: 100%" in prev_line:
+        try:
+            result = float(line[0:line.find("x")])
+            assert cur_model_name!="err"
+            results[technique][cur_model_name]=result
+            cur_model_name = "err"
+        except:
+            if not "running benchmark" in line:
+                print("err parsing line: \n", line)
+    prev_line = line
+
+techniques = [x for x in results.keys()]
+baseline_technique = techniques[baseline_technique_number]
+
+max_model_len = max([len(x) for x in models])+1
+max_technique_len = max([len(x) for x in techniques])+1
+num_techniques = len(techniques)
+print(max_model_len, max_technique_len)
+
+print("|"+"-"*max_model_len+"|"+("-"*max_model_len+"|")*num_techniques)
+print(f"|{f'model'.ljust(max_model_len)}|{''.join([f'{x.ljust(max_model_len)}|' for x in techniques])}")
+print("|"+"-"*max_model_len+"|"+("-"*max_model_len+"|")*num_techniques)
+working = {}
+faster = {}
+for model in models:
+    out = f"|{model.ljust(max_model_len)}|"
+    for technique in techniques:
+        try:
+            result = f"{results[technique][model]/results[baseline_technique][model]:0.4}"
+        except:
+            result = "err"
+
+        working[technique]=working.get(technique, 0)+(result!="err")
+        faster[technique]=faster.get(technique, 0)+(result!="err" and float(result)>=1.0)
+        out+=f"{result.ljust(max_model_len)}|"
+    print(out)
+print("|"+"-"*max_model_len+"|"+("-"*max_model_len+"|")*num_techniques)
+print(f"|{f'TOTAL COVERAGE'.ljust(max_model_len)}|{''.join([f'{working[x]/working[baseline_technique]*100:0.5}%'.ljust(max_model_len)+'|' for x in techniques])}")
+print(f"|{f'TOTAL FASTER'.ljust(max_model_len)}|{''.join([f'{faster[x]/faster[baseline_technique]*100:0.5}%'.ljust(max_model_len)+'|' for x in techniques])}")
+
+print("|"+"-"*max_model_len+"|"+("-"*max_model_len+"|")*num_techniques)
diff --git a/log.log b/log.log
new file mode 100644
index 0000000000..27cf21f27a
--- /dev/null
+++ b/log.log
@@ -0,0 +1,17016 @@
+start dynamic
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/fbgemm_gpu_py.so: undefined symbol: _ZNK5torch8autograd4Node4nameEv
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/canary_models/torchrec_dlrm/__init__.py", line 7, in <module>
+    from .data.dlrm_dataloader import get_dataloader
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/canary_models/torchrec_dlrm/data/dlrm_dataloader.py", line 13, in <module>
+    from torchrec.datasets.criteo import (
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/__init__.py", line 8, in <module>
+    import torchrec.distributed  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/__init__.py", line 36, in <module>
+    from torchrec.distributed.model_parallel import DistributedModelParallel  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/model_parallel.py", line 21, in <module>
+    from torchrec.distributed.planner import EmbeddingShardingPlanner, Topology
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/__init__.py", line 22, in <module>
+    from torchrec.distributed.planner.planners import EmbeddingShardingPlanner  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/planners.py", line 19, in <module>
+    from torchrec.distributed.planner.constants import BATCH_SIZE, MAX_SIZE
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/constants.py", line 10, in <module>
+    from torchrec.distributed.embedding_types import EmbeddingComputeKernel
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/embedding_types.py", line 14, in <module>
+    from fbgemm_gpu.split_table_batched_embeddings_ops_training import EmbeddingLocation
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/__init__.py", line 22, in <module>
+    from . import _fbgemm_gpu_docs  # noqa: F401, E402
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/_fbgemm_gpu_docs.py", line 19, in <module>
+    torch.ops.fbgemm.jagged_2d_to_dense,
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 820, in __getattr__
+    raise AttributeError(
+AttributeError: '_OpNamespace' 'fbgemm' object has no attribute 'jagged_2d_to_dense'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  BERT_pytorch                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:04,  6.46it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:04,  6.87it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:03,  7.01it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:03,  7.04it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:03,  7.04it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:03,  7.09it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:03,  7.13it/s]
running benchmark:  27%|██▋       | 8/30 [00:01<00:03,  7.15it/s]
running benchmark:  30%|███       | 9/30 [00:01<00:02,  7.15it/s]
running benchmark:  33%|███▎      | 10/30 [00:01<00:02,  7.17it/s]
running benchmark:  37%|███▋      | 11/30 [00:01<00:02,  7.11it/s]
running benchmark:  40%|████      | 12/30 [00:01<00:02,  7.09it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:02,  7.14it/s]
running benchmark:  47%|████▋     | 14/30 [00:01<00:02,  7.14it/s]
running benchmark:  50%|█████     | 15/30 [00:02<00:02,  7.12it/s]
running benchmark:  53%|█████▎    | 16/30 [00:02<00:01,  7.14it/s]
running benchmark:  57%|█████▋    | 17/30 [00:02<00:01,  7.17it/s]
running benchmark:  60%|██████    | 18/30 [00:02<00:01,  7.14it/s]
running benchmark:  63%|██████▎   | 19/30 [00:02<00:01,  7.17it/s]
running benchmark:  67%|██████▋   | 20/30 [00:02<00:01,  7.18it/s]
running benchmark:  70%|███████   | 21/30 [00:02<00:01,  7.19it/s]
running benchmark:  73%|███████▎  | 22/30 [00:03<00:01,  7.17it/s]
running benchmark:  77%|███████▋  | 23/30 [00:03<00:00,  7.16it/s]
running benchmark:  80%|████████  | 24/30 [00:03<00:00,  7.19it/s]
running benchmark:  83%|████████▎ | 25/30 [00:03<00:00,  7.21it/s]
running benchmark:  87%|████████▋ | 26/30 [00:03<00:00,  7.18it/s]
running benchmark:  90%|█████████ | 27/30 [00:03<00:00,  7.18it/s]
running benchmark:  93%|█████████▎| 28/30 [00:03<00:00,  7.18it/s]
running benchmark:  97%|█████████▋| 29/30 [00:04<00:00,  7.14it/s]
running benchmark: 100%|██████████| 30/30 [00:04<00:00,  7.04it/s]
running benchmark: 100%|██████████| 30/30 [00:04<00:00,  7.12it/s]
+30.933x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  Background_Matting                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:00, 29.44it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 37.76it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 40.22it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 41.35it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 41.93it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 42.19it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 41.01it/s]
+2.091x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model DALLE2_pytorch supports float32 only
+
loading model: 0it [00:11, ?it/s]
+WARNING:common:Model DALLE2_pytorch supports float32 only
+cuda eval  DALLE2_pytorch                     
+WARNING:common:Model DALLE2_pytorch supports float32 only
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/utils/_contextlib.py", line 115, in decorate_context
+    return func(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 95, in inner
+    model.eval()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 96, in resume_in_inner
+    out = fn(model, *args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 3322, in forward
+    device = module_device(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 81, in module_device
+    return next(module.parameters()).device
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+cuda eval  LearningToPaint                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 118.96it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 122.04it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 121.98it/s]
+3.917x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model Super_SloMo supports float32 only
+
loading model: 0it [00:04, ?it/s]
+WARNING:common:Model Super_SloMo supports float32 only
+cuda eval  Super_SloMo                        
+WARNING:common:Model Super_SloMo supports float32 only
+AUTOTUNE convolution(6x6x352x352, 32x6x7x7)
+  convolution 0.4805 ms 100.0%
+  triton_convolution_3 1.4949 ms 32.1%
+  triton_convolution_1 1.5015 ms 32.0%
+  triton_convolution_0 1.5742 ms 30.5%
+  triton_convolution_5 1.6843 ms 28.5%
+  triton_convolution_4 1.7921 ms 26.8%
+  triton_convolution_2 2.7756 ms 17.3%
+SingleProcess AUTOTUNE takes 3.6696 seconds
+AUTOTUNE convolution(6x32x352x352, 32x32x7x7)
+  convolution 1.6498 ms 100.0%
+  triton_convolution_9 2.3422 ms 70.4%
+  triton_convolution_12 2.3666 ms 69.7%
+  triton_convolution_10 2.4241 ms 68.1%
+  triton_convolution_6 2.7273 ms 60.5%
+  triton_convolution_11 2.8173 ms 58.6%
+  triton_convolution_7 3.2051 ms 51.5%
+  triton_convolution_8 5.7945 ms 28.5%
+SingleProcess AUTOTUNE takes 4.2051 seconds
+AUTOTUNE convolution(6x32x176x176, 64x32x5x5)
+  convolution 0.2524 ms 100.0%
+  triton_convolution_13 0.5010 ms 50.4%
+  triton_convolution_18 0.5207 ms 48.5%
+  triton_convolution_16 0.5513 ms 45.8%
+  triton_convolution_17 0.5739 ms 44.0%
+  triton_convolution_19 0.6147 ms 41.1%
+  triton_convolution_14 0.8714 ms 29.0%
+  triton_convolution_15 1.5069 ms 16.8%
+SingleProcess AUTOTUNE takes 5.6923 seconds
+AUTOTUNE convolution(6x64x176x176, 64x64x5x5)
+  convolution 0.4746 ms 100.0%
+  triton_convolution_25 1.0087 ms 47.1%
+  triton_convolution_20 1.0101 ms 47.0%
+  triton_convolution_23 1.0689 ms 44.4%
+  triton_convolution_24 1.1122 ms 42.7%
+  triton_convolution_26 1.1992 ms 39.6%
+  triton_convolution_21 1.8507 ms 25.6%
+  triton_convolution_22 3.0241 ms 15.7%
+SingleProcess AUTOTUNE takes 5.6536 seconds
+AUTOTUNE convolution(6x64x88x88, 128x64x3x3)
+  convolution 0.1009 ms 100.0%
+  triton_convolution_27 0.1985 ms 50.8%
+  triton_convolution_30 0.2011 ms 50.2%
+  triton_convolution_32 0.2019 ms 50.0%
+  triton_convolution_31 0.2314 ms 43.6%
+  triton_convolution_33 0.2748 ms 36.7%
+  triton_convolution_28 0.3577 ms 28.2%
+  triton_convolution_29 0.6250 ms 16.1%
+SingleProcess AUTOTUNE takes 6.1096 seconds
+AUTOTUNE convolution(6x128x88x88, 128x128x3x3)
+  convolution 0.1734 ms 100.0%
+  triton_convolution_39 0.3771 ms 46.0%
+  triton_convolution_37 0.3884 ms 44.7%
+  triton_convolution_34 0.3900 ms 44.5%
+  triton_convolution_38 0.4460 ms 38.9%
+  triton_convolution_40 0.5328 ms 32.6%
+  triton_convolution_35 0.8714 ms 19.9%
+  triton_convolution_36 1.2702 ms 13.7%
+SingleProcess AUTOTUNE takes 6.5366 seconds
+AUTOTUNE convolution(6x128x44x44, 256x128x3x3)
+  convolution 0.0865 ms 100.0%
+  triton_convolution_41 0.1723 ms 50.2%
+  triton_convolution_44 0.1962 ms 44.1%
+  triton_convolution_46 0.2180 ms 39.7%
+  triton_convolution_45 0.2528 ms 34.2%
+  triton_convolution_47 0.2662 ms 32.5%
+  triton_convolution_42 0.3284 ms 26.4%
+  triton_convolution_43 0.6086 ms 14.2%
+SingleProcess AUTOTUNE takes 7.2526 seconds
+AUTOTUNE convolution(6x256x44x44, 256x256x3x3)
+  convolution 0.1494 ms 100.0%
+  triton_convolution_48 0.3315 ms 45.1%
+  triton_convolution_51 0.3903 ms 38.3%
+  triton_convolution_53 0.4299 ms 34.8%
+  triton_convolution_52 0.5012 ms 29.8%
+  triton_convolution_54 0.5242 ms 28.5%
+  triton_convolution_49 0.7420 ms 20.1%
+  triton_convolution_50 1.2055 ms 12.4%
+SingleProcess AUTOTUNE takes 6.9560 seconds
+AUTOTUNE convolution(6x256x22x22, 512x256x3x3)
+  convolution 0.0859 ms 100.0%
+  triton_convolution_58 0.2001 ms 42.9%
+  triton_convolution_60 0.2192 ms 39.2%
+  triton_convolution_59 0.2680 ms 32.0%
+  triton_convolution_61 0.2716 ms 31.6%
+  triton_convolution_55 0.2749 ms 31.2%
+  triton_convolution_56 0.4792 ms 17.9%
+  triton_convolution_57 0.5950 ms 14.4%
+SingleProcess AUTOTUNE takes 7.3606 seconds
+AUTOTUNE convolution(6x512x22x22, 512x512x3x3)
+  convolution 0.1484 ms 100.0%
+  triton_convolution_65 0.3920 ms 37.9%
+  triton_convolution_67 0.4338 ms 34.2%
+  triton_convolution_68 0.5246 ms 28.3%
+  triton_convolution_66 0.5326 ms 27.9%
+  triton_convolution_62 0.6044 ms 24.6%
+  triton_convolution_63 0.9912 ms 15.0%
+  triton_convolution_64 1.1874 ms 12.5%
+SingleProcess AUTOTUNE takes 6.7953 seconds
+AUTOTUNE convolution(6x512x11x11, 512x512x3x3)
+  convolution 0.0732 ms 100.0%
+  triton_convolution_73 0.2374 ms 30.8%
+  triton_convolution_72 0.3923 ms 18.7%
+  triton_convolution_74 0.4227 ms 17.3%
+  triton_convolution_75 0.5379 ms 13.6%
+  triton_convolution_69 0.6048 ms 12.1%
+  triton_convolution_71 1.0003 ms 7.3%
+  triton_convolution_70 1.0683 ms 6.9%
+SingleProcess AUTOTUNE takes 6.8122 seconds
+AUTOTUNE convolution(6x1024x22x22, 512x1024x3x3)
+  convolution 0.2779 ms 100.0%
+  triton_convolution_93 0.7461 ms 37.2%
+  triton_convolution_95 0.8162 ms 34.1%
+  triton_convolution_96 1.0386 ms 26.8%
+  triton_convolution_94 1.0639 ms 26.1%
+  triton_convolution_90 1.1895 ms 23.4%
+  triton_convolution_91 2.0843 ms 13.3%
+  triton_convolution_92 2.3698 ms 11.7%
+SingleProcess AUTOTUNE takes 7.1516 seconds
+AUTOTUNE convolution(6x512x44x44, 256x512x3x3)
+  convolution 0.2845 ms 100.0%
+  triton_convolution_97 0.7492 ms 38.0%
+  triton_convolution_100 0.7769 ms 36.6%
+  triton_convolution_102 0.8718 ms 32.6%
+  triton_convolution_101 1.0231 ms 27.8%
+  triton_convolution_103 1.1442 ms 24.9%
+  triton_convolution_98 1.7737 ms 16.0%
+  triton_convolution_99 2.6973 ms 10.5%
+SingleProcess AUTOTUNE takes 6.8734 seconds
+AUTOTUNE convolution(6x256x88x88, 128x256x3x3)
+  convolution 0.3242 ms 100.0%
+  triton_convolution_116 0.7539 ms 43.0%
+  triton_convolution_111 0.7831 ms 41.4%
+  triton_convolution_114 0.8059 ms 40.2%
+  triton_convolution_115 0.9457 ms 34.3%
+  triton_convolution_117 1.1134 ms 29.1%
+  triton_convolution_112 1.9916 ms 16.3%
+  triton_convolution_113 3.0402 ms 10.7%
+SingleProcess AUTOTUNE takes 6.3334 seconds
+AUTOTUNE convolution(6x128x176x176, 64x128x3x3)
+  convolution 0.4249 ms 100.0%
+  triton_convolution_125 0.7636 ms 55.7%
+  triton_convolution_130 0.7750 ms 54.8%
+  triton_convolution_128 0.8105 ms 52.4%
+  triton_convolution_129 0.8389 ms 50.7%
+  triton_convolution_131 0.9306 ms 45.7%
+  triton_convolution_126 1.7586 ms 24.2%
+  triton_convolution_127 2.9469 ms 14.4%
+SingleProcess AUTOTUNE takes 5.6592 seconds
+AUTOTUNE convolution(6x64x352x352, 32x64x3x3)
+  convolution 0.8416 ms 100.0%
+  triton_convolution_145 0.9574 ms 87.9%
+  triton_convolution_142 0.9774 ms 86.1%
+  triton_convolution_143 1.0171 ms 82.7%
+  triton_convolution_139 1.1104 ms 75.8%
+  triton_convolution_144 1.1132 ms 75.6%
+  triton_convolution_140 1.2605 ms 66.8%
+  triton_convolution_141 2.3235 ms 36.2%
+SingleProcess AUTOTUNE takes 4.8066 seconds
+AUTOTUNE convolution(6x32x352x352, 4x32x3x3)
+  convolution 0.3403 ms 100.0%
+  triton_convolution_157 0.5217 ms 65.2%
+  triton_convolution_155 0.5765 ms 59.0%
+  triton_convolution_154 0.5803 ms 58.7%
+  triton_convolution_158 0.5882 ms 57.9%
+  triton_convolution_156 0.6502 ms 52.3%
+  triton_convolution_153 0.6953 ms 48.9%
+SingleProcess AUTOTUNE takes 3.5028 seconds
+AUTOTUNE convolution(6x20x352x352, 32x20x7x7)
+  convolution 1.4653 ms 100.0%
+  triton_convolution_162 2.7446 ms 53.4%
+  triton_convolution_159 2.9253 ms 50.1%
+  triton_convolution_163 3.0323 ms 48.3%
+  triton_convolution_165 3.1045 ms 47.2%
+  triton_convolution_160 3.1927 ms 45.9%
+  triton_convolution_164 3.3339 ms 44.0%
+  triton_convolution_161 6.1758 ms 23.7%
+SingleProcess AUTOTUNE takes 4.5879 seconds
+AUTOTUNE convolution(6x32x352x352, 5x32x3x3)
+  convolution 0.3420 ms 100.0%
+  triton_convolution_313 0.4566 ms 74.9%
+  triton_convolution_317 0.5308 ms 64.4%
+  triton_convolution_314 0.5864 ms 58.3%
+  triton_convolution_318 0.5998 ms 57.0%
+  triton_convolution_316 0.6616 ms 51.7%
+  triton_convolution_315 3.5874 ms 9.5%
+SingleProcess AUTOTUNE takes 4.6535 seconds
+AUTOTUNE convolution(6x3x352x352, 64x3x3x3)
+  convolution 0.1812 ms 100.0%
+  triton_convolution_324 0.4741 ms 38.2%
+  triton_convolution_319 0.5050 ms 35.9%
+  triton_convolution_323 0.5687 ms 31.9%
+  triton_convolution_322 0.5780 ms 31.3%
+  triton_convolution_321 0.9720 ms 18.6%
+  triton_convolution_320 1.2293 ms 14.7%
+SingleProcess AUTOTUNE takes 4.5431 seconds
+AUTOTUNE convolution(6x64x352x352, 64x64x3x3)
+  convolution 0.9368 ms 100.0%
+  triton_convolution_325 1.5644 ms 59.9%
+  triton_convolution_330 1.6048 ms 58.4%
+  triton_convolution_328 1.6292 ms 57.5%
+  triton_convolution_329 1.6396 ms 57.1%
+  triton_convolution_331 1.8196 ms 51.5%
+  triton_convolution_326 2.7817 ms 33.7%
+  triton_convolution_327 4.4771 ms 20.9%
+SingleProcess AUTOTUNE takes 5.9593 seconds
+AUTOTUNE convolution(6x64x176x176, 128x64x3x3)
+  convolution 0.3384 ms 100.0%
+  triton_convolution_332 0.6588 ms 51.4%
+  triton_convolution_335 0.6647 ms 50.9%
+  triton_convolution_337 0.7344 ms 46.1%
+  triton_convolution_336 0.8378 ms 40.4%
+  triton_convolution_338 0.9436 ms 35.9%
+  triton_convolution_333 1.3652 ms 24.8%
+  triton_convolution_334 2.3311 ms 14.5%
+SingleProcess AUTOTUNE takes 6.6120 seconds
+AUTOTUNE convolution(6x128x176x176, 128x128x3x3)
+  convolution 0.6179 ms 100.0%
+  triton_convolution_339 1.2893 ms 47.9%
+  triton_convolution_342 1.3246 ms 46.6%
+  triton_convolution_344 1.3710 ms 45.1%
+  triton_convolution_343 1.6078 ms 38.4%
+  triton_convolution_345 1.8667 ms 33.1%
+  triton_convolution_340 3.7940 ms 16.3%
+  triton_convolution_341 5.8522 ms 10.6%
+SingleProcess AUTOTUNE takes 6.2825 seconds
+AUTOTUNE convolution(6x128x88x88, 256x128x3x3)
+  convolution 0.2826 ms 100.0%
+  triton_convolution_346 0.6378 ms 44.3%
+  triton_convolution_349 0.6707 ms 42.1%
+  triton_convolution_351 0.6764 ms 41.8%
+  triton_convolution_350 0.8492 ms 33.3%
+  triton_convolution_352 0.9372 ms 30.2%
+  triton_convolution_347 1.6856 ms 16.8%
+  triton_convolution_348 2.2880 ms 12.4%
+SingleProcess AUTOTUNE takes 6.9567 seconds
+AUTOTUNE convolution(6x256x88x88, 256x256x3x3)
+  convolution 0.5358 ms 100.0%
+  triton_convolution_353 1.2556 ms 42.7%
+  triton_convolution_358 1.3465 ms 39.8%
+  triton_convolution_356 1.4374 ms 37.3%
+  triton_convolution_357 1.8613 ms 28.8%
+  triton_convolution_359 2.0278 ms 26.4%
+  triton_convolution_354 4.0064 ms 13.4%
+  triton_convolution_355 5.3476 ms 10.0%
+SingleProcess AUTOTUNE takes 7.7455 seconds
+AUTOTUNE convolution(6x256x44x44, 512x256x3x3)
+  convolution 0.2742 ms 100.0%
+  triton_convolution_415 0.6620 ms 41.4%
+  triton_convolution_418 0.7423 ms 36.9%
+  triton_convolution_420 0.7941 ms 34.5%
+  triton_convolution_419 0.8812 ms 31.1%
+  triton_convolution_421 1.0280 ms 26.7%
+  triton_convolution_416 1.3646 ms 20.1%
+  triton_convolution_417 2.3669 ms 11.6%
+SingleProcess AUTOTUNE takes 7.6315 seconds
+AUTOTUNE convolution(6x512x44x44, 512x512x3x3)
+  convolution 0.5422 ms 100.0%
+  triton_convolution_422 1.4296 ms 37.9%
+  triton_convolution_425 1.5347 ms 35.3%
+  triton_convolution_427 1.7154 ms 31.6%
+  triton_convolution_426 1.8103 ms 30.0%
+  triton_convolution_428 2.1478 ms 25.2%
+  triton_convolution_423 3.4089 ms 15.9%
+  triton_convolution_424 5.3201 ms 10.2%
+SingleProcess AUTOTUNE takes 7.3728 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:05,  5.19it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02,  9.10it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02, 10.51it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:02, 11.19it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 11.59it/s]
running benchmark:  37%|███▋      | 11/30 [00:01<00:01, 11.83it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 11.98it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 12.08it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:01, 12.13it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 12.19it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 12.22it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 12.24it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00, 12.26it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 12.30it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 12.30it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 11.77it/s]
+1.391x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  alexnet                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 77.67it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 85.34it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 87.96it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 86.77it/s]
+3.275x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_edgecnn                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 212.67it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 214.87it/s]
+1.367x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_gcn                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 127.74it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 128.37it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 128.00it/s]
+1.059x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_gin                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 282.53it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 284.10it/s]
+1.212x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_sage                     
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 148.62it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 159.57it/s]
+1.099x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  cm3leon_generate                   
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:01<00:41,  1.42s/it]
running benchmark:   7%|▋         | 2/30 [00:02<00:36,  1.30s/it]
running benchmark:  10%|█         | 3/30 [00:03<00:33,  1.25s/it]
running benchmark:  13%|█▎        | 4/30 [00:05<00:32,  1.23s/it]
running benchmark:  17%|█▋        | 5/30 [00:06<00:30,  1.22s/it]
running benchmark:  20%|██        | 6/30 [00:07<00:29,  1.21s/it]
running benchmark:  23%|██▎       | 7/30 [00:08<00:27,  1.21s/it]
running benchmark:  27%|██▋       | 8/30 [00:09<00:26,  1.20s/it]
running benchmark:  30%|███       | 9/30 [00:11<00:25,  1.21s/it]
running benchmark:  33%|███▎      | 10/30 [00:12<00:24,  1.21s/it]
running benchmark:  37%|███▋      | 11/30 [00:13<00:22,  1.21s/it]
running benchmark:  40%|████      | 12/30 [00:14<00:21,  1.20s/it]
running benchmark:  43%|████▎     | 13/30 [00:15<00:20,  1.20s/it]
running benchmark:  47%|████▋     | 14/30 [00:17<00:19,  1.20s/it]
running benchmark:  50%|█████     | 15/30 [00:18<00:18,  1.20s/it]
running benchmark:  53%|█████▎    | 16/30 [00:19<00:16,  1.20s/it]
running benchmark:  57%|█████▋    | 17/30 [00:20<00:15,  1.20s/it]
running benchmark:  60%|██████    | 18/30 [00:21<00:14,  1.20s/it]
running benchmark:  63%|██████▎   | 19/30 [00:23<00:13,  1.20s/it]
running benchmark:  67%|██████▋   | 20/30 [00:24<00:11,  1.20s/it]
running benchmark:  70%|███████   | 21/30 [00:25<00:10,  1.20s/it]
running benchmark:  73%|███████▎  | 22/30 [00:26<00:09,  1.20s/it]
running benchmark:  77%|███████▋  | 23/30 [00:27<00:08,  1.20s/it]
running benchmark:  80%|████████  | 24/30 [00:29<00:07,  1.20s/it]
running benchmark:  83%|████████▎ | 25/30 [00:30<00:05,  1.20s/it]
running benchmark:  87%|████████▋ | 26/30 [00:31<00:04,  1.20s/it]
running benchmark:  90%|█████████ | 27/30 [00:32<00:03,  1.20s/it]
running benchmark:  93%|█████████▎| 28/30 [00:33<00:02,  1.21s/it]
running benchmark:  97%|█████████▋| 29/30 [00:35<00:01,  1.21s/it]
running benchmark: 100%|██████████| 30/30 [00:36<00:00,  1.21s/it]
running benchmark: 100%|██████████| 30/30 [00:36<00:00,  1.21s/it]
+4.333x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  dcgan                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 411.58it/s]
+1.361x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+WARNING:root:demucs failed to load
+Original Error: "_thnn_fused_lstm_cell_cuda" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/__init__.py", line 31, in forward
+    return sources, self.model(mix)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/demucs/model.py", line 209, in forward
+    x = self.lstm(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/demucs/model.py", line 27, in forward
+    x = self.lstm(x)[0]
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/rnn.py", line 878, in forward
+    result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,
+RuntimeError: "_thnn_fused_lstm_cell_cuda" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  densenet121                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 24.19it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 29.05it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 30.61it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 31.34it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 31.78it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 32.02it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 32.19it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 31.42it/s]
+2.129x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_dc5 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:09, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_dc5 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+
loading model: 0it [00:05, ?it/s]
+WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+cuda eval  detectron2_fcos_r_50_fpn           
+WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+AUTOTUNE convolution(1x256x200x304, 128x256x1x1)
+  triton_convolution_115 0.0548 ms 100.0%
+  triton_convolution_111 0.0567 ms 96.7%
+  triton_convolution_116 0.0595 ms 92.2%
+  triton_convolution_114 0.0615 ms 89.2%
+  triton_convolution_117 0.0650 ms 84.4%
+  triton_convolution_112 0.0803 ms 68.3%
+  convolution 0.1286 ms 42.7%
+  triton_convolution_113 0.1794 ms 30.6%
+SingleProcess AUTOTUNE takes 5.6229 seconds
+AUTOTUNE convolution(1x512x100x152, 128x512x1x1)
+  convolution 0.0474 ms 100.0%
+  triton_convolution_148 0.0926 ms 51.2%
+  triton_convolution_147 0.1037 ms 45.7%
+  triton_convolution_150 0.1040 ms 45.6%
+  triton_convolution_145 0.1045 ms 45.3%
+  triton_convolution_144 0.1053 ms 45.0%
+  triton_convolution_149 0.1071 ms 44.2%
+  conv1x1_via_mm 0.1544 ms 30.7%
+  triton_convolution_146 0.2006 ms 23.6%
+SingleProcess AUTOTUNE takes 6.4628 seconds
+AUTOTUNE convolution(1x128x100x152, 128x128x3x3)
+  convolution 0.0904 ms 100.0%
+  triton_convolution_152 0.1744 ms 51.9%
+  triton_convolution_157 0.2057 ms 44.0%
+  triton_convolution_154 0.2677 ms 33.8%
+  triton_convolution_155 0.3344 ms 27.0%
+  triton_convolution_153 0.3585 ms 25.2%
+  triton_convolution_156 0.3673 ms 24.6%
+  triton_convolution_151 0.3891 ms 23.2%
+SingleProcess AUTOTUNE takes 5.8656 seconds
+AUTOTUNE convolution(1x128x100x152, 512x128x1x1)
+  convolution 0.0441 ms 100.0%
+  triton_convolution_162 0.0784 ms 56.2%
+  triton_convolution_161 0.0827 ms 53.3%
+  triton_convolution_164 0.0853 ms 51.7%
+  triton_convolution_163 0.0895 ms 49.2%
+  triton_convolution_159 0.0982 ms 44.9%
+  triton_convolution_160 0.1247 ms 35.3%
+  triton_convolution_158 0.1355 ms 32.5%
+  conv1x1_via_mm 0.1599 ms 27.6%
+SingleProcess AUTOTUNE takes 7.2775 seconds
+AUTOTUNE convolution(1x512x100x152, 256x512x1x1)
+  triton_convolution_211 0.0519 ms 100.0%
+  triton_convolution_210 0.0565 ms 91.8%
+  triton_convolution_212 0.0631 ms 82.2%
+  triton_convolution_213 0.0706 ms 73.5%
+  convolution 0.0846 ms 61.3%
+  triton_convolution_208 0.0986 ms 52.6%
+  triton_convolution_207 0.1487 ms 34.9%
+  triton_convolution_209 0.1764 ms 29.4%
+SingleProcess AUTOTUNE takes 7.6957 seconds
+AUTOTUNE convolution(1x256x50x76, 256x256x3x3)
+  convolution 0.0896 ms 100.0%
+  triton_convolution_220 0.1949 ms 46.0%
+  triton_convolution_215 0.2332 ms 38.4%
+  triton_convolution_217 0.2452 ms 36.5%
+  triton_convolution_218 0.3109 ms 28.8%
+  triton_convolution_216 0.3466 ms 25.8%
+  triton_convolution_219 0.3814 ms 23.5%
+  triton_convolution_214 0.4592 ms 19.5%
+SingleProcess AUTOTUNE takes 7.5863 seconds
+AUTOTUNE convolution(1x256x50x76, 1024x256x1x1)
+  convolution 0.0377 ms 100.0%
+  triton_convolution_225 0.0663 ms 56.9%
+  triton_convolution_224 0.0732 ms 51.5%
+  triton_convolution_227 0.0776 ms 48.6%
+  triton_convolution_226 0.0847 ms 44.5%
+  triton_convolution_222 0.1031 ms 36.5%
+  triton_convolution_223 0.1423 ms 26.5%
+  triton_convolution_221 0.1479 ms 25.5%
+  conv1x1_via_mm 0.1552 ms 24.3%
+SingleProcess AUTOTUNE takes 8.0819 seconds
+AUTOTUNE convolution(1x512x100x152, 1024x512x1x1)
+  triton_convolution_232 0.1319 ms 100.0%
+  convolution 0.1400 ms 94.2%
+  triton_convolution_231 0.1497 ms 88.1%
+  triton_convolution_233 0.1609 ms 82.0%
+  triton_convolution_234 0.1835 ms 71.9%
+  triton_convolution_229 0.2468 ms 53.4%
+  triton_convolution_228 0.3362 ms 39.2%
+  triton_convolution_230 0.4401 ms 30.0%
+SingleProcess AUTOTUNE takes 8.0033 seconds
+AUTOTUNE convolution(1x1024x50x76, 256x1024x1x1)
+  convolution 0.0376 ms 100.0%
+  triton_convolution_239 0.0760 ms 49.4%
+  triton_convolution_238 0.0864 ms 43.5%
+  triton_convolution_241 0.0948 ms 39.6%
+  triton_convolution_240 0.1006 ms 37.3%
+  triton_convolution_236 0.1564 ms 24.0%
+  conv1x1_via_mm 0.1745 ms 21.5%
+  triton_convolution_237 0.2122 ms 17.7%
+  triton_convolution_235 0.2318 ms 16.2%
+SingleProcess AUTOTUNE takes 6.8989 seconds
+AUTOTUNE convolution(1x256x50x76, 256x256x3x3)
+  convolution 0.0899 ms 100.0%
+  triton_convolution_248 0.1955 ms 46.0%
+  triton_convolution_245 0.2449 ms 36.7%
+  triton_convolution_243 0.2492 ms 36.1%
+  triton_convolution_246 0.3100 ms 29.0%
+  triton_convolution_244 0.3621 ms 24.8%
+  triton_convolution_247 0.3795 ms 23.7%
+  triton_convolution_242 0.4564 ms 19.7%
+SingleProcess AUTOTUNE takes 6.7545 seconds
+AUTOTUNE convolution(1x256x50x76, 1024x256x1x1)
+  convolution 0.0376 ms 100.0%
+  triton_convolution_253 0.0666 ms 56.5%
+  triton_convolution_252 0.0734 ms 51.3%
+  triton_convolution_255 0.0776 ms 48.5%
+  triton_convolution_254 0.0843 ms 44.7%
+  triton_convolution_250 0.0971 ms 38.8%
+  triton_convolution_251 0.1455 ms 25.9%
+  triton_convolution_249 0.1461 ms 25.8%
+  conv1x1_via_mm 0.1533 ms 24.5%
+SingleProcess AUTOTUNE takes 7.0479 seconds
+AUTOTUNE convolution(1x1024x50x76, 512x1024x1x1)
+  triton_convolution_344 0.0628 ms 100.0%
+  convolution 0.0885 ms 71.0%
+  triton_convolution_343 0.0960 ms 65.5%
+  triton_convolution_345 0.1033 ms 60.8%
+  triton_convolution_346 0.1245 ms 50.5%
+  triton_convolution_341 0.1776 ms 35.4%
+  triton_convolution_340 0.2685 ms 23.4%
+  triton_convolution_342 0.3289 ms 19.1%
+SingleProcess AUTOTUNE takes 7.7714 seconds
+AUTOTUNE convolution(1x512x25x38, 512x512x3x3)
+  convolution 0.0999 ms 100.0%
+  triton_convolution_353 0.3650 ms 27.4%
+  triton_convolution_351 0.4126 ms 24.2%
+  triton_convolution_348 0.4255 ms 23.5%
+  triton_convolution_350 0.4505 ms 22.2%
+  triton_convolution_349 0.6654 ms 15.0%
+  triton_convolution_352 0.6893 ms 14.5%
+  triton_convolution_347 0.8888 ms 11.2%
+SingleProcess AUTOTUNE takes 7.6453 seconds
+AUTOTUNE convolution(1x512x25x38, 2048x512x1x1)
+  convolution 0.0564 ms 100.0%
+  triton_convolution_358 0.0758 ms 74.4%
+  triton_convolution_357 0.0883 ms 63.8%
+  triton_convolution_359 0.0944 ms 59.7%
+  triton_convolution_360 0.0948 ms 59.5%
+  triton_convolution_355 0.0949 ms 59.4%
+  triton_convolution_354 0.1436 ms 39.2%
+  conv1x1_via_mm 0.1566 ms 36.0%
+  triton_convolution_356 0.1896 ms 29.7%
+SingleProcess AUTOTUNE takes 8.0976 seconds
+AUTOTUNE convolution(1x1024x50x76, 2048x1024x1x1)
+  triton_convolution_365 0.1481 ms 100.0%
+  triton_convolution_364 0.1736 ms 85.3%
+  triton_convolution_362 0.1824 ms 81.2%
+  triton_convolution_366 0.1925 ms 76.9%
+  triton_convolution_367 0.2137 ms 69.3%
+  convolution 0.2200 ms 67.3%
+  triton_convolution_361 0.3169 ms 46.7%
+  triton_convolution_363 0.6227 ms 23.8%
+SingleProcess AUTOTUNE takes 8.2863 seconds
+AUTOTUNE convolution(1x2048x25x38, 512x2048x1x1)
+  convolution 0.0650 ms 100.0%
+  triton_convolution_372 0.1144 ms 56.9%
+  conv1x1_via_mm 0.1555 ms 41.8%
+  triton_convolution_371 0.1580 ms 41.2%
+  triton_convolution_374 0.1727 ms 37.7%
+  triton_convolution_373 0.1831 ms 35.5%
+  triton_convolution_369 0.2897 ms 22.4%
+  triton_convolution_370 0.3836 ms 17.0%
+  triton_convolution_368 0.4340 ms 15.0%
+SingleProcess AUTOTUNE takes 6.9962 seconds
+AUTOTUNE convolution(1x512x25x38, 512x512x3x3)
+  convolution 0.0998 ms 100.0%
+  triton_convolution_381 0.3660 ms 27.3%
+  triton_convolution_379 0.4142 ms 24.1%
+  triton_convolution_378 0.4518 ms 22.1%
+  triton_convolution_376 0.4645 ms 21.5%
+  triton_convolution_377 0.6816 ms 14.6%
+  triton_convolution_380 0.6866 ms 14.5%
+  triton_convolution_375 0.8472 ms 11.8%
+SingleProcess AUTOTUNE takes 6.9308 seconds
+AUTOTUNE convolution(1x512x25x38, 2048x512x1x1)
+  convolution 0.0567 ms 100.0%
+  triton_convolution_386 0.0775 ms 73.2%
+  triton_convolution_385 0.0890 ms 63.7%
+  triton_convolution_383 0.0902 ms 62.8%
+  triton_convolution_387 0.0941 ms 60.2%
+  triton_convolution_388 0.0948 ms 59.8%
+  triton_convolution_382 0.1348 ms 42.0%
+  conv1x1_via_mm 0.1642 ms 34.5%
+  triton_convolution_384 0.1848 ms 30.7%
+SingleProcess AUTOTUNE takes 7.0501 seconds
+AUTOTUNE convolution(1x512x100x152, 256x512x1x1)
+  convolution 0.0634 ms 100.0%
+  triton_convolution_414 0.1471 ms 43.1%
+  conv1x1_via_mm 0.1657 ms 38.3%
+  triton_convolution_413 0.1668 ms 38.0%
+  triton_convolution_415 0.1685 ms 37.6%
+  triton_convolution_416 0.1741 ms 36.4%
+  triton_convolution_411 0.1989 ms 31.9%
+  triton_convolution_410 0.2278 ms 27.8%
+  triton_convolution_412 0.3433 ms 18.5%
+SingleProcess AUTOTUNE takes 7.9512 seconds
+AUTOTUNE convolution(1x1024x50x76, 256x1024x1x1)
+  convolution 0.0372 ms 100.0%
+  triton_convolution_421 0.0774 ms 48.0%
+  triton_convolution_420 0.0878 ms 42.3%
+  triton_convolution_423 0.0961 ms 38.6%
+  triton_convolution_422 0.1008 ms 36.8%
+  triton_convolution_418 0.1506 ms 24.7%
+  conv1x1_via_mm 0.1633 ms 22.8%
+  triton_convolution_419 0.2054 ms 18.1%
+  triton_convolution_417 0.2270 ms 16.4%
+SingleProcess AUTOTUNE takes 7.0517 seconds
+AUTOTUNE convolution(1x2048x25x38, 256x2048x1x1)
+  convolution 0.0376 ms 100.0%
+  triton_convolution_428 0.1103 ms 34.0%
+  triton_convolution_427 0.1556 ms 24.1%
+  conv1x1_via_mm 0.1566 ms 24.0%
+  triton_convolution_430 0.1715 ms 21.9%
+  triton_convolution_429 0.1796 ms 20.9%
+  triton_convolution_425 0.3022 ms 12.4%
+  triton_convolution_426 0.3787 ms 9.9%
+  triton_convolution_424 0.4530 ms 8.3%
+SingleProcess AUTOTUNE takes 7.3767 seconds
+AUTOTUNE convolution(1x256x100x152, 256x256x3x3)
+  convolution 0.2273 ms 100.0%
+  triton_convolution_437 0.6431 ms 35.3%
+  triton_convolution_432 0.7090 ms 32.1%
+  triton_convolution_434 0.7951 ms 28.6%
+  triton_convolution_435 1.0360 ms 21.9%
+  triton_convolution_436 1.2861 ms 17.7%
+  triton_convolution_433 1.5287 ms 14.9%
+  triton_convolution_431 1.6555 ms 13.7%
+SingleProcess AUTOTUNE takes 7.4565 seconds
+AUTOTUNE convolution(1x256x50x76, 256x256x3x3)
+  convolution 0.0897 ms 100.0%
+  triton_convolution_444 0.2053 ms 43.7%
+  triton_convolution_439 0.2383 ms 37.6%
+  triton_convolution_441 0.2459 ms 36.5%
+  triton_convolution_442 0.3091 ms 29.0%
+  triton_convolution_440 0.3687 ms 24.3%
+  triton_convolution_443 0.3789 ms 23.7%
+  triton_convolution_438 0.3956 ms 22.7%
+SingleProcess AUTOTUNE takes 6.9123 seconds
+AUTOTUNE convolution(1x256x25x38, 256x256x3x3)
+  convolution 0.0464 ms 100.0%
+  triton_convolution_449 0.1355 ms 34.3%
+  triton_convolution_451 0.1836 ms 25.3%
+  triton_convolution_448 0.2228 ms 20.8%
+  triton_convolution_446 0.2374 ms 19.6%
+  triton_convolution_450 0.3352 ms 13.9%
+  triton_convolution_447 0.3461 ms 13.4%
+  triton_convolution_445 0.3961 ms 11.7%
+SingleProcess AUTOTUNE takes 1.1521 seconds
+AUTOTUNE convolution(1x256x25x38, 256x256x3x3)
+  convolution 0.0402 ms 100.0%
+  triton_convolution_456 0.1306 ms 30.8%
+  triton_convolution_458 0.2270 ms 17.7%
+  triton_convolution_455 0.2505 ms 16.0%
+  triton_convolution_453 0.2606 ms 15.4%
+  triton_convolution_454 0.3320 ms 12.1%
+  triton_convolution_457 0.3497 ms 11.5%
+  triton_convolution_452 0.4008 ms 10.0%
+SingleProcess AUTOTUNE takes 7.6162 seconds
+AUTOTUNE convolution(1x256x13x19, 256x256x3x3)
+  convolution 0.0416 ms 100.0%
+  triton_convolution_463 0.1188 ms 35.0%
+  triton_convolution_461 0.1337 ms 31.1%
+  triton_convolution_465 0.1967 ms 21.2%
+  triton_convolution_462 0.2337 ms 17.8%
+  triton_convolution_460 0.2501 ms 16.6%
+  triton_convolution_464 0.3423 ms 12.2%
+  triton_convolution_459 0.3898 ms 10.7%
+SingleProcess AUTOTUNE takes 7.2758 seconds
+[2023-12-06 14:53:16,224] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (8)
+[2023-12-06 14:53:16,224] torch._dynamo.convert_frame: [WARNING]    function: 'forward' (/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/batch_norm.py:318)
+[2023-12-06 14:53:16,224] torch._dynamo.convert_frame: [WARNING]    last reason: L['self']._pos == 0                                           # ret = self[self._pos](x)  # miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/batch_norm.py:319 in forward
+[2023-12-06 14:53:16,224] torch._dynamo.convert_frame: [WARNING] To log all recompilation reasons, use TORCH_LOGS="recompiles".
+[2023-12-06 14:53:16,224] torch._dynamo.convert_frame: [WARNING] To diagnose recompilation issues, see https://pytorch.org/docs/master/compile/troubleshooting.html.
+skipping cudagraphs due to ['mutated inputs']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:01, 15.13it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:01, 14.36it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:01, 15.44it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 16.07it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:01, 15.76it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:01, 15.57it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 16.03it/s]
running benchmark:  53%|█████▎    | 16/30 [00:01<00:00, 16.19it/s]
running benchmark:  60%|██████    | 18/30 [00:01<00:00, 16.44it/s]
running benchmark:  67%|██████▋   | 20/30 [00:01<00:00, 16.68it/s]
running benchmark:  73%|███████▎  | 22/30 [00:01<00:00, 16.89it/s]
running benchmark:  80%|████████  | 24/30 [00:01<00:00, 17.03it/s]
running benchmark:  87%|████████▋ | 26/30 [00:01<00:00, 17.11it/s]
running benchmark:  93%|█████████▎| 28/30 [00:01<00:00, 17.13it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 17.20it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 16.46it/s]
+1.564x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_101_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_101_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_50_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_50_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:13, ?it/s]
+cuda eval  dlrm                               
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  9.43it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 12.80it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 13.56it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 14.05it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 14.16it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 14.38it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 14.43it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 14.44it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 14.58it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 14.44it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 14.45it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 14.32it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 14.46it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 14.53it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 14.53it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 14.28it/s]
+41.356x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model doctr_det_predictor supports float32 only
+
loading model: 0it [00:06, ?it/s]
+WARNING:common:Model doctr_det_predictor supports float32 only
+cuda eval  doctr_det_predictor                
+WARNING:common:Model doctr_det_predictor supports float32 only
+AUTOTUNE convolution(1x3x1024x1024, 64x3x7x7)
+  convolution 0.2094 ms 100.0%
+  triton_convolution_5 0.7862 ms 26.6%
+  triton_convolution_0 0.8351 ms 25.1%
+  triton_convolution_3 0.8552 ms 24.5%
+  triton_convolution_4 0.9119 ms 23.0%
+  triton_convolution_2 2.5612 ms 8.2%
+  triton_convolution_1 3.6571 ms 5.7%
+SingleProcess AUTOTUNE takes 4.0615 seconds
+AUTOTUNE mm(65536x64, 64x64)
+  triton_mm_14 0.0310 ms 100.0%
+  triton_mm_16 0.0321 ms 96.5%
+  triton_mm_6 0.0329 ms 94.3%
+  triton_mm_7 0.0330 ms 94.1%
+  triton_mm_8 0.0330 ms 94.1%
+  mm 0.0335 ms 92.6%
+  triton_mm_13 0.0338 ms 91.8%
+  triton_mm_17 0.0355 ms 87.5%
+  triton_mm_9 0.0358 ms 86.6%
+  triton_mm_15 0.0360 ms 86.1%
+SingleProcess AUTOTUNE takes 5.2301 seconds
+AUTOTUNE convolution(1x64x256x256, 64x64x3x3)
+  convolution 0.1039 ms 100.0%
+  triton_convolution_23 0.1536 ms 67.7%
+  triton_convolution_21 0.1548 ms 67.1%
+  triton_convolution_18 0.1591 ms 65.3%
+  triton_convolution_22 0.1603 ms 64.8%
+  triton_convolution_24 0.2101 ms 49.5%
+  triton_convolution_19 0.3588 ms 29.0%
+  triton_convolution_20 0.4993 ms 20.8%
+SingleProcess AUTOTUNE takes 5.1042 seconds
+AUTOTUNE mm(65536x64, 64x256)
+  triton_mm_33 0.0669 ms 100.0%
+  triton_mm_27 0.0677 ms 98.8%
+  triton_mm_26 0.0689 ms 97.1%
+  triton_mm_25 0.0702 ms 95.3%
+  mm 0.0716 ms 93.5%
+  triton_mm_35 0.0716 ms 93.4%
+  triton_mm_29 0.0849 ms 78.8%
+  triton_mm_32 0.0862 ms 77.6%
+  triton_mm_28 0.0894 ms 74.8%
+  triton_mm_36 0.0902 ms 74.1%
+SingleProcess AUTOTUNE takes 5.4769 seconds
+AUTOTUNE mm(65536x256, 256x64)
+  triton_mm_51 0.0707 ms 100.0%
+  triton_mm_50 0.0725 ms 97.5%
+  mm 0.0727 ms 97.3%
+  triton_mm_49 0.0754 ms 93.7%
+  triton_mm_59 0.0756 ms 93.5%
+  triton_mm_53 0.0756 ms 93.4%
+  triton_mm_56 0.0756 ms 93.4%
+  triton_mm_52 0.0762 ms 92.8%
+  triton_mm_54 0.0801 ms 88.3%
+  triton_mm_57 0.0818 ms 86.4%
+SingleProcess AUTOTUNE takes 5.7454 seconds
+AUTOTUNE mm(65536x256, 256x128)
+  mm 0.0783 ms 100.0%
+  triton_mm_112 0.0861 ms 90.9%
+  triton_mm_113 0.0869 ms 90.1%
+  triton_mm_111 0.0936 ms 83.6%
+  triton_mm_115 0.0958 ms 81.7%
+  triton_mm_114 0.0968 ms 80.9%
+  triton_mm_121 0.0979 ms 80.0%
+  triton_mm_118 0.1064 ms 73.6%
+  triton_mm_119 0.1201 ms 65.2%
+  triton_mm_116 0.1252 ms 62.5%
+SingleProcess AUTOTUNE takes 6.2801 seconds
+AUTOTUNE convolution(1x128x256x256, 128x128x3x3)
+  convolution 0.1290 ms 100.0%
+  triton_convolution_123 0.1902 ms 67.8%
+  triton_convolution_128 0.2124 ms 60.7%
+  triton_convolution_127 0.2159 ms 59.8%
+  triton_convolution_126 0.2236 ms 57.7%
+  triton_convolution_129 0.3526 ms 36.6%
+  triton_convolution_124 0.4094 ms 31.5%
+  triton_convolution_125 0.7647 ms 16.9%
+SingleProcess AUTOTUNE takes 6.2517 seconds
+AUTOTUNE mm(16384x128, 128x512)
+  mm 0.0431 ms 100.0%
+  triton_mm_132 0.0471 ms 91.6%
+  triton_mm_131 0.0472 ms 91.5%
+  triton_mm_130 0.0519 ms 83.2%
+  triton_mm_140 0.0546 ms 79.1%
+  triton_mm_134 0.0598 ms 72.1%
+  triton_mm_133 0.0603 ms 71.6%
+  triton_mm_137 0.0620 ms 69.6%
+  triton_mm_138 0.0808 ms 53.4%
+  triton_mm_141 0.0808 ms 53.4%
+SingleProcess AUTOTUNE takes 5.9133 seconds
+AUTOTUNE convolution(1x256x256x256, 512x256x1x1)
+  triton_convolution_145 0.1404 ms 100.0%
+  triton_convolution_147 0.1420 ms 98.9%
+  triton_convolution_148 0.1541 ms 91.1%
+  triton_convolution_146 0.1624 ms 86.4%
+  convolution 0.1686 ms 83.3%
+  triton_convolution_142 0.1970 ms 71.3%
+  triton_convolution_143 0.2732 ms 51.4%
+  triton_convolution_144 0.4425 ms 31.7%
+SingleProcess AUTOTUNE takes 6.2779 seconds
+AUTOTUNE mm(16384x512, 512x128)
+  mm 0.0503 ms 100.0%
+  triton_mm_150 0.0508 ms 99.0%
+  triton_mm_159 0.0521 ms 96.4%
+  triton_mm_151 0.0530 ms 94.9%
+  triton_mm_152 0.0538 ms 93.5%
+  triton_mm_153 0.0560 ms 89.8%
+  triton_mm_157 0.0595 ms 84.5%
+  triton_mm_149 0.0608 ms 82.6%
+  triton_mm_155 0.0621 ms 80.9%
+  triton_mm_154 0.0623 ms 80.7%
+SingleProcess AUTOTUNE takes 7.0252 seconds
+AUTOTUNE convolution(1x128x128x128, 128x128x3x3)
+  convolution 0.0852 ms 100.0%
+  triton_convolution_166 0.1735 ms 49.1%
+  triton_convolution_161 0.1809 ms 47.1%
+  triton_convolution_165 0.1851 ms 46.0%
+  triton_convolution_164 0.2070 ms 41.2%
+  triton_convolution_167 0.2574 ms 33.1%
+  triton_convolution_162 0.3364 ms 25.3%
+  triton_convolution_163 0.6397 ms 13.3%
+SingleProcess AUTOTUNE takes 6.2372 seconds
+AUTOTUNE mm(16384x512, 512x256)
+  mm 0.0626 ms 100.0%
+  triton_mm_243 0.0631 ms 99.2%
+  triton_mm_244 0.0632 ms 98.9%
+  triton_mm_245 0.0710 ms 88.1%
+  triton_mm_246 0.0714 ms 87.7%
+  triton_mm_242 0.0812 ms 77.1%
+  triton_mm_252 0.0815 ms 76.8%
+  triton_mm_250 0.0963 ms 65.0%
+  triton_mm_249 0.0975 ms 64.2%
+  triton_mm_247 0.1101 ms 56.8%
+SingleProcess AUTOTUNE takes 6.2753 seconds
+AUTOTUNE convolution(1x256x128x128, 256x256x3x3)
+  convolution 0.1011 ms 100.0%
+  triton_convolution_257 0.2082 ms 48.5%
+  triton_convolution_259 0.2219 ms 45.6%
+  triton_convolution_258 0.2456 ms 41.1%
+  triton_convolution_260 0.3179 ms 31.8%
+  triton_convolution_254 0.3894 ms 25.9%
+  triton_convolution_255 0.6003 ms 16.8%
+  triton_convolution_256 0.7187 ms 14.1%
+SingleProcess AUTOTUNE takes 6.4901 seconds
+AUTOTUNE mm(4096x256, 256x1024)
+  mm 0.0368 ms 100.0%
+  triton_mm_262 0.0386 ms 95.3%
+  triton_mm_263 0.0393 ms 93.6%
+  triton_mm_264 0.0461 ms 79.8%
+  triton_mm_271 0.0461 ms 79.8%
+  triton_mm_265 0.0469 ms 78.5%
+  triton_mm_261 0.0469 ms 78.5%
+  triton_mm_268 0.0580 ms 63.4%
+  triton_mm_269 0.0639 ms 57.6%
+  triton_mm_267 0.0661 ms 55.6%
+SingleProcess AUTOTUNE takes 6.1028 seconds
+AUTOTUNE convolution(1x512x128x128, 1024x512x1x1)
+  triton_convolution_276 0.1344 ms 100.0%
+  convolution 0.1401 ms 96.0%
+  triton_convolution_278 0.1469 ms 91.5%
+  triton_convolution_277 0.1505 ms 89.3%
+  triton_convolution_279 0.1562 ms 86.1%
+  triton_convolution_273 0.2251 ms 59.7%
+  triton_convolution_274 0.2423 ms 55.5%
+  triton_convolution_275 0.4420 ms 30.4%
+SingleProcess AUTOTUNE takes 6.0181 seconds
+AUTOTUNE mm(4096x1024, 1024x256)
+  mm 0.0375 ms 100.0%
+  triton_mm_282 0.0416 ms 90.1%
+  triton_mm_281 0.0419 ms 89.5%
+  triton_mm_280 0.0490 ms 76.5%
+  triton_mm_283 0.0499 ms 75.1%
+  triton_mm_284 0.0519 ms 72.2%
+  triton_mm_288 0.0533 ms 70.3%
+  triton_mm_286 0.0566 ms 66.2%
+  triton_mm_285 0.0582 ms 64.4%
+  triton_mm_290 0.0666 ms 56.3%
+SingleProcess AUTOTUNE takes 6.4366 seconds
+AUTOTUNE convolution(1x256x64x64, 256x256x3x3)
+  convolution 0.0853 ms 100.0%
+  triton_convolution_295 0.2025 ms 42.1%
+  triton_convolution_296 0.2126 ms 40.1%
+  triton_convolution_297 0.2161 ms 39.5%
+  triton_convolution_298 0.2563 ms 33.3%
+  triton_convolution_292 0.3071 ms 27.8%
+  triton_convolution_293 0.4957 ms 17.2%
+  triton_convolution_294 0.6503 ms 13.1%
+SingleProcess AUTOTUNE takes 6.5396 seconds
+AUTOTUNE mm(4096x1024, 1024x512)
+  mm 0.0631 ms 100.0%
+  triton_mm_436 0.0642 ms 98.3%
+  triton_mm_437 0.0665 ms 94.9%
+  triton_mm_438 0.0710 ms 88.9%
+  triton_mm_439 0.0714 ms 88.4%
+  triton_mm_445 0.0762 ms 82.9%
+  triton_mm_443 0.0849 ms 74.4%
+  triton_mm_435 0.0868 ms 72.7%
+  triton_mm_441 0.1031 ms 61.2%
+  triton_mm_440 0.1057 ms 59.7%
+SingleProcess AUTOTUNE takes 6.3045 seconds
+AUTOTUNE convolution(1x512x64x64, 512x512x3x3)
+  convolution 0.0982 ms 100.0%
+  triton_convolution_451 0.2785 ms 35.3%
+  triton_convolution_450 0.4006 ms 24.5%
+  triton_convolution_452 0.4257 ms 23.1%
+  triton_convolution_453 0.5818 ms 16.9%
+  triton_convolution_447 0.6247 ms 15.7%
+  triton_convolution_448 1.1776 ms 8.3%
+  triton_convolution_449 1.4166 ms 6.9%
+SingleProcess AUTOTUNE takes 6.8421 seconds
+AUTOTUNE mm(1024x512, 512x2048)
+  triton_mm_455 0.0372 ms 100.0%
+  mm 0.0375 ms 99.4%
+  triton_mm_456 0.0396 ms 94.1%
+  triton_mm_458 0.0434 ms 85.9%
+  triton_mm_457 0.0439 ms 84.8%
+  triton_mm_464 0.0443 ms 84.1%
+  triton_mm_454 0.0481 ms 77.4%
+  triton_mm_462 0.0514 ms 72.5%
+  triton_mm_459 0.0595 ms 62.6%
+  triton_mm_460 0.0597 ms 62.4%
+SingleProcess AUTOTUNE takes 6.4090 seconds
+AUTOTUNE convolution(1x1024x64x64, 2048x1024x1x1)
+  triton_convolution_469 0.1703 ms 100.0%
+  triton_convolution_470 0.1723 ms 98.8%
+  triton_convolution_471 0.1911 ms 89.1%
+  triton_convolution_472 0.1957 ms 87.0%
+  convolution 0.2139 ms 79.6%
+  triton_convolution_467 0.2437 ms 69.9%
+  triton_convolution_466 0.2480 ms 68.7%
+  triton_convolution_468 0.5752 ms 29.6%
+SingleProcess AUTOTUNE takes 6.2113 seconds
+AUTOTUNE mm(1024x2048, 2048x512)
+  mm 0.0424 ms 100.0%
+  triton_mm_476 0.0450 ms 94.1%
+  triton_mm_477 0.0451 ms 94.0%
+  triton_mm_474 0.0499 ms 85.0%
+  triton_mm_475 0.0507 ms 83.6%
+  triton_mm_481 0.0621 ms 68.2%
+  triton_mm_478 0.0624 ms 67.9%
+  triton_mm_479 0.0626 ms 67.7%
+  triton_mm_473 0.0737 ms 57.5%
+  triton_mm_482 0.0820 ms 51.6%
+SingleProcess AUTOTUNE takes 6.2607 seconds
+AUTOTUNE convolution(1x512x32x32, 512x512x3x3)
+  convolution 0.0908 ms 100.0%
+  triton_convolution_489 0.2671 ms 34.0%
+  triton_convolution_488 0.3950 ms 23.0%
+  triton_convolution_490 0.4211 ms 21.6%
+  triton_convolution_491 0.4894 ms 18.5%
+  triton_convolution_485 0.6104 ms 14.9%
+  triton_convolution_486 1.0281 ms 8.8%
+  triton_convolution_487 1.2878 ms 7.0%
+SingleProcess AUTOTUNE takes 6.9597 seconds
+AUTOTUNE mm(1024x2048, 2048x256)
+  mm 0.0332 ms 100.0%
+  triton_mm_543 0.0341 ms 97.4%
+  triton_mm_540 0.0431 ms 77.0%
+  triton_mm_541 0.0437 ms 76.0%
+  triton_mm_538 0.0444 ms 74.9%
+  triton_mm_539 0.0445 ms 74.6%
+  triton_mm_536 0.0486 ms 68.4%
+  triton_mm_537 0.0499 ms 66.5%
+  triton_mm_544 0.0528 ms 62.9%
+  triton_mm_535 0.0711 ms 46.7%
+SingleProcess AUTOTUNE takes 6.4188 seconds
+AUTOTUNE mm(65536x256, 256x256)
+  mm 0.1208 ms 100.0%
+  triton_mm_573 0.1271 ms 95.0%
+  triton_mm_572 0.1280 ms 94.3%
+  triton_mm_575 0.1544 ms 78.2%
+  triton_mm_571 0.1553 ms 77.8%
+  triton_mm_574 0.1583 ms 76.3%
+  triton_mm_581 0.1659 ms 72.8%
+  triton_mm_578 0.1823 ms 66.2%
+  triton_mm_579 0.2158 ms 56.0%
+  triton_mm_576 0.2377 ms 50.8%
+SingleProcess AUTOTUNE takes 6.9839 seconds
+AUTOTUNE convolution(1x256x256x256, 64x256x3x3)
+  convolution 0.3135 ms 100.0%
+  triton_convolution_588 0.5772 ms 54.3%
+  triton_convolution_583 0.5833 ms 53.7%
+  triton_convolution_586 0.6431 ms 48.8%
+  triton_convolution_587 0.6662 ms 47.1%
+  triton_convolution_589 0.8547 ms 36.7%
+  triton_convolution_584 1.6492 ms 19.0%
+  triton_convolution_585 2.6317 ms 11.9%
+SingleProcess AUTOTUNE takes 5.3146 seconds
+AUTOTUNE convolution(1x256x128x128, 64x256x3x3)
+  convolution 0.1104 ms 100.0%
+  triton_convolution_590 0.1744 ms 63.3%
+  triton_convolution_595 0.2164 ms 51.0%
+  triton_convolution_594 0.2456 ms 45.0%
+  triton_convolution_593 0.2602 ms 42.4%
+  triton_convolution_596 0.2678 ms 41.2%
+  triton_convolution_591 0.5463 ms 20.2%
+  triton_convolution_592 0.6592 ms 16.8%
+SingleProcess AUTOTUNE takes 5.6784 seconds
+AUTOTUNE convolution(1x256x64x64, 64x256x3x3)
+  convolution 0.0451 ms 100.0%
+  triton_convolution_602 0.1072 ms 42.0%
+  triton_convolution_601 0.1156 ms 39.0%
+  triton_convolution_600 0.1305 ms 34.5%
+  triton_convolution_597 0.1377 ms 32.7%
+  triton_convolution_603 0.2571 ms 17.5%
+  triton_convolution_598 0.5049 ms 8.9%
+  triton_convolution_599 0.6556 ms 6.9%
+SingleProcess AUTOTUNE takes 5.2021 seconds
+AUTOTUNE convolution(1x256x32x32, 64x256x3x3)
+  convolution 0.0373 ms 100.0%
+  triton_convolution_609 0.1046 ms 35.7%
+  triton_convolution_608 0.1135 ms 32.9%
+  triton_convolution_607 0.1295 ms 28.8%
+  triton_convolution_604 0.1337 ms 27.9%
+  triton_convolution_610 0.2479 ms 15.0%
+  triton_convolution_605 0.4830 ms 7.7%
+  triton_convolution_606 0.6409 ms 5.8%
+SingleProcess AUTOTUNE takes 5.8870 seconds
+[2023-12-06 14:58:15,672] [1/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+skipping cudagraphs due to ['non-cuda device in graph']
+Run failed with return code:  -11
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model doctr_reco_predictor supports float32 only
+
loading model: 0it [00:07, ?it/s]
+WARNING:common:Model doctr_reco_predictor supports float32 only
+cuda eval  doctr_reco_predictor               
+WARNING:common:Model doctr_reco_predictor supports float32 only
+AUTOTUNE convolution(1x3x32x128, 64x3x3x3)
+  convolution 0.0109 ms 100.0%
+  triton_convolution_4 0.0129 ms 84.7%
+  triton_convolution_0 0.0137 ms 80.1%
+  triton_convolution_3 0.0162 ms 67.6%
+  triton_convolution_5 0.0230 ms 47.6%
+  triton_convolution_2 0.0385 ms 28.4%
+  triton_convolution_1 0.0533 ms 20.5%
+SingleProcess AUTOTUNE takes 4.3184 seconds
+AUTOTUNE convolution(1x64x32x128, 64x64x3x3)
+  convolution 0.0260 ms 100.0%
+  triton_convolution_11 0.0331 ms 78.8%
+  triton_convolution_10 0.0358 ms 72.7%
+  triton_convolution_9 0.0403 ms 64.6%
+  triton_convolution_6 0.0405 ms 64.2%
+  triton_convolution_12 0.0748 ms 34.8%
+  triton_convolution_7 0.1463 ms 17.8%
+  triton_convolution_8 0.1648 ms 15.8%
+SingleProcess AUTOTUNE takes 5.1706 seconds
+AUTOTUNE convolution(1x64x16x64, 128x64x3x3)
+  convolution 0.0255 ms 100.0%
+  triton_convolution_17 0.0349 ms 73.1%
+  triton_convolution_18 0.0378 ms 67.6%
+  triton_convolution_13 0.0481 ms 53.1%
+  triton_convolution_16 0.0535 ms 47.8%
+  triton_convolution_19 0.0740 ms 34.5%
+  triton_convolution_14 0.1316 ms 19.4%
+  triton_convolution_15 0.1622 ms 15.7%
+SingleProcess AUTOTUNE takes 5.8005 seconds
+AUTOTUNE convolution(1x128x16x64, 128x128x3x3)
+  convolution 0.0301 ms 100.0%
+  triton_convolution_24 0.0628 ms 48.0%
+  triton_convolution_25 0.0682 ms 44.2%
+  triton_convolution_20 0.0888 ms 33.9%
+  triton_convolution_23 0.1034 ms 29.2%
+  triton_convolution_26 0.1387 ms 21.7%
+  triton_convolution_21 0.2700 ms 11.2%
+  triton_convolution_22 0.3136 ms 9.6%
+SingleProcess AUTOTUNE takes 5.8914 seconds
+AUTOTUNE convolution(1x128x8x32, 256x128x3x3)
+  convolution 0.0301 ms 100.0%
+  triton_convolution_31 0.0639 ms 47.2%
+  triton_convolution_30 0.0982 ms 30.7%
+  triton_convolution_29 0.1083 ms 27.8%
+  triton_convolution_32 0.1106 ms 27.2%
+  triton_convolution_33 0.1372 ms 22.0%
+  triton_convolution_27 0.1403 ms 21.5%
+  triton_convolution_28 0.2647 ms 11.4%
+SingleProcess AUTOTUNE takes 6.0502 seconds
+AUTOTUNE convolution(1x256x8x32, 256x256x3x3)
+  convolution 0.0475 ms 100.0%
+  triton_convolution_38 0.1203 ms 39.5%
+  triton_convolution_37 0.1997 ms 23.8%
+  triton_convolution_36 0.2080 ms 22.8%
+  triton_convolution_39 0.2118 ms 22.4%
+  triton_convolution_40 0.2637 ms 18.0%
+  triton_convolution_34 0.2746 ms 17.3%
+  triton_convolution_35 0.5518 ms 8.6%
+SingleProcess AUTOTUNE takes 6.5770 seconds
+AUTOTUNE convolution(1x256x4x32, 512x256x3x3)
+  convolution 0.0454 ms 100.0%
+  triton_convolution_52 0.1225 ms 37.1%
+  triton_convolution_54 0.1385 ms 32.8%
+  triton_convolution_51 0.1896 ms 24.0%
+  triton_convolution_50 0.2007 ms 22.6%
+  triton_convolution_49 0.2012 ms 22.6%
+  triton_convolution_53 0.2131 ms 21.3%
+  triton_convolution_48 0.2834 ms 16.0%
+SingleProcess AUTOTUNE takes 5.0601 seconds
+AUTOTUNE convolution(1x512x4x32, 512x512x3x3)
+  convolution 0.0502 ms 100.0%
+  triton_convolution_59 0.2389 ms 21.0%
+  triton_convolution_61 0.2687 ms 18.7%
+  triton_convolution_57 0.2927 ms 17.1%
+  triton_convolution_58 0.3947 ms 12.7%
+  triton_convolution_60 0.4206 ms 11.9%
+  triton_convolution_56 0.4372 ms 11.5%
+  triton_convolution_55 0.6253 ms 8.0%
+SingleProcess AUTOTUNE takes 5.8892 seconds
+AUTOTUNE convolution(1x512x2x32, 512x512x3x3)
+  convolution 0.0520 ms 100.0%
+  triton_convolution_75 0.2162 ms 24.1%
+  triton_convolution_73 0.2371 ms 21.9%
+  triton_convolution_71 0.2507 ms 20.8%
+  triton_convolution_72 0.2613 ms 19.9%
+  triton_convolution_70 0.3133 ms 16.6%
+  triton_convolution_74 0.4272 ms 12.2%
+  triton_convolution_69 0.6131 ms 8.5%
+SingleProcess AUTOTUNE takes 4.8147 seconds
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/doctr/models/recognition/crnn/pytorch.py", line 212, in forward
+    logits = self.linear(logits)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 249, in impl
+    self.push(fn_var.call_function(self, self.popn(nargs), {}))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in call_function
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in <listcomp>
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 56, in realize
+    self._cache.realize(self.parents_tracker)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 22, in realize
+    self.vt = VariableBuilder(tx, self.source)(self.value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 380, in _wrap
+    return type_dispatch(self, value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 824, in wrap_listlike
+    output = [
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 825, in <listcomp>
+    VariableBuilder(self.tx, GetItemSource(self.get_source(), i))(item)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 395, in _wrap
+    return self.wrap_tensor(value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1049, in wrap_tensor
+    tensor_variable = wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/ao/torchao/quantization/subclass.py", line 154, in __torch_dispatch__
+    new = args[0]._change_shape(args[0].shape[::-1])
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  drq                                
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 113.63it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 120.53it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 120.42it/s]
+17.111x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  fastNLP_Bert                       
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/models/bert.py", line 265, in forward
+    sequence_output = self.bert(words)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/embeddings/bert_embedding.py", line 137, in forward
+    outputs = self.model(words)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/embeddings/bert_embedding.py", line 445, in forward
+    max_word_piece_length = batch_word_pieces_length.sum(dim=-1).max().item()  # 表示word piece的长度(包括padding)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/embeddings/bert_embedding.py", line 482, in resume_in_forward
+    bert_outputs, pooled_cls = self.encoder(word_pieces, token_type_ids=token_type_ids, attention_mask=attn_masks,
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/modules/encoder/bert.py", line 509, in forward
+    extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  functorch_dp_cifar10               
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 176.49it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 178.98it/s]
+6.657x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/functorch_maml_omniglot/__init__.py", line 73, in __init__
+    self.meta_inputs = torch.load(f'{root}/maml_omniglot/batch.pt')
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 998, in load
+    with _open_file_like(f, 'rb') as opened_file:
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 445, in _open_file_like
+    return _open_file(name_or_buffer, mode)
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 426, in __init__
+    super().__init__(open(name, mode))
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/batch.pt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  hf_Albert                          
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/albert/modeling_albert.py", line 986, in forward
+    outputs = self.albert(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/albert/modeling_albert.py", line 725, in forward
+    extended_attention_mask = extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+cuda eval  hf_Bart                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:03<01:42,  3.53s/it]
running benchmark:   7%|▋         | 2/30 [00:07<01:38,  3.50s/it]
running benchmark:  10%|█         | 3/30 [00:10<01:34,  3.49s/it]
running benchmark:  13%|█▎        | 4/30 [00:14<01:31,  3.50s/it]
running benchmark:  17%|█▋        | 5/30 [00:17<01:27,  3.51s/it]
running benchmark:  20%|██        | 6/30 [00:21<01:24,  3.52s/it]
running benchmark:  23%|██▎       | 7/30 [00:24<01:20,  3.51s/it]
running benchmark:  27%|██▋       | 8/30 [00:28<01:17,  3.51s/it]
running benchmark:  30%|███       | 9/30 [00:31<01:13,  3.50s/it]
running benchmark:  33%|███▎      | 10/30 [00:35<01:09,  3.49s/it]
running benchmark:  37%|███▋      | 11/30 [00:38<01:06,  3.50s/it]
running benchmark:  40%|████      | 12/30 [00:42<01:02,  3.49s/it]
running benchmark:  43%|████▎     | 13/30 [00:45<00:59,  3.49s/it]
running benchmark:  47%|████▋     | 14/30 [00:48<00:55,  3.49s/it]
running benchmark:  50%|█████     | 15/30 [00:52<00:52,  3.49s/it]
running benchmark:  53%|█████▎    | 16/30 [00:55<00:48,  3.49s/it]
running benchmark:  57%|█████▋    | 17/30 [00:59<00:45,  3.49s/it]
running benchmark:  60%|██████    | 18/30 [01:02<00:42,  3.50s/it]
running benchmark:  63%|██████▎   | 19/30 [01:06<00:38,  3.50s/it]
running benchmark:  67%|██████▋   | 20/30 [01:09<00:34,  3.49s/it]
running benchmark:  70%|███████   | 21/30 [01:13<00:31,  3.50s/it]
running benchmark:  73%|███████▎  | 22/30 [01:17<00:28,  3.50s/it]
running benchmark:  77%|███████▋  | 23/30 [01:20<00:24,  3.50s/it]
running benchmark:  80%|████████  | 24/30 [01:24<00:21,  3.51s/it]
running benchmark:  83%|████████▎ | 25/30 [01:27<00:17,  3.52s/it]
running benchmark:  87%|████████▋ | 26/30 [01:31<00:14,  3.52s/it]
running benchmark:  90%|█████████ | 27/30 [01:34<00:10,  3.51s/it]
running benchmark:  93%|█████████▎| 28/30 [01:38<00:07,  3.52s/it]
running benchmark:  97%|█████████▋| 29/30 [01:41<00:03,  3.51s/it]
running benchmark: 100%|██████████| 30/30 [01:45<00:00,  3.50s/it]
running benchmark: 100%|██████████| 30/30 [01:45<00:00,  3.50s/it]
+949.965x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+cuda eval  hf_BigBird                         
+[2023-12-06 15:07:50,506] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:07:53,071] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:07:54,898] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:07:56,383] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:07:57,869] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:07:59,377] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:08:00,869] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:08:02,366] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:08:03,834] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:08:05,307] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:08:06,795] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 15:08:08,273] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+skipping cudagraphs due to ['non-cuda device in graph']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:32<15:52, 32.86s/it]
running benchmark:   7%|▋         | 2/30 [01:03<14:50, 31.82s/it]
running benchmark:  10%|█         | 3/30 [01:34<14:05, 31.32s/it]
running benchmark:  13%|█▎        | 4/30 [02:05<13:27, 31.06s/it]
running benchmark:  17%|█▋        | 5/30 [02:36<12:56, 31.07s/it]
running benchmark:  20%|██        | 6/30 [03:07<12:25, 31.08s/it]
running benchmark:  23%|██▎       | 7/30 [03:39<11:58, 31.24s/it]
running benchmark:  27%|██▋       | 8/30 [04:10<11:30, 31.36s/it]
running benchmark:  30%|███       | 9/30 [04:42<11:00, 31.43s/it]
running benchmark:  33%|███▎      | 10/30 [05:13<10:28, 31.41s/it]
running benchmark:  37%|███▋      | 11/30 [05:45<09:58, 31.49s/it]
running benchmark:  40%|████      | 12/30 [06:16<09:27, 31.52s/it]
running benchmark:  43%|████▎     | 13/30 [06:48<08:56, 31.53s/it]
running benchmark:  47%|████▋     | 14/30 [07:20<08:26, 31.65s/it]
running benchmark:  50%|█████     | 15/30 [07:52<07:54, 31.65s/it]
running benchmark:  53%|█████▎    | 16/30 [08:23<07:23, 31.65s/it]
running benchmark:  57%|█████▋    | 17/30 [08:55<06:51, 31.64s/it]
running benchmark:  60%|██████    | 18/30 [09:26<06:17, 31.42s/it]
running benchmark:  63%|██████▎   | 19/30 [09:57<05:44, 31.28s/it]
running benchmark:  67%|██████▋   | 20/30 [10:29<05:17, 31.71s/it]
running benchmark:  70%|███████   | 21/30 [11:00<04:43, 31.51s/it]
running benchmark:  73%|███████▎  | 22/30 [11:32<04:11, 31.40s/it]
running benchmark:  77%|███████▋  | 23/30 [12:02<03:38, 31.21s/it]
running benchmark:  80%|████████  | 24/30 [12:33<03:06, 31.10s/it]
running benchmark:  83%|████████▎ | 25/30 [13:04<02:35, 31.01s/it]
running benchmark:  87%|████████▋ | 26/30 [13:35<02:03, 30.98s/it]
running benchmark:  90%|█████████ | 27/30 [14:06<01:33, 31.03s/it]
running benchmark:  93%|█████████▎| 28/30 [14:37<01:01, 30.99s/it]
running benchmark:  97%|█████████▋| 29/30 [15:08<00:31, 31.09s/it]
running benchmark: 100%|██████████| 30/30 [15:40<00:00, 31.20s/it]
running benchmark: 100%|██████████| 30/30 [15:40<00:00, 31.34s/it]
+1496.081x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  hf_DistilBert                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:02<01:04,  2.23s/it]
running benchmark:   7%|▋         | 2/30 [00:04<01:02,  2.23s/it]
running benchmark:  10%|█         | 3/30 [00:06<00:59,  2.21s/it]
running benchmark:  13%|█▎        | 4/30 [00:08<00:56,  2.19s/it]
running benchmark:  17%|█▋        | 5/30 [00:10<00:54,  2.18s/it]
running benchmark:  20%|██        | 6/30 [00:13<00:51,  2.16s/it]
running benchmark:  23%|██▎       | 7/30 [00:15<00:50,  2.18s/it]
running benchmark:  27%|██▋       | 8/30 [00:17<00:47,  2.18s/it]
running benchmark:  30%|███       | 9/30 [00:19<00:45,  2.17s/it]
running benchmark:  33%|███▎      | 10/30 [00:21<00:43,  2.19s/it]
running benchmark:  37%|███▋      | 11/30 [00:24<00:41,  2.18s/it]
running benchmark:  40%|████      | 12/30 [00:26<00:39,  2.18s/it]
running benchmark:  43%|████▎     | 13/30 [00:28<00:36,  2.17s/it]
running benchmark:  47%|████▋     | 14/30 [00:30<00:34,  2.17s/it]
running benchmark:  50%|█████     | 15/30 [00:32<00:32,  2.16s/it]
running benchmark:  53%|█████▎    | 16/30 [00:34<00:30,  2.15s/it]
running benchmark:  57%|█████▋    | 17/30 [00:36<00:27,  2.15s/it]
running benchmark:  60%|██████    | 18/30 [00:39<00:25,  2.14s/it]
running benchmark:  63%|██████▎   | 19/30 [00:41<00:23,  2.14s/it]
running benchmark:  67%|██████▋   | 20/30 [00:43<00:21,  2.13s/it]
running benchmark:  70%|███████   | 21/30 [00:45<00:19,  2.13s/it]
running benchmark:  73%|███████▎  | 22/30 [00:47<00:17,  2.14s/it]
running benchmark:  77%|███████▋  | 23/30 [00:49<00:14,  2.14s/it]
running benchmark:  80%|████████  | 24/30 [00:51<00:12,  2.14s/it]
running benchmark:  83%|████████▎ | 25/30 [00:53<00:10,  2.13s/it]
running benchmark:  87%|████████▋ | 26/30 [00:56<00:08,  2.13s/it]
running benchmark:  90%|█████████ | 27/30 [00:58<00:06,  2.13s/it]
running benchmark:  93%|█████████▎| 28/30 [01:00<00:04,  2.13s/it]
running benchmark:  97%|█████████▋| 29/30 [01:02<00:02,  2.13s/it]
running benchmark: 100%|██████████| 30/30 [01:04<00:00,  2.14s/it]
running benchmark: 100%|██████████| 30/30 [01:04<00:00,  2.15s/it]
+1231.677x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  hf_GPT2                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:07<03:47,  7.84s/it]
running benchmark:   7%|▋         | 2/30 [00:15<03:37,  7.77s/it]
running benchmark:  10%|█         | 3/30 [00:23<03:31,  7.82s/it]
running benchmark:  13%|█▎        | 4/30 [00:30<03:15,  7.52s/it]
running benchmark:  17%|█▋        | 5/30 [00:37<03:07,  7.48s/it]
running benchmark:  20%|██        | 6/30 [00:45<02:58,  7.45s/it]
running benchmark:  23%|██▎       | 7/30 [00:52<02:50,  7.43s/it]
running benchmark:  27%|██▋       | 8/30 [01:00<02:43,  7.42s/it]
running benchmark:  30%|███       | 9/30 [01:07<02:35,  7.42s/it]
running benchmark:  33%|███▎      | 10/30 [01:14<02:28,  7.40s/it]
running benchmark:  37%|███▋      | 11/30 [01:22<02:20,  7.41s/it]
running benchmark:  40%|████      | 12/30 [01:29<02:13,  7.42s/it]
running benchmark:  43%|████▎     | 13/30 [01:37<02:06,  7.43s/it]
running benchmark:  47%|████▋     | 14/30 [01:44<01:59,  7.45s/it]
running benchmark:  50%|█████     | 15/30 [01:52<01:52,  7.47s/it]
running benchmark:  53%|█████▎    | 16/30 [01:59<01:45,  7.51s/it]
running benchmark:  57%|█████▋    | 17/30 [02:07<01:37,  7.52s/it]
running benchmark:  60%|██████    | 18/30 [02:14<01:30,  7.51s/it]
running benchmark:  63%|██████▎   | 19/30 [02:22<01:22,  7.48s/it]
running benchmark:  67%|██████▋   | 20/30 [02:30<01:15,  7.60s/it]
running benchmark:  70%|███████   | 21/30 [02:38<01:09,  7.69s/it]
running benchmark:  73%|███████▎  | 22/30 [02:45<01:01,  7.73s/it]
running benchmark:  77%|███████▋  | 23/30 [02:53<00:54,  7.77s/it]
running benchmark:  80%|████████  | 24/30 [03:01<00:46,  7.80s/it]
running benchmark:  83%|████████▎ | 25/30 [03:09<00:39,  7.83s/it]
running benchmark:  87%|████████▋ | 26/30 [03:17<00:31,  7.85s/it]
running benchmark:  90%|█████████ | 27/30 [03:25<00:23,  7.88s/it]
running benchmark:  93%|█████████▎| 28/30 [03:33<00:15,  7.89s/it]
running benchmark:  97%|█████████▋| 29/30 [03:41<00:07,  7.90s/it]
running benchmark: 100%|██████████| 30/30 [03:49<00:00,  7.89s/it]
running benchmark: 100%|██████████| 30/30 [03:49<00:00,  7.63s/it]
+1953.228x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:19, ?it/s]
+cuda eval  hf_GPT2_large                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:11<05:46, 11.97s/it]
running benchmark:   7%|▋         | 2/30 [00:23<05:30, 11.81s/it]
running benchmark:  10%|█         | 3/30 [00:35<05:19, 11.84s/it]
running benchmark:  13%|█▎        | 4/30 [00:47<05:08, 11.86s/it]
running benchmark:  17%|█▋        | 5/30 [00:59<04:56, 11.88s/it]
running benchmark:  20%|██        | 6/30 [01:11<04:44, 11.86s/it]
running benchmark:  23%|██▎       | 7/30 [01:23<04:33, 11.87s/it]
running benchmark:  27%|██▋       | 8/30 [01:34<04:21, 11.88s/it]
running benchmark:  30%|███       | 9/30 [01:46<04:09, 11.90s/it]
running benchmark:  33%|███▎      | 10/30 [01:58<03:57, 11.90s/it]
running benchmark:  37%|███▋      | 11/30 [02:10<03:46, 11.92s/it]
running benchmark:  40%|████      | 12/30 [02:22<03:34, 11.93s/it]
running benchmark:  43%|████▎     | 13/30 [02:34<03:23, 11.96s/it]
running benchmark:  47%|████▋     | 14/30 [02:46<03:11, 11.97s/it]
running benchmark:  50%|█████     | 15/30 [02:58<03:00, 12.00s/it]
running benchmark:  53%|█████▎    | 16/30 [03:10<02:48, 12.03s/it]
running benchmark:  57%|█████▋    | 17/30 [03:22<02:36, 12.04s/it]
running benchmark:  60%|██████    | 18/30 [03:34<02:24, 12.02s/it]
running benchmark:  63%|██████▎   | 19/30 [03:46<02:12, 12.02s/it]
running benchmark:  67%|██████▋   | 20/30 [03:59<02:00, 12.04s/it]
running benchmark:  70%|███████   | 21/30 [04:11<01:48, 12.06s/it]
running benchmark:  73%|███████▎  | 22/30 [04:23<01:36, 12.08s/it]
running benchmark:  77%|███████▋  | 23/30 [04:35<01:24, 12.08s/it]
running benchmark:  80%|████████  | 24/30 [04:47<01:12, 12.06s/it]
running benchmark:  83%|████████▎ | 25/30 [04:59<01:00, 12.04s/it]
running benchmark:  87%|████████▋ | 26/30 [05:11<00:48, 12.02s/it]
running benchmark:  90%|█████████ | 27/30 [05:23<00:35, 11.99s/it]
running benchmark:  93%|█████████▎| 28/30 [05:35<00:23, 11.98s/it]
running benchmark:  97%|█████████▋| 29/30 [05:47<00:11, 12.00s/it]
running benchmark: 100%|██████████| 30/30 [05:59<00:00, 12.00s/it]
running benchmark: 100%|██████████| 30/30 [05:59<00:00, 11.98s/it]
+706.528x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  hf_Longformer                      
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/longformer/modeling_longformer.py", line 1844, in forward
+    outputs = self.longformer(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/longformer/modeling_longformer.py", line 1739, in forward
+    extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape)[
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  hf_Reformer                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:02, 13.37it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:01, 13.68it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:01, 13.69it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 13.81it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:01, 13.82it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:01, 13.89it/s]
running benchmark:  47%|████▋     | 14/30 [00:01<00:01, 13.79it/s]
running benchmark:  53%|█████▎    | 16/30 [00:01<00:01, 13.60it/s]
running benchmark:  60%|██████    | 18/30 [00:01<00:00, 13.50it/s]
running benchmark:  67%|██████▋   | 20/30 [00:01<00:00, 13.57it/s]
running benchmark:  73%|███████▎  | 22/30 [00:01<00:00, 13.71it/s]
running benchmark:  80%|████████  | 24/30 [00:01<00:00, 13.81it/s]
running benchmark:  87%|████████▋ | 26/30 [00:01<00:00, 13.74it/s]
running benchmark:  93%|█████████▎| 28/30 [00:02<00:00, 13.74it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 13.72it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 13.71it/s]
+20.614x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  hf_T5                              
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1709, in forward
+    encoder_outputs = self.encoder(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+cuda eval  hf_T5_base                         
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1709, in forward
+    encoder_outputs = self.encoder(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  hf_T5_generate                     
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 221, in forward
+    return self.model.generate(inputs, self.generation_config)
+  File "/home/cdhernandez/local/pytorch/torch/utils/_contextlib.py", line 115, in decorate_context
+    return func(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1426, in generate
+    generation_config = copy.deepcopy(generation_config)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1429, in resume_in_generate
+    self._validate_model_kwargs(model_kwargs.copy())
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1429, in resume_in_generate
+    self._validate_model_kwargs(model_kwargs.copy())
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1432, in resume_in_generate
+    logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1433, in resume_in_generate
+    stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1492, in resume_in_generate
+    model_kwargs = self._prepare_encoder_decoder_kwargs_for_generation(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 661, in _prepare_encoder_decoder_kwargs_for_generation
+    model_kwargs["encoder_outputs"]: ModelOutput = encoder(**encoder_kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:18, ?it/s]
+cuda eval  hf_T5_large                        
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1709, in forward
+    encoder_outputs = self.encoder(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  hf_Whisper                         
+AUTOTUNE int_mm(12000x256, 256x256, 12000x256)
+  triton_mm_8 0.0215 ms 100.0%
+  triton_mm_9 0.0220 ms 97.5%
+  triton_mm_10 0.0220 ms 97.4%
+  triton_mm_7 0.0222 ms 96.9%
+  triton_mm_2 0.0222 ms 96.5%
+  triton_mm_4 0.0223 ms 96.1%
+  triton_mm_1 0.0235 ms 91.5%
+  triton_mm_0 0.0240 ms 89.5%
+  triton_mm_3 0.0273 ms 78.6%
+  triton_mm_5 0.0397 ms 54.0%
+SingleProcess AUTOTUNE takes 7.0686 seconds
+AUTOTUNE int_mm(12000x256, 256x1536, 12000x1536)
+  triton_mm_45 0.0716 ms 100.0%
+  triton_mm_46 0.0724 ms 98.9%
+  triton_mm_51 0.0811 ms 88.3%
+  triton_mm_44 0.0813 ms 88.0%
+  triton_mm_48 0.0836 ms 85.7%
+  triton_mm_52 0.0846 ms 84.6%
+  triton_mm_54 0.0861 ms 83.1%
+  triton_mm_53 0.0867 ms 82.5%
+  triton_mm_47 0.0900 ms 79.5%
+  triton_mm_49 0.2009 ms 35.6%
+SingleProcess AUTOTUNE takes 7.1610 seconds
+AUTOTUNE int_mm(12000x1536, 1536x256, 12000x256)
+  triton_mm_64 0.0374 ms 100.0%
+  triton_mm_65 0.0392 ms 95.6%
+  triton_mm_63 0.0537 ms 69.8%
+  triton_mm_57 0.0627 ms 59.7%
+  triton_mm_59 0.0629 ms 59.5%
+  triton_mm_62 0.0636 ms 58.8%
+  triton_mm_56 0.0664 ms 56.4%
+  triton_mm_58 0.0697 ms 53.7%
+  triton_mm_55 0.0769 ms 48.7%
+  triton_mm_60 0.1393 ms 26.9%
+SingleProcess AUTOTUNE takes 7.2671 seconds
+AUTOTUNE int_mm(8x256, 256x2, 8x2)
+  triton_mm_411 0.0074 ms 100.0%
+  triton_mm_412 0.0077 ms 96.3%
+  triton_mm_410 0.0081 ms 91.2%
+  triton_mm_409 0.0083 ms 89.2%
+  triton_mm_408 0.0085 ms 87.5%
+  triton_mm_407 0.0100 ms 74.1%
+SingleProcess AUTOTUNE takes 2.1472 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:02, 12.10it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:02, 12.47it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:01, 12.65it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 12.80it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:01, 12.88it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:01, 12.93it/s]
running benchmark:  47%|████▋     | 14/30 [00:01<00:01, 12.99it/s]
running benchmark:  53%|█████▎    | 16/30 [00:01<00:01, 13.02it/s]
running benchmark:  60%|██████    | 18/30 [00:01<00:00, 12.90it/s]
running benchmark:  67%|██████▋   | 20/30 [00:01<00:00, 12.91it/s]
running benchmark:  73%|███████▎  | 22/30 [00:01<00:00, 12.97it/s]
running benchmark:  80%|████████  | 24/30 [00:01<00:00, 13.05it/s]
running benchmark:  87%|████████▋ | 26/30 [00:02<00:00, 13.03it/s]
running benchmark:  93%|█████████▎| 28/30 [00:02<00:00, 13.08it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 13.03it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 12.93it/s]
+20.518x
+
loading model: 0it [00:00, ?it/s]
+
Downloading config.json:   0%|          | 0.00/4.19k [00:00<?, ?B/s]
Downloading config.json: 100%|██████████| 4.19k/4.19k [00:00<00:00, 20.6MB/s]
+
+
Downloading pytorch_model.bin:   0%|          | 0.00/605M [00:00<?, ?B/s]
+
Downloading pytorch_model.bin:   3%|▎         | 21.0M/605M [00:00<00:03, 173MB/s]
+
Downloading pytorch_model.bin:   7%|▋         | 41.9M/605M [00:00<00:03, 171MB/s]
+
Downloading pytorch_model.bin:  12%|█▏        | 73.4M/605M [00:00<00:02, 228MB/s]
+
Downloading pytorch_model.bin:  19%|█▉        | 115M/605M [00:00<00:01, 271MB/s] 
+
Downloading pytorch_model.bin:  26%|██▌       | 157M/605M [00:00<00:01, 293MB/s]
+
Downloading pytorch_model.bin:  31%|███       | 189M/605M [00:00<00:01, 294MB/s]
+
Downloading pytorch_model.bin:  36%|███▋      | 220M/605M [00:00<00:01, 297MB/s]
+
Downloading pytorch_model.bin:  42%|████▏     | 252M/605M [00:00<00:01, 297MB/s]
+
Downloading pytorch_model.bin:  47%|████▋     | 283M/605M [00:01<00:01, 299MB/s]
+
Downloading pytorch_model.bin:  52%|█████▏    | 315M/605M [00:01<00:00, 302MB/s]
+
Downloading pytorch_model.bin:  57%|█████▋    | 346M/605M [00:01<00:00, 305MB/s]
+
Downloading pytorch_model.bin:  62%|██████▏   | 377M/605M [00:01<00:00, 307MB/s]
+
Downloading pytorch_model.bin:  69%|██████▉   | 419M/605M [00:01<00:00, 309MB/s]
+
Downloading pytorch_model.bin:  76%|███████▌  | 461M/605M [00:01<00:00, 320MB/s]
+
Downloading pytorch_model.bin:  83%|████████▎ | 503M/605M [00:01<00:00, 332MB/s]
+
Downloading pytorch_model.bin:  90%|█████████ | 545M/605M [00:01<00:00, 313MB/s]
+
Downloading pytorch_model.bin:  97%|█████████▋| 587M/605M [00:01<00:00, 310MB/s]
Downloading pytorch_model.bin: 100%|██████████| 605M/605M [00:02<00:00, 297MB/s]
+
+
Downloading (…)rocessor_config.json:   0%|          | 0.00/316 [00:00<?, ?B/s]
Downloading (…)rocessor_config.json: 100%|██████████| 316/316 [00:00<00:00, 3.05MB/s]
+
+
Downloading tokenizer_config.json:   0%|          | 0.00/568 [00:00<?, ?B/s]
Downloading tokenizer_config.json: 100%|██████████| 568/568 [00:00<00:00, 5.94MB/s]
+
+
Downloading vocab.json:   0%|          | 0.00/862k [00:00<?, ?B/s]
Downloading vocab.json: 100%|██████████| 862k/862k [00:00<00:00, 16.5MB/s]
+
+
Downloading merges.txt:   0%|          | 0.00/525k [00:00<?, ?B/s]
Downloading merges.txt: 100%|██████████| 525k/525k [00:00<00:00, 65.4MB/s]
+
+
Downloading tokenizer.json:   0%|          | 0.00/2.22M [00:00<?, ?B/s]
Downloading tokenizer.json: 100%|██████████| 2.22M/2.22M [00:00<00:00, 49.2MB/s]
+
+
Downloading (…)cial_tokens_map.json:   0%|          | 0.00/389 [00:00<?, ?B/s]
Downloading (…)cial_tokens_map.json: 100%|██████████| 389/389 [00:00<00:00, 4.04MB/s]
+
loading model: 0it [00:08, ?it/s]
+WARNING:root:hf_clip failed to load
+Original Error: 'str' object has no attribute 'shape'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 1142, in forward
+    vision_outputs = self.vision_model(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 883, in forward
+    hidden_states = self.embeddings(pixel_values)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 194, in forward
+    batch_size = pixel_values.shape[0]
+AttributeError: 'str' object has no attribute 'shape'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  lennard_jones                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 1411.51it/s]
+1.414x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  llama                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  8.55it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:03,  8.84it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:03,  8.85it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:02,  8.87it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02,  8.92it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:02,  8.99it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:02,  9.02it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:02,  9.00it/s]
running benchmark:  30%|███       | 9/30 [00:01<00:02,  9.03it/s]
running benchmark:  33%|███▎      | 10/30 [00:01<00:02,  9.04it/s]
running benchmark:  37%|███▋      | 11/30 [00:01<00:02,  9.03it/s]
running benchmark:  40%|████      | 12/30 [00:01<00:02,  8.98it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01,  8.93it/s]
running benchmark:  47%|████▋     | 14/30 [00:01<00:01,  8.93it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01,  8.93it/s]
running benchmark:  53%|█████▎    | 16/30 [00:01<00:01,  8.94it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:01,  8.94it/s]
running benchmark:  60%|██████    | 18/30 [00:02<00:01,  8.90it/s]
running benchmark:  63%|██████▎   | 19/30 [00:02<00:01,  8.86it/s]
running benchmark:  67%|██████▋   | 20/30 [00:02<00:01,  8.86it/s]
running benchmark:  70%|███████   | 21/30 [00:02<00:01,  8.80it/s]
running benchmark:  73%|███████▎  | 22/30 [00:02<00:00,  8.79it/s]
running benchmark:  77%|███████▋  | 23/30 [00:02<00:00,  8.81it/s]
running benchmark:  80%|████████  | 24/30 [00:02<00:00,  8.76it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00,  8.67it/s]
running benchmark:  87%|████████▋ | 26/30 [00:02<00:00,  8.60it/s]
running benchmark:  90%|█████████ | 27/30 [00:03<00:00,  8.54it/s]
running benchmark:  93%|█████████▎| 28/30 [00:03<00:00,  8.63it/s]
running benchmark:  97%|█████████▋| 29/30 [00:03<00:00,  8.78it/s]
running benchmark: 100%|██████████| 30/30 [00:03<00:00,  8.79it/s]
running benchmark: 100%|██████████| 30/30 [00:03<00:00,  8.85it/s]
+41.302x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:59, ?it/s]
+cuda eval  llama_v2_7b_16h                    
+AUTOTUNE int_mm(512x4096, 4096x4096, 512x4096)
+  triton_mm_9 0.0657 ms 100.0%
+  triton_mm_10 0.0664 ms 99.0%
+  triton_mm_8 0.0876 ms 75.0%
+  triton_mm_1 0.1089 ms 60.3%
+  triton_mm_2 0.1101 ms 59.7%
+  triton_mm_3 0.1126 ms 58.3%
+  triton_mm_4 0.1138 ms 57.7%
+  triton_mm_0 0.1225 ms 53.6%
+  triton_mm_7 0.1393 ms 47.2%
+  triton_mm_5 0.2387 ms 27.5%
+SingleProcess AUTOTUNE takes 7.5083 seconds
+AUTOTUNE int_mm(512x4096, 4096x11008, 512x11008)
+  triton_mm_53 0.1271 ms 100.0%
+  triton_mm_54 0.1277 ms 99.5%
+  triton_mm_52 0.1983 ms 64.1%
+  triton_mm_45 0.2272 ms 55.9%
+  triton_mm_46 0.2289 ms 55.5%
+  triton_mm_48 0.2433 ms 52.2%
+  triton_mm_47 0.2434 ms 52.2%
+  triton_mm_51 0.2653 ms 47.9%
+  triton_mm_44 0.2956 ms 43.0%
+  triton_mm_49 0.6084 ms 20.9%
+SingleProcess AUTOTUNE takes 7.1348 seconds
+AUTOTUNE int_mm(512x11008, 11008x4096, 512x4096)
+  triton_mm_75 0.1492 ms 100.0%
+  triton_mm_76 0.1496 ms 99.8%
+  triton_mm_74 0.2147 ms 69.5%
+  triton_mm_67 0.2449 ms 61.0%
+  triton_mm_68 0.2517 ms 59.3%
+  triton_mm_70 0.2774 ms 53.8%
+  triton_mm_69 0.2794 ms 53.4%
+  triton_mm_66 0.3099 ms 48.2%
+  triton_mm_73 0.3336 ms 44.7%
+  triton_mm_72 0.6237 ms 23.9%
+SingleProcess AUTOTUNE takes 7.4188 seconds
+AUTOTUNE int_mm(512x4096, 4096x32000, 512x32000)
+  triton_mm_1241 0.3076 ms 100.0%
+  triton_mm_1242 0.3095 ms 99.4%
+  triton_mm_1240 0.5504 ms 55.9%
+  triton_mm_1233 0.5870 ms 52.4%
+  triton_mm_1234 0.5894 ms 52.2%
+  triton_mm_1235 0.6339 ms 48.5%
+  triton_mm_1236 0.6344 ms 48.5%
+  triton_mm_1239 0.6526 ms 47.1%
+  triton_mm_1232 0.8036 ms 38.3%
+  triton_mm_1237 1.7366 ms 17.7%
+SingleProcess AUTOTUNE takes 7.3910 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:08,  3.33it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:07,  3.60it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:07,  3.72it/s]
running benchmark:  13%|█▎        | 4/30 [00:01<00:06,  3.78it/s]
running benchmark:  17%|█▋        | 5/30 [00:01<00:06,  3.81it/s]
running benchmark:  20%|██        | 6/30 [00:01<00:06,  3.82it/s]
running benchmark:  23%|██▎       | 7/30 [00:01<00:06,  3.83it/s]
running benchmark:  27%|██▋       | 8/30 [00:02<00:05,  3.83it/s]
running benchmark:  30%|███       | 9/30 [00:02<00:05,  3.84it/s]
running benchmark:  33%|███▎      | 10/30 [00:02<00:05,  3.84it/s]
running benchmark:  37%|███▋      | 11/30 [00:02<00:04,  3.84it/s]
running benchmark:  40%|████      | 12/30 [00:03<00:04,  3.85it/s]
running benchmark:  43%|████▎     | 13/30 [00:03<00:04,  3.84it/s]
running benchmark:  47%|████▋     | 14/30 [00:03<00:04,  3.85it/s]
running benchmark:  50%|█████     | 15/30 [00:03<00:03,  3.86it/s]
running benchmark:  53%|█████▎    | 16/30 [00:04<00:03,  3.86it/s]
running benchmark:  57%|█████▋    | 17/30 [00:04<00:03,  3.83it/s]
running benchmark:  60%|██████    | 18/30 [00:04<00:03,  3.82it/s]
running benchmark:  63%|██████▎   | 19/30 [00:04<00:02,  3.84it/s]
running benchmark:  67%|██████▋   | 20/30 [00:05<00:02,  3.85it/s]
running benchmark:  70%|███████   | 21/30 [00:05<00:02,  3.84it/s]
running benchmark:  73%|███████▎  | 22/30 [00:05<00:02,  3.86it/s]
running benchmark:  77%|███████▋  | 23/30 [00:06<00:01,  3.86it/s]
running benchmark:  80%|████████  | 24/30 [00:06<00:01,  3.86it/s]
running benchmark:  83%|████████▎ | 25/30 [00:06<00:01,  3.85it/s]
running benchmark:  87%|████████▋ | 26/30 [00:06<00:01,  3.85it/s]
running benchmark:  90%|█████████ | 27/30 [00:07<00:00,  3.84it/s]
running benchmark:  93%|█████████▎| 28/30 [00:07<00:00,  3.85it/s]
running benchmark:  97%|█████████▋| 29/30 [00:07<00:00,  3.83it/s]
running benchmark: 100%|██████████| 30/30 [00:07<00:00,  3.83it/s]
running benchmark: 100%|██████████| 30/30 [00:07<00:00,  3.83it/s]
+16.801x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/__init__.py", line 78, in __init__
+    self.meta_inputs = torch.load(f'{root}/batch.pt')
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 998, in load
+    with _open_file_like(f, 'rb') as opened_file:
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 445, in _open_file_like
+    return _open_file(name_or_buffer, mode)
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 426, in __init__
+    super().__init__(open(name, mode))
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/batch.pt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  mnasnet1_0                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 97.96it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 102.18it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 102.75it/s]
+3.899x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  mobilenet_v2                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 52.96it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 59.88it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 62.22it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 63.30it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 62.16it/s]
+9.044x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+WARNING:root:mobilenet_v2_quantized_qat failed to load
+The eval test only supports CPU.
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/mobilenet_v2_quantized_qat/__init__.py", line 21, in __init__
+    raise NotImplementedError("The eval test only supports CPU.")
+NotImplementedError: The eval test only supports CPU.
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  mobilenet_v3_large                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 73.24it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 76.60it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 77.60it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 77.35it/s]
+5.919x
+
loading model: 0it [00:00, ?it/s]NCCL version 2.19.3+cuda12.0
+
loading model: 0it [00:04, ?it/s]
+cuda eval  moco                               
+--Return--
+> /home/cdhernandez/local/ao/torchao/quantization/subclass.py(196)__torch_dispatch__()->None
+-> breakpoint()
+(Pdb) TIMEOUT
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:10, ?it/s]
+number of parameters: 123.69M
+num decayed parameter tensors: 50, with 124,354,560 parameters
+num non-decayed parameter tensors: 98, with 121,344 parameters
+using fused AdamW: True
+cuda eval  nanogpt                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:14,  2.03it/s]
running benchmark:   7%|▋         | 2/30 [00:01<00:14,  1.88it/s]
running benchmark:  10%|█         | 3/30 [00:01<00:14,  1.85it/s]
running benchmark:  13%|█▎        | 4/30 [00:02<00:13,  1.90it/s]
running benchmark:  17%|█▋        | 5/30 [00:02<00:13,  1.89it/s]
running benchmark:  20%|██        | 6/30 [00:03<00:12,  1.93it/s]
running benchmark:  23%|██▎       | 7/30 [00:03<00:11,  1.94it/s]
running benchmark:  27%|██▋       | 8/30 [00:04<00:11,  1.96it/s]
running benchmark:  30%|███       | 9/30 [00:04<00:10,  2.08it/s]
running benchmark:  33%|███▎      | 10/30 [00:05<00:09,  2.09it/s]
running benchmark:  37%|███▋      | 11/30 [00:05<00:10,  1.88it/s]
running benchmark:  40%|████      | 12/30 [00:06<00:09,  1.82it/s]
running benchmark:  43%|████▎     | 13/30 [00:07<00:10,  1.65it/s]
running benchmark:  47%|████▋     | 14/30 [00:07<00:10,  1.51it/s]
running benchmark:  50%|█████     | 15/30 [00:08<00:09,  1.54it/s]
running benchmark:  53%|█████▎    | 16/30 [00:09<00:08,  1.60it/s]
running benchmark:  57%|█████▋    | 17/30 [00:09<00:07,  1.63it/s]
running benchmark:  60%|██████    | 18/30 [00:10<00:07,  1.60it/s]
running benchmark:  63%|██████▎   | 19/30 [00:10<00:06,  1.66it/s]
running benchmark:  67%|██████▋   | 20/30 [00:11<00:05,  1.70it/s]
running benchmark:  70%|███████   | 21/30 [00:12<00:05,  1.64it/s]
running benchmark:  73%|███████▎  | 22/30 [00:12<00:04,  1.73it/s]
running benchmark:  77%|███████▋  | 23/30 [00:13<00:03,  1.80it/s]
running benchmark:  80%|████████  | 24/30 [00:13<00:03,  1.93it/s]
running benchmark:  83%|████████▎ | 25/30 [00:13<00:02,  1.93it/s]
running benchmark:  87%|████████▋ | 26/30 [00:14<00:02,  1.95it/s]
running benchmark:  90%|█████████ | 27/30 [00:15<00:01,  1.80it/s]
running benchmark:  93%|█████████▎| 28/30 [00:15<00:01,  1.78it/s]
running benchmark:  97%|█████████▋| 29/30 [00:16<00:00,  1.74it/s]
running benchmark: 100%|██████████| 30/30 [00:16<00:00,  1.82it/s]
running benchmark: 100%|██████████| 30/30 [00:16<00:00,  1.79it/s]
+243.964x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:10, ?it/s]
+cuda eval  nvidia_deeprecommender             
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 144.67it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 158.60it/s]
+0.890x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  opacus_cifar10                     
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 702, in call_function
+    return variables.UserFunctionVariable(fn, source=source).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1167, in call_getattr
+    return obj.var_getattr(tx, name).clone(source=source)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/user_defined.py", line 504, in var_getattr
+    return VariableBuilder(tx, source)(subobj)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 395, in _wrap
+    return self.wrap_tensor(value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 971, in wrap_tensor
+    return self.tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/opacus/grad_sample/grad_sample_module.py", line 148, in forward
+    return self._module(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/vision/torchvision/models/resnet.py", line 285, in forward
+    return self._forward_impl(x)
+  File "/home/cdhernandez/local/vision/torchvision/models/resnet.py", line 280, in _forward_impl
+    x = self.fc(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1562, in _call_impl
+    result = forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-1_5:
+- configuration_phi.py
+. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.
+
+
Downloading modeling_phi.py:   0%|          | 0.00/33.4k [00:00<?, ?B/s]
Downloading modeling_phi.py: 100%|██████████| 33.4k/33.4k [00:00<00:00, 51.3MB/s]
+A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-1_5:
+- modeling_phi.py
+. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.
+
loading model: 0it [01:16, ?it/s]
+cuda eval  phi_1_5                            
+AUTOTUNE int_mm(512x2048, 2048x6144, 512x6144)
+  triton_mm_9 0.0425 ms 100.0%
+  triton_mm_10 0.0426 ms 99.7%
+  triton_mm_8 0.0714 ms 59.6%
+  triton_mm_1 0.0741 ms 57.3%
+  triton_mm_2 0.0756 ms 56.2%
+  triton_mm_7 0.0761 ms 55.8%
+  triton_mm_4 0.0765 ms 55.5%
+  triton_mm_3 0.0810 ms 52.5%
+  triton_mm_0 0.1144 ms 37.1%
+  triton_mm_5 0.1891 ms 22.5%
+SingleProcess AUTOTUNE takes 8.0958 seconds
+AUTOTUNE bmm(32x512x64, 32x64x512)
+  triton_bmm_13 0.0382 ms 100.0%
+  triton_bmm_12 0.0387 ms 98.9%
+  triton_bmm_19 0.0391 ms 97.7%
+  triton_bmm_11 0.0396 ms 96.4%
+  bmm 0.0420 ms 91.1%
+  triton_bmm_21 0.0420 ms 90.9%
+  triton_bmm_18 0.0491 ms 77.9%
+  triton_bmm_14 0.0505 ms 75.7%
+  triton_bmm_15 0.0514 ms 74.4%
+  triton_bmm_20 0.0536 ms 71.4%
+SingleProcess AUTOTUNE takes 6.6945 seconds
+AUTOTUNE bmm(32x512x512, 32x512x64)
+  triton_bmm_24 0.0262 ms 100.0%
+  triton_bmm_25 0.0265 ms 98.8%
+  triton_bmm_26 0.0281 ms 93.3%
+  triton_bmm_31 0.0284 ms 92.2%
+  triton_bmm_27 0.0287 ms 91.1%
+  triton_bmm_29 0.0295 ms 88.6%
+  bmm 0.0302 ms 86.7%
+  triton_bmm_28 0.0303 ms 86.3%
+  triton_bmm_32 0.0332 ms 78.9%
+  triton_bmm_23 0.0333 ms 78.6%
+SingleProcess AUTOTUNE takes 4.6003 seconds
+AUTOTUNE int_mm(512x2048, 2048x2048, 512x2048)
+  triton_mm_43 0.0318 ms 100.0%
+  triton_mm_45 0.0401 ms 79.4%
+  triton_mm_36 0.0409 ms 77.9%
+  triton_mm_37 0.0412 ms 77.4%
+  triton_mm_38 0.0426 ms 74.8%
+  triton_mm_44 0.0485 ms 65.6%
+  triton_mm_35 0.0510 ms 62.4%
+  triton_mm_42 0.0676 ms 47.1%
+  triton_mm_41 0.0677 ms 47.0%
+  triton_mm_40 0.0681 ms 46.8%
+SingleProcess AUTOTUNE takes 7.8055 seconds
+AUTOTUNE int_mm(512x2048, 2048x8192, 512x8192)
+  triton_mm_56 0.0773 ms 100.0%
+  triton_mm_55 0.0781 ms 99.0%
+  triton_mm_54 0.0879 ms 88.0%
+  triton_mm_50 0.0945 ms 81.9%
+  triton_mm_47 0.0980 ms 78.9%
+  triton_mm_48 0.1004 ms 77.0%
+  triton_mm_49 0.1015 ms 76.2%
+  triton_mm_46 0.1348 ms 57.4%
+  triton_mm_53 0.1366 ms 56.6%
+  triton_mm_52 0.2375 ms 32.6%
+SingleProcess AUTOTUNE takes 8.0539 seconds
+AUTOTUNE int_mm(512x8192, 8192x2048, 512x2048)
+  triton_mm_65 0.0965 ms 100.0%
+  triton_mm_66 0.1143 ms 84.5%
+  triton_mm_67 0.1152 ms 83.7%
+  triton_mm_60 0.1380 ms 69.9%
+  triton_mm_58 0.1386 ms 69.6%
+  triton_mm_59 0.1401 ms 68.9%
+  triton_mm_61 0.1476 ms 65.4%
+  triton_mm_57 0.1872 ms 51.5%
+  triton_mm_63 0.2335 ms 41.3%
+  triton_mm_62 0.2341 ms 41.2%
+SingleProcess AUTOTUNE takes 8.0376 seconds
+AUTOTUNE int_mm(512x2048, 2048x51200, 512x51200)
+  triton_mm_1641 0.2894 ms 100.0%
+  triton_mm_1642 0.2920 ms 99.1%
+  triton_mm_1633 0.4724 ms 61.3%
+  triton_mm_1640 0.4747 ms 61.0%
+  triton_mm_1634 0.4761 ms 60.8%
+  triton_mm_1636 0.5113 ms 56.6%
+  triton_mm_1635 0.5359 ms 54.0%
+  triton_mm_1639 0.5375 ms 53.8%
+  triton_mm_1632 0.6279 ms 46.1%
+  triton_mm_1638 1.4442 ms 20.0%
+SingleProcess AUTOTUNE takes 8.0618 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:07,  3.96it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:06,  4.24it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:06,  4.34it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:05,  4.36it/s]
running benchmark:  17%|█▋        | 5/30 [00:01<00:05,  4.44it/s]
running benchmark:  20%|██        | 6/30 [00:01<00:05,  4.47it/s]
running benchmark:  23%|██▎       | 7/30 [00:01<00:05,  4.48it/s]
running benchmark:  27%|██▋       | 8/30 [00:01<00:04,  4.51it/s]
running benchmark:  30%|███       | 9/30 [00:02<00:04,  4.47it/s]
running benchmark:  33%|███▎      | 10/30 [00:02<00:04,  4.44it/s]
running benchmark:  37%|███▋      | 11/30 [00:02<00:04,  4.41it/s]
running benchmark:  40%|████      | 12/30 [00:02<00:04,  4.42it/s]
running benchmark:  43%|████▎     | 13/30 [00:02<00:03,  4.42it/s]
running benchmark:  47%|████▋     | 14/30 [00:03<00:03,  4.42it/s]
running benchmark:  50%|█████     | 15/30 [00:03<00:03,  4.42it/s]
running benchmark:  53%|█████▎    | 16/30 [00:03<00:03,  4.48it/s]
running benchmark:  57%|█████▋    | 17/30 [00:03<00:02,  4.50it/s]
running benchmark:  60%|██████    | 18/30 [00:04<00:02,  4.52it/s]
running benchmark:  63%|██████▎   | 19/30 [00:04<00:02,  4.55it/s]
running benchmark:  67%|██████▋   | 20/30 [00:04<00:02,  4.57it/s]
running benchmark:  70%|███████   | 21/30 [00:04<00:01,  4.59it/s]
running benchmark:  73%|███████▎  | 22/30 [00:04<00:01,  4.57it/s]
running benchmark:  77%|███████▋  | 23/30 [00:05<00:01,  4.57it/s]
running benchmark:  80%|████████  | 24/30 [00:05<00:01,  4.52it/s]
running benchmark:  83%|████████▎ | 25/30 [00:05<00:01,  4.51it/s]
running benchmark:  87%|████████▋ | 26/30 [00:05<00:00,  4.51it/s]
running benchmark:  90%|█████████ | 27/30 [00:06<00:00,  4.50it/s]
running benchmark:  93%|█████████▎| 28/30 [00:06<00:00,  4.52it/s]
running benchmark:  97%|█████████▋| 29/30 [00:06<00:00,  4.50it/s]
running benchmark: 100%|██████████| 30/30 [00:06<00:00,  4.48it/s]
running benchmark: 100%|██████████| 30/30 [00:06<00:00,  4.47it/s]
+18.507x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  phlippe_densenet                   
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 55.69it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 71.68it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 76.01it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 74.64it/s]
+5.653x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  phlippe_resnet                     
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 174.28it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 176.63it/s]
+5.906x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+cuda eval  pyhpc_equation_of_state            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 222.45it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 224.27it/s]
+16.558x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  pyhpc_isoneutral_mixing            
+skipping cudagraphs due to ['mutated inputs']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 85.92it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 86.72it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 85.56it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 85.86it/s]
+6.051x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+
loading model: 0it [00:01, ?it/s]
+WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+cuda eval  pyhpc_turbulent_kinetic_energy     
+WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 66.45it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 78.75it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 82.89it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 81.77it/s]
+8.241x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/__init__.py", line 13, in <module>
+    from .train_cyclegan import prepare_training_loop
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/train_cyclegan.py", line 27, in <module>
+    from .util.visualizer import Visualizer
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/util/visualizer.py", line 6, in <module>
+    from . import util, html
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/util/html.py", line 1, in <module>
+    import dominate
+ModuleNotFoundError: No module named 'dominate'
+Failed to import user benchmark module dynamo, error: No module named 'dominate'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/__init__.py", line 52, in __init__
+    self.data_loader = self.get_data_loader(config)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/__init__.py", line 68, in get_data_loader
+    celeba_loader = get_loader(config.celeba_image_dir, config.attr_path, config.selected_attrs,
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 82, in get_loader
+    dataset = CelebA(image_dir, attr_path, selected_attrs, transform, mode)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 24, in __init__
+    self.preprocess()
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 33, in preprocess
+    lines = [line.rstrip() for line in open(self.attr_path, 'r')]
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/data/.data/pytorch_stargan_inputs/data/celeba/list_attr_celeba.txt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  pytorch_unet                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 34.79it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 41.35it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 43.55it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 44.64it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 45.29it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 45.68it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 44.37it/s]
+1.805x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  resnet152                          
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  7.81it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 21.54it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 26.32it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 28.64it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 29.64it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 30.19it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 30.13it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 30.38it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 28.42it/s]
+2.581x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  resnet18                           
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 121.34it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 126.80it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 125.39it/s]
+6.817x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  resnet50                           
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 65.87it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 71.73it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 73.48it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 73.04it/s]
+2.183x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+WARNING:root:resnet50_quantized_qat failed to load
+The eval test only supports CPU.
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/resnet50_quantized_qat/__init__.py", line 21, in __init__
+    raise NotImplementedError("The eval test only supports CPU.")
+NotImplementedError: The eval test only supports CPU.
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  resnext50_32x4d                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 56.70it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 58.33it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 58.90it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 59.19it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 59.34it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 58.96it/s]
+6.940x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/__init__.py", line 27, in __init__
+    self.model = sam_model_registry[model_type](checkpoint=sam_checkpoint)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/build_sam.py", line 19, in build_sam_vit_h
+    return _build_sam(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/build_sam.py", line 108, in _build_sam
+    with open(checkpoint, "rb") as f:
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/.data/sam_vit_h_4b8939.pth'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  shufflenet_v2_x1_0                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 74.39it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 79.19it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 80.82it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 80.31it/s]
+4.292x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  soft_actor_critic                  
+AUTOTUNE mm(256x3, 3x1024)
+  triton_mm_6 0.0067 ms 100.0%
+  triton_mm_8 0.0067 ms 100.0%
+  triton_mm_9 0.0067 ms 100.0%
+  triton_mm_0 0.0068 ms 99.5%
+  triton_mm_5 0.0069 ms 96.8%
+  triton_mm_4 0.0076 ms 88.6%
+  triton_mm_10 0.0076 ms 88.2%
+  mm 0.0078 ms 86.4%
+  triton_mm_1 0.0078 ms 86.1%
+  triton_mm_2 0.0078 ms 85.7%
+SingleProcess AUTOTUNE takes 3.8899 seconds
+AUTOTUNE int_mm(256x1024, 1024x1024, 256x1024)
+  triton_mm_19 0.0147 ms 100.0%
+  triton_mm_16 0.0199 ms 73.8%
+  triton_mm_17 0.0201 ms 73.1%
+  triton_mm_15 0.0204 ms 72.3%
+  triton_mm_14 0.0205 ms 71.7%
+  triton_mm_12 0.0233 ms 63.2%
+  triton_mm_13 0.0233 ms 63.1%
+  triton_mm_21 0.0274 ms 53.7%
+  triton_mm_11 0.0275 ms 53.6%
+  triton_mm_20 0.0281 ms 52.5%
+SingleProcess AUTOTUNE takes 7.3635 seconds
+AUTOTUNE int_mm(256x1024, 1024x2, 256x2)
+  triton_mm_31 0.0108 ms 100.0%
+  triton_mm_32 0.0125 ms 86.2%
+  triton_mm_28 0.0127 ms 84.6%
+  triton_mm_30 0.0132 ms 81.8%
+  triton_mm_27 0.0137 ms 78.7%
+  triton_mm_25 0.0147 ms 73.0%
+  triton_mm_23 0.0196 ms 54.8%
+  triton_mm_24 0.0207 ms 52.0%
+  triton_mm_26 0.0247 ms 43.6%
+  triton_mm_22 0.0252 ms 42.7%
+SingleProcess AUTOTUNE takes 3.8153 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 169.15it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 178.30it/s]
+13.441x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/speech_transformer/__init__.py", line 15, in <module>
+    from .config import SpeechTransformerTrainConfig, SpeechTransformerEvalConfig
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/speech_transformer/config.py", line 4, in <module>
+    import kaldi_io
+ModuleNotFoundError: No module named 'kaldi_io'
+Failed to import user benchmark module dynamo, error: No module named 'kaldi_io'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  squeezenet1_1                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 296.67it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 296.01it/s]
+3.640x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/stable_diffusion_text_encoder/__init__.py", line 11, in <module>
+    from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+ModuleNotFoundError: No module named 'diffusers'
+Failed to import user benchmark module dynamo, error: No module named 'diffusers'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/stable_diffusion_unet/__init__.py", line 11, in <module>
+    from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+ModuleNotFoundError: No module named 'diffusers'
+Failed to import user benchmark module dynamo, error: No module named 'diffusers'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/timm_efficientdet/__init__.py", line 12, in <module>
+    from effdet import create_model, create_loader
+ModuleNotFoundError: No module named 'effdet'
+Failed to import user benchmark module dynamo, error: No module named 'effdet'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  timm_efficientnet                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 39.49it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 47.40it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 50.38it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 51.52it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 51.93it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 50.68it/s]
+2.702x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  timm_nfnet                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:04,  5.88it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 10.75it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 12.62it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 13.58it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 14.11it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 14.44it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 14.65it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 14.78it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 14.87it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 14.95it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 15.00it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 15.04it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 15.06it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 15.06it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 15.06it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 14.34it/s]
+1.920x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  timm_regnet                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:09,  3.12it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 13.60it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 20.09it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 24.29it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 27.07it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 28.92it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 30.17it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 31.01it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 25.02it/s]
+1.459x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  timm_resnest                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 77.01it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 87.74it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 91.38it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 89.70it/s]
+2.054x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  timm_vision_transformer            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  9.24it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 10.09it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02, 10.19it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:02, 10.19it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:02, 10.25it/s]
running benchmark:  37%|███▋      | 11/30 [00:01<00:01, 10.30it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 10.31it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 10.33it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:01, 10.29it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:01, 10.19it/s]
running benchmark:  70%|███████   | 21/30 [00:02<00:00, 10.10it/s]
running benchmark:  77%|███████▋  | 23/30 [00:02<00:00, 10.11it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00, 10.17it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 10.20it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 10.29it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 10.23it/s]
+20.370x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:20, ?it/s]
+cuda eval  timm_vision_transformer_large      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:28,  1.03it/s]
running benchmark:   7%|▋         | 2/30 [00:01<00:22,  1.27it/s]
running benchmark:  10%|█         | 3/30 [00:02<00:19,  1.37it/s]
running benchmark:  13%|█▎        | 4/30 [00:02<00:18,  1.43it/s]
running benchmark:  17%|█▋        | 5/30 [00:03<00:17,  1.46it/s]
running benchmark:  20%|██        | 6/30 [00:04<00:16,  1.48it/s]
running benchmark:  23%|██▎       | 7/30 [00:04<00:15,  1.49it/s]
running benchmark:  27%|██▋       | 8/30 [00:05<00:14,  1.50it/s]
running benchmark:  30%|███       | 9/30 [00:06<00:13,  1.51it/s]
running benchmark:  33%|███▎      | 10/30 [00:06<00:13,  1.51it/s]
running benchmark:  37%|███▋      | 11/30 [00:07<00:12,  1.51it/s]
running benchmark:  40%|████      | 12/30 [00:08<00:11,  1.52it/s]
running benchmark:  43%|████▎     | 13/30 [00:08<00:11,  1.52it/s]
running benchmark:  47%|████▋     | 14/30 [00:09<00:10,  1.52it/s]
running benchmark:  50%|█████     | 15/30 [00:10<00:09,  1.52it/s]
running benchmark:  53%|█████▎    | 16/30 [00:10<00:09,  1.52it/s]
running benchmark:  57%|█████▋    | 17/30 [00:11<00:08,  1.52it/s]
running benchmark:  60%|██████    | 18/30 [00:12<00:07,  1.52it/s]
running benchmark:  63%|██████▎   | 19/30 [00:12<00:07,  1.52it/s]
running benchmark:  67%|██████▋   | 20/30 [00:13<00:06,  1.52it/s]
running benchmark:  70%|███████   | 21/30 [00:14<00:05,  1.52it/s]
running benchmark:  73%|███████▎  | 22/30 [00:14<00:05,  1.52it/s]
running benchmark:  77%|███████▋  | 23/30 [00:15<00:04,  1.52it/s]
running benchmark:  80%|████████  | 24/30 [00:16<00:03,  1.52it/s]
running benchmark:  83%|████████▎ | 25/30 [00:16<00:03,  1.52it/s]
running benchmark:  87%|████████▋ | 26/30 [00:17<00:02,  1.52it/s]
running benchmark:  90%|█████████ | 27/30 [00:18<00:01,  1.52it/s]
running benchmark:  93%|█████████▎| 28/30 [00:18<00:01,  1.52it/s]
running benchmark:  97%|█████████▋| 29/30 [00:19<00:00,  1.52it/s]
running benchmark: 100%|██████████| 30/30 [00:20<00:00,  1.52it/s]
running benchmark: 100%|██████████| 30/30 [00:20<00:00,  1.50it/s]
+6.676x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  timm_vovnet                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 64.21it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 73.87it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 76.79it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 75.92it/s]
+2.084x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/torch_multimodal_clip/__init__.py", line 27, in __init__
+    self.image = Image.open(os.path.join(self.data_folder, self.image_name))
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/PIL/Image.py", line 3218, in open
+    fp = builtins.open(filename, "rb")
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/torch_multimodal_clip/.data/pizza.jpg'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model tts_angular supports float32 only
+
loading model: 0it [00:00, ?it/s]
+WARNING:common:Model tts_angular supports float32 only
+cuda eval  tts_angular                        
+WARNING:common:Model tts_angular supports float32 only
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/tts_angular/model.py", line 59, in forward
+    d = self.layers(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/container.py", line 217, in forward
+    input = module(input)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/tts_angular/model.py", line 19, in forward
+    return self.linear(o)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 249, in impl
+    self.push(fn_var.call_function(self, self.popn(nargs), {}))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in call_function
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in <listcomp>
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 56, in realize
+    self._cache.realize(self.parents_tracker)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 22, in realize
+    self.vt = VariableBuilder(tx, self.source)(self.value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 380, in _wrap
+    return type_dispatch(self, value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 824, in wrap_listlike
+    output = [
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 825, in <listcomp>
+    VariableBuilder(self.tx, GetItemSource(self.get_source(), i))(item)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 395, in _wrap
+    return self.wrap_tensor(value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1049, in wrap_tensor
+    tensor_variable = wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/ao/torchao/quantization/subclass.py", line 154, in __torch_dispatch__
+    new = args[0]._change_shape(args[0].shape[::-1])
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  vgg16                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:07,  3.84it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:07,  3.96it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:06,  4.01it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:06,  4.03it/s]
running benchmark:  17%|█▋        | 5/30 [00:01<00:06,  4.04it/s]
running benchmark:  20%|██        | 6/30 [00:01<00:06,  3.96it/s]
running benchmark:  23%|██▎       | 7/30 [00:01<00:05,  4.00it/s]
running benchmark:  27%|██▋       | 8/30 [00:02<00:05,  4.00it/s]
running benchmark:  30%|███       | 9/30 [00:02<00:05,  4.03it/s]
running benchmark:  33%|███▎      | 10/30 [00:02<00:05,  4.00it/s]
running benchmark:  37%|███▋      | 11/30 [00:02<00:04,  4.02it/s]
running benchmark:  40%|████      | 12/30 [00:03<00:04,  3.98it/s]
running benchmark:  43%|████▎     | 13/30 [00:03<00:04,  4.01it/s]
running benchmark:  47%|████▋     | 14/30 [00:03<00:03,  4.04it/s]
running benchmark:  50%|█████     | 15/30 [00:03<00:03,  4.05it/s]
running benchmark:  53%|█████▎    | 16/30 [00:03<00:03,  4.06it/s]
running benchmark:  57%|█████▋    | 17/30 [00:04<00:03,  4.07it/s]
running benchmark:  60%|██████    | 18/30 [00:04<00:02,  4.09it/s]
running benchmark:  63%|██████▎   | 19/30 [00:04<00:02,  4.09it/s]
running benchmark:  67%|██████▋   | 20/30 [00:04<00:02,  4.10it/s]
running benchmark:  70%|███████   | 21/30 [00:05<00:02,  4.09it/s]
running benchmark:  73%|███████▎  | 22/30 [00:05<00:01,  4.09it/s]
running benchmark:  77%|███████▋  | 23/30 [00:05<00:01,  4.09it/s]
running benchmark:  80%|████████  | 24/30 [00:05<00:01,  4.09it/s]
running benchmark:  83%|████████▎ | 25/30 [00:06<00:01,  4.08it/s]
running benchmark:  87%|████████▋ | 26/30 [00:06<00:00,  4.08it/s]
running benchmark:  90%|█████████ | 27/30 [00:06<00:00,  4.08it/s]
running benchmark:  93%|█████████▎| 28/30 [00:06<00:00,  4.08it/s]
running benchmark:  97%|█████████▋| 29/30 [00:07<00:00,  4.04it/s]
running benchmark: 100%|██████████| 30/30 [00:07<00:00,  4.06it/s]
running benchmark: 100%|██████████| 30/30 [00:07<00:00,  4.05it/s]
+162.315x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model vision_maskrcnn supports float32 only
+
loading model: 0it [00:07, ?it/s]
+WARNING:common:Model vision_maskrcnn supports float32 only
+cuda eval  vision_maskrcnn                    
+WARNING:common:Model vision_maskrcnn supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:06,  4.23it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:05,  4.78it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:05,  5.19it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:04,  5.42it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:04,  5.50it/s]
running benchmark:  20%|██        | 6/30 [00:01<00:04,  5.52it/s]
running benchmark:  23%|██▎       | 7/30 [00:01<00:04,  5.15it/s]
running benchmark:  27%|██▋       | 8/30 [00:01<00:04,  5.34it/s]
running benchmark:  30%|███       | 9/30 [00:01<00:03,  5.45it/s]
running benchmark:  33%|███▎      | 10/30 [00:01<00:03,  5.49it/s]
running benchmark:  37%|███▋      | 11/30 [00:02<00:03,  5.58it/s]
running benchmark:  40%|████      | 12/30 [00:02<00:03,  5.63it/s]
running benchmark:  43%|████▎     | 13/30 [00:02<00:02,  5.68it/s]
running benchmark:  47%|████▋     | 14/30 [00:02<00:02,  5.73it/s]
running benchmark:  50%|█████     | 15/30 [00:02<00:02,  5.77it/s]
running benchmark:  53%|█████▎    | 16/30 [00:02<00:02,  5.81it/s]
running benchmark:  57%|█████▋    | 17/30 [00:03<00:02,  5.82it/s]
running benchmark:  60%|██████    | 18/30 [00:03<00:02,  5.84it/s]
running benchmark:  63%|██████▎   | 19/30 [00:03<00:01,  5.85it/s]
running benchmark:  67%|██████▋   | 20/30 [00:03<00:01,  5.87it/s]
running benchmark:  70%|███████   | 21/30 [00:03<00:01,  5.88it/s]
running benchmark:  73%|███████▎  | 22/30 [00:03<00:01,  5.88it/s]
running benchmark:  77%|███████▋  | 23/30 [00:04<00:01,  5.85it/s]
running benchmark:  80%|████████  | 24/30 [00:04<00:01,  5.85it/s]
running benchmark:  83%|████████▎ | 25/30 [00:04<00:00,  5.84it/s]
running benchmark:  87%|████████▋ | 26/30 [00:04<00:00,  5.84it/s]
running benchmark:  90%|█████████ | 27/30 [00:04<00:00,  5.88it/s]
running benchmark:  93%|█████████▎| 28/30 [00:04<00:00,  5.89it/s]
running benchmark:  97%|█████████▋| 29/30 [00:05<00:00,  5.87it/s]
running benchmark: 100%|██████████| 30/30 [00:05<00:00,  5.84it/s]
running benchmark: 100%|██████████| 30/30 [00:05<00:00,  5.67it/s]
+3.690x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/yolov3/__init__.py", line 28, in <module>
+    assert os.path.exists(DATA_DIR), "Couldn't find coco128 data dir, please run install.py again."
+AssertionError: Couldn't find coco128 data dir, please run install.py again.
+Run failed with return code:  1
+Output:  None
+Error:  None
+speedup             gmean=0.00x mean=96.332x
+abs_latency         gmean=0.00x mean=8.390x
+compilation_latency mean=23.731 seconds
+compression_ratio   mean=0.929x
+eager_peak_mem      gmean=0.00x mean=0.599x
+dynamo_peak_mem     gmean=0.00x mean=0.615x
+calls_captured      gmean=0.00x mean=347.378x
+unique_graphs       gmean=0.00x mean=2.027x
+graph_breaks        gmean=0.00x mean=0.878x
+unique_graph_breaks gmean=0.00x mean=0.203x
+start int8 weight only
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/fbgemm_gpu_py.so: undefined symbol: _ZNK5torch8autograd4Node4nameEv
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/canary_models/torchrec_dlrm/__init__.py", line 7, in <module>
+    from .data.dlrm_dataloader import get_dataloader
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/canary_models/torchrec_dlrm/data/dlrm_dataloader.py", line 13, in <module>
+    from torchrec.datasets.criteo import (
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/__init__.py", line 8, in <module>
+    import torchrec.distributed  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/__init__.py", line 36, in <module>
+    from torchrec.distributed.model_parallel import DistributedModelParallel  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/model_parallel.py", line 21, in <module>
+    from torchrec.distributed.planner import EmbeddingShardingPlanner, Topology
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/__init__.py", line 22, in <module>
+    from torchrec.distributed.planner.planners import EmbeddingShardingPlanner  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/planners.py", line 19, in <module>
+    from torchrec.distributed.planner.constants import BATCH_SIZE, MAX_SIZE
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/constants.py", line 10, in <module>
+    from torchrec.distributed.embedding_types import EmbeddingComputeKernel
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/embedding_types.py", line 14, in <module>
+    from fbgemm_gpu.split_table_batched_embeddings_ops_training import EmbeddingLocation
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/__init__.py", line 22, in <module>
+    from . import _fbgemm_gpu_docs  # noqa: F401, E402
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/_fbgemm_gpu_docs.py", line 19, in <module>
+    torch.ops.fbgemm.jagged_2d_to_dense,
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 820, in __getattr__
+    raise AttributeError(
+AttributeError: '_OpNamespace' 'fbgemm' object has no attribute 'jagged_2d_to_dense'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  BERT_pytorch                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 31.35it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 32.56it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 32.97it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 33.24it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 33.38it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 33.28it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 33.60it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 33.32it/s]
+6.180x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  Background_Matting                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 35.45it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 40.02it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 41.45it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 42.06it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 42.43it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 42.65it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 41.87it/s]
+2.104x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model DALLE2_pytorch supports float32 only
+
loading model: 0it [00:12, ?it/s]
+WARNING:common:Model DALLE2_pytorch supports float32 only
+cuda eval  DALLE2_pytorch                     
+WARNING:common:Model DALLE2_pytorch supports float32 only
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/utils/_contextlib.py", line 115, in decorate_context
+    return func(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 95, in inner
+    model.eval()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 96, in resume_in_inner
+    out = fn(model, *args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 3322, in forward
+    device = module_device(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 81, in module_device
+    return next(module.parameters()).device
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  LearningToPaint                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 176.99it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 186.40it/s]
+2.108x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model Super_SloMo supports float32 only
+
loading model: 0it [00:03, ?it/s]
+WARNING:common:Model Super_SloMo supports float32 only
+cuda eval  Super_SloMo                        
+WARNING:common:Model Super_SloMo supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:09,  3.05it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:03,  6.93it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02,  8.99it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:02, 10.20it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 10.94it/s]
running benchmark:  37%|███▋      | 11/30 [00:01<00:01, 11.42it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 11.72it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 11.94it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:01, 12.06it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 12.15it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 12.21it/s]
running benchmark:  77%|███████▋  | 23/30 [00:02<00:00, 12.25it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00, 12.28it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 12.30it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 12.31it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 11.25it/s]
+1.389x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  alexnet                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 140.76it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 145.44it/s]
+1.329x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_edgecnn                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 210.51it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 212.90it/s]
+1.362x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_gcn                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 123.73it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 124.69it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 124.62it/s]
+1.068x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  basic_gnn_gin                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 327.36it/s]
+1.215x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_sage                     
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 167.94it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 173.54it/s]
+1.239x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+cuda eval  cm3leon_generate                   
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:01<00:35,  1.23s/it]
running benchmark:   7%|▋         | 2/30 [00:02<00:34,  1.22s/it]
running benchmark:  10%|█         | 3/30 [00:03<00:32,  1.21s/it]
running benchmark:  13%|█▎        | 4/30 [00:04<00:31,  1.21s/it]
running benchmark:  17%|█▋        | 5/30 [00:06<00:30,  1.21s/it]
running benchmark:  20%|██        | 6/30 [00:07<00:28,  1.20s/it]
running benchmark:  23%|██▎       | 7/30 [00:08<00:27,  1.20s/it]
running benchmark:  27%|██▋       | 8/30 [00:09<00:26,  1.22s/it]
running benchmark:  30%|███       | 9/30 [00:10<00:25,  1.23s/it]
running benchmark:  33%|███▎      | 10/30 [00:12<00:24,  1.23s/it]
running benchmark:  37%|███▋      | 11/30 [00:13<00:23,  1.22s/it]
running benchmark:  40%|████      | 12/30 [00:14<00:21,  1.21s/it]
running benchmark:  43%|████▎     | 13/30 [00:15<00:20,  1.21s/it]
running benchmark:  47%|████▋     | 14/30 [00:16<00:19,  1.21s/it]
running benchmark:  50%|█████     | 15/30 [00:18<00:18,  1.20s/it]
running benchmark:  53%|█████▎    | 16/30 [00:19<00:16,  1.21s/it]
running benchmark:  57%|█████▋    | 17/30 [00:20<00:15,  1.20s/it]
running benchmark:  60%|██████    | 18/30 [00:21<00:14,  1.22s/it]
running benchmark:  63%|██████▎   | 19/30 [00:23<00:13,  1.21s/it]
running benchmark:  67%|██████▋   | 20/30 [00:24<00:12,  1.21s/it]
running benchmark:  70%|███████   | 21/30 [00:25<00:10,  1.21s/it]
running benchmark:  73%|███████▎  | 22/30 [00:26<00:09,  1.23s/it]
running benchmark:  77%|███████▋  | 23/30 [00:27<00:08,  1.24s/it]
running benchmark:  80%|████████  | 24/30 [00:29<00:07,  1.23s/it]
running benchmark:  83%|████████▎ | 25/30 [00:30<00:06,  1.23s/it]
running benchmark:  87%|████████▋ | 26/30 [00:31<00:04,  1.23s/it]
running benchmark:  90%|█████████ | 27/30 [00:32<00:03,  1.22s/it]
running benchmark:  93%|█████████▎| 28/30 [00:34<00:02,  1.22s/it]
running benchmark:  97%|█████████▋| 29/30 [00:35<00:01,  1.21s/it]
running benchmark: 100%|██████████| 30/30 [00:36<00:00,  1.20s/it]
running benchmark: 100%|██████████| 30/30 [00:36<00:00,  1.22s/it]
+4.360x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  dcgan                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 455.28it/s]
+1.378x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:demucs failed to load
+Original Error: "_thnn_fused_lstm_cell_cuda" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/__init__.py", line 31, in forward
+    return sources, self.model(mix)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/demucs/model.py", line 209, in forward
+    x = self.lstm(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/demucs/model.py", line 27, in forward
+    x = self.lstm(x)[0]
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/rnn.py", line 878, in forward
+    result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,
+RuntimeError: "_thnn_fused_lstm_cell_cuda" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  densenet121                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:03,  8.45it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 16.64it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 22.07it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 25.74it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 28.24it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 29.95it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 31.14it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 26.04it/s]
+1.998x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_dc5 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_dc5 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+
loading model: 0it [00:05, ?it/s]
+WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+cuda eval  detectron2_fcos_r_50_fpn           
+WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+[2023-12-06 17:06:54,309] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (8)
+[2023-12-06 17:06:54,309] torch._dynamo.convert_frame: [WARNING]    function: 'forward' (/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/batch_norm.py:318)
+[2023-12-06 17:06:54,309] torch._dynamo.convert_frame: [WARNING]    last reason: L['self']._pos == 0                                           # ret = self[self._pos](x)  # miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/batch_norm.py:319 in forward
+[2023-12-06 17:06:54,309] torch._dynamo.convert_frame: [WARNING] To log all recompilation reasons, use TORCH_LOGS="recompiles".
+[2023-12-06 17:06:54,309] torch._dynamo.convert_frame: [WARNING] To diagnose recompilation issues, see https://pytorch.org/docs/master/compile/troubleshooting.html.
+skipping cudagraphs due to ['mutated inputs']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:01, 16.62it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 19.03it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 19.64it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 19.94it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 20.07it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 19.66it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 19.37it/s]
running benchmark:  73%|███████▎  | 22/30 [00:01<00:00, 19.82it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 19.97it/s]
running benchmark:  93%|█████████▎| 28/30 [00:01<00:00, 20.12it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 19.83it/s]
+1.536x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_101_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_101_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_50_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_50_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:12, ?it/s]
+cuda eval  dlrm                               
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 204.84it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 208.20it/s]
+1.857x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model doctr_det_predictor supports float32 only
+
loading model: 0it [00:06, ?it/s]
+WARNING:common:Model doctr_det_predictor supports float32 only
+cuda eval  doctr_det_predictor                
+WARNING:common:Model doctr_det_predictor supports float32 only
+[2023-12-06 17:08:48,698] [1/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+skipping cudagraphs due to ['non-cuda device in graph']
+malloc(): unaligned tcache chunk detected
+Run failed with return code:  -6
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model doctr_reco_predictor supports float32 only
+
loading model: 0it [00:06, ?it/s]
+WARNING:common:Model doctr_reco_predictor supports float32 only
+cuda eval  doctr_reco_predictor               
+WARNING:common:Model doctr_reco_predictor supports float32 only
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/doctr/models/recognition/crnn/pytorch.py", line 212, in forward
+    logits = self.linear(logits)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 249, in impl
+    self.push(fn_var.call_function(self, self.popn(nargs), {}))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in call_function
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in <listcomp>
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 56, in realize
+    self._cache.realize(self.parents_tracker)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 22, in realize
+    self.vt = VariableBuilder(tx, self.source)(self.value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 380, in _wrap
+    return type_dispatch(self, value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 824, in wrap_listlike
+    output = [
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 825, in <listcomp>
+    VariableBuilder(self.tx, GetItemSource(self.get_source(), i))(item)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 395, in _wrap
+    return self.wrap_tensor(value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1049, in wrap_tensor
+    tensor_variable = wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/ao/torchao/quantization/subclass.py", line 154, in __torch_dispatch__
+    new = args[0]._change_shape(args[0].shape[::-1])
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  drq                                
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 399.04it/s]
+4.974x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  fastNLP_Bert                       
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/models/bert.py", line 265, in forward
+    sequence_output = self.bert(words)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/embeddings/bert_embedding.py", line 137, in forward
+    outputs = self.model(words)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/embeddings/bert_embedding.py", line 445, in forward
+    max_word_piece_length = batch_word_pieces_length.sum(dim=-1).max().item()  # 表示word piece的长度(包括padding)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/embeddings/bert_embedding.py", line 482, in resume_in_forward
+    bert_outputs, pooled_cls = self.encoder(word_pieces, token_type_ids=token_type_ids, attention_mask=attn_masks,
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/modules/encoder/bert.py", line 509, in forward
+    extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  functorch_dp_cifar10               
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 239.42it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 240.11it/s]
+4.517x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/functorch_maml_omniglot/__init__.py", line 73, in __init__
+    self.meta_inputs = torch.load(f'{root}/maml_omniglot/batch.pt')
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 998, in load
+    with _open_file_like(f, 'rb') as opened_file:
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 445, in _open_file_like
+    return _open_file(name_or_buffer, mode)
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 426, in __init__
+    super().__init__(open(name, mode))
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/batch.pt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  hf_Albert                          
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/albert/modeling_albert.py", line 986, in forward
+    outputs = self.albert(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/albert/modeling_albert.py", line 725, in forward
+    extended_attention_mask = extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  hf_Bart                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  9.22it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 13.95it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:01, 20.25it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 23.48it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 25.34it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 26.47it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 27.19it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 27.67it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 27.99it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 28.28it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 28.50it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 25.72it/s]
+7.325x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+cuda eval  hf_BigBird                         
+[2023-12-06 17:12:09,723] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:11,806] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:12,945] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:14,085] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:15,544] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:16,677] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:17,807] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:18,942] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:20,069] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:21,194] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:22,332] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 17:12:23,467] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+skipping cudagraphs due to ['non-cuda device in graph']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:02, 10.89it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:02, 11.14it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:02, 11.26it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 11.28it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:01, 11.19it/s]
running benchmark:  40%|████      | 12/30 [00:01<00:01, 11.21it/s]
running benchmark:  47%|████▋     | 14/30 [00:01<00:01, 11.20it/s]
running benchmark:  53%|█████▎    | 16/30 [00:01<00:01, 11.19it/s]
running benchmark:  60%|██████    | 18/30 [00:01<00:01, 11.21it/s]
running benchmark:  67%|██████▋   | 20/30 [00:01<00:00, 11.22it/s]
running benchmark:  73%|███████▎  | 22/30 [00:01<00:00, 11.16it/s]
running benchmark:  80%|████████  | 24/30 [00:02<00:00, 11.17it/s]
running benchmark:  87%|████████▋ | 26/30 [00:02<00:00, 11.22it/s]
running benchmark:  93%|█████████▎| 28/30 [00:02<00:00, 11.20it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 11.20it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 11.20it/s]
+2.859x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  hf_DistilBert                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 54.36it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 61.72it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 64.13it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 65.32it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 64.06it/s]
+7.156x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  hf_GPT2                            
+AUTOTUNE mm(1024x768, 768x50264)
+  mm 0.4116 ms 100.0%
+  triton_mm_866 0.5038 ms 81.7%
+  triton_mm_865 0.5598 ms 73.5%
+  triton_mm_868 0.5721 ms 71.9%
+  triton_mm_867 0.5904 ms 69.7%
+  triton_mm_864 0.7119 ms 57.8%
+  triton_mm_872 0.7138 ms 57.7%
+  triton_mm_874 0.8649 ms 47.6%
+  triton_mm_871 0.8814 ms 46.7%
+  triton_mm_870 1.2215 ms 33.7%
+SingleProcess AUTOTUNE takes 5.3196 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 61.85it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 64.64it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 65.63it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 66.04it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 65.51it/s]
+3.271x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:18, ?it/s]
+cuda eval  hf_GPT2_large                      
+AUTOTUNE mm(1024x1280, 1280x50264)
+  mm 0.6187 ms 100.0%
+  triton_mm_2594 0.8222 ms 75.3%
+  triton_mm_2593 0.9105 ms 68.0%
+  triton_mm_2596 0.9182 ms 67.4%
+  triton_mm_2595 0.9534 ms 64.9%
+  triton_mm_2592 1.1534 ms 53.6%
+  triton_mm_2600 1.1575 ms 53.5%
+  triton_mm_2599 1.3903 ms 44.5%
+  triton_mm_2602 1.4438 ms 42.9%
+  triton_mm_2597 1.9600 ms 31.6%
+SingleProcess AUTOTUNE takes 4.7814 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:02,  9.86it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 14.61it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 16.73it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 17.89it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 18.57it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 19.00it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 19.24it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 19.39it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 19.52it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 19.60it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 19.66it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 19.64it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 19.61it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 19.64it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 19.69it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 18.93it/s]
+2.005x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+cuda eval  hf_Longformer                      
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/longformer/modeling_longformer.py", line 1844, in forward
+    outputs = self.longformer(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/longformer/modeling_longformer.py", line 1739, in forward
+    extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape)[
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  hf_Reformer                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:00, 45.80it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 48.70it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 48.95it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 49.25it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 50.29it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 49.68it/s]
+5.241x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  hf_T5                              
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1709, in forward
+    encoder_outputs = self.encoder(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:09, ?it/s]
+cuda eval  hf_T5_base                         
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1709, in forward
+    encoder_outputs = self.encoder(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+cuda eval  hf_T5_generate                     
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 221, in forward
+    return self.model.generate(inputs, self.generation_config)
+  File "/home/cdhernandez/local/pytorch/torch/utils/_contextlib.py", line 115, in decorate_context
+    return func(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1426, in generate
+    generation_config = copy.deepcopy(generation_config)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1429, in resume_in_generate
+    self._validate_model_kwargs(model_kwargs.copy())
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1429, in resume_in_generate
+    self._validate_model_kwargs(model_kwargs.copy())
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1432, in resume_in_generate
+    logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1433, in resume_in_generate
+    stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1492, in resume_in_generate
+    model_kwargs = self._prepare_encoder_decoder_kwargs_for_generation(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 661, in _prepare_encoder_decoder_kwargs_for_generation
+    model_kwargs["encoder_outputs"]: ModelOutput = encoder(**encoder_kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:19, ?it/s]
+cuda eval  hf_T5_large                        
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1709, in forward
+    encoder_outputs = self.encoder(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  hf_Whisper                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 55.89it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 59.90it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 61.68it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 62.57it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 61.72it/s]
+3.965x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:hf_clip failed to load
+Original Error: 'str' object has no attribute 'shape'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 1142, in forward
+    vision_outputs = self.vision_model(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 883, in forward
+    hidden_states = self.embeddings(pixel_values)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 194, in forward
+    batch_size = pixel_values.shape[0]
+AttributeError: 'str' object has no attribute 'shape'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  lennard_jones                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 584.40it/s]
+5.102x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  llama                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 36.42it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 38.83it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 39.61it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 39.73it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 39.80it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 39.52it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 39.58it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 39.41it/s]
+7.589x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [01:00, ?it/s]
+cuda eval  llama_v2_7b_16h                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:05,  5.54it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02,  9.39it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02, 10.70it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:02, 11.34it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 11.70it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 11.93it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 12.06it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 12.15it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:01, 12.19it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 12.22it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 12.26it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 12.29it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00, 12.30it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 12.30it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 12.28it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 11.84it/s]
+1.287x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/__init__.py", line 78, in __init__
+    self.meta_inputs = torch.load(f'{root}/batch.pt')
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 998, in load
+    with _open_file_like(f, 'rb') as opened_file:
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 445, in _open_file_like
+    return _open_file(name_or_buffer, mode)
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 426, in __init__
+    super().__init__(open(name, mode))
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/batch.pt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  mnasnet1_0                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 115.21it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 118.32it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 118.28it/s]
+3.213x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  mobilenet_v2                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 113.40it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 120.20it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 119.33it/s]
+4.729x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+WARNING:root:mobilenet_v2_quantized_qat failed to load
+The eval test only supports CPU.
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/mobilenet_v2_quantized_qat/__init__.py", line 21, in __init__
+    raise NotImplementedError("The eval test only supports CPU.")
+NotImplementedError: The eval test only supports CPU.
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  mobilenet_v3_large                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 89.74it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 99.11it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 99.88it/s]
+4.187x
+
loading model: 0it [00:00, ?it/s]NCCL version 2.19.3+cuda12.0
+
loading model: 0it [00:03, ?it/s]
+cuda eval  moco                               
+--Return--
+> /home/cdhernandez/local/ao/torchao/quantization/subclass.py(196)__torch_dispatch__()->None
+-> breakpoint()
+(Pdb) TIMEOUT
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+number of parameters: 123.69M
+num decayed parameter tensors: 50, with 124,354,560 parameters
+num non-decayed parameter tensors: 98, with 121,344 parameters
+using fused AdamW: True
+cuda eval  nanogpt                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 52.20it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 53.28it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 53.40it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 53.00it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 52.95it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 52.99it/s]
+9.043x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  nvidia_deeprecommender             
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 201.82it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 204.93it/s]
+0.921x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  opacus_cifar10                     
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 702, in call_function
+    return variables.UserFunctionVariable(fn, source=source).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1167, in call_getattr
+    return obj.var_getattr(tx, name).clone(source=source)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/user_defined.py", line 504, in var_getattr
+    return VariableBuilder(tx, source)(subobj)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 395, in _wrap
+    return self.wrap_tensor(value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 971, in wrap_tensor
+    return self.tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 1
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/opacus/grad_sample/grad_sample_module.py", line 148, in forward
+    return self._module(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/vision/torchvision/models/resnet.py", line 285, in forward
+    return self._forward_impl(x)
+  File "/home/cdhernandez/local/vision/torchvision/models/resnet.py", line 280, in _forward_impl
+    x = self.fc(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1562, in _call_impl
+    result = forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:29, ?it/s]
+cuda eval  phi_1_5                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  7.64it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 11.46it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 12.63it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 13.18it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 13.42it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 13.63it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 13.71it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 13.83it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 13.94it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 13.87it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 13.88it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 13.84it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 13.81it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 13.67it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 13.69it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 13.50it/s]
+2.950x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  phlippe_densenet                   
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 74.91it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 87.24it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 91.25it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 89.41it/s]
+4.769x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+cuda eval  phlippe_resnet                     
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 199.10it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 211.99it/s]
+4.083x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  pyhpc_equation_of_state            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 224.72it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 225.30it/s]
+16.833x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  pyhpc_isoneutral_mixing            
+skipping cudagraphs due to ['mutated inputs']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 85.90it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 84.52it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 86.04it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 86.13it/s]
+6.106x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+
loading model: 0it [00:01, ?it/s]
+WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+cuda eval  pyhpc_turbulent_kinetic_energy     
+WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 67.76it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 81.50it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 84.23it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 82.02it/s]
+8.195x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/__init__.py", line 13, in <module>
+    from .train_cyclegan import prepare_training_loop
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/train_cyclegan.py", line 27, in <module>
+    from .util.visualizer import Visualizer
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/util/visualizer.py", line 6, in <module>
+    from . import util, html
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/util/html.py", line 1, in <module>
+    import dominate
+ModuleNotFoundError: No module named 'dominate'
+Failed to import user benchmark module dynamo, error: No module named 'dominate'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/__init__.py", line 52, in __init__
+    self.data_loader = self.get_data_loader(config)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/__init__.py", line 68, in get_data_loader
+    celeba_loader = get_loader(config.celeba_image_dir, config.attr_path, config.selected_attrs,
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 82, in get_loader
+    dataset = CelebA(image_dir, attr_path, selected_attrs, transform, mode)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 24, in __init__
+    self.preprocess()
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 33, in preprocess
+    lines = [line.rstrip() for line in open(self.attr_path, 'r')]
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/data/.data/pytorch_stargan_inputs/data/celeba/list_attr_celeba.txt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  pytorch_unet                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 37.42it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 42.77it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 44.48it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 45.27it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 45.72it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 45.98it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 45.03it/s]
+1.816x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  resnet152                          
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 25.33it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 30.48it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 32.05it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 32.78it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 33.20it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 33.34it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 33.55it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 32.73it/s]
+2.375x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  resnet18                           
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 177.52it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 187.07it/s]
+4.439x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  resnet50                           
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 75.27it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 80.82it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 82.52it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 81.80it/s]
+1.830x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+WARNING:root:resnet50_quantized_qat failed to load
+The eval test only supports CPU.
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/resnet50_quantized_qat/__init__.py", line 21, in __init__
+    raise NotImplementedError("The eval test only supports CPU.")
+NotImplementedError: The eval test only supports CPU.
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  resnext50_32x4d                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 104.58it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 106.78it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 106.98it/s]
+3.697x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/__init__.py", line 27, in __init__
+    self.model = sam_model_registry[model_type](checkpoint=sam_checkpoint)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/build_sam.py", line 19, in build_sam_vit_h
+    return _build_sam(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/build_sam.py", line 108, in _build_sam
+    with open(checkpoint, "rb") as f:
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/.data/sam_vit_h_4b8939.pth'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  shufflenet_v2_x1_0                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 93.51it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 96.86it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 97.86it/s]
+3.407x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  soft_actor_critic                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 680.83it/s]
+3.252x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/speech_transformer/__init__.py", line 15, in <module>
+    from .config import SpeechTransformerTrainConfig, SpeechTransformerEvalConfig
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/speech_transformer/config.py", line 4, in <module>
+    import kaldi_io
+ModuleNotFoundError: No module named 'kaldi_io'
+Failed to import user benchmark module dynamo, error: No module named 'kaldi_io'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  squeezenet1_1                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 271.22it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 273.54it/s]
+3.495x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/stable_diffusion_text_encoder/__init__.py", line 11, in <module>
+    from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+ModuleNotFoundError: No module named 'diffusers'
+Failed to import user benchmark module dynamo, error: No module named 'diffusers'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/stable_diffusion_unet/__init__.py", line 11, in <module>
+    from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+ModuleNotFoundError: No module named 'diffusers'
+Failed to import user benchmark module dynamo, error: No module named 'diffusers'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/timm_efficientdet/__init__.py", line 12, in <module>
+    from effdet import create_model, create_loader
+ModuleNotFoundError: No module named 'effdet'
+Failed to import user benchmark module dynamo, error: No module named 'effdet'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  timm_efficientnet                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:00, 47.27it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 55.61it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 58.14it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 59.27it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 58.18it/s]
+2.212x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  timm_nfnet                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:04,  6.69it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 11.48it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 13.19it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 14.01it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 14.47it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 14.71it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 14.89it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 14.99it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 15.11it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 15.18it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 15.24it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 15.28it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 15.30it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 15.31it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 15.31it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 14.68it/s]
+1.857x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  timm_regnet                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 23.71it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 29.72it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 31.76it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 32.76it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 33.32it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 33.62it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 33.81it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 32.80it/s]
+1.358x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  timm_resnest                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 97.40it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 103.46it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 104.15it/s]
+1.667x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  timm_vision_transformer            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 37.14it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 41.34it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 42.64it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 42.65it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 43.14it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 43.44it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 42.79it/s]
+4.501x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:20, ?it/s]
+cuda eval  timm_vision_transformer_large      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:17,  1.65it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:11,  2.50it/s]
running benchmark:  10%|█         | 3/30 [00:01<00:09,  2.99it/s]
running benchmark:  13%|█▎        | 4/30 [00:01<00:07,  3.30it/s]
running benchmark:  17%|█▋        | 5/30 [00:01<00:07,  3.49it/s]
running benchmark:  20%|██        | 6/30 [00:01<00:06,  3.62it/s]
running benchmark:  23%|██▎       | 7/30 [00:02<00:06,  3.71it/s]
running benchmark:  27%|██▋       | 8/30 [00:02<00:05,  3.77it/s]
running benchmark:  30%|███       | 9/30 [00:02<00:05,  3.81it/s]
running benchmark:  33%|███▎      | 10/30 [00:02<00:05,  3.83it/s]
running benchmark:  37%|███▋      | 11/30 [00:03<00:04,  3.85it/s]
running benchmark:  40%|████      | 12/30 [00:03<00:04,  3.87it/s]
running benchmark:  43%|████▎     | 13/30 [00:03<00:04,  3.88it/s]
running benchmark:  47%|████▋     | 14/30 [00:03<00:04,  3.89it/s]
running benchmark:  50%|█████     | 15/30 [00:04<00:03,  3.89it/s]
running benchmark:  53%|█████▎    | 16/30 [00:04<00:03,  3.89it/s]
running benchmark:  57%|█████▋    | 17/30 [00:04<00:03,  3.90it/s]
running benchmark:  60%|██████    | 18/30 [00:04<00:03,  3.90it/s]
running benchmark:  63%|██████▎   | 19/30 [00:05<00:02,  3.89it/s]
running benchmark:  67%|██████▋   | 20/30 [00:05<00:02,  3.89it/s]
running benchmark:  70%|███████   | 21/30 [00:05<00:02,  3.89it/s]
running benchmark:  73%|███████▎  | 22/30 [00:05<00:02,  3.89it/s]
running benchmark:  77%|███████▋  | 23/30 [00:06<00:01,  3.88it/s]
running benchmark:  80%|████████  | 24/30 [00:06<00:01,  3.89it/s]
running benchmark:  83%|████████▎ | 25/30 [00:06<00:01,  3.89it/s]
running benchmark:  87%|████████▋ | 26/30 [00:07<00:01,  3.89it/s]
running benchmark:  90%|█████████ | 27/30 [00:07<00:00,  3.88it/s]
running benchmark:  93%|█████████▎| 28/30 [00:07<00:00,  3.88it/s]
running benchmark:  97%|█████████▋| 29/30 [00:07<00:00,  3.88it/s]
running benchmark: 100%|██████████| 30/30 [00:08<00:00,  3.88it/s]
running benchmark: 100%|██████████| 30/30 [00:08<00:00,  3.72it/s]
+1.335x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  timm_vovnet                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 78.70it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 84.72it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 86.72it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 85.99it/s]
+1.752x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/torch_multimodal_clip/__init__.py", line 27, in __init__
+    self.image = Image.open(os.path.join(self.data_folder, self.image_name))
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/PIL/Image.py", line 3218, in open
+    fp = builtins.open(filename, "rb")
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/torch_multimodal_clip/.data/pizza.jpg'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model tts_angular supports float32 only
+
loading model: 0it [00:00, ?it/s]
+WARNING:common:Model tts_angular supports float32 only
+cuda eval  tts_angular                        
+WARNING:common:Model tts_angular supports float32 only
+ERROR:common:Backend eager failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/tts_angular/model.py", line 59, in forward
+    d = self.layers(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/container.py", line 217, in forward
+    input = module(input)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/tts_angular/model.py", line 19, in forward
+    return self.linear(o)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+  File "/home/cdhernandez/local/ao/torchao/quantization/subclass.py", line 144, in __torch_dispatch__
+    return cls._quantized_op(mat1, w_qtensor, bias)
+  File "/home/cdhernandez/local/ao/torchao/quantization/subclass.py", line 307, in _quantized_op
+    act_mat = act_mat.view(-1, act_mat.shape[-1])
+RuntimeError: view size is not compatible with input tensor's size and stride (at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead.
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  vgg16                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 209.25it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 211.51it/s]
+1.388x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model vision_maskrcnn supports float32 only
+
loading model: 0it [00:05, ?it/s]
+WARNING:common:Model vision_maskrcnn supports float32 only
+cuda eval  vision_maskrcnn                    
+WARNING:common:Model vision_maskrcnn supports float32 only
+Run failed with return code:  -11
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/yolov3/__init__.py", line 28, in <module>
+    assert os.path.exists(DATA_DIR), "Couldn't find coco128 data dir, please run install.py again."
+AssertionError: Couldn't find coco128 data dir, please run install.py again.
+Run failed with return code:  1
+Output:  None
+Error:  None
+speedup             gmean=0.00x mean=2.565x
+abs_latency         gmean=0.00x mean=8.545x
+compilation_latency mean=16.538 seconds
+compression_ratio   mean=0.862x
+eager_peak_mem      gmean=0.00x mean=0.572x
+dynamo_peak_mem     gmean=0.00x mean=0.628x
+calls_captured      gmean=0.00x mean=329.608x
+unique_graphs       gmean=0.00x mean=1.541x
+graph_breaks        gmean=0.00x mean=0.581x
+unique_graph_breaks gmean=0.00x mean=0.162x
+start int4 weight only
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/fbgemm_gpu_py.so: undefined symbol: _ZNK5torch8autograd4Node4nameEv
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/canary_models/torchrec_dlrm/__init__.py", line 7, in <module>
+    from .data.dlrm_dataloader import get_dataloader
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/canary_models/torchrec_dlrm/data/dlrm_dataloader.py", line 13, in <module>
+    from torchrec.datasets.criteo import (
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/__init__.py", line 8, in <module>
+    import torchrec.distributed  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/__init__.py", line 36, in <module>
+    from torchrec.distributed.model_parallel import DistributedModelParallel  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/model_parallel.py", line 21, in <module>
+    from torchrec.distributed.planner import EmbeddingShardingPlanner, Topology
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/__init__.py", line 22, in <module>
+    from torchrec.distributed.planner.planners import EmbeddingShardingPlanner  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/planners.py", line 19, in <module>
+    from torchrec.distributed.planner.constants import BATCH_SIZE, MAX_SIZE
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/constants.py", line 10, in <module>
+    from torchrec.distributed.embedding_types import EmbeddingComputeKernel
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/embedding_types.py", line 14, in <module>
+    from fbgemm_gpu.split_table_batched_embeddings_ops_training import EmbeddingLocation
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/__init__.py", line 22, in <module>
+    from . import _fbgemm_gpu_docs  # noqa: F401, E402
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/_fbgemm_gpu_docs.py", line 19, in <module>
+    torch.ops.fbgemm.jagged_2d_to_dense,
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 820, in __getattr__
+    raise AttributeError(
+AttributeError: '_OpNamespace' 'fbgemm' object has no attribute 'jagged_2d_to_dense'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  BERT_pytorch                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:04,  7.16it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 12.39it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 14.27it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 15.20it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 15.72it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 16.05it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 16.26it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 16.39it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 16.48it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 16.54it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 16.58it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 16.61it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 16.63it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 16.64it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 16.66it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 15.97it/s]
+1.148x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  Background_Matting                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:00, 27.33it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 36.79it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 39.73it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 41.08it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 41.76it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 42.16it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 40.70it/s]
+2.104x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model DALLE2_pytorch supports float32 only
+
loading model: 0it [00:13, ?it/s]
+WARNING:common:Model DALLE2_pytorch supports float32 only
+cuda eval  DALLE2_pytorch                     
+WARNING:common:Model DALLE2_pytorch supports float32 only
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/utils/_contextlib.py", line 115, in decorate_context
+    return func(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 95, in inner
+    model.eval()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 96, in resume_in_inner
+    out = fn(model, *args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 3322, in forward
+    device = module_device(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py", line 81, in module_device
+    return next(module.parameters()).device
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+cuda eval  LearningToPaint                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 198.58it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 200.83it/s]
+2.088x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model Super_SloMo supports float32 only
+
loading model: 0it [00:04, ?it/s]
+WARNING:common:Model Super_SloMo supports float32 only
+cuda eval  Super_SloMo                        
+WARNING:common:Model Super_SloMo supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:05,  5.26it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02,  9.17it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02, 10.58it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:02, 11.28it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 11.67it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 11.92it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 12.07it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 12.18it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:01, 12.25it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 12.29it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 12.34it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 12.37it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00, 12.40it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 12.41it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 12.41it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 11.87it/s]
+1.388x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  alexnet                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 122.26it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 128.24it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 127.74it/s]
+1.276x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_edgecnn                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 202.98it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 208.03it/s]
+1.377x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_gcn                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 83.74it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 111.34it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 111.79it/s]
+1.074x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_gin                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 353.39it/s]
+1.214x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  basic_gnn_sage                     
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 165.16it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 172.03it/s]
+1.240x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  cm3leon_generate                   
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:01<00:36,  1.24s/it]
running benchmark:   7%|▋         | 2/30 [00:02<00:34,  1.23s/it]
running benchmark:  10%|█         | 3/30 [00:03<00:32,  1.21s/it]
running benchmark:  13%|█▎        | 4/30 [00:04<00:31,  1.20s/it]
running benchmark:  17%|█▋        | 5/30 [00:06<00:29,  1.20s/it]
running benchmark:  20%|██        | 6/30 [00:07<00:28,  1.20s/it]
running benchmark:  23%|██▎       | 7/30 [00:08<00:27,  1.19s/it]
running benchmark:  27%|██▋       | 8/30 [00:09<00:26,  1.19s/it]
running benchmark:  30%|███       | 9/30 [00:10<00:25,  1.19s/it]
running benchmark:  33%|███▎      | 10/30 [00:11<00:23,  1.19s/it]
running benchmark:  37%|███▋      | 11/30 [00:13<00:22,  1.19s/it]
running benchmark:  40%|████      | 12/30 [00:14<00:21,  1.19s/it]
running benchmark:  43%|████▎     | 13/30 [00:15<00:20,  1.19s/it]
running benchmark:  47%|████▋     | 14/30 [00:16<00:19,  1.19s/it]
running benchmark:  50%|█████     | 15/30 [00:17<00:17,  1.19s/it]
running benchmark:  53%|█████▎    | 16/30 [00:19<00:16,  1.19s/it]
running benchmark:  57%|█████▋    | 17/30 [00:20<00:15,  1.19s/it]
running benchmark:  60%|██████    | 18/30 [00:21<00:14,  1.19s/it]
running benchmark:  63%|██████▎   | 19/30 [00:22<00:13,  1.19s/it]
running benchmark:  67%|██████▋   | 20/30 [00:23<00:11,  1.19s/it]
running benchmark:  70%|███████   | 21/30 [00:25<00:10,  1.20s/it]
running benchmark:  73%|███████▎  | 22/30 [00:26<00:09,  1.20s/it]
running benchmark:  77%|███████▋  | 23/30 [00:27<00:08,  1.20s/it]
running benchmark:  80%|████████  | 24/30 [00:28<00:07,  1.20s/it]
running benchmark:  83%|████████▎ | 25/30 [00:29<00:05,  1.20s/it]
running benchmark:  87%|████████▋ | 26/30 [00:31<00:04,  1.20s/it]
running benchmark:  90%|█████████ | 27/30 [00:32<00:03,  1.20s/it]
running benchmark:  93%|█████████▎| 28/30 [00:33<00:02,  1.20s/it]
running benchmark:  97%|█████████▋| 29/30 [00:34<00:01,  1.20s/it]
running benchmark: 100%|██████████| 30/30 [00:35<00:00,  1.20s/it]
running benchmark: 100%|██████████| 30/30 [00:35<00:00,  1.20s/it]
+4.313x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  dcgan                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 451.21it/s]
+1.382x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+WARNING:root:demucs failed to load
+Original Error: "_thnn_fused_lstm_cell_cuda" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/__init__.py", line 31, in forward
+    return sources, self.model(mix)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/demucs/model.py", line 209, in forward
+    x = self.lstm(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/demucs/model.py", line 27, in forward
+    x = self.lstm(x)[0]
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/rnn.py", line 878, in forward
+    result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,
+RuntimeError: "_thnn_fused_lstm_cell_cuda" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  densenet121                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:11,  2.44it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02, 11.59it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 18.16it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 22.72it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 26.04it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 28.39it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 30.04it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 31.18it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 23.68it/s]
+1.986x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_dc5 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:09, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_dc5 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+
loading model: 0it [00:07, ?it/s]
+WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+cuda eval  detectron2_fcos_r_50_fpn           
+WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+[2023-12-06 18:21:08,932] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (8)
+[2023-12-06 18:21:08,932] torch._dynamo.convert_frame: [WARNING]    function: 'forward' (/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/batch_norm.py:318)
+[2023-12-06 18:21:08,932] torch._dynamo.convert_frame: [WARNING]    last reason: L['self']._pos == 0                                           # ret = self[self._pos](x)  # miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/batch_norm.py:319 in forward
+[2023-12-06 18:21:08,932] torch._dynamo.convert_frame: [WARNING] To log all recompilation reasons, use TORCH_LOGS="recompiles".
+[2023-12-06 18:21:08,932] torch._dynamo.convert_frame: [WARNING] To diagnose recompilation issues, see https://pytorch.org/docs/master/compile/troubleshooting.html.
+skipping cudagraphs due to ['mutated inputs']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:01, 17.32it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 19.22it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 18.80it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:01, 19.64it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 19.84it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 19.82it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 20.03it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 20.24it/s]
running benchmark:  80%|████████  | 24/30 [00:01<00:00, 20.40it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 20.51it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 20.63it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 20.08it/s]
+1.541x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_101_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_101_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_50_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_50_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:12, ?it/s]
+cuda eval  dlrm                               
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 172.51it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 175.59it/s]
+1.190x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model doctr_det_predictor supports float32 only
+
loading model: 0it [00:06, ?it/s]
+WARNING:common:Model doctr_det_predictor supports float32 only
+cuda eval  doctr_det_predictor                
+WARNING:common:Model doctr_det_predictor supports float32 only
+[2023-12-06 18:23:06,810] [1/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+skipping cudagraphs due to ['non-cuda device in graph']
+malloc(): unaligned tcache chunk detected
+Run failed with return code:  -6
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model doctr_reco_predictor supports float32 only
+
loading model: 0it [00:05, ?it/s]
+WARNING:common:Model doctr_reco_predictor supports float32 only
+cuda eval  doctr_reco_predictor               
+WARNING:common:Model doctr_reco_predictor supports float32 only
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/doctr/models/recognition/crnn/pytorch.py", line 212, in forward
+    logits = self.linear(logits)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 249, in impl
+    self.push(fn_var.call_function(self, self.popn(nargs), {}))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in call_function
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in <listcomp>
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 56, in realize
+    self._cache.realize(self.parents_tracker)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 22, in realize
+    self.vt = VariableBuilder(tx, self.source)(self.value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 380, in _wrap
+    return type_dispatch(self, value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 824, in wrap_listlike
+    output = [
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 825, in <listcomp>
+    VariableBuilder(self.tx, GetItemSource(self.get_source(), i))(item)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 395, in _wrap
+    return self.wrap_tensor(value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1049, in wrap_tensor
+    tensor_variable = wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/ao/torchao/quantization/subclass.py", line 154, in __torch_dispatch__
+    new = args[0]._change_shape(args[0].shape[::-1])
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  drq                                
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 491.17it/s]
+4.781x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  fastNLP_Bert                       
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/models/bert.py", line 265, in forward
+    sequence_output = self.bert(words)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/embeddings/bert_embedding.py", line 137, in forward
+    outputs = self.model(words)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/embeddings/bert_embedding.py", line 445, in forward
+    max_word_piece_length = batch_word_pieces_length.sum(dim=-1).max().item()  # 表示word piece的长度(包括padding)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/embeddings/bert_embedding.py", line 482, in resume_in_forward
+    bert_outputs, pooled_cls = self.encoder(word_pieces, token_type_ids=token_type_ids, attention_mask=attn_masks,
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fastNLP/modules/encoder/bert.py", line 509, in forward
+    extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  functorch_dp_cifar10               
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 253.54it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 253.09it/s]
+4.393x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/functorch_maml_omniglot/__init__.py", line 73, in __init__
+    self.meta_inputs = torch.load(f'{root}/maml_omniglot/batch.pt')
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 998, in load
+    with _open_file_like(f, 'rb') as opened_file:
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 445, in _open_file_like
+    return _open_file(name_or_buffer, mode)
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 426, in __init__
+    super().__init__(open(name, mode))
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/batch.pt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  hf_Albert                          
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/albert/modeling_albert.py", line 986, in forward
+    outputs = self.albert(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/albert/modeling_albert.py", line 725, in forward
+    extended_attention_mask = extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+cuda eval  hf_Bart                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:01, 14.11it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 18.73it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 20.41it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 21.05it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 21.37it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 21.63it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 21.91it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 22.12it/s]
running benchmark:  87%|████████▋ | 26/30 [00:01<00:00, 22.12it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 22.23it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 21.44it/s]
+2.517x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  hf_BigBird                         
+[2023-12-06 18:26:29,574] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:31,721] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:32,900] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:34,080] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:35,582] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:36,754] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:37,919] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:39,102] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:40,275] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:41,431] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:42,584] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 18:26:43,751] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+skipping cudagraphs due to ['non-cuda device in graph']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:06,  4.15it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:06,  4.43it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:05,  4.56it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:05,  4.60it/s]
running benchmark:  17%|█▋        | 5/30 [00:01<00:05,  4.63it/s]
running benchmark:  20%|██        | 6/30 [00:01<00:05,  4.66it/s]
running benchmark:  23%|██▎       | 7/30 [00:01<00:04,  4.67it/s]
running benchmark:  27%|██▋       | 8/30 [00:01<00:04,  4.67it/s]
running benchmark:  30%|███       | 9/30 [00:01<00:04,  4.67it/s]
running benchmark:  33%|███▎      | 10/30 [00:02<00:04,  4.68it/s]
running benchmark:  37%|███▋      | 11/30 [00:02<00:04,  4.68it/s]
running benchmark:  40%|████      | 12/30 [00:02<00:03,  4.68it/s]
running benchmark:  43%|████▎     | 13/30 [00:02<00:03,  4.68it/s]
running benchmark:  47%|████▋     | 14/30 [00:03<00:03,  4.68it/s]
running benchmark:  50%|█████     | 15/30 [00:03<00:03,  4.69it/s]
running benchmark:  53%|█████▎    | 16/30 [00:03<00:02,  4.69it/s]
running benchmark:  57%|█████▋    | 17/30 [00:03<00:02,  4.69it/s]
running benchmark:  60%|██████    | 18/30 [00:03<00:02,  4.70it/s]
running benchmark:  63%|██████▎   | 19/30 [00:04<00:02,  4.70it/s]
running benchmark:  67%|██████▋   | 20/30 [00:04<00:02,  4.70it/s]
running benchmark:  70%|███████   | 21/30 [00:04<00:01,  4.68it/s]
running benchmark:  73%|███████▎  | 22/30 [00:04<00:01,  4.69it/s]
running benchmark:  77%|███████▋  | 23/30 [00:04<00:01,  4.70it/s]
running benchmark:  80%|████████  | 24/30 [00:05<00:01,  4.69it/s]
running benchmark:  83%|████████▎ | 25/30 [00:05<00:01,  4.69it/s]
running benchmark:  87%|████████▋ | 26/30 [00:05<00:00,  4.70it/s]
running benchmark:  90%|█████████ | 27/30 [00:05<00:00,  4.69it/s]
running benchmark:  93%|█████████▎| 28/30 [00:05<00:00,  4.69it/s]
running benchmark:  97%|█████████▋| 29/30 [00:06<00:00,  4.69it/s]
running benchmark: 100%|██████████| 30/30 [00:06<00:00,  4.69it/s]
running benchmark: 100%|██████████| 30/30 [00:06<00:00,  4.67it/s]
+1.452x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  hf_DistilBert                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:00, 41.97it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 46.26it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 47.64it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 47.86it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 48.30it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 48.75it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 47.84it/s]
+2.315x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+cuda eval  hf_GPT2                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:00, 29.98it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 35.62it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 37.61it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 38.75it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 38.89it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 39.13it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 39.35it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 38.44it/s]
+1.834x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:19, ?it/s]
+cuda eval  hf_GPT2_large                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:04,  6.68it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 10.84it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02, 12.30it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 13.04it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 13.49it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 13.72it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 13.85it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 13.90it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 14.01it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 14.08it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 14.13it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 14.13it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 14.12it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 14.04it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 14.10it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 13.62it/s]
+1.564x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+cuda eval  hf_Longformer                      
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/longformer/modeling_longformer.py", line 1844, in forward
+    outputs = self.longformer(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/longformer/modeling_longformer.py", line 1739, in forward
+    extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape)[
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  hf_Reformer                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 31.28it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 33.27it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 33.82it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 33.79it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 33.89it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 34.11it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 34.17it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 33.88it/s]
+1.284x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+cuda eval  hf_T5                              
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1709, in forward
+    encoder_outputs = self.encoder(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+cuda eval  hf_T5_base                         
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1709, in forward
+    encoder_outputs = self.encoder(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  hf_T5_generate                     
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 221, in forward
+    return self.model.generate(inputs, self.generation_config)
+  File "/home/cdhernandez/local/pytorch/torch/utils/_contextlib.py", line 115, in decorate_context
+    return func(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1426, in generate
+    generation_config = copy.deepcopy(generation_config)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1429, in resume_in_generate
+    self._validate_model_kwargs(model_kwargs.copy())
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1429, in resume_in_generate
+    self._validate_model_kwargs(model_kwargs.copy())
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1432, in resume_in_generate
+    logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1433, in resume_in_generate
+    stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 1492, in resume_in_generate
+    model_kwargs = self._prepare_encoder_decoder_kwargs_for_generation(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/generation/utils.py", line 661, in _prepare_encoder_decoder_kwargs_for_generation
+    model_kwargs["encoder_outputs"]: ModelOutput = encoder(**encoder_kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:16, ?it/s]
+cuda eval  hf_T5_large                        
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1264, in CALL_FUNCTION_KW
+    self.call_function(fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1110, in call_getattr
+    return obj.var_getattr(tx, name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 217, in var_getattr
+    ).call_function(tx, [(self)], {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 291, in call_function
+    return self.obj.call_method(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 494, in call_method
+    return wrap_values(module.named_parameters(**get_kwargs("recurse")))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 430, in wrap_values
+    tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/framework/huggingface/model_factory.py", line 55, in forward
+    return self.model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1709, in forward
+    encoder_outputs = self.encoder(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1046, in forward
+    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 895, in get_extended_attention_mask
+    dtype = self.dtype
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 825, in dtype
+    return get_parameter_dtype(self)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/modeling_utils.py", line 197, in get_parameter_dtype
+    for t in parameter.parameters():
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  hf_Whisper                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:04,  6.59it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 11.24it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 12.87it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 13.66it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 14.10it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 14.39it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 14.58it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 14.70it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 14.74it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 14.79it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 14.84it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 14.85it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 14.87it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 14.86it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 14.85it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 14.30it/s]
+1.298x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:hf_clip failed to load
+Original Error: 'str' object has no attribute 'shape'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 1142, in forward
+    vision_outputs = self.vision_model(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 883, in forward
+    hidden_states = self.embeddings(pixel_values)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 194, in forward
+    batch_size = pixel_values.shape[0]
+AttributeError: 'str' object has no attribute 'shape'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  lennard_jones                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 534.51it/s]
+5.301x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  llama                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 25.87it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 29.67it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 31.34it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 32.22it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 32.75it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 33.16it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 33.25it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 32.37it/s]
+2.501x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [01:00, ?it/s]
+cuda eval  llama_v2_7b_16h                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:27,  1.04it/s]
running benchmark:   7%|▋         | 2/30 [00:01<00:17,  1.60it/s]
running benchmark:  10%|█         | 3/30 [00:01<00:13,  1.93it/s]
running benchmark:  13%|█▎        | 4/30 [00:02<00:12,  2.14it/s]
running benchmark:  17%|█▋        | 5/30 [00:02<00:11,  2.27it/s]
running benchmark:  20%|██        | 6/30 [00:02<00:10,  2.36it/s]
running benchmark:  23%|██▎       | 7/30 [00:03<00:09,  2.42it/s]
running benchmark:  27%|██▋       | 8/30 [00:03<00:08,  2.46it/s]
running benchmark:  30%|███       | 9/30 [00:04<00:08,  2.49it/s]
running benchmark:  33%|███▎      | 10/30 [00:04<00:07,  2.51it/s]
running benchmark:  37%|███▋      | 11/30 [00:04<00:07,  2.52it/s]
running benchmark:  40%|████      | 12/30 [00:05<00:07,  2.53it/s]
running benchmark:  43%|████▎     | 13/30 [00:05<00:06,  2.54it/s]
running benchmark:  47%|████▋     | 14/30 [00:06<00:06,  2.54it/s]
running benchmark:  50%|█████     | 15/30 [00:06<00:05,  2.55it/s]
running benchmark:  53%|█████▎    | 16/30 [00:06<00:05,  2.55it/s]
running benchmark:  57%|█████▋    | 17/30 [00:07<00:05,  2.55it/s]
running benchmark:  60%|██████    | 18/30 [00:07<00:04,  2.55it/s]
running benchmark:  63%|██████▎   | 19/30 [00:08<00:04,  2.55it/s]
running benchmark:  67%|██████▋   | 20/30 [00:08<00:03,  2.55it/s]
running benchmark:  70%|███████   | 21/30 [00:08<00:03,  2.55it/s]
running benchmark:  73%|███████▎  | 22/30 [00:09<00:03,  2.55it/s]
running benchmark:  77%|███████▋  | 23/30 [00:09<00:02,  2.55it/s]
running benchmark:  80%|████████  | 24/30 [00:09<00:02,  2.56it/s]
running benchmark:  83%|████████▎ | 25/30 [00:10<00:01,  2.56it/s]
running benchmark:  87%|████████▋ | 26/30 [00:10<00:01,  2.55it/s]
running benchmark:  90%|█████████ | 27/30 [00:11<00:01,  2.55it/s]
running benchmark:  93%|█████████▎| 28/30 [00:11<00:00,  2.55it/s]
running benchmark:  97%|█████████▋| 29/30 [00:11<00:00,  2.55it/s]
running benchmark: 100%|██████████| 30/30 [00:12<00:00,  2.55it/s]
running benchmark: 100%|██████████| 30/30 [00:12<00:00,  2.44it/s]
+1.044x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/__init__.py", line 78, in __init__
+    self.meta_inputs = torch.load(f'{root}/batch.pt')
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 998, in load
+    with _open_file_like(f, 'rb') as opened_file:
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 445, in _open_file_like
+    return _open_file(name_or_buffer, mode)
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 426, in __init__
+    super().__init__(open(name, mode))
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/batch.pt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  mnasnet1_0                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 121.49it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 125.95it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 125.52it/s]
+3.020x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  mobilenet_v2                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 123.85it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 125.98it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 125.98it/s]
+4.717x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+WARNING:root:mobilenet_v2_quantized_qat failed to load
+The eval test only supports CPU.
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/mobilenet_v2_quantized_qat/__init__.py", line 21, in __init__
+    raise NotImplementedError("The eval test only supports CPU.")
+NotImplementedError: The eval test only supports CPU.
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  mobilenet_v3_large                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 82.28it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 95.15it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 96.90it/s]
+4.141x
+
loading model: 0it [00:00, ?it/s]NCCL version 2.19.3+cuda12.0
+
loading model: 0it [00:05, ?it/s]
+cuda eval  moco                               
+--Return--
+> /home/cdhernandez/local/ao/torchao/quantization/subclass.py(196)__torch_dispatch__()->None
+-> breakpoint()
+(Pdb) TIMEOUT
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+number of parameters: 123.69M
+num decayed parameter tensors: 50, with 124,354,560 parameters
+num non-decayed parameter tensors: 98, with 121,344 parameters
+using fused AdamW: True
+cuda eval  nanogpt                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 51.50it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 52.80it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 53.03it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 53.18it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 53.36it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 53.10it/s]
+7.953x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  nvidia_deeprecommender             
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 202.38it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 205.39it/s]
+0.921x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  opacus_cifar10                     
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 90, in realize_and_forward
+    return getattr(self.realize(), name)(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 328, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1213, in CALL_FUNCTION
+    self.call_function(fn, args, {})
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/nn_module.py", line 702, in call_function
+    return variables.UserFunctionVariable(fn, source=source).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1252, in CALL_FUNCTION_EX
+    self.call_function(fn, argsvars.items, kwargsvars.items)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 652, in call_function
+    self.push(fn.call_function(self, args, kwargs))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 294, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 248, in call_function
+    return super().call_function(tx, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/functions.py", line 81, in call_function
+    return tx.inline_user_function_return(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 688, in inline_user_function_return
+    return InliningInstructionTranslator.inline_call(self, fn, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2256, in inline_call
+    return cls.inline_call_(parent, func, args, kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2371, in inline_call_
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 1303, in LOAD_ATTR
+    result = BuiltinVariable(getattr).call_function(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 636, in call_function
+    result = handler(tx, *args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 1167, in call_getattr
+    return obj.var_getattr(tx, name).clone(source=source)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/user_defined.py", line 504, in var_getattr
+    return VariableBuilder(tx, source)(subobj)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 395, in _wrap
+    return self.wrap_tensor(value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 971, in wrap_tensor
+    return self.tx.output.register_attr_or_module(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 788, in register_attr_or_module
+    return wrap_name(name)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/output_graph.py", line 719, in wrap_name
+    return wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/opacus/grad_sample/grad_sample_module.py", line 148, in forward
+    return self._module(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/vision/torchvision/models/resnet.py", line 285, in forward
+    return self._forward_impl(x)
+  File "/home/cdhernandez/local/vision/torchvision/models/resnet.py", line 280, in _forward_impl
+    x = self.fc(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1562, in _call_impl
+    result = forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:28, ?it/s]
+cuda eval  phi_1_5                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:11,  2.51it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:07,  3.81it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:05,  4.57it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:05,  5.04it/s]
running benchmark:  17%|█▋        | 5/30 [00:01<00:04,  5.34it/s]
running benchmark:  20%|██        | 6/30 [00:01<00:04,  5.53it/s]
running benchmark:  23%|██▎       | 7/30 [00:01<00:04,  5.66it/s]
running benchmark:  27%|██▋       | 8/30 [00:01<00:03,  5.75it/s]
running benchmark:  30%|███       | 9/30 [00:01<00:03,  5.82it/s]
running benchmark:  33%|███▎      | 10/30 [00:01<00:03,  5.86it/s]
running benchmark:  37%|███▋      | 11/30 [00:02<00:03,  5.89it/s]
running benchmark:  40%|████      | 12/30 [00:02<00:03,  5.91it/s]
running benchmark:  43%|████▎     | 13/30 [00:02<00:02,  5.93it/s]
running benchmark:  47%|████▋     | 14/30 [00:02<00:02,  5.93it/s]
running benchmark:  50%|█████     | 15/30 [00:02<00:02,  5.94it/s]
running benchmark:  53%|█████▎    | 16/30 [00:02<00:02,  5.94it/s]
running benchmark:  57%|█████▋    | 17/30 [00:03<00:02,  5.95it/s]
running benchmark:  60%|██████    | 18/30 [00:03<00:02,  5.95it/s]
running benchmark:  63%|██████▎   | 19/30 [00:03<00:01,  5.95it/s]
running benchmark:  67%|██████▋   | 20/30 [00:03<00:01,  5.95it/s]
running benchmark:  70%|███████   | 21/30 [00:03<00:01,  5.96it/s]
running benchmark:  73%|███████▎  | 22/30 [00:03<00:01,  5.96it/s]
running benchmark:  77%|███████▋  | 23/30 [00:04<00:01,  5.96it/s]
running benchmark:  80%|████████  | 24/30 [00:04<00:01,  5.97it/s]
running benchmark:  83%|████████▎ | 25/30 [00:04<00:00,  5.97it/s]
running benchmark:  87%|████████▋ | 26/30 [00:04<00:00,  5.98it/s]
running benchmark:  90%|█████████ | 27/30 [00:04<00:00,  5.98it/s]
running benchmark:  93%|█████████▎| 28/30 [00:04<00:00,  5.98it/s]
running benchmark:  97%|█████████▋| 29/30 [00:05<00:00,  5.98it/s]
running benchmark: 100%|██████████| 30/30 [00:05<00:00,  5.98it/s]
running benchmark: 100%|██████████| 30/30 [00:05<00:00,  5.71it/s]
+1.128x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+cuda eval  phlippe_densenet                   
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 81.93it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 89.50it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 93.89it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 91.93it/s]
+4.717x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  phlippe_resnet                     
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 197.39it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 198.32it/s]
+5.094x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+cuda eval  pyhpc_equation_of_state            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 218.25it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 220.75it/s]
+17.037x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  pyhpc_isoneutral_mixing            
+skipping cudagraphs due to ['mutated inputs']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 87.13it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 88.69it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 87.72it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 87.69it/s]
+6.087x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+
loading model: 0it [00:01, ?it/s]
+WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+cuda eval  pyhpc_turbulent_kinetic_energy     
+WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 69.92it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 79.75it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 82.65it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 81.36it/s]
+8.569x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/__init__.py", line 13, in <module>
+    from .train_cyclegan import prepare_training_loop
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/train_cyclegan.py", line 27, in <module>
+    from .util.visualizer import Visualizer
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/util/visualizer.py", line 6, in <module>
+    from . import util, html
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/util/html.py", line 1, in <module>
+    import dominate
+ModuleNotFoundError: No module named 'dominate'
+Failed to import user benchmark module dynamo, error: No module named 'dominate'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/__init__.py", line 52, in __init__
+    self.data_loader = self.get_data_loader(config)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/__init__.py", line 68, in get_data_loader
+    celeba_loader = get_loader(config.celeba_image_dir, config.attr_path, config.selected_attrs,
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 82, in get_loader
+    dataset = CelebA(image_dir, attr_path, selected_attrs, transform, mode)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 24, in __init__
+    self.preprocess()
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 33, in preprocess
+    lines = [line.rstrip() for line in open(self.attr_path, 'r')]
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/data/.data/pytorch_stargan_inputs/data/celeba/list_attr_celeba.txt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  pytorch_unet                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 37.41it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 42.76it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 44.46it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 45.25it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 45.70it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 45.96it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 45.01it/s]
+1.818x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  resnet152                          
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 25.62it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 30.94it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 32.67it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 33.53it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 33.89it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 33.98it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 34.22it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 33.48it/s]
+2.304x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  resnet18                           
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 228.44it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 230.69it/s]
+3.639x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  resnet50                           
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 72.37it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 79.58it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 81.64it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 80.67it/s]
+1.822x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+WARNING:root:resnet50_quantized_qat failed to load
+The eval test only supports CPU.
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/resnet50_quantized_qat/__init__.py", line 21, in __init__
+    raise NotImplementedError("The eval test only supports CPU.")
+NotImplementedError: The eval test only supports CPU.
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  resnext50_32x4d                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 96.30it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 102.29it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 103.12it/s]
+3.828x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/__init__.py", line 27, in __init__
+    self.model = sam_model_registry[model_type](checkpoint=sam_checkpoint)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/build_sam.py", line 19, in build_sam_vit_h
+    return _build_sam(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/build_sam.py", line 108, in _build_sam
+    with open(checkpoint, "rb") as f:
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/.data/sam_vit_h_4b8939.pth'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  shufflenet_v2_x1_0                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 77.74it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 87.65it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 91.15it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 89.48it/s]
+3.600x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  soft_actor_critic                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 689.57it/s]
+3.327x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/speech_transformer/__init__.py", line 15, in <module>
+    from .config import SpeechTransformerTrainConfig, SpeechTransformerEvalConfig
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/speech_transformer/config.py", line 4, in <module>
+    import kaldi_io
+ModuleNotFoundError: No module named 'kaldi_io'
+Failed to import user benchmark module dynamo, error: No module named 'kaldi_io'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  squeezenet1_1                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 208.44it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 224.04it/s]
+4.246x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/stable_diffusion_text_encoder/__init__.py", line 11, in <module>
+    from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+ModuleNotFoundError: No module named 'diffusers'
+Failed to import user benchmark module dynamo, error: No module named 'diffusers'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/stable_diffusion_unet/__init__.py", line 11, in <module>
+    from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+ModuleNotFoundError: No module named 'diffusers'
+Failed to import user benchmark module dynamo, error: No module named 'diffusers'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/timm_efficientdet/__init__.py", line 12, in <module>
+    from effdet import create_model, create_loader
+ModuleNotFoundError: No module named 'effdet'
+Failed to import user benchmark module dynamo, error: No module named 'effdet'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  timm_efficientnet                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  7.45it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 36.89it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 47.52it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 52.76it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 55.74it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 49.25it/s]
+2.207x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  timm_nfnet                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:04,  6.74it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 11.51it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 13.22it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 14.01it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 14.48it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 14.70it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 14.91it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:00, 15.03it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 15.12it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 15.18it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 15.21it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 15.25it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 15.26it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 15.27it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 15.26it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 14.67it/s]
+1.861x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  timm_regnet                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 23.82it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 29.81it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 31.84it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 32.81it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 33.35it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 33.68it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 33.87it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 32.85it/s]
+1.353x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  timm_resnest                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 79.90it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 96.77it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 102.11it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 98.95it/s] 
+1.678x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  timm_vision_transformer            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:06,  4.83it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:03,  8.99it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02, 10.63it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:02, 11.47it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 11.95it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 12.26it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 12.45it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 12.58it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:01, 12.67it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 12.73it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 12.78it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 12.81it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00, 12.83it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 12.84it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 12.85it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 12.19it/s]
+1.046x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:19, ?it/s]
+cuda eval  timm_vision_transformer_large      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:06<02:57,  6.11s/it]
running benchmark:   7%|▋         | 2/30 [00:08<01:50,  3.95s/it]
running benchmark:  10%|█         | 3/30 [00:11<01:28,  3.27s/it]
running benchmark:  13%|█▎        | 4/30 [00:13<01:16,  2.94s/it]
running benchmark:  17%|█▋        | 5/30 [00:15<01:09,  2.76s/it]
running benchmark:  20%|██        | 6/30 [00:18<01:03,  2.66s/it]
running benchmark:  23%|██▎       | 7/30 [00:20<00:59,  2.59s/it]
running benchmark:  27%|██▋       | 8/30 [00:23<00:55,  2.54s/it]
running benchmark:  30%|███       | 9/30 [00:25<00:52,  2.51s/it]
running benchmark:  33%|███▎      | 10/30 [00:28<00:49,  2.49s/it]
running benchmark:  37%|███▋      | 11/30 [00:30<00:47,  2.48s/it]
running benchmark:  40%|████      | 12/30 [00:33<00:44,  2.47s/it]
running benchmark:  43%|████▎     | 13/30 [00:35<00:41,  2.46s/it]
running benchmark:  47%|████▋     | 14/30 [00:37<00:39,  2.46s/it]
running benchmark:  50%|█████     | 15/30 [00:40<00:36,  2.46s/it]
running benchmark:  53%|█████▎    | 16/30 [00:42<00:34,  2.45s/it]
running benchmark:  57%|█████▋    | 17/30 [00:45<00:31,  2.45s/it]
running benchmark:  60%|██████    | 18/30 [00:47<00:29,  2.45s/it]
running benchmark:  63%|██████▎   | 19/30 [00:50<00:26,  2.45s/it]
running benchmark:  67%|██████▋   | 20/30 [00:52<00:24,  2.45s/it]
running benchmark:  70%|███████   | 21/30 [00:55<00:22,  2.45s/it]
running benchmark:  73%|███████▎  | 22/30 [00:57<00:19,  2.45s/it]
running benchmark:  77%|███████▋  | 23/30 [00:59<00:17,  2.45s/it]
running benchmark:  80%|████████  | 24/30 [01:02<00:14,  2.45s/it]
running benchmark:  83%|████████▎ | 25/30 [01:04<00:12,  2.45s/it]
running benchmark:  87%|████████▋ | 26/30 [01:07<00:09,  2.45s/it]
running benchmark:  90%|█████████ | 27/30 [01:09<00:07,  2.45s/it]
running benchmark:  93%|█████████▎| 28/30 [01:12<00:04,  2.45s/it]
running benchmark:  97%|█████████▋| 29/30 [01:14<00:02,  2.45s/it]
running benchmark: 100%|██████████| 30/30 [01:17<00:00,  2.45s/it]
running benchmark: 100%|██████████| 30/30 [01:17<00:00,  2.57s/it]
+1.024x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  timm_vovnet                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 72.12it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 81.53it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 84.73it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 83.58it/s]
+1.761x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/torch_multimodal_clip/__init__.py", line 27, in __init__
+    self.image = Image.open(os.path.join(self.data_folder, self.image_name))
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/PIL/Image.py", line 3218, in open
+    fp = builtins.open(filename, "rb")
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/torch_multimodal_clip/.data/pizza.jpg'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model tts_angular supports float32 only
+
loading model: 0it [00:01, ?it/s]
+WARNING:common:Model tts_angular supports float32 only
+cuda eval  tts_angular                        
+WARNING:common:Model tts_angular supports float32 only
+ERROR:common:Backend dynamo failed in warmup()
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 2378, in warmup
+    fn(model, example_inputs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 489, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/tts_angular/model.py", line 59, in forward
+    d = self.layers(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/container.py", line 217, in forward
+    input = module(input)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/tts_angular/model.py", line 19, in forward
+    return self.linear(o)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/linear.py", line 116, in forward
+    return F.linear(input, self.weight, self.bias)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/eval_frame.py", line 655, in catch_errors
+    return callback(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 722, in _convert_frame
+    result = inner_convert(frame, cache_entry, hooks, frame_state)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 383, in _convert_frame_assert
+    compiled_product = _compile(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 646, in _compile
+    guarded_code = compile_inner(code, one_graph, hooks, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 244, in time_wrapper
+    r = func(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 562, in compile_inner
+    out_code = transform_code_object(code, transform)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/bytecode_transformation.py", line 1033, in transform_code_object
+    transformations(instructions, code_options)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 151, in _fn
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/convert_frame.py", line 527, in transform
+    tracer.run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 2123, in run
+    super().run()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 818, in run
+    and self.step()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 781, in step
+    getattr(self, inst.opname)(inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 470, in wrapper
+    return inner_fn(self, inst)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/symbolic_convert.py", line 249, in impl
+    self.push(fn_var.call_function(self, self.popn(nargs), {}))
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in call_function
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builtin.py", line 479, in <listcomp>
+    args = [v.realize() for v in args]
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 56, in realize
+    self._cache.realize(self.parents_tracker)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/lazy.py", line 22, in realize
+    self.vt = VariableBuilder(tx, self.source)(self.value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 380, in _wrap
+    return type_dispatch(self, value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 824, in wrap_listlike
+    output = [
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 825, in <listcomp>
+    VariableBuilder(self.tx, GetItemSource(self.get_source(), i))(item)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 244, in __call__
+    vt = self._wrap(value).clone(**self.options())
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 395, in _wrap
+    return self.wrap_tensor(value)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1049, in wrap_tensor
+    tensor_variable = wrap_fx_proxy(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1291, in wrap_fx_proxy
+    return wrap_fx_proxy_cls(target_cls=TensorVariable, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1401, in wrap_fx_proxy_cls
+    example_value = wrap_to_fake_tensor_and_record(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1767, in wrap_to_fake_tensor_and_record
+    fake_e = wrap_fake_exception(
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/utils.py", line 1026, in wrap_fake_exception
+    return fn()
+  File "/home/cdhernandez/local/pytorch/torch/_dynamo/variables/builder.py", line 1768, in <lambda>
+    lambda: tx.fake_mode.from_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 1884, in from_tensor
+    return self.fake_tensor_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 396, in __call__
+    return self.from_real_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 353, in from_real_tensor
+    out = self.meta_converter(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 710, in __call__
+    r = self.meta_tensor(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 530, in meta_tensor
+    r = transform_subclass(
+  File "/home/cdhernandez/local/pytorch/torch/utils/_python_dispatch.py", line 168, in transform_subclass
+    transformed_tensors_dict[attr] = callback(attr, getattr(t, attr))
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 532, in <lambda>
+    lambda attr, inner_t: callback(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/fake_tensor.py", line 348, in mk_fake_tensor
+    make_meta_t(),
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 533, in <lambda>
+    lambda: empty_create(
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 460, in empty_create
+    ) = sym_sizes_strides_storage_offset(inner_t, inner_src)
+  File "/home/cdhernandez/local/pytorch/torch/_subclasses/meta_utils.py", line 246, in sym_sizes_strides_storage_offset
+    return shape_env.create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2108, in create_symbolic_sizes_strides_storage_offset
+    return self._create_symbolic_sizes_strides_storage_offset(
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/recording.py", line 226, in wrapper
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/fx/experimental/symbolic_shapes.py", line 2158, in _create_symbolic_sizes_strides_storage_offset
+    assert len(dynamic_dims) == dim, f"{len(dynamic_dims)} != {dim}"
+AssertionError: 2 != 4
+
+from user code:
+   File "/home/cdhernandez/local/ao/torchao/quantization/subclass.py", line 154, in __torch_dispatch__
+    new = args[0]._change_shape(args[0].shape[::-1])
+
+Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
+
+
+You can suppress this exception and fall back to eager by setting:
+    import torch._dynamo
+    torch._dynamo.config.suppress_errors = True
+
+Run failed with return code:  255
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  vgg16                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 287.59it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 287.20it/s]
+1.673x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model vision_maskrcnn supports float32 only
+
loading model: 0it [00:05, ?it/s]
+WARNING:common:Model vision_maskrcnn supports float32 only
+cuda eval  vision_maskrcnn                    
+WARNING:common:Model vision_maskrcnn supports float32 only
+AUTOTUNE convolution(31x256x14x14, 256x256x3x3)
+  convolution 0.0873 ms 100.0%
+  triton_convolution_769 0.2035 ms 42.9%
+  triton_convolution_771 0.2211 ms 39.5%
+  triton_convolution_770 0.2560 ms 34.1%
+  triton_convolution_772 0.2656 ms 32.9%
+  triton_convolution_766 0.3074 ms 28.4%
+  triton_convolution_767 0.4987 ms 17.5%
+  triton_convolution_768 0.5935 ms 14.7%
+SingleProcess AUTOTUNE takes 7.0614 seconds
+AUTOTUNE addmm(24304x91, 24304x256, 256x91)
+  triton_mm_795 0.0431 ms 100.0%
+  triton_mm_796 0.0451 ms 95.6%
+  triton_mm_794 0.0466 ms 92.6%
+  triton_mm_797 0.0480 ms 89.7%
+  bias_addmm 0.0494 ms 87.2%
+  triton_mm_798 0.0528 ms 81.6%
+  triton_mm_801 0.0531 ms 81.2%
+  triton_mm_799 0.0577 ms 74.8%
+  triton_mm_804 0.0605 ms 71.2%
+  addmm 0.0607 ms 71.0%
+SingleProcess AUTOTUNE takes 6.7994 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:02,  9.81it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 11.91it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 12.51it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 12.70it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 12.79it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 13.00it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 13.01it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 13.04it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 13.12it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 13.23it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 13.26it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 13.24it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 13.26it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 13.21it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 13.13it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 13.00it/s]
+1.083x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/yolov3/__init__.py", line 28, in <module>
+    assert os.path.exists(DATA_DIR), "Couldn't find coco128 data dir, please run install.py again."
+AssertionError: Couldn't find coco128 data dir, please run install.py again.
+Run failed with return code:  1
+Output:  None
+Error:  None
+speedup             gmean=0.00x mean=2.098x
+abs_latency         gmean=0.00x mean=29.388x
+compilation_latency mean=17.506 seconds
+compression_ratio   mean=0.802x
+eager_peak_mem      gmean=0.00x mean=0.492x
+dynamo_peak_mem     gmean=0.00x mean=0.562x
+calls_captured      gmean=0.00x mean=347.270x
+unique_graphs       gmean=0.00x mean=2.027x
+graph_breaks        gmean=0.00x mean=0.878x
+unique_graph_breaks gmean=0.00x mean=0.203x
+start baseline
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/fbgemm_gpu_py.so: undefined symbol: _ZNK5torch8autograd4Node4nameEv
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/canary_models/torchrec_dlrm/__init__.py", line 7, in <module>
+    from .data.dlrm_dataloader import get_dataloader
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/canary_models/torchrec_dlrm/data/dlrm_dataloader.py", line 13, in <module>
+    from torchrec.datasets.criteo import (
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/__init__.py", line 8, in <module>
+    import torchrec.distributed  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/__init__.py", line 36, in <module>
+    from torchrec.distributed.model_parallel import DistributedModelParallel  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/model_parallel.py", line 21, in <module>
+    from torchrec.distributed.planner import EmbeddingShardingPlanner, Topology
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/__init__.py", line 22, in <module>
+    from torchrec.distributed.planner.planners import EmbeddingShardingPlanner  # noqa
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/planners.py", line 19, in <module>
+    from torchrec.distributed.planner.constants import BATCH_SIZE, MAX_SIZE
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/planner/constants.py", line 10, in <module>
+    from torchrec.distributed.embedding_types import EmbeddingComputeKernel
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/torchrec/distributed/embedding_types.py", line 14, in <module>
+    from fbgemm_gpu.split_table_batched_embeddings_ops_training import EmbeddingLocation
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/__init__.py", line 22, in <module>
+    from . import _fbgemm_gpu_docs  # noqa: F401, E402
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/fbgemm_gpu/_fbgemm_gpu_docs.py", line 19, in <module>
+    torch.ops.fbgemm.jagged_2d_to_dense,
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 820, in __getattr__
+    raise AttributeError(
+AttributeError: '_OpNamespace' 'fbgemm' object has no attribute 'jagged_2d_to_dense'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  BERT_pytorch                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:07,  3.75it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 29.03it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 43.70it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 52.78it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 44.41it/s]
+3.313x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  Background_Matting                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 35.22it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 39.84it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 41.31it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 41.95it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 42.38it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 42.57it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 41.77it/s]
+2.105x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model DALLE2_pytorch supports float32 only
+
loading model: 0it [00:10, ?it/s]
+WARNING:common:Model DALLE2_pytorch supports float32 only
+cuda eval  DALLE2_pytorch                     
+WARNING:common:Model DALLE2_pytorch supports float32 only
+AUTOTUNE mm(154x512, 512x1536)
+  triton_mm_8 0.0151 ms 100.0%
+  triton_mm_5 0.0172 ms 88.1%
+  triton_mm_6 0.0173 ms 87.6%
+  triton_mm_3 0.0174 ms 86.9%
+  triton_mm_4 0.0181 ms 83.4%
+  triton_mm_1 0.0182 ms 83.3%
+  triton_mm_2 0.0182 ms 83.1%
+  mm 0.0193 ms 78.3%
+  triton_mm_9 0.0195 ms 77.8%
+  triton_mm_0 0.0232 ms 65.2%
+SingleProcess AUTOTUNE takes 6.3495 seconds
+[2023-12-06 19:28:02,477] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+AUTOTUNE mm(154x512, 512x512)
+  triton_mm_17 0.0117 ms 100.0%
+  triton_mm_21 0.0118 ms 99.2%
+  triton_mm_18 0.0121 ms 97.1%
+  triton_mm_20 0.0138 ms 85.2%
+  mm 0.0146 ms 80.7%
+  triton_mm_14 0.0170 ms 69.0%
+  triton_mm_15 0.0170 ms 69.0%
+  triton_mm_16 0.0172 ms 68.1%
+  triton_mm_13 0.0175 ms 67.1%
+  triton_mm_12 0.0225 ms 52.2%
+SingleProcess AUTOTUNE takes 6.6042 seconds
+AUTOTUNE mm(154x512, 512x2048)
+  mm 0.0150 ms 100.0%
+  triton_mm_32 0.0152 ms 98.9%
+  triton_mm_30 0.0173 ms 87.0%
+  triton_mm_28 0.0180 ms 83.5%
+  triton_mm_27 0.0181 ms 83.0%
+  triton_mm_29 0.0182 ms 82.7%
+  triton_mm_26 0.0182 ms 82.6%
+  triton_mm_25 0.0186 ms 80.8%
+  triton_mm_33 0.0227 ms 66.4%
+  triton_mm_24 0.0236 ms 63.6%
+SingleProcess AUTOTUNE takes 6.3516 seconds
+AUTOTUNE mm(154x2048, 2048x512)
+  mm 0.0193 ms 100.0%
+  triton_mm_41 0.0260 ms 74.2%
+  triton_mm_42 0.0261 ms 73.7%
+  triton_mm_45 0.0278 ms 69.3%
+  triton_mm_44 0.0323 ms 59.6%
+  triton_mm_39 0.0436 ms 44.2%
+  triton_mm_40 0.0449 ms 42.9%
+  triton_mm_38 0.0452 ms 42.6%
+  triton_mm_36 0.0683 ms 28.2%
+  triton_mm_37 0.0755 ms 25.5%
+SingleProcess AUTOTUNE takes 6.6746 seconds
+[2023-12-06 19:28:22,521] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:22,769] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:23,018] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:23,256] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:23,486] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:23,712] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:23,942] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:24,176] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:24,409] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:24,644] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:28:24,876] [4/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+AUTOTUNE mm(2x512, 512x512)
+  triton_mm_582 0.0104 ms 100.0%
+  triton_mm_581 0.0108 ms 95.6%
+  triton_mm_585 0.0110 ms 93.9%
+  triton_mm_584 0.0113 ms 91.8%
+  triton_mm_580 0.0115 ms 90.0%
+  mm 0.0126 ms 82.4%
+  triton_mm_579 0.0127 ms 81.9%
+  triton_mm_578 0.0140 ms 74.0%
+  triton_mm_577 0.0151 ms 68.5%
+  triton_mm_576 0.0209 ms 49.5%
+SingleProcess AUTOTUNE takes 4.1315 seconds
+skipping cudagraphs due to ['non-cuda device in graph']
+AUTOTUNE mm(2x512, 512x1024)
+  triton_mm_596 0.0113 ms 100.0%
+  triton_mm_593 0.0113 ms 99.7%
+  triton_mm_594 0.0116 ms 97.8%
+  triton_mm_592 0.0120 ms 94.1%
+  triton_mm_597 0.0123 ms 91.9%
+  triton_mm_591 0.0129 ms 87.4%
+  mm 0.0135 ms 83.6%
+  triton_mm_590 0.0143 ms 79.1%
+  triton_mm_589 0.0154 ms 73.2%
+  triton_mm_588 0.0214 ms 52.8%
+SingleProcess AUTOTUNE takes 4.0763 seconds
+AUTOTUNE mm(2x1024, 1024x1024)
+  triton_mm_605 0.0150 ms 100.0%
+  triton_mm_606 0.0151 ms 99.8%
+  mm 0.0154 ms 97.9%
+  triton_mm_608 0.0156 ms 96.5%
+  triton_mm_604 0.0171 ms 87.9%
+  triton_mm_609 0.0177 ms 84.8%
+  triton_mm_603 0.0189 ms 79.5%
+  triton_mm_602 0.0215 ms 70.0%
+  triton_mm_601 0.0233 ms 64.5%
+  triton_mm_600 0.0350 ms 43.0%
+SingleProcess AUTOTUNE takes 5.2870 seconds
+AUTOTUNE addmm(2x512, 2x1024, 1024x512)
+  triton_mm_618 0.0150 ms 100.0%
+  bias_addmm 0.0151 ms 99.4%
+  triton_mm_617 0.0151 ms 99.4%
+  triton_mm_620 0.0163 ms 91.8%
+  triton_mm_616 0.0172 ms 87.2%
+  triton_mm_621 0.0175 ms 85.7%
+  addmm 0.0191 ms 78.4%
+  triton_mm_615 0.0193 ms 77.6%
+  triton_mm_614 0.0216 ms 69.2%
+  triton_mm_613 0.0237 ms 63.3%
+SingleProcess AUTOTUNE takes 5.0822 seconds
+AUTOTUNE mm(520x512, 512x128)
+  triton_mm_630 0.0116 ms 100.0%
+  triton_mm_633 0.0118 ms 97.8%
+  triton_mm_629 0.0122 ms 94.8%
+  mm 0.0135 ms 85.7%
+  triton_mm_632 0.0142 ms 81.1%
+  triton_mm_626 0.0166 ms 69.6%
+  triton_mm_625 0.0168 ms 68.8%
+  triton_mm_627 0.0170 ms 67.9%
+  triton_mm_628 0.0172 ms 67.0%
+  triton_mm_624 0.0223 ms 51.8%
+SingleProcess AUTOTUNE takes 6.0956 seconds
+AUTOTUNE mm(520x512, 512x512)
+  mm 0.0138 ms 100.0%
+  triton_mm_644 0.0142 ms 97.3%
+  triton_mm_642 0.0167 ms 82.6%
+  triton_mm_641 0.0168 ms 82.4%
+  triton_mm_639 0.0171 ms 80.7%
+  triton_mm_638 0.0174 ms 79.6%
+  triton_mm_640 0.0175 ms 78.8%
+  triton_mm_637 0.0176 ms 78.7%
+  triton_mm_645 0.0209 ms 66.1%
+  triton_mm_636 0.0228 ms 60.6%
+SingleProcess AUTOTUNE takes 6.3077 seconds
+AUTOTUNE bmm(2x2080x64, 2x64x261)
+  triton_bmm_650 0.0138 ms 100.0%
+  triton_bmm_648 0.0142 ms 96.6%
+  triton_bmm_649 0.0145 ms 95.1%
+  triton_bmm_658 0.0145 ms 95.1%
+  triton_bmm_657 0.0148 ms 92.9%
+  triton_bmm_659 0.0150 ms 91.5%
+  triton_bmm_655 0.0156 ms 88.5%
+  triton_bmm_652 0.0156 ms 88.1%
+  triton_bmm_651 0.0163 ms 84.5%
+  triton_bmm_654 0.0177 ms 77.8%
+SingleProcess AUTOTUNE takes 6.2312 seconds
+AUTOTUNE bmm(2x2080x261, 2x261x64)
+  triton_bmm_663 0.0146 ms 100.0%
+  triton_bmm_661 0.0148 ms 98.5%
+  triton_bmm_668 0.0162 ms 90.0%
+  triton_bmm_666 0.0172 ms 85.1%
+  bmm 0.0174 ms 84.0%
+  triton_bmm_660 0.0188 ms 78.0%
+  triton_bmm_669 0.0198 ms 73.9%
+  triton_bmm_665 0.0204 ms 71.6%
+  triton_bmm_662 0.0208 ms 70.4%
+  triton_bmm_664 0.0215 ms 68.1%
+SingleProcess AUTOTUNE takes 5.4410 seconds
+AUTOTUNE bmm(2x1x512, 2x512x1)
+  triton_bmm_987 0.0087 ms 100.0%
+  triton_bmm_986 0.0092 ms 94.8%
+  triton_bmm_988 0.0092 ms 94.5%
+  triton_bmm_989 0.0097 ms 90.1%
+  triton_bmm_985 0.0111 ms 78.9%
+  bmm 0.0123 ms 71.3%
+  triton_bmm_984 0.0149 ms 58.5%
+  triton_bmm_991 0.0223 ms 39.1%
+  triton_bmm_990 0.0226 ms 38.7%
+SingleProcess AUTOTUNE takes 2.8219 seconds
+AUTOTUNE mm(77x512, 512x1536)
+  triton_mm_998 0.0130 ms 100.0%
+  triton_mm_997 0.0138 ms 94.4%
+  triton_mm_1000 0.0146 ms 89.5%
+  triton_mm_1001 0.0149 ms 87.5%
+  mm 0.0162 ms 80.3%
+  triton_mm_996 0.0176 ms 74.2%
+  triton_mm_995 0.0178 ms 73.2%
+  triton_mm_993 0.0179 ms 72.8%
+  triton_mm_994 0.0179 ms 72.8%
+  triton_mm_992 0.0240 ms 54.3%
+SingleProcess AUTOTUNE takes 6.1631 seconds
+[2023-12-06 19:30:02,893] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+AUTOTUNE mm(77x512, 512x512)
+  triton_mm_1013 0.0120 ms 100.0%
+  triton_mm_1010 0.0120 ms 99.7%
+  triton_mm_1009 0.0121 ms 99.2%
+  mm 0.0131 ms 91.2%
+  triton_mm_1012 0.0136 ms 88.0%
+  triton_mm_1008 0.0168 ms 71.2%
+  triton_mm_1006 0.0171 ms 70.2%
+  triton_mm_1005 0.0173 ms 69.1%
+  triton_mm_1007 0.0174 ms 68.7%
+  triton_mm_1004 0.0228 ms 52.5%
+SingleProcess AUTOTUNE takes 6.3475 seconds
+AUTOTUNE mm(77x512, 512x2048)
+  triton_mm_1022 0.0136 ms 100.0%
+  mm 0.0151 ms 89.8%
+  triton_mm_1024 0.0154 ms 88.1%
+  triton_mm_1025 0.0160 ms 84.8%
+  triton_mm_1021 0.0175 ms 77.5%
+  triton_mm_1020 0.0179 ms 76.0%
+  triton_mm_1019 0.0181 ms 75.0%
+  triton_mm_1018 0.0181 ms 74.8%
+  triton_mm_1017 0.0186 ms 73.0%
+  triton_mm_1016 0.0238 ms 57.0%
+SingleProcess AUTOTUNE takes 6.3198 seconds
+AUTOTUNE mm(77x2048, 2048x512)
+  mm 0.0178 ms 100.0%
+  triton_mm_1033 0.0257 ms 69.0%
+  triton_mm_1034 0.0259 ms 68.6%
+  triton_mm_1037 0.0271 ms 65.6%
+  triton_mm_1036 0.0324 ms 54.8%
+  triton_mm_1031 0.0441 ms 40.3%
+  triton_mm_1032 0.0445 ms 39.9%
+  triton_mm_1030 0.0447 ms 39.7%
+  triton_mm_1029 0.0454 ms 39.1%
+  triton_mm_1028 0.0690 ms 25.7%
+SingleProcess AUTOTUNE takes 6.4915 seconds
+[2023-12-06 19:30:22,413] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:22,634] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:22,852] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:23,074] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:23,299] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:23,961] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:24,189] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:24,417] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:24,635] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:24,853] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 19:30:25,078] [27/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+AUTOTUNE mm(1x512, 512x512)
+  triton_mm_1577 0.0112 ms 100.0%
+  triton_mm_1573 0.0113 ms 98.9%
+  triton_mm_1576 0.0114 ms 98.3%
+  triton_mm_1574 0.0119 ms 93.8%
+  mm 0.0125 ms 89.7%
+  triton_mm_1572 0.0125 ms 89.6%
+  triton_mm_1571 0.0131 ms 85.6%
+  triton_mm_1570 0.0147 ms 76.4%
+  triton_mm_1569 0.0150 ms 74.8%
+  triton_mm_1568 0.0205 ms 54.7%
+SingleProcess AUTOTUNE takes 4.2800 seconds
+skipping cudagraphs due to ['non-cuda device in graph']
+AUTOTUNE mm(1x128, 128x512)
+  mm 0.0069 ms 100.0%
+  triton_mm_1586 0.0074 ms 93.5%
+  triton_mm_1584 0.0076 ms 91.6%
+  triton_mm_1588 0.0078 ms 88.6%
+  triton_mm_1585 0.0079 ms 88.2%
+  triton_mm_1582 0.0083 ms 83.8%
+  triton_mm_1583 0.0083 ms 83.8%
+  triton_mm_1581 0.0086 ms 80.4%
+  triton_mm_1580 0.0092 ms 75.1%
+  triton_mm_1589 0.0102 ms 68.3%
+SingleProcess AUTOTUNE takes 4.2822 seconds
+AUTOTUNE addmm(1x256, 1x512, 512x256)
+  bias_addmm 0.0094 ms 100.0%
+  triton_mm_1597 0.0104 ms 90.2%
+  addmm 0.0106 ms 88.8%
+  triton_mm_1600 0.0112 ms 83.7%
+  triton_mm_1598 0.0112 ms 83.4%
+  triton_mm_1601 0.0115 ms 81.4%
+  triton_mm_1596 0.0117 ms 79.8%
+  triton_mm_1595 0.0135 ms 69.3%
+  triton_mm_1594 0.0141 ms 66.6%
+  triton_mm_1593 0.0150 ms 62.6%
+SingleProcess AUTOTUNE takes 4.5426 seconds
+AUTOTUNE mm(1x512, 512x512)
+  mm 0.0079 ms 100.0%
+  triton_mm_1610 0.0106 ms 74.3%
+  triton_mm_1609 0.0108 ms 72.8%
+  triton_mm_1613 0.0110 ms 71.3%
+  triton_mm_1612 0.0113 ms 69.7%
+  triton_mm_1608 0.0120 ms 65.8%
+  triton_mm_1607 0.0132 ms 59.7%
+  triton_mm_1606 0.0136 ms 58.0%
+  triton_mm_1605 0.0143 ms 55.0%
+  triton_mm_1604 0.0204 ms 38.5%
+SingleProcess AUTOTUNE takes 4.4615 seconds
+AUTOTUNE addmm(77x128, 77x512, 512x128)
+  triton_mm_1625 0.0115 ms 100.0%
+  triton_mm_1622 0.0120 ms 96.3%
+  triton_mm_1621 0.0122 ms 94.5%
+  bias_addmm 0.0123 ms 93.5%
+  triton_mm_1624 0.0145 ms 79.5%
+  addmm 0.0156 ms 73.8%
+  triton_mm_1618 0.0177 ms 65.2%
+  triton_mm_1617 0.0177 ms 65.0%
+  triton_mm_1620 0.0180 ms 63.9%
+  triton_mm_1619 0.0182 ms 63.3%
+SingleProcess AUTOTUNE takes 6.8521 seconds
+AUTOTUNE convolution(1x3x128x128, 64x3x3x3)
+  triton_convolution_1628 0.0193 ms 100.0%
+  triton_convolution_1633 0.0225 ms 85.9%
+  triton_convolution_1632 0.0238 ms 81.0%
+  convolution 0.0241 ms 80.1%
+  triton_convolution_1631 0.0268 ms 72.0%
+  triton_convolution_1630 0.0377 ms 51.2%
+  triton_convolution_1629 0.0551 ms 35.0%
+SingleProcess AUTOTUNE takes 3.9722 seconds
+AUTOTUNE convolution(1x3x128x128, 32x3x7x7)
+  convolution 0.0281 ms 100.0%
+  triton_convolution_1634 0.0543 ms 51.7%
+  triton_convolution_1637 0.0606 ms 46.3%
+  triton_convolution_1638 0.0609 ms 46.1%
+  triton_convolution_1639 0.0711 ms 39.5%
+  triton_convolution_1635 0.1052 ms 26.7%
+  triton_convolution_1636 0.1453 ms 19.3%
+SingleProcess AUTOTUNE takes 3.3674 seconds
+AUTOTUNE convolution(1x3x128x128, 32x3x15x15)
+  convolution 0.0924 ms 100.0%
+  triton_convolution_1640 0.2295 ms 40.2%
+  triton_convolution_1643 0.2399 ms 38.5%
+  triton_convolution_1644 0.2478 ms 37.3%
+  triton_convolution_1645 0.3002 ms 30.8%
+  triton_convolution_1641 0.4502 ms 20.5%
+  triton_convolution_1642 0.6021 ms 15.3%
+SingleProcess AUTOTUNE takes 3.4128 seconds
+AUTOTUNE mm(1x512, 512x256)
+  mm 0.0074 ms 100.0%
+  triton_mm_1675 0.0102 ms 72.4%
+  triton_mm_1678 0.0106 ms 69.8%
+  triton_mm_1679 0.0106 ms 69.8%
+  triton_mm_1676 0.0106 ms 69.6%
+  triton_mm_1674 0.0115 ms 64.2%
+  triton_mm_1673 0.0124 ms 59.4%
+  triton_mm_1672 0.0131 ms 56.3%
+  triton_mm_1671 0.0145 ms 51.0%
+  triton_mm_1670 0.0193 ms 38.2%
+SingleProcess AUTOTUNE takes 4.1773 seconds
+AUTOTUNE convolution(1x128x64x64, 128x128x3x3)
+  convolution 0.0424 ms 100.0%
+  triton_convolution_1766 0.1075 ms 39.5%
+  triton_convolution_1765 0.1146 ms 37.0%
+  triton_convolution_1764 0.1151 ms 36.9%
+  triton_convolution_1760 0.1190 ms 35.7%
+  triton_convolution_1761 0.1201 ms 35.3%
+  triton_convolution_1763 0.1230 ms 34.5%
+  triton_convolution_1762 0.2449 ms 17.3%
+SingleProcess AUTOTUNE takes 5.9779 seconds
+AUTOTUNE mm(6x128, 128x1024)
+  triton_mm_1798 0.0079 ms 100.0%
+  triton_mm_1801 0.0080 ms 98.0%
+  triton_mm_1799 0.0081 ms 97.6%
+  triton_mm_1797 0.0081 ms 97.0%
+  triton_mm_1795 0.0083 ms 95.0%
+  triton_mm_1796 0.0083 ms 95.0%
+  triton_mm_1794 0.0083 ms 94.6%
+  triton_mm_1793 0.0094 ms 83.4%
+  mm 0.0098 ms 80.4%
+  triton_mm_1800 0.0102 ms 77.4%
+SingleProcess AUTOTUNE takes 4.0162 seconds
+AUTOTUNE mm(4096x128, 128x512)
+  triton_mm_1815 0.0191 ms 100.0%
+  triton_mm_1807 0.0192 ms 99.7%
+  triton_mm_1806 0.0193 ms 99.0%
+  triton_mm_1805 0.0207 ms 92.1%
+  triton_mm_1812 0.0215 ms 88.8%
+  mm 0.0220 ms 86.8%
+  triton_mm_1809 0.0229 ms 83.4%
+  triton_mm_1808 0.0231 ms 82.6%
+  triton_mm_1810 0.0262 ms 73.0%
+  triton_mm_1813 0.0268 ms 71.2%
+SingleProcess AUTOTUNE takes 6.3821 seconds
+AUTOTUNE bmm(8x4096x64, 8x64x7)
+  triton_bmm_1827 0.0151 ms 100.0%
+  triton_bmm_1824 0.0151 ms 99.6%
+  triton_bmm_1817 0.0154 ms 97.9%
+  triton_bmm_1828 0.0154 ms 97.7%
+  triton_bmm_1818 0.0158 ms 95.3%
+  triton_bmm_1820 0.0159 ms 94.8%
+  triton_bmm_1825 0.0159 ms 94.6%
+  triton_bmm_1822 0.0161 ms 93.6%
+  triton_bmm_1823 0.0162 ms 93.3%
+  triton_bmm_1826 0.0162 ms 93.1%
+SingleProcess AUTOTUNE takes 4.1472 seconds
+AUTOTUNE bmm(8x4096x7, 8x7x64)
+  triton_bmm_1834 0.0117 ms 100.0%
+  triton_bmm_1838 0.0117 ms 100.0%
+  triton_bmm_1829 0.0119 ms 98.4%
+  triton_bmm_1830 0.0119 ms 98.4%
+  triton_bmm_1839 0.0119 ms 98.4%
+  triton_bmm_1835 0.0122 ms 96.3%
+  triton_bmm_1832 0.0122 ms 96.1%
+  triton_bmm_1837 0.0122 ms 96.1%
+  triton_bmm_1833 0.0125 ms 93.9%
+  triton_bmm_1836 0.0125 ms 93.9%
+SingleProcess AUTOTUNE takes 3.8753 seconds
+AUTOTUNE mm(4096x512, 512x128)
+  mm 0.0183 ms 100.0%
+  triton_mm_1844 0.0189 ms 96.5%
+  triton_mm_1843 0.0190 ms 96.3%
+  triton_mm_1841 0.0203 ms 90.1%
+  triton_mm_1842 0.0204 ms 89.8%
+  triton_mm_1846 0.0234 ms 78.2%
+  triton_mm_1845 0.0240 ms 76.1%
+  triton_mm_1848 0.0249 ms 73.4%
+  triton_mm_1840 0.0273 ms 66.9%
+  triton_mm_1849 0.0297 ms 61.5%
+SingleProcess AUTOTUNE takes 6.1658 seconds
+AUTOTUNE convolution(1x128x64x64, 128x128x3x3)
+  convolution 0.0390 ms 100.0%
+  triton_convolution_1856 0.0685 ms 57.0%
+  triton_convolution_1857 0.0706 ms 55.3%
+  triton_convolution_1852 0.0941 ms 41.5%
+  triton_convolution_1855 0.1048 ms 37.3%
+  triton_convolution_1858 0.1480 ms 26.4%
+  triton_convolution_1853 0.2553 ms 15.3%
+  triton_convolution_1854 0.4272 ms 9.1%
+SingleProcess AUTOTUNE takes 6.6394 seconds
+AUTOTUNE convolution(1x256x32x32, 256x256x3x3)
+  convolution 0.0485 ms 100.0%
+  triton_convolution_1960 0.1355 ms 35.8%
+  triton_convolution_1962 0.1948 ms 24.9%
+  triton_convolution_1959 0.2230 ms 21.8%
+  triton_convolution_1957 0.2250 ms 21.6%
+  triton_convolution_1961 0.3377 ms 14.4%
+  triton_convolution_1956 0.3919 ms 12.4%
+  triton_convolution_1958 0.4604 ms 10.5%
+SingleProcess AUTOTUNE takes 6.8583 seconds
+AUTOTUNE mm(1024x256, 256x512)
+  triton_mm_2003 0.0130 ms 100.0%
+  triton_mm_2005 0.0131 ms 99.0%
+  triton_mm_2002 0.0133 ms 97.6%
+  triton_mm_2004 0.0136 ms 95.3%
+  mm 0.0137 ms 94.9%
+  triton_mm_2006 0.0161 ms 80.9%
+  triton_mm_2001 0.0165 ms 78.8%
+  triton_mm_2007 0.0165 ms 78.7%
+  triton_mm_2009 0.0169 ms 76.7%
+  triton_mm_2010 0.0189 ms 68.6%
+SingleProcess AUTOTUNE takes 6.5078 seconds
+AUTOTUNE bmm(8x1024x64, 8x64x7)
+  triton_bmm_2019 0.0092 ms 100.0%
+  triton_bmm_2021 0.0092 ms 99.7%
+  triton_bmm_2013 0.0092 ms 99.3%
+  triton_bmm_2014 0.0092 ms 99.3%
+  triton_bmm_2016 0.0092 ms 99.3%
+  triton_bmm_2018 0.0092 ms 99.3%
+  triton_bmm_2017 0.0095 ms 97.0%
+  triton_bmm_2015 0.0097 ms 94.4%
+  triton_bmm_2024 0.0099 ms 92.6%
+  triton_bmm_2020 0.0101 ms 91.1%
+SingleProcess AUTOTUNE takes 1.6053 seconds
+AUTOTUNE bmm(8x1024x7, 8x7x64)
+  triton_bmm_2031 0.0076 ms 100.0%
+  triton_bmm_2035 0.0079 ms 97.2%
+  triton_bmm_2025 0.0081 ms 94.5%
+  triton_bmm_2028 0.0081 ms 94.5%
+  triton_bmm_2034 0.0081 ms 94.1%
+  triton_bmm_2030 0.0083 ms 92.3%
+  triton_bmm_2029 0.0083 ms 91.9%
+  triton_bmm_2032 0.0083 ms 91.9%
+  triton_bmm_2033 0.0085 ms 89.5%
+  triton_bmm_2026 0.0086 ms 89.2%
+SingleProcess AUTOTUNE takes 1.4881 seconds
+AUTOTUNE mm(1024x512, 512x256)
+  triton_mm_2044 0.0142 ms 100.0%
+  mm 0.0146 ms 97.8%
+  triton_mm_2042 0.0168 ms 84.8%
+  triton_mm_2041 0.0170 ms 83.8%
+  triton_mm_2038 0.0176 ms 81.1%
+  triton_mm_2039 0.0178 ms 80.2%
+  triton_mm_2040 0.0180 ms 79.3%
+  triton_mm_2037 0.0180 ms 79.0%
+  triton_mm_2045 0.0196 ms 72.6%
+  triton_mm_2036 0.0227 ms 62.7%
+SingleProcess AUTOTUNE takes 6.4908 seconds
+AUTOTUNE convolution(1x256x32x32, 256x256x3x3)
+  convolution 0.0432 ms 100.0%
+  triton_convolution_2052 0.1268 ms 34.1%
+  triton_convolution_2051 0.2003 ms 21.6%
+  triton_convolution_2053 0.2095 ms 20.6%
+  triton_convolution_2054 0.2918 ms 14.8%
+  triton_convolution_2048 0.3149 ms 13.7%
+  triton_convolution_2049 0.5302 ms 8.2%
+  triton_convolution_2050 0.8952 ms 4.8%
+SingleProcess AUTOTUNE takes 7.5470 seconds
+AUTOTUNE mm(1x512, 512x1024)
+  mm 0.0094 ms 100.0%
+  triton_mm_2145 0.0113 ms 83.8%
+  triton_mm_2146 0.0113 ms 83.6%
+  triton_mm_2149 0.0120 ms 78.9%
+  triton_mm_2148 0.0120 ms 78.7%
+  triton_mm_2144 0.0127 ms 74.5%
+  triton_mm_2143 0.0131 ms 72.0%
+  triton_mm_2142 0.0148 ms 64.0%
+  triton_mm_2141 0.0154 ms 61.2%
+  triton_mm_2140 0.0212 ms 44.6%
+SingleProcess AUTOTUNE takes 4.3357 seconds
+AUTOTUNE convolution(1x512x16x16, 512x512x3x3)
+  convolution 0.0660 ms 100.0%
+  triton_convolution_2156 0.2426 ms 27.2%
+  triton_convolution_2158 0.3784 ms 17.5%
+  triton_convolution_2154 0.3906 ms 16.9%
+  triton_convolution_2153 0.4307 ms 15.3%
+  triton_convolution_2155 0.4340 ms 15.2%
+  triton_convolution_2157 0.6535 ms 10.1%
+  triton_convolution_2152 0.7388 ms 8.9%
+SingleProcess AUTOTUNE takes 6.3883 seconds
+AUTOTUNE mm(256x512, 512x512)
+  triton_mm_2202 0.0119 ms 100.0%
+  triton_mm_2203 0.0124 ms 96.1%
+  triton_mm_2205 0.0139 ms 85.5%
+  triton_mm_2206 0.0141 ms 83.9%
+  mm 0.0150 ms 79.1%
+  triton_mm_2200 0.0171 ms 69.5%
+  triton_mm_2199 0.0173 ms 68.7%
+  triton_mm_2201 0.0173 ms 68.7%
+  triton_mm_2198 0.0175 ms 67.7%
+  triton_mm_2197 0.0221 ms 53.7%
+SingleProcess AUTOTUNE takes 6.3981 seconds
+AUTOTUNE bmm(8x256x64, 8x64x7)
+  triton_bmm_2210 0.0072 ms 100.0%
+  triton_bmm_2212 0.0072 ms 100.0%
+  triton_bmm_2214 0.0076 ms 94.1%
+  triton_bmm_2215 0.0076 ms 94.1%
+  triton_bmm_2217 0.0076 ms 94.1%
+  triton_bmm_2211 0.0081 ms 88.9%
+  triton_bmm_2213 0.0083 ms 86.5%
+  triton_bmm_2209 0.0083 ms 86.2%
+  triton_bmm_2216 0.0083 ms 86.2%
+  triton_bmm_2218 0.0088 ms 81.8%
+SingleProcess AUTOTUNE takes 1.6100 seconds
+AUTOTUNE bmm(8x256x7, 8x7x64)
+  triton_bmm_2230 0.0067 ms 100.0%
+  triton_bmm_2227 0.0071 ms 93.9%
+  triton_bmm_2224 0.0071 ms 93.7%
+  triton_bmm_2226 0.0072 ms 93.3%
+  triton_bmm_2231 0.0072 ms 92.9%
+  triton_bmm_2222 0.0074 ms 90.1%
+  triton_bmm_2221 0.0076 ms 88.2%
+  triton_bmm_2229 0.0076 ms 87.8%
+  triton_bmm_2225 0.0081 ms 82.9%
+  triton_bmm_2223 0.0085 ms 78.9%
+SingleProcess AUTOTUNE takes 1.4920 seconds
+AUTOTUNE convolution(1x512x16x16, 512x512x3x3)
+  convolution 0.0542 ms 100.0%
+  triton_convolution_2248 0.2491 ms 21.7%
+  triton_convolution_2247 0.3928 ms 13.8%
+  triton_convolution_2249 0.4189 ms 12.9%
+  triton_convolution_2246 0.4756 ms 11.4%
+  triton_convolution_2250 0.5758 ms 9.4%
+  triton_convolution_2244 0.6384 ms 8.5%
+  triton_convolution_2245 1.2097 ms 4.5%
+SingleProcess AUTOTUNE takes 7.4778 seconds
+AUTOTUNE mm(1x512, 512x2048)
+  triton_mm_2342 0.0123 ms 100.0%
+  triton_mm_2344 0.0130 ms 94.6%
+  triton_mm_2340 0.0135 ms 91.2%
+  triton_mm_2339 0.0142 ms 86.5%
+  triton_mm_2341 0.0144 ms 85.3%
+  triton_mm_2345 0.0148 ms 82.8%
+  triton_mm_2338 0.0149 ms 82.4%
+  triton_mm_2337 0.0160 ms 76.8%
+  triton_mm_2336 0.0219 ms 56.1%
+  triton_mm_2343 0.0236 ms 52.0%
+SingleProcess AUTOTUNE takes 4.4611 seconds
+AUTOTUNE convolution(1x1024x16x16, 1024x1024x3x3)
+  convolution 0.1833 ms 100.0%
+  triton_convolution_2352 0.5526 ms 33.2%
+  triton_convolution_2349 1.0660 ms 17.2%
+  triton_convolution_2350 1.0914 ms 16.8%
+  triton_convolution_2354 1.1026 ms 16.6%
+  triton_convolution_2351 1.3243 ms 13.8%
+  triton_convolution_2348 1.6191 ms 11.3%
+  triton_convolution_2353 1.9643 ms 9.3%
+SingleProcess AUTOTUNE takes 6.5250 seconds
+AUTOTUNE mm(262x128, 128x1024)
+  mm 0.0096 ms 100.0%
+  triton_mm_2357 0.0102 ms 93.7%
+  triton_mm_2363 0.0103 ms 93.1%
+  triton_mm_2361 0.0106 ms 90.1%
+  triton_mm_2360 0.0107 ms 89.5%
+  triton_mm_2356 0.0108 ms 88.2%
+  triton_mm_2364 0.0110 ms 86.9%
+  triton_mm_2359 0.0111 ms 86.4%
+  triton_mm_2358 0.0112 ms 85.6%
+  triton_mm_2355 0.0116 ms 82.6%
+SingleProcess AUTOTUNE takes 5.9904 seconds
+AUTOTUNE mm(256x1024, 1024x512)
+  triton_mm_2372 0.0167 ms 100.0%
+  triton_mm_2373 0.0172 ms 97.6%
+  mm 0.0176 ms 95.1%
+  triton_mm_2375 0.0205 ms 81.6%
+  triton_mm_2376 0.0214 ms 78.3%
+  triton_mm_2370 0.0260 ms 64.3%
+  triton_mm_2371 0.0263 ms 63.7%
+  triton_mm_2369 0.0272 ms 61.6%
+  triton_mm_2368 0.0278 ms 60.1%
+  triton_mm_2367 0.0377 ms 44.3%
+SingleProcess AUTOTUNE takes 6.1873 seconds
+AUTOTUNE bmm(8x256x64, 8x64x263)
+  triton_bmm_2379 0.0109 ms 100.0%
+  triton_bmm_2381 0.0110 ms 99.7%
+  triton_bmm_2388 0.0110 ms 99.4%
+  triton_bmm_2380 0.0111 ms 98.7%
+  triton_bmm_2382 0.0114 ms 96.1%
+  triton_bmm_2383 0.0114 ms 96.1%
+  triton_bmm_2390 0.0115 ms 95.3%
+  triton_bmm_2389 0.0116 ms 94.0%
+  triton_bmm_2385 0.0120 ms 91.4%
+  triton_bmm_2387 0.0138 ms 79.5%
+SingleProcess AUTOTUNE takes 6.9611 seconds
+AUTOTUNE bmm(8x256x263, 8x263x64)
+  triton_bmm_2397 0.0123 ms 100.0%
+  triton_bmm_2396 0.0130 ms 94.8%
+  triton_bmm_2394 0.0141 ms 87.3%
+  triton_bmm_2392 0.0146 ms 84.4%
+  triton_bmm_2400 0.0151 ms 81.4%
+  triton_bmm_2399 0.0157 ms 78.0%
+  bmm 0.0168 ms 73.3%
+  triton_bmm_2391 0.0189 ms 65.0%
+  triton_bmm_2393 0.0204 ms 60.4%
+  triton_bmm_2395 0.0208 ms 59.1%
+SingleProcess AUTOTUNE takes 5.7534 seconds
+AUTOTUNE mm(256x512, 512x1024)
+  triton_mm_2411 0.0148 ms 100.0%
+  triton_mm_2409 0.0175 ms 85.0%
+  triton_mm_2408 0.0175 ms 84.8%
+  triton_mm_2406 0.0178 ms 83.3%
+  triton_mm_2407 0.0180 ms 82.6%
+  triton_mm_2405 0.0181 ms 82.0%
+  triton_mm_2404 0.0183 ms 81.3%
+  mm 0.0187 ms 79.5%
+  triton_mm_2412 0.0196 ms 75.6%
+  triton_mm_2403 0.0229 ms 64.7%
+SingleProcess AUTOTUNE takes 6.6181 seconds
+AUTOTUNE convolution(1x1024x16x16, 1024x1024x3x3)
+  convolution 0.1594 ms 100.0%
+  triton_convolution_2419 0.4875 ms 32.7%
+  triton_convolution_2418 0.7883 ms 20.2%
+  triton_convolution_2420 0.8412 ms 18.9%
+  triton_convolution_2417 0.9399 ms 17.0%
+  triton_convolution_2421 1.1244 ms 14.2%
+  triton_convolution_2415 1.3130 ms 12.1%
+  triton_convolution_2416 2.5819 ms 6.2%
+SingleProcess AUTOTUNE takes 7.5278 seconds
+AUTOTUNE mm(256x1024, 1024x64)
+  mm 0.0136 ms 100.0%
+  triton_mm_2428 0.0161 ms 84.3%
+  triton_mm_2427 0.0163 ms 83.1%
+  triton_mm_2431 0.0170 ms 79.7%
+  triton_mm_2430 0.0196 ms 69.4%
+  triton_mm_2425 0.0202 ms 67.1%
+  triton_mm_2423 0.0223 ms 60.8%
+  triton_mm_2424 0.0258 ms 52.6%
+  triton_mm_2426 0.0266 ms 51.0%
+  triton_mm_2422 0.0372 ms 36.4%
+SingleProcess AUTOTUNE takes 5.3948 seconds
+AUTOTUNE bmm(1x4096x32, 1x32x257)
+  triton_bmm_2452 0.0115 ms 100.0%
+  triton_bmm_2454 0.0118 ms 97.0%
+  triton_bmm_2457 0.0121 ms 95.2%
+  triton_bmm_2456 0.0121 ms 94.7%
+  triton_bmm_2446 0.0122 ms 94.2%
+  triton_bmm_2448 0.0125 ms 91.8%
+  triton_bmm_2449 0.0128 ms 89.8%
+  triton_bmm_2450 0.0128 ms 89.8%
+  triton_bmm_2447 0.0130 ms 88.6%
+  triton_bmm_2453 0.0132 ms 86.7%
+SingleProcess AUTOTUNE takes 4.9137 seconds
+AUTOTUNE bmm(1x4096x257, 1x257x32)
+  triton_bmm_2463 0.0134 ms 100.0%
+  triton_bmm_2461 0.0137 ms 98.4%
+  triton_bmm_2466 0.0141 ms 95.0%
+  triton_bmm_2459 0.0144 ms 93.3%
+  triton_bmm_2464 0.0145 ms 92.9%
+  triton_bmm_2467 0.0162 ms 83.0%
+  triton_bmm_2460 0.0178 ms 75.5%
+  triton_bmm_2458 0.0187 ms 72.0%
+  bmm 0.0187 ms 71.9%
+  triton_bmm_2462 0.0196 ms 68.7%
+SingleProcess AUTOTUNE takes 5.1166 seconds
+AUTOTUNE convolution(1x1536x16x16, 1024x1536x3x3)
+  convolution 0.2393 ms 100.0%
+  triton_convolution_2584 0.8321 ms 28.8%
+  triton_convolution_2582 1.6275 ms 14.7%
+  triton_convolution_2586 1.6416 ms 14.6%
+  triton_convolution_2581 1.7161 ms 13.9%
+  triton_convolution_2583 1.9816 ms 12.1%
+  triton_convolution_2580 2.5503 ms 9.4%
+  triton_convolution_2585 2.9249 ms 8.2%
+SingleProcess AUTOTUNE takes 6.9053 seconds
+AUTOTUNE mm(6x128, 128x1024)
+  triton_mm_2595 0.0076 ms 100.0%
+  triton_mm_2592 0.0076 ms 99.6%
+  triton_mm_2591 0.0076 ms 98.7%
+  triton_mm_2589 0.0078 ms 96.3%
+  triton_mm_2593 0.0080 ms 94.8%
+  triton_mm_2590 0.0087 ms 86.4%
+  triton_mm_2588 0.0088 ms 85.8%
+  triton_mm_2587 0.0094 ms 80.0%
+  mm 0.0097 ms 78.1%
+  triton_mm_2594 0.0108 ms 69.9%
+SingleProcess AUTOTUNE takes 4.1011 seconds
+AUTOTUNE bmm(8x256x64, 8x64x7)
+  triton_bmm_2614 0.0074 ms 100.0%
+  triton_bmm_2616 0.0074 ms 100.0%
+  triton_bmm_2617 0.0076 ms 96.7%
+  triton_bmm_2612 0.0078 ms 94.3%
+  triton_bmm_2615 0.0081 ms 91.7%
+  triton_bmm_2613 0.0083 ms 89.5%
+  triton_bmm_2619 0.0083 ms 88.8%
+  triton_bmm_2620 0.0085 ms 87.2%
+  triton_bmm_2611 0.0085 ms 86.5%
+  triton_bmm_2618 0.0085 ms 86.5%
+SingleProcess AUTOTUNE takes 4.2192 seconds
+AUTOTUNE bmm(8x256x7, 8x7x64)
+  triton_bmm_2629 0.0069 ms 100.0%
+  triton_bmm_2632 0.0069 ms 100.0%
+  triton_bmm_2628 0.0072 ms 96.9%
+  triton_bmm_2623 0.0074 ms 93.9%
+  triton_bmm_2624 0.0074 ms 93.9%
+  triton_bmm_2626 0.0076 ms 91.2%
+  triton_bmm_2631 0.0076 ms 91.2%
+  triton_bmm_2633 0.0076 ms 91.2%
+  triton_bmm_2627 0.0088 ms 79.2%
+  triton_bmm_2630 0.0092 ms 75.6%
+SingleProcess AUTOTUNE takes 4.0791 seconds
+AUTOTUNE convolution(1x1536x16x16, 1024x1536x1x1)
+  convolution 0.0250 ms 100.0%
+  triton_convolution_2657 0.0938 ms 26.7%
+  triton_convolution_2656 0.1267 ms 19.8%
+  triton_convolution_2659 0.1333 ms 18.8%
+  triton_convolution_2658 0.1454 ms 17.2%
+  conv1x1_via_mm 0.1523 ms 16.4%
+  triton_convolution_2655 0.1618 ms 15.5%
+  triton_convolution_2654 0.1930 ms 13.0%
+  triton_convolution_2653 0.2527 ms 9.9%
+SingleProcess AUTOTUNE takes 6.5036 seconds
+AUTOTUNE convolution(1x1024x16x16, 2048x1024x1x1)
+  convolution 0.0342 ms 100.0%
+  triton_convolution_2848 0.0721 ms 47.5%
+  triton_convolution_2847 0.0902 ms 38.0%
+  triton_convolution_2850 0.0948 ms 36.1%
+  triton_convolution_2849 0.1033 ms 33.2%
+  triton_convolution_2846 0.1297 ms 26.4%
+  triton_convolution_2845 0.1309 ms 26.2%
+  conv1x1_via_mm 0.1607 ms 21.3%
+  triton_convolution_2844 0.1666 ms 20.5%
+SingleProcess AUTOTUNE takes 6.6732 seconds
+AUTOTUNE convolution(1x768x32x32, 512x768x3x3)
+  convolution 0.1486 ms 100.0%
+  triton_convolution_2869 0.5554 ms 26.8%
+  triton_convolution_2867 0.5602 ms 26.5%
+  triton_convolution_2866 0.6690 ms 22.2%
+  triton_convolution_2864 0.6905 ms 21.5%
+  triton_convolution_2868 1.0240 ms 14.5%
+  triton_convolution_2863 1.2562 ms 11.8%
+  triton_convolution_2865 1.4108 ms 10.5%
+SingleProcess AUTOTUNE takes 7.3011 seconds
+AUTOTUNE mm(1024x512, 512x512)
+  mm 0.0176 ms 100.0%
+  triton_mm_2886 0.0178 ms 98.9%
+  triton_mm_2885 0.0180 ms 97.9%
+  triton_mm_2884 0.0180 ms 97.7%
+  triton_mm_2883 0.0182 ms 96.7%
+  triton_mm_2887 0.0227 ms 77.5%
+  triton_mm_2890 0.0230 ms 76.5%
+  triton_mm_2888 0.0231 ms 75.9%
+  triton_mm_2882 0.0236 ms 74.3%
+  triton_mm_2891 0.0281 ms 62.6%
+SingleProcess AUTOTUNE takes 2.1168 seconds
+AUTOTUNE bmm(8x1024x64, 8x64x7)
+  triton_bmm_2896 0.0093 ms 100.0%
+  triton_bmm_2897 0.0095 ms 98.0%
+  triton_bmm_2899 0.0096 ms 97.3%
+  triton_bmm_2901 0.0097 ms 96.0%
+  triton_bmm_2900 0.0099 ms 94.2%
+  triton_bmm_2894 0.0099 ms 93.9%
+  triton_bmm_2895 0.0100 ms 92.8%
+  triton_bmm_2898 0.0102 ms 91.5%
+  triton_bmm_2902 0.0102 ms 91.2%
+  triton_bmm_2903 0.0103 ms 90.7%
+SingleProcess AUTOTUNE takes 1.6145 seconds
+AUTOTUNE bmm(8x1024x7, 8x7x64)
+  triton_bmm_2915 0.0083 ms 100.0%
+  triton_bmm_2916 0.0084 ms 99.2%
+  triton_bmm_2907 0.0086 ms 96.6%
+  triton_bmm_2909 0.0088 ms 94.2%
+  triton_bmm_2912 0.0088 ms 93.8%
+  triton_bmm_2906 0.0090 ms 91.8%
+  triton_bmm_2914 0.0092 ms 89.9%
+  triton_bmm_2908 0.0095 ms 87.5%
+  triton_bmm_2913 0.0095 ms 87.5%
+  triton_bmm_2910 0.0097 ms 85.8%
+SingleProcess AUTOTUNE takes 1.4925 seconds
+AUTOTUNE convolution(1x512x32x32, 512x512x3x3)
+  convolution 0.0906 ms 100.0%
+  triton_convolution_2933 0.2820 ms 32.1%
+  triton_convolution_2932 0.3947 ms 23.0%
+  triton_convolution_2934 0.4131 ms 21.9%
+  triton_convolution_2935 0.5742 ms 15.8%
+  triton_convolution_2929 0.6204 ms 14.6%
+  triton_convolution_2930 1.0628 ms 8.5%
+  triton_convolution_2931 1.7595 ms 5.1%
+SingleProcess AUTOTUNE takes 1.5454 seconds
+AUTOTUNE convolution(1x768x32x32, 512x768x1x1)
+  convolution 0.0222 ms 100.0%
+  triton_convolution_2940 0.0545 ms 40.6%
+  triton_convolution_2939 0.0695 ms 31.9%
+  triton_convolution_2942 0.0750 ms 29.5%
+  triton_convolution_2941 0.0791 ms 28.0%
+  triton_convolution_2937 0.0986 ms 22.5%
+  conv1x1_via_mm 0.1294 ms 17.1%
+  triton_convolution_2936 0.1369 ms 16.2%
+  triton_convolution_2938 0.1654 ms 13.4%
+SingleProcess AUTOTUNE takes 7.3604 seconds
+AUTOTUNE convolution(1x512x32x32, 1024x512x1x1)
+  convolution 0.0233 ms 100.0%
+  triton_convolution_3131 0.0458 ms 50.9%
+  triton_convolution_3130 0.0528 ms 44.2%
+  triton_convolution_3133 0.0563 ms 41.4%
+  triton_convolution_3132 0.0588 ms 39.7%
+  triton_convolution_3128 0.0697 ms 33.5%
+  triton_convolution_3127 0.0883 ms 26.4%
+  triton_convolution_3129 0.1193 ms 19.6%
+  conv1x1_via_mm 0.1366 ms 17.1%
+SingleProcess AUTOTUNE takes 7.1216 seconds
+AUTOTUNE convolution(1x384x64x64, 256x384x3x3)
+  convolution 0.1261 ms 100.0%
+  triton_convolution_3152 0.3128 ms 40.3%
+  triton_convolution_3149 0.3680 ms 34.3%
+  triton_convolution_3147 0.3746 ms 33.7%
+  triton_convolution_3150 0.4803 ms 26.3%
+  triton_convolution_3151 0.5773 ms 21.8%
+  triton_convolution_3146 0.6935 ms 18.2%
+  triton_convolution_3148 0.7742 ms 16.3%
+SingleProcess AUTOTUNE takes 7.0005 seconds
+AUTOTUNE mm(4096x256, 256x512)
+  triton_mm_3167 0.0247 ms 100.0%
+  triton_mm_3166 0.0259 ms 95.4%
+  mm 0.0261 ms 94.6%
+  triton_mm_3175 0.0272 ms 90.8%
+  triton_mm_3172 0.0291 ms 84.9%
+  triton_mm_3169 0.0298 ms 82.8%
+  triton_mm_3168 0.0300 ms 82.4%
+  triton_mm_3165 0.0303 ms 81.4%
+  triton_mm_3173 0.0349 ms 70.7%
+  triton_mm_3171 0.0375 ms 65.8%
+SingleProcess AUTOTUNE takes 1.5827 seconds
+AUTOTUNE bmm(8x4096x64, 8x64x7)
+  triton_bmm_3185 0.0151 ms 100.0%
+  triton_bmm_3188 0.0152 ms 98.9%
+  triton_bmm_3178 0.0153 ms 98.7%
+  triton_bmm_3187 0.0153 ms 98.5%
+  triton_bmm_3184 0.0154 ms 97.7%
+  triton_bmm_3177 0.0156 ms 96.5%
+  triton_bmm_3186 0.0159 ms 94.6%
+  triton_bmm_3179 0.0160 ms 94.0%
+  triton_bmm_3180 0.0163 ms 92.4%
+  triton_bmm_3182 0.0164 ms 91.6%
+SingleProcess AUTOTUNE takes 1.6050 seconds
+AUTOTUNE bmm(8x4096x7, 8x7x64)
+  triton_bmm_3189 0.0130 ms 100.0%
+  triton_bmm_3199 0.0130 ms 99.8%
+  triton_bmm_3190 0.0136 ms 95.5%
+  triton_bmm_3197 0.0139 ms 93.5%
+  triton_bmm_3191 0.0140 ms 93.1%
+  triton_bmm_3198 0.0146 ms 89.2%
+  triton_bmm_3192 0.0147 ms 88.5%
+  triton_bmm_3196 0.0153 ms 84.8%
+  triton_bmm_3195 0.0164 ms 79.0%
+  triton_bmm_3193 0.0165 ms 78.5%
+SingleProcess AUTOTUNE takes 1.4860 seconds
+AUTOTUNE mm(4096x512, 512x256)
+  mm 0.0243 ms 100.0%
+  triton_mm_3202 0.0247 ms 98.4%
+  triton_mm_3201 0.0252 ms 96.7%
+  triton_mm_3200 0.0310 ms 78.4%
+  triton_mm_3204 0.0311 ms 78.3%
+  triton_mm_3203 0.0313 ms 77.7%
+  triton_mm_3208 0.0327 ms 74.4%
+  triton_mm_3206 0.0338 ms 72.0%
+  triton_mm_3205 0.0338 ms 72.0%
+  triton_mm_3207 0.0384 ms 63.3%
+SingleProcess AUTOTUNE takes 1.5832 seconds
+AUTOTUNE convolution(1x256x64x64, 256x256x3x3)
+  convolution 0.0850 ms 100.0%
+  triton_convolution_3215 0.2028 ms 41.9%
+  triton_convolution_3217 0.2116 ms 40.2%
+  triton_convolution_3216 0.2376 ms 35.8%
+  triton_convolution_3218 0.2833 ms 30.0%
+  triton_convolution_3212 0.3116 ms 27.3%
+  triton_convolution_3213 0.4963 ms 17.1%
+  triton_convolution_3214 0.8468 ms 10.0%
+SingleProcess AUTOTUNE takes 1.0139 seconds
+AUTOTUNE convolution(1x384x64x64, 256x384x1x1)
+  convolution 0.0214 ms 100.0%
+  triton_convolution_3223 0.0355 ms 60.4%
+  triton_convolution_3222 0.0478 ms 44.8%
+  triton_convolution_3225 0.0494 ms 43.4%
+  triton_convolution_3224 0.0502 ms 42.6%
+  triton_convolution_3220 0.0564 ms 37.9%
+  triton_convolution_3219 0.0747 ms 28.7%
+  triton_convolution_3221 0.0905 ms 23.7%
+  conv1x1_via_mm 0.1343 ms 15.9%
+SingleProcess AUTOTUNE takes 7.1202 seconds
+[2023-12-06 19:37:58,437] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (8)
+[2023-12-06 19:37:58,437] torch._dynamo.convert_frame: [WARNING]    function: '<lambda>' (/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py:2356)
+[2023-12-06 19:37:58,437] torch._dynamo.convert_frame: [WARNING]    last reason: len(L['down_hiddens']) == 12                                  # connect_skip = lambda fmap: torch.cat((fmap, down_hiddens.pop() * self.skip_connect_scale), dim = 1)  # miniconda3/envs/pytorch/lib/python3.10/site-packages/dalle2_pytorch/dalle2_pytorch.py:2356 in <lambda>
+[2023-12-06 19:37:58,437] torch._dynamo.convert_frame: [WARNING] To log all recompilation reasons, use TORCH_LOGS="recompiles".
+[2023-12-06 19:37:58,437] torch._dynamo.convert_frame: [WARNING] To diagnose recompilation issues, see https://pytorch.org/docs/master/compile/troubleshooting.html.
+AUTOTUNE convolution(1x256x64x64, 512x256x1x1)
+  convolution 0.0265 ms 100.0%
+  triton_convolution_3414 0.0470 ms 56.4%
+  triton_convolution_3411 0.0504 ms 52.6%
+  triton_convolution_3413 0.0627 ms 42.3%
+  triton_convolution_3410 0.0656 ms 40.4%
+  triton_convolution_3416 0.0680 ms 39.0%
+  triton_convolution_3415 0.0686 ms 38.6%
+  triton_convolution_3412 0.1217 ms 21.8%
+  conv1x1_via_mm 0.1252 ms 21.2%
+SingleProcess AUTOTUNE takes 7.2147 seconds
+AUTOTUNE convolution(1x256x128x128, 128x256x3x3)
+  convolution 0.1630 ms 100.0%
+  triton_convolution_3430 0.3422 ms 47.6%
+  triton_convolution_3435 0.4279 ms 38.1%
+  triton_convolution_3432 0.5241 ms 31.1%
+  triton_convolution_3433 0.6718 ms 24.3%
+  triton_convolution_3434 0.7593 ms 21.5%
+  triton_convolution_3429 0.7875 ms 20.7%
+  triton_convolution_3431 1.0484 ms 15.6%
+SingleProcess AUTOTUNE takes 6.5273 seconds
+AUTOTUNE convolution(1x128x128x128, 128x128x3x3)
+  convolution 0.0934 ms 100.0%
+  triton_convolution_3437 0.1770 ms 52.8%
+  triton_convolution_3442 0.2263 ms 41.3%
+  triton_convolution_3439 0.2761 ms 33.8%
+  triton_convolution_3440 0.3409 ms 27.4%
+  triton_convolution_3441 0.3866 ms 24.2%
+  triton_convolution_3436 0.3945 ms 23.7%
+  triton_convolution_3438 0.5277 ms 17.7%
+SingleProcess AUTOTUNE takes 1.0254 seconds
+AUTOTUNE convolution(1x256x128x128, 128x256x1x1)
+  convolution 0.0308 ms 100.0%
+  triton_convolution_3447 0.0535 ms 57.6%
+  triton_convolution_3444 0.0569 ms 54.2%
+  triton_convolution_3443 0.0597 ms 51.6%
+  triton_convolution_3448 0.0679 ms 45.4%
+  triton_convolution_3446 0.0719 ms 42.9%
+  triton_convolution_3449 0.0728 ms 42.3%
+  triton_convolution_3445 0.1262 ms 24.4%
+  conv1x1_via_mm 0.1325 ms 23.2%
+SingleProcess AUTOTUNE takes 6.1270 seconds
+AUTOTUNE mm(1x16, 16x64)
+  triton_mm_3549 0.0062 ms 100.0%
+  triton_mm_3551 0.0062 ms 100.0%
+  mm 0.0065 ms 96.5%
+  triton_mm_3554 0.0066 ms 94.2%
+  triton_mm_3552 0.0067 ms 93.8%
+  triton_mm_3555 0.0067 ms 93.5%
+  triton_mm_3550 0.0067 ms 93.3%
+  triton_mm_3553 0.0067 ms 93.3%
+SingleProcess AUTOTUNE takes 1.8976 seconds
+AUTOTUNE addmm(1x256, 1x64, 64x256)
+  triton_mm_3564 0.0072 ms 100.0%
+  triton_mm_3560 0.0074 ms 97.4%
+  triton_mm_3565 0.0074 ms 97.0%
+  bias_addmm 0.0077 ms 92.9%
+  triton_mm_3558 0.0078 ms 92.2%
+  triton_mm_3556 0.0078 ms 91.4%
+  triton_mm_3557 0.0078 ms 91.4%
+  triton_mm_3559 0.0078 ms 91.4%
+  triton_mm_3561 0.0078 ms 91.4%
+  triton_mm_3562 0.0078 ms 91.4%
+SingleProcess AUTOTUNE takes 4.3911 seconds
+AUTOTUNE convolution(1x6x256x256, 8x6x3x3)
+  triton_convolution_3571 0.0300 ms 100.0%
+  triton_convolution_3569 0.0311 ms 96.3%
+  triton_convolution_3572 0.0319 ms 94.1%
+  triton_convolution_3568 0.0345 ms 86.8%
+  convolution 0.0442 ms 67.8%
+  triton_convolution_3570 0.0570 ms 52.6%
+SingleProcess AUTOTUNE takes 2.5305 seconds
+AUTOTUNE convolution(1x6x256x256, 4x6x7x7)
+  triton_convolution_3574 0.1205 ms 100.0%
+  convolution 0.1230 ms 98.0%
+  triton_convolution_3576 0.1244 ms 96.9%
+  triton_convolution_3577 0.1250 ms 96.4%
+  triton_convolution_3573 0.1346 ms 89.5%
+  triton_convolution_3575 0.2476 ms 48.7%
+SingleProcess AUTOTUNE takes 2.4360 seconds
+AUTOTUNE convolution(1x6x256x256, 4x6x15x15)
+  triton_convolution_3579 0.5118 ms 100.0%
+  convolution 0.5207 ms 98.3%
+  triton_convolution_3582 0.5333 ms 96.0%
+  triton_convolution_3581 0.5377 ms 95.2%
+  triton_convolution_3578 0.5734 ms 89.3%
+  triton_convolution_3580 1.0712 ms 47.8%
+SingleProcess AUTOTUNE takes 2.4038 seconds
+AUTOTUNE addmm(1x64, 1x64, 64x64)
+  triton_mm_3589 0.0071 ms 100.0%
+  triton_mm_3585 0.0072 ms 99.6%
+  triton_mm_3587 0.0072 ms 99.6%
+  triton_mm_3586 0.0074 ms 96.5%
+  bias_addmm 0.0075 ms 95.5%
+  triton_mm_3584 0.0076 ms 93.3%
+  triton_mm_3588 0.0077 ms 92.9%
+  triton_mm_3583 0.0083 ms 85.8%
+  triton_mm_3591 0.0085 ms 83.5%
+  triton_mm_3590 0.0090 ms 79.4%
+SingleProcess AUTOTUNE takes 2.9913 seconds
+AUTOTUNE mm(1x64, 64x32)
+  triton_mm_3593 0.0069 ms 100.0%
+  triton_mm_3594 0.0069 ms 99.5%
+  triton_mm_3595 0.0069 ms 99.1%
+  triton_mm_3592 0.0075 ms 91.3%
+  triton_mm_3596 0.0076 ms 90.3%
+  triton_mm_3597 0.0081 ms 85.3%
+  triton_mm_3598 0.0083 ms 83.0%
+  triton_mm_3599 0.0088 ms 78.5%
+  mm 0.0096 ms 71.7%
+SingleProcess AUTOTUNE takes 2.7713 seconds
+AUTOTUNE convolution(1x16x256x256, 16x16x3x3)
+  triton_convolution_3600 0.0331 ms 100.0%
+  triton_convolution_3603 0.0342 ms 96.7%
+  triton_convolution_3601 0.0351 ms 94.2%
+  triton_convolution_3604 0.0360 ms 91.9%
+  convolution 0.0376 ms 87.9%
+  triton_convolution_3602 0.0504 ms 65.5%
+SingleProcess AUTOTUNE takes 2.9664 seconds
+AUTOTUNE convolution(1x16x128x128, 16x16x3x3)
+  triton_convolution_3654 0.0160 ms 100.0%
+  triton_convolution_3655 0.0166 ms 96.3%
+  triton_convolution_3658 0.0176 ms 91.3%
+  triton_convolution_3657 0.0179 ms 89.8%
+  convolution 0.0217 ms 74.0%
+  triton_convolution_3656 0.0438 ms 36.6%
+SingleProcess AUTOTUNE takes 0.7639 seconds
+AUTOTUNE mm(2x128, 128x1024)
+  triton_mm_3683 0.0081 ms 100.0%
+  triton_mm_3685 0.0081 ms 100.0%
+  triton_mm_3682 0.0081 ms 99.6%
+  triton_mm_3680 0.0083 ms 96.9%
+  triton_mm_3681 0.0083 ms 96.9%
+  triton_mm_3679 0.0085 ms 94.4%
+  triton_mm_3678 0.0088 ms 91.3%
+  triton_mm_3677 0.0095 ms 85.1%
+  mm 0.0096 ms 84.0%
+  triton_mm_3686 0.0099 ms 81.6%
+SingleProcess AUTOTUNE takes 1.6047 seconds
+AUTOTUNE mm(16384x16, 16x512)
+  triton_mm_3694 0.0265 ms 100.0%
+  triton_mm_3698 0.0269 ms 98.3%
+  triton_mm_3699 0.0271 ms 97.8%
+  triton_mm_3689 0.0272 ms 97.3%
+  triton_mm_3697 0.0272 ms 97.3%
+  triton_mm_3695 0.0277 ms 95.5%
+  triton_mm_3693 0.0278 ms 95.2%
+  triton_mm_3691 0.0280 ms 94.6%
+  triton_mm_3692 0.0281 ms 94.3%
+  triton_mm_3690 0.0282 ms 93.7%
+SingleProcess AUTOTUNE takes 3.3535 seconds
+AUTOTUNE bmm(8x16384x64, 8x64x3)
+  triton_bmm_3702 0.0404 ms 100.0%
+  triton_bmm_3704 0.0411 ms 98.3%
+  triton_bmm_3707 0.0414 ms 97.4%
+  triton_bmm_3701 0.0414 ms 97.4%
+  triton_bmm_3703 0.0416 ms 97.1%
+  triton_bmm_3708 0.0417 ms 96.7%
+  triton_bmm_3700 0.0424 ms 95.2%
+  triton_bmm_3705 0.0424 ms 95.2%
+  triton_bmm_3711 0.0428 ms 94.2%
+  triton_bmm_3710 0.0430 ms 93.8%
+SingleProcess AUTOTUNE takes 1.5603 seconds
+AUTOTUNE bmm(8x16384x3, 8x3x64)
+  triton_bmm_3712 0.0286 ms 100.0%
+  triton_bmm_3722 0.0289 ms 98.9%
+  triton_bmm_3713 0.0301 ms 95.0%
+  triton_bmm_3714 0.0321 ms 89.0%
+  triton_bmm_3720 0.0327 ms 87.5%
+  triton_bmm_3719 0.0337 ms 85.0%
+  triton_bmm_3715 0.0349 ms 81.9%
+  triton_bmm_3721 0.0352 ms 81.3%
+  triton_bmm_3716 0.0373 ms 76.6%
+  triton_bmm_3718 0.0439 ms 65.1%
+SingleProcess AUTOTUNE takes 1.4588 seconds
+AUTOTUNE mm(16384x512, 512x16)
+  triton_mm_3729 0.0395 ms 100.0%
+  triton_mm_3726 0.0397 ms 99.4%
+  triton_mm_3731 0.0403 ms 97.9%
+  triton_mm_3732 0.0403 ms 97.9%
+  triton_mm_3724 0.0404 ms 97.9%
+  triton_mm_3728 0.0404 ms 97.8%
+  triton_mm_3727 0.0409 ms 96.5%
+  triton_mm_3725 0.0411 ms 96.1%
+  triton_mm_3723 0.0424 ms 93.1%
+  mm 0.0428 ms 92.4%
+SingleProcess AUTOTUNE takes 4.4858 seconds
+AUTOTUNE convolution(1x16x128x128, 16x16x3x3)
+  triton_convolution_3735 0.0145 ms 100.0%
+  triton_convolution_3738 0.0157 ms 92.5%
+  triton_convolution_3739 0.0170 ms 85.7%
+  triton_convolution_3736 0.0189 ms 76.8%
+  convolution 0.0306 ms 47.4%
+  triton_convolution_3737 0.0603 ms 24.1%
+SingleProcess AUTOTUNE takes 2.5997 seconds
+AUTOTUNE mm(1x64, 64x64)
+  triton_mm_3819 0.0065 ms 100.0%
+  triton_mm_3818 0.0067 ms 96.7%
+  triton_mm_3817 0.0072 ms 90.4%
+  triton_mm_3820 0.0072 ms 90.2%
+  triton_mm_3821 0.0076 ms 84.5%
+  triton_mm_3816 0.0078 ms 82.4%
+  triton_mm_3822 0.0078 ms 82.4%
+  triton_mm_3824 0.0088 ms 73.7%
+  triton_mm_3823 0.0089 ms 72.8%
+  mm 0.0098 ms 65.8%
+SingleProcess AUTOTUNE takes 3.0326 seconds
+AUTOTUNE convolution(1x32x64x64, 32x32x3x3)
+  triton_convolution_3829 0.0148 ms 100.0%
+  triton_convolution_3825 0.0150 ms 98.1%
+  triton_convolution_3830 0.0161 ms 91.8%
+  triton_convolution_3828 0.0193 ms 76.3%
+  convolution 0.0197 ms 74.7%
+  triton_convolution_3826 0.0260 ms 56.8%
+  triton_convolution_3831 0.0294 ms 50.1%
+  triton_convolution_3827 0.0687 ms 21.5%
+SingleProcess AUTOTUNE takes 4.2605 seconds
+AUTOTUNE mm(4096x32, 32x512)
+  triton_mm_3875 0.0129 ms 100.0%
+  triton_mm_3872 0.0131 ms 98.8%
+  triton_mm_3867 0.0131 ms 98.5%
+  triton_mm_3873 0.0133 ms 97.1%
+  triton_mm_3871 0.0135 ms 95.5%
+  triton_mm_3877 0.0137 ms 94.4%
+  triton_mm_3868 0.0139 ms 93.3%
+  triton_mm_3876 0.0139 ms 92.9%
+  triton_mm_3869 0.0141 ms 91.6%
+  triton_mm_3870 0.0149 ms 86.5%
+SingleProcess AUTOTUNE takes 4.0246 seconds
+AUTOTUNE bmm(8x4096x64, 8x64x3)
+  triton_bmm_3887 0.0150 ms 100.0%
+  triton_bmm_3879 0.0151 ms 98.9%
+  triton_bmm_3886 0.0155 ms 96.9%
+  triton_bmm_3890 0.0156 ms 96.1%
+  triton_bmm_3889 0.0156 ms 95.9%
+  triton_bmm_3880 0.0158 ms 94.9%
+  triton_bmm_3884 0.0160 ms 93.8%
+  triton_bmm_3883 0.0163 ms 91.9%
+  triton_bmm_3881 0.0165 ms 90.9%
+  triton_bmm_3888 0.0165 ms 90.9%
+SingleProcess AUTOTUNE takes 1.5959 seconds
+AUTOTUNE bmm(8x4096x3, 8x3x64)
+  triton_bmm_3901 0.0126 ms 100.0%
+  triton_bmm_3891 0.0128 ms 99.0%
+  triton_bmm_3899 0.0133 ms 95.0%
+  triton_bmm_3892 0.0133 ms 94.7%
+  triton_bmm_3893 0.0139 ms 91.2%
+  triton_bmm_3900 0.0139 ms 91.2%
+  triton_bmm_3894 0.0140 ms 90.2%
+  triton_bmm_3898 0.0151 ms 83.7%
+  triton_bmm_3895 0.0163 ms 77.6%
+  triton_bmm_3897 0.0165 ms 76.5%
+SingleProcess AUTOTUNE takes 1.4867 seconds
+AUTOTUNE mm(4096x512, 512x32)
+  triton_mm_3905 0.0154 ms 100.0%
+  triton_mm_3907 0.0154 ms 100.0%
+  triton_mm_3910 0.0156 ms 98.4%
+  triton_mm_3908 0.0157 ms 97.6%
+  mm 0.0158 ms 97.0%
+  triton_mm_3911 0.0164 ms 93.7%
+  triton_mm_3906 0.0169 ms 90.9%
+  triton_mm_3903 0.0170 ms 90.6%
+  triton_mm_3904 0.0183 ms 84.1%
+  triton_mm_3902 0.0241 ms 63.7%
+SingleProcess AUTOTUNE takes 4.5345 seconds
+AUTOTUNE convolution(1x32x64x64, 32x32x3x3)
+  triton_convolution_3919 0.0157 ms 100.0%
+  triton_convolution_3918 0.0173 ms 90.7%
+  triton_convolution_3917 0.0193 ms 81.1%
+  triton_convolution_3914 0.0216 ms 72.5%
+  convolution 0.0220 ms 71.4%
+  triton_convolution_3920 0.0261 ms 60.0%
+  triton_convolution_3915 0.0514 ms 30.5%
+  triton_convolution_3916 0.1118 ms 14.0%
+SingleProcess AUTOTUNE takes 4.5679 seconds
+AUTOTUNE mm(1x64, 64x128)
+  triton_mm_4008 0.0065 ms 100.0%
+  triton_mm_4005 0.0067 ms 96.7%
+  triton_mm_4007 0.0067 ms 96.7%
+  triton_mm_4009 0.0067 ms 96.7%
+  triton_mm_4011 0.0071 ms 90.6%
+  triton_mm_4006 0.0072 ms 90.2%
+  triton_mm_4003 0.0074 ms 87.8%
+  triton_mm_4004 0.0076 ms 84.9%
+  triton_mm_4012 0.0079 ms 82.1%
+  triton_mm_4010 0.0083 ms 78.0%
+SingleProcess AUTOTUNE takes 3.8398 seconds
+AUTOTUNE convolution(1x64x32x32, 64x64x3x3)
+  convolution 0.0236 ms 100.0%
+  triton_convolution_4020 0.0345 ms 68.4%
+  triton_convolution_4019 0.0367 ms 64.3%
+  triton_convolution_4015 0.0407 ms 57.9%
+  triton_convolution_4018 0.0423 ms 55.7%
+  triton_convolution_4021 0.0586 ms 40.3%
+  triton_convolution_4016 0.0627 ms 37.6%
+  triton_convolution_4017 0.1235 ms 19.1%
+SingleProcess AUTOTUNE takes 5.5091 seconds
+AUTOTUNE mm(1024x64, 64x512)
+  triton_mm_4062 0.0097 ms 100.0%
+  mm 0.0097 ms 99.8%
+  triton_mm_4060 0.0098 ms 99.2%
+  triton_mm_4061 0.0100 ms 97.0%
+  triton_mm_4064 0.0100 ms 97.0%
+  triton_mm_4063 0.0102 ms 94.8%
+  triton_mm_4071 0.0103 ms 93.9%
+  triton_mm_4069 0.0105 ms 92.5%
+  triton_mm_4068 0.0107 ms 90.6%
+  triton_mm_4065 0.0109 ms 88.7%
+SingleProcess AUTOTUNE takes 6.0579 seconds
+AUTOTUNE bmm(8x1024x64, 8x64x3)
+  triton_bmm_4078 0.0094 ms 100.0%
+  triton_bmm_4072 0.0095 ms 99.3%
+  triton_bmm_4073 0.0095 ms 99.0%
+  triton_bmm_4075 0.0095 ms 99.0%
+  triton_bmm_4076 0.0097 ms 97.0%
+  triton_bmm_4081 0.0097 ms 97.0%
+  triton_bmm_4074 0.0098 ms 95.8%
+  triton_bmm_4077 0.0100 ms 94.2%
+  triton_bmm_4079 0.0102 ms 92.5%
+  triton_bmm_4080 0.0102 ms 92.5%
+SingleProcess AUTOTUNE takes 1.6110 seconds
+AUTOTUNE bmm(8x1024x3, 8x3x64)
+  triton_bmm_4094 0.0083 ms 100.0%
+  triton_bmm_4084 0.0085 ms 97.4%
+  triton_bmm_4085 0.0085 ms 97.4%
+  triton_bmm_4093 0.0088 ms 94.9%
+  triton_bmm_4086 0.0088 ms 94.5%
+  triton_bmm_4091 0.0088 ms 94.5%
+  triton_bmm_4092 0.0091 ms 91.9%
+  triton_bmm_4087 0.0093 ms 89.7%
+  triton_bmm_4090 0.0095 ms 87.8%
+  triton_bmm_4088 0.0097 ms 85.8%
+SingleProcess AUTOTUNE takes 1.4971 seconds
+AUTOTUNE mm(1024x512, 512x64)
+  triton_mm_4100 0.0120 ms 100.0%
+  triton_mm_4101 0.0120 ms 100.0%
+  triton_mm_4104 0.0125 ms 95.9%
+  triton_mm_4098 0.0140 ms 85.4%
+  triton_mm_4103 0.0140 ms 85.4%
+  mm 0.0147 ms 81.7%
+  triton_mm_4096 0.0155 ms 77.6%
+  triton_mm_4097 0.0175 ms 68.6%
+  triton_mm_4099 0.0177 ms 67.7%
+  triton_mm_4095 0.0235 ms 51.0%
+SingleProcess AUTOTUNE takes 5.6797 seconds
+AUTOTUNE convolution(1x64x32x32, 64x64x3x3)
+  convolution 0.0240 ms 100.0%
+  triton_convolution_4112 0.0352 ms 68.2%
+  triton_convolution_4111 0.0370 ms 64.8%
+  triton_convolution_4107 0.0404 ms 59.3%
+  triton_convolution_4110 0.0421 ms 57.0%
+  triton_convolution_4113 0.0813 ms 29.5%
+  triton_convolution_4108 0.1265 ms 19.0%
+  triton_convolution_4109 0.2088 ms 11.5%
+SingleProcess AUTOTUNE takes 6.4137 seconds
+AUTOTUNE mm(1x64, 64x256)
+  triton_mm_4204 0.0067 ms 100.0%
+  triton_mm_4205 0.0067 ms 100.0%
+  triton_mm_4201 0.0072 ms 93.3%
+  triton_mm_4203 0.0072 ms 93.3%
+  mm 0.0073 ms 92.1%
+  triton_mm_4200 0.0075 ms 88.9%
+  triton_mm_4207 0.0076 ms 87.8%
+  triton_mm_4202 0.0077 ms 86.5%
+  triton_mm_4199 0.0078 ms 85.7%
+  triton_mm_4208 0.0080 ms 83.8%
+SingleProcess AUTOTUNE takes 3.9285 seconds
+AUTOTUNE convolution(1x128x16x16, 128x128x3x3)
+  convolution 0.0265 ms 100.0%
+  triton_convolution_4215 0.0632 ms 41.9%
+  triton_convolution_4216 0.0941 ms 28.1%
+  triton_convolution_4213 0.0992 ms 26.7%
+  triton_convolution_4211 0.1029 ms 25.8%
+  triton_convolution_4217 0.1050 ms 25.2%
+  triton_convolution_4212 0.1176 ms 22.5%
+  triton_convolution_4214 0.1185 ms 22.4%
+SingleProcess AUTOTUNE takes 1.1876 seconds
+AUTOTUNE mm(256x128, 128x512)
+  triton_mm_4261 0.0083 ms 100.0%
+  triton_mm_4262 0.0083 ms 100.0%
+  triton_mm_4264 0.0092 ms 90.3%
+  mm 0.0096 ms 86.4%
+  triton_mm_4265 0.0098 ms 84.7%
+  triton_mm_4257 0.0101 ms 82.0%
+  triton_mm_4258 0.0101 ms 82.0%
+  triton_mm_4256 0.0104 ms 80.0%
+  triton_mm_4259 0.0106 ms 78.5%
+  triton_mm_4260 0.0110 ms 75.6%
+SingleProcess AUTOTUNE takes 1.6074 seconds
+AUTOTUNE bmm(8x256x64, 8x64x3)
+  triton_bmm_4269 0.0074 ms 100.0%
+  triton_bmm_4273 0.0074 ms 100.0%
+  triton_bmm_4270 0.0076 ms 97.1%
+  triton_bmm_4274 0.0077 ms 95.9%
+  triton_bmm_4271 0.0078 ms 94.3%
+  triton_bmm_4268 0.0081 ms 91.7%
+  triton_bmm_4276 0.0083 ms 88.8%
+  triton_bmm_4277 0.0083 ms 88.8%
+  triton_bmm_4272 0.0085 ms 86.7%
+  triton_bmm_4275 0.0085 ms 86.5%
+SingleProcess AUTOTUNE takes 1.6145 seconds
+AUTOTUNE bmm(8x256x3, 8x3x64)
+  triton_bmm_4289 0.0069 ms 100.0%
+  triton_bmm_4285 0.0072 ms 96.9%
+  triton_bmm_4280 0.0074 ms 93.9%
+  triton_bmm_4286 0.0074 ms 93.5%
+  triton_bmm_4290 0.0076 ms 91.8%
+  triton_bmm_4283 0.0076 ms 91.2%
+  triton_bmm_4281 0.0078 ms 88.6%
+  triton_bmm_4288 0.0080 ms 86.8%
+  triton_bmm_4282 0.0085 ms 81.3%
+  triton_bmm_4287 0.0091 ms 76.1%
+SingleProcess AUTOTUNE takes 1.5014 seconds
+AUTOTUNE mm(256x512, 512x128)
+  triton_mm_4297 0.0115 ms 100.0%
+  triton_mm_4296 0.0119 ms 96.8%
+  triton_mm_4300 0.0120 ms 96.3%
+  mm 0.0129 ms 89.3%
+  triton_mm_4299 0.0136 ms 84.7%
+  triton_mm_4293 0.0166 ms 69.5%
+  triton_mm_4292 0.0170 ms 67.7%
+  triton_mm_4294 0.0170 ms 67.7%
+  triton_mm_4295 0.0170 ms 67.7%
+  triton_mm_4291 0.0221 ms 52.2%
+SingleProcess AUTOTUNE takes 1.6051 seconds
+AUTOTUNE convolution(1x128x16x16, 128x128x3x3)
+  convolution 0.0293 ms 100.0%
+  triton_convolution_4307 0.0668 ms 43.8%
+  triton_convolution_4308 0.0716 ms 40.9%
+  triton_convolution_4303 0.0926 ms 31.6%
+  triton_convolution_4306 0.1045 ms 28.0%
+  triton_convolution_4305 0.1180 ms 24.8%
+  triton_convolution_4309 0.1533 ms 19.1%
+  triton_convolution_4304 0.2571 ms 11.4%
+SingleProcess AUTOTUNE takes 1.1804 seconds
+AUTOTUNE mm(1x64, 64x512)
+  mm 0.0069 ms 100.0%
+  triton_mm_4397 0.0069 ms 99.5%
+  triton_mm_4399 0.0069 ms 99.5%
+  triton_mm_4401 0.0070 ms 99.1%
+  triton_mm_4403 0.0073 ms 94.7%
+  triton_mm_4400 0.0076 ms 91.1%
+  triton_mm_4395 0.0076 ms 90.8%
+  triton_mm_4396 0.0076 ms 90.8%
+  triton_mm_4398 0.0078 ms 88.9%
+  triton_mm_4402 0.0078 ms 88.2%
+SingleProcess AUTOTUNE takes 3.8845 seconds
+AUTOTUNE convolution(1x256x16x16, 256x256x3x3)
+  convolution 0.0363 ms 100.0%
+  triton_convolution_4411 0.1215 ms 29.8%
+  triton_convolution_4409 0.1926 ms 18.8%
+  triton_convolution_4413 0.1960 ms 18.5%
+  triton_convolution_4410 0.2224 ms 16.3%
+  triton_convolution_4408 0.2262 ms 16.0%
+  triton_convolution_4412 0.3316 ms 10.9%
+  triton_convolution_4407 0.3682 ms 9.8%
+SingleProcess AUTOTUNE takes 1.4512 seconds
+AUTOTUNE mm(256x256, 256x512)
+  triton_mm_4431 0.0095 ms 100.0%
+  triton_mm_4432 0.0099 ms 96.0%
+  triton_mm_4434 0.0106 ms 89.5%
+  mm 0.0107 ms 89.2%
+  triton_mm_4435 0.0109 ms 87.1%
+  triton_mm_4428 0.0126 ms 75.2%
+  triton_mm_4427 0.0127 ms 75.0%
+  triton_mm_4429 0.0127 ms 75.0%
+  triton_mm_4430 0.0133 ms 71.3%
+  triton_mm_4426 0.0145 ms 65.4%
+SingleProcess AUTOTUNE takes 1.6114 seconds
+AUTOTUNE mm(256x512, 512x256)
+  triton_mm_4470 0.0118 ms 100.0%
+  triton_mm_4467 0.0121 ms 97.1%
+  triton_mm_4466 0.0122 ms 96.8%
+  mm 0.0135 ms 87.2%
+  triton_mm_4469 0.0143 ms 82.5%
+  triton_mm_4464 0.0170 ms 69.2%
+  triton_mm_4463 0.0172 ms 68.3%
+  triton_mm_4465 0.0172 ms 68.3%
+  triton_mm_4462 0.0177 ms 66.4%
+  triton_mm_4461 0.0223 ms 52.7%
+SingleProcess AUTOTUNE takes 1.5954 seconds
+AUTOTUNE convolution(1x256x16x16, 256x256x3x3)
+  convolution 0.0379 ms 100.0%
+  triton_convolution_4477 0.1270 ms 29.8%
+  triton_convolution_4476 0.2000 ms 18.9%
+  triton_convolution_4478 0.2139 ms 17.7%
+  triton_convolution_4475 0.2412 ms 15.7%
+  triton_convolution_4479 0.2930 ms 12.9%
+  triton_convolution_4473 0.3174 ms 11.9%
+  triton_convolution_4474 0.5301 ms 7.1%
+SingleProcess AUTOTUNE takes 1.1926 seconds
+AUTOTUNE mm(256x256, 256x64)
+  triton_mm_4489 0.0088 ms 100.0%
+  triton_mm_4485 0.0092 ms 95.1%
+  triton_mm_4486 0.0097 ms 90.4%
+  triton_mm_4488 0.0105 ms 83.5%
+  triton_mm_4481 0.0108 ms 81.1%
+  triton_mm_4483 0.0109 ms 80.1%
+  mm 0.0118 ms 74.5%
+  triton_mm_4482 0.0124 ms 70.6%
+  triton_mm_4484 0.0127 ms 69.2%
+  triton_mm_4480 0.0148 ms 59.4%
+SingleProcess AUTOTUNE takes 5.5965 seconds
+AUTOTUNE convolution(1x384x16x16, 256x384x3x3)
+  convolution 0.0418 ms 100.0%
+  triton_convolution_4641 0.1699 ms 24.6%
+  triton_convolution_4643 0.2828 ms 14.8%
+  triton_convolution_4639 0.2939 ms 14.2%
+  triton_convolution_4640 0.3295 ms 12.7%
+  triton_convolution_4638 0.3523 ms 11.9%
+  triton_convolution_4642 0.4932 ms 8.5%
+  triton_convolution_4637 0.5457 ms 7.7%
+SingleProcess AUTOTUNE takes 1.4045 seconds
+AUTOTUNE convolution(1x384x16x16, 256x384x1x1)
+  convolution 0.0129 ms 100.0%
+  triton_convolution_4714 0.0281 ms 45.8%
+  triton_convolution_4713 0.0426 ms 30.3%
+  triton_convolution_4716 0.0451 ms 28.6%
+  triton_convolution_4712 0.0462 ms 27.9%
+  triton_convolution_4715 0.0474 ms 27.2%
+  triton_convolution_4711 0.0549 ms 23.5%
+  triton_convolution_4710 0.0712 ms 18.1%
+  conv1x1_via_mm 0.1244 ms 10.4%
+SingleProcess AUTOTUNE takes 1.3465 seconds
+AUTOTUNE convolution(1x256x16x16, 512x256x1x1)
+  convolution 0.0112 ms 100.0%
+  triton_convolution_4905 0.0219 ms 51.2%
+  triton_convolution_4903 0.0336 ms 33.3%
+  triton_convolution_4904 0.0338 ms 33.1%
+  triton_convolution_4907 0.0349 ms 32.1%
+  triton_convolution_4906 0.0367 ms 30.5%
+  triton_convolution_4902 0.0407 ms 27.5%
+  triton_convolution_4901 0.0501 ms 22.3%
+  conv1x1_via_mm 0.1283 ms 8.7%
+SingleProcess AUTOTUNE takes 1.3579 seconds
+AUTOTUNE convolution(1x192x32x32, 128x192x3x3)
+  convolution 0.0324 ms 100.0%
+  triton_convolution_4924 0.0923 ms 35.1%
+  triton_convolution_4925 0.1413 ms 22.9%
+  triton_convolution_4926 0.1458 ms 22.2%
+  triton_convolution_4920 0.1571 ms 20.6%
+  triton_convolution_4923 0.1715 ms 18.9%
+  triton_convolution_4921 0.1817 ms 17.8%
+  triton_convolution_4922 0.3507 ms 9.2%
+SingleProcess AUTOTUNE takes 6.0607 seconds
+AUTOTUNE mm(1024x128, 128x512)
+  triton_mm_4941 0.0104 ms 100.0%
+  triton_mm_4940 0.0107 ms 97.9%
+  triton_mm_4942 0.0111 ms 93.9%
+  triton_mm_4943 0.0114 ms 91.8%
+  mm 0.0116 ms 89.6%
+  triton_mm_4939 0.0118 ms 88.3%
+  triton_mm_4945 0.0124 ms 83.8%
+  triton_mm_4944 0.0128 ms 81.5%
+  triton_mm_4950 0.0132 ms 78.7%
+  triton_mm_4947 0.0135 ms 77.3%
+SingleProcess AUTOTUNE takes 1.6094 seconds
+AUTOTUNE mm(1024x512, 512x128)
+  triton_mm_4980 0.0126 ms 100.0%
+  triton_mm_4979 0.0127 ms 98.7%
+  triton_mm_4983 0.0142 ms 88.7%
+  triton_mm_4982 0.0146 ms 86.4%
+  mm 0.0162 ms 77.5%
+  triton_mm_4977 0.0172 ms 72.9%
+  triton_mm_4976 0.0175 ms 72.0%
+  triton_mm_4975 0.0177 ms 70.9%
+  triton_mm_4978 0.0178 ms 70.8%
+  triton_mm_4974 0.0228 ms 55.2%
+SingleProcess AUTOTUNE takes 1.5967 seconds
+AUTOTUNE convolution(1x128x32x32, 128x128x3x3)
+  convolution 0.0300 ms 100.0%
+  triton_convolution_4990 0.0664 ms 45.2%
+  triton_convolution_4991 0.0713 ms 42.1%
+  triton_convolution_4986 0.0932 ms 32.2%
+  triton_convolution_4989 0.1052 ms 28.5%
+  triton_convolution_4992 0.1492 ms 20.1%
+  triton_convolution_4987 0.2572 ms 11.7%
+  triton_convolution_4988 0.4236 ms 7.1%
+SingleProcess AUTOTUNE takes 0.9848 seconds
+AUTOTUNE convolution(1x192x32x32, 128x192x1x1)
+  convolution 0.0110 ms 100.0%
+  triton_convolution_4997 0.0178 ms 61.6%
+  triton_convolution_4998 0.0206 ms 53.3%
+  triton_convolution_4993 0.0221 ms 49.7%
+  triton_convolution_4996 0.0287 ms 38.2%
+  triton_convolution_4999 0.0304 ms 36.2%
+  triton_convolution_4994 0.0329 ms 33.3%
+  triton_convolution_4995 0.0499 ms 22.0%
+  conv1x1_via_mm 0.1248 ms 8.8%
+SingleProcess AUTOTUNE takes 6.3327 seconds
+AUTOTUNE convolution(1x128x32x32, 256x128x1x1)
+  convolution 0.0096 ms 100.0%
+  triton_convolution_5188 0.0141 ms 67.8%
+  triton_convolution_5187 0.0239 ms 40.0%
+  triton_convolution_5190 0.0253 ms 37.8%
+  triton_convolution_5185 0.0255 ms 37.5%
+  triton_convolution_5189 0.0256 ms 37.4%
+  triton_convolution_5184 0.0304 ms 31.4%
+  triton_convolution_5186 0.0379 ms 25.2%
+  conv1x1_via_mm 0.1215 ms 7.9%
+SingleProcess AUTOTUNE takes 7.1801 seconds
+AUTOTUNE convolution(1x96x64x64, 64x96x3x3)
+  convolution 0.0307 ms 100.0%
+  triton_convolution_5208 0.0607 ms 50.5%
+  triton_convolution_5207 0.0608 ms 50.4%
+  triton_convolution_5206 0.0637 ms 48.1%
+  triton_convolution_5203 0.0644 ms 47.6%
+  triton_convolution_5209 0.0819 ms 37.4%
+  triton_convolution_5204 0.0979 ms 31.3%
+  triton_convolution_5205 0.1814 ms 16.9%
+SingleProcess AUTOTUNE takes 5.2967 seconds
+AUTOTUNE mm(4096x64, 64x512)
+  triton_mm_5232 0.0156 ms 100.0%
+  triton_mm_5224 0.0161 ms 96.8%
+  triton_mm_5230 0.0163 ms 95.9%
+  triton_mm_5223 0.0166 ms 94.0%
+  triton_mm_5222 0.0168 ms 93.1%
+  mm 0.0173 ms 90.4%
+  triton_mm_5226 0.0187 ms 83.7%
+  triton_mm_5233 0.0192 ms 81.2%
+  triton_mm_5225 0.0193 ms 81.1%
+  triton_mm_5231 0.0195 ms 80.1%
+SingleProcess AUTOTUNE takes 1.5976 seconds
+AUTOTUNE mm(4096x512, 512x64)
+  mm 0.0158 ms 100.0%
+  triton_mm_5260 0.0163 ms 96.7%
+  triton_mm_5265 0.0165 ms 95.7%
+  triton_mm_5258 0.0180 ms 87.7%
+  triton_mm_5262 0.0183 ms 86.0%
+  triton_mm_5263 0.0187 ms 84.6%
+  triton_mm_5261 0.0188 ms 83.7%
+  triton_mm_5259 0.0199 ms 79.3%
+  triton_mm_5266 0.0227 ms 69.4%
+  triton_mm_5257 0.0245 ms 64.4%
+SingleProcess AUTOTUNE takes 1.6146 seconds
+AUTOTUNE convolution(1x64x64x64, 64x64x3x3)
+  convolution 0.0255 ms 100.0%
+  triton_convolution_5274 0.0359 ms 71.1%
+  triton_convolution_5273 0.0378 ms 67.5%
+  triton_convolution_5269 0.0417 ms 61.2%
+  triton_convolution_5272 0.0430 ms 59.3%
+  triton_convolution_5275 0.0797 ms 32.0%
+  triton_convolution_5270 0.1337 ms 19.1%
+  triton_convolution_5271 0.2172 ms 11.8%
+SingleProcess AUTOTUNE takes 0.9894 seconds
+AUTOTUNE convolution(1x96x64x64, 64x96x1x1)
+  convolution 0.0100 ms 100.0%
+  triton_convolution_5281 0.0129 ms 77.2%
+  triton_convolution_5280 0.0134 ms 74.6%
+  triton_convolution_5276 0.0141 ms 70.9%
+  triton_convolution_5279 0.0162 ms 61.8%
+  triton_convolution_5282 0.0221 ms 45.1%
+  triton_convolution_5277 0.0241 ms 41.4%
+  triton_convolution_5278 0.0319 ms 31.3%
+  conv1x1_via_mm 0.1236 ms 8.1%
+SingleProcess AUTOTUNE takes 5.5973 seconds
+AUTOTUNE convolution(1x64x64x64, 128x64x1x1)
+  convolution 0.0102 ms 100.0%
+  triton_convolution_5471 0.0124 ms 82.0%
+  triton_convolution_5467 0.0139 ms 73.7%
+  triton_convolution_5472 0.0156 ms 65.6%
+  triton_convolution_5468 0.0194 ms 52.7%
+  triton_convolution_5470 0.0201 ms 50.7%
+  triton_convolution_5473 0.0203 ms 50.4%
+  triton_convolution_5469 0.0260 ms 39.2%
+  conv1x1_via_mm 0.1230 ms 8.3%
+SingleProcess AUTOTUNE takes 5.9804 seconds
+AUTOTUNE convolution(1x48x128x128, 32x48x3x3)
+  triton_convolution_5483 0.0308 ms 100.0%
+  convolution 0.0357 ms 86.4%
+  triton_convolution_5487 0.0399 ms 77.3%
+  triton_convolution_5484 0.0403 ms 76.6%
+  triton_convolution_5488 0.0452 ms 68.2%
+  triton_convolution_5486 0.0482 ms 64.0%
+  triton_convolution_5489 0.0529 ms 58.2%
+  triton_convolution_5485 0.1015 ms 30.3%
+SingleProcess AUTOTUNE takes 4.6617 seconds
+AUTOTUNE mm(16384x32, 32x512)
+  triton_mm_5510 0.0297 ms 100.0%
+  triton_mm_5502 0.0298 ms 99.8%
+  triton_mm_5506 0.0299 ms 99.4%
+  triton_mm_5504 0.0301 ms 98.6%
+  triton_mm_5507 0.0306 ms 97.2%
+  triton_mm_5508 0.0306 ms 97.2%
+  triton_mm_5503 0.0309 ms 96.1%
+  triton_mm_5505 0.0321 ms 92.6%
+  triton_mm_5509 0.0322 ms 92.2%
+  triton_mm_5511 0.0326 ms 91.1%
+SingleProcess AUTOTUNE takes 1.5799 seconds
+AUTOTUNE mm(16384x512, 512x32)
+  triton_mm_5540 0.0412 ms 100.0%
+  triton_mm_5538 0.0415 ms 99.2%
+  triton_mm_5539 0.0422 ms 97.7%
+  triton_mm_5545 0.0422 ms 97.6%
+  triton_mm_5541 0.0423 ms 97.5%
+  triton_mm_5542 0.0432 ms 95.3%
+  mm 0.0434 ms 95.1%
+  triton_mm_5546 0.0434 ms 94.9%
+  triton_mm_5537 0.0435 ms 94.8%
+  triton_mm_5543 0.0437 ms 94.2%
+SingleProcess AUTOTUNE takes 1.5680 seconds
+AUTOTUNE convolution(1x32x128x128, 32x32x3x3)
+  triton_convolution_5553 0.0241 ms 100.0%
+  triton_convolution_5554 0.0243 ms 99.2%
+  triton_convolution_5552 0.0273 ms 88.4%
+  triton_convolution_5555 0.0275 ms 87.7%
+  triton_convolution_5549 0.0276 ms 87.6%
+  convolution 0.0309 ms 78.1%
+  triton_convolution_5550 0.0522 ms 46.2%
+  triton_convolution_5551 0.1112 ms 21.7%
+SingleProcess AUTOTUNE takes 0.9996 seconds
+AUTOTUNE convolution(1x48x128x128, 32x48x1x1)
+  triton_convolution_5556 0.0129 ms 100.0%
+  convolution 0.0130 ms 99.3%
+  triton_convolution_5560 0.0141 ms 91.2%
+  triton_convolution_5557 0.0144 ms 89.8%
+  triton_convolution_5561 0.0151 ms 85.2%
+  triton_convolution_5559 0.0167 ms 77.1%
+  triton_convolution_5562 0.0173 ms 74.5%
+  triton_convolution_5558 0.0235 ms 54.8%
+  conv1x1_via_mm 0.1347 ms 9.6%
+SingleProcess AUTOTUNE takes 4.1733 seconds
+AUTOTUNE convolution(1x32x128x128, 64x32x1x1)
+  convolution 0.0126 ms 100.0%
+  triton_convolution_5745 0.0135 ms 92.9%
+  triton_convolution_5741 0.0142 ms 88.5%
+  triton_convolution_5746 0.0160 ms 78.4%
+  triton_convolution_5742 0.0169 ms 74.3%
+  triton_convolution_5744 0.0192 ms 65.5%
+  triton_convolution_5743 0.0206 ms 60.9%
+  triton_convolution_5747 0.0224 ms 56.1%
+  conv1x1_via_mm 0.1234 ms 10.2%
+SingleProcess AUTOTUNE takes 4.4871 seconds
+AUTOTUNE convolution(1x32x256x256, 16x32x3x3)
+  triton_convolution_5760 0.0438 ms 100.0%
+  triton_convolution_5759 0.0456 ms 96.0%
+  convolution 0.0550 ms 79.7%
+  triton_convolution_5756 0.0553 ms 79.2%
+  triton_convolution_5761 0.0558 ms 78.5%
+  triton_convolution_5757 0.0593 ms 73.9%
+  triton_convolution_5758 0.0794 ms 55.2%
+SingleProcess AUTOTUNE takes 3.4034 seconds
+AUTOTUNE convolution(1x32x256x256, 16x32x1x1)
+  triton_convolution_5771 0.0181 ms 100.0%
+  triton_convolution_5770 0.0192 ms 94.2%
+  triton_convolution_5772 0.0200 ms 90.4%
+  triton_convolution_5767 0.0203 ms 89.1%
+  triton_convolution_5768 0.0203 ms 89.1%
+  convolution 0.0212 ms 85.1%
+  triton_convolution_5769 0.0233 ms 77.6%
+  conv1x1_via_mm 0.1296 ms 14.0%
+SingleProcess AUTOTUNE takes 3.0276 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:06,  4.72it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:05,  4.81it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:05,  4.78it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:05,  4.69it/s]
running benchmark:  17%|█▋        | 5/30 [00:01<00:05,  4.76it/s]
running benchmark:  20%|██        | 6/30 [00:01<00:05,  4.72it/s]
running benchmark:  23%|██▎       | 7/30 [00:01<00:04,  4.72it/s]
running benchmark:  27%|██▋       | 8/30 [00:01<00:04,  4.75it/s]
running benchmark:  30%|███       | 9/30 [00:01<00:04,  4.79it/s]
running benchmark:  33%|███▎      | 10/30 [00:02<00:04,  4.79it/s]
running benchmark:  37%|███▋      | 11/30 [00:02<00:03,  4.75it/s]
running benchmark:  40%|████      | 12/30 [00:02<00:03,  4.78it/s]
running benchmark:  43%|████▎     | 13/30 [00:02<00:03,  4.82it/s]
running benchmark:  47%|████▋     | 14/30 [00:02<00:03,  4.79it/s]
running benchmark:  50%|█████     | 15/30 [00:03<00:03,  4.79it/s]
running benchmark:  53%|█████▎    | 16/30 [00:03<00:02,  4.79it/s]
running benchmark:  57%|█████▋    | 17/30 [00:03<00:02,  4.82it/s]
running benchmark:  60%|██████    | 18/30 [00:03<00:02,  4.63it/s]
running benchmark:  63%|██████▎   | 19/30 [00:04<00:02,  4.40it/s]
running benchmark:  67%|██████▋   | 20/30 [00:04<00:02,  4.48it/s]
running benchmark:  70%|███████   | 21/30 [00:04<00:01,  4.60it/s]
running benchmark:  73%|███████▎  | 22/30 [00:04<00:01,  4.66it/s]
running benchmark:  77%|███████▋  | 23/30 [00:04<00:01,  4.70it/s]
running benchmark:  80%|████████  | 24/30 [00:05<00:01,  4.74it/s]
running benchmark:  83%|████████▎ | 25/30 [00:05<00:01,  4.75it/s]
running benchmark:  87%|████████▋ | 26/30 [00:05<00:00,  4.74it/s]
running benchmark:  90%|█████████ | 27/30 [00:05<00:00,  4.73it/s]
running benchmark:  93%|█████████▎| 28/30 [00:05<00:00,  4.73it/s]
running benchmark:  97%|█████████▋| 29/30 [00:06<00:00,  4.67it/s]
running benchmark: 100%|██████████| 30/30 [00:06<00:00,  4.59it/s]
running benchmark: 100%|██████████| 30/30 [00:06<00:00,  4.70it/s]
+2.001x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+cuda eval  LearningToPaint                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 213.79it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 215.73it/s]
+2.031x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model Super_SloMo supports float32 only
+
loading model: 0it [00:03, ?it/s]
+WARNING:common:Model Super_SloMo supports float32 only
+cuda eval  Super_SloMo                        
+WARNING:common:Model Super_SloMo supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:05,  5.02it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:03,  8.99it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02, 10.48it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:02, 11.23it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 11.63it/s]
running benchmark:  37%|███▋      | 11/30 [00:01<00:01, 11.88it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 12.04it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 12.14it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:01, 12.19it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 12.22it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 12.25it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 12.28it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00, 12.29it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 12.30it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 12.31it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 11.78it/s]
+1.386x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  alexnet                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 148.15it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 155.99it/s]
+1.364x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  basic_gnn_edgecnn                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 212.74it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 214.86it/s]
+1.369x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_gcn                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  9.63it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 79.08it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 101.89it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 91.40it/s] 
+1.063x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_gin                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 355.92it/s]
+1.221x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  basic_gnn_sage                     
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 156.01it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 163.13it/s]
+1.100x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  cm3leon_generate                   
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:01<00:34,  1.19s/it]
running benchmark:   7%|▋         | 2/30 [00:02<00:33,  1.21s/it]
running benchmark:  10%|█         | 3/30 [00:03<00:32,  1.20s/it]
running benchmark:  13%|█▎        | 4/30 [00:04<00:30,  1.19s/it]
running benchmark:  17%|█▋        | 5/30 [00:05<00:29,  1.19s/it]
running benchmark:  20%|██        | 6/30 [00:07<00:28,  1.19s/it]
running benchmark:  23%|██▎       | 7/30 [00:08<00:27,  1.21s/it]
running benchmark:  27%|██▋       | 8/30 [00:09<00:26,  1.21s/it]
running benchmark:  30%|███       | 9/30 [00:10<00:25,  1.21s/it]
running benchmark:  33%|███▎      | 10/30 [00:12<00:24,  1.20s/it]
running benchmark:  37%|███▋      | 11/30 [00:13<00:22,  1.20s/it]
running benchmark:  40%|████      | 12/30 [00:14<00:21,  1.20s/it]
running benchmark:  43%|████▎     | 13/30 [00:15<00:20,  1.20s/it]
running benchmark:  47%|████▋     | 14/30 [00:16<00:19,  1.20s/it]
running benchmark:  50%|█████     | 15/30 [00:18<00:17,  1.20s/it]
running benchmark:  53%|█████▎    | 16/30 [00:19<00:16,  1.21s/it]
running benchmark:  57%|█████▋    | 17/30 [00:20<00:15,  1.20s/it]
running benchmark:  60%|██████    | 18/30 [00:21<00:14,  1.21s/it]
running benchmark:  63%|██████▎   | 19/30 [00:22<00:13,  1.21s/it]
running benchmark:  67%|██████▋   | 20/30 [00:24<00:12,  1.22s/it]
running benchmark:  70%|███████   | 21/30 [00:25<00:11,  1.23s/it]
running benchmark:  73%|███████▎  | 22/30 [00:26<00:09,  1.23s/it]
running benchmark:  77%|███████▋  | 23/30 [00:27<00:08,  1.24s/it]
running benchmark:  80%|████████  | 24/30 [00:29<00:07,  1.25s/it]
running benchmark:  83%|████████▎ | 25/30 [00:30<00:06,  1.25s/it]
running benchmark:  87%|████████▋ | 26/30 [00:31<00:04,  1.23s/it]
running benchmark:  90%|█████████ | 27/30 [00:32<00:03,  1.22s/it]
running benchmark:  93%|█████████▎| 28/30 [00:33<00:02,  1.22s/it]
running benchmark:  97%|█████████▋| 29/30 [00:35<00:01,  1.21s/it]
running benchmark: 100%|██████████| 30/30 [00:36<00:00,  1.21s/it]
running benchmark: 100%|██████████| 30/30 [00:36<00:00,  1.21s/it]
+4.357x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  dcgan                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 490.93it/s]
+1.360x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+WARNING:root:demucs failed to load
+Original Error: "_thnn_fused_lstm_cell_cuda" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/__init__.py", line 31, in forward
+    return sources, self.model(mix)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/demucs/model.py", line 209, in forward
+    x = self.lstm(x)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/demucs/demucs/model.py", line 27, in forward
+    x = self.lstm(x)[0]
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/rnn.py", line 878, in forward
+    result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,
+RuntimeError: "_thnn_fused_lstm_cell_cuda" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  densenet121                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 25.82it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 30.79it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 32.38it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 33.10it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 33.50it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 33.73it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 33.91it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 33.15it/s]
+2.066x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:10, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_dc5 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_101_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:08, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_dc5 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+WARNING:root:detectron2_fasterrcnn_r_50_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+
loading model: 0it [00:05, ?it/s]
+WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+cuda eval  detectron2_fcos_r_50_fpn           
+WARNING:common:Model detectron2_fcos_r_50_fpn supports float32 only
+[2023-12-06 19:56:18,051] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (8)
+[2023-12-06 19:56:18,051] torch._dynamo.convert_frame: [WARNING]    function: 'forward' (/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/batch_norm.py:318)
+[2023-12-06 19:56:18,051] torch._dynamo.convert_frame: [WARNING]    last reason: L['self']._pos == 0                                           # ret = self[self._pos](x)  # miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/batch_norm.py:319 in forward
+[2023-12-06 19:56:18,051] torch._dynamo.convert_frame: [WARNING] To log all recompilation reasons, use TORCH_LOGS="recompiles".
+[2023-12-06 19:56:18,051] torch._dynamo.convert_frame: [WARNING] To diagnose recompilation issues, see https://pytorch.org/docs/master/compile/troubleshooting.html.
+skipping cudagraphs due to ['mutated inputs']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:01, 18.69it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 19.77it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 20.15it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 19.48it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 19.79it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 20.01it/s]
running benchmark:  67%|██████▋   | 20/30 [00:01<00:00, 20.18it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 20.31it/s]
running benchmark:  87%|████████▋ | 26/30 [00:01<00:00, 20.20it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 20.25it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 19.91it/s]
+1.535x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_101_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_101_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_50_c4 failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 477, in forward
+    box_features = self._shared_roi_transform(
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 456, in _shared_roi_transform
+    x = self.pooler(features, boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 246, in forward
+    return self.level_poolers[0](x[0], pooler_fmt_boxes)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+WARNING:root:detectron2_maskrcnn_r_50_fpn failed to load
+Original Error: "roi_align_forward_kernel" not implemented for 'BFloat16'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 150, in forward
+    return self.inference(batched_inputs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/meta_arch/rcnn.py", line 213, in inference
+    results, _ = self.roi_heads(images, features, proposals, None)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 747, in forward
+    pred_instances = self._forward_box(features, proposals)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/roi_heads/roi_heads.py", line 798, in _forward_box
+    box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/modeling/poolers.py", line 261, in forward
+    output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level))
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/detectron2/layers/roi_align.py", line 58, in forward
+    return roi_align(
+  File "/home/cdhernandez/local/vision/torchvision/ops/roi_align.py", line 238, in roi_align
+    return torch.ops.torchvision.roi_align(
+  File "/home/cdhernandez/local/pytorch/torch/_ops.py", line 753, in __call__
+    return self._op(*args, **kwargs or {})
+RuntimeError: "roi_align_forward_kernel" not implemented for 'BFloat16'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:13, ?it/s]
+cuda eval  dlrm                               
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 282.54it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 282.35it/s]
+1.143x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model doctr_det_predictor supports float32 only
+
loading model: 0it [00:06, ?it/s]
+WARNING:common:Model doctr_det_predictor supports float32 only
+cuda eval  doctr_det_predictor                
+WARNING:common:Model doctr_det_predictor supports float32 only
+[2023-12-06 19:58:14,149] [1/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+skipping cudagraphs due to ['non-cuda device in graph']
+malloc(): unaligned tcache chunk detected
+Run failed with return code:  -6
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model doctr_reco_predictor supports float32 only
+
loading model: 0it [00:06, ?it/s]
+WARNING:common:Model doctr_reco_predictor supports float32 only
+cuda eval  doctr_reco_predictor               
+WARNING:common:Model doctr_reco_predictor supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 227.24it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 227.03it/s]
+1.865x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  drq                                
+AUTOTUNE mm(1x39200, 39200x50)
+  mm 0.0214 ms 100.0%
+  triton_mm_31 0.2118 ms 10.1%
+  triton_mm_32 0.2357 ms 9.1%
+  triton_mm_33 0.2569 ms 8.3%
+  triton_mm_34 0.2738 ms 7.8%
+  triton_mm_30 0.3042 ms 7.0%
+  triton_mm_29 0.4410 ms 4.8%
+  triton_mm_28 0.8653 ms 2.5%
+  triton_mm_36 1.0548 ms 2.0%
+  triton_mm_35 1.2301 ms 1.7%
+SingleProcess AUTOTUNE takes 2.9413 seconds
+AUTOTUNE mm(1x50, 50x1024)
+  triton_mm_39 0.0069 ms 100.0%
+  triton_mm_41 0.0069 ms 100.0%
+  triton_mm_43 0.0069 ms 100.0%
+  triton_mm_38 0.0071 ms 96.4%
+  triton_mm_42 0.0074 ms 92.7%
+  triton_mm_44 0.0076 ms 91.1%
+  mm 0.0076 ms 90.7%
+  triton_mm_47 0.0076 ms 90.7%
+  triton_mm_37 0.0076 ms 90.0%
+  triton_mm_46 0.0079 ms 87.4%
+SingleProcess AUTOTUNE takes 3.7602 seconds
+AUTOTUNE mm(1x1024, 1024x1024)
+  mm 0.0112 ms 100.0%
+  triton_mm_54 0.0137 ms 81.6%
+  triton_mm_55 0.0137 ms 81.6%
+  triton_mm_57 0.0144 ms 77.8%
+  triton_mm_53 0.0146 ms 76.6%
+  triton_mm_58 0.0155 ms 72.5%
+  triton_mm_52 0.0168 ms 66.7%
+  triton_mm_51 0.0199 ms 56.3%
+  triton_mm_50 0.0211 ms 53.2%
+  triton_mm_49 0.0324 ms 34.6%
+SingleProcess AUTOTUNE takes 3.6427 seconds
+AUTOTUNE addmm(1x2, 1x1024, 1024x2)
+  triton_mm_64 0.0108 ms 100.0%
+  triton_mm_63 0.0115 ms 94.1%
+  triton_mm_65 0.0117 ms 91.8%
+  triton_mm_66 0.0127 ms 85.1%
+  bias_addmm 0.0143 ms 75.2%
+  triton_mm_62 0.0152 ms 71.1%
+  addmm 0.0190 ms 56.7%
+  triton_mm_61 0.0221 ms 48.8%
+  triton_mm_68 0.0288 ms 37.4%
+  triton_mm_67 0.0293 ms 36.8%
+SingleProcess AUTOTUNE takes 2.8192 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 736.46it/s]
+2.853x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  fastNLP_Bert                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:00, 49.67it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:00, 60.50it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 65.47it/s]
running benchmark:  93%|█████████▎| 28/30 [00:00<00:00, 67.46it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 65.29it/s]
+2.741x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  functorch_dp_cifar10               
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 250.12it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 249.08it/s]
+4.828x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/functorch_maml_omniglot/__init__.py", line 73, in __init__
+    self.meta_inputs = torch.load(f'{root}/maml_omniglot/batch.pt')
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 998, in load
+    with _open_file_like(f, 'rb') as opened_file:
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 445, in _open_file_like
+    return _open_file(name_or_buffer, mode)
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 426, in __init__
+    super().__init__(open(name, mode))
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/batch.pt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  hf_Albert                          
+AUTOTUNE addmm(512x768, 512x128, 128x768)
+  triton_mm_8 0.0087 ms 100.0%
+  triton_mm_3 0.0092 ms 93.8%
+  triton_mm_4 0.0093 ms 93.1%
+  triton_mm_1 0.0096 ms 90.6%
+  triton_mm_9 0.0096 ms 90.0%
+  triton_mm_2 0.0097 ms 89.7%
+  triton_mm_5 0.0098 ms 88.6%
+  bias_addmm 0.0102 ms 85.2%
+  triton_mm_6 0.0102 ms 85.1%
+  triton_mm_0 0.0104 ms 83.6%
+SingleProcess AUTOTUNE takes 5.6043 seconds
+AUTOTUNE mm(512x768, 768x768)
+  mm 0.0129 ms 100.0%
+  triton_mm_20 0.0139 ms 92.9%
+  triton_mm_15 0.0158 ms 81.7%
+  triton_mm_16 0.0164 ms 78.5%
+  triton_mm_18 0.0168 ms 76.6%
+  triton_mm_17 0.0173 ms 74.4%
+  triton_mm_21 0.0175 ms 73.8%
+  triton_mm_13 0.0197 ms 65.5%
+  triton_mm_14 0.0198 ms 65.0%
+  triton_mm_12 0.0279 ms 46.2%
+SingleProcess AUTOTUNE takes 5.1096 seconds
+AUTOTUNE mm(512x768, 768x3072)
+  mm 0.0206 ms 100.0%
+  triton_mm_62 0.0232 ms 88.6%
+  triton_mm_61 0.0233 ms 88.1%
+  triton_mm_64 0.0238 ms 86.4%
+  triton_mm_63 0.0243 ms 84.5%
+  triton_mm_68 0.0291 ms 70.8%
+  triton_mm_60 0.0320 ms 64.2%
+  triton_mm_67 0.0378 ms 54.4%
+  triton_mm_70 0.0412 ms 50.0%
+  triton_mm_69 0.0447 ms 46.0%
+SingleProcess AUTOTUNE takes 6.0215 seconds
+AUTOTUNE mm(512x3072, 3072x768)
+  mm 0.0270 ms 100.0%
+  triton_mm_80 0.0344 ms 78.3%
+  triton_mm_76 0.0411 ms 65.6%
+  triton_mm_75 0.0416 ms 64.9%
+  triton_mm_77 0.0443 ms 61.0%
+  triton_mm_78 0.0443 ms 61.0%
+  triton_mm_81 0.0453 ms 59.5%
+  triton_mm_74 0.0556 ms 48.6%
+  triton_mm_73 0.0559 ms 48.3%
+  triton_mm_72 0.0697 ms 38.7%
+SingleProcess AUTOTUNE takes 4.8257 seconds
+AUTOTUNE mm(512x768, 768x128)
+  mm 0.0112 ms 100.0%
+  triton_mm_882 0.0122 ms 91.6%
+  triton_mm_885 0.0125 ms 89.7%
+  triton_mm_881 0.0128 ms 87.7%
+  triton_mm_884 0.0132 ms 84.5%
+  triton_mm_879 0.0159 ms 70.4%
+  triton_mm_880 0.0161 ms 69.7%
+  triton_mm_878 0.0188 ms 59.5%
+  triton_mm_877 0.0191 ms 58.7%
+  triton_mm_876 0.0276 ms 40.6%
+SingleProcess AUTOTUNE takes 4.9291 seconds
+AUTOTUNE addmm(512x30000, 512x128, 128x30000)
+  triton_mm_889 0.0506 ms 100.0%
+  triton_mm_890 0.0515 ms 98.3%
+  triton_mm_888 0.0555 ms 91.1%
+  triton_mm_895 0.0569 ms 88.8%
+  triton_mm_891 0.0587 ms 86.2%
+  triton_mm_892 0.0592 ms 85.4%
+  bias_addmm 0.0632 ms 80.0%
+  triton_mm_896 0.0647 ms 78.1%
+  triton_mm_898 0.0726 ms 69.6%
+  triton_mm_897 0.0958 ms 52.8%
+SingleProcess AUTOTUNE takes 5.0208 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 59.48it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 73.83it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 78.63it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 77.24it/s]
+6.361x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+cuda eval  hf_Bart                            
+AUTOTUNE mm(512x768, 768x50265)
+  triton_mm_1297 0.2542 ms 100.0%
+  triton_mm_1298 0.2549 ms 99.7%
+  triton_mm_1299 0.2932 ms 86.7%
+  triton_mm_1300 0.2959 ms 85.9%
+  triton_mm_1303 0.3072 ms 82.8%
+  triton_mm_1296 0.3197 ms 79.5%
+  triton_mm_1304 0.4562 ms 55.7%
+  triton_mm_1306 0.4839 ms 52.5%
+  triton_mm_1305 0.5810 ms 43.8%
+  triton_mm_1302 0.6129 ms 41.5%
+SingleProcess AUTOTUNE takes 5.9643 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  9.66it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:00, 25.46it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 46.96it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 57.11it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 62.63it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 53.41it/s]
+4.151x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+cuda eval  hf_BigBird                         
+[2023-12-06 20:02:58,645] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+AUTOTUNE addmm(4096x768, 4096x768, 768x768)
+  bias_addmm 0.0316 ms 100.0%
+  triton_mm_2 0.0385 ms 82.0%
+  triton_mm_3 0.0421 ms 75.0%
+  triton_mm_1 0.0425 ms 74.2%
+  triton_mm_4 0.0433 ms 72.9%
+  addmm 0.0433 ms 72.8%
+  triton_mm_7 0.0440 ms 71.7%
+  triton_mm_8 0.0498 ms 63.3%
+  triton_mm_0 0.0588 ms 53.7%
+  triton_mm_10 0.0723 ms 43.6%
+SingleProcess AUTOTUNE takes 5.6290 seconds
+AUTOTUNE mm(4096x768, 768x768)
+  mm 0.0306 ms 100.0%
+  triton_mm_218 0.0373 ms 82.1%
+  triton_mm_217 0.0404 ms 75.8%
+  triton_mm_219 0.0415 ms 73.8%
+  triton_mm_220 0.0416 ms 73.7%
+  triton_mm_223 0.0441 ms 69.4%
+  triton_mm_224 0.0482 ms 63.5%
+  triton_mm_216 0.0571 ms 53.6%
+  triton_mm_226 0.0710 ms 43.1%
+  triton_mm_225 0.0794 ms 38.6%
+SingleProcess AUTOTUNE takes 5.2190 seconds
+AUTOTUNE mm(4096x768, 768x3072)
+  mm 0.1010 ms 100.0%
+  triton_mm_229 0.1108 ms 91.2%
+  triton_mm_230 0.1116 ms 90.5%
+  triton_mm_235 0.1300 ms 77.7%
+  triton_mm_231 0.1319 ms 76.6%
+  triton_mm_232 0.1327 ms 76.1%
+  triton_mm_228 0.1407 ms 71.8%
+  triton_mm_236 0.1533 ms 65.9%
+  triton_mm_238 0.2351 ms 43.0%
+  triton_mm_234 0.2821 ms 35.8%
+SingleProcess AUTOTUNE takes 5.3341 seconds
+AUTOTUNE mm(4096x3072, 3072x768)
+  mm 0.0856 ms 100.0%
+  triton_mm_242 0.1220 ms 70.1%
+  triton_mm_241 0.1264 ms 67.7%
+  triton_mm_243 0.1291 ms 66.3%
+  triton_mm_244 0.1299 ms 65.9%
+  triton_mm_248 0.1541 ms 55.5%
+  triton_mm_240 0.1621 ms 52.8%
+  triton_mm_247 0.1906 ms 44.9%
+  triton_mm_245 0.2714 ms 31.5%
+  triton_mm_246 0.2720 ms 31.5%
+SingleProcess AUTOTUNE takes 5.0817 seconds
+[2023-12-06 20:03:22,315] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:23,641] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:24,948] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:26,273] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:27,626] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:28,981] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:30,341] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:31,683] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:32,974] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:34,295] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+[2023-12-06 20:03:35,602] [0/0] torch._inductor.utils: [WARNING] DeviceCopy in input program
+AUTOTUNE addmm(4096x50358, 4096x768, 768x50358)
+  bias_addmm 2.3274 ms 100.0%
+  triton_mm_3037 2.3277 ms 100.0%
+  triton_mm_3038 2.4641 ms 94.5%
+  triton_mm_3039 2.8108 ms 82.8%
+  triton_mm_3040 2.9675 ms 78.4%
+  triton_mm_3036 3.0073 ms 77.4%
+  addmm 3.0679 ms 75.9%
+  triton_mm_3044 3.0872 ms 75.4%
+  triton_mm_3043 3.8977 ms 59.7%
+  triton_mm_3046 4.1048 ms 56.7%
+SingleProcess AUTOTUNE takes 6.4605 seconds
+skipping cudagraphs due to ['non-cuda device in graph']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:02, 13.15it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:01, 13.70it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:01, 13.79it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 13.76it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:01, 13.82it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:01, 13.86it/s]
running benchmark:  47%|████▋     | 14/30 [00:01<00:01, 13.84it/s]
running benchmark:  53%|█████▎    | 16/30 [00:01<00:01, 13.89it/s]
running benchmark:  60%|██████    | 18/30 [00:01<00:00, 14.00it/s]
running benchmark:  67%|██████▋   | 20/30 [00:01<00:00, 13.97it/s]
running benchmark:  73%|███████▎  | 22/30 [00:01<00:00, 13.93it/s]
running benchmark:  80%|████████  | 24/30 [00:01<00:00, 13.97it/s]
running benchmark:  87%|████████▋ | 26/30 [00:01<00:00, 14.02it/s]
running benchmark:  93%|█████████▎| 28/30 [00:02<00:00, 13.99it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 13.95it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 13.89it/s]
+2.343x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  hf_DistilBert                      
+AUTOTUNE addmm(512x768, 512x768, 768x768)
+  bias_addmm 0.0143 ms 100.0%
+  triton_mm_20 0.0150 ms 95.7%
+  triton_mm_15 0.0164 ms 87.3%
+  triton_mm_16 0.0172 ms 83.6%
+  triton_mm_18 0.0172 ms 83.3%
+  triton_mm_17 0.0173 ms 82.7%
+  triton_mm_21 0.0182 ms 78.7%
+  addmm 0.0184 ms 77.9%
+  triton_mm_13 0.0200 ms 71.7%
+  triton_mm_14 0.0200 ms 71.7%
+SingleProcess AUTOTUNE takes 5.3474 seconds
+AUTOTUNE addmm(512x30522, 512x768, 768x30522)
+  triton_mm_589 0.1673 ms 100.0%
+  triton_mm_590 0.1828 ms 91.6%
+  bias_addmm 0.2004 ms 83.5%
+  triton_mm_591 0.2110 ms 79.3%
+  triton_mm_592 0.2271 ms 73.7%
+  triton_mm_596 0.2297 ms 72.9%
+  triton_mm_588 0.2459 ms 68.1%
+  addmm 0.2631 ms 63.6%
+  triton_mm_595 0.3279 ms 51.0%
+  triton_mm_598 0.3352 ms 49.9%
+SingleProcess AUTOTUNE takes 6.1550 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 101.76it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 126.80it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 126.03it/s]
+4.383x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  hf_GPT2                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 64.17it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 67.75it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 68.20it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 68.69it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 68.15it/s]
+3.337x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:19, ?it/s]
+cuda eval  hf_GPT2_large                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:09,  3.16it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:07,  3.91it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02,  9.22it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 11.79it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 13.83it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 15.45it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 16.68it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:00, 17.60it/s]
running benchmark:  60%|██████    | 18/30 [00:01<00:00, 18.52it/s]
running benchmark:  67%|██████▋   | 20/30 [00:01<00:00, 18.88it/s]
running benchmark:  73%|███████▎  | 22/30 [00:01<00:00, 19.18it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 19.48it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 19.57it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 19.80it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 15.52it/s]
+2.042x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+cuda eval  hf_Longformer                      
+AUTOTUNE bmm(180x512x64, 180x64x512)
+  triton_bmm_25 0.0889 ms 100.0%
+  triton_bmm_26 0.0921 ms 96.5%
+  bmm 0.0985 ms 90.2%
+  triton_bmm_32 0.1012 ms 87.8%
+  triton_bmm_27 0.1069 ms 83.2%
+  triton_bmm_28 0.1138 ms 78.1%
+  triton_bmm_34 0.1147 ms 77.5%
+  triton_bmm_24 0.1184 ms 75.1%
+  triton_bmm_31 0.1380 ms 64.4%
+  triton_bmm_35 0.1656 ms 53.7%
+SingleProcess AUTOTUNE takes 4.2406 seconds
+AUTOTUNE bmm(192x256x768, 192x768x64)
+  triton_bmm_49 0.1100 ms 100.0%
+  triton_bmm_51 0.1135 ms 96.9%
+  triton_bmm_52 0.1140 ms 96.5%
+  triton_bmm_56 0.1166 ms 94.3%
+  triton_bmm_54 0.1254 ms 87.7%
+  triton_bmm_48 0.1282 ms 85.8%
+  triton_bmm_55 0.1347 ms 81.7%
+  triton_bmm_50 0.1407 ms 78.1%
+  bmm 0.1541 ms 71.4%
+  triton_bmm_57 0.1558 ms 70.6%
+SingleProcess AUTOTUNE takes 4.3490 seconds
+AUTOTUNE addmm(4096x50272, 4096x768, 768x50272)
+  bias_addmm 1.5488 ms 100.0%
+  triton_mm_1166 1.9554 ms 79.2%
+  triton_mm_1165 1.9666 ms 78.8%
+  addmm 2.0870 ms 74.2%
+  triton_mm_1167 2.3149 ms 66.9%
+  triton_mm_1168 2.3324 ms 66.4%
+  triton_mm_1164 2.6459 ms 58.5%
+  triton_mm_1172 2.7645 ms 56.0%
+  triton_mm_1174 3.1868 ms 48.6%
+  triton_mm_1171 3.4290 ms 45.2%
+SingleProcess AUTOTUNE takes 5.8343 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:02, 13.95it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:01, 15.92it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:01, 16.66it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 17.10it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:01, 17.33it/s]
running benchmark:  40%|████      | 12/30 [00:00<00:01, 17.46it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 17.52it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 17.57it/s]
running benchmark:  60%|██████    | 18/30 [00:01<00:00, 17.57it/s]
running benchmark:  67%|██████▋   | 20/30 [00:01<00:00, 17.51it/s]
running benchmark:  73%|███████▎  | 22/30 [00:01<00:00, 17.53it/s]
running benchmark:  80%|████████  | 24/30 [00:01<00:00, 17.58it/s]
running benchmark:  87%|████████▋ | 26/30 [00:01<00:00, 17.64it/s]
running benchmark:  93%|█████████▎| 28/30 [00:01<00:00, 17.68it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 17.68it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 17.36it/s]
+1.998x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  hf_Reformer                        
+AUTOTUNE mm(4096x256, 256x768)
+  mm 0.0179 ms 100.0%
+  triton_mm_2 0.0197 ms 90.6%
+  triton_mm_7 0.0203 ms 88.2%
+  triton_mm_1 0.0206 ms 86.9%
+  triton_mm_3 0.0214 ms 83.4%
+  triton_mm_4 0.0216 ms 82.9%
+  triton_mm_0 0.0229 ms 78.0%
+  triton_mm_8 0.0241 ms 74.3%
+  triton_mm_10 0.0304 ms 58.9%
+  triton_mm_6 0.0364 ms 49.1%
+SingleProcess AUTOTUNE takes 5.1070 seconds
+AUTOTUNE mm(4096x768, 768x256)
+  mm 0.0199 ms 100.0%
+  triton_mm_60 0.0220 ms 90.3%
+  triton_mm_59 0.0227 ms 87.7%
+  triton_mm_66 0.0232 ms 85.8%
+  triton_mm_62 0.0237 ms 84.1%
+  triton_mm_61 0.0243 ms 82.1%
+  triton_mm_58 0.0312 ms 63.8%
+  triton_mm_63 0.0323 ms 61.7%
+  triton_mm_64 0.0326 ms 61.0%
+  triton_mm_67 0.0348 ms 57.2%
+SingleProcess AUTOTUNE takes 4.7087 seconds
+AUTOTUNE mm(4096x256, 256x512)
+  triton_mm_72 0.0158 ms 100.0%
+  triton_mm_71 0.0162 ms 97.6%
+  triton_mm_70 0.0167 ms 94.5%
+  mm 0.0172 ms 91.8%
+  triton_mm_74 0.0177 ms 89.5%
+  triton_mm_73 0.0177 ms 89.2%
+  triton_mm_78 0.0183 ms 86.2%
+  triton_mm_77 0.0196 ms 80.5%
+  triton_mm_80 0.0225 ms 70.2%
+  triton_mm_75 0.0271 ms 58.3%
+SingleProcess AUTOTUNE takes 4.7282 seconds
+AUTOTUNE mm(4096x512, 512x256)
+  mm 0.0174 ms 100.0%
+  triton_mm_83 0.0177 ms 98.7%
+  triton_mm_84 0.0178 ms 97.8%
+  triton_mm_90 0.0184 ms 94.8%
+  triton_mm_86 0.0184 ms 94.6%
+  triton_mm_85 0.0189 ms 92.2%
+  triton_mm_82 0.0228 ms 76.4%
+  triton_mm_87 0.0247 ms 70.7%
+  triton_mm_88 0.0249 ms 70.1%
+  triton_mm_91 0.0262 ms 66.6%
+SingleProcess AUTOTUNE takes 4.7769 seconds
+AUTOTUNE addmm(4096x320, 4096x512, 512x320)
+  triton_mm_566 0.0185 ms 100.0%
+  bias_addmm 0.0185 ms 99.8%
+  triton_mm_572 0.0192 ms 96.3%
+  triton_mm_565 0.0197 ms 93.8%
+  triton_mm_568 0.0199 ms 92.8%
+  triton_mm_567 0.0206 ms 89.7%
+  triton_mm_564 0.0240 ms 76.9%
+  addmm 0.0244 ms 75.5%
+  triton_mm_569 0.0286 ms 64.6%
+  triton_mm_571 0.0288 ms 64.0%
+SingleProcess AUTOTUNE takes 5.5893 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 69.74it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 72.94it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 72.11it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 72.62it/s]
+3.683x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  hf_T5                              
+AUTOTUNE bmm(8x2048x64, 8x64x2048)
+  triton_bmm_26 0.0600 ms 100.0%
+  triton_bmm_25 0.0603 ms 99.6%
+  triton_bmm_27 0.0699 ms 85.9%
+  triton_bmm_28 0.0702 ms 85.5%
+  triton_bmm_32 0.0728 ms 82.5%
+  triton_bmm_24 0.0794 ms 75.6%
+  triton_bmm_34 0.0858 ms 69.9%
+  triton_bmm_31 0.0873 ms 68.8%
+  triton_bmm_33 0.1092 ms 55.0%
+  bmm 0.1306 ms 46.0%
+SingleProcess AUTOTUNE takes 4.2774 seconds
+AUTOTUNE bmm(8x2048x2048, 8x2048x64)
+  triton_bmm_56 0.0697 ms 100.0%
+  bmm 0.0708 ms 98.5%
+  triton_bmm_49 0.0725 ms 96.1%
+  triton_bmm_50 0.0729 ms 95.6%
+  triton_bmm_51 0.0736 ms 94.7%
+  triton_bmm_52 0.0749 ms 93.1%
+  triton_bmm_54 0.0776 ms 89.9%
+  triton_bmm_53 0.0802 ms 87.0%
+  triton_bmm_48 0.0889 ms 78.5%
+  triton_bmm_57 0.0891 ms 78.2%
+SingleProcess AUTOTUNE takes 4.2191 seconds
+AUTOTUNE mm(2048x512, 512x2048)
+  triton_mm_157 0.0322 ms 100.0%
+  triton_mm_158 0.0322 ms 100.0%
+  triton_mm_160 0.0368 ms 87.3%
+  triton_mm_159 0.0373 ms 86.3%
+  mm 0.0398 ms 80.8%
+  triton_mm_156 0.0424 ms 75.9%
+  triton_mm_164 0.0428 ms 75.2%
+  triton_mm_163 0.0442 ms 72.7%
+  triton_mm_166 0.0645 ms 49.9%
+  triton_mm_162 0.0718 ms 44.8%
+SingleProcess AUTOTUNE takes 4.4741 seconds
+AUTOTUNE mm(2048x2048, 2048x512)
+  mm 0.0351 ms 100.0%
+  triton_mm_170 0.0430 ms 81.6%
+  triton_mm_169 0.0431 ms 81.5%
+  triton_mm_176 0.0446 ms 78.7%
+  triton_mm_171 0.0473 ms 74.3%
+  triton_mm_172 0.0475 ms 73.9%
+  triton_mm_168 0.0534 ms 65.7%
+  triton_mm_173 0.0670 ms 52.4%
+  triton_mm_174 0.0684 ms 51.3%
+  triton_mm_175 0.0690 ms 50.9%
+SingleProcess AUTOTUNE takes 4.5250 seconds
+AUTOTUNE mm(2048x512, 512x32128)
+  mm 0.3414 ms 100.0%
+  triton_mm_1586 0.4116 ms 82.9%
+  triton_mm_1585 0.4209 ms 81.1%
+  triton_mm_1591 0.4657 ms 73.3%
+  triton_mm_1588 0.4817 ms 70.9%
+  triton_mm_1587 0.4881 ms 69.9%
+  triton_mm_1584 0.5344 ms 63.9%
+  triton_mm_1592 0.5772 ms 59.1%
+  triton_mm_1594 0.8193 ms 41.7%
+  triton_mm_1593 1.0448 ms 32.7%
+SingleProcess AUTOTUNE takes 5.2677 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:01, 16.57it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 22.70it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 24.80it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 25.89it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 26.52it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 26.79it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 27.03it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 27.22it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 27.38it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 27.50it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 26.40it/s]
+2.769x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+cuda eval  hf_T5_base                         
+AUTOTUNE bmm(12x2048x64, 12x64x2048)
+  triton_bmm_25 0.0857 ms 100.0%
+  triton_bmm_26 0.0867 ms 98.9%
+  triton_bmm_27 0.1003 ms 85.5%
+  triton_bmm_28 0.1004 ms 85.4%
+  triton_bmm_32 0.1058 ms 81.0%
+  triton_bmm_24 0.1141 ms 75.1%
+  triton_bmm_34 0.1226 ms 69.9%
+  triton_bmm_31 0.1255 ms 68.3%
+  triton_bmm_33 0.1606 ms 53.4%
+  bmm 0.1931 ms 44.4%
+SingleProcess AUTOTUNE takes 4.3848 seconds
+AUTOTUNE bmm(12x2048x2048, 12x2048x64)
+  bmm 0.0889 ms 100.0%
+  triton_bmm_52 0.0896 ms 99.2%
+  triton_bmm_50 0.0896 ms 99.1%
+  triton_bmm_49 0.0906 ms 98.1%
+  triton_bmm_51 0.0969 ms 91.7%
+  triton_bmm_56 0.0984 ms 90.3%
+  triton_bmm_55 0.1020 ms 87.1%
+  triton_bmm_48 0.1028 ms 86.5%
+  triton_bmm_54 0.1110 ms 80.1%
+  triton_bmm_53 0.1146 ms 77.6%
+SingleProcess AUTOTUNE takes 4.2835 seconds
+AUTOTUNE mm(2048x768, 768x32128)
+  mm 0.4870 ms 100.0%
+  triton_mm_3170 0.5966 ms 81.6%
+  triton_mm_3169 0.6126 ms 79.5%
+  triton_mm_3175 0.6640 ms 73.3%
+  triton_mm_3171 0.6893 ms 70.7%
+  triton_mm_3172 0.7031 ms 69.3%
+  triton_mm_3168 0.7868 ms 61.9%
+  triton_mm_3176 0.8317 ms 58.6%
+  triton_mm_3178 1.1662 ms 41.8%
+  triton_mm_3177 1.4678 ms 33.2%
+SingleProcess AUTOTUNE takes 5.1584 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:12,  2.38it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:06,  4.29it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:04,  5.77it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:03,  6.89it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:03,  7.71it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:02,  8.31it/s]
running benchmark:  23%|██▎       | 7/30 [00:01<00:02,  8.74it/s]
running benchmark:  27%|██▋       | 8/30 [00:01<00:02,  9.05it/s]
running benchmark:  30%|███       | 9/30 [00:01<00:02,  9.24it/s]
running benchmark:  33%|███▎      | 10/30 [00:01<00:02,  9.37it/s]
running benchmark:  37%|███▋      | 11/30 [00:01<00:02,  9.46it/s]
running benchmark:  40%|████      | 12/30 [00:01<00:01,  9.53it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01,  9.59it/s]
running benchmark:  47%|████▋     | 14/30 [00:01<00:01,  9.65it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01,  9.69it/s]
running benchmark:  53%|█████▎    | 16/30 [00:01<00:01,  9.71it/s]
running benchmark:  57%|█████▋    | 17/30 [00:02<00:01,  9.73it/s]
running benchmark:  60%|██████    | 18/30 [00:02<00:01,  9.75it/s]
running benchmark:  63%|██████▎   | 19/30 [00:02<00:01,  9.76it/s]
running benchmark:  67%|██████▋   | 20/30 [00:02<00:01,  9.77it/s]
running benchmark:  70%|███████   | 21/30 [00:02<00:00,  9.78it/s]
running benchmark:  73%|███████▎  | 22/30 [00:02<00:00,  9.78it/s]
running benchmark:  77%|███████▋  | 23/30 [00:02<00:00,  9.77it/s]
running benchmark:  80%|████████  | 24/30 [00:02<00:00,  9.77it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00,  9.75it/s]
running benchmark:  87%|████████▋ | 26/30 [00:02<00:00,  9.75it/s]
running benchmark:  90%|█████████ | 27/30 [00:03<00:00,  9.76it/s]
running benchmark:  93%|█████████▎| 28/30 [00:03<00:00,  9.77it/s]
running benchmark:  97%|█████████▋| 29/30 [00:03<00:00,  9.78it/s]
running benchmark: 100%|██████████| 30/30 [00:03<00:00,  9.75it/s]
running benchmark: 100%|██████████| 30/30 [00:03<00:00,  8.85it/s]
+2.778x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:06, ?it/s]
+cuda eval  hf_T5_generate                     
+AUTOTUNE mm(1x512, 512x512)
+  mm 0.0090 ms 100.0%
+  triton_mm_581 0.0096 ms 93.0%
+  triton_mm_584 0.0099 ms 90.9%
+  triton_mm_582 0.0102 ms 88.1%
+  triton_mm_580 0.0107 ms 83.8%
+  triton_mm_585 0.0107 ms 83.8%
+  triton_mm_579 0.0114 ms 78.9%
+  triton_mm_578 0.0129 ms 69.5%
+  triton_mm_577 0.0137 ms 65.4%
+  triton_mm_576 0.0189 ms 47.5%
+SingleProcess AUTOTUNE takes 3.9093 seconds
+AUTOTUNE mm(1x512, 512x512)
+  mm 0.0092 ms 100.0%
+  triton_mm_594 0.0096 ms 95.0%
+  triton_mm_596 0.0099 ms 92.9%
+  triton_mm_592 0.0101 ms 90.5%
+  triton_mm_593 0.0102 ms 89.9%
+  triton_mm_597 0.0107 ms 85.6%
+  triton_mm_591 0.0119 ms 76.7%
+  triton_mm_590 0.0124 ms 73.7%
+  triton_mm_589 0.0132 ms 69.3%
+  triton_mm_588 0.0184 ms 49.7%
+SingleProcess AUTOTUNE takes 3.9971 seconds
+AUTOTUNE bmm(8x1x64, 8x64x1)
+  triton_bmm_601 0.0061 ms 100.0%
+  triton_bmm_602 0.0061 ms 100.0%
+  triton_bmm_603 0.0061 ms 100.0%
+  triton_bmm_604 0.0061 ms 100.0%
+  triton_bmm_600 0.0066 ms 92.2%
+  triton_bmm_607 0.0068 ms 88.8%
+  triton_bmm_605 0.0069 ms 88.4%
+  triton_bmm_606 0.0069 ms 88.0%
+  bmm 0.0073 ms 83.0%
+SingleProcess AUTOTUNE takes 2.2296 seconds
+AUTOTUNE bmm(8x1x1, 8x1x64)
+  triton_bmm_621 0.0061 ms 100.0%
+  triton_bmm_623 0.0061 ms 100.0%
+  triton_bmm_626 0.0061 ms 100.0%
+  triton_bmm_622 0.0067 ms 90.5%
+  triton_bmm_625 0.0068 ms 89.6%
+  triton_bmm_624 0.0068 ms 89.2%
+  triton_bmm_620 0.0068 ms 89.2%
+  bmm 0.0092 ms 66.2%
+SingleProcess AUTOTUNE takes 2.0466 seconds
+AUTOTUNE bmm(8x1x64, 8x64x2048)
+  triton_bmm_672 0.0091 ms 100.0%
+  triton_bmm_669 0.0094 ms 96.3%
+  triton_bmm_670 0.0096 ms 94.4%
+  triton_bmm_666 0.0097 ms 93.7%
+  triton_bmm_671 0.0099 ms 91.9%
+  triton_bmm_665 0.0100 ms 91.0%
+  triton_bmm_667 0.0100 ms 91.0%
+  triton_bmm_664 0.0100 ms 90.7%
+  triton_bmm_673 0.0101 ms 89.9%
+  triton_bmm_663 0.0102 ms 89.4%
+SingleProcess AUTOTUNE takes 3.4739 seconds
+AUTOTUNE bmm(8x1x2048, 8x2048x64)
+  bmm 0.0139 ms 100.0%
+  triton_bmm_690 0.0194 ms 71.6%
+  triton_bmm_691 0.0198 ms 70.1%
+  triton_bmm_692 0.0208 ms 66.6%
+  triton_bmm_693 0.0223 ms 62.1%
+  triton_bmm_689 0.0232 ms 59.6%
+  triton_bmm_688 0.0311 ms 44.5%
+  triton_bmm_687 0.0560 ms 24.8%
+  triton_bmm_695 0.0873 ms 15.9%
+  triton_bmm_694 0.0934 ms 14.8%
+SingleProcess AUTOTUNE takes 3.0264 seconds
+AUTOTUNE mm(1x512, 512x2048)
+  mm 0.0105 ms 100.0%
+  triton_mm_714 0.0109 ms 95.6%
+  triton_mm_716 0.0110 ms 95.1%
+  triton_mm_712 0.0114 ms 91.9%
+  triton_mm_711 0.0119 ms 88.1%
+  triton_mm_713 0.0122 ms 85.8%
+  triton_mm_717 0.0129 ms 80.9%
+  triton_mm_710 0.0140 ms 75.0%
+  triton_mm_709 0.0141 ms 74.0%
+  triton_mm_708 0.0200 ms 52.3%
+SingleProcess AUTOTUNE takes 3.5233 seconds
+AUTOTUNE mm(1x2048, 2048x512)
+  mm 0.0103 ms 100.0%
+  triton_mm_725 0.0200 ms 51.8%
+  triton_mm_726 0.0202 ms 51.1%
+  triton_mm_728 0.0210 ms 49.2%
+  triton_mm_729 0.0217 ms 47.6%
+  triton_mm_724 0.0222 ms 46.5%
+  triton_mm_723 0.0258 ms 40.0%
+  triton_mm_722 0.0328 ms 31.5%
+  triton_mm_721 0.0346 ms 29.9%
+  triton_mm_720 0.0570 ms 18.1%
+SingleProcess AUTOTUNE takes 3.9050 seconds
+AUTOTUNE mm(1x512, 512x32128)
+  triton_mm_1514 0.0395 ms 100.0%
+  triton_mm_1513 0.0395 ms 99.9%
+  triton_mm_1518 0.0396 ms 99.7%
+  triton_mm_1516 0.0397 ms 99.4%
+  triton_mm_1515 0.0410 ms 96.3%
+  mm 0.0421 ms 93.6%
+  triton_mm_1520 0.0467 ms 84.5%
+  triton_mm_1512 0.0469 ms 84.1%
+  triton_mm_1519 0.0469 ms 84.0%
+  triton_mm_1522 0.0494 ms 79.8%
+SingleProcess AUTOTUNE takes 4.0804 seconds
+AUTOTUNE bmm(8x1x64, 8x64x2)
+  triton_bmm_1550 0.0062 ms 100.0%
+  triton_bmm_1549 0.0062 ms 99.5%
+  bmm 0.0066 ms 93.7%
+  triton_bmm_1548 0.0066 ms 93.7%
+  triton_bmm_1552 0.0067 ms 92.6%
+  triton_bmm_1553 0.0068 ms 91.0%
+  triton_bmm_1551 0.0069 ms 89.8%
+  triton_bmm_1554 0.0071 ms 87.3%
+  triton_bmm_1555 0.0076 ms 81.3%
+SingleProcess AUTOTUNE takes 2.2545 seconds
+AUTOTUNE bmm(8x1x2, 8x2x64)
+  triton_bmm_1569 0.0061 ms 100.0%
+  triton_bmm_1572 0.0061 ms 100.0%
+  triton_bmm_1571 0.0063 ms 96.0%
+  triton_bmm_1570 0.0067 ms 91.1%
+  triton_bmm_1568 0.0069 ms 88.4%
+  triton_bmm_1574 0.0070 ms 87.4%
+  triton_bmm_1573 0.0071 ms 85.2%
+  bmm 0.0092 ms 66.2%
+SingleProcess AUTOTUNE takes 2.0702 seconds
+AUTOTUNE bmm(8x1x64, 8x64x3)
+  triton_bmm_2354 0.0063 ms 100.0%
+  triton_bmm_2355 0.0063 ms 100.0%
+  triton_bmm_2353 0.0068 ms 93.0%
+  triton_bmm_2352 0.0068 ms 93.0%
+  triton_bmm_2357 0.0069 ms 91.2%
+  triton_bmm_2356 0.0070 ms 90.0%
+  bmm 0.0078 ms 81.1%
+  triton_bmm_2358 0.0078 ms 80.8%
+  triton_bmm_2359 0.0079 ms 80.5%
+SingleProcess AUTOTUNE takes 3.2895 seconds
+AUTOTUNE bmm(8x1x3, 8x3x64)
+  triton_bmm_2372 0.0066 ms 100.0%
+  triton_bmm_2374 0.0066 ms 100.0%
+  triton_bmm_2378 0.0069 ms 95.8%
+  triton_bmm_2375 0.0070 ms 94.5%
+  triton_bmm_2377 0.0070 ms 93.8%
+  triton_bmm_2376 0.0072 ms 91.8%
+  triton_bmm_2373 0.0073 ms 90.4%
+  bmm 0.0104 ms 63.4%
+SingleProcess AUTOTUNE takes 2.3096 seconds
+AUTOTUNE bmm(8x1x64, 8x64x4)
+  triton_bmm_3157 0.0063 ms 100.0%
+  triton_bmm_3159 0.0063 ms 100.0%
+  triton_bmm_3160 0.0063 ms 100.0%
+  bmm 0.0066 ms 96.1%
+  triton_bmm_3156 0.0068 ms 93.0%
+  triton_bmm_3158 0.0069 ms 91.7%
+  triton_bmm_3161 0.0070 ms 90.8%
+  triton_bmm_3163 0.0073 ms 86.5%
+  triton_bmm_3162 0.0078 ms 80.8%
+SingleProcess AUTOTUNE takes 1.4498 seconds
+AUTOTUNE bmm(8x1x4, 8x4x64)
+  triton_bmm_3182 0.0063 ms 100.0%
+  triton_bmm_3179 0.0064 ms 98.0%
+  triton_bmm_3176 0.0066 ms 95.6%
+  triton_bmm_3181 0.0070 ms 90.4%
+  triton_bmm_3180 0.0071 ms 88.3%
+  triton_bmm_3178 0.0072 ms 87.4%
+  triton_bmm_3177 0.0073 ms 86.8%
+  bmm 0.0097 ms 64.8%
+SingleProcess AUTOTUNE takes 1.0059 seconds
+AUTOTUNE bmm(8x1x64, 8x64x5)
+  triton_bmm_3964 0.0063 ms 100.0%
+  triton_bmm_3965 0.0064 ms 99.5%
+  triton_bmm_3960 0.0068 ms 92.5%
+  triton_bmm_3962 0.0069 ms 92.1%
+  triton_bmm_3961 0.0070 ms 90.0%
+  triton_bmm_3963 0.0071 ms 88.8%
+  triton_bmm_3966 0.0073 ms 86.5%
+  triton_bmm_3967 0.0073 ms 86.5%
+  bmm 0.0074 ms 86.1%
+SingleProcess AUTOTUNE takes 1.1249 seconds
+AUTOTUNE bmm(8x1x5, 8x5x64)
+  triton_bmm_3983 0.0066 ms 100.0%
+  triton_bmm_3980 0.0066 ms 99.5%
+  triton_bmm_3982 0.0066 ms 99.5%
+  triton_bmm_3984 0.0066 ms 99.5%
+  triton_bmm_3985 0.0066 ms 99.5%
+  triton_bmm_3986 0.0067 ms 97.6%
+  triton_bmm_3981 0.0074 ms 88.7%
+  bmm 0.0101 ms 64.7%
+SingleProcess AUTOTUNE takes 1.0123 seconds
+AUTOTUNE bmm(8x1x64, 8x64x6)
+  triton_bmm_4765 0.0063 ms 100.0%
+  triton_bmm_4767 0.0063 ms 100.0%
+  triton_bmm_4768 0.0063 ms 100.0%
+  triton_bmm_4769 0.0070 ms 90.4%
+  triton_bmm_4766 0.0071 ms 88.8%
+  triton_bmm_4771 0.0073 ms 86.5%
+  bmm 0.0074 ms 85.7%
+  triton_bmm_4764 0.0074 ms 85.5%
+  triton_bmm_4770 0.0079 ms 80.5%
+SingleProcess AUTOTUNE takes 1.1267 seconds
+AUTOTUNE bmm(8x1x6, 8x6x64)
+  triton_bmm_4787 0.0066 ms 100.0%
+  triton_bmm_4784 0.0066 ms 99.5%
+  triton_bmm_4786 0.0066 ms 99.5%
+  triton_bmm_4790 0.0068 ms 97.2%
+  triton_bmm_4789 0.0071 ms 91.9%
+  triton_bmm_4785 0.0073 ms 90.1%
+  triton_bmm_4788 0.0073 ms 90.1%
+  bmm 0.0097 ms 67.4%
+SingleProcess AUTOTUNE takes 1.0225 seconds
+AUTOTUNE bmm(8x1x64, 8x64x7)
+  triton_bmm_5571 0.0063 ms 100.0%
+  triton_bmm_5573 0.0066 ms 96.6%
+  triton_bmm_5569 0.0070 ms 91.0%
+  triton_bmm_5572 0.0071 ms 89.6%
+  triton_bmm_5570 0.0071 ms 88.8%
+  triton_bmm_5574 0.0073 ms 86.5%
+  bmm 0.0074 ms 86.1%
+  triton_bmm_5568 0.0076 ms 82.8%
+  triton_bmm_5575 0.0079 ms 80.2%
+SingleProcess AUTOTUNE takes 1.1196 seconds
+AUTOTUNE bmm(8x1x7, 8x7x64)
+  triton_bmm_5593 0.0066 ms 100.0%
+  triton_bmm_5589 0.0066 ms 99.5%
+  triton_bmm_5594 0.0069 ms 95.3%
+  triton_bmm_5591 0.0071 ms 92.8%
+  triton_bmm_5588 0.0073 ms 90.3%
+  triton_bmm_5592 0.0073 ms 89.9%
+  triton_bmm_5590 0.0074 ms 88.7%
+  bmm 0.0108 ms 60.7%
+SingleProcess AUTOTUNE takes 1.0055 seconds
+AUTOTUNE bmm(8x1x64, 8x64x8)
+  triton_bmm_6373 0.0063 ms 100.0%
+  triton_bmm_6374 0.0063 ms 100.0%
+  triton_bmm_6375 0.0063 ms 100.0%
+  triton_bmm_6377 0.0064 ms 98.5%
+  triton_bmm_6372 0.0068 ms 92.5%
+  triton_bmm_6376 0.0071 ms 88.8%
+  triton_bmm_6378 0.0073 ms 86.5%
+  triton_bmm_6379 0.0073 ms 86.5%
+  bmm 0.0074 ms 85.7%
+SingleProcess AUTOTUNE takes 1.1227 seconds
+AUTOTUNE bmm(8x1x8, 8x8x64)
+  triton_bmm_6398 0.0063 ms 100.0%
+  triton_bmm_6392 0.0066 ms 95.6%
+  triton_bmm_6394 0.0066 ms 95.6%
+  triton_bmm_6397 0.0071 ms 89.3%
+  triton_bmm_6395 0.0071 ms 88.3%
+  triton_bmm_6396 0.0072 ms 87.9%
+  triton_bmm_6393 0.0073 ms 86.0%
+  bmm 0.0094 ms 66.8%
+SingleProcess AUTOTUNE takes 1.0101 seconds
+[2023-12-06 20:15:17,750] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (8)
+[2023-12-06 20:15:17,750] torch._dynamo.convert_frame: [WARNING]    function: 'forward' (/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py:1645)
+[2023-12-06 20:15:17,750] torch._dynamo.convert_frame: [WARNING]    last reason: ___check_obj_id(L['past_key_values'], 7628576)                # mask_seq_length = past_key_values[0][0].shape[2] + seq_length if past_key_values is not None else seq_length  # miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py:1026 in forward
+[2023-12-06 20:15:17,750] torch._dynamo.convert_frame: [WARNING] To log all recompilation reasons, use TORCH_LOGS="recompiles".
+[2023-12-06 20:15:17,750] torch._dynamo.convert_frame: [WARNING] To diagnose recompilation issues, see https://pytorch.org/docs/master/compile/troubleshooting.html.
+AUTOTUNE bmm(8x1x64, 8x64x9)
+  triton_bmm_7178 0.0064 ms 100.0%
+  triton_bmm_7179 0.0064 ms 100.0%
+  triton_bmm_7181 0.0069 ms 93.3%
+  triton_bmm_7180 0.0071 ms 90.5%
+  triton_bmm_7177 0.0071 ms 90.3%
+  triton_bmm_7182 0.0074 ms 87.0%
+  triton_bmm_7176 0.0076 ms 84.8%
+  triton_bmm_7183 0.0080 ms 80.4%
+  bmm 0.0081 ms 79.1%
+SingleProcess AUTOTUNE takes 2.3522 seconds
+AUTOTUNE bmm(8x1x9, 8x9x64)
+  triton_bmm_7196 0.0068 ms 100.0%
+  triton_bmm_7198 0.0068 ms 100.0%
+  triton_bmm_7200 0.0068 ms 100.0%
+  triton_bmm_7202 0.0069 ms 99.5%
+  triton_bmm_7199 0.0071 ms 96.0%
+  triton_bmm_7201 0.0071 ms 96.0%
+  triton_bmm_7197 0.0074 ms 92.0%
+  bmm 0.0100 ms 68.2%
+SingleProcess AUTOTUNE takes 1.9660 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:02<01:25,  2.94s/it]
running benchmark:   7%|▋         | 2/30 [00:05<01:18,  2.80s/it]
running benchmark:  10%|█         | 3/30 [00:08<01:14,  2.76s/it]
running benchmark:  13%|█▎        | 4/30 [00:11<01:10,  2.73s/it]
running benchmark:  17%|█▋        | 5/30 [00:13<01:07,  2.71s/it]
running benchmark:  20%|██        | 6/30 [00:16<01:05,  2.72s/it]
running benchmark:  23%|██▎       | 7/30 [00:19<01:02,  2.71s/it]
running benchmark:  27%|██▋       | 8/30 [00:21<00:59,  2.71s/it]
running benchmark:  30%|███       | 9/30 [00:24<00:56,  2.71s/it]
running benchmark:  33%|███▎      | 10/30 [00:27<00:54,  2.72s/it]
running benchmark:  37%|███▋      | 11/30 [00:30<00:51,  2.73s/it]
running benchmark:  40%|████      | 12/30 [00:32<00:48,  2.72s/it]
running benchmark:  43%|████▎     | 13/30 [00:35<00:46,  2.73s/it]
running benchmark:  47%|████▋     | 14/30 [00:38<00:43,  2.74s/it]
running benchmark:  50%|█████     | 15/30 [00:40<00:40,  2.73s/it]
running benchmark:  53%|█████▎    | 16/30 [00:43<00:38,  2.72s/it]
running benchmark:  57%|█████▋    | 17/30 [00:46<00:35,  2.73s/it]
running benchmark:  60%|██████    | 18/30 [00:49<00:32,  2.75s/it]
running benchmark:  63%|██████▎   | 19/30 [00:51<00:30,  2.76s/it]
running benchmark:  67%|██████▋   | 20/30 [00:54<00:27,  2.75s/it]
running benchmark:  70%|███████   | 21/30 [00:57<00:24,  2.76s/it]
running benchmark:  73%|███████▎  | 22/30 [01:00<00:22,  2.77s/it]
running benchmark:  77%|███████▋  | 23/30 [01:03<00:19,  2.77s/it]
running benchmark:  80%|████████  | 24/30 [01:05<00:16,  2.76s/it]
running benchmark:  83%|████████▎ | 25/30 [01:08<00:13,  2.79s/it]
running benchmark:  87%|████████▋ | 26/30 [01:11<00:11,  2.78s/it]
running benchmark:  90%|█████████ | 27/30 [01:14<00:08,  2.76s/it]
running benchmark:  93%|█████████▎| 28/30 [01:16<00:05,  2.76s/it]
running benchmark:  97%|█████████▋| 29/30 [01:19<00:02,  2.76s/it]
running benchmark: 100%|██████████| 30/30 [01:22<00:00,  2.76s/it]
running benchmark: 100%|██████████| 30/30 [01:22<00:00,  2.75s/it]
+7.306x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:16, ?it/s]
+cuda eval  hf_T5_large                        
+AUTOTUNE mm(512x1024, 1024x1024)
+  mm 0.0154 ms 100.0%
+  triton_mm_3 0.0190 ms 80.9%
+  triton_mm_4 0.0191 ms 80.5%
+  triton_mm_8 0.0206 ms 74.7%
+  triton_mm_2 0.0228 ms 67.2%
+  triton_mm_1 0.0230 ms 66.7%
+  triton_mm_5 0.0264 ms 58.2%
+  triton_mm_6 0.0267 ms 57.5%
+  triton_mm_9 0.0278 ms 55.2%
+  triton_mm_0 0.0348 ms 44.2%
+SingleProcess AUTOTUNE takes 4.8676 seconds
+AUTOTUNE bmm(16x512x64, 16x64x512)
+  triton_bmm_25 0.0152 ms 100.0%
+  triton_bmm_26 0.0156 ms 97.5%
+  triton_bmm_32 0.0159 ms 95.8%
+  triton_bmm_27 0.0164 ms 92.4%
+  triton_bmm_28 0.0165 ms 92.2%
+  triton_bmm_24 0.0170 ms 89.6%
+  triton_bmm_34 0.0185 ms 82.2%
+  bmm 0.0187 ms 81.4%
+  triton_bmm_31 0.0198 ms 76.6%
+  triton_bmm_33 0.0202 ms 75.3%
+SingleProcess AUTOTUNE takes 4.5118 seconds
+AUTOTUNE bmm(16x512x512, 16x512x64)
+  triton_bmm_52 0.0162 ms 100.0%
+  triton_bmm_56 0.0170 ms 95.3%
+  triton_bmm_51 0.0175 ms 92.7%
+  triton_bmm_49 0.0181 ms 89.4%
+  triton_bmm_50 0.0181 ms 89.2%
+  triton_bmm_54 0.0182 ms 88.9%
+  bmm 0.0186 ms 87.2%
+  triton_bmm_53 0.0197 ms 82.1%
+  triton_bmm_57 0.0202 ms 80.3%
+  triton_bmm_48 0.0258 ms 62.8%
+SingleProcess AUTOTUNE takes 4.4360 seconds
+AUTOTUNE mm(512x1024, 1024x4096)
+  triton_mm_157 0.0337 ms 100.0%
+  triton_mm_158 0.0339 ms 99.5%
+  mm 0.0355 ms 94.9%
+  triton_mm_159 0.0402 ms 83.8%
+  triton_mm_160 0.0403 ms 83.6%
+  triton_mm_164 0.0408 ms 82.5%
+  triton_mm_156 0.0420 ms 80.2%
+  triton_mm_166 0.0606 ms 55.6%
+  triton_mm_165 0.0684 ms 49.3%
+  triton_mm_161 0.0694 ms 48.6%
+SingleProcess AUTOTUNE takes 5.0741 seconds
+AUTOTUNE mm(512x4096, 4096x1024)
+  mm 0.0358 ms 100.0%
+  triton_mm_171 0.0525 ms 68.2%
+  triton_mm_172 0.0530 ms 67.6%
+  triton_mm_176 0.0572 ms 62.6%
+  triton_mm_170 0.0732 ms 49.0%
+  triton_mm_169 0.0740 ms 48.5%
+  triton_mm_173 0.0786 ms 45.6%
+  triton_mm_174 0.0795 ms 45.1%
+  triton_mm_177 0.0897 ms 40.0%
+  triton_mm_168 0.0942 ms 38.0%
+SingleProcess AUTOTUNE takes 4.5752 seconds
+AUTOTUNE mm(512x1024, 1024x32128)
+  mm 0.1649 ms 100.0%
+  triton_mm_6337 0.1922 ms 85.8%
+  triton_mm_6338 0.1938 ms 85.1%
+  triton_mm_6340 0.2212 ms 74.6%
+  triton_mm_6339 0.2220 ms 74.3%
+  triton_mm_6336 0.2600 ms 63.4%
+  triton_mm_6344 0.2614 ms 63.1%
+  triton_mm_6343 0.3308 ms 49.9%
+  triton_mm_6346 0.3976 ms 41.5%
+  triton_mm_6345 0.4674 ms 35.3%
+SingleProcess AUTOTUNE takes 5.7757 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  7.33it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 10.69it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 12.50it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 13.48it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 14.04it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 14.43it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 14.72it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 14.88it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 14.87it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 15.05it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 15.15it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 15.18it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 15.20it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 15.20it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 15.20it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 14.48it/s]
+4.698x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  hf_Whisper                         
+AUTOTUNE mm(12000x256, 256x256)
+  mm 0.0188 ms 100.0%
+  triton_mm_7 0.0215 ms 87.6%
+  triton_mm_0 0.0217 ms 86.9%
+  triton_mm_3 0.0218 ms 86.3%
+  triton_mm_2 0.0220 ms 85.6%
+  triton_mm_1 0.0222 ms 84.6%
+  triton_mm_4 0.0223 ms 84.2%
+  triton_mm_8 0.0240 ms 78.3%
+  triton_mm_10 0.0316 ms 59.6%
+  triton_mm_6 0.0353 ms 53.3%
+SingleProcess AUTOTUNE takes 4.5461 seconds
+AUTOTUNE mm(12000x256, 256x1536)
+  mm 0.0629 ms 100.0%
+  triton_mm_50 0.0681 ms 92.4%
+  triton_mm_49 0.0687 ms 91.6%
+  triton_mm_55 0.0785 ms 80.1%
+  triton_mm_48 0.0815 ms 77.2%
+  triton_mm_51 0.0821 ms 76.6%
+  triton_mm_52 0.0821 ms 76.6%
+  triton_mm_56 0.0942 ms 66.8%
+  triton_mm_58 0.1199 ms 52.5%
+  triton_mm_54 0.1708 ms 36.8%
+SingleProcess AUTOTUNE takes 4.4912 seconds
+AUTOTUNE mm(12000x1536, 1536x256)
+  mm 0.0546 ms 100.0%
+  triton_mm_63 0.0738 ms 74.0%
+  triton_mm_64 0.0743 ms 73.4%
+  triton_mm_61 0.0753 ms 72.5%
+  triton_mm_62 0.0753 ms 72.4%
+  triton_mm_68 0.0776 ms 70.3%
+  triton_mm_60 0.0920 ms 59.3%
+  triton_mm_67 0.1060 ms 51.4%
+  triton_mm_66 0.1363 ms 40.0%
+  triton_mm_65 0.1366 ms 39.9%
+SingleProcess AUTOTUNE takes 4.7721 seconds
+AUTOTUNE addmm(12000x256, 12000x256, 256x256)
+  bias_addmm 0.0204 ms 100.0%
+  triton_mm_432 0.0225 ms 90.7%
+  triton_mm_439 0.0229 ms 89.1%
+  triton_mm_435 0.0235 ms 86.8%
+  triton_mm_434 0.0237 ms 85.8%
+  triton_mm_436 0.0241 ms 84.5%
+  triton_mm_433 0.0243 ms 83.8%
+  triton_mm_440 0.0254 ms 80.2%
+  addmm 0.0306 ms 66.6%
+  triton_mm_442 0.0317 ms 64.2%
+SingleProcess AUTOTUNE takes 5.3463 seconds
+AUTOTUNE addmm(8x2, 8x256, 256x2)
+  triton_mm_447 0.0077 ms 100.0%
+  triton_mm_446 0.0079 ms 97.2%
+  triton_mm_449 0.0079 ms 96.8%
+  triton_mm_448 0.0081 ms 94.3%
+  triton_mm_445 0.0087 ms 88.2%
+  triton_mm_444 0.0111 ms 69.4%
+  bias_addmm 0.0120 ms 64.2%
+  triton_mm_451 0.0123 ms 62.5%
+  triton_mm_450 0.0129 ms 59.7%
+  addmm 0.0151 ms 50.7%
+SingleProcess AUTOTUNE takes 2.8126 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 73.52it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 76.30it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 77.12it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 76.92it/s]
+3.466x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:07, ?it/s]
+WARNING:root:hf_clip failed to load
+Original Error: 'str' object has no attribute 'shape'
+Eager model failed to run
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1931, in validate_model
+    self.model_iter_fn(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 537, in forward_pass
+    return mod(*inputs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 1142, in forward
+    vision_outputs = self.vision_model(
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 883, in forward
+    hidden_states = self.embeddings(pixel_values)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1512, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/cdhernandez/local/pytorch/torch/nn/modules/module.py", line 1521, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 194, in forward
+    batch_size = pixel_values.shape[0]
+AttributeError: 'str' object has no attribute 'shape'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 473, in load_model
+    self.validate_model(model, example_inputs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1934, in validate_model
+    raise NotImplementedError("Eager model failed to run") from e
+NotImplementedError: Eager model failed to run
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  lennard_jones                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 1416.64it/s]
+1.421x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  llama                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:00, 71.38it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 73.50it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 74.30it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 73.91it/s]
+5.392x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [01:01, ?it/s]
+cuda eval  llama_v2_7b_16h                    
+AUTOTUNE mm(512x4096, 4096x4096)
+  mm 0.0954 ms 100.0%
+  triton_mm_1 0.1059 ms 90.1%
+  triton_mm_2 0.1068 ms 89.4%
+  triton_mm_3 0.1284 ms 74.3%
+  triton_mm_4 0.1291 ms 73.9%
+  triton_mm_8 0.1301 ms 73.3%
+  triton_mm_0 0.1383 ms 69.0%
+  triton_mm_10 0.2285 ms 41.8%
+  triton_mm_5 0.2380 ms 40.1%
+  triton_mm_6 0.2395 ms 39.8%
+SingleProcess AUTOTUNE takes 4.6384 seconds
+AUTOTUNE mm(512x4096, 4096x11008)
+  mm 0.2214 ms 100.0%
+  triton_mm_50 0.2807 ms 78.9%
+  triton_mm_49 0.2813 ms 78.7%
+  triton_mm_51 0.2992 ms 74.0%
+  triton_mm_52 0.2996 ms 73.9%
+  triton_mm_56 0.3365 ms 65.8%
+  triton_mm_48 0.3515 ms 63.0%
+  triton_mm_55 0.4845 ms 45.7%
+  triton_mm_58 0.5749 ms 38.5%
+  triton_mm_53 0.6157 ms 36.0%
+SingleProcess AUTOTUNE takes 5.0664 seconds
+AUTOTUNE mm(512x11008, 11008x4096)
+  mm 0.2179 ms 100.0%
+  triton_mm_73 0.2912 ms 74.8%
+  triton_mm_74 0.2921 ms 74.6%
+  triton_mm_75 0.3414 ms 63.8%
+  triton_mm_76 0.3421 ms 63.7%
+  triton_mm_80 0.3492 ms 62.4%
+  triton_mm_72 0.3616 ms 60.3%
+  triton_mm_82 0.5888 ms 37.0%
+  triton_mm_77 0.6389 ms 34.1%
+  triton_mm_79 0.6424 ms 33.9%
+SingleProcess AUTOTUNE takes 4.7952 seconds
+AUTOTUNE mm(512x4096, 4096x32000)
+  mm 0.5845 ms 100.0%
+  triton_mm_1345 0.7356 ms 79.5%
+  triton_mm_1346 0.7376 ms 79.3%
+  triton_mm_1347 0.8441 ms 69.3%
+  triton_mm_1348 0.8453 ms 69.2%
+  triton_mm_1352 0.9971 ms 58.6%
+  triton_mm_1344 1.0007 ms 58.4%
+  triton_mm_1351 1.2037 ms 48.6%
+  triton_mm_1354 1.5991 ms 36.6%
+  triton_mm_1349 1.8113 ms 32.3%
+SingleProcess AUTOTUNE takes 5.4826 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  9.28it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:01, 16.60it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 18.43it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:01, 19.20it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 19.53it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 19.76it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 19.94it/s]
running benchmark:  73%|███████▎  | 22/30 [00:01<00:00, 20.03it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 20.10it/s]
running benchmark:  93%|█████████▎| 28/30 [00:01<00:00, 20.15it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 19.49it/s]
+1.377x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/__init__.py", line 78, in __init__
+    self.meta_inputs = torch.load(f'{root}/batch.pt')
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 998, in load
+    with _open_file_like(f, 'rb') as opened_file:
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 445, in _open_file_like
+    return _open_file(name_or_buffer, mode)
+  File "/home/cdhernandez/local/pytorch/torch/serialization.py", line 426, in __init__
+    super().__init__(open(name, mode))
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/maml_omniglot/batch.pt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  mnasnet1_0                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 95.51it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 114.08it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 114.54it/s]
+3.378x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  mobilenet_v2                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 109.82it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 123.74it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 122.73it/s]
+5.165x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+WARNING:root:mobilenet_v2_quantized_qat failed to load
+The eval test only supports CPU.
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/mobilenet_v2_quantized_qat/__init__.py", line 21, in __init__
+    raise NotImplementedError("The eval test only supports CPU.")
+NotImplementedError: The eval test only supports CPU.
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  mobilenet_v3_large                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 92.67it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 103.23it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 103.81it/s]
+4.410x
+
loading model: 0it [00:00, ?it/s]NCCL version 2.19.3+cuda12.0
+
loading model: 0it [00:02, ?it/s]
+cuda eval  moco                               
+[rank0]:[2023-12-06 20:27:44,450] [0/0] torch._dynamo.variables.torch: [WARNING] Profiler function <class 'torch.autograd.profiler.record_function'> will be ignored
+[rank0]:[2023-12-06 20:27:48,999] [1/0_1] torch._dynamo.backends.distributed: [WARNING] Some buckets were extended beyond their requested parameter capacities in order to ensure each subgraph has an output node, required for fx graph partitioning. This can be the case when a subgraph would have only contained nodes performing inplace mutation, and returning no logical outputs. This should not be a problem, unless it results in too few graph partitions for optimal DDP performance.
+[rank0]:[2023-12-06 20:27:49,024] [1/0_1] torch._dynamo.backends.distributed: [WARNING] DDPOptimizer extended these buckets to ensure per-subgraph output nodes:
+[rank0]:[2023-12-06 20:27:49,024] [1/0_1] torch._dynamo.backends.distributed: [WARNING] ┌─────────┬─────────────┬────────────────────────┐
+[rank0]:[2023-12-06 20:27:49,024] [1/0_1] torch._dynamo.backends.distributed: [WARNING] │   Index │   Extra Ops │   Extra Param Size (b) │
+[rank0]:[2023-12-06 20:27:49,024] [1/0_1] torch._dynamo.backends.distributed: [WARNING] ├─────────┼─────────────┼────────────────────────┤
+[rank0]:[2023-12-06 20:27:49,024] [1/0_1] torch._dynamo.backends.distributed: [WARNING] │       0 │         157 │               44910720 │
+[rank0]:[2023-12-06 20:27:49,024] [1/0_1] torch._dynamo.backends.distributed: [WARNING] └─────────┴─────────────┴────────────────────────┘
+skipping cudagraphs due to ['mutated inputs']
+[rank0]:[2023-12-06 20:28:11,802] [5/0_1] torch._inductor.utils: [WARNING] DeviceCopy in input program
+skipping cudagraphs due to ['non-cuda device in graph']
+[rank0]:[W CUDAGraph.cpp:145] Warning: Waiting for pending NCCL work to finish before starting graph capture. (function operator())
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:00, 27.18it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 28.48it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 29.52it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 30.00it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 30.23it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 30.28it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 30.53it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 30.73it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 30.20it/s]
+2.131x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+number of parameters: 123.69M
+num decayed parameter tensors: 50, with 124,354,560 parameters
+num non-decayed parameter tensors: 98, with 121,344 parameters
+using fused AdamW: True
+cuda eval  nanogpt                            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 128.57it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 134.39it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 133.97it/s]
+6.656x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+cuda eval  nvidia_deeprecommender             
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 202.48it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 205.58it/s]
+0.922x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  opacus_cifar10                     
+AUTOTUNE addmm(64x10, 64x512, 512x10)
+  triton_mm_143 0.0093 ms 100.0%
+  triton_mm_142 0.0099 ms 94.8%
+  triton_mm_144 0.0099 ms 94.8%
+  triton_mm_141 0.0102 ms 91.4%
+  triton_mm_145 0.0105 ms 88.8%
+  bias_addmm 0.0125 ms 74.9%
+  triton_mm_140 0.0130 ms 71.9%
+  addmm 0.0157 ms 59.3%
+  triton_mm_139 0.0190 ms 49.2%
+  triton_mm_147 0.0198 ms 47.2%
+SingleProcess AUTOTUNE takes 2.9628 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 223.32it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 222.30it/s]
+4.592x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:25, ?it/s]
+cuda eval  phi_1_5                            
+AUTOTUNE addmm(512x6144, 512x2048, 2048x6144)
+  bias_addmm 0.0627 ms 100.0%
+  addmm 0.0771 ms 81.2%
+  triton_mm_3 0.0898 ms 69.8%
+  triton_mm_4 0.0910 ms 68.8%
+  triton_mm_1 0.0945 ms 66.3%
+  triton_mm_2 0.0951 ms 65.9%
+  triton_mm_8 0.1085 ms 57.8%
+  triton_mm_0 0.1150 ms 54.5%
+  triton_mm_7 0.1358 ms 46.1%
+  triton_mm_6 0.1860 ms 33.7%
+SingleProcess AUTOTUNE takes 5.4715 seconds
+AUTOTUNE mm(512x2048, 2048x2048)
+  mm 0.0351 ms 100.0%
+  triton_mm_38 0.0446 ms 78.8%
+  triton_mm_37 0.0447 ms 78.6%
+  triton_mm_44 0.0461 ms 76.3%
+  triton_mm_39 0.0477 ms 73.6%
+  triton_mm_40 0.0478 ms 73.5%
+  triton_mm_36 0.0552 ms 63.7%
+  triton_mm_41 0.0669 ms 52.5%
+  triton_mm_42 0.0684 ms 51.4%
+  triton_mm_43 0.0693 ms 50.7%
+SingleProcess AUTOTUNE takes 4.8148 seconds
+AUTOTUNE mm(512x2048, 2048x8192)
+  triton_mm_49 0.0976 ms 100.0%
+  triton_mm_50 0.1078 ms 90.5%
+  triton_mm_51 0.1108 ms 88.0%
+  triton_mm_52 0.1110 ms 87.9%
+  mm 0.1153 ms 84.7%
+  triton_mm_56 0.1320 ms 73.9%
+  triton_mm_48 0.1377 ms 70.9%
+  triton_mm_55 0.1920 ms 50.8%
+  triton_mm_53 0.2313 ms 42.2%
+  triton_mm_54 0.2349 ms 41.6%
+SingleProcess AUTOTUNE takes 4.5932 seconds
+AUTOTUNE mm(512x8192, 8192x2048)
+  mm 0.0916 ms 100.0%
+  triton_mm_62 0.1462 ms 62.7%
+  triton_mm_61 0.1474 ms 62.2%
+  triton_mm_68 0.1539 ms 59.5%
+  triton_mm_64 0.1623 ms 56.5%
+  triton_mm_63 0.1626 ms 56.4%
+  triton_mm_60 0.1881 ms 48.7%
+  triton_mm_65 0.2374 ms 38.6%
+  triton_mm_66 0.2411 ms 38.0%
+  triton_mm_67 0.2452 ms 37.4%
+SingleProcess AUTOTUNE takes 4.8840 seconds
+AUTOTUNE mm(512x2048, 2048x51200)
+  mm 0.4877 ms 100.0%
+  triton_mm_1729 0.5938 ms 82.1%
+  triton_mm_1730 0.5969 ms 81.7%
+  triton_mm_1731 0.6800 ms 71.7%
+  triton_mm_1732 0.6875 ms 70.9%
+  triton_mm_1736 0.8100 ms 60.2%
+  triton_mm_1728 0.8131 ms 60.0%
+  triton_mm_1735 0.9389 ms 51.9%
+  triton_mm_1738 1.2581 ms 38.8%
+  triton_mm_1733 1.4711 ms 33.2%
+SingleProcess AUTOTUNE takes 5.2881 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:01, 16.46it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 20.07it/s]
running benchmark:  27%|██▋       | 8/30 [00:00<00:01, 21.25it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 21.92it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 22.29it/s]
running benchmark:  57%|█████▋    | 17/30 [00:00<00:00, 22.44it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 22.64it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 22.73it/s]
running benchmark:  87%|████████▋ | 26/30 [00:01<00:00, 22.84it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 22.96it/s]
running benchmark: 100%|██████████| 30/30 [00:01<00:00, 22.29it/s]
+2.974x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+cuda eval  phlippe_densenet                   
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 56.74it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 73.60it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 78.51it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 76.84it/s]
+6.121x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+cuda eval  phlippe_resnet                     
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 256.60it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 256.97it/s]
+4.246x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  pyhpc_equation_of_state            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 194.17it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 206.53it/s]
+16.964x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  pyhpc_isoneutral_mixing            
+skipping cudagraphs due to ['mutated inputs']
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 84.47it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 85.73it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 86.18it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 85.88it/s]
+6.063x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+
loading model: 0it [00:01, ?it/s]
+WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+cuda eval  pyhpc_turbulent_kinetic_energy     
+WARNING:common:Model pyhpc_turbulent_kinetic_energy supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:00, 49.29it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 74.36it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 80.67it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 78.84it/s]
+8.151x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/__init__.py", line 13, in <module>
+    from .train_cyclegan import prepare_training_loop
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/train_cyclegan.py", line 27, in <module>
+    from .util.visualizer import Visualizer
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/util/visualizer.py", line 6, in <module>
+    from . import util, html
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/util/html.py", line 1, in <module>
+    import dominate
+ModuleNotFoundError: No module named 'dominate'
+Failed to import user benchmark module dynamo, error: No module named 'dominate'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/__init__.py", line 52, in __init__
+    self.data_loader = self.get_data_loader(config)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/__init__.py", line 68, in get_data_loader
+    celeba_loader = get_loader(config.celeba_image_dir, config.attr_path, config.selected_attrs,
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 82, in get_loader
+    dataset = CelebA(image_dir, attr_path, selected_attrs, transform, mode)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 24, in __init__
+    self.preprocess()
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/pytorch_stargan/data_loader.py", line 33, in preprocess
+    lines = [line.rstrip() for line in open(self.attr_path, 'r')]
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/data/.data/pytorch_stargan_inputs/data/celeba/list_attr_celeba.txt'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  pytorch_unet                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  13%|█▎        | 4/30 [00:00<00:00, 37.44it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 42.78it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 44.51it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 45.31it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 45.74it/s]
running benchmark:  97%|█████████▋| 29/30 [00:00<00:00, 46.01it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 45.06it/s]
+1.818x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  resnet152                          
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:01, 25.50it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 30.92it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 32.72it/s]
running benchmark:  50%|█████     | 15/30 [00:00<00:00, 33.57it/s]
running benchmark:  63%|██████▎   | 19/30 [00:00<00:00, 34.11it/s]
running benchmark:  77%|███████▋  | 23/30 [00:00<00:00, 34.45it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 34.62it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 33.68it/s]
+2.435x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  resnet18                           
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 232.42it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 237.30it/s]
+3.687x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  resnet50                           
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 69.04it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 79.27it/s]
running benchmark:  83%|████████▎ | 25/30 [00:00<00:00, 82.49it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 81.49it/s]
+1.893x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+WARNING:root:resnet50_quantized_qat failed to load
+The eval test only supports CPU.
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/resnet50_quantized_qat/__init__.py", line 21, in __init__
+    raise NotImplementedError("The eval test only supports CPU.")
+NotImplementedError: The eval test only supports CPU.
+
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  resnext50_32x4d                    
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 106.42it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 107.50it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 108.93it/s]
+3.985x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/__init__.py", line 27, in __init__
+    self.model = sam_model_registry[model_type](checkpoint=sam_checkpoint)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/build_sam.py", line 19, in build_sam_vit_h
+    return _build_sam(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/build_sam.py", line 108, in _build_sam
+    with open(checkpoint, "rb") as f:
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/sam/.data/sam_vit_h_4b8939.pth'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  shufflenet_v2_x1_0                 
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 95.69it/s]
running benchmark:  70%|███████   | 21/30 [00:00<00:00, 98.88it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 99.04it/s]
+3.714x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:03, ?it/s]
+cuda eval  soft_actor_critic                  
+AUTOTUNE mm(256x1024, 1024x1024)
+  mm 0.0134 ms 100.0%
+  triton_mm_19 0.0156 ms 85.7%
+  triton_mm_20 0.0181 ms 74.2%
+  triton_mm_15 0.0187 ms 71.7%
+  triton_mm_14 0.0188 ms 71.5%
+  triton_mm_16 0.0194 ms 69.3%
+  triton_mm_17 0.0196 ms 68.6%
+  triton_mm_12 0.0228 ms 58.9%
+  triton_mm_13 0.0236 ms 56.8%
+  triton_mm_11 0.0340 ms 39.4%
+SingleProcess AUTOTUNE takes 4.8827 seconds
+AUTOTUNE addmm(256x2, 256x1024, 1024x2)
+  bias_addmm 0.0126 ms 100.0%
+  triton_mm_29 0.0130 ms 96.9%
+  triton_mm_31 0.0133 ms 95.0%
+  triton_mm_28 0.0136 ms 92.7%
+  triton_mm_32 0.0144 ms 87.8%
+  triton_mm_26 0.0146 ms 86.4%
+  triton_mm_27 0.0164 ms 77.0%
+  addmm 0.0168 ms 75.1%
+  triton_mm_24 0.0195 ms 65.0%
+  triton_mm_25 0.0210 ms 60.1%
+SingleProcess AUTOTUNE takes 4.4334 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 1097.67it/s]
+1.374x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/speech_transformer/__init__.py", line 15, in <module>
+    from .config import SpeechTransformerTrainConfig, SpeechTransformerEvalConfig
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/speech_transformer/config.py", line 4, in <module>
+    import kaldi_io
+ModuleNotFoundError: No module named 'kaldi_io'
+Failed to import user benchmark module dynamo, error: No module named 'kaldi_io'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  squeezenet1_1                      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 310.63it/s]
+3.480x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/stable_diffusion_text_encoder/__init__.py", line 11, in <module>
+    from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+ModuleNotFoundError: No module named 'diffusers'
+Failed to import user benchmark module dynamo, error: No module named 'diffusers'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/stable_diffusion_unet/__init__.py", line 11, in <module>
+    from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+ModuleNotFoundError: No module named 'diffusers'
+Failed to import user benchmark module dynamo, error: No module named 'diffusers'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/timm_efficientdet/__init__.py", line 12, in <module>
+    from effdet import create_model, create_loader
+ModuleNotFoundError: No module named 'effdet'
+Failed to import user benchmark module dynamo, error: No module named 'effdet'
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  timm_efficientnet                  
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 51.67it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 57.56it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 59.43it/s]
running benchmark:  90%|█████████ | 27/30 [00:00<00:00, 60.35it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 59.41it/s]
+2.272x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:05, ?it/s]
+cuda eval  timm_nfnet                         
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:04,  6.78it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 11.56it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:01, 13.28it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 14.08it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 14.53it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 14.82it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:01, 14.98it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:00, 15.10it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:00, 15.12it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 15.19it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 15.23it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 15.27it/s]
running benchmark:  83%|████████▎ | 25/30 [00:01<00:00, 15.30it/s]
running benchmark:  90%|█████████ | 27/30 [00:01<00:00, 15.31it/s]
running benchmark:  97%|█████████▋| 29/30 [00:01<00:00, 15.33it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 14.72it/s]
+1.867x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:04, ?it/s]
+cuda eval  timm_regnet                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:01, 19.65it/s]
running benchmark:  20%|██        | 6/30 [00:00<00:00, 28.81it/s]
running benchmark:  33%|███▎      | 10/30 [00:00<00:00, 31.48it/s]
running benchmark:  47%|████▋     | 14/30 [00:00<00:00, 32.70it/s]
running benchmark:  60%|██████    | 18/30 [00:00<00:00, 33.34it/s]
running benchmark:  73%|███████▎  | 22/30 [00:00<00:00, 33.73it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 33.97it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 34.12it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 32.81it/s]
+1.380x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  timm_resnest                       
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:00, 86.19it/s]
running benchmark:  67%|██████▋   | 20/30 [00:00<00:00, 99.67it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 101.17it/s]
+1.726x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:01, ?it/s]
+cuda eval  timm_vision_transformer            
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:00, 109.10it/s]
running benchmark:  80%|████████  | 24/30 [00:00<00:00, 117.58it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 117.45it/s]
+1.400x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:18, ?it/s]
+cuda eval  timm_vision_transformer_large      
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:14,  1.96it/s]
running benchmark:   7%|▋         | 2/30 [00:00<00:09,  3.03it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:07,  3.66it/s]
running benchmark:  13%|█▎        | 4/30 [00:01<00:06,  4.07it/s]
running benchmark:  17%|█▋        | 5/30 [00:01<00:05,  4.33it/s]
running benchmark:  20%|██        | 6/30 [00:01<00:05,  4.51it/s]
running benchmark:  23%|██▎       | 7/30 [00:01<00:04,  4.63it/s]
running benchmark:  27%|██▋       | 8/30 [00:01<00:04,  4.71it/s]
running benchmark:  30%|███       | 9/30 [00:02<00:04,  4.77it/s]
running benchmark:  33%|███▎      | 10/30 [00:02<00:04,  4.80it/s]
running benchmark:  37%|███▋      | 11/30 [00:02<00:03,  4.82it/s]
running benchmark:  40%|████      | 12/30 [00:02<00:03,  4.84it/s]
running benchmark:  43%|████▎     | 13/30 [00:02<00:03,  4.86it/s]
running benchmark:  47%|████▋     | 14/30 [00:03<00:03,  4.87it/s]
running benchmark:  50%|█████     | 15/30 [00:03<00:03,  4.87it/s]
running benchmark:  53%|█████▎    | 16/30 [00:03<00:02,  4.88it/s]
running benchmark:  57%|█████▋    | 17/30 [00:03<00:02,  4.88it/s]
running benchmark:  60%|██████    | 18/30 [00:03<00:02,  4.89it/s]
running benchmark:  63%|██████▎   | 19/30 [00:04<00:02,  4.88it/s]
running benchmark:  67%|██████▋   | 20/30 [00:04<00:02,  4.88it/s]
running benchmark:  70%|███████   | 21/30 [00:04<00:01,  4.87it/s]
running benchmark:  73%|███████▎  | 22/30 [00:04<00:01,  4.88it/s]
running benchmark:  77%|███████▋  | 23/30 [00:05<00:01,  4.88it/s]
running benchmark:  80%|████████  | 24/30 [00:05<00:01,  4.87it/s]
running benchmark:  83%|████████▎ | 25/30 [00:05<00:01,  4.87it/s]
running benchmark:  87%|████████▋ | 26/30 [00:05<00:00,  4.88it/s]
running benchmark:  90%|█████████ | 27/30 [00:05<00:00,  4.89it/s]
running benchmark:  93%|█████████▎| 28/30 [00:06<00:00,  4.88it/s]
running benchmark:  97%|█████████▋| 29/30 [00:06<00:00,  4.87it/s]
running benchmark: 100%|██████████| 30/30 [00:06<00:00,  4.87it/s]
running benchmark: 100%|██████████| 30/30 [00:06<00:00,  4.65it/s]
+0.996x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  timm_vovnet                        
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:00, 69.42it/s]
running benchmark:  53%|█████▎    | 16/30 [00:00<00:00, 81.48it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 85.63it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 84.18it/s]
+1.796x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 440, in load_model
+    benchmark = benchmark_cls(
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/util/model.py", line 24, in __call__
+    obj = type.__call__(cls, *args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/torch_multimodal_clip/__init__.py", line 27, in __init__
+    self.image = Image.open(os.path.join(self.data_folder, self.image_name))
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/site-packages/PIL/Image.py", line 3218, in open
+    fp = builtins.open(filename, "rb")
+FileNotFoundError: [Errno 2] No such file or directory: '/home/cdhernandez/local/benchmark/torchbenchmark/models/torch_multimodal_clip/.data/pizza.jpg'
+Run failed with return code:  1
+Output:  None
+Error:  None
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model tts_angular supports float32 only
+
loading model: 0it [00:00, ?it/s]
+WARNING:common:Model tts_angular supports float32 only
+cuda eval  tts_angular                        
+WARNING:common:Model tts_angular supports float32 only
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:  43%|████▎     | 13/30 [00:00<00:00, 126.39it/s]
running benchmark:  87%|████████▋ | 26/30 [00:00<00:00, 126.63it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 126.53it/s]
+0.988x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:02, ?it/s]
+cuda eval  vgg16                              
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 292.71it/s]
running benchmark: 100%|██████████| 30/30 [00:00<00:00, 292.17it/s]
+1.575x
+
loading model: 0it [00:00, ?it/s]WARNING:common:Model vision_maskrcnn supports float32 only
+
loading model: 0it [00:05, ?it/s]
+WARNING:common:Model vision_maskrcnn supports float32 only
+cuda eval  vision_maskrcnn                    
+WARNING:common:Model vision_maskrcnn supports float32 only
+AUTOTUNE mm(1000x12544, 12544x1024)
+  mm 0.2529 ms 100.0%
+  triton_mm_768 0.3532 ms 71.6%
+  triton_mm_767 0.3605 ms 70.1%
+  triton_mm_766 0.4360 ms 58.0%
+  triton_mm_769 0.4556 ms 55.5%
+  triton_mm_770 0.4646 ms 54.4%
+  triton_mm_774 0.4768 ms 53.0%
+  triton_mm_776 0.5425 ms 46.6%
+  triton_mm_771 0.5659 ms 44.7%
+  triton_mm_772 0.5698 ms 44.4%
+SingleProcess AUTOTUNE takes 6.3264 seconds
+AUTOTUNE mm(1000x1024, 1024x1024)
+  mm 0.0374 ms 100.0%
+  triton_mm_780 0.0379 ms 98.7%
+  triton_mm_779 0.0387 ms 96.7%
+  triton_mm_781 0.0489 ms 76.5%
+  triton_mm_782 0.0490 ms 76.5%
+  triton_mm_778 0.0502 ms 74.6%
+  triton_mm_786 0.0513 ms 73.0%
+  triton_mm_788 0.0529 ms 70.8%
+  triton_mm_784 0.0550 ms 68.1%
+  triton_mm_783 0.0554 ms 67.6%
+SingleProcess AUTOTUNE takes 6.3618 seconds
+AUTOTUNE addmm(1000x91, 1000x1024, 1024x91)
+  triton_mm_796 0.0179 ms 100.0%
+  triton_mm_795 0.0188 ms 95.1%
+  triton_mm_799 0.0198 ms 90.3%
+  triton_mm_798 0.0225 ms 79.4%
+  addmm 0.0274 ms 65.2%
+  triton_mm_793 0.0279 ms 64.1%
+  triton_mm_791 0.0281 ms 63.6%
+  triton_mm_792 0.0294 ms 60.9%
+  triton_mm_794 0.0307 ms 58.2%
+  triton_mm_790 0.0393 ms 45.6%
+SingleProcess AUTOTUNE takes 7.1002 seconds
+AUTOTUNE addmm(1000x364, 1000x1024, 1024x364)
+  bias_addmm 0.0211 ms 100.0%
+  triton_mm_810 0.0220 ms 95.6%
+  addmm 0.0264 ms 79.9%
+  triton_mm_807 0.0267 ms 78.9%
+  triton_mm_808 0.0271 ms 77.8%
+  triton_mm_805 0.0277 ms 76.1%
+  triton_mm_803 0.0280 ms 75.2%
+  triton_mm_806 0.0280 ms 75.2%
+  triton_mm_804 0.0285 ms 74.0%
+  triton_mm_811 0.0386 ms 54.7%
+SingleProcess AUTOTUNE takes 6.5778 seconds
+AUTOTUNE convolution(34x256x14x14, 256x256x3x3)
+  convolution 0.0883 ms 100.0%
+  triton_convolution_817 0.2055 ms 43.0%
+  triton_convolution_819 0.2379 ms 37.1%
+  triton_convolution_818 0.2574 ms 34.3%
+  triton_convolution_820 0.2624 ms 33.6%
+  triton_convolution_814 0.3105 ms 28.4%
+  triton_convolution_815 0.5031 ms 17.5%
+  triton_convolution_816 1.0525 ms 8.4%
+SingleProcess AUTOTUNE takes 6.8707 seconds
+AUTOTUNE addmm(26656x91, 26656x256, 256x91)
+  triton_mm_843 0.0470 ms 100.0%
+  triton_mm_844 0.0473 ms 99.3%
+  triton_mm_845 0.0505 ms 93.0%
+  bias_addmm 0.0513 ms 91.6%
+  triton_mm_842 0.0513 ms 91.6%
+  triton_mm_849 0.0536 ms 87.7%
+  triton_mm_846 0.0550 ms 85.4%
+  triton_mm_847 0.0609 ms 77.1%
+  triton_mm_852 0.0629 ms 74.7%
+  addmm 0.0634 ms 74.1%
+SingleProcess AUTOTUNE takes 7.2566 seconds
+
running benchmark:   0%|          | 0/30 [00:00<?, ?it/s]
running benchmark:   3%|▎         | 1/30 [00:00<00:03,  7.48it/s]
running benchmark:  10%|█         | 3/30 [00:00<00:02, 10.64it/s]
running benchmark:  17%|█▋        | 5/30 [00:00<00:02, 11.44it/s]
running benchmark:  23%|██▎       | 7/30 [00:00<00:01, 11.71it/s]
running benchmark:  30%|███       | 9/30 [00:00<00:01, 11.97it/s]
running benchmark:  37%|███▋      | 11/30 [00:00<00:01, 12.23it/s]
running benchmark:  43%|████▎     | 13/30 [00:01<00:01, 12.43it/s]
running benchmark:  50%|█████     | 15/30 [00:01<00:01, 12.48it/s]
running benchmark:  57%|█████▋    | 17/30 [00:01<00:01, 12.47it/s]
running benchmark:  63%|██████▎   | 19/30 [00:01<00:00, 12.51it/s]
running benchmark:  70%|███████   | 21/30 [00:01<00:00, 12.53it/s]
running benchmark:  77%|███████▋  | 23/30 [00:01<00:00, 12.52it/s]
running benchmark:  83%|████████▎ | 25/30 [00:02<00:00, 12.59it/s]
running benchmark:  90%|█████████ | 27/30 [00:02<00:00, 12.65it/s]
running benchmark:  97%|█████████▋| 29/30 [00:02<00:00, 12.69it/s]
running benchmark: 100%|██████████| 30/30 [00:02<00:00, 12.30it/s]
+1.083x
+
loading model: 0it [00:00, ?it/s]
loading model: 0it [00:00, ?it/s]
+Traceback (most recent call last):
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 36, in <module>
+    run()
+  File "/home/cdhernandez/local/benchmark/run_benchmark.py", line 30, in run
+    benchmark.run(bm_args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/run.py", line 24, in run
+    main(TorchBenchmarkRunner(), original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3131, in main
+    process_entry(0, runner, original_dir, args)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3088, in process_entry
+    return maybe_fresh_cache(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 1714, in inner
+    return fn(*args, **kwargs)
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/common.py", line 3556, in run
+    ) = runner.load_model(
+  File "/home/cdhernandez/local/benchmark/userbenchmark/dynamo/dynamobench/torchbench.py", line 380, in load_model
+    module = importlib.import_module(c)
+  File "/home/cdhernandez/local/miniconda3/envs/pytorch/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
+  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
+  File "/home/cdhernandez/local/benchmark/torchbenchmark/models/yolov3/__init__.py", line 28, in <module>
+    assert os.path.exists(DATA_DIR), "Couldn't find coco128 data dir, please run install.py again."
+AssertionError: Couldn't find coco128 data dir, please run install.py again.
+Run failed with return code:  1
+Output:  None
+Error:  None
+speedup             gmean=0.00x mean=2.790x
+abs_latency         gmean=0.00x mean=15.090x
+compilation_latency mean=45.905 seconds
+compression_ratio   mean=1.014x
+eager_peak_mem      gmean=0.00x mean=0.909x
+dynamo_peak_mem     gmean=0.00x mean=0.984x
+calls_captured      gmean=0.00x mean=611.270x
+unique_graphs       gmean=0.00x mean=4.230x
+graph_breaks        gmean=0.00x mean=1.892x
+unique_graph_breaks gmean=0.00x mean=0.514x
+start int4 weight only batchsize 1
+usage: run_benchmark.py [-h] [--filter FILTER] [--exclude EXCLUDE]
+                        [--exclude-exact EXCLUDE_EXACT]
+                        [--total-partitions {1,2,3,4,5,6,7,8,9}]
+                        [--partition-id PARTITION_ID] [--devices DEVICES]
+                        [--device-index DEVICE_INDEX] [--repeat REPEAT]
+                        [--iterations-per-run ITERATIONS_PER_RUN]
+                        [--randomize-input] [--threads THREADS] [--nopython]
+                        [--no-skip] [--prims-nvfuser] [--dump-raw-metrics]
+                        [--log-operator-inputs] [--channels-last]
+                        [--batch-size BATCH_SIZE] [--iterations ITERATIONS]
+                        [--batch-size-file BATCH_SIZE_FILE] [--cosine]
+                        [--cpp-wrapper] [--freezing] [--ci] [--dashboard]
+                        [--skip-fp64-check] [--fast] [--only ONLY]
+                        [--multiprocess]
+                        [--quantization {int8dynamic,int8weightonly,int4weightonly}]
+                        [--ddp] [--fsdp] [--no-optimize-ddp]
+                        [--distributed-master-port DISTRIBUTED_MASTER_PORT]
+                        [--dynamic-shapes] [--dynamic-batch-only]
+                        [--specialize-int] [--use-eval-mode]
+                        [--skip-accuracy-check]
+                        [--generate-aot-autograd-stats] [--inductor-settings]
+                        [--suppress-errors] [--output OUTPUT]
+                        [--output-directory OUTPUT_DIRECTORY]
+                        [--baseline BASELINE] [--part PART]
+                        [--export-profiler-trace]
+                        [--profiler-trace-name PROFILER_TRACE_NAME]
+                        [--diff-branch DIFF_BRANCH] [--tag TAG] [--explain]
+                        [--stats] [--print-memory] [--print-dataframe-summary]
+                        [--cold-start-latency] [--disable-cudagraphs]
+                        [--disable-split-reductions]
+                        [--disable-persistent-reductions]
+                        [--disable-divisible-by-16]
+                        [--inductor-compile-mode INDUCTOR_COMPILE_MODE]
+                        [--print-graph-breaks] [--log-graph-breaks]
+                        [--trace-on-xla] [--xla-tolerance XLA_TOLERANCE]
+                        [--collect-outputs]
+                        [--enable-activation-checkpointing] [--timing]
+                        [--progress] [--timeout TIMEOUT]
+                        [--per_process_memory_fraction PER_PROCESS_MEMORY_FRACTION]
+                        [--no-translation-validation] [--minify] [--nnc]
+                        [--float16 | --bfloat16 | --float32 | --amp]
+                        [--verbose | --quiet]
+                        [--coverage | --overhead | --speedup-dynamo-ts | --speedup-fx2trt | --speedup-fx2trt-fp16 | --print-fx | --print-aten-ops | --inductor | --export | --export-aot-inductor | --xla | --torchscript-onnx | --dynamo-onnx | --dynamo-onnx-aot-inline | --backend {aot_eager,aot_eager_decomp_partition,aot_eager_default_partitioner,aot_torchxla_trace_once,aot_torchxla_trivial,aot_ts,cudagraphs,dynamo_accuracy_minifier_backend,dynamo_minifier_backend,eager,eager_debug,inductor,non_leaf_compile_error_TESTING_ONLY,onnxrt,openxla,openxla_eval,pre_dispatch_eager,relu_accuracy_error_TESTING_ONLY,relu_compile_error_TESTING_ONLY,relu_runtime_error_TESTING_ONLY,torchxla_trace_once,torchxla_trivial,ts,tvm} | --nothing | --log-conv-args | --recompile-profiler | --find-batch-sizes]
+                        (--accuracy | --performance | --tolerance)
+                        (--training | --inference)
+run_benchmark.py: error: unrecognized arguments: --batchsize 1
+start baseline batchsize 1
+usage: run_benchmark.py [-h] [--filter FILTER] [--exclude EXCLUDE]
+                        [--exclude-exact EXCLUDE_EXACT]
+                        [--total-partitions {1,2,3,4,5,6,7,8,9}]
+                        [--partition-id PARTITION_ID] [--devices DEVICES]
+                        [--device-index DEVICE_INDEX] [--repeat REPEAT]
+                        [--iterations-per-run ITERATIONS_PER_RUN]
+                        [--randomize-input] [--threads THREADS] [--nopython]
+                        [--no-skip] [--prims-nvfuser] [--dump-raw-metrics]
+                        [--log-operator-inputs] [--channels-last]
+                        [--batch-size BATCH_SIZE] [--iterations ITERATIONS]
+                        [--batch-size-file BATCH_SIZE_FILE] [--cosine]
+                        [--cpp-wrapper] [--freezing] [--ci] [--dashboard]
+                        [--skip-fp64-check] [--fast] [--only ONLY]
+                        [--multiprocess]
+                        [--quantization {int8dynamic,int8weightonly,int4weightonly}]
+                        [--ddp] [--fsdp] [--no-optimize-ddp]
+                        [--distributed-master-port DISTRIBUTED_MASTER_PORT]
+                        [--dynamic-shapes] [--dynamic-batch-only]
+                        [--specialize-int] [--use-eval-mode]
+                        [--skip-accuracy-check]
+                        [--generate-aot-autograd-stats] [--inductor-settings]
+                        [--suppress-errors] [--output OUTPUT]
+                        [--output-directory OUTPUT_DIRECTORY]
+                        [--baseline BASELINE] [--part PART]
+                        [--export-profiler-trace]
+                        [--profiler-trace-name PROFILER_TRACE_NAME]
+                        [--diff-branch DIFF_BRANCH] [--tag TAG] [--explain]
+                        [--stats] [--print-memory] [--print-dataframe-summary]
+                        [--cold-start-latency] [--disable-cudagraphs]
+                        [--disable-split-reductions]
+                        [--disable-persistent-reductions]
+                        [--disable-divisible-by-16]
+                        [--inductor-compile-mode INDUCTOR_COMPILE_MODE]
+                        [--print-graph-breaks] [--log-graph-breaks]
+                        [--trace-on-xla] [--xla-tolerance XLA_TOLERANCE]
+                        [--collect-outputs]
+                        [--enable-activation-checkpointing] [--timing]
+                        [--progress] [--timeout TIMEOUT]
+                        [--per_process_memory_fraction PER_PROCESS_MEMORY_FRACTION]
+                        [--no-translation-validation] [--minify] [--nnc]
+                        [--float16 | --bfloat16 | --float32 | --amp]
+                        [--verbose | --quiet]
+                        [--coverage | --overhead | --speedup-dynamo-ts | --speedup-fx2trt | --speedup-fx2trt-fp16 | --print-fx | --print-aten-ops | --inductor | --export | --export-aot-inductor | --xla | --torchscript-onnx | --dynamo-onnx | --dynamo-onnx-aot-inline | --backend {aot_eager,aot_eager_decomp_partition,aot_eager_default_partitioner,aot_torchxla_trace_once,aot_torchxla_trivial,aot_ts,cudagraphs,dynamo_accuracy_minifier_backend,dynamo_minifier_backend,eager,eager_debug,inductor,non_leaf_compile_error_TESTING_ONLY,onnxrt,openxla,openxla_eval,pre_dispatch_eager,relu_accuracy_error_TESTING_ONLY,relu_compile_error_TESTING_ONLY,relu_runtime_error_TESTING_ONLY,torchxla_trace_once,torchxla_trivial,ts,tvm} | --nothing | --log-conv-args | --recompile-profiler | --find-batch-sizes]
+                        (--accuracy | --performance | --tolerance)
+                        (--training | --inference)
+run_benchmark.py: error: unrecognized arguments: --batchsize 1
diff --git a/log1.log b/log1.log
new file mode 100644
index 0000000000..50dcb0f473
Binary files /dev/null and b/log1.log differ
diff --git a/torchao_benchmarks.sh b/torchao_benchmarks.sh
new file mode 100644
index 0000000000..a67803a03d
--- /dev/null
+++ b/torchao_benchmarks.sh
@@ -0,0 +1,14 @@
+echo "start dynamic"
+python run_benchmark.py dynamo --bfloat16 --inductor --performance --inference --quantization int8dynamic --inductor-compile-mode max-autotune
+echo "start int8 weight only"
+python run_benchmark.py dynamo --bfloat16 --inductor --performance --inference --quantization int8weightonly --inductor-compile-mode max-autotune
+echo "start int4 weight only"
+python run_benchmark.py dynamo --bfloat16 --inductor --performance --inference --quantization int4weightonly --inductor-compile-mode max-autotune
+echo "start baseline"
+python run_benchmark.py dynamo --bfloat16 --inductor --performance --inference --inductor-compile-mode max-autotune
+
+
+echo "start int4 weight only batchsize 1"
+python run_benchmark.py dynamo --bfloat16 --inductor --performance --inference --quantization int4weightonly --inductor-compile-mode max-autotune --batch_size 1
+echo "start baseline batchsize 1"
+python run_benchmark.py dynamo --bfloat16 --inductor --performance --inference --inductor-compile-mode max-autotune --batch_size 1
diff --git a/userbenchmark/dynamo/dynamobench/common.py b/userbenchmark/dynamo/dynamobench/common.py
index b11c3cbf62..5fb2c584ce 100644
--- a/userbenchmark/dynamo/dynamobench/common.py
+++ b/userbenchmark/dynamo/dynamobench/common.py
@@ -73,6 +73,12 @@
 
 from torch.utils import _pytree as pytree
 from torch.utils._pytree import tree_map, tree_map_only
+import torchao
+from torchao.quantization import (
+    change_linear_weights_to_int8_dqtensors,
+    change_linear_weights_to_int8_woqtensors,
+    change_linear_weights_to_int4_woqtensors
+)
 
 from tqdm.auto import tqdm, trange
 
@@ -2714,11 +2720,17 @@ def get_example_inputs(self):
         action="store_true",
         help="Create n processes based on the number of devices (distributed use case).",
     )
+    parser.add_argument(
+        "--quantization",
+        choices=["int8dynamic", "int8weightonly", "int4weightonly"],
+        help="Apply quantization to the model before running it",
+    )
     parser.add_argument(
         "--ddp",
         action="store_true",
         help="Wraps model in DDP before running it, and uses dynamo DDPOptmizer (graph breaks) by default.",
     )
+
     parser.add_argument(
         "--fsdp",
         action="store_true",
@@ -3547,6 +3559,20 @@ def run(runner, args, original_dir=None):
                                     batch_size=batch_size,
                                     extra_args=extra_args,
                                 )
+                            if args.quantization:
+                                torch._dynamo.config.automatic_dynamic_shapes = False
+                                torch._dynamo.config.force_parameter_static_shapes = False
+                                torch._dynamo.config.cache_size_limit = 1000
+                                assert "cuda" in device
+                                if args.quantization=="int8dynamic":
+                                    torch._inductor.config.force_fuse_int_mm_with_mul = True
+                                    change_linear_weights_to_int8_dqtensors(model)
+                                elif args.quantization=="int8weightonly":
+                                    torch._inductor.config.use_mixed_mm = True
+                                    change_linear_weights_to_int8_woqtensors(model)
+                                elif args.quantization=="int4weightonly":
+                                    change_linear_weights_to_int4_woqtensors(model)
+
                 except NotImplementedError as e:
                     print(e)
                     import traceback
diff --git a/userbenchmark/dynamo/dynamobench/torchbench.py b/userbenchmark/dynamo/dynamobench/torchbench.py
index 9919332e1d..d6d8fd2495 100755
--- a/userbenchmark/dynamo/dynamobench/torchbench.py
+++ b/userbenchmark/dynamo/dynamobench/torchbench.py
@@ -257,7 +257,7 @@ def setup_torchbench_cwd():
     "pytorch_unet": 2,
 }
 
-FORCE_AMP_FOR_FP16_BF16_MODELS = {
+FORCE_FP32_FOR_FP16_BF16_MODELS = {
     "DALLE2_pytorch",
     "doctr_det_predictor",
     "doctr_reco_predictor",
@@ -265,9 +265,12 @@ def setup_torchbench_cwd():
     "tts_angular",
     "pyhpc_turbulent_kinetic_energy",
     "detectron2_fcos_r_50_fpn",
+    "vision_maskrcnn", # fails without quantization for fp16
 }
 
-FORCE_FP16_FOR_BF16_MODELS = {"vision_maskrcnn"}
+FORCE_AMP_FOR_FP16_BF16_MODELS = {}
+
+FORCE_FP16_FOR_BF16_MODELS = {}
 
 # models in canary_models that we should run anyway
 CANARY_MODELS = {
@@ -319,6 +322,10 @@ def skip_not_suitable_for_training_models(self):
     def failing_fx2trt_models(self):
         return TRT_NOT_YET_WORKING
 
+    @property
+    def fp32_only_models(self):
+        return FORCE_FP32_FOR_FP16_BF16_MODELS
+
     @property
     def force_amp_for_fp16_bf16_models(self):
         return FORCE_AMP_FOR_FP16_BF16_MODELS