From c30d44d264ceb54bb9de246d429b92f98e1057ba Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Wed, 19 Jun 2024 11:06:29 -0700 Subject: [PATCH] Re-enable detectron2 model accuracy checks. --- .../models/detectron2_fasterrcnn_r_101_c4/metadata.yaml | 2 -- .../models/detectron2_fasterrcnn_r_101_dc5/metadata.yaml | 1 - .../models/detectron2_fasterrcnn_r_101_fpn/metadata.yaml | 1 - .../models/detectron2_fasterrcnn_r_50_c4/metadata.yaml | 1 - .../models/detectron2_fasterrcnn_r_50_dc5/metadata.yaml | 1 - .../models/detectron2_fasterrcnn_r_50_fpn/metadata.yaml | 1 - torchbenchmark/models/detectron2_maskrcnn/__init__.py | 3 --- torchbenchmark/models/detectron2_maskrcnn/metadata.yaml | 1 - .../models/detectron2_maskrcnn_r_101_c4/metadata.yaml | 1 - .../models/detectron2_maskrcnn_r_101_fpn/metadata.yaml | 1 - .../models/detectron2_maskrcnn_r_50_c4/metadata.yaml | 1 - .../models/detectron2_maskrcnn_r_50_fpn/metadata.yaml | 1 - torchbenchmark/models/maml/__init__.py | 3 --- torchbenchmark/util/framework/detectron2/model_factory.py | 4 +--- 14 files changed, 1 insertion(+), 21 deletions(-) diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_101_c4/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_101_c4/metadata.yaml index a534730975..a8ee22c2a7 100644 --- a/torchbenchmark/models/detectron2_fasterrcnn_r_101_c4/metadata.yaml +++ b/torchbenchmark/models/detectron2_fasterrcnn_r_101_c4/metadata.yaml @@ -6,5 +6,3 @@ eval_deterministic: false eval_nograd: true train_benchmark: false train_deterministic: false -not_implemented: - - test: example diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_101_dc5/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_101_dc5/metadata.yaml index fab3cd3390..9c80a54250 100644 --- a/torchbenchmark/models/detectron2_fasterrcnn_r_101_dc5/metadata.yaml +++ b/torchbenchmark/models/detectron2_fasterrcnn_r_101_dc5/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_101_fpn/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_101_fpn/metadata.yaml index fab3cd3390..9c80a54250 100644 --- a/torchbenchmark/models/detectron2_fasterrcnn_r_101_fpn/metadata.yaml +++ b/torchbenchmark/models/detectron2_fasterrcnn_r_101_fpn/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_50_c4/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_50_c4/metadata.yaml index 7f98221a8e..2479d75cd3 100644 --- a/torchbenchmark/models/detectron2_fasterrcnn_r_50_c4/metadata.yaml +++ b/torchbenchmark/models/detectron2_fasterrcnn_r_50_c4/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_50_dc5/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_50_dc5/metadata.yaml index 7f98221a8e..2479d75cd3 100644 --- a/torchbenchmark/models/detectron2_fasterrcnn_r_50_dc5/metadata.yaml +++ b/torchbenchmark/models/detectron2_fasterrcnn_r_50_dc5/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/detectron2_fasterrcnn_r_50_fpn/metadata.yaml b/torchbenchmark/models/detectron2_fasterrcnn_r_50_fpn/metadata.yaml index fab3cd3390..9c80a54250 100644 --- a/torchbenchmark/models/detectron2_fasterrcnn_r_50_fpn/metadata.yaml +++ b/torchbenchmark/models/detectron2_fasterrcnn_r_50_fpn/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/detectron2_maskrcnn/__init__.py b/torchbenchmark/models/detectron2_maskrcnn/__init__.py index 206eeeaa46..2d5753e325 100644 --- a/torchbenchmark/models/detectron2_maskrcnn/__init__.py +++ b/torchbenchmark/models/detectron2_maskrcnn/__init__.py @@ -58,9 +58,6 @@ class Model(BenchmarkModel): model_file = os.path.join(MODEL_DIR, ".data", f"{MODEL_NAME}.pkl") DEFAULT_TRAIN_BSIZE = 1 DEFAULT_EVAL_BSIZE = 1 - # Skip correctness check, because the output tensor can't be verified using - # cosine similarity or torch.close() - SKIP_CORRECTNESS_CHECK = True def __init__(self, test, device, batch_size=None, extra_args=[]): super().__init__( diff --git a/torchbenchmark/models/detectron2_maskrcnn/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn/metadata.yaml index fab3cd3390..9c80a54250 100644 --- a/torchbenchmark/models/detectron2_maskrcnn/metadata.yaml +++ b/torchbenchmark/models/detectron2_maskrcnn/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/detectron2_maskrcnn_r_101_c4/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn_r_101_c4/metadata.yaml index 20a7683ad1..901ee7a33a 100644 --- a/torchbenchmark/models/detectron2_maskrcnn_r_101_c4/metadata.yaml +++ b/torchbenchmark/models/detectron2_maskrcnn_r_101_c4/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/detectron2_maskrcnn_r_101_fpn/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn_r_101_fpn/metadata.yaml index fab3cd3390..9c80a54250 100644 --- a/torchbenchmark/models/detectron2_maskrcnn_r_101_fpn/metadata.yaml +++ b/torchbenchmark/models/detectron2_maskrcnn_r_101_fpn/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/detectron2_maskrcnn_r_50_c4/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn_r_50_c4/metadata.yaml index 7f98221a8e..2479d75cd3 100644 --- a/torchbenchmark/models/detectron2_maskrcnn_r_50_c4/metadata.yaml +++ b/torchbenchmark/models/detectron2_maskrcnn_r_50_c4/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/detectron2_maskrcnn_r_50_fpn/metadata.yaml b/torchbenchmark/models/detectron2_maskrcnn_r_50_fpn/metadata.yaml index fab3cd3390..9c80a54250 100644 --- a/torchbenchmark/models/detectron2_maskrcnn_r_50_fpn/metadata.yaml +++ b/torchbenchmark/models/detectron2_maskrcnn_r_50_fpn/metadata.yaml @@ -6,6 +6,5 @@ eval_deterministic: false eval_nograd: true not_implemented: - device: cpu -- test: example train_benchmark: false train_deterministic: false diff --git a/torchbenchmark/models/maml/__init__.py b/torchbenchmark/models/maml/__init__.py index c8250c5739..ae005c8ae9 100644 --- a/torchbenchmark/models/maml/__init__.py +++ b/torchbenchmark/models/maml/__init__.py @@ -16,9 +16,6 @@ class Model(BenchmarkModel): DEFAULT_EVAL_BSIZE = 1 ALLOW_CUSTOMIZE_BSIZE = False CANNOT_SET_CUSTOM_OPTIMIZER = True - # Skip correctness check, because maml runs backward and optimizer in eval() - # Which will return non-deterministic results - SKIP_CORRECTNESS_CHECK = True def __init__(self, test, device, batch_size=None, extra_args=[]): super().__init__( diff --git a/torchbenchmark/util/framework/detectron2/model_factory.py b/torchbenchmark/util/framework/detectron2/model_factory.py index 0eece4bb53..4c41e8f35b 100644 --- a/torchbenchmark/util/framework/detectron2/model_factory.py +++ b/torchbenchmark/util/framework/detectron2/model_factory.py @@ -90,9 +90,7 @@ class Detectron2Model(BenchmarkModel): # Default batch sizes DEFAULT_TRAIN_BSIZE = 1 DEFAULT_EVAL_BSIZE = 1 - # Skip correctness check, because the output tensor can't be verified using - # cosine similarity or torch.close() - SKIP_CORRECTNESS_CHECK = True + DISABLE_DETERMINISM = True def __init__(self, variant, test, device, batch_size=None, extra_args=[]): super().__init__(