Merge branch 'master' of github.com:cms-ml/MLProf

.flake8

@@ -4,7 +4,7 @@
 max-line-length = 120
 
 # codes of errors to ignore
-ignore = E128, E306, E402, E722, E731, W504
+ignore = E128, E306, E402, E702, E722, E731, W504
 
 # enforce double quotes
 inline-quotes = double

.github/workflows/lint_and_test.yml

@@ -0,0 +1,48 @@
+name: Lint and test
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ⬇️
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Setup python 🐍
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies ☕️
+        run: |
+          pip install -U pip setuptools wheel
+          pip install -r sandboxes/dev.txt
+
+      - name: Lint 🔍
+        run: flake8 mlprof
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ⬇️
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Setup python 🐍
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies ☕️
+        run: |
+          pip install -U pip setuptools wheel
+          pip install -r sandboxes/dev.txt
+
+      - name: Typecheck 🥊
+        run: mypy mlprof

.gitignore

@@ -14,6 +14,7 @@
 *.hdf5
 *.out
 *.parquet
+*.vscode
 .coverage
 coverage*.xml
 __pycache__
@@ -26,3 +27,4 @@ software
 data
 .data
 .law
+.python-version

README.md

@@ -1,6 +1,6 @@
 # MLProf
 
-[![Lint and test](https://github.com/uhh-cms/MLProf/actions/workflows/lint.yml/badge.svg)](https://github.com/uhh-cms/MLProf/actions/workflows/lint.yml)
+[![Lint and test](https://github.com/cms-ml/MLProf/actions/workflows/lint_and_test.yml/badge.svg)](https://github.com/cms-ml/MLProf/actions/workflows/lint_and_test.yml)
 
 Tools for automated ML model performance tests in CMSSW (CMSSW version 13 and above).
 

cmssw/MLProf/RuntimeMeasurement/plugins/ONNXInference.cc

@@ -149,7 +149,10 @@ ONNXInference::ONNXInference(const edm::ParameterSet& iConfig, const ONNXRuntime
   // initialize the input data arrays
   // note there is only one element in the FloatArrays type (i.e. vector<vector<float>>) variable
   for (int i = 0; i < nInputs_; i++) {
-    inputArrays_.emplace_back(batchSize_ * flatInputSizes_[i], 0);
+    // multiply the size of all dimensions in an input
+    int full_size_input = std::accumulate(begin(input_shapes_[i]), end(input_shapes_[i]), 1, std::multiplies<int>());
+    // initialize inputArrays_ with 0s at first
+    inputArrays_.emplace_back(full_size_input, 0);
   }
 }
 

cmssw/install_sandbox_tfaot.sh

@@ -18,9 +18,6 @@ action() {
     # define additional model variables
     local tool_name="tfaot-model-mlprof-test"
 
-    # temporarily prepend the local cms_tfaot path
-    export PYTHONPATH="${MLP_BASE}/modules/cms-tfaot:${PYTHONPATH}"
-
     # remove existing code
     rm -rf MLProf
 

examples/cnn/create.py

@@ -0,0 +1,106 @@
+# coding: utf-8
+
+"""
+Test script to create a simple CNN model.
+Best to be executed in a CMSSW environment with TensorFlow and cmsml installed or a dedicated virtual environment.
+
+Signature: f32(1024 * n_channels) -> f32(8)
+
+1024 could correspond to a pixelated 32x32 "image" of an eta-phi plane.
+"""
+
+import os
+import subprocess
+
+import cmsml  # type: ignore[import-untyped]
+
+
+def create_model(
+    model_dir: str,
+    postfix: str = r"i{n_in}c{n_channels}v{n_convs}l{n_layers}u{n_units}",
+    n_in: int = 32,
+    n_channels: int = 1,
+    n_filters: int = 32,
+    n_convs: int = 3,
+    n_out: int = 8,
+    n_layers: int = 10,
+    n_units: int = 128,
+) -> None:
+    # get tensorflow
+    tf, _, tf_version = cmsml.tensorflow.import_tf()
+    print("creating simple model")
+    print(f"location  : {model_dir}")
+    print(f"TF version: {'.'.join(map(str, tf_version))}")
+
+    # set random seeds to get deterministic results for testing
+    tf.keras.utils.set_random_seed(1)
+
+    # define input layer
+    x = tf.keras.Input(shape=(n_channels * n_in**2,), dtype=tf.float32, name="input")
+
+    # reshape
+    a = tf.keras.layers.Reshape((n_in, n_in, n_channels))(x)
+
+    # convolutions and pooling
+    for _ in range(n_convs):
+        a = tf.keras.layers.Conv2D(n_filters, (3, 3), activation="elu")(a)
+        a = tf.keras.layers.MaxPooling2D((2, 2))(a)
+
+    # flatten
+    a = tf.keras.layers.Flatten()(a)
+
+    # model layers
+    for _ in range(n_layers):
+        a = tf.keras.layers.Dense(n_units, activation="elu")(a)
+
+    # output layer
+    y = tf.keras.layers.Dense(n_out, activation="softmax", name="output", dtype=tf.float32)(a)
+
+    # define the model
+    model = tf.keras.Model(inputs=[x], outputs=[y])
+
+    # test evaluation
+    print(model([tf.constant([list(range(n_channels * n_in**2))], dtype=tf.float32)]))
+
+    # save it as a frozen graph
+    _postfix = postfix.format(
+        n_in=n_in,
+        n_out=n_out,
+        n_channels=n_channels,
+        n_filters=n_filters,
+        n_convs=n_convs,
+        n_layers=n_layers,
+        n_units=n_units,
+    )
+    cmsml.tensorflow.save_graph(
+        os.path.join(model_dir, f"frozen_graph_{_postfix}.pb"),
+        model,
+        variables_to_constants=True,
+    )
+
+    # create a SavedModel
+    saved_model_dir = os.path.join(model_dir, f"saved_model_{_postfix}")
+    tf.saved_model.save(model, saved_model_dir)
+
+    # convert to onnx
+    cmd = [
+        "python",
+        "-m", "tf2onnx.convert",
+        "--saved-model", saved_model_dir,
+        "--output", os.path.join(model_dir, f"onnx_graph_{_postfix}.onnx"),
+    ]
+    subprocess.run(cmd, check=True)
+
+
+def main():
+    this_dir = os.path.dirname(os.path.abspath(__file__))
+
+    create_model(this_dir, n_in=32, n_channels=1, n_convs=1, n_layers=10)
+    create_model(this_dir, n_in=32, n_channels=1, n_convs=3, n_layers=10)
+    create_model(this_dir, n_in=64, n_channels=3, n_convs=4, n_layers=10)
+    create_model(this_dir, n_in=32, n_channels=1, n_convs=3, n_layers=20)
+    create_model(this_dir, n_in=64, n_channels=3, n_convs=4, n_layers=20)
+
+
+if __name__ == "__main__":
+    main()

examples/cnn/model_onnx_i32c1v1l10u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_onnx
+  label: ONNX 32x1$\circledast$1$\rightarrow$10x128
+  version: "1.0.0"
+  inference_engine: onnx
+  file: ./onnx_graph_i32c1v1l10u128.onnx
+  inputs:
+    - name: input
+      shape: [1024]
+  outputs:
+    - name: output

examples/cnn/model_onnx_i32c1v3l10u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_onnx
+  label: ONNX 32x1$\circledast$3$\rightarrow$10x128
+  version: "1.0.0"
+  inference_engine: onnx
+  file: ./onnx_graph_i32c1v3l10u128.onnx
+  inputs:
+    - name: input
+      shape: [1024]
+  outputs:
+    - name: output

examples/cnn/model_onnx_i32c1v3l20u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_onnx
+  label: ONNX 32x1$\circledast$3$\rightarrow$20x128
+  version: "1.0.0"
+  inference_engine: onnx
+  file: ./onnx_graph_i32c1v3l20u128.onnx
+  inputs:
+    - name: input
+      shape: [1024]
+  outputs:
+    - name: output

examples/cnn/model_onnx_i64c3v4l10u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_onnx
+  label: ONNX 64x3$\circledast$4$\rightarrow$10x128
+  version: "1.0.0"
+  inference_engine: onnx
+  file: ./onnx_graph_i64c3v4l10u128.onnx
+  inputs:
+    - name: input
+      shape: [12288]
+  outputs:
+    - name: output

examples/cnn/model_onnx_i64c3v4l20u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_onnx
+  label: ONNX 64x3$\circledast$4$\rightarrow$20x128
+  version: "1.0.0"
+  inference_engine: onnx
+  file: ./onnx_graph_i64c3v4l20u128.onnx
+  inputs:
+    - name: input
+      shape: [12288]
+  outputs:
+    - name: output

examples/cnn/model_tf_i32c1v1l10u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_tf
+  label: TF 32x1$\circledast$1$\rightarrow$10x128
+  version: "1.0.0"
+  inference_engine: tf
+  file: ./frozen_graph_i32c1v1l10u128.pb
+  inputs:
+    - name: input
+      shape: [1024]
+  outputs:
+    - name: Identity

examples/cnn/model_tf_i32c1v3l10u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_tf
+  label: TF 32x1$\circledast$3$\rightarrow$10x128
+  version: "1.0.0"
+  inference_engine: tf
+  file: ./frozen_graph_i32c1v3l10u128.pb
+  inputs:
+    - name: input
+      shape: [1024]
+  outputs:
+    - name: Identity

examples/cnn/model_tf_i32c1v3l20u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_tf
+  label: TF 32x1$\circledast$3$\rightarrow$20x128
+  version: "1.0.0"
+  inference_engine: tf
+  file: ./frozen_graph_i32c1v3l20u128.pb
+  inputs:
+    - name: input
+      shape: [1024]
+  outputs:
+    - name: Identity

examples/cnn/model_tf_i64c3v4l10u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_tf
+  label: TF 64x3$\circledast$4$\rightarrow$10x128
+  version: "1.0.0"
+  inference_engine: tf
+  file: ./frozen_graph_i64c3v4l10u128.pb
+  inputs:
+    - name: input
+      shape: [12288]
+  outputs:
+    - name: Identity

examples/cnn/model_tf_i64c3v4l20u128.yaml

@@ -0,0 +1,11 @@
+model:
+  name: simple_cnn_tf
+  label: TF 64x3$\circledast$4$\rightarrow$20x128
+  version: "1.0.0"
+  inference_engine: tf
+  file: ./frozen_graph_i64c3v4l20u128.pb
+  inputs:
+    - name: input
+      shape: [12288]
+  outputs:
+    - name: Identity