From caad40eeb1d0dc58d2cd955385ac1bc89965e366 Mon Sep 17 00:00:00 2001
From: Khalique <15948690+kahmed10@users.noreply.github.com>
Date: Tue, 9 Jun 2020 02:56:12 -0500
Subject: [PATCH 1/8] add tf backend

---
 v0.7/medical_imaging/3d-unet/Dockerfile |  2 +-
 v0.7/medical_imaging/3d-unet/Makefile   |  8 +++
 v0.7/medical_imaging/3d-unet/README.md  |  5 +-
 v0.7/medical_imaging/3d-unet/run.py     |  9 ++-
 v0.7/medical_imaging/3d-unet/tf_SUT.py  | 77 +++++++++++++++++++++++++
 5 files changed, 95 insertions(+), 6 deletions(-)
 create mode 100644 v0.7/medical_imaging/3d-unet/tf_SUT.py

diff --git a/v0.7/medical_imaging/3d-unet/Dockerfile b/v0.7/medical_imaging/3d-unet/Dockerfile
index ea08c8056e..0c1c8f80ca 100644
--- a/v0.7/medical_imaging/3d-unet/Dockerfile
+++ b/v0.7/medical_imaging/3d-unet/Dockerfile
@@ -27,7 +27,7 @@ RUN cd /tmp \
  && rm -rf inference
 
 # Install dependencies
-RUN python3 -m pip install onnx onnxruntime numpy==1.18.0 Pillow==7.0.0
+RUN python3 -m pip install onnx onnxruntime numpy==1.18.0 Pillow==7.0.0 tensorflow
 
 # Install nnUnet
 COPY nnUnet /workspace/nnUnet
diff --git a/v0.7/medical_imaging/3d-unet/Makefile b/v0.7/medical_imaging/3d-unet/Makefile
index 119dc2859a..3ac6518ec9 100644
--- a/v0.7/medical_imaging/3d-unet/Makefile
+++ b/v0.7/medical_imaging/3d-unet/Makefile
@@ -35,6 +35,7 @@ RESULT_DIR := $(BUILD_DIR)/result
 MLPERF_CONF := $(BUILD_DIR)/mlperf.conf
 PYTORCH_MODEL := $(RESULT_DIR)/fold_4.zip
 ONNX_MODEL := $(MODEL_DIR)/192_224_192.onnx
+TF_MODEL := $(MODEL_DIR)/192_224_192.pb
 
 # Env variables needed by nnUnet
 export nnUNet_raw_data_base=$(RAW_DATA_DIR)
@@ -106,6 +107,13 @@ download_onnx_model:
 		echo "For now, please manually download ONNX model to $(ONNX_MODEL)"; \
 	fi
 
+.PHONY: download_tf_model
+download_tf_model:
+	# Will download model from Zenodo
+	@if [ ! -e $(TF_MODEL) ]; then \
+		echo "For now, please manually download ONNX model to $(TF_MODEL)"; \
+	fi
+
 .PHONY: build_docker
 build_docker:
 	@echo "Building docker image..."
diff --git a/v0.7/medical_imaging/3d-unet/README.md b/v0.7/medical_imaging/3d-unet/README.md
index 9251f29c52..c016d89650 100644
--- a/v0.7/medical_imaging/3d-unet/README.md
+++ b/v0.7/medical_imaging/3d-unet/README.md
@@ -36,15 +36,16 @@ Please run the following commands:
 - `export DOWNLOAD_DATA_DIR=<path/to/MICCAI_BraTS_2019_Data_Training>`: point to location of downloaded BraTS 2019 Training dataset.
 - **Temporary:** Download the (192, 224, 192) PyTorch model named `fold_4.zip` to `build/result/`.
 - **Temporary:** Download the (192, 224, 192) ONNX model named `192_224_192.onnx` to `build/`.
+- **Temporary:** Download the (192, 224, 192) TF model named `192_224_192.pb` to `build/`.
 - `make setup`: initialize submodule and download models.
 - `make build_docker`: build docker image.
 - `make launch_docker`: launch docker container with an interaction session.
 - `make preprocess_data`: preprocess the BraTS 2019 dataset.
-- `python3 run.py --backend=[pytorch|onnxruntime] --scenario=[Offline|SingleStream|MultiStream|Server] [--accuracy]`: run the harness inside the docker container. Performance or Accuracy results will be printed in console.
+- `python3 run.py --backend=[tf|pytorch|onnxruntime] --scenario=[Offline|SingleStream|MultiStream|Server] [--accuracy]`: run the harness inside the docker container. Performance or Accuracy results will be printed in console.
 
 ## Details
 
-- SUT implementations are in [pytorch_SUT.py](pytorch_SUT.py) and [onnxruntime_SUT.py](onnxruntime_SUT.py). QSL implementation is in [brats_QSL.py](brats_QSL.py).
+- SUT implementations are in [pytorch_SUT.py](pytorch_SUT.py), [onnxruntime_SUT.py](onnxruntime_SUT.py) and [tf_SUT.py](tf_SUT.py). QSL implementation is in [brats_QSL.py](brats_QSL.py).
 - The script [brats_eval.py](brats_eval.py) parses LoadGen accuracy log, post-processes it, and computes the accuracy.
 - Preprocessing and evaluation (including post-processing) are not included in the timed path.
 - The input to the SUT is a volume of size `[4, 192, 224, 192]`. The output from SUT is a volume of size `[4, 192, 224, 192]` with predicted label logits for each voxel.
diff --git a/v0.7/medical_imaging/3d-unet/run.py b/v0.7/medical_imaging/3d-unet/run.py
index a21623b193..67828b62aa 100644
--- a/v0.7/medical_imaging/3d-unet/run.py
+++ b/v0.7/medical_imaging/3d-unet/run.py
@@ -24,14 +24,14 @@
 
 def get_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--backend", choices=["pytorch","onnxruntime"], default="pytorch", help="Backend")
+    parser.add_argument("--backend", choices=["pytorch","onnxruntime","tf"], default="pytorch", help="Backend")
     parser.add_argument("--scenario", choices=["SingleStream", "Offline", "Server", "MultiStream"], default="Offline", help="Scenario")
     parser.add_argument("--accuracy", action="store_true", help="enable accuracy pass")
     parser.add_argument("--mlperf_conf", default="build/mlperf.conf", help="mlperf rules config")
     parser.add_argument("--user_conf", default="user.conf", help="mlperf rules config")
     parser.add_argument("--model_dir", default="build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1",
         help="Path to the directory containing plans.pkl")
-    parser.add_argument("--onnx_model", default="build/model/192_224_192.onnx", help="Path to the ONNX model")
+    parser.add_argument("--model", help="Path to the ONNX or TF model")
     parser.add_argument("--preprocessed_data_dir", default="build/preprocessed_data", help="path to preprocessed data")
     parser.add_argument("--performance_count", type=int, default=16, help="performance count")
     args = parser.parse_args()
@@ -52,7 +52,10 @@ def main():
         sut = get_pytorch_sut(args.model_dir, args.preprocessed_data_dir, args.performance_count)
     elif args.backend == "onnxruntime":
         from onnxruntime_SUT import get_onnxruntime_sut
-        sut = get_onnxruntime_sut(args.onnx_model, args.preprocessed_data_dir, args.performance_count)
+        sut = get_onnxruntime_sut(args.model, args.preprocessed_data_dir, args.performance_count)
+    elif args.backend == "tf"
+        from tf_SUT import get_tf_sut
+        sut = get_tf_sut(args.model, args.preprocessed_data_dir, args.performance_count)
     else:
         raise ValueError("Unknown backend: {:}".format(args.backend))
 
diff --git a/v0.7/medical_imaging/3d-unet/tf_SUT.py b/v0.7/medical_imaging/3d-unet/tf_SUT.py
new file mode 100644
index 0000000000..b9da38bc86
--- /dev/null
+++ b/v0.7/medical_imaging/3d-unet/tf_SUT.py
@@ -0,0 +1,77 @@
+# coding=utf-8
+# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import array
+import json
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+
+import mlperf_loadgen as lg
+import numpy as np
+import tensorflow as tf
+from tensorflow.core.framework import graph_pb2
+
+from brats_QSL import get_brats_QSL
+
+
+class _3DUNET_TF_SUT():
+    def __init__(self, model_path, preprocessed_data_dir, performance_count):
+        print("Loading TF model...")
+        graph_def = graph_pb2.GraphDef()
+        with open(model_path, "rb") as f:
+            graph_def.ParseFromString(f.read())
+        g = tf.compat.v1.import_graph_def(graph_def, name='')
+        self.sess = tf.compat.v1.Session(graph=g)
+        self.input = g.get_tensor_by_name("import/input:0")
+        self.output = g.get_tensor_by_name("import/output:0")
+
+        print("Constructing SUT...")
+        self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries,
+                                   self.process_latencies)
+        self.qsl = get_brats_QSL(preprocessed_data_dir, performance_count)
+        print("Finished constructing SUT.")
+
+    def issue_queries(self, query_samples):
+        for i in range(len(query_samples)):
+            data = self.qsl.get_features(query_samples[i].index)
+
+            print("Processing sample id {:d} with shape = {:}".format(
+                query_samples[i].index, data.shape))
+
+            before_softmax = self.sess.run(
+                self.output, feed_dict={self.input: data[np.newaxis, ...]})[0]
+            softmax = np.softmax(before_softmax, axis=0).astype(np.float16)
+
+            response_array = array.array("B", softmax.tobytes())
+            bi = response_array.buffer_info()
+            response = lg.QuerySampleResponse(query_samples[i].id, bi[0],
+                                              bi[1])
+            lg.QuerySamplesComplete([response])
+
+    def flush_queries(self):
+        pass
+
+    def process_latencies(self, latencies_ns):
+        pass
+
+    def __del__(self):
+        lg.DestroySUT(self.sut)
+        print("Finished destroying SUT.")
+
+
+def get_tf_sut(model_path, preprocessed_data_dir, performance_count):
+    return _3DUNET_TF_SUT(model_path, preprocessed_data_dir, performance_count)

From ce7480bc9b333999b9ec1fec226a91815469a017 Mon Sep 17 00:00:00 2001
From: Khalique <15948690+kahmed10@users.noreply.github.com>
Date: Mon, 15 Jun 2020 20:22:44 -0500
Subject: [PATCH 2/8] fix bugs

---
 v0.7/medical_imaging/3d-unet/Dockerfile |  1 +
 v0.7/medical_imaging/3d-unet/README.md  |  1 +
 v0.7/medical_imaging/3d-unet/run.py     | 52 +++++++++++++++++++------
 v0.7/medical_imaging/3d-unet/tf_SUT.py  |  3 +-
 4 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/v0.7/medical_imaging/3d-unet/Dockerfile b/v0.7/medical_imaging/3d-unet/Dockerfile
index 0c1c8f80ca..a8d23f3cbe 100644
--- a/v0.7/medical_imaging/3d-unet/Dockerfile
+++ b/v0.7/medical_imaging/3d-unet/Dockerfile
@@ -27,6 +27,7 @@ RUN cd /tmp \
  && rm -rf inference
 
 # Install dependencies
+RUN python3 -m pip install wrapt --upgrade --ignore-installed
 RUN python3 -m pip install onnx onnxruntime numpy==1.18.0 Pillow==7.0.0 tensorflow
 
 # Install nnUnet
diff --git a/v0.7/medical_imaging/3d-unet/README.md b/v0.7/medical_imaging/3d-unet/README.md
index c016d89650..89f225f6ed 100644
--- a/v0.7/medical_imaging/3d-unet/README.md
+++ b/v0.7/medical_imaging/3d-unet/README.md
@@ -15,6 +15,7 @@ The chosen model is 3D-Unet in [nnUnet](https://github.com/MIC-DKFZ/nnUNet) perf
 | ----- | --------- | -------- | ------- | ---------- | ------------ | --------- | ----- |
 | 3D-Unet | PyTorch | mean = 0.82400, whole tumor = 0.8922, tumor core = 0.8158, enhancing tumor = 0.7640 | last 20% of BraTS 2019 Training Dataset (67 samples) | [from zenodo](???) | Trained in PyTorch using codes from [nnUnet](https://github.com/MIC-DKFZ/nnUNet) on the first 80% of BraTS 2019 Training Dataset. | fp32 | |
 | 3D-Unet | ONNX | mean = 0.82400, whole tumor = 0.8922, tumor core = 0.8158, enhancing tumor = 0.7640 | last 20% of BraTS 2019 Training Dataset (67 samples) | [from zenodo](???) | Converted from the PyTorch model using ??? script. | fp32 | |
+| 3D-Unet | Tensorflow | mean = 0.82400, whole tumor = 0.8922, tumor core = 0.8158, enhancing tumor = 0.7640 | last 20% of BraTS 2019 Training Dataset (67 samples) | [from zenodo](???) | Converted from the PyTorch model using ??? script. | fp32 | |
 
 ## Disclaimer
 This benchmark app is a reference implementation that is not meant to be the fastest implementation possible.
diff --git a/v0.7/medical_imaging/3d-unet/run.py b/v0.7/medical_imaging/3d-unet/run.py
index 67828b62aa..4a19301dea 100644
--- a/v0.7/medical_imaging/3d-unet/run.py
+++ b/v0.7/medical_imaging/3d-unet/run.py
@@ -22,21 +22,44 @@
 import mlperf_loadgen as lg
 import subprocess
 
+
 def get_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--backend", choices=["pytorch","onnxruntime","tf"], default="pytorch", help="Backend")
-    parser.add_argument("--scenario", choices=["SingleStream", "Offline", "Server", "MultiStream"], default="Offline", help="Scenario")
-    parser.add_argument("--accuracy", action="store_true", help="enable accuracy pass")
-    parser.add_argument("--mlperf_conf", default="build/mlperf.conf", help="mlperf rules config")
-    parser.add_argument("--user_conf", default="user.conf", help="mlperf rules config")
-    parser.add_argument("--model_dir", default="build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1",
+    parser.add_argument("--backend",
+                        choices=["pytorch", "onnxruntime", "tf"],
+                        default="pytorch",
+                        help="Backend")
+    parser.add_argument(
+        "--scenario",
+        choices=["SingleStream", "Offline", "Server", "MultiStream"],
+        default="Offline",
+        help="Scenario")
+    parser.add_argument("--accuracy",
+                        action="store_true",
+                        help="enable accuracy pass")
+    parser.add_argument("--mlperf_conf",
+                        default="build/mlperf.conf",
+                        help="mlperf rules config")
+    parser.add_argument("--user_conf",
+                        default="user.conf",
+                        help="mlperf rules config")
+    parser.add_argument(
+        "--model_dir",
+        default=
+        "build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1",
         help="Path to the directory containing plans.pkl")
     parser.add_argument("--model", help="Path to the ONNX or TF model")
-    parser.add_argument("--preprocessed_data_dir", default="build/preprocessed_data", help="path to preprocessed data")
-    parser.add_argument("--performance_count", type=int, default=16, help="performance count")
+    parser.add_argument("--preprocessed_data_dir",
+                        default="build/preprocessed_data",
+                        help="path to preprocessed data")
+    parser.add_argument("--performance_count",
+                        type=int,
+                        default=16,
+                        help="performance count")
     args = parser.parse_args()
     return args
 
+
 scenario_map = {
     "SingleStream": lg.TestScenario.SingleStream,
     "Offline": lg.TestScenario.Offline,
@@ -44,18 +67,22 @@ def get_args():
     "MultiStream": lg.TestScenario.MultiStream
 }
 
+
 def main():
     args = get_args()
 
     if args.backend == "pytorch":
         from pytorch_SUT import get_pytorch_sut
-        sut = get_pytorch_sut(args.model_dir, args.preprocessed_data_dir, args.performance_count)
+        sut = get_pytorch_sut(args.model_dir, args.preprocessed_data_dir,
+                              args.performance_count)
     elif args.backend == "onnxruntime":
         from onnxruntime_SUT import get_onnxruntime_sut
-        sut = get_onnxruntime_sut(args.model, args.preprocessed_data_dir, args.performance_count)
-    elif args.backend == "tf"
+        sut = get_onnxruntime_sut(args.model, args.preprocessed_data_dir,
+                                  args.performance_count)
+    elif args.backend == "tf":
         from tf_SUT import get_tf_sut
-        sut = get_tf_sut(args.model, args.preprocessed_data_dir, args.performance_count)
+        sut = get_tf_sut(args.model, args.preprocessed_data_dir,
+                         args.performance_count)
     else:
         raise ValueError("Unknown backend: {:}".format(args.backend))
 
@@ -94,5 +121,6 @@ def main():
     print("Destroying QSL...")
     lg.DestroyQSL(sut.qsl.qsl)
 
+
 if __name__ == "__main__":
     main()
diff --git a/v0.7/medical_imaging/3d-unet/tf_SUT.py b/v0.7/medical_imaging/3d-unet/tf_SUT.py
index b9da38bc86..b112c9652a 100644
--- a/v0.7/medical_imaging/3d-unet/tf_SUT.py
+++ b/v0.7/medical_imaging/3d-unet/tf_SUT.py
@@ -34,7 +34,8 @@ def __init__(self, model_path, preprocessed_data_dir, performance_count):
         graph_def = graph_pb2.GraphDef()
         with open(model_path, "rb") as f:
             graph_def.ParseFromString(f.read())
-        g = tf.compat.v1.import_graph_def(graph_def, name='')
+        with tf.Graph().as_default() as g:
+            g = tf.compat.v1.import_graph_def(graph_def)
         self.sess = tf.compat.v1.Session(graph=g)
         self.input = g.get_tensor_by_name("import/input:0")
         self.output = g.get_tensor_by_name("import/output:0")

From 5db2fbe473482b6a2748929d614dbaff43993f00 Mon Sep 17 00:00:00 2001
From: Khalique <15948690+kahmed10@users.noreply.github.com>
Date: Wed, 17 Jun 2020 00:13:11 -0500
Subject: [PATCH 3/8] fix bugs and test w 1 sample

---
 v0.7/medical_imaging/3d-unet/tf_SUT.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/v0.7/medical_imaging/3d-unet/tf_SUT.py b/v0.7/medical_imaging/3d-unet/tf_SUT.py
index b112c9652a..ced51aacb1 100644
--- a/v0.7/medical_imaging/3d-unet/tf_SUT.py
+++ b/v0.7/medical_imaging/3d-unet/tf_SUT.py
@@ -35,7 +35,7 @@ def __init__(self, model_path, preprocessed_data_dir, performance_count):
         with open(model_path, "rb") as f:
             graph_def.ParseFromString(f.read())
         with tf.Graph().as_default() as g:
-            g = tf.compat.v1.import_graph_def(graph_def)
+            tf.compat.v1.import_graph_def(graph_def)
         self.sess = tf.compat.v1.Session(graph=g)
         self.input = g.get_tensor_by_name("import/input:0")
         self.output = g.get_tensor_by_name("import/output:0")
@@ -55,7 +55,8 @@ def issue_queries(self, query_samples):
 
             before_softmax = self.sess.run(
                 self.output, feed_dict={self.input: data[np.newaxis, ...]})[0]
-            softmax = np.softmax(before_softmax, axis=0).astype(np.float16)
+            softmax = tf.nn.softmax(before_softmax,
+                                    axis=0).numpy().astype(np.float16)
 
             response_array = array.array("B", softmax.tobytes())
             bi = response_array.buffer_info()

From c280db635ef95b07399028a42c38fff7f520518c Mon Sep 17 00:00:00 2001
From: Khalique <15948690+kahmed10@users.noreply.github.com>
Date: Wed, 17 Jun 2020 00:21:22 -0500
Subject: [PATCH 4/8] add commands to makefile and update readme

---
 v0.7/medical_imaging/3d-unet/Makefile  | 13 +++++++++++--
 v0.7/medical_imaging/3d-unet/README.md |  2 +-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/v0.7/medical_imaging/3d-unet/Makefile b/v0.7/medical_imaging/3d-unet/Makefile
index 3ac6518ec9..8f44074713 100644
--- a/v0.7/medical_imaging/3d-unet/Makefile
+++ b/v0.7/medical_imaging/3d-unet/Makefile
@@ -87,6 +87,7 @@ download_model:
 	@echo "Download models..."
 	@$(MAKE) -f $(MAKEFILE_NAME) download_pytorch_model
 	@$(MAKE) -f $(MAKEFILE_NAME) download_onnx_model
+	@$(MAKE) -f $(MAKEFILE_NAME) download_tf_model
 
 .PHONY: download_pytorch_model
 download_pytorch_model:
@@ -162,11 +163,19 @@ run_pytorch_accuracy: mkdir_postprocessed_data
 
 .PHONY: run_onnxruntime_performance
 run_onnxruntime_performance:
-	@python3 run.py --backend=onnxruntime
+	@python3 run.py --backend=onnxruntime --model=build/model/192_224_192.onnx
 
 .PHONY: run_onnxruntime_accuracy
 run_onnxruntime_accuracy: mkdir_postprocessed_data
-	@python3 run.py --backend=onnxruntime --accuracy
+	@python3 run.py --backend=onnxruntime --model=build/model/192_224_192.onnx --accuracy
+
+.PHONY: run_tf_performance
+run_tf_performance:
+	@python3 run.py --backend=tf --model=build/model/192_224_192.pb
+
+.PHONY: run_tf_accuracy
+run_tf_accuracy: mkdir_postprocessed_data
+	@python3 run.py --backend=tf --model=build/model/192_224_192.pb --accuracy
 
 .PHONY: evaluate
 evaluate:
diff --git a/v0.7/medical_imaging/3d-unet/README.md b/v0.7/medical_imaging/3d-unet/README.md
index 89f225f6ed..4c3cb26c69 100644
--- a/v0.7/medical_imaging/3d-unet/README.md
+++ b/v0.7/medical_imaging/3d-unet/README.md
@@ -42,7 +42,7 @@ Please run the following commands:
 - `make build_docker`: build docker image.
 - `make launch_docker`: launch docker container with an interaction session.
 - `make preprocess_data`: preprocess the BraTS 2019 dataset.
-- `python3 run.py --backend=[tf|pytorch|onnxruntime] --scenario=[Offline|SingleStream|MultiStream|Server] [--accuracy]`: run the harness inside the docker container. Performance or Accuracy results will be printed in console.
+- `python3 run.py --backend=[tf|pytorch|onnxruntime] --scenario=[Offline|SingleStream|MultiStream|Server] [--accuracy] --model=[path/to/model_file(tf/onnx only)]`: run the harness inside the docker container. Performance or Accuracy results will be printed in console.
 
 ## Details
 

From 4ab7690e3ebbc1a1ba56bf492437f3de92119758 Mon Sep 17 00:00:00 2001
From: Khalique <15948690+kahmed10@users.noreply.github.com>
Date: Mon, 22 Jun 2020 20:49:36 -0500
Subject: [PATCH 5/8] add onnx to tf conversion script

---
 .../3d-unet/unet_onnx_to_tf.py                | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 v0.7/medical_imaging/3d-unet/unet_onnx_to_tf.py

diff --git a/v0.7/medical_imaging/3d-unet/unet_onnx_to_tf.py b/v0.7/medical_imaging/3d-unet/unet_onnx_to_tf.py
new file mode 100644
index 0000000000..cd606823cc
--- /dev/null
+++ b/v0.7/medical_imaging/3d-unet/unet_onnx_to_tf.py
@@ -0,0 +1,59 @@
+# coding=utf-8
+# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+
+import argparse
+import onnx
+import onnx_tf
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--onnx_model", help="Path to the ONNX model")
+    parser.add_argument("--output_name",
+                        default="192_224_192.pb",
+                        help="Name of output model")
+    parser.add_argument("--output_dir",
+                        default="build/model",
+                        help="Directory to save output model")
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    print("Loading ONNX model...")
+    onnx_model = onnx.load(args.onnx_model)
+
+    print("Converting ONNX model to TF...")
+    tf_model = onnx_tf.backend.prepare(onnx_model)
+
+    if not os.path.exists(args.output_dir):
+        os.makedirs(args.output_dir)
+
+    output_path = "./{}/{}".format(args.output_dir, args.output_name)
+
+    tf_model.export_graph(output_path)
+
+    print("Successfully exported model {}".format(output_path))
+
+
+if __name__ == "__main__":
+    main()

From 7d4a3574653cd676f690419616df5bc33189595d Mon Sep 17 00:00:00 2001
From: Po-Han Huang <pohanh@nvidia.com>
Date: Tue, 23 Jun 2020 00:20:26 -0700
Subject: [PATCH 6/8] 3D-Unet: Switch to 224x224x160 model and upload them to
 Zenodo

---
 v0.7/medical_imaging/3d-unet/Dockerfile       |   1 +
 v0.7/medical_imaging/3d-unet/Makefile         |  67 ++++++++-------
 v0.7/medical_imaging/3d-unet/README.md        |  24 +-----
 v0.7/medical_imaging/3d-unet/brats_eval.py    |   3 +-
 .../3d-unet/fold1_validation.npy              | Bin 0 -> 6828 bytes
 .../3d-unet/fold4_validation.npy              | Bin 6828 -> 0 bytes
 v0.7/medical_imaging/3d-unet/preprocess.py    |   8 +-
 v0.7/medical_imaging/3d-unet/pytorch_SUT.py   |   2 +-
 v0.7/medical_imaging/3d-unet/tf_SUT.py        |   4 -
 .../3d-unet/unet_onnx_to_tf.py                |   8 +-
 .../3d-unet/unet_pytorch_to_onnx.py           |  76 ++++++++++++++++++
 11 files changed, 131 insertions(+), 62 deletions(-)
 create mode 100644 v0.7/medical_imaging/3d-unet/fold1_validation.npy
 delete mode 100644 v0.7/medical_imaging/3d-unet/fold4_validation.npy
 create mode 100644 v0.7/medical_imaging/3d-unet/unet_pytorch_to_onnx.py

diff --git a/v0.7/medical_imaging/3d-unet/Dockerfile b/v0.7/medical_imaging/3d-unet/Dockerfile
index a8d23f3cbe..6e3d5e3012 100644
--- a/v0.7/medical_imaging/3d-unet/Dockerfile
+++ b/v0.7/medical_imaging/3d-unet/Dockerfile
@@ -29,6 +29,7 @@ RUN cd /tmp \
 # Install dependencies
 RUN python3 -m pip install wrapt --upgrade --ignore-installed
 RUN python3 -m pip install onnx onnxruntime numpy==1.18.0 Pillow==7.0.0 tensorflow
+RUN python3 -m pip install tensorflow-addons https://github.com/onnx/onnx-tensorflow/archive/master.zip
 
 # Install nnUnet
 COPY nnUnet /workspace/nnUnet
diff --git a/v0.7/medical_imaging/3d-unet/Makefile b/v0.7/medical_imaging/3d-unet/Makefile
index 8f44074713..39bc91d658 100644
--- a/v0.7/medical_imaging/3d-unet/Makefile
+++ b/v0.7/medical_imaging/3d-unet/Makefile
@@ -33,9 +33,10 @@ POSTPROCESSED_DATA_DIR := $(BUILD_DIR)/postprocessed_data
 MODEL_DIR := $(BUILD_DIR)/model
 RESULT_DIR := $(BUILD_DIR)/result
 MLPERF_CONF := $(BUILD_DIR)/mlperf.conf
-PYTORCH_MODEL := $(RESULT_DIR)/fold_4.zip
-ONNX_MODEL := $(MODEL_DIR)/192_224_192.onnx
-TF_MODEL := $(MODEL_DIR)/192_224_192.pb
+PYTORCH_MODEL := $(RESULT_DIR)/fold_1.zip
+ONNX_MODEL := $(MODEL_DIR)/224_224_160.onnx
+ONNX_DYNAMIC_BS_MODEL := $(MODEL_DIR)/224_224_160_dynamic_bs.onnx
+TF_MODEL := $(MODEL_DIR)/224_224_160.pb
 
 # Env variables needed by nnUnet
 export nnUNet_raw_data_base=$(RAW_DATA_DIR)
@@ -44,16 +45,15 @@ export RESULTS_FOLDER=$(RESULT_DIR)
 
 HAS_GPU := $(shell command -v nvidia-smi 2> /dev/null)
 
-ifndef $HAS_GPU
-    DOCKER_RUN_CMD := docker run
+ifeq ($(HAS_GPU),)
+	DOCKER_RUN_CMD := docker run
 else
-   # Handle different nvidia-docker version
-   ifneq ($(wildcard /usr/bin/nvidia-docker),)
-	DOCKER_RUN_CMD := nvidia-docker run
-   else
-	DOCKER_RUN_CMD := docker run --gpus=all
-   endif
-
+	# Handle different nvidia-docker version
+	ifneq ($(wildcard /usr/bin/nvidia-docker),)
+		DOCKER_RUN_CMD := nvidia-docker run
+	else
+		DOCKER_RUN_CMD := docker run --gpus=all
+	endif
 endif
 
 .PHONY: setup
@@ -91,28 +91,41 @@ download_model:
 
 .PHONY: download_pytorch_model
 download_pytorch_model:
-	# Will download model from Zenodo
-	# @if [ ! -e $(PYTORCH_MODEL)/model.pytorch ]; then \
-	# 	wget -O ; \
-	# fi
-	# For now, assume that fold_4.zip is in build/result
+	@echo "Downloading PyTorch model from Zenodo..."
 	@if [ ! -e $(PYTORCH_MODEL) ]; then \
-		echo "For now, please manually download PyTorch model to $(PYTORCH_MODEL)/"; \
+		wget -O $(PYTORCH_MODEL) https://zenodo.org/record/3904106/files/fold_1.zip?download=1 \
+		&& cd $(RESULT_DIR) && unzip -o fold_1.zip; \
 	fi
-	@cd $(RESULT_DIR) && unzip -o fold_4.zip
 
 .PHONY: download_onnx_model
 download_onnx_model:
-	# Will download model from Zenodo
+	@echo "Downloading ONNX model from Zenodo..."
 	@if [ ! -e $(ONNX_MODEL) ]; then \
-		echo "For now, please manually download ONNX model to $(ONNX_MODEL)"; \
+		wget -O $(ONNX_MODEL) https://zenodo.org/record/3904138/files/224_224_160.onnx?download=1; \
+	fi
+	@if [ ! -e $(ONNX_DYNAMIC_BS_MODEL) ]; then \
+		wget -O $(ONNX_DYNAMIC_BS_MODEL) https://zenodo.org/record/3904138/files/224_224_160_dyanmic_bs.onnx?download=1; \
 	fi
 
 .PHONY: download_tf_model
 download_tf_model:
-	# Will download model from Zenodo
+	@echo "Downloading TF model from Zenodo..."
+	@if [ ! -e $(TF_MODEL) ]; then \
+		wget -O $(TF_MODEL) https://zenodo.org/record/3904146/files/224_224_160.pb?download=1; \
+	fi
+
+.PHONY: convert_onnx_model
+convert_onnx_model: download_pytorch_model
+	@echo "Converting PyTorch model to ONNX model..."
+	@if [ ! -e $(ONNX_MODEL) ]; then \
+		python3 unet_pytorch_to_onnx.py; \
+	fi
+
+.PHONY: convert_tf_model
+convert_tf_model: convert_onnx_model
+	@echo "Converting ONNX model to TF model..."
 	@if [ ! -e $(TF_MODEL) ]; then \
-		echo "For now, please manually download ONNX model to $(TF_MODEL)"; \
+		python3 unet_onnx_to_tf.py; \
 	fi
 
 .PHONY: build_docker
@@ -163,19 +176,19 @@ run_pytorch_accuracy: mkdir_postprocessed_data
 
 .PHONY: run_onnxruntime_performance
 run_onnxruntime_performance:
-	@python3 run.py --backend=onnxruntime --model=build/model/192_224_192.onnx
+	@python3 run.py --backend=onnxruntime --model=build/model/224_224_160.onnx
 
 .PHONY: run_onnxruntime_accuracy
 run_onnxruntime_accuracy: mkdir_postprocessed_data
-	@python3 run.py --backend=onnxruntime --model=build/model/192_224_192.onnx --accuracy
+	@python3 run.py --backend=onnxruntime --model=build/model/224_224_160.onnx --accuracy
 
 .PHONY: run_tf_performance
 run_tf_performance:
-	@python3 run.py --backend=tf --model=build/model/192_224_192.pb
+	@python3 run.py --backend=tf --model=build/model/224_224_160.pb
 
 .PHONY: run_tf_accuracy
 run_tf_accuracy: mkdir_postprocessed_data
-	@python3 run.py --backend=tf --model=build/model/192_224_192.pb --accuracy
+	@python3 run.py --backend=tf --model=build/model/224_224_160.pb --accuracy
 
 .PHONY: evaluate
 evaluate:
diff --git a/v0.7/medical_imaging/3d-unet/README.md b/v0.7/medical_imaging/3d-unet/README.md
index 4c3cb26c69..c24c0a0a1b 100644
--- a/v0.7/medical_imaging/3d-unet/README.md
+++ b/v0.7/medical_imaging/3d-unet/README.md
@@ -1,7 +1,5 @@
 # MLPerf Inference Benchmarks for Medical Image 3D Segmentation
 
-This is the reference implementation for MLPerf Inference benchmarks for Medical Image 3D Segmentation.
-
 The chosen model is 3D-Unet in [nnUnet](https://github.com/MIC-DKFZ/nnUNet) performing [BraTS 2019](https://www.med.upenn.edu/cbica/brats2019/data.html) brain tumor segmentation task.
 
 ## Prerequisites
@@ -13,19 +11,15 @@ The chosen model is 3D-Unet in [nnUnet](https://github.com/MIC-DKFZ/nnUNet) perf
 
 | model | framework | accuracy | dataset | model link | model source | precision | notes |
 | ----- | --------- | -------- | ------- | ---------- | ------------ | --------- | ----- |
-| 3D-Unet | PyTorch | mean = 0.82400, whole tumor = 0.8922, tumor core = 0.8158, enhancing tumor = 0.7640 | last 20% of BraTS 2019 Training Dataset (67 samples) | [from zenodo](???) | Trained in PyTorch using codes from [nnUnet](https://github.com/MIC-DKFZ/nnUNet) on the first 80% of BraTS 2019 Training Dataset. | fp32 | |
-| 3D-Unet | ONNX | mean = 0.82400, whole tumor = 0.8922, tumor core = 0.8158, enhancing tumor = 0.7640 | last 20% of BraTS 2019 Training Dataset (67 samples) | [from zenodo](???) | Converted from the PyTorch model using ??? script. | fp32 | |
-| 3D-Unet | Tensorflow | mean = 0.82400, whole tumor = 0.8922, tumor core = 0.8158, enhancing tumor = 0.7640 | last 20% of BraTS 2019 Training Dataset (67 samples) | [from zenodo](???) | Converted from the PyTorch model using ??? script. | fp32 | |
+| 3D-Unet | PyTorch | mean = 0.85203, whole tumor = 0.9147, tumor core = 0.8645, enhancing tumor = 0.7769 | The second 20% of BraTS 2019 Training Dataset (fold 1, 67 samples) | [from zenodo](https://zenodo.org/record/3904106) | Trained in PyTorch using codes from [nnUnet](https://github.com/MIC-DKFZ/nnUNet) on the first 20% and the last 60% (fold 1) of BraTS 2019 Training Dataset. | fp32 | |
+| 3D-Unet | ONNX | mean = 0.85203, whole tumor = 0.9147, tumor core = 0.8645, enhancing tumor = 0.7769 | The second 20% of BraTS 2019 Training Dataset (fold 1, 67 samples) | [from zenodo](https://zenodo.org/record/3904138) | Converted from the PyTorch model using [script](unet_pytorch_to_onnx.py). | fp32 | |
+| 3D-Unet | Tensorflow | mean = 0.85203, whole tumor = 0.9147, tumor core = 0.8645, enhancing tumor = 0.7769 | The second 20% of BraTS 2019 Training Dataset (fold 1, 67 samples) | [from zenodo](https://zenodo.org/record/3904146) | Converted from the PyTorch model using [script](unet_onnx_to_tf.py). | fp32 | |
 
 ## Disclaimer
 This benchmark app is a reference implementation that is not meant to be the fastest implementation possible.
 
 ## TODO
 
-[ ] Update the models (PyTorch and ONNX) to the final volume size (160, 224, 224).
-[ ] Upload the models to Zenodo, and fill in Zenodo link, and modify Makefile so that it downloads models from Zenodo.
-[ ] Update the accuracy metric.
-[ ] Add PyTorch -> ONNX script.
 [ ] Update the onnxruntime in the docker container to a version which supports 3D ConvTranspose op.
 
 ## Commands
@@ -35,9 +29,6 @@ Please download [BraTS 2019](https://www.med.upenn.edu/cbica/brats2019/data.html
 Please run the following commands:
 
 - `export DOWNLOAD_DATA_DIR=<path/to/MICCAI_BraTS_2019_Data_Training>`: point to location of downloaded BraTS 2019 Training dataset.
-- **Temporary:** Download the (192, 224, 192) PyTorch model named `fold_4.zip` to `build/result/`.
-- **Temporary:** Download the (192, 224, 192) ONNX model named `192_224_192.onnx` to `build/`.
-- **Temporary:** Download the (192, 224, 192) TF model named `192_224_192.pb` to `build/`.
 - `make setup`: initialize submodule and download models.
 - `make build_docker`: build docker image.
 - `make launch_docker`: launch docker container with an interaction session.
@@ -46,11 +37,4 @@ Please run the following commands:
 
 ## Details
 
-- SUT implementations are in [pytorch_SUT.py](pytorch_SUT.py), [onnxruntime_SUT.py](onnxruntime_SUT.py) and [tf_SUT.py](tf_SUT.py). QSL implementation is in [brats_QSL.py](brats_QSL.py).
-- The script [brats_eval.py](brats_eval.py) parses LoadGen accuracy log, post-processes it, and computes the accuracy.
-- Preprocessing and evaluation (including post-processing) are not included in the timed path.
-- The input to the SUT is a volume of size `[4, 192, 224, 192]`. The output from SUT is a volume of size `[4, 192, 224, 192]` with predicted label logits for each voxel.
-
-## License
-
-Apache License 2.0
+- SUT implementations are in [pytorch_SUT.py](pytorch_SUT.py), [onnxruntime_
\ No newline at end of file
diff --git a/v0.7/medical_imaging/3d-unet/brats_eval.py b/v0.7/medical_imaging/3d-unet/brats_eval.py
index d70e565898..6fece895cc 100644
--- a/v0.7/medical_imaging/3d-unet/brats_eval.py
+++ b/v0.7/medical_imaging/3d-unet/brats_eval.py
@@ -84,8 +84,7 @@ def load_loadgen_log(log_file, result_dtype, dictionaries):
 
     assert len(predictions) == len(dictionaries), "Number of predictions does not match number of samples in validation set!"
 
-    # TODO: need to change to [160, 224, 224]
-    padded_shape = [192, 224, 192]
+    padded_shape = [224, 224, 160]
     results = [None for i in range(len(predictions))]
     for prediction in predictions:
         qsl_idx = prediction["qsl_idx"]
diff --git a/v0.7/medical_imaging/3d-unet/fold1_validation.npy b/v0.7/medical_imaging/3d-unet/fold1_validation.npy
new file mode 100644
index 0000000000000000000000000000000000000000..e157b024d0cb54e85b8dc4dcc818c26e4a1df803
GIT binary patch
literal 6828
zcmchb%}x|i6oi{CPcgfeWEP47YTPIRe?b(+zYXcY7#2iKk8xvs3SZdz63U#+LX+_p
zot*UT?#ZcJRp*|2;n(5p;OOI8_r3ekpU)R(WB+i{KbmcB_0viJ`D*+&4qr~M#`8HJ
z_J_;GT<;eb!`Hc<Z*A{PH>Q)Hla;UaXI<Aly^d4ei6@FtF(^(H$BH|O2MWG#DsC%y
zcUQr?<*{D<xmsd)4&c?R6$adP6z2557`sY<p3fDGA1lbsok07FwF-~FVZ2iQ!!QQ<
z--R(=D?i#KwK`J%mp6d%HjME``O|d;26`p^K&MY(40_xC4Pd1Gpiig$0OLbgtC{k<
zN4m$i?uqoO7hxV?q<i9fVWjH{zu7~Q>l_UFNeYAgmzqlMgJ2LzKb(J0I7zKoZ)rb#
z9!m1S?-qM4eUCF1bEb}b_VwI31JkSgyh^|N7sfs1ujkSP#{Fw<NgkZv(JzI;`JG7m
zf#1ZIBE<v0;gH~gKAtxmk{IZe^20prN1DgBxUfd7U8^-{9$w=K9%#h8!NT)IFENIB
zaBr<u&52G4t&HK?O?X~07+X8|w5lujCp^dRgy+V91NjqLvF=<)jE&h+t2%G?rz8e`
zfY)LlJWw9aCf0{}+l%VAUawqN6@$5>JN1eem4hCeV33=<1Xx}xjJf!Ggmbpf+HmJ=
zJSqmoiQ-tn@BRNm-YuWu{Ehtf#5%UX0Ke*kAGNMpW55B7x?Wmi;0?NQPici8=vLQc
zYaU=M`;))`$8`o?0)zau&RSzoPt;bjcEO;gYR_uLgSk_y+*5uSgZywwtU<3ia#KqQ
LKPm>}T35dTRJi#7

literal 0
HcmV?d00001

diff --git a/v0.7/medical_imaging/3d-unet/fold4_validation.npy b/v0.7/medical_imaging/3d-unet/fold4_validation.npy
deleted file mode 100644
index 96bd1c4956844a9d987738b63f7f86cfd8e24574..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 6828
zcmchb%Syvg5QgK{r^qe}U1+u5aN`B<t=EdWDM2GH#7Zh|#Ha9uoewf{=)y&qzk%VL
zq<FshXXf1e84UXu_jB22_EkJplgGN)%8Tt`w^wxX;(1(u)aBb~Tt8L(a8SNZD*HZp
zDc>u5Uszl3EOzqmy!A7Glx5jrV<T&^U9;M(G`lgoHd{8^G~@TK*^(LGR?PTDpKso`
z?%xy1ShrD?47BwE8GAMs@jN!8zhj2~wAe-SPW*V<#Ogl=?0d5`X3>m(<F}GAFa|MH
zGR|#$Ov$*k@i8^yY6gtk88FnfOD-P7I#@rMbIP2<V2-F6>bgS5a0WWmb;Z8P+N;b5
zjQtrf4#W)dt*)y(W1NU}z+f*_<{XB4p0ggR>k5W)O=RwIE>hMm48~WU;b3q+SDs5?
zkPBK>?K(qwW^jgbKZkK@7NLXnn;PX@%U!_lROEnrNcztGpv}4cKgMdqN$)#R4E*<6
z!uRyvrebU)Oa+7cuad!Cnw(=EyJ#IS$W5A`XvSKjK~Y!u@3A2tn7>Eqa0W3#i$Vu;
z&;6#a(BXcghw;(k%&1s70}XD`4ELLSu$Evd1{#=uihTImK|au+WVjBG9lW+V=S<5!
zMa+o}W0DK}PuD7Y4Q0SVhtEee@I>gqfA%ECriJ4*W!>z|EaLy7)A3<!f1((~CC#&s
zPaDM`UehshP87rEhv(VHr)|TyvA>%>U-_Iv3;ndVA{hAYIVWcHX}z`-3~~=w#UMuL
pNY8N5I$)501tX0=Iux<;IwBYJ)438AtCYdM>ve`66~mb@egTQD^@IQb

diff --git a/v0.7/medical_imaging/3d-unet/preprocess.py b/v0.7/medical_imaging/3d-unet/preprocess.py
index 551a74c853..0f2332511b 100644
--- a/v0.7/medical_imaging/3d-unet/preprocess.py
+++ b/v0.7/medical_imaging/3d-unet/preprocess.py
@@ -35,7 +35,7 @@ def get_args():
     parser.add_argument("--raw_data_dir", default="build/raw_data/nnUNet_raw_data/Task043_BraTS2019/imagesTr",
         help="Path to the directory containing raw nii.gz files")
     parser.add_argument("--preprocessed_data_dir", default="build/preprocessed_data", help="Path to the directory containing preprocessed data")
-    parser.add_argument("--validation_fold_file", default="fold4_validation.npy", help="Path to the npy file storing all the sample names for the validation fold")
+    parser.add_argument("--validation_fold_file", default="fold1_validation.npy", help="Path to the npy file storing all the sample names for the validation fold")
     parser.add_argument("--num_threads_preprocessing", type=int, default=12, help="Number of threads to run the preprocessing with")
     args = parser.parse_args()
     return args
@@ -75,8 +75,8 @@ def main():
 
     print("Preparing for preprocessing data...")
 
-    # Validation set is fold 4
-    fold = 4
+    # Validation set is fold 1
+    fold = 1
     validation_fold_file = args.validation_fold_file
 
     # Make sure the model exists
@@ -91,7 +91,7 @@ def main():
     raw_data_dir = args.raw_data_dir
     preprocessed_data_dir = args.preprocessed_data_dir
 
-    # Open npy containing validation images from specific fold (e.g. 4)
+    # Open npy containing validation images from specific fold (e.g. 1)
     with open(validation_fold_file, "rb") as f:
         validation_files = numpy.load(f)
 
diff --git a/v0.7/medical_imaging/3d-unet/pytorch_SUT.py b/v0.7/medical_imaging/3d-unet/pytorch_SUT.py
index e077ddafb5..9002791426 100644
--- a/v0.7/medical_imaging/3d-unet/pytorch_SUT.py
+++ b/v0.7/medical_imaging/3d-unet/pytorch_SUT.py
@@ -71,5 +71,5 @@ def flush_queries(self):
     def process_latencies(self, latencies_ns):
         pass
 
-def get_pytorch_sut(model_dir, preprocessed_data_dir, performance_count, folds=4, checkpoint_name="model_best"):
+def get_pytorch_sut(model_dir, preprocessed_data_dir, performance_count, folds=1, checkpoint_name="model_best"):
     return _3DUNET_PyTorch_SUT(model_dir, preprocessed_data_dir, performance_count, folds, checkpoint_name)
diff --git a/v0.7/medical_imaging/3d-unet/tf_SUT.py b/v0.7/medical_imaging/3d-unet/tf_SUT.py
index ced51aacb1..47d864a489 100644
--- a/v0.7/medical_imaging/3d-unet/tf_SUT.py
+++ b/v0.7/medical_imaging/3d-unet/tf_SUT.py
@@ -70,10 +70,6 @@ def flush_queries(self):
     def process_latencies(self, latencies_ns):
         pass
 
-    def __del__(self):
-        lg.DestroySUT(self.sut)
-        print("Finished destroying SUT.")
-
 
 def get_tf_sut(model_path, preprocessed_data_dir, performance_count):
     return _3DUNET_TF_SUT(model_path, preprocessed_data_dir, performance_count)
diff --git a/v0.7/medical_imaging/3d-unet/unet_onnx_to_tf.py b/v0.7/medical_imaging/3d-unet/unet_onnx_to_tf.py
index cd606823cc..472bcd61b1 100644
--- a/v0.7/medical_imaging/3d-unet/unet_onnx_to_tf.py
+++ b/v0.7/medical_imaging/3d-unet/unet_onnx_to_tf.py
@@ -22,12 +22,13 @@
 import onnx
 import onnx_tf
 
-
 def get_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--onnx_model", help="Path to the ONNX model")
+    parser.add_argument("--onnx_model",
+                        default="build/model/224_224_160.onnx",
+                        help="Path to the ONNX model")
     parser.add_argument("--output_name",
-                        default="192_224_192.pb",
+                        default="224_224_160.pb",
                         help="Name of output model")
     parser.add_argument("--output_dir",
                         default="build/model",
@@ -35,7 +36,6 @@ def get_args():
     args = parser.parse_args()
     return args
 
-
 def main():
     args = get_args()
 
diff --git a/v0.7/medical_imaging/3d-unet/unet_pytorch_to_onnx.py b/v0.7/medical_imaging/3d-unet/unet_pytorch_to_onnx.py
new file mode 100644
index 0000000000..a4990a6b6c
--- /dev/null
+++ b/v0.7/medical_imaging/3d-unet/unet_pytorch_to_onnx.py
@@ -0,0 +1,76 @@
+# coding=utf-8
+# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+
+import argparse
+import onnx
+import torch
+
+sys.path.insert(0, os.path.join(os.getcwd(), "nnUnet"))
+from nnunet.training.model_restore import load_model_and_checkpoint_files
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_dir",
+                        default="build/result/nnUNet/3d_fullres/Task043_BraTS2019/nnUNetTrainerV2__nnUNetPlansv2.mlperf.1",
+                        help="Path to the PyTorch model")
+    parser.add_argument("--output_name",
+                        default="224_224_160.onnx",
+                        help="Name of output model")
+    parser.add_argument("--dynamic_bs_output_name",
+                        default="224_224_160_dyanmic_bs.onnx",
+                        help="Name of output model")
+    parser.add_argument("--output_dir",
+                        default="build/model",
+                        help="Directory to save output model")
+    args = parser.parse_args()
+    return args
+
+def main():
+    args = get_args()
+
+    print("Converting PyTorch model to ONNX...")
+
+    if not os.path.exists(args.output_dir):
+        os.makedirs(args.output_dir)
+
+    output_path = "./{}/{}".format(args.output_dir, args.output_name)
+    dynamic_bs_output_path = "./{}/{}".format(args.output_dir, args.dynamic_bs_output_name)
+
+    print("Loading Pytorch model...")
+    checkpoint_name = "model_best"
+    folds = 1
+    trainer, params = load_model_and_checkpoint_files(args.model_dir, folds, fp16=False, checkpoint_name=checkpoint_name)
+    trainer.load_checkpoint_ram(params[0], False)
+    height = 224
+    width = 224
+    depth = 160
+    channels = 4
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    dummy_input = torch.rand([1, channels, height, width, depth]).float().to(device)
+    torch.onnx.export(trainer.network, dummy_input, output_path, opset_version=11,
+                      input_names=['input'], output_names=['output'])
+    torch.onnx.export(trainer.network, dummy_input, dynamic_bs_output_path, opset_version=11,
+                      input_names=['input'], output_names=['output'],
+                      dynamic_axes=({"input": {0: "batch_size"}, "output": {0: "batch_size"}}))
+
+    print("Successfully exported model {} and {}".format(output_path, dynamic_bs_output_path))
+
+if __name__ == "__main__":
+    main()

From e386a2abb22be48b1168b6b9627435bcb658fd5d Mon Sep 17 00:00:00 2001
From: Tony Reina <g.anthony.reina@intel.com>
Date: Wed, 1 Jul 2020 21:01:45 -0700
Subject: [PATCH 7/8] Added OpenVINO inference SUT

---
 v0.7/medical_imaging/3d-unet/ov_SUT.py | 76 ++++++++++++++++++++++++++
 v0.7/medical_imaging/3d-unet/run.py    |  6 +-
 2 files changed, 81 insertions(+), 1 deletion(-)
 create mode 100644 v0.7/medical_imaging/3d-unet/ov_SUT.py

diff --git a/v0.7/medical_imaging/3d-unet/ov_SUT.py b/v0.7/medical_imaging/3d-unet/ov_SUT.py
new file mode 100644
index 0000000000..4ac06b944e
--- /dev/null
+++ b/v0.7/medical_imaging/3d-unet/ov_SUT.py
@@ -0,0 +1,76 @@
+# coding=utf-8
+# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import array
+import json
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+
+import mlperf_loadgen as lg
+import numpy as np
+
+from brats_QSL import get_brats_QSL
+
+from openvino.inference_engine import IECore
+from scipy.special import softmax
+
+class _3DUNET_OV_SUT():
+    def __init__(self, model_path, preprocessed_data_dir, performance_count):
+        print("Loading OV model...")
+
+        model_xml = model_path
+        model_bin = os.path.splitext(model_xml)[0] + '.bin'
+        
+        ie = IECore()
+        net = ie.read_network(model=model_xml, weights=model_bin)
+
+        self.input_name = next(iter(net.inputs))
+        self.output_name = 'output'
+
+        self.exec_net = ie.load_network(network=net, device_name='CPU')
+
+        print("Constructing SUT...")
+        self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries,
+                                   self.process_latencies)
+        self.qsl = get_brats_QSL(preprocessed_data_dir, performance_count)
+        print("Finished constructing SUT.")
+
+    def issue_queries(self, query_samples):
+        for i in range(len(query_samples)):
+            data = self.qsl.get_features(query_samples[i].index)
+
+            print("Processing sample id {:d} with shape = {:}".format(
+                query_samples[i].index, data.shape))
+
+            before_softmax = self.exec_net.infer(inputs={self.input_name: data[np.newaxis, ...]})[self.output_name]
+            after_softmax = softmax(before_softmax, axis=1).astype(np.float16)
+
+            response_array = array.array("B", after_softmax.tobytes())
+            bi = response_array.buffer_info()
+            response = lg.QuerySampleResponse(query_samples[i].id, bi[0],
+                                              bi[1])
+            lg.QuerySamplesComplete([response])
+
+    def flush_queries(self):
+        pass
+
+    def process_latencies(self, latencies_ns):
+        pass
+
+
+def get_ov_sut(model_path, preprocessed_data_dir, performance_count):
+    return _3DUNET_OV_SUT(model_path, preprocessed_data_dir, performance_count)
diff --git a/v0.7/medical_imaging/3d-unet/run.py b/v0.7/medical_imaging/3d-unet/run.py
index 4a19301dea..9a4e024cb3 100644
--- a/v0.7/medical_imaging/3d-unet/run.py
+++ b/v0.7/medical_imaging/3d-unet/run.py
@@ -26,7 +26,7 @@
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--backend",
-                        choices=["pytorch", "onnxruntime", "tf"],
+                        choices=["pytorch", "onnxruntime", "tf", "ov"],
                         default="pytorch",
                         help="Backend")
     parser.add_argument(
@@ -83,6 +83,10 @@ def main():
         from tf_SUT import get_tf_sut
         sut = get_tf_sut(args.model, args.preprocessed_data_dir,
                          args.performance_count)
+    elif args.backend == "ov":
+        from ov_SUT import get_ov_sut
+        sut = get_ov_sut(args.model, args.preprocessed_data_dir,
+                         args.performance_count)
     else:
         raise ValueError("Unknown backend: {:}".format(args.backend))
 

From 22fb9e98df4034e03f60cb1dfb488ce449e49320 Mon Sep 17 00:00:00 2001
From: Tony Reina <37851530+tonyreina@users.noreply.github.com>
Date: Thu, 2 Jul 2020 15:04:48 -0700
Subject: [PATCH 8/8] Update ov_SUT.py

---
 v0.7/medical_imaging/3d-unet/ov_SUT.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/v0.7/medical_imaging/3d-unet/ov_SUT.py b/v0.7/medical_imaging/3d-unet/ov_SUT.py
index 4ac06b944e..9549d66416 100644
--- a/v0.7/medical_imaging/3d-unet/ov_SUT.py
+++ b/v0.7/medical_imaging/3d-unet/ov_SUT.py
@@ -1,5 +1,5 @@
 # coding=utf-8
-# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2020 INTEL CORPORATION. All rights reserved.
 # Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany
 #
 # Licensed under the Apache License, Version 2.0 (the "License");