From 9ae0b0d1344759dc67ced1417dd16452064b9ab1 Mon Sep 17 00:00:00 2001
From: Hashim Sharif <hsharif3@miranda.cs.illinois.edu>
Date: Tue, 23 Mar 2021 20:39:36 -0500
Subject: [PATCH] Fixing paths to Keras h5 files based on new model params
location
---
hpvm/test/dnn_benchmarks/keras/alexnet.py | 3 +-
hpvm/test/dnn_benchmarks/keras/alexnet2.py | 2 +-
.../dnn_benchmarks/keras/alexnet_imagenet.py | 2 +-
hpvm/test/dnn_benchmarks/keras/lenet.py | 2 +-
.../dnn_benchmarks/keras/mobilenet_cifar10.py | 2 +-
.../dnn_benchmarks/keras/resnet18_cifar10.py | 2 +-
.../dnn_benchmarks/keras/resnet50_imagenet.py | 2 +-
.../dnn_benchmarks/keras/test_benchmarks.py | 229 ++++++++++++++++++
.../dnn_benchmarks/keras/vgg16_cifar10.py | 2 +-
.../dnn_benchmarks/keras/vgg16_cifar100.py | 2 +-
.../dnn_benchmarks/keras/vgg16_imagenet.py | 2 +-
11 files changed, 239 insertions(+), 11 deletions(-)
create mode 100644 hpvm/test/dnn_benchmarks/keras/test_benchmarks.py
diff --git a/hpvm/test/dnn_benchmarks/keras/alexnet.py b/hpvm/test/dnn_benchmarks/keras/alexnet.py
index 7adfd5be5f..86019e80e5 100644
--- a/hpvm/test/dnn_benchmarks/keras/alexnet.py
+++ b/hpvm/test/dnn_benchmarks/keras/alexnet.py
@@ -143,9 +143,8 @@ if __name__ == '__main__':
# *** Below are Parameters specific to each benchmark *****
reload_dir = MODEL_PARAMS_DIR + '/alexnet_cifar10/'
## Either the HPVM weights are loaded (above) or the Keras Model from the path below
- keras_model_file = MODEL_PARAMS_DIR + '/alexnet_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/alexnet_cifar10.h5'
data_dir = 'data/alexnet_cifar10_hpvm/' # if reloading weights, data_dir can be set to empty string (value is ignored)
-
src_dir = 'src/alexnet_cifar10_src_hpvm/' # Directory where HPVM sources are downloaded
num_classes = 10 # Specify num out output classes - CIFAR10 has `10` classes
batch_size = 500 # Batch Size set to 500 - Adjust this value based on your GPU memory
diff --git a/hpvm/test/dnn_benchmarks/keras/alexnet2.py b/hpvm/test/dnn_benchmarks/keras/alexnet2.py
index 447d9d32ad..16bd92b23e 100644
--- a/hpvm/test/dnn_benchmarks/keras/alexnet2.py
+++ b/hpvm/test/dnn_benchmarks/keras/alexnet2.py
@@ -136,7 +136,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/alexnet2_cifar10/'
- keras_model_file = MODEL_PARAMS_DIR + '/alexnet2_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/alexnet2_cifar10.h5'
data_dir = 'data/alexnet2_cifar10/'
src_dir = 'src/alexnet2_cifar10_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/alexnet_imagenet.py b/hpvm/test/dnn_benchmarks/keras/alexnet_imagenet.py
index 9be1ff648c..9f5bbe12c9 100644
--- a/hpvm/test/dnn_benchmarks/keras/alexnet_imagenet.py
+++ b/hpvm/test/dnn_benchmarks/keras/alexnet_imagenet.py
@@ -93,7 +93,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/alexnet_imagenet/'
- keras_model_file = MODEL_PARAMS_DIR + '/alexnet_imagenet/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/alexnet_imagenet.h5'
data_dir = 'data/alexnet_imagenet/'
src_dir = 'src/alexnet_imagenet_src/'
num_classes = 1000
diff --git a/hpvm/test/dnn_benchmarks/keras/lenet.py b/hpvm/test/dnn_benchmarks/keras/lenet.py
index 9de3c28b15..e158d70913 100644
--- a/hpvm/test/dnn_benchmarks/keras/lenet.py
+++ b/hpvm/test/dnn_benchmarks/keras/lenet.py
@@ -102,7 +102,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/lenet_mnist/'
- keras_model_file = MODEL_PARAMS_DIR + '/lenet_mnist/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/lenet_mnist.h5'
data_dir = 'data/lenet_mnist/'
src_dir = 'src/lenet_mnist_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/mobilenet_cifar10.py b/hpvm/test/dnn_benchmarks/keras/mobilenet_cifar10.py
index 4987f3235f..83b0444522 100644
--- a/hpvm/test/dnn_benchmarks/keras/mobilenet_cifar10.py
+++ b/hpvm/test/dnn_benchmarks/keras/mobilenet_cifar10.py
@@ -182,7 +182,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/mobilenet_cifar10/'
- keras_model_file = MODEL_PARAMS_DIR + '/mobilenet_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/mobilenet_cifar10.h5'
data_dir = 'data/mobilenet_cifar10/'
src_dir = 'src/mobilenet_cifar10_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/resnet18_cifar10.py b/hpvm/test/dnn_benchmarks/keras/resnet18_cifar10.py
index 1a77cf1b2c..8ef31e3ff6 100644
--- a/hpvm/test/dnn_benchmarks/keras/resnet18_cifar10.py
+++ b/hpvm/test/dnn_benchmarks/keras/resnet18_cifar10.py
@@ -554,7 +554,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/resnet18_cifar10/'
- keras_model_file = MODEL_PARAMS_DIR + '/resnet18_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/resnet18_cifar10.h5'
data_dir = 'data/resnet18_cifar10/'
src_dir = 'src/resnet18_cifar10_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/resnet50_imagenet.py b/hpvm/test/dnn_benchmarks/keras/resnet50_imagenet.py
index 6674faffab..c1632750a9 100644
--- a/hpvm/test/dnn_benchmarks/keras/resnet50_imagenet.py
+++ b/hpvm/test/dnn_benchmarks/keras/resnet50_imagenet.py
@@ -141,7 +141,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/resnet50_imagenet/'
- keras_model_file = MODEL_PARAMS_DIR + '/resnet50_imagenet/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/resnet50_imagenet.h5'
data_dir = 'data/resnet50_imagenet/'
src_dir = 'src/resnet50_imagenet_src/'
num_classes = 1000
diff --git a/hpvm/test/dnn_benchmarks/keras/test_benchmarks.py b/hpvm/test/dnn_benchmarks/keras/test_benchmarks.py
new file mode 100644
index 0000000000..27fceb44d2
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/keras/test_benchmarks.py
@@ -0,0 +1,229 @@
+
+
+import os
+import subprocess
+
+import site
+from pathlib import Path
+
+import torch
+from torch.utils.data.dataloader import DataLoader
+from torch.utils.data.dataset import Subset
+
+#site.addsitedir(Path(__file__).parent.parent.absolute().as_posix())
+#from predtuner import PipedBinaryApp, config_pylogger
+
+
+
+class Benchmark:
+
+ def __init__(self, binary_path, test_accuracy):
+
+ self.binary_path = binary_path
+ self.test_accuracy = test_accuracy
+ self.epsilon = 0.05 # Adding some slack for accuracy difference
+
+
+ def getPath(self):
+ return self.binary_path
+
+
+ def readAccuracy(self, accuracy_file):
+
+ f = open(accuracy_file, "r") # File with final benchmark accuracy
+ acc_str = f.read()
+ return float(acc_str)
+
+
+ def runKeras(self):
+
+ # Test Bechmark accuracy with pretrained weights (hpvm_relaod)
+ run_cmd = "python3 " + self.binary_path + " keras_reload "
+ try:
+ subprocess.call(run_cmd, shell=True)
+ except:
+ return False
+
+ accuracy = self.readAccuracy("final_accuracy")
+
+ print ("accuracy = ", accuracy, " test_accuracy = ", self.test_accuracy)
+
+ test_success = False
+ if (abs(self.test_accuracy - accuracy) < self.epsilon):
+ print ("Test for " + self. binary_path + " Passed ")
+ test_success = True
+ else:
+ print ("Test Failed for " + self.binary_path)
+ test_success = False
+
+ return test_success
+
+
+ def runHPVM(self):
+
+ # Test Bechmark accuracy with pretrained weights (hpvm_relaod)
+ run_cmd = "python3 " + self.binary_path + " keras_reload frontend compile compile_tuner"
+ try:
+ subprocess.call(run_cmd, shell=True)
+ except:
+ return False
+
+ working_dir = open("working_dir.txt").read()
+ cur_dir = os.getcwd()
+
+ os.chdir(working_dir)
+ binary_path = "./HPVM_binary"
+
+ try:
+ subprocess.call(binary_path, shell=True)
+ except:
+ return False
+
+ accuracy = self.readAccuracy("final_accuracy")
+ print ("accuracy = ", accuracy, " test_accuracy = ", self.test_accuracy)
+
+ test_success = False
+ if (abs(self.test_accuracy - accuracy) < self.epsilon):
+ print ("Test for " + self. binary_path + " Passed ")
+ test_success = True
+ else:
+ print ("Test Failed for " + self.binary_path)
+ test_success = False
+
+ os.chdir(cur_dir) # Change back to original working directory
+
+ return test_success
+
+
+"""
+ def runApproxTuner(self):
+
+ working_dir = open("working_dir.txt").read()
+ cur_dir = os.getcwd()
+
+ os.chdir(working_dir)
+ binary_path = "./HPVM_tuner_binary"
+
+ full_binary_path = str(cur_dir) + "/" + working_dir + "/" + binary_path
+ full_json_path = str(cur_dir) + "/" + working_dir + "/tuner.json"
+
+ app = PipedBinaryApp("TestHPVMApp", full_binary_path, full_json_path)
+ # Tuning procedure is exactly the same as that for PyTorch DNN.
+ # Please refer to `./tune_vgg16_cifar10.py` for details.
+ tuner = app.get_tuner()
+ tuner.tune(5000, 3.0, 3.0, True, 50, cost_model="cost_linear", qos_model="qos_p1")
+
+ tuner.dump_configs("configs.json")
+ fig = tuner.plot_configs(show_qos_loss=True)
+ fig.savefig("configs.png", dpi=300)
+ app.dump_hpvm_configs(tuner.best_configs, "hpvm_confs.txt")
+
+ os.chdir(cur_dir) # Change back to original working directory
+"""
+
+
+
+class BenchmarkTests:
+
+ def __init__(self):
+
+ self.benchmarks = []
+ self.passed_tests = []
+ self.failed_tests = []
+ self.passed_hpvm_tests = []
+ self.failed_hpvm_tests = []
+
+
+ def addBenchmark(self, benchmark):
+
+ self.benchmarks.append(benchmark)
+
+
+ def runKerasTests(self):
+
+ for benchmark in self.benchmarks:
+ test_success = benchmark.runKeras()
+
+ if not test_success:
+ self.failed_tests.append(benchmark.getPath())
+ else:
+ self.passed_tests.append(benchmark.getPath())
+
+
+ def runHPVMTests(self):
+
+ for benchmark in self.benchmarks:
+ test_success = benchmark.runHPVM()
+
+ if not test_success:
+ self.failed_hpvm_tests.append(benchmark.getPath())
+ else:
+ self.passed_hpvm_tests.append(benchmark.getPath())
+
+
+ def printKerasSummary(self):
+
+ failed_test_count = len(self.failed_tests)
+ passed_test_count = len(self.passed_tests)
+
+ print (" Tests Passed = " + str(passed_test_count) + " / " + str(len(self.benchmarks)))
+ print ("******* Passed Tests ** \n")
+ for passed_test in self.passed_tests:
+ print ("Passed: " + passed_test)
+
+ print (" Tests Failed = " + str(failed_test_count) + " / " + str(len(self.benchmarks)))
+ print ("****** Failed Tests *** \n")
+ for failed_test in self.failed_tests:
+ print ("Failed: " + failed_test)
+
+
+ def printHPVMSummary(self):
+
+ failed_test_count = len(self.failed_hpvm_tests)
+ passed_test_count = len(self.passed_hpvm_tests)
+
+ print (" Tests Passed = " + str(passed_test_count) + " / " + str(len(self.benchmarks)))
+ print ("******* Passed Tests ** \n")
+ for passed_test in self.passed_hpvm_tests:
+ print ("Passed: " + passed_test)
+
+ print (" Tests Failed = " + str(failed_test_count) + " / " + str(len(self.benchmarks)))
+ print ("****** Failed Tests *** \n")
+ for failed_test in self.failed_hpvm_tests:
+ print ("Failed: " + failed_test)
+
+
+
+
+if __name__ == "__main__":
+
+ testMgr = BenchmarkTests()
+ AlexNet = Benchmark("alexnet.py", 79.28)
+ AlexNet_ImageNet = Benchmark("alexnet_imagenet.py", 56.30)
+ AlexNet2 = Benchmark("alexnet2.py", 84.98)
+ LeNet = Benchmark("lenet.py", 98.70)
+ MobileNet = Benchmark("mobilenet_cifar10.py", 84.42)
+ ResNet18 = Benchmark("resnet18_cifar10.py", 89.56)
+ ResNet50 = Benchmark("resnet50_imagenet.py", 75.10)
+ VGG16_cifar10 = Benchmark("vgg16_cifar10.py", 89.96)
+ VGG16_cifar100 = Benchmark("vgg16_cifar100.py", 66.50)
+ VGG16_ImageNet = Benchmark("vgg16_imagenet.py", 69.46)
+
+ testMgr.addBenchmark(AlexNet)
+ #testMgr.addBenchmark(AlexNet_ImageNet)
+ testMgr.addBenchmark(AlexNet2)
+ testMgr.addBenchmark(LeNet)
+ testMgr.addBenchmark(MobileNet)
+ testMgr.addBenchmark(ResNet18)
+ #testMgr.addBenchmark(ResNet50)
+ testMgr.addBenchmark(VGG16_cifar10)
+ testMgr.addBenchmark(VGG16_cifar100)
+ #testMgr.addBenchmark(VGG16_ImageNet)
+
+ #testMgr.runKerasTests()
+ #testMgr.printKerasSummary()
+
+ testMgr.runHPVMTests()
+ testMgr.printHPVMSummary()
+
+
diff --git a/hpvm/test/dnn_benchmarks/keras/vgg16_cifar10.py b/hpvm/test/dnn_benchmarks/keras/vgg16_cifar10.py
index 53207a039c..9f298ebfa6 100644
--- a/hpvm/test/dnn_benchmarks/keras/vgg16_cifar10.py
+++ b/hpvm/test/dnn_benchmarks/keras/vgg16_cifar10.py
@@ -184,7 +184,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/vgg16_cifar10/'
- keras_model_file = MODEL_PARAMS_DIR + '/vgg16_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/vgg16_cifar10.h5'
data_dir = 'data/vgg16_cifar10/'
src_dir = 'src/vgg16_cifar10_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/vgg16_cifar100.py b/hpvm/test/dnn_benchmarks/keras/vgg16_cifar100.py
index 6fcfd3249e..4f6168d0f0 100644
--- a/hpvm/test/dnn_benchmarks/keras/vgg16_cifar100.py
+++ b/hpvm/test/dnn_benchmarks/keras/vgg16_cifar100.py
@@ -199,7 +199,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/vgg16_cifar100/'
- keras_model_file = MODEL_PARAMS_DIR + '/vgg16_cifar100/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/vgg16_cifar100.h5'
data_dir = 'data/vgg16_cifar100/'
src_dir = 'src/vgg16_cifar100_src/'
num_classes = 100
diff --git a/hpvm/test/dnn_benchmarks/keras/vgg16_imagenet.py b/hpvm/test/dnn_benchmarks/keras/vgg16_imagenet.py
index a881d2340f..c84b429407 100644
--- a/hpvm/test/dnn_benchmarks/keras/vgg16_imagenet.py
+++ b/hpvm/test/dnn_benchmarks/keras/vgg16_imagenet.py
@@ -126,7 +126,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/vgg16_imagenet/'
- keras_model_file = MODEL_PARAMS_DIR + '/vgg16_imagenet/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/vgg16_imagenet.h5'
data_dir = 'data/vgg16_imagenet/'
src_dir = 'src/vgg16_imagenet_src/'
num_classes = 1000
--
GitLab