diff --git a/hpvm/test/dnn_benchmarks/keras/alexnet.py b/hpvm/test/dnn_benchmarks/keras/alexnet.py
index 7adfd5be5f2a91058e68ec39e6c73fe9d7b60492..86019e80e5a713d711b9d9c6e45cf91d4b01b9a5 100644
--- a/hpvm/test/dnn_benchmarks/keras/alexnet.py
+++ b/hpvm/test/dnn_benchmarks/keras/alexnet.py
@@ -143,9 +143,8 @@ if __name__ == '__main__':
# *** Below are Parameters specific to each benchmark *****
reload_dir = MODEL_PARAMS_DIR + '/alexnet_cifar10/'
## Either the HPVM weights are loaded (above) or the Keras Model from the path below
- keras_model_file = MODEL_PARAMS_DIR + '/alexnet_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/alexnet_cifar10.h5'
data_dir = 'data/alexnet_cifar10_hpvm/' # if reloading weights, data_dir can be set to empty string (value is ignored)
-
src_dir = 'src/alexnet_cifar10_src_hpvm/' # Directory where HPVM sources are downloaded
num_classes = 10 # Specify num out output classes - CIFAR10 has `10` classes
batch_size = 500 # Batch Size set to 500 - Adjust this value based on your GPU memory
diff --git a/hpvm/test/dnn_benchmarks/keras/alexnet2.py b/hpvm/test/dnn_benchmarks/keras/alexnet2.py
index 447d9d32ad6273c37b433a6bcc107613f164e68b..16bd92b23e3659be2f851d14e5030b8a436c3ac2 100644
--- a/hpvm/test/dnn_benchmarks/keras/alexnet2.py
+++ b/hpvm/test/dnn_benchmarks/keras/alexnet2.py
@@ -136,7 +136,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/alexnet2_cifar10/'
- keras_model_file = MODEL_PARAMS_DIR + '/alexnet2_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/alexnet2_cifar10.h5'
data_dir = 'data/alexnet2_cifar10/'
src_dir = 'src/alexnet2_cifar10_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/alexnet_imagenet.py b/hpvm/test/dnn_benchmarks/keras/alexnet_imagenet.py
index 9be1ff648c2ad27f99d4e1bccde8c4bb199dc174..9f5bbe12c93f667cae3bec37089a83b51df5311e 100644
--- a/hpvm/test/dnn_benchmarks/keras/alexnet_imagenet.py
+++ b/hpvm/test/dnn_benchmarks/keras/alexnet_imagenet.py
@@ -93,7 +93,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/alexnet_imagenet/'
- keras_model_file = MODEL_PARAMS_DIR + '/alexnet_imagenet/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/alexnet_imagenet.h5'
data_dir = 'data/alexnet_imagenet/'
src_dir = 'src/alexnet_imagenet_src/'
num_classes = 1000
diff --git a/hpvm/test/dnn_benchmarks/keras/lenet.py b/hpvm/test/dnn_benchmarks/keras/lenet.py
index 9de3c28b15aa097d1412880c21537c016a7392ef..e158d709136f3d593a74aa8bc770faa1bd59707f 100644
--- a/hpvm/test/dnn_benchmarks/keras/lenet.py
+++ b/hpvm/test/dnn_benchmarks/keras/lenet.py
@@ -102,7 +102,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/lenet_mnist/'
- keras_model_file = MODEL_PARAMS_DIR + '/lenet_mnist/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/lenet_mnist.h5'
data_dir = 'data/lenet_mnist/'
src_dir = 'src/lenet_mnist_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/mobilenet_cifar10.py b/hpvm/test/dnn_benchmarks/keras/mobilenet_cifar10.py
index 4987f3235f2458fc8ad78b7157941b648bc0e1c0..83b04445221d8826f62699a7fc2e45310654e857 100644
--- a/hpvm/test/dnn_benchmarks/keras/mobilenet_cifar10.py
+++ b/hpvm/test/dnn_benchmarks/keras/mobilenet_cifar10.py
@@ -182,7 +182,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/mobilenet_cifar10/'
- keras_model_file = MODEL_PARAMS_DIR + '/mobilenet_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/mobilenet_cifar10.h5'
data_dir = 'data/mobilenet_cifar10/'
src_dir = 'src/mobilenet_cifar10_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/resnet18_cifar10.py b/hpvm/test/dnn_benchmarks/keras/resnet18_cifar10.py
index 1a77cf1b2ce91c16a775e4a99df13dec2282e247..8ef31e3ff6dc2f252dbfa50fb557449220d01f39 100644
--- a/hpvm/test/dnn_benchmarks/keras/resnet18_cifar10.py
+++ b/hpvm/test/dnn_benchmarks/keras/resnet18_cifar10.py
@@ -554,7 +554,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/resnet18_cifar10/'
- keras_model_file = MODEL_PARAMS_DIR + '/resnet18_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/resnet18_cifar10.h5'
data_dir = 'data/resnet18_cifar10/'
src_dir = 'src/resnet18_cifar10_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/resnet50_imagenet.py b/hpvm/test/dnn_benchmarks/keras/resnet50_imagenet.py
index 6674faffab6e4c57b12607e01fda983b846dfa74..c1632750a91dc85815da15a1d0df718d00073311 100644
--- a/hpvm/test/dnn_benchmarks/keras/resnet50_imagenet.py
+++ b/hpvm/test/dnn_benchmarks/keras/resnet50_imagenet.py
@@ -141,7 +141,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/resnet50_imagenet/'
- keras_model_file = MODEL_PARAMS_DIR + '/resnet50_imagenet/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/resnet50_imagenet.h5'
data_dir = 'data/resnet50_imagenet/'
src_dir = 'src/resnet50_imagenet_src/'
num_classes = 1000
diff --git a/hpvm/test/dnn_benchmarks/keras/test_benchmarks.py b/hpvm/test/dnn_benchmarks/keras/test_benchmarks.py
new file mode 100644
index 0000000000000000000000000000000000000000..27fceb44d2466604f4536561c7786407f3c571eb
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/keras/test_benchmarks.py
@@ -0,0 +1,229 @@
+
+
+import os
+import subprocess
+
+import site
+from pathlib import Path
+
+import torch
+from torch.utils.data.dataloader import DataLoader
+from torch.utils.data.dataset import Subset
+
+#site.addsitedir(Path(__file__).parent.parent.absolute().as_posix())
+#from predtuner import PipedBinaryApp, config_pylogger
+
+
+
+class Benchmark:
+
+ def __init__(self, binary_path, test_accuracy):
+
+ self.binary_path = binary_path
+ self.test_accuracy = test_accuracy
+ self.epsilon = 0.05 # Adding some slack for accuracy difference
+
+
+ def getPath(self):
+ return self.binary_path
+
+
+ def readAccuracy(self, accuracy_file):
+
+ f = open(accuracy_file, "r") # File with final benchmark accuracy
+ acc_str = f.read()
+ return float(acc_str)
+
+
+ def runKeras(self):
+
+ # Test Bechmark accuracy with pretrained weights (hpvm_relaod)
+ run_cmd = "python3 " + self.binary_path + " keras_reload "
+ try:
+ subprocess.call(run_cmd, shell=True)
+ except:
+ return False
+
+ accuracy = self.readAccuracy("final_accuracy")
+
+ print ("accuracy = ", accuracy, " test_accuracy = ", self.test_accuracy)
+
+ test_success = False
+ if (abs(self.test_accuracy - accuracy) < self.epsilon):
+ print ("Test for " + self. binary_path + " Passed ")
+ test_success = True
+ else:
+ print ("Test Failed for " + self.binary_path)
+ test_success = False
+
+ return test_success
+
+
+ def runHPVM(self):
+
+ # Test Bechmark accuracy with pretrained weights (hpvm_relaod)
+ run_cmd = "python3 " + self.binary_path + " keras_reload frontend compile compile_tuner"
+ try:
+ subprocess.call(run_cmd, shell=True)
+ except:
+ return False
+
+ working_dir = open("working_dir.txt").read()
+ cur_dir = os.getcwd()
+
+ os.chdir(working_dir)
+ binary_path = "./HPVM_binary"
+
+ try:
+ subprocess.call(binary_path, shell=True)
+ except:
+ return False
+
+ accuracy = self.readAccuracy("final_accuracy")
+ print ("accuracy = ", accuracy, " test_accuracy = ", self.test_accuracy)
+
+ test_success = False
+ if (abs(self.test_accuracy - accuracy) < self.epsilon):
+ print ("Test for " + self. binary_path + " Passed ")
+ test_success = True
+ else:
+ print ("Test Failed for " + self.binary_path)
+ test_success = False
+
+ os.chdir(cur_dir) # Change back to original working directory
+
+ return test_success
+
+
+"""
+ def runApproxTuner(self):
+
+ working_dir = open("working_dir.txt").read()
+ cur_dir = os.getcwd()
+
+ os.chdir(working_dir)
+ binary_path = "./HPVM_tuner_binary"
+
+ full_binary_path = str(cur_dir) + "/" + working_dir + "/" + binary_path
+ full_json_path = str(cur_dir) + "/" + working_dir + "/tuner.json"
+
+ app = PipedBinaryApp("TestHPVMApp", full_binary_path, full_json_path)
+ # Tuning procedure is exactly the same as that for PyTorch DNN.
+ # Please refer to `./tune_vgg16_cifar10.py` for details.
+ tuner = app.get_tuner()
+ tuner.tune(5000, 3.0, 3.0, True, 50, cost_model="cost_linear", qos_model="qos_p1")
+
+ tuner.dump_configs("configs.json")
+ fig = tuner.plot_configs(show_qos_loss=True)
+ fig.savefig("configs.png", dpi=300)
+ app.dump_hpvm_configs(tuner.best_configs, "hpvm_confs.txt")
+
+ os.chdir(cur_dir) # Change back to original working directory
+"""
+
+
+
+class BenchmarkTests:
+
+ def __init__(self):
+
+ self.benchmarks = []
+ self.passed_tests = []
+ self.failed_tests = []
+ self.passed_hpvm_tests = []
+ self.failed_hpvm_tests = []
+
+
+ def addBenchmark(self, benchmark):
+
+ self.benchmarks.append(benchmark)
+
+
+ def runKerasTests(self):
+
+ for benchmark in self.benchmarks:
+ test_success = benchmark.runKeras()
+
+ if not test_success:
+ self.failed_tests.append(benchmark.getPath())
+ else:
+ self.passed_tests.append(benchmark.getPath())
+
+
+ def runHPVMTests(self):
+
+ for benchmark in self.benchmarks:
+ test_success = benchmark.runHPVM()
+
+ if not test_success:
+ self.failed_hpvm_tests.append(benchmark.getPath())
+ else:
+ self.passed_hpvm_tests.append(benchmark.getPath())
+
+
+ def printKerasSummary(self):
+
+ failed_test_count = len(self.failed_tests)
+ passed_test_count = len(self.passed_tests)
+
+ print (" Tests Passed = " + str(passed_test_count) + " / " + str(len(self.benchmarks)))
+ print ("******* Passed Tests ** \n")
+ for passed_test in self.passed_tests:
+ print ("Passed: " + passed_test)
+
+ print (" Tests Failed = " + str(failed_test_count) + " / " + str(len(self.benchmarks)))
+ print ("****** Failed Tests *** \n")
+ for failed_test in self.failed_tests:
+ print ("Failed: " + failed_test)
+
+
+ def printHPVMSummary(self):
+
+ failed_test_count = len(self.failed_hpvm_tests)
+ passed_test_count = len(self.passed_hpvm_tests)
+
+ print (" Tests Passed = " + str(passed_test_count) + " / " + str(len(self.benchmarks)))
+ print ("******* Passed Tests ** \n")
+ for passed_test in self.passed_hpvm_tests:
+ print ("Passed: " + passed_test)
+
+ print (" Tests Failed = " + str(failed_test_count) + " / " + str(len(self.benchmarks)))
+ print ("****** Failed Tests *** \n")
+ for failed_test in self.failed_hpvm_tests:
+ print ("Failed: " + failed_test)
+
+
+
+
+if __name__ == "__main__":
+
+ testMgr = BenchmarkTests()
+ AlexNet = Benchmark("alexnet.py", 79.28)
+ AlexNet_ImageNet = Benchmark("alexnet_imagenet.py", 56.30)
+ AlexNet2 = Benchmark("alexnet2.py", 84.98)
+ LeNet = Benchmark("lenet.py", 98.70)
+ MobileNet = Benchmark("mobilenet_cifar10.py", 84.42)
+ ResNet18 = Benchmark("resnet18_cifar10.py", 89.56)
+ ResNet50 = Benchmark("resnet50_imagenet.py", 75.10)
+ VGG16_cifar10 = Benchmark("vgg16_cifar10.py", 89.96)
+ VGG16_cifar100 = Benchmark("vgg16_cifar100.py", 66.50)
+ VGG16_ImageNet = Benchmark("vgg16_imagenet.py", 69.46)
+
+ testMgr.addBenchmark(AlexNet)
+ #testMgr.addBenchmark(AlexNet_ImageNet)
+ testMgr.addBenchmark(AlexNet2)
+ testMgr.addBenchmark(LeNet)
+ testMgr.addBenchmark(MobileNet)
+ testMgr.addBenchmark(ResNet18)
+ #testMgr.addBenchmark(ResNet50)
+ testMgr.addBenchmark(VGG16_cifar10)
+ testMgr.addBenchmark(VGG16_cifar100)
+ #testMgr.addBenchmark(VGG16_ImageNet)
+
+ #testMgr.runKerasTests()
+ #testMgr.printKerasSummary()
+
+ testMgr.runHPVMTests()
+ testMgr.printHPVMSummary()
+
+
diff --git a/hpvm/test/dnn_benchmarks/keras/vgg16_cifar10.py b/hpvm/test/dnn_benchmarks/keras/vgg16_cifar10.py
index 53207a039cafa9a89a34641ab887b21936c10ec1..9f298ebfa6a385bbeee48a2bbf6c193d8849a4d9 100644
--- a/hpvm/test/dnn_benchmarks/keras/vgg16_cifar10.py
+++ b/hpvm/test/dnn_benchmarks/keras/vgg16_cifar10.py
@@ -184,7 +184,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/vgg16_cifar10/'
- keras_model_file = MODEL_PARAMS_DIR + '/vgg16_cifar10/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/vgg16_cifar10.h5'
data_dir = 'data/vgg16_cifar10/'
src_dir = 'src/vgg16_cifar10_src/'
num_classes = 10
diff --git a/hpvm/test/dnn_benchmarks/keras/vgg16_cifar100.py b/hpvm/test/dnn_benchmarks/keras/vgg16_cifar100.py
index 6fcfd3249e24ace232d5c5c6e70e5e8d2842f773..4f6168d0f022223b2f355791f2bd7b11d5e04440 100644
--- a/hpvm/test/dnn_benchmarks/keras/vgg16_cifar100.py
+++ b/hpvm/test/dnn_benchmarks/keras/vgg16_cifar100.py
@@ -199,7 +199,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/vgg16_cifar100/'
- keras_model_file = MODEL_PARAMS_DIR + '/vgg16_cifar100/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/vgg16_cifar100.h5'
data_dir = 'data/vgg16_cifar100/'
src_dir = 'src/vgg16_cifar100_src/'
num_classes = 100
diff --git a/hpvm/test/dnn_benchmarks/keras/vgg16_imagenet.py b/hpvm/test/dnn_benchmarks/keras/vgg16_imagenet.py
index a881d2340f93ed8dac2c86bdd57c1f7be5e1330f..c84b429407cf491c5d6f4b6cb30b1c34e971612c 100644
--- a/hpvm/test/dnn_benchmarks/keras/vgg16_imagenet.py
+++ b/hpvm/test/dnn_benchmarks/keras/vgg16_imagenet.py
@@ -126,7 +126,7 @@ if __name__ == '__main__':
### Parameters specific to each benchmark
reload_dir = MODEL_PARAMS_DIR + '/vgg16_imagenet/'
- keras_model_file = MODEL_PARAMS_DIR + '/vgg16_imagenet/model.h5'
+ keras_model_file = MODEL_PARAMS_DIR + '/keras/vgg16_imagenet.h5'
data_dir = 'data/vgg16_imagenet/'
src_dir = 'src/vgg16_imagenet_src/'
num_classes = 1000