Adding successful automated tests for HPVM build

hpvm/projects/keras/frontend/promise_translator.py

@@ -1157,7 +1157,7 @@ class PromiseRtTranslator:
     
   def translate(self, model, weights_dir, x_test):
 
-    print ("\n\n\n **** PromiseRT Translator ****** \n\n\n")
+    #print ("\n\n\n **** PromiseRT Translator ****** \n\n\n")
     root_node = self.dfg.root_node
     state = State()
 

hpvm/projects/keras/scripts/test_benchmarks.py

 
 
-
+import os
 import subprocess
 
 class Benchmark:
@@ -16,14 +16,14 @@ class Benchmark:
         return self.binary_path
 
     
-    def readAccuracy(self):
+    def readAccuracy(self, accuracy_file):
 
-        f = open("final_accuracy", "r") # File with final benchmark accuracy 
+        f = open(accuracy_file, "r") # File with final benchmark accuracy 
         acc_str = f.read()
         return float(acc_str)
     
         
-    def run(self):
+    def runKeras(self):
 
         # Test Bechmark accuracy with pretrained weights (hpvm_relaod)
         run_cmd = "python " + self.binary_path + " hpvm_reload "
@@ -32,7 +32,7 @@ class Benchmark:
         except:
             return False
 
-        accuracy = self.readAccuracy()
+        accuracy = self.readAccuracy("final_accuracy")
 
         print ("accuracy = ", accuracy, " test_accuracy = ", self.test_accuracy) 
 
@@ -45,7 +45,43 @@ class Benchmark:
             test_success = False
 
         return test_success
-    
+
+
+    def runHPVM(self):
+
+        # Test Bechmark accuracy with pretrained weights (hpvm_relaod)
+        run_cmd = "python " + self.binary_path + " hpvm_reload frontend compile"
+        try:
+            subprocess.call(run_cmd, shell=True)
+        except:
+            return False
+
+        working_dir = open("working_dir.txt").read()
+        cur_dir = os.getcwd()
+        
+        os.chdir(working_dir)
+        binary_path =  "./HPVM_binary"
+        
+        try:
+            subprocess.call(binary_path, shell=True)
+        except:
+            return False
+        
+        accuracy = self.readAccuracy("final_accuracy")
+        print ("accuracy = ", accuracy, " test_accuracy = ", self.test_accuracy) 
+
+        test_success = False
+        if (abs(self.test_accuracy - accuracy) < self.epsilon):
+            print ("Test for " + self. binary_path + " Passed ")
+            test_success = True
+        else:
+            print ("Test Failed for " + self.binary_path)
+            test_success = False
+
+        os.chdir(cur_dir)  # Change back to original working directory
+        
+        return test_success
+ 
         
 
 class BenchmarkTests:
@@ -55,6 +91,8 @@ class BenchmarkTests:
         self.benchmarks = []
         self.passed_tests = []
         self.failed_tests = []
+        self.passed_hpvm_tests = []
+        self.failed_hpvm_tests = []
 
 
     def addBenchmark(self, benchmark):
@@ -62,17 +100,29 @@ class BenchmarkTests:
         self.benchmarks.append(benchmark)
 
 
-    def runTests(self):
+    def runKerasTests(self):
 
         for benchmark in self.benchmarks:
-            test_success = benchmark.run()
+            test_success = benchmark.runKeras()
 
             if not test_success:
                 self.failed_tests.append(benchmark.getPath())
             else:
                 self.passed_tests.append(benchmark.getPath())
 
-    def printSummary(self):
+
+    def runHPVMTests(self):
+
+        for benchmark in self.benchmarks:
+            test_success = benchmark.runHPVM()
+
+            if not test_success:
+                self.failed_hpvm_tests.append(benchmark.getPath())
+            else:
+                self.passed_hpvm_tests.append(benchmark.getPath())
+
+                
+    def printKerasSummary(self):
 
         failed_test_count = len(self.failed_tests)
         passed_test_count = len(self.passed_tests)
@@ -88,6 +138,22 @@ class BenchmarkTests:
             print ("Failed: " + failed_test)
             
 
+    def printHPVMSummary(self):
+
+        failed_test_count = len(self.failed_hpvm_tests)
+        passed_test_count = len(self.passed_hpvm_tests)
+        
+        print (" Tests Passed  = " + str(passed_test_count) + " / " + str(len(self.benchmarks)))
+        print ("******* Passed Tests ** \n")
+        for passed_test in self.passed_hpvm_tests:
+            print ("Passed: " + passed_test)
+
+        print (" Tests Failed  = " + str(failed_test_count) + " / " + str(len(self.benchmarks)))
+        print ("****** Failed Tests *** \n")
+        for failed_test in self.failed_hpvm_tests:
+            print ("Failed: " + failed_test)
+            
+
         
             
 if __name__ == "__main__":
@@ -109,7 +175,9 @@ if __name__ == "__main__":
     testMgr.addBenchmark(VGG16_cifar10)
     testMgr.addBenchmark(VGG16_cifar100)
 
-    testMgr.runTests()
-    testMgr.printSummary()
-    
+    testMgr.runKerasTests()
+    testMgr.printKerasSummary()
     
+    testMgr.runHPVMTests()
+    testMgr.printHPVMSummary()
+

hpvm/projects/keras/src/Benchmark.py

@@ -69,6 +69,11 @@ class Benchmark:
             print ("\n\n ERROR: HPVM Compilation Failed!! \n\n")
             sys.exit(1)
 
+        f = open("working_dir.txt", "w+")
+        f.write(working_dir)
+        f.close()
+       
+            
         
     def printUsage(self):