Added accuracy checking for frontend-generated code

hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt

@@ -49,16 +49,17 @@ foreach(dir ${entries})
 endforeach(dir)
 
 # Install an accuracy comparator under build/bin for test suite.
-set(BIN_DIR ${LLVM_BINARY_DIR}/${LLVM_TOOLS_INSTALL_DIR})
+set(BIN_DIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 add_custom_command(
   OUTPUT ${BIN_DIR}/check_dnn_acc.py
   COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py ${BIN_DIR}
   COMMAND chmod +x ${BIN_DIR}/check_dnn_acc.py
   DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py
 )
+add_custom_target(check_dnn_acc DEPENDS ${BIN_DIR}/check_dnn_acc.py)
 
 message(STATUS "List of HPVM-C DNN benchmarks: ${test_compile_targets}")
-add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets} ${BIN_DIR}/check_dnn_acc.py)
+add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets})
 message(STATUS "Target name for compiling all DNN benchmarks: dnn_benchmarks")
 
 # --[ llvm-lit test setup
@@ -73,6 +74,6 @@ configure_lit_site_cfg(
 )
 add_lit_testsuite(check-hpvm-dnn "Running HPVM DNNs"
   ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS dnn_benchmarks  # Compile all dnn benchmarks to run them
+  DEPENDS dnn_benchmarks check_dnn_acc # Compile all dnn benchmarks to run them
   ARGS "-j1"  # Run DNN benchmarks sequentially
 )

hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt

@@ -10,5 +10,9 @@ configure_lit_site_cfg(
 )
 add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for package torch2hpvm"
   ${CMAKE_CURRENT_BINARY_DIR}
+  # We depend on check_dnn_acc.py defined in ../hpvm-c/
+  # to compare the inference accuracy of our frontend-generated binary
+  # to that of the baseline.
+  DEPENDS check_dnn_acc
   ARGS "-j1"  # Run frontend generation sequentially
 )

hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test

 RUN: test_frontend.py alexnet2_cifar10
+RUN: check_dnn_acc.py final_accuracy alexnet2_cifar10

hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test

 RUN: test_frontend.py alexnet_cifar10
+RUN: check_dnn_acc.py final_accuracy alexnet_cifar10

hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test

 RUN: test_frontend.py alexnet_imagenet
+RUN: check_dnn_acc.py final_accuracy alexnet_imagenet

hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test

 RUN: test_frontend.py lenet_mnist
+RUN: check_dnn_acc.py final_accuracy lenet_mnist

hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py

@@ -28,5 +28,9 @@ config.test_exec_root = current_binary_dir
 # Tweak the PATH to include the tools dir.
 llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
 
+# Add substitution for check_dnn_acc.py which goes under build/bin.
+llvm_config.add_tool_substitutions(
+    ["check_dnn_acc.py"], os.path.join(config.llvm_obj_root, "bin")
+)
 # Add substitution for our main script in this directory.
 llvm_config.add_tool_substitutions(["test_frontend.py"], config.test_source_root)

hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test

 RUN: test_frontend.py mobilenet_cifar10
+RUN: check_dnn_acc.py final_accuracy mobilenet_cifar10

hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test

 RUN: test_frontend.py resnet18_cifar10
+RUN: check_dnn_acc.py final_accuracy resnet18_cifar10

hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test

 RUN: test_frontend.py resnet50_imagenet
+RUN: check_dnn_acc.py final_accuracy resnet50_imagenet

hpvm/test/dnn_benchmarks/pytorch/test_frontend.py

@@ -14,16 +14,16 @@ site.addsitedir(os.path.dirname(__file__))
 import dnn
 
 benchmarks = {
-    "lenet_mnist": (dnn.LeNet, 1, 28, 5000),
-    "alexnet_cifar10": (dnn.AlexNet, 3, 32, 5000),
-    "alexnet2_cifar10": (dnn.AlexNet2, 3, 32, 5000),
+    "lenet_mnist": (dnn.LeNet, 1, 28, 1000),
+    "alexnet_cifar10": (dnn.AlexNet, 3, 32, 500),
+    "alexnet2_cifar10": (dnn.AlexNet2, 3, 32, 500),
     "alexnet_imagenet": (dnn.AlexNetImageNet, 3, 224, 500),
-    "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 5000),
-    "resnet18_cifar10": (dnn.ResNet18, 3, 32, 5000),
-    "resnet50_imagenet": (dnn.ResNet50, 3, 224, 100),
-    "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 5000),
-    "vgg16_cifar100": (dnn.VGG16Cifar100, 3, 32, 5000),
-    "vgg16_imagenet": (dnn.VGG16ImageNet, 3, 224, 100),
+    "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 500),
+    "resnet18_cifar10": (dnn.ResNet18, 3, 32, 500),
+    "resnet50_imagenet": (dnn.ResNet50, 3, 224, 25),
+    "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 500),
+    "vgg16_cifar100": (dnn.VGG16Cifar100, 3, 32, 500),
+    "vgg16_imagenet": (dnn.VGG16ImageNet, 3, 224, 10),
 }
 self_folder = Path(__file__).parent
 netname = argv[1]
@@ -44,6 +44,7 @@ bin_testset = BinDataset(
 model: Module = model_cls()
 checkpoint = self_folder / "../model_params/pytorch" / f"{netname}.pth.tar"
 model.load_state_dict(torch.load(checkpoint.as_posix()))
+print(model)
 
 build_dir = codegen_dir / "build"
 target_binary = build_dir / netname

hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test

 RUN: test_frontend.py vgg16_cifar10
+RUN: check_dnn_acc.py final_accuracy vgg16_cifar10

hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test

 RUN: test_frontend.py vgg16_cifar100
+RUN: check_dnn_acc.py final_accuracy vgg16_cifar100

hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test

 RUN: test_frontend.py vgg16_imagenet
+RUN: check_dnn_acc.py final_accuracy vgg16_imagenet