diff --git a/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt b/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt index 487adc4fa58e0b1ad13402156fdbcbdbe6026aea..f56312e9c3dabf22731bdc910672748c67ddf50d 100644 --- a/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt +++ b/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt @@ -49,16 +49,17 @@ foreach(dir ${entries}) endforeach(dir) # Install an accuracy comparator under build/bin for test suite. -set(BIN_DIR ${LLVM_BINARY_DIR}/${LLVM_TOOLS_INSTALL_DIR}) +set(BIN_DIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) add_custom_command( OUTPUT ${BIN_DIR}/check_dnn_acc.py COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py ${BIN_DIR} COMMAND chmod +x ${BIN_DIR}/check_dnn_acc.py DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py ) +add_custom_target(check_dnn_acc DEPENDS ${BIN_DIR}/check_dnn_acc.py) message(STATUS "List of HPVM-C DNN benchmarks: ${test_compile_targets}") -add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets} ${BIN_DIR}/check_dnn_acc.py) +add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets}) message(STATUS "Target name for compiling all DNN benchmarks: dnn_benchmarks") # --[ llvm-lit test setup @@ -73,6 +74,6 @@ configure_lit_site_cfg( ) add_lit_testsuite(check-hpvm-dnn "Running HPVM DNNs" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS dnn_benchmarks # Compile all dnn benchmarks to run them + DEPENDS dnn_benchmarks check_dnn_acc # Compile all dnn benchmarks to run them ARGS "-j1" # Run DNN benchmarks sequentially ) diff --git a/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt index b184d2995b7433d07ca45eab32a276f6896c6f41..778593a57ddfc3a6abcc4ed045f02614535739f8 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt +++ b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt @@ -10,5 +10,9 @@ configure_lit_site_cfg( ) add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for package torch2hpvm" ${CMAKE_CURRENT_BINARY_DIR} + # We depend on check_dnn_acc.py defined in ../hpvm-c/ + # to compare the inference accuracy of our frontend-generated binary + # to that of the baseline. + DEPENDS check_dnn_acc ARGS "-j1" # Run frontend generation sequentially ) diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test index 378f8f80eb3ca463f0577dde506f802b6b3eef58..4adf30226b19179be066f150b36ef3bd4a010636 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test +++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test @@ -1 +1,2 @@ RUN: test_frontend.py alexnet2_cifar10 +RUN: check_dnn_acc.py final_accuracy alexnet2_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test index de7283c9a1cbe3cbc995b6605f5a8c69f444fc47..cffec91e415cda256a72de5a04abb956336519d7 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test +++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test @@ -1 +1,2 @@ RUN: test_frontend.py alexnet_cifar10 +RUN: check_dnn_acc.py final_accuracy alexnet_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test index 565f9d6f495d53066d153133639dafe7d757cf28..126de1bfe80106bbd803ace37534cd38ab54a67c 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test +++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test @@ -1 +1,2 @@ RUN: test_frontend.py alexnet_imagenet +RUN: check_dnn_acc.py final_accuracy alexnet_imagenet diff --git a/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test b/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test index 9c0d3ebbbd490f3062bc43a741c55ab879eb3edf..b87a976bcd1bfa8d637f1298d5259bccb8781419 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test +++ b/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test @@ -1 +1,2 @@ RUN: test_frontend.py lenet_mnist +RUN: check_dnn_acc.py final_accuracy lenet_mnist diff --git a/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py b/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py index 6a098a5d76b1096d97836d18a204cdd87f0c1788..34473d24bea3565d0e2865c7026b43538f927ce7 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py +++ b/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py @@ -28,5 +28,9 @@ config.test_exec_root = current_binary_dir # Tweak the PATH to include the tools dir. llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True) +# Add substitution for check_dnn_acc.py which goes under build/bin. +llvm_config.add_tool_substitutions( + ["check_dnn_acc.py"], os.path.join(config.llvm_obj_root, "bin") +) # Add substitution for our main script in this directory. llvm_config.add_tool_substitutions(["test_frontend.py"], config.test_source_root) diff --git a/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test index af9065e1847fa48dbfd366e87184d2df205ab341..9964887b420a3896c83eff0114a419ad10740dc1 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test +++ b/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test @@ -1 +1,2 @@ RUN: test_frontend.py mobilenet_cifar10 +RUN: check_dnn_acc.py final_accuracy mobilenet_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test index 5d08067c6ebd178fde6682f44c3c775a5beb28df..71e0881a3f6d81a2982ac3fbd2dddd849f23a08b 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test +++ b/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test @@ -1 +1,2 @@ RUN: test_frontend.py resnet18_cifar10 +RUN: check_dnn_acc.py final_accuracy resnet18_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test index de08c21c43d7d1efe36eea1627ff3c0fd2f21011..b1ff2e6a92f506da299c1f94ebec10ddd1958159 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test +++ b/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test @@ -1 +1,2 @@ RUN: test_frontend.py resnet50_imagenet +RUN: check_dnn_acc.py final_accuracy resnet50_imagenet diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py b/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py index 89a49d511dd82361ff5f8bfb82243b3ad635d1a3..3c20c6ea5a472a693156b4881b58d4e0f1fc8575 100755 --- a/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py +++ b/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py @@ -14,16 +14,16 @@ site.addsitedir(os.path.dirname(__file__)) import dnn benchmarks = { - "lenet_mnist": (dnn.LeNet, 1, 28, 5000), - "alexnet_cifar10": (dnn.AlexNet, 3, 32, 5000), - "alexnet2_cifar10": (dnn.AlexNet2, 3, 32, 5000), + "lenet_mnist": (dnn.LeNet, 1, 28, 1000), + "alexnet_cifar10": (dnn.AlexNet, 3, 32, 500), + "alexnet2_cifar10": (dnn.AlexNet2, 3, 32, 500), "alexnet_imagenet": (dnn.AlexNetImageNet, 3, 224, 500), - "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 5000), - "resnet18_cifar10": (dnn.ResNet18, 3, 32, 5000), - "resnet50_imagenet": (dnn.ResNet50, 3, 224, 100), - "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 5000), - "vgg16_cifar100": (dnn.VGG16Cifar100, 3, 32, 5000), - "vgg16_imagenet": (dnn.VGG16ImageNet, 3, 224, 100), + "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 500), + "resnet18_cifar10": (dnn.ResNet18, 3, 32, 500), + "resnet50_imagenet": (dnn.ResNet50, 3, 224, 25), + "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 500), + "vgg16_cifar100": (dnn.VGG16Cifar100, 3, 32, 500), + "vgg16_imagenet": (dnn.VGG16ImageNet, 3, 224, 10), } self_folder = Path(__file__).parent netname = argv[1] @@ -44,6 +44,7 @@ bin_testset = BinDataset( model: Module = model_cls() checkpoint = self_folder / "../model_params/pytorch" / f"{netname}.pth.tar" model.load_state_dict(torch.load(checkpoint.as_posix())) +print(model) build_dir = codegen_dir / "build" target_binary = build_dir / netname diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test index 4010f21fc1e60a09c11dac1065a2d833135c09c9..5544c75d2823fb31da6624e109c81567770d18ad 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test +++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test @@ -1 +1,2 @@ RUN: test_frontend.py vgg16_cifar10 +RUN: check_dnn_acc.py final_accuracy vgg16_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test index 23b43e6288d8d79aa74471b38955238579e1c66a..66bd69ee377b4dd84071e3c63ec631f3c041512a 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test +++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test @@ -1 +1,2 @@ RUN: test_frontend.py vgg16_cifar100 +RUN: check_dnn_acc.py final_accuracy vgg16_cifar100 diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test index e864c1e9437d1ec65b5091514e88a8f069c9272f..6529998ec4e4d62d14fc6b99d42474f3161d2eb7 100644 --- a/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test +++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test @@ -1 +1,2 @@ RUN: test_frontend.py vgg16_imagenet +RUN: check_dnn_acc.py final_accuracy vgg16_imagenet