diff --git a/hpvm/test/CMakeLists.txt b/hpvm/test/CMakeLists.txt index 3c4f26472317f511edaab98c5e4a4f8ed7ba2dfb..4ff98a5386d91ce50b755d7e507a84e0fbe1c4dd 100644 --- a/hpvm/test/CMakeLists.txt +++ b/hpvm/test/CMakeLists.txt @@ -8,5 +8,6 @@ set(CLANG_CXX ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/clang++) add_subdirectory(hpvm_pass) # Passes test suite add_subdirectory(benchmarks) add_subdirectory(dnn_benchmarks/hpvm-c) # HPVM-C DNN accuracy test suite +add_subdirectory(dnn_benchmarks/pytorch) # Torch frontend test suite add_subdirectory(dnn_benchmarks/tensor-rt-src) # tensor_runtime DNN (build only, no tests) add_subdirectory(dnn_benchmarks/profiling) # hpvm-profiler test suite diff --git a/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b184d2995b7433d07ca45eab32a276f6896c6f41 --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt @@ -0,0 +1,14 @@ +# --[ llvm-lit test setup +# lit.cfg.py looks for tests in CMAKE_CURRENT_BINARY_DIR (see lit.cfg.py) +# as most of the tests require some kind of compilation / generation +# which is best done over there. +configure_lit_site_cfg( + ../../lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py + MAIN_CONFIG + ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py +) +add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for package torch2hpvm" + ${CMAKE_CURRENT_BINARY_DIR} + ARGS "-j1" # Run frontend generation sequentially +) diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test new file mode 100644 index 0000000000000000000000000000000000000000..378f8f80eb3ca463f0577dde506f802b6b3eef58 --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test @@ -0,0 +1 @@ +RUN: test_frontend.py alexnet2_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test new file mode 100644 index 0000000000000000000000000000000000000000..de7283c9a1cbe3cbc995b6605f5a8c69f444fc47 --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test @@ -0,0 +1 @@ +RUN: test_frontend.py alexnet_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test new file mode 100644 index 0000000000000000000000000000000000000000..565f9d6f495d53066d153133639dafe7d757cf28 --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test @@ -0,0 +1 @@ +RUN: test_frontend.py alexnet_imagenet diff --git a/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test b/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test new file mode 100644 index 0000000000000000000000000000000000000000..9c0d3ebbbd490f3062bc43a741c55ab879eb3edf --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test @@ -0,0 +1 @@ +RUN: test_frontend.py lenet_mnist diff --git a/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py b/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py new file mode 100644 index 0000000000000000000000000000000000000000..6a098a5d76b1096d97836d18a204cdd87f0c1788 --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py @@ -0,0 +1,32 @@ +# -*- Python -*- + +# Configuration file for the 'lit' test runner. + +import os + +import lit.formats +from lit.llvm import llvm_config + +# name: The name of this test suite. +config.name = "HPVM-Torch2HPVM" + +# testFormat: The test format to use to interpret tests. +config.test_format = lit.formats.ShTest(False) + +# suffixes: A list of file extensions to treat as test files. This is overriden +# by individual lit.local.cfg files in the test subdirectories. +config.suffixes = [".test"] + +# test_source_root: The root path where tests are located. +config.test_source_root = os.path.dirname(__file__) + +# test_exec_root: The root path where tests should be run. +current_source_dir = os.path.dirname(os.path.relpath(__file__, config.llvm_src_root)) +current_binary_dir = os.path.join(config.llvm_obj_root, current_source_dir) +config.test_exec_root = current_binary_dir + +# Tweak the PATH to include the tools dir. +llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True) + +# Add substitution for our main script in this directory. +llvm_config.add_tool_substitutions(["test_frontend.py"], config.test_source_root) diff --git a/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test new file mode 100644 index 0000000000000000000000000000000000000000..af9065e1847fa48dbfd366e87184d2df205ab341 --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test @@ -0,0 +1 @@ +RUN: test_frontend.py mobilenet_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test new file mode 100644 index 0000000000000000000000000000000000000000..5d08067c6ebd178fde6682f44c3c775a5beb28df --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test @@ -0,0 +1 @@ +RUN: test_frontend.py resnet18_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test new file mode 100644 index 0000000000000000000000000000000000000000..de08c21c43d7d1efe36eea1627ff3c0fd2f21011 --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test @@ -0,0 +1 @@ +RUN: test_frontend.py resnet50_imagenet diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py b/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py old mode 100644 new mode 100755 index 2fb1de17ee226571e6cd6b808640bf35280932db..89a49d511dd82361ff5f8bfb82243b3ad635d1a3 --- a/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py +++ b/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py @@ -1,52 +1,55 @@ +#!/usr/bin/env python3 import os import shutil import site from pathlib import Path from subprocess import run -import torch +from sys import argv +import torch from torch2hpvm import BinDataset, ModelExporter from torch.nn import Module site.addsitedir(os.path.dirname(__file__)) import dnn -benchmarks = [ - (dnn.LeNet, 1, 28, 5000, "lenet_mnist"), - (dnn.AlexNet, 3, 32, 5000, "alexnet_cifar10"), - (dnn.AlexNet2, 3, 32, 5000, "alexnet2_cifar10"), - (dnn.AlexNetImageNet, 3, 224, 500, "alexnet_imagenet"), - (dnn.MobileNet, 3, 32, 5000, "mobilenet_cifar10"), - (dnn.ResNet18, 3, 32, 5000, "resnet18_cifar10"), - (dnn.ResNet50, 3, 224, 100, "resnet50_imagenet"), - (dnn.VGG16Cifar10, 3, 32, 5000, "vgg16_cifar10"), - (dnn.VGG16Cifar100, 3, 32, 5000, "vgg16_cifar100"), - (dnn.VGG16ImageNet, 3, 224, 100, "vgg16_imagenet"), -] +benchmarks = { + "lenet_mnist": (dnn.LeNet, 1, 28, 5000), + "alexnet_cifar10": (dnn.AlexNet, 3, 32, 5000), + "alexnet2_cifar10": (dnn.AlexNet2, 3, 32, 5000), + "alexnet_imagenet": (dnn.AlexNetImageNet, 3, 224, 500), + "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 5000), + "resnet18_cifar10": (dnn.ResNet18, 3, 32, 5000), + "resnet50_imagenet": (dnn.ResNet50, 3, 224, 100), + "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 5000), + "vgg16_cifar100": (dnn.VGG16Cifar100, 3, 32, 5000), + "vgg16_imagenet": (dnn.VGG16ImageNet, 3, 224, 100), +} self_folder = Path(__file__).parent -for model_cls, nch, img_size, batch_size, pathname in benchmarks: - codegen_dir = Path(f"/tmp/{pathname}") - print(f"Generating {pathname} to {codegen_dir}") - if codegen_dir.exists(): - shutil.rmtree(codegen_dir) +netname = argv[1] +model_cls, nch, img_size, batch_size = benchmarks[netname] +codegen_dir = Path(f"./{netname}") +print(f"Generating {netname} to {codegen_dir}") +if codegen_dir.exists(): + shutil.rmtree(codegen_dir) - params = self_folder / "../model_params" / pathname - dataset_shape = 5000, nch, img_size, img_size - bin_tuneset = BinDataset( - params / "tune_input.bin", params / "tune_labels.bin", dataset_shape - ) - bin_testset = BinDataset( - params / "test_input.bin", params / "test_labels.bin", dataset_shape - ) - model: Module = model_cls() - checkpoint = self_folder / "../model_params/pytorch" / f"{pathname}.pth.tar" - model.load_state_dict(torch.load(checkpoint.as_posix())) +params = self_folder / "../model_params" / netname +dataset_shape = 5000, nch, img_size, img_size +bin_tuneset = BinDataset( + params / "tune_input.bin", params / "tune_labels.bin", dataset_shape +) +bin_testset = BinDataset( + params / "test_input.bin", params / "test_labels.bin", dataset_shape +) +model: Module = model_cls() +checkpoint = self_folder / "../model_params/pytorch" / f"{netname}.pth.tar" +model.load_state_dict(torch.load(checkpoint.as_posix())) - build_dir = codegen_dir / "build" - target_binary = build_dir / pathname - conf_file = self_folder / "../hpvm-c/benchmarks" / pathname / "data/tuner_confs.txt" - exporter = ModelExporter( - model, bin_tuneset, bin_testset, codegen_dir, config_file=conf_file - ) - exporter.generate(batch_size=batch_size).compile(target_binary, build_dir) - run([str(target_binary), "test"], check=True) +build_dir = codegen_dir / "build" +target_binary = build_dir / netname +conf_file = self_folder / "../hpvm-c/benchmarks" / netname / "data/tuner_confs.txt" +exporter = ModelExporter( + model, bin_tuneset, bin_testset, codegen_dir, config_file=conf_file +) +exporter.generate(batch_size=batch_size).compile(target_binary, build_dir) +run([str(target_binary), "test"], check=True) diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test new file mode 100644 index 0000000000000000000000000000000000000000..4010f21fc1e60a09c11dac1065a2d833135c09c9 --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test @@ -0,0 +1 @@ +RUN: test_frontend.py vgg16_cifar10 diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test new file mode 100644 index 0000000000000000000000000000000000000000..23b43e6288d8d79aa74471b38955238579e1c66a --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test @@ -0,0 +1 @@ +RUN: test_frontend.py vgg16_cifar100 diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test new file mode 100644 index 0000000000000000000000000000000000000000..e864c1e9437d1ec65b5091514e88a8f069c9272f --- /dev/null +++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test @@ -0,0 +1 @@ +RUN: test_frontend.py vgg16_imagenet