Created test case for pytorch frontend

hpvm/test/CMakeLists.txt

@@ -8,5 +8,6 @@ set(CLANG_CXX ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/clang++)
 add_subdirectory(hpvm_pass)  # Passes test suite
 add_subdirectory(benchmarks)
 add_subdirectory(dnn_benchmarks/hpvm-c)  # HPVM-C DNN accuracy test suite
+add_subdirectory(dnn_benchmarks/pytorch)  # Torch frontend test suite
 add_subdirectory(dnn_benchmarks/tensor-rt-src)  # tensor_runtime DNN (build only, no tests)
 add_subdirectory(dnn_benchmarks/profiling)  # hpvm-profiler test suite

hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt

+# --[ llvm-lit test setup
+# lit.cfg.py looks for tests in CMAKE_CURRENT_BINARY_DIR (see lit.cfg.py)
+# as most of the tests require some kind of compilation / generation
+# which is best done over there.
+configure_lit_site_cfg(
+  ../../lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+  MAIN_CONFIG
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+)
+add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for package torch2hpvm"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  ARGS "-j1"  # Run frontend generation sequentially
+)

hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test

+RUN: test_frontend.py alexnet2_cifar10

hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test

+RUN: test_frontend.py alexnet_cifar10

hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test

+RUN: test_frontend.py alexnet_imagenet

hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test

+RUN: test_frontend.py lenet_mnist

hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py

+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+
+import lit.formats
+from lit.llvm import llvm_config
+
+# name: The name of this test suite.
+config.name = "HPVM-Torch2HPVM"
+
+# testFormat: The test format to use to interpret tests.
+config.test_format = lit.formats.ShTest(False)
+
+# suffixes: A list of file extensions to treat as test files. This is overriden
+# by individual lit.local.cfg files in the test subdirectories.
+config.suffixes = [".test"]
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# test_exec_root: The root path where tests should be run.
+current_source_dir = os.path.dirname(os.path.relpath(__file__, config.llvm_src_root))
+current_binary_dir = os.path.join(config.llvm_obj_root, current_source_dir)
+config.test_exec_root = current_binary_dir
+
+# Tweak the PATH to include the tools dir.
+llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
+
+# Add substitution for our main script in this directory.
+llvm_config.add_tool_substitutions(["test_frontend.py"], config.test_source_root)

hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test

+RUN: test_frontend.py mobilenet_cifar10

hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test

+RUN: test_frontend.py resnet18_cifar10

hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test

+RUN: test_frontend.py resnet50_imagenet

hpvm/test/dnn_benchmarks/pytorch/test_frontend.py

+#!/usr/bin/env python3
 import os
 import shutil
 import site
 from pathlib import Path
 from subprocess import run
-import torch
+from sys import argv
 
+import torch
 from torch2hpvm import BinDataset, ModelExporter
 from torch.nn import Module
 
 site.addsitedir(os.path.dirname(__file__))
 import dnn
 
-benchmarks = [
-    (dnn.LeNet, 1, 28, 5000, "lenet_mnist"),
-    (dnn.AlexNet, 3, 32, 5000, "alexnet_cifar10"),
-    (dnn.AlexNet2, 3, 32, 5000, "alexnet2_cifar10"),
-    (dnn.AlexNetImageNet, 3, 224, 500, "alexnet_imagenet"),
-    (dnn.MobileNet, 3, 32, 5000, "mobilenet_cifar10"),
-    (dnn.ResNet18, 3, 32, 5000, "resnet18_cifar10"),
-    (dnn.ResNet50, 3, 224, 100, "resnet50_imagenet"),
-    (dnn.VGG16Cifar10, 3, 32, 5000, "vgg16_cifar10"),
-    (dnn.VGG16Cifar100, 3, 32, 5000, "vgg16_cifar100"),
-    (dnn.VGG16ImageNet, 3, 224, 100, "vgg16_imagenet"),
-]
+benchmarks = {
+    "lenet_mnist": (dnn.LeNet, 1, 28, 5000),
+    "alexnet_cifar10": (dnn.AlexNet, 3, 32, 5000),
+    "alexnet2_cifar10": (dnn.AlexNet2, 3, 32, 5000),
+    "alexnet_imagenet": (dnn.AlexNetImageNet, 3, 224, 500),
+    "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 5000),
+    "resnet18_cifar10": (dnn.ResNet18, 3, 32, 5000),
+    "resnet50_imagenet": (dnn.ResNet50, 3, 224, 100),
+    "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 5000),
+    "vgg16_cifar100": (dnn.VGG16Cifar100, 3, 32, 5000),
+    "vgg16_imagenet": (dnn.VGG16ImageNet, 3, 224, 100),
+}
 self_folder = Path(__file__).parent
-for model_cls, nch, img_size, batch_size, pathname in benchmarks:
-    codegen_dir = Path(f"/tmp/{pathname}")
-    print(f"Generating {pathname} to {codegen_dir}")
-    if codegen_dir.exists():
-        shutil.rmtree(codegen_dir)
+netname = argv[1]
+model_cls, nch, img_size, batch_size = benchmarks[netname]
+codegen_dir = Path(f"./{netname}")
+print(f"Generating {netname} to {codegen_dir}")
+if codegen_dir.exists():
+    shutil.rmtree(codegen_dir)
 
-    params = self_folder / "../model_params" / pathname
-    dataset_shape = 5000, nch, img_size, img_size
-    bin_tuneset = BinDataset(
-        params / "tune_input.bin", params / "tune_labels.bin", dataset_shape
-    )
-    bin_testset = BinDataset(
-        params / "test_input.bin", params / "test_labels.bin", dataset_shape
-    )
-    model: Module = model_cls()
-    checkpoint = self_folder / "../model_params/pytorch" / f"{pathname}.pth.tar"
-    model.load_state_dict(torch.load(checkpoint.as_posix()))
+params = self_folder / "../model_params" / netname
+dataset_shape = 5000, nch, img_size, img_size
+bin_tuneset = BinDataset(
+    params / "tune_input.bin", params / "tune_labels.bin", dataset_shape
+)
+bin_testset = BinDataset(
+    params / "test_input.bin", params / "test_labels.bin", dataset_shape
+)
+model: Module = model_cls()
+checkpoint = self_folder / "../model_params/pytorch" / f"{netname}.pth.tar"
+model.load_state_dict(torch.load(checkpoint.as_posix()))
 
-    build_dir = codegen_dir / "build"
-    target_binary = build_dir / pathname
-    conf_file = self_folder / "../hpvm-c/benchmarks" / pathname / "data/tuner_confs.txt"
-    exporter = ModelExporter(
-        model, bin_tuneset, bin_testset, codegen_dir, config_file=conf_file
-    )
-    exporter.generate(batch_size=batch_size).compile(target_binary, build_dir)
-    run([str(target_binary), "test"], check=True)
+build_dir = codegen_dir / "build"
+target_binary = build_dir / netname
+conf_file = self_folder / "../hpvm-c/benchmarks" / netname / "data/tuner_confs.txt"
+exporter = ModelExporter(
+    model, bin_tuneset, bin_testset, codegen_dir, config_file=conf_file
+)
+exporter.generate(batch_size=batch_size).compile(target_binary, build_dir)
+run([str(target_binary), "test"], check=True)

hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test

+RUN: test_frontend.py vgg16_cifar10

hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test

+RUN: test_frontend.py vgg16_cifar100

hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test

+RUN: test_frontend.py vgg16_imagenet