From 717e55dcc8dda303d394caa4ea46f18dbc5ec201 Mon Sep 17 00:00:00 2001
From: Yifan Zhao <yifanz16@illinois.edu>
Date: Fri, 2 Apr 2021 07:19:37 -0500
Subject: [PATCH] Created test case for pytorch frontend
---
hpvm/test/CMakeLists.txt | 1 +
.../dnn_benchmarks/pytorch/CMakeLists.txt | 14 ++++
.../pytorch/alexnet2_cifar10.test | 1 +
.../pytorch/alexnet_cifar10.test | 1 +
.../pytorch/alexnet_imagenet.test | 1 +
.../dnn_benchmarks/pytorch/lenet_mnist.test | 1 +
hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py | 32 ++++++++
.../pytorch/mobilenet_cifar10.test | 1 +
.../pytorch/resnet18_cifar10.test | 1 +
.../pytorch/resnet50_imagenet.test | 1 +
.../dnn_benchmarks/pytorch/test_frontend.py | 77 ++++++++++---------
.../dnn_benchmarks/pytorch/vgg16_cifar10.test | 1 +
.../pytorch/vgg16_cifar100.test | 1 +
.../pytorch/vgg16_imagenet.test | 1 +
14 files changed, 97 insertions(+), 37 deletions(-)
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test
mode change 100644 => 100755 hpvm/test/dnn_benchmarks/pytorch/test_frontend.py
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test
diff --git a/hpvm/test/CMakeLists.txt b/hpvm/test/CMakeLists.txt
index 3c4f264723..4ff98a5386 100644
--- a/hpvm/test/CMakeLists.txt
+++ b/hpvm/test/CMakeLists.txt
@@ -8,5 +8,6 @@ set(CLANG_CXX ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/clang++)
add_subdirectory(hpvm_pass) # Passes test suite
add_subdirectory(benchmarks)
add_subdirectory(dnn_benchmarks/hpvm-c) # HPVM-C DNN accuracy test suite
+add_subdirectory(dnn_benchmarks/pytorch) # Torch frontend test suite
add_subdirectory(dnn_benchmarks/tensor-rt-src) # tensor_runtime DNN (build only, no tests)
add_subdirectory(dnn_benchmarks/profiling) # hpvm-profiler test suite
diff --git a/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt
new file mode 100644
index 0000000000..b184d2995b
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt
@@ -0,0 +1,14 @@
+# --[ llvm-lit test setup
+# lit.cfg.py looks for tests in CMAKE_CURRENT_BINARY_DIR (see lit.cfg.py)
+# as most of the tests require some kind of compilation / generation
+# which is best done over there.
+configure_lit_site_cfg(
+ ../../lit.site.cfg.py.in
+ ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+ MAIN_CONFIG
+ ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+)
+add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for package torch2hpvm"
+ ${CMAKE_CURRENT_BINARY_DIR}
+ ARGS "-j1" # Run frontend generation sequentially
+)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test
new file mode 100644
index 0000000000..378f8f80eb
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test
@@ -0,0 +1 @@
+RUN: test_frontend.py alexnet2_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test
new file mode 100644
index 0000000000..de7283c9a1
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test
@@ -0,0 +1 @@
+RUN: test_frontend.py alexnet_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test
new file mode 100644
index 0000000000..565f9d6f49
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test
@@ -0,0 +1 @@
+RUN: test_frontend.py alexnet_imagenet
diff --git a/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test b/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test
new file mode 100644
index 0000000000..9c0d3ebbbd
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test
@@ -0,0 +1 @@
+RUN: test_frontend.py lenet_mnist
diff --git a/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py b/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py
new file mode 100644
index 0000000000..6a098a5d76
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py
@@ -0,0 +1,32 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+
+import lit.formats
+from lit.llvm import llvm_config
+
+# name: The name of this test suite.
+config.name = "HPVM-Torch2HPVM"
+
+# testFormat: The test format to use to interpret tests.
+config.test_format = lit.formats.ShTest(False)
+
+# suffixes: A list of file extensions to treat as test files. This is overriden
+# by individual lit.local.cfg files in the test subdirectories.
+config.suffixes = [".test"]
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# test_exec_root: The root path where tests should be run.
+current_source_dir = os.path.dirname(os.path.relpath(__file__, config.llvm_src_root))
+current_binary_dir = os.path.join(config.llvm_obj_root, current_source_dir)
+config.test_exec_root = current_binary_dir
+
+# Tweak the PATH to include the tools dir.
+llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
+
+# Add substitution for our main script in this directory.
+llvm_config.add_tool_substitutions(["test_frontend.py"], config.test_source_root)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test
new file mode 100644
index 0000000000..af9065e184
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test
@@ -0,0 +1 @@
+RUN: test_frontend.py mobilenet_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test
new file mode 100644
index 0000000000..5d08067c6e
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test
@@ -0,0 +1 @@
+RUN: test_frontend.py resnet18_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test
new file mode 100644
index 0000000000..de08c21c43
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test
@@ -0,0 +1 @@
+RUN: test_frontend.py resnet50_imagenet
diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py b/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py
old mode 100644
new mode 100755
index 2fb1de17ee..89a49d511d
--- a/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py
@@ -1,52 +1,55 @@
+#!/usr/bin/env python3
import os
import shutil
import site
from pathlib import Path
from subprocess import run
-import torch
+from sys import argv
+import torch
from torch2hpvm import BinDataset, ModelExporter
from torch.nn import Module
site.addsitedir(os.path.dirname(__file__))
import dnn
-benchmarks = [
- (dnn.LeNet, 1, 28, 5000, "lenet_mnist"),
- (dnn.AlexNet, 3, 32, 5000, "alexnet_cifar10"),
- (dnn.AlexNet2, 3, 32, 5000, "alexnet2_cifar10"),
- (dnn.AlexNetImageNet, 3, 224, 500, "alexnet_imagenet"),
- (dnn.MobileNet, 3, 32, 5000, "mobilenet_cifar10"),
- (dnn.ResNet18, 3, 32, 5000, "resnet18_cifar10"),
- (dnn.ResNet50, 3, 224, 100, "resnet50_imagenet"),
- (dnn.VGG16Cifar10, 3, 32, 5000, "vgg16_cifar10"),
- (dnn.VGG16Cifar100, 3, 32, 5000, "vgg16_cifar100"),
- (dnn.VGG16ImageNet, 3, 224, 100, "vgg16_imagenet"),
-]
+benchmarks = {
+ "lenet_mnist": (dnn.LeNet, 1, 28, 5000),
+ "alexnet_cifar10": (dnn.AlexNet, 3, 32, 5000),
+ "alexnet2_cifar10": (dnn.AlexNet2, 3, 32, 5000),
+ "alexnet_imagenet": (dnn.AlexNetImageNet, 3, 224, 500),
+ "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 5000),
+ "resnet18_cifar10": (dnn.ResNet18, 3, 32, 5000),
+ "resnet50_imagenet": (dnn.ResNet50, 3, 224, 100),
+ "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 5000),
+ "vgg16_cifar100": (dnn.VGG16Cifar100, 3, 32, 5000),
+ "vgg16_imagenet": (dnn.VGG16ImageNet, 3, 224, 100),
+}
self_folder = Path(__file__).parent
-for model_cls, nch, img_size, batch_size, pathname in benchmarks:
- codegen_dir = Path(f"/tmp/{pathname}")
- print(f"Generating {pathname} to {codegen_dir}")
- if codegen_dir.exists():
- shutil.rmtree(codegen_dir)
+netname = argv[1]
+model_cls, nch, img_size, batch_size = benchmarks[netname]
+codegen_dir = Path(f"./{netname}")
+print(f"Generating {netname} to {codegen_dir}")
+if codegen_dir.exists():
+ shutil.rmtree(codegen_dir)
- params = self_folder / "../model_params" / pathname
- dataset_shape = 5000, nch, img_size, img_size
- bin_tuneset = BinDataset(
- params / "tune_input.bin", params / "tune_labels.bin", dataset_shape
- )
- bin_testset = BinDataset(
- params / "test_input.bin", params / "test_labels.bin", dataset_shape
- )
- model: Module = model_cls()
- checkpoint = self_folder / "../model_params/pytorch" / f"{pathname}.pth.tar"
- model.load_state_dict(torch.load(checkpoint.as_posix()))
+params = self_folder / "../model_params" / netname
+dataset_shape = 5000, nch, img_size, img_size
+bin_tuneset = BinDataset(
+ params / "tune_input.bin", params / "tune_labels.bin", dataset_shape
+)
+bin_testset = BinDataset(
+ params / "test_input.bin", params / "test_labels.bin", dataset_shape
+)
+model: Module = model_cls()
+checkpoint = self_folder / "../model_params/pytorch" / f"{netname}.pth.tar"
+model.load_state_dict(torch.load(checkpoint.as_posix()))
- build_dir = codegen_dir / "build"
- target_binary = build_dir / pathname
- conf_file = self_folder / "../hpvm-c/benchmarks" / pathname / "data/tuner_confs.txt"
- exporter = ModelExporter(
- model, bin_tuneset, bin_testset, codegen_dir, config_file=conf_file
- )
- exporter.generate(batch_size=batch_size).compile(target_binary, build_dir)
- run([str(target_binary), "test"], check=True)
+build_dir = codegen_dir / "build"
+target_binary = build_dir / netname
+conf_file = self_folder / "../hpvm-c/benchmarks" / netname / "data/tuner_confs.txt"
+exporter = ModelExporter(
+ model, bin_tuneset, bin_testset, codegen_dir, config_file=conf_file
+)
+exporter.generate(batch_size=batch_size).compile(target_binary, build_dir)
+run([str(target_binary), "test"], check=True)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test
new file mode 100644
index 0000000000..4010f21fc1
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test
@@ -0,0 +1 @@
+RUN: test_frontend.py vgg16_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test
new file mode 100644
index 0000000000..23b43e6288
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test
@@ -0,0 +1 @@
+RUN: test_frontend.py vgg16_cifar100
diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test
new file mode 100644
index 0000000000..e864c1e943
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test
@@ -0,0 +1 @@
+RUN: test_frontend.py vgg16_imagenet
--
GitLab