Skip to content
Snippets Groups Projects
Commit 475b44d2 authored by Yifan Zhao's avatar Yifan Zhao
Browse files

Added test cases for tuning (coming from torch frontend)

parent 8dcf6d55
No related branches found
No related tags found
No related merge requests found
Showing
with 135 additions and 11 deletions
# --[ llvm-lit test setup # --[ llvm-lit test setup for test_frontend/
# lit.cfg.py looks for tests in CMAKE_CURRENT_BINARY_DIR (see lit.cfg.py)
# as most of the tests require some kind of compilation / generation
# which is best done over there.
configure_lit_site_cfg( configure_lit_site_cfg(
../../lit.site.cfg.py.in ../../lit.site.cfg.py.in
${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py ${CMAKE_CURRENT_BINARY_DIR}/test_frontend/lit.site.cfg.py
MAIN_CONFIG MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py ${CMAKE_CURRENT_SOURCE_DIR}/test_frontend/lit.cfg.py
) )
add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for package torch2hpvm" add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for HPVM PyTorch frontend"
${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/test_frontend
# We depend on check_dnn_acc.py defined in ../hpvm-c/ # We depend on check_dnn_acc.py defined in ../hpvm-c/
# to compare the inference accuracy of our frontend-generated binary # to compare the inference accuracy of our frontend-generated binary
# to that of the baseline. # to that of the baseline.
DEPENDS check_dnn_acc DEPENDS check_dnn_acc
ARGS "-j1" # Run frontend generation sequentially ARGS "-j1" # Run frontend generation sequentially
) )
# --[ llvm-lit test setup for test_tuning/
configure_lit_site_cfg(
../../lit.site.cfg.py.in
${CMAKE_CURRENT_BINARY_DIR}/test_tuning/lit.site.cfg.py
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/test_tuning/lit.cfg.py
)
add_lit_testsuite(check-hpvm-tuning "Run tests for autotuning procedure"
${CMAKE_CURRENT_BINARY_DIR}/test_tuning
ARGS "-j1" # Run tuning tests sequentially
)
#!/usr/bin/env python3 #!/usr/bin/env python3
import os
import shutil import shutil
import site import site
from pathlib import Path from pathlib import Path
...@@ -10,7 +9,7 @@ import torch ...@@ -10,7 +9,7 @@ import torch
from torch2hpvm import BinDataset, ModelExporter from torch2hpvm import BinDataset, ModelExporter
from torch.nn import Module from torch.nn import Module
site.addsitedir(os.path.dirname(__file__)) site.addsitedir(Path(__file__).parent.parent.absolute())
import dnn import dnn
benchmarks = { benchmarks = {
...@@ -33,7 +32,7 @@ print(f"Generating {netname} to {codegen_dir}") ...@@ -33,7 +32,7 @@ print(f"Generating {netname} to {codegen_dir}")
if codegen_dir.exists(): if codegen_dir.exists():
shutil.rmtree(codegen_dir) shutil.rmtree(codegen_dir)
params = self_folder / "../model_params" / netname params = self_folder / "../../model_params" / netname
dataset_shape = 5000, nch, img_size, img_size dataset_shape = 5000, nch, img_size, img_size
bin_tuneset = BinDataset( bin_tuneset = BinDataset(
params / "tune_input.bin", params / "tune_labels.bin", dataset_shape params / "tune_input.bin", params / "tune_labels.bin", dataset_shape
......
RUN: test_tuning.py alexnet_imagenet 0
RUN: test_tuning.py lenet_mnist 1
# -*- Python -*-
# Configuration file for the 'lit' test runner.
import os
import lit.formats
from lit.llvm import llvm_config
# name: The name of this test suite.
config.name = "HPVM-Predtuner"
# testFormat: The test format to use to interpret tests.
config.test_format = lit.formats.ShTest(False)
# suffixes: A list of file extensions to treat as test files. This is overriden
# by individual lit.local.cfg files in the test subdirectories.
config.suffixes = [".test"]
# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)
# test_exec_root: The root path where tests should be run.
current_source_dir = os.path.dirname(os.path.relpath(__file__, config.llvm_src_root))
current_binary_dir = os.path.join(config.llvm_obj_root, current_source_dir)
config.test_exec_root = current_binary_dir
# Tweak the PATH to include the tools dir.
llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
# Add substitution for our main script in this directory.
llvm_config.add_tool_substitutions(["test_tuning.py"], config.test_source_root)
RUN: test_tuning.py mobilenet_cifar10 0
RUN: test_tuning.py resnet18_cifar10 0
import os #!/usr/bin/env python3
import shutil import shutil
import site import site
from pathlib import Path from pathlib import Path
from sys import argv
import torch import torch
from predtuner import config_pylogger from predtuner import config_pylogger
...@@ -9,41 +10,31 @@ from predtuner.pipedbin import PipedBinaryApp ...@@ -9,41 +10,31 @@ from predtuner.pipedbin import PipedBinaryApp
from torch2hpvm import BinDataset, ModelExporter from torch2hpvm import BinDataset, ModelExporter
from torch.nn import Module from torch.nn import Module
site.addsitedir(os.path.dirname(__file__)) site.addsitedir(Path(__file__).parent.parent.absolute())
import dnn import dnn
# Set up logger to put log file in /tmp # Set up logger to put log file in /tmp
msg_logger = config_pylogger(output_dir="/tmp", verbose=True) msg_logger = config_pylogger(output_dir="/tmp", verbose=True)
benchmarks = {
"lenet_mnist": (dnn.LeNet, 1, 28, 1000),
"alexnet_imagenet": (dnn.AlexNetImageNet, 3, 224, 100),
"mobilenet_cifar10": (dnn.MobileNet, 3, 32, 500),
"resnet18_cifar10": (dnn.ResNet18, 3, 32, 500),
"vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 500),
}
model_param = Path(__file__).parent / "../../model_params"
benchmarks = [ def generate(model_cls, nch, img_size, batch_size, netname):
(dnn.LeNet, 1, 28, 500, "lenet_mnist"), codegen_dir = Path(f"./{netname}")
(dnn.AlexNet, 3, 32, 500, "alexnet_cifar10"),
(dnn.AlexNet2, 3, 32, 500, "alexnet2_cifar10"),
(dnn.AlexNetImageNet, 3, 224, 100, "alexnet_imagenet"),
(dnn.MobileNet, 3, 32, 500, "mobilenet_cifar10"),
(dnn.ResNet18, 3, 32, 500, "resnet18_cifar10"),
(dnn.ResNet50, 3, 224, 50, "resnet50_imagenet"),
(dnn.VGG16Cifar10, 3, 32, 500, "vgg16_cifar10"),
(dnn.VGG16Cifar100, 3, 32, 500, "vgg16_cifar100"),
(dnn.VGG16ImageNet, 3, 224, 50, "vgg16_imagenet"),
]
model_param = Path(__file__).parent / "../model_params"
def generate(model_cls, nch, img_size, batch_size, pathname):
codegen_dir = Path(f"/tmp/{pathname}_tune")
build_dir = codegen_dir / "build" build_dir = codegen_dir / "build"
metadata_file = codegen_dir / "ops.json" metadata_file = codegen_dir / "ops.json"
binary_file = build_dir / pathname binary_file = build_dir / netname
build_dir = codegen_dir / "build" build_dir = codegen_dir / "build"
# if binary_file.is_file() and metadata_file.is_file():
# return binary_file, metadata_file
print(f"Generating {pathname} to {codegen_dir}")
if codegen_dir.exists(): if codegen_dir.exists():
shutil.rmtree(codegen_dir) shutil.rmtree(codegen_dir)
params = model_param / pathname params = model_param / netname
dataset_shape = 5000, nch, img_size, img_size dataset_shape = 5000, nch, img_size, img_size
bin_tuneset = BinDataset( bin_tuneset = BinDataset(
params / "tune_input.bin", params / "tune_labels.bin", dataset_shape params / "tune_input.bin", params / "tune_labels.bin", dataset_shape
...@@ -52,7 +43,7 @@ def generate(model_cls, nch, img_size, batch_size, pathname): ...@@ -52,7 +43,7 @@ def generate(model_cls, nch, img_size, batch_size, pathname):
params / "test_input.bin", params / "test_labels.bin", dataset_shape params / "test_input.bin", params / "test_labels.bin", dataset_shape
) )
model: Module = model_cls() model: Module = model_cls()
checkpoint = model_param / f"{pathname}.pth.tar" checkpoint = model_param / f"pytorch/{netname}.pth.tar"
model.load_state_dict(torch.load(checkpoint.as_posix())) model.load_state_dict(torch.load(checkpoint.as_posix()))
exporter = ModelExporter( exporter = ModelExporter(
model, bin_tuneset, bin_testset, codegen_dir, target="hpvm_tensor_inspect" model, bin_tuneset, bin_testset, codegen_dir, target="hpvm_tensor_inspect"
...@@ -62,19 +53,27 @@ def generate(model_cls, nch, img_size, batch_size, pathname): ...@@ -62,19 +53,27 @@ def generate(model_cls, nch, img_size, batch_size, pathname):
def main(): def main():
for model_cls, nch, img_size, batch_size, pathname in benchmarks: netname, is_pred = argv[1:]
print(f"Testing {pathname}") is_pred = int(is_pred)
binary_file, metadata_file = generate( model_cls, nch, img_size, batch_size = benchmarks[netname]
model_cls, nch, img_size, batch_size, pathname binary_file, metadata_file = generate(
) model_cls, nch, img_size, batch_size, netname
app = PipedBinaryApp("test", binary_file, metadata_file) )
tuner = app.get_tuner() print(Path.cwd(), binary_file, metadata_file)
tuner.tune(100, 3.0, 3.0, True, 50, cost_model="cost_linear") app = PipedBinaryApp("test", binary_file, metadata_file)
tuner.dump_configs("configs.json") tuner = app.get_tuner()
fig = tuner.plot_configs(show_qos_loss=True) tuner.tune(
fig.savefig("configs.png", dpi=300) 5,
app.dump_hpvm_configs(tuner.best_configs, "hpvm_confs.txt") 3.0,
is_threshold_relative=True,
cost_model="cost_linear",
qos_model="qos_p1" if is_pred else "none",
)
tuner.dump_configs("configs.json")
fig = tuner.plot_configs(show_qos_loss=True)
fig.savefig("configs.png", dpi=300)
app.dump_hpvm_configs(tuner.best_configs, "hpvm_confs.txt")
if __name__ == "__main__": if __name__ == "__main__":
main() main()
\ No newline at end of file
RUN: test_tuning.py vgg16_cifar10 0
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment