From 1be5fb23609e3d4ec0d72a8a58f0670b6812037f Mon Sep 17 00:00:00 2001
From: Yifan Zhao <yifanz16@illinois.edu>
Date: Tue, 13 Jul 2021 05:53:22 -0500
Subject: [PATCH] Bring miniera model up to date
---
.../miniera/{miniera.pth => miniera.ckpt} | Bin 1908903 -> 1909486 bytes
hpvm/test/epoch_dnn/miniera.py | 62 +++++++++++++
hpvm/test/epoch_dnn/torch_dnn/__init__.py | 1 +
hpvm/test/epoch_dnn/torch_dnn/_utils.py | 47 ++++++++++
.../epoch_dnn/torch_dnn/miniera/__init__.py | 3 +-
.../test/epoch_dnn/torch_dnn/miniera/model.py | 84 ++++++++++++++----
hpvm/test/epoch_dnn/yolo.py | 70 ++-------------
7 files changed, 184 insertions(+), 83 deletions(-)
rename hpvm/test/epoch_dnn/assets/miniera/{miniera.pth => miniera.ckpt} (99%)
create mode 100644 hpvm/test/epoch_dnn/miniera.py
create mode 100644 hpvm/test/epoch_dnn/torch_dnn/_utils.py
diff --git a/hpvm/test/epoch_dnn/assets/miniera/miniera.pth b/hpvm/test/epoch_dnn/assets/miniera/miniera.ckpt
similarity index 99%
rename from hpvm/test/epoch_dnn/assets/miniera/miniera.pth
rename to hpvm/test/epoch_dnn/assets/miniera/miniera.ckpt
index 6796117d7821bc9574ffbb7194f21a03dc559e73..7bf0e767bd063e42c16aa10f6eca4649f553efc9 100644
GIT binary patch
delta 2590
zcmZ`*TXz#x6i(WvX#!~rlv@h60%}l+llw%JMo~n@R2jUCmm1RP(1}em`7$TmbQ~00
zig+pBD&F7x0k~FQT%rqo;z7hme*$aypvxy-+%uE337sbETWe<TGvD6dx$J$u|Aqbi
z8=Jo%8}LLVpU2}-s+z3AD|n?f^fp=6#*5{G&Xg)T$hMVi)l*EX$?BeRX;NjCQf0Dm
zT&d|Lt%6RQAr?(Ud2~rZ8mqG+QwkF$nZbAQLU?EQZV~>7w2535g`Lo9Q&Cx~9M_{<
z^n_yZq37byJIp{NBKhslwOVmfK~T0s1`m|AB0ECQWDH`B)JtQs$z&{^%EXf~E*s~u
zR_Y|9T&*LNKiWCU(0f?$M1*0X)A}>?MT9{iK+j%={`0Dp&^m_oA~r|^&8*Cpw8l$C
z9Rn(5qz2XYhzXvfA2zBXVVDAKqCmp%3^ohi8EkQ=w=#r947v!8HwAB7$!(Gxp3Cq_
z4%^jG-m{C??`X)dg)j_7^26Syj$tR*oeVof>|EBdBqJWRW$aoRDwWPAGBKXxaaTje
z-G>F!)^U%eBjQl+X1G_xeJ(muO&#~UkZ0lvK9S;+*kj2!NHU_fjGVI`3=g!*;G9@7
zW+E;k;i4nu=t#PSie(Zpj&!6*M~)(;t>wtgAY;kk9qKH@gCh32$jCHhJVe~)%C+Wc
zmnD;U*wQgi?E4xza?Y+AiR5z~&E?2B!5(3FRK$K485u{$V{S6yJfAi>p2On}9S0gZ
zp0IQrbf}+XI3z-7qdq3LoT8^(sk0oHOvI5_Lx()MZV$;ACGxz-o6mL6;%UK4kF$8@
zylEec!wjN`Bhvcjv7qhb1hF^Mv#uc$d^%0r_Bm^wycFPhC%_C|u+o>D^j~CnNyN)5
z(r=y#6R)_EXH)4I9q}=X+(+cET2te*lP(Be!8ePePJklAxCnVg>Wy<S;l`eb#WJZx
z8j2b+1&jTd%|3%kD|6K$E-}0&VybSSECpy@3eyTJP83-Y)ApQIA`)3W&<s@V3f3%*
z5WKo=Xp}wF4CtzD4Yka#T?}C|19ivQce^vn;kbblt9fHMY2egq-Z)Mhczrc*0&f^F
z$ZJxrD+4F-rU7%ck5f2f;H}lXX}oRV>}uW&-Z3z<>^0ka1CLn)=a${=-Yn(}oOird
z%W8UQtKwY)7aULPB-@)9?-_XC-4`?Qfq@U*opH<?xajy==SbI%vFYMTd}QEb+gsPi
z?JXh~%jNN+Jf-6k1E0!`?pavsp3$WN(P?c$DHmkbROG3uR;sXI-m>ghm)`N2fd!{G
zPHUACbwYeTD)p1>aYbhZhd`ILnu1HC_8PX>t3{^b3ssY>$&l-{no?l}XYiM!_U4=@
z)vQ4#z8bCTHmz1=rRmaXrABuEYUMh<9>X^_L77als!Y_&)cx`87{04V&vcAg_lV`T
zywqL1Eyhf<RClx*sa4qRv{ULeDZgf0Z#0mu+lJaz#@A+STmZV;jEy$xZ#O!{yr<pB
zr>(LZAmhH}@QsogY#Z9BLv6Q#X8BmFL>K=0Jv_Jg?;qhF?}B_iJa_i$(Bjo0Z(#WH
zkAwcM#i0$`=R%8rj)Z&W&)vEaCJZj#y0NV%w|Mh9eRA&Fh=mS9C!verBXkq|gdReG
z5G1T2tR;j9y@WnOKVcnVJz)c3fUuFUiLjZlg|Kzu+DJHX`3Jv$V4HR4vwl-M{@dHt
U8@ck+KR5SvUFyGb=-1rH|4}0OX#fBK
delta 2002
zcmZ{l+jA3T7{+&VPMU^N%Bd|HDryjn$!3%6ia=CE?3yZzhj`$ShO}Fu4Nvp6;9*?^
z(-yd?(+kJHzzgs2A~RmdIOvTt{3XtC;fyz4`F{H)CE1C|JTp5_cAodQ@4nxg$A7RN
ze`A~XzI!}AAssd$rz#0al4^~m)n+=AzFuFLw-|g|N8~BTpGsB2^kb7XmaFr11Zs3q
zvtp;OHmdBLFk=Y10eVhX3u-Q->N!O(AXM@5)N3mUmoNEe7$Rp4DP<%LKmRgBQ^tf5
zrE8fXcE{o*Lkw{f!<Bd!$045Cg=!NCi%6`1<*B3<i>itpR@g{Vfe|WTBwN^N$SsVD
z$XyI$CU*Bcq0)WAJ%cmVLM~UpUMpOd4mp&%uY>J^Gwe^5lX91PTojpLm^5+V33pl%
z*iQ{Ek<aQ>;^_`{vV(n)V^4|1XBZBdc-HCR&Ud+=8yKo+SyfT-Jm)?`*eMsgB>K&8
zxEK2cQRE22Q4=rrVCMyPx?4fi-+r;0meW+cMBF8+km1~=7P6dM5s^6t)kMAryV%7p
zbVK`xMbk4y1x1d%NugQ?yW|du13gtP`MT|XSrj?O@QR7!J={gXed2#gXbN@aWCy#{
z!G4uvza|o2XE<fz4L5PF_T&^eI6XL0(RC$*H?8orR0>JNeTGuYl3Xr@*6@}g)8!hb
zMYzH6wuy2jMpG@q-I1Odn5-1_LP5nlydUz6!|<*MZs9EFHAUn(hWAXozs<Wn(;o~D
zE$A6d!-qUHNU#;&f}qRzks%wwHGC{`KVkUP#QAMhr!&VF2F9v-HkVa!ks8ZWFS)5L
zR5|FZ2&^$&Hc?-(F;|Jw^Iu%7v+Ct4t76_g@m8uru7m{}pSg=>wZ7b3Xe{B1AvY^Q
zhkkSS60X`<v|N*~qt33vl8uJf&ElF3cwGg{Hkw{H2WDf%>#A6_ah<zP(}fb{^0;B+
zbMMUsd|~5DuUo`dHf*n};cFX?*VS>;#y7&{E0bmfw`|-d$Kg#7Ryu=kZM4K~T`!9@
z8|(d^f(;vYgy*jA#vEI$-oRZO_gr_SIh(%eI}b_h{qE^j#E{6n$J5Dl=F!*dW;(mZ
z-)E}gOa%H(u}wW|TkOA3|D9r&hx^P6s^*^ENS{k4-$PN4=`NC3zbPhk$YZ+yPw{%X
z{b%ApWUM`YRE`90kH*`haXFe?f0zi6f}{{BOp1`Aq!?+46ekUn5~LlZ5z<c5C}|gI
zjI^7yhqRZpkF=jOPMRQ1e*cgPw|_b=hc~zWjc#n+9olIBG8~A=Ki`T)<n{ZHxP0yV
okNlC+B!Aibb7ktEV}V5K!LNV+bt3S?$b;$MODEe?8F^#)Kl6kUQUCw|
diff --git a/hpvm/test/epoch_dnn/miniera.py b/hpvm/test/epoch_dnn/miniera.py
new file mode 100644
index 0000000000..398731d1ed
--- /dev/null
+++ b/hpvm/test/epoch_dnn/miniera.py
@@ -0,0 +1,62 @@
+import site
+import sys
+from os import makedirs
+from pathlib import Path
+
+import numpy as np
+from torch2hpvm import ModelExporter
+
+self_folder = Path(__file__).parent.absolute()
+site.addsitedir(self_folder.as_posix())
+
+from torch_dnn import BUFFER_NAME, quantize, run_test_over_ssh, split_and_scp
+from torch_dnn.miniera import MiniEraPL
+
+# Local configs
+ASSET_DIR = self_folder / "assets/miniera"
+QUANT_STRAT = "NONE" # Quantization method
+WORKING_DIR = Path("/tmp/miniera")
+N_IMAGES = 50
+# Remote configs
+SCP_OPTS = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P 5506"
+SSH_OPTS = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 5506"
+SCP_HOST = "root@espgate.cs.columbia.edu"
+SCP_PWD = "openesp"
+SCP_DST = "~/NV_NVDLA"
+
+if len(sys.argv) > 1:
+ WORKING_DIR = Path(sys.argv[1])
+
+# Reproducibility
+np.random.seed(42)
+
+# Create working directory
+makedirs(WORKING_DIR, exist_ok=True)
+
+# Calculate quantization scales
+ckpt = (ASSET_DIR / "miniera.ckpt").as_posix()
+model: MiniEraPL = MiniEraPL.load_from_checkpoint(ckpt, dataset_path=ASSET_DIR)
+scale_output = quantize(model, QUANT_STRAT, WORKING_DIR)
+
+# Code generation (into /tmp/miniera/hpvm-mod.nvdla)
+nvdla_buffer = WORKING_DIR / BUFFER_NAME
+print(f"Generating NVDLA buffer into {nvdla_buffer}")
+# You may replace scale_output (below) with ASSET_DIR / "scales/calib_NONE.txt"
+# to use precomputed quantization scale
+dataset = model.test_dataloader().dataset
+exporter = ModelExporter(model, dataset, WORKING_DIR, ASSET_DIR / "scales/calib_NONE.txt")
+exporter.generate(n_images=N_IMAGES).compile(WORKING_DIR / "miniera", WORKING_DIR)
+
+# SCP essential files to remote device
+input_images = exporter.dataset_dir
+split_and_scp([nvdla_buffer, input_images], SCP_HOST, SCP_DST, SCP_PWD, SCP_OPTS)
+
+# SSH to run test remotely
+n_total, correct = 0, 0
+for image_path, output in run_test_over_ssh(SCP_HOST, SCP_PWD, SCP_DST, input_images, SSH_OPTS):
+ idx, label = [int(s) for s in image_path.stem.split("_")]
+ output = np.array(output)
+ print(idx, output.argmax(), label, output)
+ n_total += 1
+ correct += int(output.argmax() == label)
+print(f"Accuracy: {correct / n_total * 100}% ({n_total} images)")
diff --git a/hpvm/test/epoch_dnn/torch_dnn/__init__.py b/hpvm/test/epoch_dnn/torch_dnn/__init__.py
index 89b65549ba..af88db3e23 100644
--- a/hpvm/test/epoch_dnn/torch_dnn/__init__.py
+++ b/hpvm/test/epoch_dnn/torch_dnn/__init__.py
@@ -1 +1,2 @@
from .quantizer import quantize
+from ._utils import split_and_scp, run_test_over_ssh, BUFFER_NAME
diff --git a/hpvm/test/epoch_dnn/torch_dnn/_utils.py b/hpvm/test/epoch_dnn/torch_dnn/_utils.py
new file mode 100644
index 0000000000..630100bb46
--- /dev/null
+++ b/hpvm/test/epoch_dnn/torch_dnn/_utils.py
@@ -0,0 +1,47 @@
+from pathlib import Path
+import pexpect
+
+# This is hardcoded in the NVDLA pass and will not change any time soon
+BUFFER_NAME = "hpvm-mod.nvdla"
+
+
+def split_and_scp(
+ local_srcs: list, host: str, remote_dst: str, password: str, options: str
+):
+ print(f"Copying files to remote host {host}...")
+ args = options.split(" ")
+ local_srcs = [str(s) for s in local_srcs]
+ args += ["-r", *local_srcs, f"{host}:{remote_dst}"]
+ child = pexpect.spawn("scp", args, timeout=None)
+ child.expect(r"password:")
+ child.sendline(password)
+ # A rough approach to at least print something when scp is alive
+ for line in child:
+ print(line.decode(), end="")
+
+
+def run_test_over_ssh(
+ host: str, password: str, working_dir: str, image_dir: Path, options: str
+):
+ print(f"Running test on remote host {host}...")
+ args = options.split(" ") + [host]
+ child = pexpect.spawn("ssh", args, timeout=None)
+ child.expect(r"password:")
+ child.sendline(password)
+ child.expect("# ") # The bash prompt
+ child.sendline(f"cd {working_dir}")
+ child.expect("# ")
+ child.delimiter = "# "
+ for image in image_dir.glob("*"):
+ remote_path = f"{image_dir.name}/{image.name}"
+ print(f"Sending {image.name} to run")
+ child.sendline(
+ f"./nvdla_runtime --loadable {BUFFER_NAME} --image {remote_path} --rawdump"
+ )
+ child.expect("# ")
+ child.sendline("cat output.dimg")
+ child.expect("# ")
+ result_lines = child.before.decode().splitlines()
+ # Should have 2 lines. First line is the command we keyed in.
+ output = [int(s) for s in result_lines[1].strip().split()]
+ yield image, output
diff --git a/hpvm/test/epoch_dnn/torch_dnn/miniera/__init__.py b/hpvm/test/epoch_dnn/torch_dnn/miniera/__init__.py
index d135d1bd40..5ed605176e 100644
--- a/hpvm/test/epoch_dnn/torch_dnn/miniera/__init__.py
+++ b/hpvm/test/epoch_dnn/torch_dnn/miniera/__init__.py
@@ -1,2 +1 @@
-from .dataset import CIFAR
-from .model import MiniERA
+from .model import MiniEraPL
diff --git a/hpvm/test/epoch_dnn/torch_dnn/miniera/model.py b/hpvm/test/epoch_dnn/torch_dnn/miniera/model.py
index 0f6fe455d8..b03df902de 100644
--- a/hpvm/test/epoch_dnn/torch_dnn/miniera/model.py
+++ b/hpvm/test/epoch_dnn/torch_dnn/miniera/model.py
@@ -2,27 +2,33 @@ from pathlib import Path
from typing import Union
import numpy as np
+import pytorch_lightning as pl
import torch
-from torch.nn import Conv2d, Linear, MaxPool2d, Module, ReLU, Sequential, Softmax
+from torch import nn
+from torch.utils.data import DataLoader
+from .dataset import CIFAR
-class MiniERA(Module):
+PathLike = Union[Path, str]
+
+
+class MiniERA(nn.Module):
def __init__(self):
super().__init__()
- self.convs = Sequential(
- Conv2d(3, 32, 3),
- ReLU(),
- Conv2d(32, 32, 3),
- ReLU(),
- MaxPool2d(2, 2),
- Conv2d(32, 64, 3),
- ReLU(),
- Conv2d(64, 64, 3),
- ReLU(),
- MaxPool2d(2, 2),
+ self.convs = nn.Sequential(
+ nn.Conv2d(3, 32, 3),
+ nn.ReLU(),
+ nn.Conv2d(32, 32, 3),
+ nn.ReLU(),
+ nn.MaxPool2d(2, 2),
+ nn.Conv2d(32, 64, 3),
+ nn.ReLU(),
+ nn.Conv2d(64, 64, 3),
+ nn.ReLU(),
+ nn.MaxPool2d(2, 2),
)
- self.fcs = Sequential(Linear(1600, 256), ReLU(), Linear(256, 5))
- self.softmax = Softmax(1)
+ self.fcs = nn.Sequential(nn.Linear(1600, 256), nn.ReLU(), nn.Linear(256, 5))
+ self.softmax = nn.Softmax(1)
def forward(self, input):
outputs = self.convs(input)
@@ -34,7 +40,7 @@ class MiniERA(Module):
# Load in model convolution weights
count = 0
for conv in self.convs:
- if not isinstance(conv, Conv2d):
+ if not isinstance(conv, nn.Conv2d):
continue
weight_np = np.fromfile(
prefix / f"conv2d_{count+1}_w.bin", dtype=np.float32
@@ -46,7 +52,7 @@ class MiniERA(Module):
# Load in model fc weights
count = 0
for linear in self.fcs:
- if not isinstance(linear, Linear):
+ if not isinstance(linear, nn.Linear):
continue
weight_np = np.fromfile(prefix / f"dense_{count+1}_w.bin", dtype=np.float32)
bias_np = np.fromfile(prefix / f"dense_{count+1}_b.bin", dtype=np.float32)
@@ -55,3 +61,47 @@ class MiniERA(Module):
linear.bias.data = torch.tensor(bias_np).reshape(linear.bias.shape)
count += 1
return self
+
+
+class MiniEraPL(pl.LightningModule):
+ def __init__(self, dataset_path: PathLike):
+ super().__init__()
+ self.network = MiniERA()
+ self.dataset_path = Path(dataset_path)
+
+ def forward(self, image):
+ prediction = self.network(image)
+ return prediction
+
+ @staticmethod
+ def _get_loss(output, targets):
+ from torch.nn.functional import cross_entropy
+
+ return cross_entropy(output, targets)
+
+ @staticmethod
+ def _get_metric(output, targets):
+ predicted = torch.argmax(output, 1)
+ return (predicted == targets).sum().item() / len(targets)
+
+ def validation_step(self, val_batch, batch_idx):
+ images, target = val_batch
+ prediction = self(images)
+ loss = self._get_loss(prediction, target)
+ accuracy = self._get_metric(prediction, target)
+ self.log("val_loss", loss)
+ self.log("val_acc", accuracy)
+ return accuracy
+
+ def test_step(self, test_batch, batch_idx):
+ images, target = test_batch
+ prediction = self(images)
+ accuracy = self._get_metric(prediction, target)
+ self.log("test_acc", accuracy)
+ return accuracy
+
+ def test_dataloader(self):
+ dataset = CIFAR.from_file(
+ self.dataset_path / "input.bin", self.dataset_path / "labels.bin"
+ )
+ return DataLoader(dataset, batch_size=128)
diff --git a/hpvm/test/epoch_dnn/yolo.py b/hpvm/test/epoch_dnn/yolo.py
index aa1ac34210..16c01fd287 100644
--- a/hpvm/test/epoch_dnn/yolo.py
+++ b/hpvm/test/epoch_dnn/yolo.py
@@ -9,59 +9,9 @@ from torch2hpvm import ModelExporter
self_folder = Path(__file__).parent.absolute()
site.addsitedir(self_folder.as_posix())
-from torch_dnn import quantize
+from torch_dnn import BUFFER_NAME, quantize, run_test_over_ssh, split_and_scp
from torch_dnn.yolo import TinyYoloPL
-# Consts (don't change)
-BUFFER_NAME = "hpvm-mod.nvdla"
-
-
-def split_and_scp(
- local_srcs: list, host: str, remote_dst: str, password: str, options: str
-):
- import pexpect
-
- print(f"Copying files to remote host {host}...")
- args = options.split(" ")
- local_srcs = [str(s) for s in local_srcs]
- args += ["-r", *local_srcs, f"{host}:{remote_dst}"]
- child = pexpect.spawn("scp", args, timeout=None)
- child.expect(r"password:")
- child.sendline(password)
- # A rough approach to at least print something when scp is alive
- for line in child:
- print(line.decode(), end="")
-
-
-def run_test_over_ssh(
- host: str, password: str, working_dir: str, image_dir: Path, options: str
-):
- import pexpect
-
- print(f"Running test on remote host {host}...")
- args = options.split(" ") + [host]
- child = pexpect.spawn("ssh", args, timeout=None)
- child.expect(r"password:")
- child.sendline(password)
- child.expect("# ") # The bash prompt
- child.sendline(f"cd {working_dir}")
- child.expect("# ")
- child.delimiter = "# "
- for image in image_dir.glob("*"):
- remote_path = f"{image_dir.name}/{image.name}"
- print(f"Sending {image.name} to run")
- child.sendline(
- f"./nvdla_runtime --loadable {BUFFER_NAME} --image {remote_path} --rawdump"
- )
- child.expect("# ")
- child.sendline("cat output.dimg")
- child.expect("# ")
- result_lines = child.before.decode().splitlines()
- # Should have 2 lines. First line is the command we keyed in.
- output = [int(s) for s in result_lines[1].strip().split()]
- yield image, output
-
-
# Local configs
ASSET_DIR = self_folder / "assets/yolo"
QUANT_STRAT = "NONE" # Quantization method
@@ -84,17 +34,16 @@ np.random.seed(42)
makedirs(WORKING_DIR, exist_ok=True)
# Calculate quantization scales
-# model = TinyYoloPL(num_classes=12)
ckpt = (ASSET_DIR / "yolo_atrv2.ckpt").as_posix()
model = TinyYoloPL.load_from_checkpoint(
ckpt, num_classes=12, dataset_path=ASSET_DIR / "atr_dataset.tar.gz"
)
-dataset = model.test_dataloader().dataset
scale_output = quantize(model, QUANT_STRAT, WORKING_DIR, gpus=1)
# Code generation (into /tmp/miniera/hpvm-mod.nvdla)
nvdla_buffer = WORKING_DIR / BUFFER_NAME
print(f"Generating NVDLA buffer into {nvdla_buffer}")
+dataset = model.test_dataloader().dataset
exporter = ModelExporter(model, dataset, WORKING_DIR, scale_output)
exporter.generate(n_images=N_IMAGES).compile(WORKING_DIR / "miniera", WORKING_DIR)
@@ -103,14 +52,7 @@ input_images = exporter.dataset_dir
split_and_scp([nvdla_buffer, input_images], SCP_HOST, SCP_DST, SCP_PWD, SCP_OPTS)
# SSH to run test remotely
-# n_total, correct = 0, 0
-# image_path, output = next(
-# run_test_over_ssh(SCP_HOST, SCP_PWD, SCP_DST, input_images, SSH_OPTS)
-# )
-# for image_path, output in run_test_over_ssh(SCP_HOST, SCP_PWD, SCP_DST, input_images, SSH_OPTS):
-# idx, label = [int(s) for s in image_path.stem.split("_")]
-# output = np.array(output)
-# print(idx, output.argmax(), label, output)
-# n_total += 1
-# correct += int(output.argmax() == label)
-# print(f"Accuracy: {correct / n_total * 100}% ({n_total} images)")
+n_total, correct = 0, 0
+image_path, output = next(
+ run_test_over_ssh(SCP_HOST, SCP_PWD, SCP_DST, input_images, SSH_OPTS)
+)
--
GitLab