From def1ad89584e22619f02d76150c7829bb7e4461b Mon Sep 17 00:00:00 2001
From: Yifan Zhao <yifanz16@illinois.edu>
Date: Sun, 14 Mar 2021 23:57:31 -0500
Subject: [PATCH] Handle relative speedup better
---
predtuner/approxapp.py | 17 +++++++++++++----
test/integrated_tuning.py | 2 ++
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/predtuner/approxapp.py b/predtuner/approxapp.py
index 0775ec8..552229d 100644
--- a/predtuner/approxapp.py
+++ b/predtuner/approxapp.py
@@ -113,6 +113,10 @@ class Config:
self.knobs = dict(sorted(knobs.items()))
self.test_qos: Optional[float] = test_qos
+ @property
+ def qos_speedup(self):
+ return self.qos, 1 / self.cost
+
T = TypeVar("T", bound=Config)
@@ -213,7 +217,7 @@ class ApproxTuner(Generic[T]):
@staticmethod
def take_best_configs(configs: List[T], n: Optional[int] = None) -> List[T]:
- points = np.array([[c.cost, c.qos] for c in configs])
+ points = np.array([c.qos_speedup for c in configs])
taken_idx = is_pareto_efficient(points, take_n=n)
return [configs[i] for i in taken_idx]
@@ -237,12 +241,11 @@ class ApproxTuner(Generic[T]):
f"No tuning session has been run; call self.tune() first."
)
- _, cost = self.app.measure_qos_cost({}, False)
fig, ax = plt.subplots()
confs = self.kept_configs
if not confs:
return fig
- qos_speedup = [(c.qos, cost / c.cost) for c in confs]
+ qos_speedup = [c.qos_speedup for c in confs]
qoses, speedups = zip(*sorted(qos_speedup, key=lambda p: p[0]))
ax.plot(qoses, speedups)
ax.scatter(qoses, speedups)
@@ -315,6 +318,8 @@ class TunerInterface(MeasurementInterface):
self.keep_thres = keep_thres
self.pbar = tqdm(total=test_limit, leave=False)
self.app_kwargs = app_kwargs
+ _, self.baseline_cost = app.measure_qos_cost({}, False, **self.app_kwargs)
+ msg_logger.debug(f"Baseline cost = {self.baseline_cost}")
# tune_thres can come in as np.float64 and opentuner doesn't like that
objective = ThresholdAccuracyMinimizeTime(float(tuner_thres))
@@ -343,6 +348,9 @@ class TunerInterface(MeasurementInterface):
cfg = desired_result.configuration.data
qos, cost = self.app.measure_qos_cost(cfg, False, **self.app_kwargs)
+ # We only want to measure cost in relative.
+ # This `cost` is inverse of speedup because opentuner needs a lower-better value.
+ cost /= self.baseline_cost
# Print a debug message for each config in tuning and keep threshold
self.print_debug_config(qos, cost)
self.pbar.update(self.progress_getter() - self.pbar.n)
@@ -362,5 +370,6 @@ class TunerInterface(MeasurementInterface):
else:
kind = "tuning and keep"
msg_logger.debug(
- f"Found config within {kind} threshold: QoS = {qos}, cost = {cost}"
+ f"Found config within {kind} threshold: QoS = {qos}, "
+ f"cost = {cost} ({1 / cost} speedup)"
)
diff --git a/test/integrated_tuning.py b/test/integrated_tuning.py
index 0ea3541..fe7f9e1 100644
--- a/test/integrated_tuning.py
+++ b/test/integrated_tuning.py
@@ -36,3 +36,5 @@ baseline, _ = app.measure_qos_cost({}, False)
tuner = app.get_tuner()
tuner.tune(100, 2.1, 3.0, True, 50, cost_model="cost_linear", qos_model="qos_p1")
tuner.dump_configs("tuner_results/test/configs.json")
+fig = tuner.plot_configs()
+fig.savefig("tuner_results/test/configs.png", dpi=300)
--
GitLab