Skip to content
Snippets Groups Projects
Commit 66d8a8ec authored by Yifan Zhao's avatar Yifan Zhao
Browse files

Handle relative speedup better

parent c751dac8
No related branches found
No related tags found
No related merge requests found
...@@ -113,6 +113,10 @@ class Config: ...@@ -113,6 +113,10 @@ class Config:
self.knobs = dict(sorted(knobs.items())) self.knobs = dict(sorted(knobs.items()))
self.test_qos: Optional[float] = test_qos self.test_qos: Optional[float] = test_qos
@property
def qos_speedup(self):
return self.qos, 1 / self.cost
T = TypeVar("T", bound=Config) T = TypeVar("T", bound=Config)
...@@ -213,7 +217,7 @@ class ApproxTuner(Generic[T]): ...@@ -213,7 +217,7 @@ class ApproxTuner(Generic[T]):
@staticmethod @staticmethod
def take_best_configs(configs: List[T], n: Optional[int] = None) -> List[T]: def take_best_configs(configs: List[T], n: Optional[int] = None) -> List[T]:
points = np.array([[c.cost, c.qos] for c in configs]) points = np.array([c.qos_speedup for c in configs])
taken_idx = is_pareto_efficient(points, take_n=n) taken_idx = is_pareto_efficient(points, take_n=n)
return [configs[i] for i in taken_idx] return [configs[i] for i in taken_idx]
...@@ -237,12 +241,11 @@ class ApproxTuner(Generic[T]): ...@@ -237,12 +241,11 @@ class ApproxTuner(Generic[T]):
f"No tuning session has been run; call self.tune() first." f"No tuning session has been run; call self.tune() first."
) )
_, cost = self.app.measure_qos_cost({}, False)
fig, ax = plt.subplots() fig, ax = plt.subplots()
confs = self.kept_configs confs = self.kept_configs
if not confs: if not confs:
return fig return fig
qos_speedup = [(c.qos, cost / c.cost) for c in confs] qos_speedup = [c.qos_speedup for c in confs]
qoses, speedups = zip(*sorted(qos_speedup, key=lambda p: p[0])) qoses, speedups = zip(*sorted(qos_speedup, key=lambda p: p[0]))
ax.plot(qoses, speedups) ax.plot(qoses, speedups)
ax.scatter(qoses, speedups) ax.scatter(qoses, speedups)
...@@ -315,6 +318,8 @@ class TunerInterface(MeasurementInterface): ...@@ -315,6 +318,8 @@ class TunerInterface(MeasurementInterface):
self.keep_thres = keep_thres self.keep_thres = keep_thres
self.pbar = tqdm(total=test_limit, leave=False) self.pbar = tqdm(total=test_limit, leave=False)
self.app_kwargs = app_kwargs self.app_kwargs = app_kwargs
_, self.baseline_cost = app.measure_qos_cost({}, False, **self.app_kwargs)
msg_logger.debug(f"Baseline cost = {self.baseline_cost}")
# tune_thres can come in as np.float64 and opentuner doesn't like that # tune_thres can come in as np.float64 and opentuner doesn't like that
objective = ThresholdAccuracyMinimizeTime(float(tuner_thres)) objective = ThresholdAccuracyMinimizeTime(float(tuner_thres))
...@@ -343,6 +348,9 @@ class TunerInterface(MeasurementInterface): ...@@ -343,6 +348,9 @@ class TunerInterface(MeasurementInterface):
cfg = desired_result.configuration.data cfg = desired_result.configuration.data
qos, cost = self.app.measure_qos_cost(cfg, False, **self.app_kwargs) qos, cost = self.app.measure_qos_cost(cfg, False, **self.app_kwargs)
# We only want to measure cost in relative.
# This `cost` is inverse of speedup because opentuner needs a lower-better value.
cost /= self.baseline_cost
# Print a debug message for each config in tuning and keep threshold # Print a debug message for each config in tuning and keep threshold
self.print_debug_config(qos, cost) self.print_debug_config(qos, cost)
self.pbar.update(self.progress_getter() - self.pbar.n) self.pbar.update(self.progress_getter() - self.pbar.n)
...@@ -362,5 +370,6 @@ class TunerInterface(MeasurementInterface): ...@@ -362,5 +370,6 @@ class TunerInterface(MeasurementInterface):
else: else:
kind = "tuning and keep" kind = "tuning and keep"
msg_logger.debug( msg_logger.debug(
f"Found config within {kind} threshold: QoS = {qos}, cost = {cost}" f"Found config within {kind} threshold: QoS = {qos}, "
f"cost = {cost} ({1 / cost} speedup)"
) )
...@@ -36,3 +36,5 @@ baseline, _ = app.measure_qos_cost({}, False) ...@@ -36,3 +36,5 @@ baseline, _ = app.measure_qos_cost({}, False)
tuner = app.get_tuner() tuner = app.get_tuner()
tuner.tune(100, 2.1, 3.0, True, 50, cost_model="cost_linear", qos_model="qos_p1") tuner.tune(100, 2.1, 3.0, True, 50, cost_model="cost_linear", qos_model="qos_p1")
tuner.dump_configs("tuner_results/test/configs.json") tuner.dump_configs("tuner_results/test/configs.json")
fig = tuner.plot_configs()
fig.savefig("tuner_results/test/configs.png", dpi=300)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment