Handle relative speedup better

predtuner/approxapp.py

@@ -113,6 +113,10 @@ class Config:
         self.knobs = dict(sorted(knobs.items()))
         self.test_qos: Optional[float] = test_qos
 
+    @property
+    def qos_speedup(self):
+        return self.qos, 1 / self.cost
+
 
 T = TypeVar("T", bound=Config)
 
@@ -213,7 +217,7 @@ class ApproxTuner(Generic[T]):
 
     @staticmethod
     def take_best_configs(configs: List[T], n: Optional[int] = None) -> List[T]:
-        points = np.array([[c.cost, c.qos] for c in configs])
+        points = np.array([c.qos_speedup for c in configs])
         taken_idx = is_pareto_efficient(points, take_n=n)
         return [configs[i] for i in taken_idx]
 
@@ -237,12 +241,11 @@ class ApproxTuner(Generic[T]):
                 f"No tuning session has been run; call self.tune() first."
             )
 
-        _, cost = self.app.measure_qos_cost({}, False)
         fig, ax = plt.subplots()
         confs = self.kept_configs
         if not confs:
             return fig
-        qos_speedup = [(c.qos, cost / c.cost) for c in confs]
+        qos_speedup = [c.qos_speedup for c in confs]
         qoses, speedups = zip(*sorted(qos_speedup, key=lambda p: p[0]))
         ax.plot(qoses, speedups)
         ax.scatter(qoses, speedups)
@@ -315,6 +318,8 @@ class TunerInterface(MeasurementInterface):
         self.keep_thres = keep_thres
         self.pbar = tqdm(total=test_limit, leave=False)
         self.app_kwargs = app_kwargs
+        _, self.baseline_cost = app.measure_qos_cost({}, False, **self.app_kwargs)
+        msg_logger.debug(f"Baseline cost = {self.baseline_cost}")
 
         # tune_thres can come in as np.float64 and opentuner doesn't like that
         objective = ThresholdAccuracyMinimizeTime(float(tuner_thres))
@@ -343,6 +348,9 @@ class TunerInterface(MeasurementInterface):
 
         cfg = desired_result.configuration.data
         qos, cost = self.app.measure_qos_cost(cfg, False, **self.app_kwargs)
+        # We only want to measure cost in relative.
+        # This `cost` is inverse of speedup because opentuner needs a lower-better value.
+        cost /= self.baseline_cost
         # Print a debug message for each config in tuning and keep threshold
         self.print_debug_config(qos, cost)
         self.pbar.update(self.progress_getter() - self.pbar.n)
@@ -362,5 +370,6 @@ class TunerInterface(MeasurementInterface):
         else:
             kind = "tuning and keep"
         msg_logger.debug(
-            f"Found config within {kind} threshold: QoS = {qos}, cost = {cost}"
+            f"Found config within {kind} threshold: QoS = {qos}, "
+            f"cost = {cost} ({1 / cost} speedup)"
         )

test/integrated_tuning.py

@@ -36,3 +36,5 @@ baseline, _ = app.measure_qos_cost({}, False)
 tuner = app.get_tuner()
 tuner.tune(100, 2.1, 3.0, True, 50, cost_model="cost_linear", qos_model="qos_p1")
 tuner.dump_configs("tuner_results/test/configs.json")
+fig = tuner.plot_configs()
+fig.savefig("tuner_results/test/configs.png", dpi=300)