Put up framework for modeled tuning

predtuner/approxapp.py

 import abc
 import logging
 from pathlib import Path
-from typing import Dict, Generic, List, NamedTuple, Optional, Tuple, TypeVar, Union
+from typing import Dict, Generic, List, Optional, Tuple, Type, TypeVar, Union
 
 import matplotlib.pyplot as plt
 import numpy as np
 from opentuner.measurement.interface import MeasurementInterface
-from opentuner.resultsdb.models import Configuration, Result
-from opentuner.search.manipulator import ConfigurationManipulator, EnumParameter
+from opentuner.search.manipulator import (ConfigurationManipulator,
+                                          EnumParameter)
 
 from ._logging import override_opentuner_config
 from ._pareto import is_pareto_efficient
@@ -61,32 +61,33 @@ class ApproxApp(abc.ABC):
 
 class Config:
     def __init__(
-        self, qos: float, calib_qos: Optional[float], perf: float, knobs: KnobsT
+        self, qos: float, perf: float, knobs: KnobsT, calib_qos: Optional[float] = None
     ) -> None:
         self.qos = qos
-        self.calib_qos = calib_qos
         self.perf = perf
         self.knobs = knobs
+        self.calib_qos: Optional[float] = calib_qos
 
 
 T = TypeVar("T", bound=Config)
 
 
-# ApproxTuner is generic over the type of the config
+# IOpenTuner is generic over the type of the config
 # So that the user can use custom Config inherited from Config
 # (in which case they need to override `get_all_configs_from_db`).
 class ApproxTuner(Generic[T]):
     def __init__(self, app: ApproxApp) -> None:
         self.app = app
+        self._tuned = False
         self.all_configs = []
         self.kept_configs = []
         self.best_configs = []
+        # The following will be filled after self.tune() is called
         self.keep_threshold = None
-        self._db = None
 
     @property
     def tuned(self) -> bool:
-        return not self._db is None
+        return self._tuned
 
     def tune(
         self,
@@ -95,54 +96,44 @@ class ApproxTuner(Generic[T]):
         qos_keep_threshold: Optional[float] = None,
         is_threshold_relative: bool = False,
         take_best_n: Optional[int] = None,
-        calibrate: bool = True
+        calibrate: bool = True,
+        **kwargs
         # TODO: more parameters + opentuner param forwarding
-    ) -> List[Config]:
-        """Generate an optimal set of approximation configurations for the model."""
+    ) -> List[T]:
         from opentuner.tuningrunmain import TuningRunMain
 
         from ._dbloader import read_opentuner_db
 
-        # By default, keep_threshold == tuner_threshold
         opentuner_args = opentuner_default_args()
-        qos_keep_threshold = qos_keep_threshold or qos_tuner_threshold
-        if is_threshold_relative:
-            baseline_qos, _ = self.app.measure_qos_perf({}, False)
-            qos_tuner_threshold = baseline_qos - qos_tuner_threshold
-            qos_keep_threshold = baseline_qos - qos_keep_threshold
-        opentuner_args.test_limit = max_iter
-        tuner = TunerInterface(
-            opentuner_args, self.app, qos_tuner_threshold, qos_keep_threshold, max_iter,
+        tuner = self._get_tuner_interface(
+            opentuner_args,
+            max_iter,
+            qos_tuner_threshold,
+            qos_keep_threshold,
+            is_threshold_relative,
+            self._get_app_kwargs(**kwargs),
         )
+        assert self.keep_threshold is not None
         trm = TuningRunMain(tuner, opentuner_args)
         # TuningRunMain.__init__ initializes its own logger, so we'll override it and use ours
         override_opentuner_config()
         # This is where opentuner runs
         trm.main()
-
         # Parse and store results
-        self._db = opentuner_args.database
-        self.keep_threshold = qos_keep_threshold
-        self.all_configs = list(
-            self.get_all_configs_from_db(read_opentuner_db(self._db))
-        )
+        self._tuned = True
+        config_ty = self._get_config_class()
+        self.all_configs = [
+            config_ty(result.accuracy, result.time, configuration.data)
+            for result, configuration in read_opentuner_db(opentuner_args.database)
+        ]
         self.kept_configs = [
-            cfg for cfg in self.all_configs if cfg.qos > qos_keep_threshold
+            cfg for cfg in self.all_configs if cfg.qos > self.keep_threshold
         ]
         self.best_configs = self.take_best_configs(self.kept_configs, take_best_n)
         if calibrate:
             self.calibrate_configs_(self.best_configs)
         return self.best_configs
 
-    @classmethod
-    def get_all_configs_from_db(
-        cls, results_configs: List[Tuple[Result, Configuration]]
-    ) -> Tuple[T]:
-        return tuple(
-            Config(result.accuracy, None, result.time, configuration.data)
-            for result, configuration in results_configs
-        )
-
     def calibrate_configs_(self, configs: List[T]):
         from tqdm import tqdm
 
@@ -154,9 +145,7 @@ class ApproxTuner(Generic[T]):
             msg_logger.debug(f"Calibration: {cfg.qos} (mean) -> {cfg.calib_qos} (mean)")
 
     @staticmethod
-    def take_best_configs(
-        configs: List[Config], n: Optional[int] = None
-    ) -> List[Config]:
+    def take_best_configs(configs: List[T], n: Optional[int] = None) -> List[T]:
         points = np.array([[c.perf, c.qos] for c in configs])
         taken_idx = is_pareto_efficient(points, take_n=n)
         return [configs[i] for i in taken_idx]
@@ -193,6 +182,38 @@ class ApproxTuner(Generic[T]):
         ax.set_ylabel("speedup")
         return fig
 
+    def _get_tuner_interface(
+        self,
+        opentuner_args,
+        max_iter: int,
+        qos_tuner_threshold: float,
+        qos_keep_threshold: Optional[float],
+        is_threshold_relative: bool,
+        app_kwargs: dict,
+    ) -> "TunerInterface":
+        # By default, keep_threshold == tuner_threshold
+        self.keep_threshold = qos_keep_threshold or qos_tuner_threshold
+        if is_threshold_relative:
+            baseline_qos, _ = self.app.measure_qos_perf({}, False)
+            qos_tuner_threshold = baseline_qos - qos_tuner_threshold
+            self.keep_threshold = baseline_qos - self.keep_threshold
+        opentuner_args.test_limit = max_iter
+        return TunerInterface(
+            opentuner_args,
+            self.app,
+            qos_tuner_threshold,
+            self.keep_threshold,
+            max_iter,
+            **app_kwargs,
+        )
+
+    def _get_app_kwargs(self, **kwargs):
+        return {}
+
+    @classmethod
+    def _get_config_class(cls) -> Type[Config]:
+        return Config
+
 
 def opentuner_default_args():
     from opentuner import default_argparser
@@ -208,6 +229,7 @@ class TunerInterface(MeasurementInterface):
         tuner_thres: float,
         keep_thres: float,
         test_limit: int,
+        **app_kwargs,
     ):
         from opentuner.measurement.inputmanager import FixedInputManager
         from opentuner.search.objective import ThresholdAccuracyMinimizeTime
@@ -217,6 +239,7 @@ class TunerInterface(MeasurementInterface):
         self.tune_thres = tuner_thres
         self.keep_thres = keep_thres
         self.pbar = tqdm(total=test_limit, leave=False)
+        self.app_kwargs = app_kwargs
 
         objective = ThresholdAccuracyMinimizeTime(tuner_thres)
         input_manager = FixedInputManager(size=len(self.app.op_knobs))

predtuner/modeledapp.py

 import abc
-from typing import Callable, Dict, List, Tuple, Union
+import logging
+from typing import Callable, Dict, List, Optional, Tuple, Type, Union
 
 import torch
 
-from .approxapp import ApproxApp, KnobsT
+from .approxapp import ApproxApp, ApproxTuner, Config, KnobsT
+
+msg_logger = logging.getLogger(__name__)
 
 
 class ModeledApp(ApproxApp, abc.ABC):
@@ -65,7 +68,7 @@ class ModeledApp(ApproxApp, abc.ABC):
                     f'"{qos_model}" is an invalid value for qos_model '
                     f"(choose from {list(self._qos_models.keys())})"
                 )
-            qos = self._qos_models[qos_model].measure_qos(with_approxes, is_calibration)
+            qos = self._qos_models[qos_model].measure_qos(with_approxes)
         # Same goes for perf
         if perf_model != "none":
             if perf_model not in self._perf_models:
@@ -73,10 +76,13 @@ class ModeledApp(ApproxApp, abc.ABC):
                     f'"{perf_model}" is an invalid value for perf_model '
                     f"(choose from {list(self._perf_models.keys())})"
                 )
-            perf = self._perf_models[perf_model].measure_perf(with_approxes, is_calibration)
+            perf = self._perf_models[perf_model].measure_perf(with_approxes)
         assert qos is not None and perf is not None
         return qos, perf
 
+    def get_tuner(self) -> "ApproxModeledTuner":
+        return ApproxModeledTuner(self)
+
 
 class IPerfModel(abc.ABC):
     """Abstract base class for models that provide performance prediction."""
@@ -163,7 +169,7 @@ class QoSModelP1(IQoSModel):
 
     def measure_qos(self, with_approxes: KnobsT) -> float:
         """Implementation of model."""
-        pass
+        return 0.0
 
 
 class QoSModelP2(IQoSModel):
@@ -189,4 +195,64 @@ class QoSModelP2(IQoSModel):
 
     def measure_qos(self, with_approxes: KnobsT) -> float:
         """Implementation of model."""
-        pass
+        return 0.0
+
+
+class ValConfig(Config):
+    def __init__(
+        self,
+        qos: float,
+        perf: float,
+        knobs: KnobsT,
+        calib_qos: Optional[float] = None,
+        validated_qos: Optional[float] = None,
+    ) -> None:
+        super().__init__(qos, perf, knobs, calib_qos)
+        self.validated_qos = validated_qos
+
+
+class ApproxModeledTuner(ApproxTuner):
+    def tune(
+        self,
+        max_iter: int,
+        qos_tuner_threshold: float,
+        qos_keep_threshold: Optional[float] = None,
+        is_threshold_relative: bool = False,
+        take_best_n: Optional[int] = None,
+        calibrate: bool = True,
+        validate: Optional[bool] = None,
+        perf_model: str = "none",
+        qos_model: str = "none",
+    ) -> List[ValConfig]:
+        ret = super().tune(
+            max_iter=max_iter,
+            qos_tuner_threshold=qos_tuner_threshold,
+            qos_keep_threshold=qos_keep_threshold,
+            is_threshold_relative=is_threshold_relative,
+            take_best_n=take_best_n,
+            calibrate=calibrate,
+            perf_model=perf_model,
+            qos_model=qos_model,
+        )
+        if validate is None:
+            validate = qos_model != "none"
+        if validate:
+            self.validate_configs_(self.best_configs)
+        return ret
+
+    def validate_configs_(self, configs: List[ValConfig]):
+        from tqdm import tqdm
+
+        for cfg in tqdm(configs, leave=False):
+            cfg: ValConfig
+            if cfg.validated_qos is not None:
+                continue
+            cfg.validated_qos, _ = self.app.measure_qos_perf(cfg.knobs, False)
+            msg_logger.debug(f"Validation: {cfg.qos} (mean) -> {cfg.calib_qos} (mean)")
+
+    def _get_app_kwargs(self, perf_model: str, qos_model: str):
+        return {"perf_model": perf_model, "qos_model": qos_model}
+
+    @classmethod
+    def _get_config_class(cls) -> Type[Config]:
+        return ValConfig

test/test_torchapp.py

@@ -55,6 +55,12 @@ class TestTorchAppTuning(TorchAppSetUp):
         if len(tuner.kept_configs) >= 10:
             self.assertEqual(len(tuner.best_configs), 10)
 
+    def test_enum_models(self):
+        self.assertSetEqual(
+            set(model.name for model in self.app.get_models()),
+            {"perf_linear", "qos_p1", "qos_p2"},
+        )
+
 
 class TestTorchAppTunerResult(TorchAppSetUp):
     @classmethod
@@ -80,3 +86,30 @@ class TestTorchAppTunerResult(TorchAppSetUp):
         configs = self.tuner.best_configs
         for c in configs:
             self.assertAlmostEqual(c.calib_qos, c.qos)
+
+
+class TestModeledTuning(TorchAppSetUp):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.baseline, _ = cls.app.measure_qos_perf({}, False)
+
+    def test_qos_p1(self):
+        tuner = self.app.get_tuner()
+        tuner.tune(
+            100,
+            3.0,
+            is_threshold_relative=True,
+            perf_model="perf_linear",
+            qos_model="qos_p1",
+        )
+
+    def test_qos_p2(self):
+        tuner = self.app.get_tuner()
+        tuner.tune(
+            100,
+            3.0,
+            is_threshold_relative=True,
+            perf_model="perf_linear",
+            qos_model="qos_p2",
+        )