From fe456002e20bdbe68a065637c70c6992f8b3e5bb Mon Sep 17 00:00:00 2001
From: Yifan Zhao <yifanz16@illinois.edu>
Date: Sat, 23 Jan 2021 23:34:30 -0600
Subject: [PATCH] Put up framework for modeled tuning
---
predtuner/approxapp.py | 101 ++++++++++++++++++++++++----------------
predtuner/modeledapp.py | 78 ++++++++++++++++++++++++++++---
test/test_torchapp.py | 33 +++++++++++++
3 files changed, 167 insertions(+), 45 deletions(-)
diff --git a/predtuner/approxapp.py b/predtuner/approxapp.py
index cd24abc..fbc7eda 100644
--- a/predtuner/approxapp.py
+++ b/predtuner/approxapp.py
@@ -1,13 +1,13 @@
import abc
import logging
from pathlib import Path
-from typing import Dict, Generic, List, NamedTuple, Optional, Tuple, TypeVar, Union
+from typing import Dict, Generic, List, Optional, Tuple, Type, TypeVar, Union
import matplotlib.pyplot as plt
import numpy as np
from opentuner.measurement.interface import MeasurementInterface
-from opentuner.resultsdb.models import Configuration, Result
-from opentuner.search.manipulator import ConfigurationManipulator, EnumParameter
+from opentuner.search.manipulator import (ConfigurationManipulator,
+ EnumParameter)
from ._logging import override_opentuner_config
from ._pareto import is_pareto_efficient
@@ -61,32 +61,33 @@ class ApproxApp(abc.ABC):
class Config:
def __init__(
- self, qos: float, calib_qos: Optional[float], perf: float, knobs: KnobsT
+ self, qos: float, perf: float, knobs: KnobsT, calib_qos: Optional[float] = None
) -> None:
self.qos = qos
- self.calib_qos = calib_qos
self.perf = perf
self.knobs = knobs
+ self.calib_qos: Optional[float] = calib_qos
T = TypeVar("T", bound=Config)
-# ApproxTuner is generic over the type of the config
+# IOpenTuner is generic over the type of the config
# So that the user can use custom Config inherited from Config
# (in which case they need to override `get_all_configs_from_db`).
class ApproxTuner(Generic[T]):
def __init__(self, app: ApproxApp) -> None:
self.app = app
+ self._tuned = False
self.all_configs = []
self.kept_configs = []
self.best_configs = []
+ # The following will be filled after self.tune() is called
self.keep_threshold = None
- self._db = None
@property
def tuned(self) -> bool:
- return not self._db is None
+ return self._tuned
def tune(
self,
@@ -95,54 +96,44 @@ class ApproxTuner(Generic[T]):
qos_keep_threshold: Optional[float] = None,
is_threshold_relative: bool = False,
take_best_n: Optional[int] = None,
- calibrate: bool = True
+ calibrate: bool = True,
+ **kwargs
# TODO: more parameters + opentuner param forwarding
- ) -> List[Config]:
- """Generate an optimal set of approximation configurations for the model."""
+ ) -> List[T]:
from opentuner.tuningrunmain import TuningRunMain
from ._dbloader import read_opentuner_db
- # By default, keep_threshold == tuner_threshold
opentuner_args = opentuner_default_args()
- qos_keep_threshold = qos_keep_threshold or qos_tuner_threshold
- if is_threshold_relative:
- baseline_qos, _ = self.app.measure_qos_perf({}, False)
- qos_tuner_threshold = baseline_qos - qos_tuner_threshold
- qos_keep_threshold = baseline_qos - qos_keep_threshold
- opentuner_args.test_limit = max_iter
- tuner = TunerInterface(
- opentuner_args, self.app, qos_tuner_threshold, qos_keep_threshold, max_iter,
+ tuner = self._get_tuner_interface(
+ opentuner_args,
+ max_iter,
+ qos_tuner_threshold,
+ qos_keep_threshold,
+ is_threshold_relative,
+ self._get_app_kwargs(**kwargs),
)
+ assert self.keep_threshold is not None
trm = TuningRunMain(tuner, opentuner_args)
# TuningRunMain.__init__ initializes its own logger, so we'll override it and use ours
override_opentuner_config()
# This is where opentuner runs
trm.main()
-
# Parse and store results
- self._db = opentuner_args.database
- self.keep_threshold = qos_keep_threshold
- self.all_configs = list(
- self.get_all_configs_from_db(read_opentuner_db(self._db))
- )
+ self._tuned = True
+ config_ty = self._get_config_class()
+ self.all_configs = [
+ config_ty(result.accuracy, result.time, configuration.data)
+ for result, configuration in read_opentuner_db(opentuner_args.database)
+ ]
self.kept_configs = [
- cfg for cfg in self.all_configs if cfg.qos > qos_keep_threshold
+ cfg for cfg in self.all_configs if cfg.qos > self.keep_threshold
]
self.best_configs = self.take_best_configs(self.kept_configs, take_best_n)
if calibrate:
self.calibrate_configs_(self.best_configs)
return self.best_configs
- @classmethod
- def get_all_configs_from_db(
- cls, results_configs: List[Tuple[Result, Configuration]]
- ) -> Tuple[T]:
- return tuple(
- Config(result.accuracy, None, result.time, configuration.data)
- for result, configuration in results_configs
- )
-
def calibrate_configs_(self, configs: List[T]):
from tqdm import tqdm
@@ -154,9 +145,7 @@ class ApproxTuner(Generic[T]):
msg_logger.debug(f"Calibration: {cfg.qos} (mean) -> {cfg.calib_qos} (mean)")
@staticmethod
- def take_best_configs(
- configs: List[Config], n: Optional[int] = None
- ) -> List[Config]:
+ def take_best_configs(configs: List[T], n: Optional[int] = None) -> List[T]:
points = np.array([[c.perf, c.qos] for c in configs])
taken_idx = is_pareto_efficient(points, take_n=n)
return [configs[i] for i in taken_idx]
@@ -193,6 +182,38 @@ class ApproxTuner(Generic[T]):
ax.set_ylabel("speedup")
return fig
+ def _get_tuner_interface(
+ self,
+ opentuner_args,
+ max_iter: int,
+ qos_tuner_threshold: float,
+ qos_keep_threshold: Optional[float],
+ is_threshold_relative: bool,
+ app_kwargs: dict,
+ ) -> "TunerInterface":
+ # By default, keep_threshold == tuner_threshold
+ self.keep_threshold = qos_keep_threshold or qos_tuner_threshold
+ if is_threshold_relative:
+ baseline_qos, _ = self.app.measure_qos_perf({}, False)
+ qos_tuner_threshold = baseline_qos - qos_tuner_threshold
+ self.keep_threshold = baseline_qos - self.keep_threshold
+ opentuner_args.test_limit = max_iter
+ return TunerInterface(
+ opentuner_args,
+ self.app,
+ qos_tuner_threshold,
+ self.keep_threshold,
+ max_iter,
+ **app_kwargs,
+ )
+
+ def _get_app_kwargs(self, **kwargs):
+ return {}
+
+ @classmethod
+ def _get_config_class(cls) -> Type[Config]:
+ return Config
+
def opentuner_default_args():
from opentuner import default_argparser
@@ -208,6 +229,7 @@ class TunerInterface(MeasurementInterface):
tuner_thres: float,
keep_thres: float,
test_limit: int,
+ **app_kwargs,
):
from opentuner.measurement.inputmanager import FixedInputManager
from opentuner.search.objective import ThresholdAccuracyMinimizeTime
@@ -217,6 +239,7 @@ class TunerInterface(MeasurementInterface):
self.tune_thres = tuner_thres
self.keep_thres = keep_thres
self.pbar = tqdm(total=test_limit, leave=False)
+ self.app_kwargs = app_kwargs
objective = ThresholdAccuracyMinimizeTime(tuner_thres)
input_manager = FixedInputManager(size=len(self.app.op_knobs))
diff --git a/predtuner/modeledapp.py b/predtuner/modeledapp.py
index 6745999..d5f3873 100644
--- a/predtuner/modeledapp.py
+++ b/predtuner/modeledapp.py
@@ -1,9 +1,12 @@
import abc
-from typing import Callable, Dict, List, Tuple, Union
+import logging
+from typing import Callable, Dict, List, Optional, Tuple, Type, Union
import torch
-from .approxapp import ApproxApp, KnobsT
+from .approxapp import ApproxApp, ApproxTuner, Config, KnobsT
+
+msg_logger = logging.getLogger(__name__)
class ModeledApp(ApproxApp, abc.ABC):
@@ -65,7 +68,7 @@ class ModeledApp(ApproxApp, abc.ABC):
f'"{qos_model}" is an invalid value for qos_model '
f"(choose from {list(self._qos_models.keys())})"
)
- qos = self._qos_models[qos_model].measure_qos(with_approxes, is_calibration)
+ qos = self._qos_models[qos_model].measure_qos(with_approxes)
# Same goes for perf
if perf_model != "none":
if perf_model not in self._perf_models:
@@ -73,10 +76,13 @@ class ModeledApp(ApproxApp, abc.ABC):
f'"{perf_model}" is an invalid value for perf_model '
f"(choose from {list(self._perf_models.keys())})"
)
- perf = self._perf_models[perf_model].measure_perf(with_approxes, is_calibration)
+ perf = self._perf_models[perf_model].measure_perf(with_approxes)
assert qos is not None and perf is not None
return qos, perf
+ def get_tuner(self) -> "ApproxModeledTuner":
+ return ApproxModeledTuner(self)
+
class IPerfModel(abc.ABC):
"""Abstract base class for models that provide performance prediction."""
@@ -163,7 +169,7 @@ class QoSModelP1(IQoSModel):
def measure_qos(self, with_approxes: KnobsT) -> float:
"""Implementation of model."""
- pass
+ return 0.0
class QoSModelP2(IQoSModel):
@@ -189,4 +195,64 @@ class QoSModelP2(IQoSModel):
def measure_qos(self, with_approxes: KnobsT) -> float:
"""Implementation of model."""
- pass
+ return 0.0
+
+
+class ValConfig(Config):
+ def __init__(
+ self,
+ qos: float,
+ perf: float,
+ knobs: KnobsT,
+ calib_qos: Optional[float] = None,
+ validated_qos: Optional[float] = None,
+ ) -> None:
+ super().__init__(qos, perf, knobs, calib_qos)
+ self.validated_qos = validated_qos
+
+
+class ApproxModeledTuner(ApproxTuner):
+ def tune(
+ self,
+ max_iter: int,
+ qos_tuner_threshold: float,
+ qos_keep_threshold: Optional[float] = None,
+ is_threshold_relative: bool = False,
+ take_best_n: Optional[int] = None,
+ calibrate: bool = True,
+ validate: Optional[bool] = None,
+ perf_model: str = "none",
+ qos_model: str = "none",
+ ) -> List[ValConfig]:
+ ret = super().tune(
+ max_iter=max_iter,
+ qos_tuner_threshold=qos_tuner_threshold,
+ qos_keep_threshold=qos_keep_threshold,
+ is_threshold_relative=is_threshold_relative,
+ take_best_n=take_best_n,
+ calibrate=calibrate,
+ perf_model=perf_model,
+ qos_model=qos_model,
+ )
+ if validate is None:
+ validate = qos_model != "none"
+ if validate:
+ self.validate_configs_(self.best_configs)
+ return ret
+
+ def validate_configs_(self, configs: List[ValConfig]):
+ from tqdm import tqdm
+
+ for cfg in tqdm(configs, leave=False):
+ cfg: ValConfig
+ if cfg.validated_qos is not None:
+ continue
+ cfg.validated_qos, _ = self.app.measure_qos_perf(cfg.knobs, False)
+ msg_logger.debug(f"Validation: {cfg.qos} (mean) -> {cfg.calib_qos} (mean)")
+
+ def _get_app_kwargs(self, perf_model: str, qos_model: str):
+ return {"perf_model": perf_model, "qos_model": qos_model}
+
+ @classmethod
+ def _get_config_class(cls) -> Type[Config]:
+ return ValConfig
diff --git a/test/test_torchapp.py b/test/test_torchapp.py
index d2752d7..98b941b 100644
--- a/test/test_torchapp.py
+++ b/test/test_torchapp.py
@@ -55,6 +55,12 @@ class TestTorchAppTuning(TorchAppSetUp):
if len(tuner.kept_configs) >= 10:
self.assertEqual(len(tuner.best_configs), 10)
+ def test_enum_models(self):
+ self.assertSetEqual(
+ set(model.name for model in self.app.get_models()),
+ {"perf_linear", "qos_p1", "qos_p2"},
+ )
+
class TestTorchAppTunerResult(TorchAppSetUp):
@classmethod
@@ -80,3 +86,30 @@ class TestTorchAppTunerResult(TorchAppSetUp):
configs = self.tuner.best_configs
for c in configs:
self.assertAlmostEqual(c.calib_qos, c.qos)
+
+
+class TestModeledTuning(TorchAppSetUp):
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ cls.baseline, _ = cls.app.measure_qos_perf({}, False)
+
+ def test_qos_p1(self):
+ tuner = self.app.get_tuner()
+ tuner.tune(
+ 100,
+ 3.0,
+ is_threshold_relative=True,
+ perf_model="perf_linear",
+ qos_model="qos_p1",
+ )
+
+ def test_qos_p2(self):
+ tuner = self.app.get_tuner()
+ tuner.tune(
+ 100,
+ 3.0,
+ is_threshold_relative=True,
+ perf_model="perf_linear",
+ qos_model="qos_p2",
+ )
--
GitLab