Put down API design in code

notes.md

@@ -7,9 +7,7 @@
     - Useful when we want to support multiple knobs in an op
 
 - Application:
-  - List of operators (just a name) -- constant
-  - List of knobs (each with a name) -- constant
-  - List of knobs (by knob name) for each layer -- constant
+  - List of knobs for each operator -- constant
     - (We provide a knob value for "baseline" and sneak that in each layer)
   - Argparser extra arguments -- static method
   - How to measure QoS & performance given a configuration -- method

predtuner/apps/approxapp.py

+import abc
+from pathlib import Path
+from typing import Dict, List, Tuple, Union
+
+
+class ApproxKnob:
+    def __init__(self, name: str, **kwargs):
+        self.name = name
+        self.kwargs = kwargs
+
+    def coexists_with(self, other: "ApproxKnob") -> bool:
+        return False
+
+    def __repr__(self):
+        return f'Knob"{self.name}"({self.kwargs})'
+
+
+KnobsT = Dict[str, str]
+PathLike = Union[Path, str]
+
+
+class ApproxApp(abc.ABC):
+    """Generic approximable application with operator & knob enumeration,
+    and measures its own QoS and performance given a configuration."""
+
+    @property
+    @abc.abstractmethod
+    def op_knobs(self) -> Dict[str, List[ApproxKnob]]:
+        """Get a mapping from each operator (identified by str) to a list of applicable knobs."""
+        pass
+
+    @abc.abstractmethod
+    def measure_qos_perf(
+        self, with_approxes: KnobsT, is_testset: bool
+    ) -> Tuple[float, float]:
+        pass
+
+    def get_tuner(self) -> "ApproxTuner":
+        """We implement this function. Sets up an ApproxTuner instance
+        which the user can directly call `tune()` on with opentuner parameters."""
+        return ApproxTuner()  # TODO
+
+
+class Config:
+    pass  # TODO: work this out later
+
+
+class ApproxTuner:
+    def tune(
+        self,
+        qos_threshold: float,
+        accuracy_convention: str = "absolute",
+        **kwargs,  # many opentuner parameters with defaults, omitted
+    ):
+        """Generate an optimal set of approximation configurations for the model."""
+        pass  # TODO
+
+    # More helpers for selecting a config omitted for brevity
+
+    def get_all_configs(self) -> List[Config]:
+        return []  # TODO
+
+    # TODO
+    # Work out details of saving / loading
+    # Important to keep association between model, weights, and configs
+    # Especially when retraining is involved
+    def store_configs(self, path: PathLike):
+        pass
+
+    def load_configs(self, path: PathLike):
+        pass

predtuner/apps/modeledapp.py

+import abc
+from typing import Dict, Tuple
+
+import torch
+
+from .approxapp import ApproxApp, ApproxKnob, KnobsT
+
+
+class ModeledApp(ApproxApp, abc.ABC):
+    """Approximable application that inherits at least 1 interface for performance/QoS modeling.
+
+    It's invalid to inherit from this class without also implementing at least 1 interface
+    provided in this set of API;
+    for non-modeling application, inherit from `ApproxApp` instead.
+    """
+
+    @abc.abstractmethod
+    def measure_qos(self, with_approxes: KnobsT, is_testset: bool) -> float:
+        """User should fill in this hole if not using any QoS model.
+    Otherwise this function will not be called and can be empty."""
+        pass
+
+    @abc.abstractmethod
+    def measure_perf(self, with_approxes: KnobsT, is_testset: bool) -> float:
+        """User should fill in this hole if not using any performance model.
+    Otherwise this function will not be called and can be empty."""
+        pass
+
+    def measure_qos_perf(
+        self,
+        with_approxes: KnobsT,
+        is_testset: bool,
+        perf_model: str = "none",
+        qos_model: str = "none",
+    ) -> Tuple[float, float]:
+        """We provide this with the right qos and perf function.
+
+        Need to detect self capability using `isinstance(self, ...)`
+        and check input parameter, to decide which model to use.
+        The non-modeled part will be obtained by calling the respective
+        `measure_qos` and `measure_perf` function (a bit of dirty
+        dispatching work to do here).
+        """
+        pass
+
+
+class IPerfModeled(abc.ABC):
+    """Interface to be inherited by user App which allows performance to be model-derived."""
+
+    @property
+    @abc.abstractmethod
+    def op_knobs_cost(self) -> Dict[str, Dict[ApproxKnob, float]]:
+        """Get a scalar cost of each operator applied with each knob.
+        The ops and knobs listed here should be strictly equal to `ApproxApp.ops_knobs()`"""
+        pass
+
+    def measure_perf(self, with_approxes: KnobsT, is_testset: bool) -> float:
+        """We implement this using a weighted linear performance model."""
+        pass
+
+
+class IQoSModeledP1(abc.ABC):
+    """Interface that allows QoS model `P1` to be applied to user-defined App."""
+
+    @abc.abstractmethod
+    def get_tensor_output(
+        self, with_approxes: KnobsT, is_testset: bool
+    ) -> torch.Tensor:
+        """Run the tensor-based application with config `with_approxes` applied,
+        and return a single tensor result.
+
+        Note that while we require the return value to be a PyTorch tensor,
+        user is free to implement this on non-PyTorch applications.
+        """
+        pass
+
+    @abc.abstractmethod
+    def qos_from_output(self, tensor_output: torch.Tensor) -> float:
+        """Compute a Quality of Service level from the tensor output of application."""
+        pass
+
+    def measure_qos(self, with_approxes: KnobsT, is_testset: bool) -> float:
+        """We implement this using a QoS model P1."""
+        pass
+
+
+class IQoSModeledP2(abc.ABC):
+    """Interface that allows QoS model `P2` to be applied to user-defined App."""
+
+    @abc.abstractmethod
+    def _measure_qos(self, with_approxes: KnobsT, is_testset: bool) -> torch.Tensor:
+        """An internal QoS-measuring method that does the same thing as `measure_qos_p2`.
+
+        The point is P2 queries some QoS results and caches them before tuning starts,
+        and then defines a `measure_qos` that doesn't run the application during tuning
+        (to reduce overhead).
+        """
+        pass
+
+    def measure_qos(self, with_approxes: KnobsT, is_testset: bool) -> float:
+        """We implement this using a QoS model P1."""
+        pass
+

predtuner/apps/torchapp.py

+import abc
+from typing import Set
+
+from torch.utils.data.dataloader import DataLoader
+
+from .approxapp import ApproxKnob
+from .modeledapp import IPerfModeled, IQoSModeledP1, IQoSModeledP2, ModeledApp
+
+
+class TorchApproxKnob(ApproxKnob):
+    """Defines an approximation knob that knows
+    its own expected speedup ratio and what Modules it can apply to,
+    and can be applied to a torch.nn.Module to return an approximated Module."""
+
+    pass
+
+
+class TorchApp(ModeledApp, IPerfModeled, IQoSModeledP1, IQoSModeledP2, abc.ABC):
+    """Approximable PyTorch Modules (tensor output assumed).
+  
+    Automatically derives performance model and QoS models P1&P2."""
+
+    @property
+    @abc.abstractmethod
+    def all_knobs(self) -> Set[TorchApproxKnob]:
+        """User defines a set of all knobs available; we'll dispatch them to each layer (op)."""
+        pass
+
+    @abc.abstractmethod
+    def get_input_data(self, testset: bool) -> DataLoader:
+        """User defines the input dataset to traverse."""
+        pass
+
+    # User also needs to define `IQoSModeledP1.qos_from_output` (QoS metric, omitted)
+
+    # We implement `ApproxApp.op_knobs`,
+    # `IPerfModeled.op_knobs_cost`,
+    # `IQoSModeledP1.get_tensor_output`
+    # and `IQoSModeledP2._measure_qos`. (Omitted)