Merge branch 'main' into temp

predtuner/torchapp.py

@@ -10,9 +10,9 @@ from torch.utils.data.dataloader import DataLoader
 from ._logging import PathLike
 from .approxapp import ApproxKnob, KnobsT
 from .modeledapp import (
-    IPerfModel,
+    ICostModel,
     IQoSModel,
-    LinearPerfModel,
+    LinearCostModel,
     ModeledApp,
     QoSModelP1,
     QoSModelP2,
@@ -34,15 +34,27 @@ class TorchApproxKnob(ApproxKnob):
     @property
     @abc.abstractmethod
     def expected_speedup(self) -> float:
+        """The speedup this knob is expected to provide. Used for cost prediction."""
         pass
 
     @abc.abstractmethod
     def is_applicable(self, op: Module) -> bool:
+        """Returns True if this knob can be applied to this Module.
+        
+        :param op: the module to check availability for.
+        :type op: torch.nn.Module
+        :rtype: torch.nn.Module
+        """
         pass
 
     @abc.abstractmethod
     def apply(self, op: Module) -> Module:
-        """Applies knob to `module` and returns an approximated `module`."""
+        """Applies knob to a Module and returns an approximated Module.
+        
+        :param op: the module to apply approximation on.
+        :type op: torch.nn.Module
+        :rtype: torch.nn.Module
+        """
         pass
 
 
@@ -53,40 +65,35 @@ class TorchApp(ModeledApp, abc.ABC):
     r"""Adaptor for approximable PyTorch Modules with tensor output.
 
     A TorchApp stores the PyTorch Module, datasets for tuning and calibration,
-    set of available TorchApproxKnob each of which may be applied to some layer in the Module,
+    set of available `TorchApproxKnob` each of which may be applied to some layer in the Module,
     and the quality of service (QoS) metric of application (e.g., accuracy).
-    It provides empirical tuning and predictive tuning capability (see `TorchApp.tune()`),
-
-    Parameters
-    ----------
-    app_name:
-        Name of the application, which is used as an identifier in tuning sessions, etc.
-    module:
-        The PyTorch module to tune.
-    tune_dataloader:
-        A dataset to use as inputs to module during tuning. (PyTorch DataLoader is conceptually
-        an enumerable, batched dataset.)
-    test_dataloader:
-        A input dataset used for QoS testing (see `test_config` parameter of `ApproxModeledTuner.tune`).
-    knobs:
-        A set of knobs to be considered. Each knob has an `is_applicable()` method
-        which is used to determine which layer it can apply to.
-    tensor_to_qos:
-        QoS metric function which computes QoS from the module's output.
-    combine_qos:
-        A function to combine each batch's QoS into one value.
-        When QoS is accuracy this will most likely be `mean()` (which is the default).
-    device:
-        The device to store module and perform inference on. By default is "cuda"
-        if CUDA is available, otherwise "cpu".
-    model_storage_folder:
-        A folder to store the serialized QoS models into.
-        `QoSModelP1` will be serialized into `model_storage_folder / "p1.pkl"`,
-        and `QoSModelP2` into `model_storage_folder / "p2.json"`.
-        See `QoSModelP1` and `QoSModelP2` for details.
-
-    Attributes
-    ----------
+    It provides empirical tuning and predictive tuning capability,
+    automatically supporting `.modeledapp.LinearCostModel`,
+    `.modeledapp.QoSModelP1`, and `.modeledapp.QoSModelP2`.
+
+    In contrast to `.approxapp.ApproxApp` and `.modeledapp.ModeledApp`,
+    there should be no need to inherit from `TorchApp` in most use cases.
+
+    :param app_name: Name of the application, which is used as an identifier in tuning sessions, etc.
+    :param module: The PyTorch module to tune.
+    :param tune_dataloader: A `torch.utils.data.Dataset` dataset to use as inputs to module during tuning.
+    :param test_dataloader: A `torch.utils.data.Dataset` dataset used for QoS testing
+           (see `test_configs` parameter of `ApproxModeledTuner.tune`).
+    :param knobs: A set of `TorchApproxKnob` to be considered. Each knob has an `is_applicable()` method
+           which is used to determine which layer it can apply to.
+           `.approxes.get_knobs_from_file` returns a set of builtin knobs that will exactly fit here.
+    :param tensor_to_qos: QoS metric function which computes QoS from the module's output.
+           `.torchutil.accuracy` computes the classification accuracy which can be applied here.
+    :param combine_qos: A function to combine each batch's QoS into one value.
+           When QoS is Classification Accuracy, this will most likely be `numpy.mean`
+           (which is the default value).
+    :param target_device: The target device that this application should be tuned on.
+    :param torch_device: The PyTorch device where the model inference is run on.
+           This device should be able to run the implementations of the knobs
+           available for this app on `target_device`.
+    :param model_storage_folder: A folder to store the serialized QoS models into.
+           `QoSModelP1` will be serialized into ``model_storage_folder / "p1.pkl"``,
+           and `QoSModelP2` into ``model_storage_folder / "p2.json"``.
     """
 
     def __init__(
@@ -98,7 +105,7 @@ class TorchApp(ModeledApp, abc.ABC):
         knobs: Set[TorchApproxKnob],
         tensor_to_qos: Callable[[torch.Tensor, Any], float],
         combine_qos: Callable[[np.ndarray], float] = np.mean,
-        tuning_device: str = None,
+        target_device: str = None,
         torch_device: Union[torch.device, str] = _default_device,
         model_storage_folder: Optional[PathLike] = None,
     ) -> None:
@@ -107,7 +114,7 @@ class TorchApp(ModeledApp, abc.ABC):
         self.tune_loader = tune_dataloader
         self.test_loader = test_dataloader
         self.name_to_knob = {
-            k.name: k for k in self._check_and_filter_knob(knobs, tuning_device)
+            k.name: k for k in self._check_and_filter_knob(knobs, target_device)
         }
         self.tensor_to_qos = tensor_to_qos
         self.combine_qos = combine_qos
@@ -132,17 +139,17 @@ class TorchApp(ModeledApp, abc.ABC):
             self._op_costs[op_name] = summary.loc[op_name, "flops"]
 
         # Init parent class last
-        super().__init__(op_knobs, tuning_device)
+        super().__init__(op_knobs, target_device)
 
     @property
     def name(self) -> str:
         """Returns the name of application."""
         return self.app_name
 
-    def get_models(self) -> List[Union[IPerfModel, IQoSModel]]:
+    def get_models(self) -> List[Union[ICostModel, IQoSModel]]:
         """Returns a list of predictive tuning models.
 
-        TorchApp in particular derives 1 performance model (LinearPerfModel)
+        TorchApp in particular derives 1 performance model (LinearCostModel)
         and 2 QoS models (QoSModelP1, QoSModelP2) automatically.
         """
 
@@ -162,7 +169,7 @@ class TorchApp(ModeledApp, abc.ABC):
         p1_storage = self.model_storage / "p1.pkl" if self.model_storage else None
         p2_storage = self.model_storage / "p2.json" if self.model_storage else None
         return [
-            LinearPerfModel(self, self._op_costs, self._knob_speedups),
+            LinearCostModel(self, self._op_costs, self._knob_speedups),
             QoSModelP1(
                 self, self._get_raw_output_valset, batched_valset_qos, p1_storage
             ),

predtuner/torchutil/common_qos.py

@@ -2,6 +2,12 @@ from torch import Tensor
 
 
 def accuracy(output: Tensor, target: Tensor) -> float:
+    """The "classification accuracy" metric (return value is between 0 and 100).
+    
+    :param output: Probability distribution output from the model.
+    :param target: A 1d-tensor of labels, one for each input image, from the dataset.
+    """
+
     _, pred_labels = output.max(1)
     n_correct = (pred_labels == target).sum().item()
     return n_correct / len(output) * 100

setup.py

 import setuptools
 
-with open("README.md", "r", encoding="utf-8") as fh:
+with open("README.rst", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 setuptools.setup(
@@ -10,14 +10,18 @@ setuptools.setup(
     author_email="yifanz16@illinois.edu",
     description="A package for predictive and empirical approximation autotuning",
     long_description=long_description,
-    long_description_content_type="text/markdown",
+    long_description_content_type="text/x-rst",
     url="https://github.com/Evan-Zhao/predictive-tuner",
-    packages=["predtuner"],
+    packages=setuptools.find_packages(),
+    package_data={
+        "predtuner.approxes": ["default_approx_params.json"]
+    },
+    include_package_data=True,
     install_requires=[
         "matplotlib>=3.3",
         "networkx>=2.5",
-        "torch==1.7.1",
-        "torchvision==0.8.2",
+        "torch>=1.5.1",
+        "torchvision>=0.6",
         "tqdm>=4.50",
         "pandas>=1.1",
         "jsonpickle>=1.5",

test/test_torchapp.py

@@ -41,7 +41,7 @@ class TestTorchAppTuning(TorchAppSetUp):
         self.assertEqual(self.app.baseline_knob.name, "11")
 
     def test_cpu_knobs(self):
-        app = TorchApp(**self.app_args, tuning_device="cpu")
+        app = TorchApp(**self.app_args, target_device="cpu")
         n_knobs = {op: len(ks) for op, ks in app.op_knobs.items()}
         for op_name, op in app.midx.name_to_module.items():
             nknob = 28 if isinstance(op, Conv2d) else 1
@@ -49,7 +49,7 @@ class TestTorchAppTuning(TorchAppSetUp):
         self.assertEqual(app.baseline_knob.name, "11")
 
     def test_gpu_knobs(self):
-        app = TorchApp(**self.app_args, tuning_device="gpu")
+        app = TorchApp(**self.app_args, target_device="gpu")
         n_knobs = {op: len(ks) for op, ks in app.op_knobs.items()}
         for op_name, op in app.midx.name_to_module.items():
             nknob = 28 if isinstance(op, Conv2d) else 1