diff --git a/.gitignore b/.gitignore
index a298af1e5435b5e90f256adaecd94c5af1a51261..99a234f7db027ae66da39cc04a9f6f3abdfd2d8c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,4 +27,5 @@ tuner_results
tuner_results/
*.sh
*.ipynb
-logistics/
\ No newline at end of file
+logistics/
+networkx_doc/
\ No newline at end of file
diff --git a/doc/Makefile b/doc/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..b7000de4853b1c4a9f75cdc1ced9f351ab02284d
--- /dev/null
+++ b/doc/Makefile
@@ -0,0 +1,109 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest epub
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " pickle to make pickle files"
+ @echo " epub to make an epub"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+ @echo " gitwash to update the gitwash documentation"
+
+
+clean:
+ -rm -rf build/*
+
+dist: html
+ test -d build/latex || make latex
+ make -C build/latex all-pdf
+ -rm -rf build/dist
+ (cd build/html; cp -r . ../../build/dist)
+ (cd build/dist && tar czf ../dist.tar.gz .)
+
+html:
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html
+ @echo
+ @echo "Build finished. The HTML pages are in build/html."
+
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) build/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in build/dirhtml."
+
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) build/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) build/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) build/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in build/htmlhelp."
+
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) build/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in build/qthelp, like this:"
+ @echo "# qcollectiongenerator build/qthelp/test.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile build/qthelp/test.qhc"
+
+epub:
+ $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) build/epub
+ @echo
+ @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+
+latex:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) build/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in build/latex."
+ @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
+ "run these through (pdf)latex."
+
+changes:
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) build/changes
+ @echo
+ @echo "The overview file is in build/changes."
+
+linkcheck:
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) build/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in build/linkcheck/output.txt."
+
+doctest:
+ $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) build/doctest
+ @echo "Testing of doctests in the sources finished, look at the " \
+ "results in build/doctest/output.txt."
+
+latexpdf: latex
+ @echo "Running LaTeX files through latexmk..."
+ $(MAKE) -C build/latex all-pdf
+ @echo "latexmk finished; the PDF files are in build/latex."
diff --git a/doc/README.md b/doc/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..5e0af1e3f90895ee5cdbc2927ccedf8970053c32
--- /dev/null
+++ b/doc/README.md
@@ -0,0 +1,27 @@
+# Building docs
+
+We use Sphinx for generating the API and reference documentation.
+## Instructions
+
+Install the following Python packages needed to build the documentation by entering:
+
+```bash
+pip install sphinx sphinx-autodoc-typehints sphinx-rtd-theme
+```
+
+To build the HTML documentation, enter::
+
+```bash
+make html
+```
+
+in the ``doc/`` directory. This will generate a ``build/html`` subdirectory
+containing the built documentation.
+
+To build the PDF documentation, enter::
+
+```bash
+make latexpdf
+```
+
+You will need to have LaTeX installed for this.
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..12f28f5c0af5b4ba58545c094c660337223233cf
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,154 @@
+from datetime import date
+import sphinx_rtd_theme
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+# General configuration
+# ---------------------
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [
+ "sphinx.ext.autosummary",
+ "sphinx.ext.autodoc",
+ "sphinx_autodoc_typehints",
+ "sphinx.ext.coverage",
+ "sphinx.ext.doctest",
+ "sphinx.ext.intersphinx",
+ "sphinx.ext.mathjax",
+ "sphinx.ext.todo",
+ "sphinx.ext.viewcode",
+ "numpydoc",
+]
+always_document_param_types = True
+
+# generate autosummary pages
+autosummary_generate = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix of source filenames.
+source_suffix = ".rst"
+
+# The encoding of source files.
+source_encoding = "utf-8"
+
+# The master toctree document.
+master_doc = "index"
+
+# General substitutions.
+project = "PredTuner"
+copyright = f"2020-{date.today().year}, University of Illinois"
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+# unused_docs = ['']
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+add_module_names = False
+
+# show_authors = True
+
+# The name of the Pygments (syntax highlighting) style to use.
+# pygments_style = 'friendly'
+pygments_style = "sphinx"
+
+# A list of prefixs that are ignored when creating the module index. (new in Sphinx 0.6)
+# modindex_common_prefix = ["networkx."]
+
+# doctest_global_setup = "import networkx as nx"
+
+# Options for HTML output
+# -----------------------
+
+
+html_theme = "sphinx_rtd_theme"
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+html_theme_options = {
+ "canonical_url": "https://networkx.org/documentation/stable/",
+ "navigation_depth": 3,
+ "logo_only": True,
+}
+
+# html_logo = "_static/networkx_logo.svg"
+
+# The style sheet to use for HTML and HTML Help pages. A file of that name
+# must exist either in Sphinx' static/ path, or in one of the custom paths
+# given in html_static_path.
+# html_style = ''
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = "%b %d, %Y"
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Content template for the index page.
+# html_index = 'index.html'
+
+# Custom sidebar templates, maps page names to templates.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# templates.
+# html_additional_pages = {'': ''}
+
+# If true, the reST sources are included in the HTML build as _sources/<name>.
+html_copy_source = False
+
+# Options for LaTeX output
+# ------------------------
+
+# Use a latex engine that allows for unicode characters in docstrings
+latex_engine = "xelatex"
+# The paper size ('letter' or 'a4').
+latex_paper_size = "letter"
+
+# The font size ('10pt', '11pt' or '12pt').
+# latex_font_size = '10pt'
+
+latex_appendices = ["tutorial"]
+
+# Intersphinx mapping
+intersphinx_mapping = {
+ "python": ("https://docs.python.org/3/", None),
+ "numpy": ("https://numpy.org/doc/stable/", None),
+ "matplotlib": ("https://matplotlib.org", None),
+ "scipy": ("https://docs.scipy.org/doc/scipy/reference", None),
+ "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None),
+ "pytorch": ("https://pytorch.org/docs/stable", None),
+}
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+default_role = "obj"
+
+numpydoc_show_class_members = False
+
+
+def setup(app):
+ app.add_css_file("custom.css")
+ app.add_js_file("copybutton.js")
diff --git a/doc/getting_started.rst b/doc/getting_started.rst
new file mode 100644
index 0000000000000000000000000000000000000000..75381aa720fdee717d20db115ecbc74e73deff4c
--- /dev/null
+++ b/doc/getting_started.rst
@@ -0,0 +1,2 @@
+Getting Started
+===================
diff --git a/doc/index.rst b/doc/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..6866a98b3d779063329219cd2c49f8b2c6feb41d
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1,46 @@
+.. _contents:
+
+PredTuner
+=============================
+
+PredTuner is a Python library for predictive approximation autotuning.
+
+PredTuner performs autotuning on approximation choices for a program
+using an error-predictive proxy instead of executing the program,
+to greatly speedup autotuning while getting results of comparable quality.
+
+PredTuner is a contribution of [ApproxTuner]
+(https://ppopp21.sigplan.org/details/PPoPP-2021-main-conference/41/ApproxTuner-A-Compiler-and-Runtime-System-for-Adaptive-Approximations).
+
+Short-term Goals
+- Measure accuracy impact of approximations
+- Obtain a tuned, approximated CNN in <5 lines of code
+- Easy to manage multiple approximation configs
+- Easy to load and manage prior tuning results
+- Flexible retraining support
+Possible Long-term Goals
+- High-performance implementations of approximate layers
+- Allow users to register their own approximations
+- Support for other frameworks: TF, ONNX, JAX
+
+Documentation
+-------------
+
+.. only:: html
+
+ :Release: |version|
+ :Date: |today|
+
+.. toctree::
+ :maxdepth: 1
+
+ getting_started
+ reference/index
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+* :ref:`glossary`
diff --git a/doc/reference/index.rst b/doc/reference/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..2a3a0a68b70eb678e956900f9a7291e11bc8fbe6
--- /dev/null
+++ b/doc/reference/index.rst
@@ -0,0 +1,11 @@
+PyTorch Autotuning API
+======================
+
+.. autoclass:: predtuner.torchapp.TorchApp
+ :members:
+ :undoc-members:
+
+.. autoclass:: predtuner.modeledapp.ApproxModeledTuner
+ :members:
+ :inherited-members:
+ :undoc-members:
diff --git a/predtuner/torchapp.py b/predtuner/torchapp.py
index 0160c0cc14950abe0df068d84934c76224ca0939..3ad4f8768a961c07e87064fc1ebf164b0d606a1f 100644
--- a/predtuner/torchapp.py
+++ b/predtuner/torchapp.py
@@ -44,10 +44,43 @@ _default_device = f"cuda" if torch.cuda.is_available() else "cpu"
class TorchApp(ModeledApp, abc.ABC):
- """Approximable PyTorch Modules (tensor output assumed).
- Automatically derives performance model and QoS models P1&P2.
-
- knobs: User defines a set of all knobs available; we'll dispatch them to each layer (op).
+ r"""Adaptor for approximable PyTorch Modules with tensor output.
+
+ A TorchApp stores the PyTorch Module, datasets for tuning and calibration,
+ set of available TorchApproxKnob each of which may be applied to some layer in the Module,
+ and the quality of service (QoS) metric of application (e.g., accuracy).
+ It provides empirical tuning and predictive tuning capability (see `TorchApp.tune()`),
+
+ Parameters
+ ----------
+ app_name:
+ Name of the application, which is used as an identifier in tuning sessions, etc.
+ module:
+ The PyTorch module to tune.
+ tune_dataloader:
+ A dataset to use as inputs to module during tuning. (PyTorch DataLoader is conceptually
+ an enumerable, batched dataset.)
+ test_dataloader:
+ A input dataset used for QoS testing (see `test_config` parameter of `ApproxModeledTuner.tune`).
+ knobs:
+ A set of knobs to be considered. Each knob has an `is_applicable()` method
+ which is used to determine which layer it can apply to.
+ tensor_to_qos:
+ QoS metric function which computes QoS from the module's output.
+ combine_qos:
+ A function to combine each batch's QoS into one value.
+ When QoS is accuracy this will most likely be `mean()` (which is the default).
+ device:
+ The device to store module and perform inference on. By default is "cuda"
+ if CUDA is available, otherwise "cpu".
+ model_storage_folder:
+ A folder to store the serialized QoS models into.
+ `QoSModelP1` will be serialized into `model_storage_folder / "p1.pkl"`,
+ and `QoSModelP2` into `model_storage_folder / "p2.json"`.
+ See `QoSModelP1` and `QoSModelP2` for details.
+
+ Attributes
+ ----------
"""
def __init__(
@@ -60,7 +93,7 @@ class TorchApp(ModeledApp, abc.ABC):
tensor_to_qos: Callable[[torch.Tensor, Any], float],
combine_qos: Callable[[np.ndarray], float] = np.mean,
device: Union[torch.device, str] = _default_device,
- model_storage_folder: Optional[PathLike] = None
+ model_storage_folder: Optional[PathLike] = None,
) -> None:
self.app_name = app_name
self.module = module
@@ -70,7 +103,9 @@ class TorchApp(ModeledApp, abc.ABC):
self.tensor_to_qos = tensor_to_qos
self.combine_qos = combine_qos
self.device = device
- self.model_storage = Path(model_storage_folder) if model_storage_folder else None
+ self.model_storage = (
+ Path(model_storage_folder) if model_storage_folder else None
+ )
self.module = self.module.to(device)
self.midx = ModuleIndexer(module)
@@ -92,13 +127,21 @@ class TorchApp(ModeledApp, abc.ABC):
@property
def name(self) -> str:
+ """Returns the name of application."""
return self.app_name
@property
def op_knobs(self) -> Dict[str, List[ApproxKnob]]:
+ """Returns a list of applicable knobs for each operator (layer) in module."""
return self._op_knobs
def get_models(self) -> List[Union[IPerfModel, IQoSModel]]:
+ """Returns a list of predictive tuning models.
+
+ TorchApp in particular derives 1 performance model (LinearPerfModel)
+ and 2 QoS models (QoSModelP1, QoSModelP2) automatically.
+ """
+
def batched_valset_qos(tensor_output: torch.Tensor):
dataset_len = len(self.tune_loader.dataset)
assert len(tensor_output) == dataset_len
@@ -115,7 +158,9 @@ class TorchApp(ModeledApp, abc.ABC):
p2_storage = self.model_storage / "p2.json" if self.model_storage else None
return [
LinearPerfModel(self._op_costs, self._knob_speedups),
- QoSModelP1(self, self._get_raw_output_valset, batched_valset_qos, p1_storage),
+ QoSModelP1(
+ self, self._get_raw_output_valset, batched_valset_qos, p1_storage
+ ),
QoSModelP2(self, p2_storage),
]
@@ -123,6 +168,9 @@ class TorchApp(ModeledApp, abc.ABC):
def empirical_measure_qos_perf(
self, with_approxes: KnobsT, is_test: bool
) -> Tuple[float, float]:
+ """Measure the QoS and performance of Module with given approximation
+ empirically (i.e., by running the Module on the dataset)."""
+
from time import time_ns
dataloader = self.test_loader if is_test else self.tune_loader