diff --git a/doc/README.md b/doc/README.md
index 5e0af1e3f90895ee5cdbc2927ccedf8970053c32..cd9b7b68846b9c164978a10f8e8ea9bc9d846ab1 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -6,7 +6,7 @@ We use Sphinx for generating the API and reference documentation.
Install the following Python packages needed to build the documentation by entering:
```bash
-pip install sphinx sphinx-autodoc-typehints sphinx-rtd-theme
+pip install -r requirements.txt
```
To build the HTML documentation, enter::
diff --git a/doc/conf.py b/doc/conf.py
index ad8cee166b4df9acec13bca5dd73eccd4db31a9e..8fe20cee8424b1d12a07e7426885920c3dad13da 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -1,14 +1,13 @@
from datetime import date
-import sphinx_rtd_theme
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
-import os
import sys
-
-sys.path.insert(0, os.path.abspath(".."))
+from pathlib import Path
+this_folder = Path(__file__).parent
+sys.path.insert(0, (this_folder / "..").absolute().as_posix())
# General configuration
# ---------------------
@@ -18,16 +17,15 @@ sys.path.insert(0, os.path.abspath(".."))
extensions = [
"sphinx.ext.autosummary",
"sphinx.ext.autodoc",
- "sphinx_autodoc_typehints",
"sphinx.ext.coverage",
"sphinx.ext.doctest",
"sphinx.ext.intersphinx",
"sphinx.ext.mathjax",
"sphinx.ext.todo",
"sphinx.ext.viewcode",
- "numpydoc",
]
-always_document_param_types = True
+
+autodoc_typehints = "description"
# generate autosummary pages
autosummary_generate = True
@@ -48,48 +46,27 @@ master_doc = "index"
project = "PredTuner"
copyright = f"2020-{date.today().year}, University of Illinois"
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-# today = ''
-# Else, today_fmt is used as the format for a strftime call.
-# today_fmt = '%B %d, %Y'
-
-# List of documents that shouldn't be included in the build.
-# unused_docs = ['']
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-# add_function_parentheses = True
-
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = False
-# show_authors = True
-
# The name of the Pygments (syntax highlighting) style to use.
# pygments_style = 'friendly'
pygments_style = "sphinx"
# A list of prefixs that are ignored when creating the module index. (new in Sphinx 0.6)
-# modindex_common_prefix = ["networkx."]
-
-# doctest_global_setup = "import networkx as nx"
+# modindex_common_prefix = []
# Options for HTML output
# -----------------------
-
-html_theme = "sphinx_rtd_theme"
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
+html_theme = "pydata_sphinx_theme"
html_theme_options = {
- "canonical_url": "https://networkx.org/documentation/stable/",
- "navigation_depth": 3,
- "logo_only": True,
+ # "github_url": "https://gitlab.engr.illinois.edu/llvm/hpvm-beta",
+ "show_prev_next": False,
+ "search_bar_position": "sidebar",
}
-# html_logo = "_static/networkx_logo.svg"
-
# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
@@ -98,26 +75,12 @@ html_theme_options = {
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["_static"]
+html_static_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = "%b %d, %Y"
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-# html_use_smartypants = True
-
-# Content template for the index page.
-# html_index = 'index.html'
-
-# Custom sidebar templates, maps page names to templates.
-# html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# templates.
-# html_additional_pages = {'': ''}
-
# If true, the reST sources are included in the HTML build as _sources/<name>.
html_copy_source = False
@@ -129,9 +92,6 @@ latex_engine = "xelatex"
# The paper size ('letter' or 'a4').
latex_paper_size = "letter"
-# The font size ('10pt', '11pt' or '12pt').
-# latex_font_size = '10pt'
-
latex_appendices = ["tutorial"]
# Intersphinx mapping
@@ -147,10 +107,3 @@ intersphinx_mapping = {
# The reST default role (used for this markup: `text`) to use for all
# documents.
default_role = "obj"
-
-numpydoc_show_class_members = False
-
-
-def setup(app):
- app.add_css_file("custom.css")
- app.add_js_file("copybutton.js")
diff --git a/doc/reference/approx-app.rst b/doc/reference/approx-app.rst
new file mode 100644
index 0000000000000000000000000000000000000000..20d6e321b7526cad7062c90687073b33d569e76e
--- /dev/null
+++ b/doc/reference/approx-app.rst
@@ -0,0 +1,14 @@
+General Application Autotuning API
+==================================
+
+.. autoclass:: predtuner.approxapp.ApproxApp
+ :members:
+
+.. autoclass:: predtuner.approxapp.ApproxTuner
+ :members:
+
+.. autoclass:: predtuner.approxapp.ApproxKnob
+ :members:
+
+.. autoclass:: predtuner.approxapp.Config
+ :members:
diff --git a/doc/reference/index.rst b/doc/reference/index.rst
index 2a3a0a68b70eb678e956900f9a7291e11bc8fbe6..e94a2e79da525f24a8fe700a6efb140e963ec8d8 100644
--- a/doc/reference/index.rst
+++ b/doc/reference/index.rst
@@ -1,11 +1,15 @@
-PyTorch Autotuning API
-======================
+PredTuner Autotuning API
+========================
-.. autoclass:: predtuner.torchapp.TorchApp
- :members:
- :undoc-members:
+:doc:`pytorch-app` documents a high-level API for autotuning PyTorch Module.
-.. autoclass:: predtuner.modeledapp.ApproxModeledTuner
- :members:
- :inherited-members:
- :undoc-members:
+`predtuner` also supports predictive tuning of general applications that are not PyTorch Module,
+or even empirical tuning of general application that doesn't support predictive models.
+These lower-level APIs are documented in :doc:`modeled-app` and :doc:`approx-app` respectively.
+
+.. toctree::
+ :maxdepth: 1
+
+ pytorch-app
+ modeled-app
+ approx-app
diff --git a/doc/reference/modeled-app.rst b/doc/reference/modeled-app.rst
new file mode 100644
index 0000000000000000000000000000000000000000..458ffc38bb158559b2e75d0deacdec8e61effe19
--- /dev/null
+++ b/doc/reference/modeled-app.rst
@@ -0,0 +1,41 @@
+Predictive (Modeled) Autotuning API
+===================================
+
+.. autoclass:: predtuner.modeledapp.ModeledApp
+ :show-inheritance:
+ :members:
+
+.. autoclass:: predtuner.modeledapp.ApproxModeledTuner
+ :show-inheritance:
+ :members:
+
+.. autoclass:: predtuner.modeledapp.ValConfig
+ :show-inheritance:
+ :members:
+
+Predictive Model Interface
+----------------------------
+
+.. autoclass:: predtuner.modeledapp.IQoSModel
+ :members:
+
+.. autoclass:: predtuner.modeledapp.ICostModel
+ :members:
+
+Predefined Predictive Models
+----------------------------
+
+Below is a list of cost and QoS models already defined:
+
+* `predtuner.modeledapp.LinearCostModel`
+* `predtuner.modeledapp.QoSModelP1`
+* `predtuner.modeledapp.QoSModelP2`
+
+.. autoclass:: predtuner.modeledapp.LinearCostModel
+ :show-inheritance:
+
+.. autoclass:: predtuner.modeledapp.QoSModelP1
+ :show-inheritance:
+
+.. autoclass:: predtuner.modeledapp.QoSModelP2
+ :show-inheritance:
diff --git a/doc/reference/pytorch-app.rst b/doc/reference/pytorch-app.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4bd7e3d8f24d12c9fe53b54145c586d2d6a6f0e1
--- /dev/null
+++ b/doc/reference/pytorch-app.rst
@@ -0,0 +1,17 @@
+PyTorch Autotuning API
+======================
+
+.. autoclass:: predtuner.torchapp.TorchApp
+ :show-inheritance:
+ :members: get_tuner
+
+.. autofunction:: predtuner.approxes.get_knobs_from_file
+
+.. autofunction:: predtuner.torchutil.accuracy
+
+Defining New Approximation Knobs
+--------------------------------
+
+.. autoclass:: predtuner.torchapp.TorchApproxKnob
+ :show-inheritance:
+ :members:
diff --git a/doc/requirements.txt b/doc/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d4dc1a6039b99571b2800077062ae9236d39c710
--- /dev/null
+++ b/doc/requirements.txt
@@ -0,0 +1,3 @@
+sphinx>=3.5
+pydata-sphinx-theme==0.5.2
+numpydoc>=1.1
\ No newline at end of file
diff --git a/predtuner/__init__.py b/predtuner/__init__.py
index c11d18f205241f54b32405475e6e19d22d85538a..5edebe18433f6ace586253faa7af9b1b342521e4 100644
--- a/predtuner/__init__.py
+++ b/predtuner/__init__.py
@@ -2,9 +2,9 @@ from ._logging import config_pylogger
from .approxapp import ApproxApp, ApproxKnob, ApproxTuner
from .approxes import get_knobs_from_file
from .modeledapp import (
- IPerfModel,
+ ICostModel,
IQoSModel,
- LinearPerfModel,
+ LinearCostModel,
ModeledApp,
QoSModelP1,
QoSModelP2,
diff --git a/predtuner/approxapp.py b/predtuner/approxapp.py
index da12d72dd65fc45a6688416830214826314606d9..222d3cb5878f2f4c1624d6b41b43e4cc5c248758 100644
--- a/predtuner/approxapp.py
+++ b/predtuner/approxapp.py
@@ -27,6 +27,16 @@ TunerConfigT = Dict[int, int]
class ApproxKnob:
+ r"""Basic definition of an approximation knob.
+ An approximation knob is an instance of a type of approximation;
+ for example, Perforated Convolution is a type of approximation,
+ while row-perforated convolution with stride 2 is a knob.
+
+ :param name: The name of this approximation knob. Must be unique throughout.
+ :param devices: The devices this knob can be applied on.
+ Default is `None` which means all devices are supported.
+ """
+
def __init__(
self, name: str, devices: List[str] = None, baseline_priority: int = None
):
@@ -35,6 +45,10 @@ class ApproxKnob:
self.baseline_priority = baseline_priority
def exists_on_device(self, device: str) -> bool:
+ """Returns True if this knob can be applied to an `ApproxApp` on device `device`.
+
+ :param device: The device to check for.
+ """
if self.devices is None:
return True
return device in self.devices
@@ -57,50 +71,94 @@ class ApproxKnob:
class ApproxApp(abc.ABC):
"""Generic approximable application with operator & knob enumeration,
- and measures its own QoS and performance given a configuration.
-
- Parameters
- ----------
- op_knobs:
- a mapping from each operator (identified by str) to a list of applicable knobs.
+ and measures its own QoS and cost given a configuration.
+ (A configuration is a dictionary from operator name to a knob name.)
+ To use this class, inherit from it and implement `name` and `measure_qos_cost`.
+
+ :param op_knobs: a mapping from each operator (identified by str) to a list of applicable knobs.
+ :type op_knobs: Dict[str, List[ApproxKnob]]
+ :param target_device: the target device that this application should be tuned on.
+ Each knob has a number of devices it is supported on
+ (see `ApproxKnob.exists_on_device`)
+ and only knobs supported on `target_device` will be used for this application.
+ :type target_device: Optional[str]
+
+ :var baseline_knob: The baseline knob of this application.
+ This is derived by looking at all knobs defined in `op_knobs`
+ and deciding which is the baseline.
"""
def __init__(
- self, op_knobs: Dict[str, List[ApproxKnob]], tuning_device: str = None
+ self, op_knobs: Dict[str, List[ApproxKnob]], target_device: Optional[str] = None
) -> None:
super().__init__()
self.op_knobs = op_knobs
- if tuning_device:
- self.op_knobs = self._filter_knob_by_device(self.op_knobs, tuning_device)
+ if target_device:
+ self.op_knobs = self._filter_knob_by_device(self.op_knobs, target_device)
# Also modifies self.op_knobs in place.
self.baseline_knob = self._check_get_baseline_knob_(self.op_knobs)
- @abc.abstractmethod
- def measure_qos_cost(
- self, with_approxes: KnobsT, is_test: bool
- ) -> Tuple[float, float]:
- pass
+ @property
+ def ops(self) -> List[str]:
+ """A list of operators in this application.
+
+ :rtype: List[str]
+ """
+ return list(self.op_knobs)
+
+ @property
+ def knobs(self) -> List[ApproxKnob]:
+ """A list of all unique knobs (see `ApproxKnob`)
+ applicable to operators in this application.
+
+ :rtype: List[ApproxKnob]
+ """
+ knob_sets = [set(knobs) for knobs in self.op_knobs.values()]
+ return list(set.union(*knob_sets))
def get_tuner(self) -> "ApproxTuner":
- """We implement this function. Sets up an ApproxTuner instance
- which the user can directly call `tune()` on with opentuner parameters."""
+ """Sets up an ApproxTuner instance which the user can directly call
+ `tune()` on with opentuner parameters."""
return ApproxTuner(self)
@property
@abc.abstractmethod
def name(self) -> str:
- """Name of application. Acts as an identifier in many places, so
- the user should try to make it unique."""
+ """The name of this application.
+ Acts as an identifier in many places, so the user should try to make it unique.
+
+ :rtype: str
+ """
return ""
- @property
- def ops(self) -> List[str]:
- return list(self.op_knobs)
+ @abc.abstractmethod
+ def measure_qos_cost(
+ self, with_approxes: KnobsT, is_test: bool
+ ) -> Tuple[float, float]:
+ """Measures the QoS and cost (time, energy, ...) of a given configuration.
- @property
- def knobs(self) -> List[ApproxKnob]:
- knob_sets = [set(knobs) for knobs in self.op_knobs.values()]
- return list(set.union(*knob_sets))
+ :param with_approxes: The approximation configuration to measure QoS and cost for.
+ :param is_test: If True, uses a "test" dataset/mode that is held away from the tuner
+ during tuning; otherwise use "tune" dataset.
+ How the "tune" and "test" mode behave is up to the user to define.
+ """
+ pass
+
+ def add_baseline_to_knobs(self, approxes: KnobsT) -> KnobsT:
+ """For each operator not appearing in the keys of configuration `approxes`
+ (a dictionary), map it to the baseline (see `ApproxApp.baseline_knob`).
+
+ `measure_qos_cost` should call this on the incoming config
+ if you wish to be able to abbreviate the configuration
+ (for example, you can write `measure_qos_cost({})` to get the baseline QoS).
+ This ensures all operators are present when the config is sent to tuner.
+
+ :param approxes: the config to add baseline knobs to.
+ """
+ return {
+ op_name: approxes.get(op_name, self.baseline_knob.name)
+ for op_name in self.ops
+ }
@staticmethod
def _check_get_baseline_knob_(
@@ -124,14 +182,19 @@ class ApproxApp(abc.ABC):
for op, knobs in op_knobs.items()
}
- def add_baseline_to_knobs(self, approxes: KnobsT):
- return {
- op_name: approxes.get(op_name, self.baseline_knob.name)
- for op_name in self.ops
- }
-
class Config:
+ """An approximation configuration with its measurement results, including QoS and cost.
+
+ :param qos: The QoS of this config (measured on tuning mode, see `ApproxApp.measure_qos_cost`).
+ :param cost: The *relative* cost (time, energy, etc.) of this config
+ compared to the baseline config. This is essentially :math:`1 / speedup`.
+ :param knobs: The op-knob mapping in this configuration.
+ :param test_qos: The QoS of this config on test mode (see `ApproxApp.measure_qos_cost`).
+ This is optional as it is filled in only after the config-testing phase
+ (which can be opt out of). See `ApproxTuner.tune`.
+ """
+
def __init__(
self, qos: float, cost: float, knobs: KnobsT, test_qos: Optional[float] = None
) -> None:
@@ -148,10 +211,18 @@ class Config:
T = TypeVar("T", bound=Config)
-# IOpenTuner is generic over the type of the config
+# ApproxTuner is generic over the type of the config
# So that the user can use custom Config inherited from Config
# (in which case they need to override `get_all_configs_from_db`).
class ApproxTuner(Generic[T]):
+ """Supports tuning and holds all tuning results.
+ `ApproxTuner.tune` is the main method for tuning.
+
+ An instance of `ApproxTuner` can be obtained from `ApproxApp.get_tuner`.
+
+ :param app: the application to tune.
+ """
+
def __init__(self, app: ApproxApp) -> None:
self.app = app
self._tuned = False
@@ -165,6 +236,7 @@ class ApproxTuner(Generic[T]):
@property
def tuned(self) -> bool:
+ """Returns True if `tune` has been called at least once."""
return self._tuned
def tune(
@@ -178,6 +250,30 @@ class ApproxTuner(Generic[T]):
app_kwargs: dict = None
# TODO: more parameters + opentuner param forwarding
) -> List[T]:
+ """Runs a tuning session.
+
+ :param max_iter: Number of iterations to use in tuning.
+ :param qos_tuner_threshold: The QoS threshold that the tuner should aim for.
+ QoS is assumed to be a higher-better quantity.
+ This should be slightly tighter than `qos_keep_threshold`
+ to account for extra error when running on test dataset.
+ :param qos_keep_threshold: The QoS threshold beyond which we will keep the configuration.
+ By default it is equal to `qos_keep_threshold`.
+ :param is_threshold_relative: If True, the actual thresholds are considered to be
+ ``baseline_qos - given_threshold``.
+ This applies to `qos_tuner_threshold` and `qos_keep_threshold`.
+ :param take_best_n: Take the best :math:`n` configurations after tuning.
+ "Best" is defined as the configurations closest to the pareto curve
+ of the QoS-cost tradeoff space.
+ If `take_best_n` is None, only the configurations strictly on the
+ pareto curve are taken.
+ :param test_configs: If True, runs the configs on the test dataset,
+ filter the taken configs by `qos_keep_threshold`,
+ and fill the `test_qos` field of `Config`.
+ :param app_kwargs: Additional arguments to pass to
+ `ApproxApp.measure_qos_cost` during tuning.
+ """
+
from opentuner.tuningrunmain import TuningRunMain
from ._dbloader import read_opentuner_db
@@ -222,7 +318,7 @@ class ApproxTuner(Generic[T]):
self.kept_configs = [
cfg for cfg in self.all_configs if cfg.qos > self.tune_keep_threshold
]
- self.best_configs_prefilter = self.take_best_configs(self.kept_configs, take_best_n)
+ self.best_configs_prefilter = self._take_best_configs(self.kept_configs, take_best_n)
msg_logger.info(
"Tuning finished with %d configs in total, "
"%d configs above keeping threshold, "
@@ -232,43 +328,21 @@ class ApproxTuner(Generic[T]):
len(self.best_configs_prefilter),
)
if test_configs:
- msg_logger.info("Calibrating configurations on test inputs")
- self.best_configs = self.test_configs(self.best_configs_prefilter)
+ msg_logger.info("Running configurations on test inputs")
+ self.best_configs = self._test_configs(self.best_configs_prefilter)
else:
self.best_configs = self.best_configs_prefilter
return self.best_configs
- def test_configs(self, configs: List[Config]):
- from copy import deepcopy
-
- from tqdm import tqdm
-
- assert self.test_keep_threshold is not None
- if not configs:
- return []
- ret_configs = []
- total_error = 0
- for cfg in tqdm(configs, leave=False):
- cfg = deepcopy(cfg)
- assert cfg.test_qos is None
- cfg.test_qos, _ = self.app.measure_qos_cost(cfg.knobs, True)
- msg_logger.debug(f"Calibration: {cfg.qos} (mean) -> {cfg.test_qos} (mean)")
- total_error += abs(cfg.qos - cfg.test_qos)
- if cfg.test_qos > self.test_keep_threshold:
- ret_configs.append(cfg)
- else:
- msg_logger.debug("Config removed")
- mean_err = total_error / len(configs)
- msg_logger.info("QoS mean abs difference of calibration: %f", mean_err)
- return ret_configs
+ def dump_configs(self, filepath: PathLike, best_only: bool = True):
+ """Writes configuration to a JSON file.
- @staticmethod
- def take_best_configs(configs: List[T], n: Optional[int] = None) -> List[T]:
- points = np.array([(c.qos, c.speedup) for c in configs])
- taken_idx = is_pareto_efficient(points, take_n=n)
- return [configs[i] for i in taken_idx]
+ :param filepath: The JSON file to write into.
+ :param best_only: If True, only writes the "best" configuration
+ (filtered after running on test dataset, if required).
+ Otherwise, writes all configurations within the given QoS threshold.
+ """
- def dump_configs(self, filepath: PathLike, best_only: bool = True):
import os
from jsonpickle import encode
@@ -289,6 +363,16 @@ class ApproxTuner(Generic[T]):
connect_best_points: bool = False,
use_test_qos: bool = False,
) -> plt.Figure:
+ """Plots the QoS and speedup of configurations into a scatter plot.
+
+ :param show_qos_loss: If True, uses the loss of QoS (compared to the baseline)
+ instead of the absolute QoS.
+ :param connect_best_points: If True, draw a line connecting all the "best"
+ configurations (otherwise just plot a scatter plot).
+ :param use_test_qos: If True, plots with the test set QoS (`Config.test_qos`);
+ otherwise plots the tuning QoS (`Config.qos`).
+ """
+
if not self.tuned:
raise RuntimeError(
f"No tuning session has been run; call self.tune() first."
@@ -320,6 +404,36 @@ class ApproxTuner(Generic[T]):
ax.legend()
return fig
+ @staticmethod
+ def _take_best_configs(configs: List[T], n: Optional[int] = None) -> List[T]:
+ points = np.array([(c.qos, c.speedup) for c in configs])
+ taken_idx = is_pareto_efficient(points, take_n=n)
+ return [configs[i] for i in taken_idx]
+
+ def _test_configs(self, configs: List[Config]):
+ from copy import deepcopy
+
+ from tqdm import tqdm
+
+ assert self.test_keep_threshold is not None
+ if not configs:
+ return []
+ ret_configs = []
+ total_error = 0
+ for cfg in tqdm(configs, leave=False):
+ cfg = deepcopy(cfg)
+ assert cfg.test_qos is None
+ cfg.test_qos, _ = self.app.measure_qos_cost(cfg.knobs, True)
+ msg_logger.debug(f"Test dataset: {cfg.qos:.3f} -> {cfg.test_qos:.3f}")
+ total_error += abs(cfg.qos - cfg.test_qos)
+ if cfg.test_qos > self.test_keep_threshold:
+ ret_configs.append(cfg)
+ else:
+ msg_logger.debug("Config removed")
+ mean_err = total_error / len(configs)
+ msg_logger.debug("QoS changed by %f on test dataset (mean abs diff)", mean_err)
+ return ret_configs
+
def _get_tuner_interface(
self,
opentuner_args,
@@ -339,6 +453,8 @@ class ApproxTuner(Generic[T]):
# These are also abs thresholds
self.tune_keep_threshold = self.baseline_tune_qos - keep_threshold
self.test_keep_threshold = self.baseline_test_qos - keep_threshold
+ else:
+ self.tune_keep_threshold = self.test_keep_threshold = keep_threshold
opentuner_args.test_limit = max_iter
msg_logger.info(
"Tuner QoS threshold: %f; keeping configurations with QoS >= %f (tune dataset)",
@@ -411,7 +527,7 @@ class TunerInterface(MeasurementInterface):
return manipulator
def run(self, desired_result, input_, limit):
- """Run a given configuration then return performance and accuracy."""
+ """Run a given configuration then return cost and QoS."""
from opentuner.resultsdb.models import Result
cfg = desired_result.configuration.data
diff --git a/predtuner/approxes/approxes.py b/predtuner/approxes/approxes.py
index 0125de74f71a8a57e0148682e528ac687450da4a..f81265231aae55a76594c0cd7adc7395a3723c4e 100644
--- a/predtuner/approxes/approxes.py
+++ b/predtuner/approxes/approxes.py
@@ -393,6 +393,21 @@ def get_knobs_from_file(
filepath: PathLike = default_knob_file,
extra_name_to_class: Dict[str, Type[TorchApproxKnob]] = None,
) -> Set[TorchApproxKnob]:
+ """get_knobs_from_file(filepath=default_knob_file, extra_name_to_class=None)
+
+ Constructs and returns a set of `TorchApproxKnob` from a knob declaration file.
+ `default_knob_file` points to a file that is contained in the predtuner package,
+ so just calling ``get_knobs_from_file()`` should provide a set of predefined knobs already.
+
+ :param filepath: the knob declaration file (JSON) to read from.
+ :param extra_name_to_class: a mapping from the name of the approximation to the
+ class (implementation) of the approximation.
+ If not given, only the builtin approximations will be considered
+ when parsing the declaration file.
+ :type extra_name_to_class: Dict[str, Type[TorchApproxKnob]]
+ :rtype: Set[TorchApproxKnob]
+ """
+
import json
extra_name_to_class = extra_name_to_class or {}
diff --git a/predtuner/modeledapp.py b/predtuner/modeledapp.py
index a52dd11beb33e3c1cbd2a2385c01b2906cc854f5..dba07d61c8decd116f56d5b6661f1e321f7d83a0 100644
--- a/predtuner/modeledapp.py
+++ b/predtuner/modeledapp.py
@@ -17,72 +17,102 @@ msg_logger = logging.getLogger(__name__)
class ModeledApp(ApproxApp, abc.ABC):
- """Approximable application that inherits at least 1 interface for performance/QoS modeling.
+ """Like `.approxapp.ApproxApp`, but uses a model for QoS/cost measurement.
- It's invalid to inherit from this class without also implementing at least 1 interface
- provided in this set of API;
- for non-modeling application, inherit from `ApproxApp` instead.
+ To use this class, inherit from it and implement `get_models`,
+ `empirical_measure_qos_cost`, and `.approxapp.ApproxApp.name`.
+ (This class provides an implementation of `.approxapp.ApproxApp.measure_qos_cost`.)
+
+ :param op_knobs: a mapping from each operator (identified by str) to a list of applicable knobs.
+ :type op_knobs: Dict[str, List[ApproxKnob]]
+ :param target_device: the target device that this application should be tuned on.
+ See `.approxapp.ApproxApp` constructor.
+ :type target_device: Optional[str]
"""
def __init__(
- self, op_knobs: Dict[str, List[ApproxKnob]], tuning_device: str = None
+ self, op_knobs: Dict[str, List[ApproxKnob]], target_device: str = None
) -> None:
- super().__init__(op_knobs, tuning_device)
+ super().__init__(op_knobs, target_device)
models = self.get_models()
self._name_to_model = {m.name: m for m in models}
if len(self._name_to_model) != len(models):
raise ValueError("Name conflict in models")
self._cost_models = {
- model.name: model for model in models if isinstance(model, IPerfModel)
+ model.name: model for model in models if isinstance(model, ICostModel)
}
self._qos_models = {
model.name: model for model in models if isinstance(model, IQoSModel)
}
@abc.abstractmethod
- def get_models(self) -> List[Union["IPerfModel", "IQoSModel"]]:
- """Get QoS/Performance prediction models for this application."""
+ def get_models(self) -> List[Union["ICostModel", "IQoSModel"]]:
+ """A list of QoS/Cost prediction models for this application.
+
+ Cost models should inherit from `ICostModel`
+ while QoS models should inherit from `IQoSModel`.
+
+ :rtype: List[Union[ICostModel, IQoSModel]]
+ """
pass
+ @abc.abstractmethod
def empirical_measure_qos_cost(
self, with_approxes: KnobsT, is_test: bool
) -> Tuple[float, float]:
- """Measures QoS and performance by running the program with approximation.
+ """Empirically measures QoS and cost by actually
+ running the program with approximation (as opposed to using model).
- An implementation is not necessary if empirical measurement is never intended.
+ :param with_approxes: The approximation configuration to measure QoS and cost for.
+ :param is_test: If True, uses a "test" dataset/mode that is held away from the tuner
+ during tuning.
"""
- raise NotImplementedError()
def measure_qos_cost(
self,
with_approxes: KnobsT,
is_test: bool,
- qos_model: str = "none",
- cost_model: str = "none",
+ qos_model: Optional[str] = None,
+ cost_model: Optional[str] = None,
) -> Tuple[float, float]:
- """We provide this with the right qos and cost function.
-
- Empirical measurement will be called once if either `cost_model` or `qos_model`
- is "none", otherwise only use model indicated by model name.
+ """Returns the QoS and cost (time, energy, ...) of a given configuration,
+ *potentially using models*.
+
+ If either of `cost_model` or `qos_model` is None,
+ this will perform empirical measurement once to get the one that is not using a model.
+ Otherwise, no empirical measurement will be used.
+
+ Note that when running on test set (``is_test == True``), no modeling is allowed
+ (this raises a `ValueError`).
+
+ :param with_approxes: The approximation configuration to measure QoS and cost for.
+ :param is_test: If True, uses a "test" dataset/mode that is held away from the tuner
+ during tuning; otherwise use "tune" dataset.
+ :param qos_model: The QoS model to use in this measurement, keyed by model's name
+ (See `IQoSModel.name`).
+ :param cost_model: The Cost model to use in this measurement, keyed by model's name
+ (See `ICostModel.name`).
"""
# Testset measurement is always empirical
if is_test:
+ if qos_model is not None or cost_model is not None:
+ raise ValueError("Test dataset measurement is always empirical")
return self.empirical_measure_qos_cost(with_approxes, is_test)
# Run empirical measurement once if either cost or qos needs it
qos, cost = None, None
- if qos_model == "none" or cost_model == "none":
+ if qos_model is None or cost_model is None:
qos, cost = self.empirical_measure_qos_cost(with_approxes, is_test)
# If we're asked to use some qos_model, overwrite `qos` value
- # even if we already get it from empirical measure (i.e., even if cost_model == "none")
- if qos_model != "none":
+ # even if we already get it from empirical measure (i.e., even if cost_model is None)
+ if qos_model is not None:
if qos_model not in self._qos_models:
raise ValueError(
f'"{qos_model}" is an invalid value for qos_model '
f"(choose from {list(self._qos_models.keys())})"
)
qos = self._qos_models[qos_model].measure_qos(with_approxes)
- # Same goes for perf
- if cost_model != "none":
+ # Same goes for cost
+ if cost_model is not None:
if cost_model not in self._cost_models:
raise ValueError(
f'"{cost_model}" is an invalid value for cost_model '
@@ -93,14 +123,23 @@ class ModeledApp(ApproxApp, abc.ABC):
return qos, cost
def get_tuner(self) -> "ApproxModeledTuner":
+ """Sets up an ApproxTuner instance which the user can directly call
+ `tune()` on with opentuner parameters.
+
+ This returns an `ApproxModeledTuner`, different from `.approxapp.ApproxApp.get_tuner`
+ which returns an `ApproxTuner`.
+
+ :rtype: ApproxModeledTuner
+ """
+
return ApproxModeledTuner(self)
def init_model(self, model_name: str):
self._name_to_model[model_name]._init()
-class IPerfModel(abc.ABC):
- """Abstract base class for models that provide performance prediction."""
+class ICostModel(abc.ABC):
+ """Abstract base class for models that provide cost prediction."""
def __init__(self) -> None:
self._inited = False
@@ -113,7 +152,10 @@ class IPerfModel(abc.ABC):
@abc.abstractmethod
def measure_cost(self, with_approxes: KnobsT) -> float:
- """Predict the performance of application."""
+ """Predict the cost of application.
+
+ :param with_approxes: The configuration to predict cost for.
+ """
pass
def _init(self):
@@ -135,7 +177,10 @@ class IQoSModel(abc.ABC):
@abc.abstractmethod
def measure_qos(self, with_approxes: KnobsT) -> float:
- """Predict the qos of application."""
+ """Predict the QoS of application.
+
+ :param with_approxes: The configuration to predict QoS for.
+ """
pass
def _init(self):
@@ -143,8 +188,16 @@ class IQoSModel(abc.ABC):
self._inited = True
-class LinearPerfModel(IPerfModel):
- """Weighted linear performance predictor based on cost of each operator."""
+class LinearCostModel(ICostModel):
+ """Weighted linear cost predictor based on cost of each operator.
+
+ This predictor compute a weighted sum over
+ the cost of each operator and the speedup of each knob on that operator.
+
+ :param app: The `ModeledApp` to predict cost for.
+ :param op_costs: A mapping from operator name to its (baseline) cost.
+ :param knob_speedups: A mapping from knob name to its (expected) speedup.
+ """
def __init__(
self,
@@ -169,7 +222,6 @@ class LinearPerfModel(IPerfModel):
return "cost_linear"
def measure_cost(self, with_approxes: KnobsT) -> float:
- """We implement this using a weighted linear performance model."""
with_approxes = self.app.add_baseline_to_knobs(with_approxes)
return float(
sum(self.cost_df.loc[layer, knob] for layer, knob in with_approxes.items())
@@ -179,13 +231,17 @@ class LinearPerfModel(IPerfModel):
class QoSModelP1(IQoSModel):
"""QoS model `P1` in ApproxTuner.
- tensor_output_getter: Run the tensor-based application with config `with_approxes` applied,
- and return a single tensor result.
+ :param app: The `ModeledApp` to predict QoS for.
+ :param tensor_output_getter: A function that can run the
+ tensor-based application with a config and return a single tensor result.
- Note that while we require the return value to be a PyTorch tensor,
- user is free to implement this on non-PyTorch applications.
+ Note that here we require the return value to be a PyTorch tensor.
- qos_metric: Compute a Quality of Service level from the tensor output of application
+ :param qos_metric: A function that compute a QoS level from the return value
+ of `tensor_output_getter`.
+ :param storage: A `pickle` file to store this model into, if the file doesn't exist,
+ or load the model from if the file exists.
+ If not given, the model will not be stored.
"""
def __init__(
@@ -212,7 +268,6 @@ class QoSModelP1(IQoSModel):
return "qos_p1"
def measure_qos(self, with_approxes: KnobsT) -> float:
- """Implementation of model."""
assert self.baseline_tensor is not None
with_approxes = self.app.add_baseline_to_knobs(with_approxes)
delta_sum = sum(
@@ -260,7 +315,13 @@ class QoSModelP1(IQoSModel):
class QoSModelP2(IQoSModel):
- """QoS model `P2` in ApproxTuner."""
+ """QoS model `P1` in ApproxTuner.
+
+ :param app: The `ModeledApp` to predict QoS for.
+ :param storage: A JSON file to store this model into, if the file doesn't exist,
+ or load the model from if the file exists.
+ If not given, the model will not be stored.
+ """
def __init__(self, app: ModeledApp, storage: PathLike = None) -> None:
super().__init__()
@@ -344,6 +405,22 @@ class QoSModelP2(IQoSModel):
class ValConfig(Config):
+ """An `.approxapp.Config` that also optionally stores the "validation QoS".
+
+ Validation QoS is the empirically measured QoS in the "validation phase"
+ at the end of tuning (see `ApproxModeledTuner.tune`).
+
+ :param qos: The maybe-predicted QoS of this config.
+ (If tuning is empirical then this is empirical, not predicted, QoS.)
+ This is in contrast to `Config.qos`, which is always empirically measured on tuning dataset.
+ :param cost: The *relative* cost (time, energy, etc.) of this config
+ compared to the baseline config. This is essentially :math:`1 / speedup`.
+ :param knobs: The op-knob mapping in this configuration.
+ :param test_qos: The empirically measured QoS of this config on test mode.
+ :param validated_qos: The empirically measured QoS of this config on tuning mode,
+ in the validation phase. See `ApproxModeledTuner.tune`.
+ """
+
def __init__(
self,
qos: float,
@@ -368,23 +445,49 @@ class ApproxModeledTuner(ApproxTuner):
take_best_n: Optional[int] = None,
test_configs: bool = True,
validate_configs: Optional[bool] = None,
- cost_model: str = "none",
- qos_model: str = "none",
+ cost_model: Optional[str] = None,
+ qos_model: Optional[str] = None,
) -> List[ValConfig]:
+ """Runs a tuning session.
+
+ :param max_iter: Number of iterations to use in tuning.
+ :param qos_tuner_threshold: The QoS threshold that the tuner should aim for.
+ QoS is assumed to be a higher-better quantity.
+ This should be slightly tighter than `qos_keep_threshold`
+ to account for extra error when running on test dataset.
+ :param qos_keep_threshold: The QoS threshold beyond which we will keep the configuration.
+ By default it is equal to `qos_keep_threshold`.
+ :param is_threshold_relative: If True, the actual thresholds are considered to be
+ ``baseline_qos - given_threshold``.
+ This applies to `qos_tuner_threshold` and `qos_keep_threshold`.
+ :param take_best_n: Take the best :math:`n` configurations after tuning.
+ "Best" is defined as the configurations closest to the pareto curve
+ of the QoS-cost tradeoff space.
+ If `take_best_n` is None, only the configurations strictly on the
+ pareto curve are taken.
+ :param test_configs: If True, runs the configs on the test dataset,
+ filter the taken configs by `qos_keep_threshold`,
+ and fill the `test_qos` field of `ValConfig`.
+ :param validate_configs: If True, runs a validation step that empirically measures
+ the QoS of configs, filter the taken configs by `qos_keep_threshold`,
+ and fill the `validated_qos` field of `ValConfig`.
+ :param cost_model: The cost model to use for this tuning session.
+ :param qos_model: The QoS model to use for this tuning session.
+ This and `cost_model` are relayed down the line to `ModeledApp.measure_qos_cost`.
+ """
+
qos_desc = (
- "no model for qos" if qos_model == "none" else f'qos model "{qos_model}"'
+ "no model for qos" if qos_model is None else f'qos model "{qos_model}"'
)
cost_desc = (
- "no model for performance"
- if cost_model == "none"
- else f'performance model "{cost_model}"'
+ "no model for cost" if cost_model is None else f'cost model "{cost_model}"'
)
msg_logger.info("Starting tuning with %s and %s", qos_desc, cost_desc)
- if qos_model != "none":
+ if qos_model is not None:
msg_logger.info("Initializing qos model %s", qos_model)
self.app.init_model(qos_model)
- if cost_model != "none":
- msg_logger.info("Initializing performance model %s", cost_model)
+ if cost_model is not None:
+ msg_logger.info("Initializing cost model %s", cost_model)
self.app.init_model(cost_model)
super().tune(
max_iter=max_iter,
@@ -395,7 +498,7 @@ class ApproxModeledTuner(ApproxTuner):
test_configs=False, # Test configs below by ourselves
app_kwargs={"cost_model": cost_model, "qos_model": qos_model},
)
- if validate_configs is None and qos_model != "none":
+ if validate_configs is None and qos_model is not None:
msg_logger.info(
'Validating configurations due to using qos model "%s"', qos_model
)
@@ -425,12 +528,11 @@ class ApproxModeledTuner(ApproxTuner):
ret_configs = []
total_error = 0
for cfg in tqdm(configs, leave=False):
- cfg = deepcopy(cfg)
qos, _ = self.app.measure_qos_cost(cfg.knobs, test_mode)
if test_mode:
assert cfg.test_qos is None
cfg.test_qos = qos
- msg_logger.debug(f"Calibration: {cfg.qos} (mean) -> {qos} (mean)")
+ msg_logger.debug(f"Test: {cfg.qos} (mean) -> {qos} (mean)")
else:
assert cfg.validated_qos is None
cfg.validated_qos = qos
@@ -441,15 +543,16 @@ class ApproxModeledTuner(ApproxTuner):
else:
msg_logger.debug("Config removed")
mean_err = total_error / len(configs)
- if test_mode:
- msg_logger.info("QoS mean abs difference of calibration: %f", mean_err)
- else:
- msg_logger.info("QoS mean abs difference of validation: %f", mean_err)
+ dataset_name = "test" if test_mode else "tune"
+ msg_logger.info("QoS changed by %f on %s dataset (mean abs diff)", mean_err, dataset_name)
msg_logger.info("%d of %d configs remain", len(ret_configs), len(configs))
return ret_configs
def plot_configs(
- self, show_qos_loss: bool = False, connect_best_points: bool = False
+ self,
+ show_qos_loss: bool = False,
+ connect_best_points: bool = False,
+ use_test_qos: bool = False,
) -> plt.Figure:
if not self.tuned:
raise RuntimeError(
@@ -461,7 +564,14 @@ class ApproxModeledTuner(ApproxTuner):
val_qos_nones = [conf.validated_qos is None for conf in self.best_configs]
if any(val_qos_nones):
assert all(val_qos_nones)
- return super().plot_configs(show_qos_loss, connect_best_points, False)
+ return super().plot_configs(
+ show_qos_loss, connect_best_points, use_test_qos
+ )
+
+ if use_test_qos:
+ raise ValueError(
+ "use_test_qos is not yet supported for plotting predictive tuning session."
+ )
def get_points(confs, validated):
def qos_speedup(conf):
diff --git a/predtuner/torchapp.py b/predtuner/torchapp.py
index 00a0376da0f7b24db45a615c13f8d0c2c740040c..3e0678d0cc5d66969bce31926fda4fd1a67bf4fa 100644
--- a/predtuner/torchapp.py
+++ b/predtuner/torchapp.py
@@ -10,9 +10,9 @@ from torch.utils.data.dataloader import DataLoader
from ._logging import PathLike
from .approxapp import ApproxKnob, KnobsT
from .modeledapp import (
- IPerfModel,
+ ICostModel,
IQoSModel,
- LinearPerfModel,
+ LinearCostModel,
ModeledApp,
QoSModelP1,
QoSModelP2,
@@ -34,15 +34,27 @@ class TorchApproxKnob(ApproxKnob):
@property
@abc.abstractmethod
def expected_speedup(self) -> float:
+ """The speedup this knob is expected to provide. Used for cost prediction."""
pass
@abc.abstractmethod
def is_applicable(self, op: Module) -> bool:
+ """Returns True if this knob can be applied to this Module.
+
+ :param op: the module to check availability for.
+ :type op: torch.nn.Module
+ :rtype: torch.nn.Module
+ """
pass
@abc.abstractmethod
def apply(self, op: Module) -> Module:
- """Applies knob to `module` and returns an approximated `module`."""
+ """Applies knob to a Module and returns an approximated Module.
+
+ :param op: the module to apply approximation on.
+ :type op: torch.nn.Module
+ :rtype: torch.nn.Module
+ """
pass
@@ -53,40 +65,35 @@ class TorchApp(ModeledApp, abc.ABC):
r"""Adaptor for approximable PyTorch Modules with tensor output.
A TorchApp stores the PyTorch Module, datasets for tuning and calibration,
- set of available TorchApproxKnob each of which may be applied to some layer in the Module,
+ set of available `TorchApproxKnob` each of which may be applied to some layer in the Module,
and the quality of service (QoS) metric of application (e.g., accuracy).
- It provides empirical tuning and predictive tuning capability (see `TorchApp.tune()`),
-
- Parameters
- ----------
- app_name:
- Name of the application, which is used as an identifier in tuning sessions, etc.
- module:
- The PyTorch module to tune.
- tune_dataloader:
- A dataset to use as inputs to module during tuning. (PyTorch DataLoader is conceptually
- an enumerable, batched dataset.)
- test_dataloader:
- A input dataset used for QoS testing (see `test_config` parameter of `ApproxModeledTuner.tune`).
- knobs:
- A set of knobs to be considered. Each knob has an `is_applicable()` method
- which is used to determine which layer it can apply to.
- tensor_to_qos:
- QoS metric function which computes QoS from the module's output.
- combine_qos:
- A function to combine each batch's QoS into one value.
- When QoS is accuracy this will most likely be `mean()` (which is the default).
- device:
- The device to store module and perform inference on. By default is "cuda"
- if CUDA is available, otherwise "cpu".
- model_storage_folder:
- A folder to store the serialized QoS models into.
- `QoSModelP1` will be serialized into `model_storage_folder / "p1.pkl"`,
- and `QoSModelP2` into `model_storage_folder / "p2.json"`.
- See `QoSModelP1` and `QoSModelP2` for details.
-
- Attributes
- ----------
+ It provides empirical tuning and predictive tuning capability,
+ automatically supporting `.modeledapp.LinearCostModel`,
+ `.modeledapp.QoSModelP1`, and `.modeledapp.QoSModelP2`.
+
+ In contrast to `.approxapp.ApproxApp` and `.modeledapp.ModeledApp`,
+ there should be no need to inherit from `TorchApp` in most use cases.
+
+ :param app_name: Name of the application, which is used as an identifier in tuning sessions, etc.
+ :param module: The PyTorch module to tune.
+ :param tune_dataloader: A `torch.utils.data.Dataset` dataset to use as inputs to module during tuning.
+ :param test_dataloader: A `torch.utils.data.Dataset` dataset used for QoS testing
+ (see `test_configs` parameter of `ApproxModeledTuner.tune`).
+ :param knobs: A set of `TorchApproxKnob` to be considered. Each knob has an `is_applicable()` method
+ which is used to determine which layer it can apply to.
+ `.approxes.get_knobs_from_file` returns a set of builtin knobs that will exactly fit here.
+ :param tensor_to_qos: QoS metric function which computes QoS from the module's output.
+ `.torchutil.accuracy` computes the classification accuracy which can be applied here.
+ :param combine_qos: A function to combine each batch's QoS into one value.
+ When QoS is Classification Accuracy, this will most likely be `numpy.mean`
+ (which is the default value).
+ :param target_device: The target device that this application should be tuned on.
+ :param torch_device: The PyTorch device where the model inference is run on.
+ This device should be able to run the implementations of the knobs
+ available for this app on `target_device`.
+ :param model_storage_folder: A folder to store the serialized QoS models into.
+ `QoSModelP1` will be serialized into ``model_storage_folder / "p1.pkl"``,
+ and `QoSModelP2` into ``model_storage_folder / "p2.json"``.
"""
def __init__(
@@ -98,7 +105,7 @@ class TorchApp(ModeledApp, abc.ABC):
knobs: Set[TorchApproxKnob],
tensor_to_qos: Callable[[torch.Tensor, Any], float],
combine_qos: Callable[[np.ndarray], float] = np.mean,
- tuning_device: str = None,
+ target_device: str = None,
torch_device: Union[torch.device, str] = _default_device,
model_storage_folder: Optional[PathLike] = None,
) -> None:
@@ -107,7 +114,7 @@ class TorchApp(ModeledApp, abc.ABC):
self.tune_loader = tune_dataloader
self.test_loader = test_dataloader
self.name_to_knob = {
- k.name: k for k in self._check_and_filter_knob(knobs, tuning_device)
+ k.name: k for k in self._check_and_filter_knob(knobs, target_device)
}
self.tensor_to_qos = tensor_to_qos
self.combine_qos = combine_qos
@@ -132,17 +139,17 @@ class TorchApp(ModeledApp, abc.ABC):
self._op_costs[op_name] = summary.loc[op_name, "flops"]
# Init parent class last
- super().__init__(op_knobs, tuning_device)
+ super().__init__(op_knobs, target_device)
@property
def name(self) -> str:
"""Returns the name of application."""
return self.app_name
- def get_models(self) -> List[Union[IPerfModel, IQoSModel]]:
+ def get_models(self) -> List[Union[ICostModel, IQoSModel]]:
"""Returns a list of predictive tuning models.
- TorchApp in particular derives 1 performance model (LinearPerfModel)
+ TorchApp in particular derives 1 performance model (LinearCostModel)
and 2 QoS models (QoSModelP1, QoSModelP2) automatically.
"""
@@ -162,7 +169,7 @@ class TorchApp(ModeledApp, abc.ABC):
p1_storage = self.model_storage / "p1.pkl" if self.model_storage else None
p2_storage = self.model_storage / "p2.json" if self.model_storage else None
return [
- LinearPerfModel(self, self._op_costs, self._knob_speedups),
+ LinearCostModel(self, self._op_costs, self._knob_speedups),
QoSModelP1(
self, self._get_raw_output_valset, batched_valset_qos, p1_storage
),
diff --git a/predtuner/torchutil/common_qos.py b/predtuner/torchutil/common_qos.py
index a6a19f6625453bcd381cdd392f1c8a0206c206f8..b9d8dbf5a0945f274a1fa4b50724fcde5887d45a 100644
--- a/predtuner/torchutil/common_qos.py
+++ b/predtuner/torchutil/common_qos.py
@@ -2,6 +2,12 @@ from torch import Tensor
def accuracy(output: Tensor, target: Tensor) -> float:
+ """The "classification accuracy" metric (return value is between 0 and 100).
+
+ :param output: Probability distribution output from the model.
+ :param target: A 1d-tensor of labels, one for each input image, from the dataset.
+ """
+
_, pred_labels = output.max(1)
n_correct = (pred_labels == target).sum().item()
return n_correct / len(output) * 100
diff --git a/test/test_torchapp.py b/test/test_torchapp.py
index f43fdb45e5b574c00cdd748fe506ee50e2c1a5b9..e4b7359fbd213635088559c966cacd634785d8e9 100644
--- a/test/test_torchapp.py
+++ b/test/test_torchapp.py
@@ -41,7 +41,7 @@ class TestTorchAppTuning(TorchAppSetUp):
self.assertEqual(self.app.baseline_knob.name, "11")
def test_cpu_knobs(self):
- app = TorchApp(**self.app_args, tuning_device="cpu")
+ app = TorchApp(**self.app_args, target_device="cpu")
n_knobs = {op: len(ks) for op, ks in app.op_knobs.items()}
for op_name, op in app.midx.name_to_module.items():
nknob = 28 if isinstance(op, Conv2d) else 1
@@ -49,7 +49,7 @@ class TestTorchAppTuning(TorchAppSetUp):
self.assertEqual(app.baseline_knob.name, "11")
def test_gpu_knobs(self):
- app = TorchApp(**self.app_args, tuning_device="gpu")
+ app = TorchApp(**self.app_args, target_device="gpu")
n_knobs = {op: len(ks) for op, ks in app.op_knobs.items()}
for op_name, op in app.midx.name_to_module.items():
nknob = 28 if isinstance(op, Conv2d) else 1