Fixed plotting doesn't show "best" configs before filtering

predtuner/approxapp.py

@@ -157,6 +157,7 @@ class ApproxTuner(Generic[T]):
         self._tuned = False
         self.all_configs = []
         self.kept_configs = []
+        self.best_configs_prefilter = []
         self.best_configs = []
         # The following will be filled after self.tune() is called
         self.baseline_tune_qos, self.baseline_test_qos = None, None
@@ -221,18 +222,20 @@ class ApproxTuner(Generic[T]):
         self.kept_configs = [
             cfg for cfg in self.all_configs if cfg.qos > self.tune_keep_threshold
         ]
-        self.best_configs = self.take_best_configs(self.kept_configs, take_best_n)
+        self.best_configs_prefilter = self.take_best_configs(self.kept_configs, take_best_n)
         msg_logger.info(
             "Tuning finished with %d configs in total, "
             "%d configs above keeping threshold, "
             "and %d configs selected on tradeoff curve",
             len(self.all_configs),
             len(self.kept_configs),
-            len(self.best_configs),
+            len(self.best_configs_prefilter),
         )
         if test_configs:
             msg_logger.info("Calibrating configurations on test inputs")
-            self.best_configs = self.test_configs(self.best_configs)
+            self.best_configs = self.test_configs(self.best_configs_prefilter)
+        else:
+            self.best_configs = self.best_configs_prefilter
         return self.best_configs
 
     def test_configs(self, configs: List[Config]):
@@ -308,8 +311,8 @@ class ApproxTuner(Generic[T]):
 
         fig, ax = plt.subplots()
         kept_confs = get_points(self.kept_configs)
-        best_confs = get_points(self.best_configs)
-        ax.plot(kept_confs[0], kept_confs[1], "o", label="valid")
+        best_confs = get_points(self.best_configs_prefilter)
+        ax.plot(kept_confs[0], kept_confs[1], "o", label="kept")
         mode = "-o" if connect_best_points else "o"
         ax.plot(best_confs[0], best_confs[1], mode, label="best")
         ax.set_xlabel("QoS Loss" if show_qos_loss else "QoS")

predtuner/modeledapp.py

@@ -386,7 +386,7 @@ class ApproxModeledTuner(ApproxTuner):
         if cost_model != "none":
             msg_logger.info("Initializing performance model %s", cost_model)
             self.app.init_model(cost_model)
-        ret = super().tune(
+        super().tune(
             max_iter=max_iter,
             qos_tuner_threshold=qos_tuner_threshold,
             qos_keep_threshold=qos_keep_threshold,
@@ -399,14 +399,16 @@ class ApproxModeledTuner(ApproxTuner):
             msg_logger.info(
                 'Validating configurations due to using qos model "%s"', qos_model
             )
-            self.best_configs = self._update_configs(self.best_configs, False)
+            self.best_configs = self._update_configs(self.best_configs_prefilter, False)
         elif validate_configs:
             msg_logger.info("Validating configurations as user requested")
-            self.best_configs = self._update_configs(self.best_configs, False)
+            self.best_configs = self._update_configs(self.best_configs_prefilter, False)
+        else:
+            self.best_configs = self.best_configs_prefilter
         if test_configs:
             msg_logger.info("Calibrating configurations on test inputs")
             self.best_configs = self._update_configs(self.best_configs, True)
-        return ret
+        return self.best_configs
 
     def _update_configs(self, configs: List[ValConfig], test_mode: bool):
         from copy import deepcopy
@@ -476,14 +478,16 @@ class ApproxModeledTuner(ApproxTuner):
 
         fig, ax = plt.subplots()
         kept_confs = get_points(self.kept_configs, False)
-        best_confs = get_points(self.best_configs, False)
+        best_confs = get_points(self.best_configs_prefilter, False)
         best_confs_val = get_points(self.best_configs, True)
-        ax.plot(kept_confs[0], kept_confs[1], "o", label="valid")
+        ax.plot(kept_confs[0], kept_confs[1], "o", label="kept")
         mode = "-o" if connect_best_points else "o"
         ax.plot(best_confs[0], best_confs[1], mode, label="best")
         mode = "-o" if connect_best_points else "o"
         ax.plot(best_confs_val[0], best_confs_val[1], mode, label="best_validated")
-        ax.set_xlabel("QoS Loss (Tune dataset)" if show_qos_loss else "QoS (Tune dataset)")
+        ax.set_xlabel(
+            "QoS Loss (Tune dataset)" if show_qos_loss else "QoS (Tune dataset)"
+        )
         ax.set_ylabel("Speedup (x)")
         ax.legend()
         return fig