Merge pull request #109 from automl/bugfix/fanova-constant-hp

bugfix/fANOVA constant hyperparameters

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Fix api examples (#68).
 - Reset inputs to fix error when subsequently selecting runs with different configspaces, objectives or budgets (#106).
 - Fix errors due to changing inputs before runselection (#64).
+- For fANOVA, remove constant hyperparameters from configspace (#9).
 
 # Version 1.1.3
 

deepcave/plugins/hyperparameter/importances.py

@@ -3,6 +3,7 @@
 import plotly.graph_objs as go
 from dash import dcc, html
 from dash.exceptions import PreventUpdate
+from ConfigSpace import ConfigurationSpace, Constant
 
 from deepcave.config import Config
 from deepcave.evaluators.fanova import fANOVA as GlobalEvaluator
@@ -177,6 +178,26 @@ def process(run, inputs):
         if n_trees is None:
             raise RuntimeError("Please specify the number of trees.")
 
+        # Handle constant values in fANOVA: As the fANOVA implementation relies on pyrfr and pyrfr cannot be applied
+        # to constant hyperparameters (see https://github.com/automl/fanova/issues/81), as a workaround we remove
+        # constant hyperparameters before calculation.
+        # Note: This will break if there are conditions or forbiddens including constant hyperparameters.
+        hp_dict = run.configspace.get_hyperparameters_dict()
+        if method == "global" and any([type(v) == Constant for v in hp_dict.values()]):
+            hp_dict_wo_const = {
+                k: v for k, v in hp_dict.items() if type(v) != Constant}
+            configspace_wo_const = ConfigurationSpace()
+            for k in hp_dict_wo_const.keys():
+                configspace_wo_const.add_hyperparameter(hp_dict_wo_const[k])
+            configspace_wo_const.add_conditions(run.configspace.get_conditions())
+            configspace_wo_const.add_forbidden_clauses(run.configspace.get_forbiddens())
+            run.configspace = configspace_wo_const
+
+            configs_wo_const = []
+            for n in range(len(run.configs)):
+                configs_wo_const.append({k: v for k,v in run.configs[n].items() if k in hp_dict_wo_const.keys()})
+            run.configs = configs_wo_const
+
         hp_names = run.configspace.get_hyperparameter_names()
         budgets = run.get_budgets(include_combined=True)
 

docs/plugins/importances.rst

@@ -8,8 +8,13 @@ This plugin is capable of answering following questions:
 * How much differs the importance between the budgets?
 
 
-.. warning:: 
+.. warning::
     This page is under construction.
 
 
 .. image:: ../images/plugins/importances.png
+
+.. warning::
+    As the fANOVA implementation relies on pyrfr and pyrfr cannot be applied to constant hyperparameters,
+    as a workaround we remove constant hyperparameters before calculation.
+    This will break if there are conditions or forbiddens including constant hyperparameters.