diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e68a6f7..beb3d82d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Fix api examples (#68). - Reset inputs to fix error when subsequently selecting runs with different configspaces, objectives or budgets (#106). - Fix errors due to changing inputs before runselection (#64). +- For fANOVA, remove constant hyperparameters from configspace (#9). # Version 1.1.3 diff --git a/deepcave/plugins/hyperparameter/importances.py b/deepcave/plugins/hyperparameter/importances.py index 5b197b50..ceb91f5a 100644 --- a/deepcave/plugins/hyperparameter/importances.py +++ b/deepcave/plugins/hyperparameter/importances.py @@ -3,6 +3,7 @@ import plotly.graph_objs as go from dash import dcc, html from dash.exceptions import PreventUpdate +from ConfigSpace import ConfigurationSpace, Constant from deepcave.config import Config from deepcave.evaluators.fanova import fANOVA as GlobalEvaluator @@ -177,6 +178,26 @@ def process(run, inputs): if n_trees is None: raise RuntimeError("Please specify the number of trees.") + # Handle constant values in fANOVA: As the fANOVA implementation relies on pyrfr and pyrfr cannot be applied + # to constant hyperparameters (see https://github.com/automl/fanova/issues/81), as a workaround we remove + # constant hyperparameters before calculation. + # Note: This will break if there are conditions or forbiddens including constant hyperparameters. + hp_dict = run.configspace.get_hyperparameters_dict() + if method == "global" and any([type(v) == Constant for v in hp_dict.values()]): + hp_dict_wo_const = { + k: v for k, v in hp_dict.items() if type(v) != Constant} + configspace_wo_const = ConfigurationSpace() + for k in hp_dict_wo_const.keys(): + configspace_wo_const.add_hyperparameter(hp_dict_wo_const[k]) + configspace_wo_const.add_conditions(run.configspace.get_conditions()) + configspace_wo_const.add_forbidden_clauses(run.configspace.get_forbiddens()) + run.configspace = configspace_wo_const + + configs_wo_const = [] + for n in range(len(run.configs)): + configs_wo_const.append({k: v for k,v in run.configs[n].items() if k in hp_dict_wo_const.keys()}) + run.configs = configs_wo_const + hp_names = run.configspace.get_hyperparameter_names() budgets = run.get_budgets(include_combined=True) diff --git a/docs/plugins/importances.rst b/docs/plugins/importances.rst index e3a39f7c..227e4569 100644 --- a/docs/plugins/importances.rst +++ b/docs/plugins/importances.rst @@ -8,8 +8,13 @@ This plugin is capable of answering following questions: * How much differs the importance between the budgets? -.. warning:: +.. warning:: This page is under construction. .. image:: ../images/plugins/importances.png + +.. warning:: + As the fANOVA implementation relies on pyrfr and pyrfr cannot be applied to constant hyperparameters, + as a workaround we remove constant hyperparameters before calculation. + This will break if there are conditions or forbiddens including constant hyperparameters. \ No newline at end of file