From 8ea1554a60b56996388cc940ae4f3c075e059c29 Mon Sep 17 00:00:00 2001 From: Sarah Krebs Date: Mon, 8 Jan 2024 17:39:20 +0100 Subject: [PATCH 1/3] For fANOVA, remove constant hyperparameters from configspace --- CHANGELOG.md | 1 + .../plugins/hyperparameter/importances.py | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ef73be..5a5f1915 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Bug-Fixes - Don't convert BOHB runs with status 'running' (consistent with SMAC). +- For fANOVA, remove constant hyperparameters from configspace (#9). # Version 1.1.3 diff --git a/deepcave/plugins/hyperparameter/importances.py b/deepcave/plugins/hyperparameter/importances.py index 261c9a9e..876ef4b6 100644 --- a/deepcave/plugins/hyperparameter/importances.py +++ b/deepcave/plugins/hyperparameter/importances.py @@ -3,6 +3,7 @@ import plotly.graph_objs as go from dash import dcc, html from dash.exceptions import PreventUpdate +from ConfigSpace import ConfigurationSpace, Constant from deepcave import config from deepcave.evaluators.fanova import fANOVA as GlobalEvaluator @@ -178,6 +179,25 @@ def process(run, inputs): if n_trees is None: raise RuntimeError("Please specify the number of trees.") + # Handle constant values in fANOVA: As the fANOVA implementation relies on pyrfr and pyrfr cannot be applied + # to constant hyperparameters (see https://github.com/automl/fanova/issues/81), as a workaround we remove + # constant hyperparameters before calculation. + # Note: This will break if there are conditions including constant hyperparameters. + hp_dict = run.configspace.get_hyperparameters_dict() + if method == "global" and any([type(v) == Constant for v in hp_dict.values()]): + hp_dict_wo_const = { + k: v for k, v in hp_dict.items() if type(v) != Constant} + configspace_wo_const = ConfigurationSpace() + for k in hp_dict_wo_const.keys(): + configspace_wo_const.add_hyperparameter(hp_dict_wo_const[k]) + configspace_wo_const.add_conditions(run.configspace.get_conditions()) + run.configspace = configspace_wo_const + + configs_wo_const = [] + for n in range(len(run.configs)): + configs_wo_const.append({k: v for k,v in run.configs[n].items() if k in hp_dict_wo_const.keys()}) + run.configs = configs_wo_const + hp_names = run.configspace.get_hyperparameter_names() budgets = run.get_budgets(include_combined=True) From dee8aaa28b7aa3d98ebe1bd11e1d9945fc9be6db Mon Sep 17 00:00:00 2001 From: Sarah Krebs Date: Tue, 9 Jan 2024 09:02:43 +0100 Subject: [PATCH 2/3] For fANOVA, remove constant hyperparameters from configspace --- deepcave/plugins/hyperparameter/importances.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deepcave/plugins/hyperparameter/importances.py b/deepcave/plugins/hyperparameter/importances.py index 876ef4b6..f47860a5 100644 --- a/deepcave/plugins/hyperparameter/importances.py +++ b/deepcave/plugins/hyperparameter/importances.py @@ -182,7 +182,7 @@ def process(run, inputs): # Handle constant values in fANOVA: As the fANOVA implementation relies on pyrfr and pyrfr cannot be applied # to constant hyperparameters (see https://github.com/automl/fanova/issues/81), as a workaround we remove # constant hyperparameters before calculation. - # Note: This will break if there are conditions including constant hyperparameters. + # Note: This will break if there are conditions or forbiddens including constant hyperparameters. hp_dict = run.configspace.get_hyperparameters_dict() if method == "global" and any([type(v) == Constant for v in hp_dict.values()]): hp_dict_wo_const = { @@ -191,6 +191,7 @@ def process(run, inputs): for k in hp_dict_wo_const.keys(): configspace_wo_const.add_hyperparameter(hp_dict_wo_const[k]) configspace_wo_const.add_conditions(run.configspace.get_conditions()) + configspace_wo_const.add_forbidden_clauses(run.configspace.get_forbiddens()) run.configspace = configspace_wo_const configs_wo_const = [] From ad20db1bb828ab5d416aecc2197e403a3f3ff71c Mon Sep 17 00:00:00 2001 From: Sarah Krebs Date: Wed, 24 Jan 2024 13:49:52 +0100 Subject: [PATCH 3/3] Add constant hyperparameter warning to importances documentation --- docs/plugins/importances.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/plugins/importances.rst b/docs/plugins/importances.rst index e3a39f7c..227e4569 100644 --- a/docs/plugins/importances.rst +++ b/docs/plugins/importances.rst @@ -8,8 +8,13 @@ This plugin is capable of answering following questions: * How much differs the importance between the budgets? -.. warning:: +.. warning:: This page is under construction. .. image:: ../images/plugins/importances.png + +.. warning:: + As the fANOVA implementation relies on pyrfr and pyrfr cannot be applied to constant hyperparameters, + as a workaround we remove constant hyperparameters before calculation. + This will break if there are conditions or forbiddens including constant hyperparameters. \ No newline at end of file