diff --git a/CHANGELOG.md b/CHANGELOG.md index beb3d82d..e83e6a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,16 @@ -# Version 1.1.4 +# Version 1.2 + +## Plugins +- Add symbolic explanations plugin (#46). ## Enhancements - Fix lower bounds of dependency versions. - Allow to load multi-objective SMAC3v2 and add example (#69) - Do not disable existing loggers. +- Update author email. +- Add exit button which first deletes running jobs and then terminates DeepCave. +- Nicer handling of Keyboard Interrupt. +- Disable debug mode. ## Bug-Fixes - Don't convert BOHB runs with status 'running' (consistent with SMAC). diff --git a/Makefile b/Makefile index 258de3a9..84b857e9 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # are usually completed in github actions. SHELL := /bin/bash -VERSION := 1.1.3 +VERSION := 1.2 NAME := DeepCAVE PACKAGE_NAME := deepcave @@ -42,7 +42,7 @@ MYPY ?= mypy PRECOMMIT ?= pre-commit FLAKE8 ?= flake8 -install: +install: $(PIP) install -e . install-dev: @@ -115,7 +115,7 @@ build: # This is done to prevent accidental publishing but provide the same conveniences publish: clean build read -p "Did you update the version number in Makefile and deepcave/__init__.py?" - + $(PIP) install twine $(PYTHON) -m twine upload --repository testpypi ${DIST}/* @echo diff --git a/deepcave/__init__.py b/deepcave/__init__.py index 75139590..84c9914c 100644 --- a/deepcave/__init__.py +++ b/deepcave/__init__.py @@ -9,7 +9,7 @@ name = "DeepCAVE" package_name = "deepcave" author = "R. Sass and E. Bergman and A. Biedenkapp and F. Hutter and M. Lindauer" -author_email = "sass@tnt.uni-hannover.de" +author_email = "s.segel@ai.uni-hannover.de" description = "An interactive framework to visualize and analyze your AutoML process in real-time." url = "automl.org" project_urls = { @@ -17,7 +17,7 @@ "Source Code": "https://github.com/automl/deepcave", } copyright = f"Copyright {datetime.date.today().strftime('%Y')}, {author}" -version = "1.1.3" +version = "1.2" _exec_file = sys.argv[0] _exec_files = ["server.py", "worker.py", "sphinx-build"] diff --git a/deepcave/cli.py b/deepcave/cli.py index a16784a3..c5c0e347 100644 --- a/deepcave/cli.py +++ b/deepcave/cli.py @@ -1,5 +1,3 @@ -from typing import Any, List - import multiprocessing import subprocess from pathlib import Path @@ -49,4 +47,7 @@ def execute(_) -> None: def main() -> None: - app.run(execute) + try: + app.run(execute) + except KeyboardInterrupt: + exit("KeyboardInterrupt.") diff --git a/deepcave/config.py b/deepcave/config.py index c568bbf5..0cc24f68 100644 --- a/deepcave/config.py +++ b/deepcave/config.py @@ -6,7 +6,7 @@ class Config: # General config TITLE: str = "DeepCAVE" - DEBUG: bool = True + DEBUG: bool = False # How often to refresh background activities (such as update the sidebar or process button for # static plugins). Value in milliseconds. REFRESH_RATE: int = 500 @@ -49,6 +49,9 @@ def PLUGINS(self) -> Dict[str, List["Plugin"]]: from deepcave.plugins.budget.budget_correlation import BudgetCorrelation from deepcave.plugins.hyperparameter.importances import Importances from deepcave.plugins.hyperparameter.pdp import PartialDependencies + from deepcave.plugins.hyperparameter.symbolic_explanations import ( + SymbolicExplanations, + ) from deepcave.plugins.objective.configuration_cube import ConfigurationCube from deepcave.plugins.objective.cost_over_time import CostOverTime from deepcave.plugins.objective.parallel_coordinates import ParallelCoordinates @@ -75,6 +78,7 @@ def PLUGINS(self) -> Dict[str, List["Plugin"]]: "Hyperparameter Analysis": [ Importances(), PartialDependencies(), + SymbolicExplanations(), ], } return plugins diff --git a/deepcave/evaluators/epm/random_forest.py b/deepcave/evaluators/epm/random_forest.py index 64d69ad3..fc4d1e61 100644 --- a/deepcave/evaluators/epm/random_forest.py +++ b/deepcave/evaluators/epm/random_forest.py @@ -180,7 +180,7 @@ def _impute_inactive(self, X: np.ndarray) -> np.ndarray: def _check_dimensions(self, X: np.ndarray, Y: Optional[np.ndarray] = None) -> None: """ - Checks if the dimensions of X and Y are correct wrt features. + Checks if the dimensions of X and Y are correct with respect to features. Parameters ---------- diff --git a/deepcave/evaluators/fanova.py b/deepcave/evaluators/fanova.py index 8507ae2a..f6873610 100644 --- a/deepcave/evaluators/fanova.py +++ b/deepcave/evaluators/fanova.py @@ -33,7 +33,7 @@ def calculate( seed: int = 0, ) -> None: """ - Get the data wrt budget and trains the forest on the encoded data. + Get the data with respect to budget and trains the forest on the encoded data. Note ---- diff --git a/deepcave/layouts/header.py b/deepcave/layouts/header.py index ca11aa03..41ac88b7 100644 --- a/deepcave/layouts/header.py +++ b/deepcave/layouts/header.py @@ -1,15 +1,22 @@ +import os +import time + import dash_bootstrap_components as dbc from dash import dcc, html from dash.dependencies import Input, Output -from deepcave import app, c +from deepcave import app, c, queue from deepcave.layouts import Layout class HeaderLayout(Layout): def register_callbacks(self) -> None: super().register_callbacks() + self._callback_update_matplotlib_mode() + self._callback_delete_jobs() + self._callback_terminate_deepcave() + def _callback_update_matplotlib_mode(self) -> None: outputs = [ Output("matplotlib-mode-toggle", "color"), Output("matplotlib-mode-badge", "children"), @@ -21,7 +28,7 @@ def register_callbacks(self) -> None: ] @app.callback(outputs, inputs) - def update_matplotlib_mode(n_clicks, pathname): + def callback(n_clicks, pathname): update = None mode = c.get("matplotlib-mode") if mode is None: @@ -37,6 +44,34 @@ def update_matplotlib_mode(n_clicks, pathname): else: return "secondary", "off", update + def _callback_delete_jobs(self) -> None: + inputs = [Input("exit-deepcave", "n_clicks")] + outputs = [ + Output("exit-deepcave", "color"), + Output("exit-deepcave", "children"), + Output("exit-deepcave", "disabled"), + ] + + @app.callback(inputs, outputs) + def callback(n_clicks): + # When clicking the Exit button, we first want to delete existing jobs and update the button + if n_clicks is not None: + queue.delete_jobs() + return "danger", "Terminated DeepCAVE", True + else: + return "primary", "Exit", False + + def _callback_terminate_deepcave(self) -> None: + inputs = [Input("exit-deepcave", "n_clicks")] + outputs = [] + + @app.callback(inputs, outputs) + def callback(n_clicks): + # Then we want to terminate DeepCAVE + if n_clicks is not None: + time.sleep(1) + os._exit(130) + def __call__(self) -> html.Header: return html.Header( className="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0 shadow", @@ -59,5 +94,8 @@ def __call__(self) -> html.Header: className="me-2", id="matplotlib-mode-toggle", ), + dbc.Button( + "Exit", color="secondary", className="me-2", id="exit-deepcave", disabled=False + ), ], ) diff --git a/deepcave/plugins/hyperparameter/pdp.py b/deepcave/plugins/hyperparameter/pdp.py index c0fe7285..5783903f 100644 --- a/deepcave/plugins/hyperparameter/pdp.py +++ b/deepcave/plugins/hyperparameter/pdp.py @@ -20,7 +20,7 @@ class PartialDependencies(StaticPlugin): id = "pdp" name = "Partial Dependencies" - icon = "far fa-grip-lines" + icon = "fas fa-grip-lines" help = "docs/plugins/partial_dependencies.rst" activate_run_selection = True @@ -149,9 +149,7 @@ def load_dependency_inputs(self, run, previous_inputs, inputs): if objective_value is None: objective_value = objective_ids[0] - if budget_value is None: budget_value = budget_ids[-1] - if hp1_value is None: hp1_value = hp_names[0] return { @@ -163,7 +161,6 @@ def load_dependency_inputs(self, run, previous_inputs, inputs): }, "hyperparameter_name_2": { "options": get_checklist_options([None] + hp_names), - "value": hp2_value, }, } @@ -237,7 +234,7 @@ def get_output_layout(register): return dcc.Graph(register("graph", "figure"), style={"height": Config.FIGURE_HEIGHT}) @staticmethod - def load_outputs(run, inputs, outputs): + def get_pdp_figure(run, inputs, outputs, show_confidence, show_ice, title=None): # Parse inputs hp1_name = inputs["hyperparameter_name_1"] hp1_idx = run.configspace.get_idx_by_hyperparameter_name(hp1_name) @@ -250,9 +247,6 @@ def load_outputs(run, inputs, outputs): hp2_idx = run.configspace.get_idx_by_hyperparameter_name(hp2_name) hp2 = run.configspace.get_hyperparameter(hp2_name) - show_confidence = inputs["show_confidence"] - show_ice = inputs["show_ice"] - objective = run.get_objective(inputs["objective_id"]) objective_name = objective.name @@ -323,6 +317,7 @@ def load_outputs(run, inputs, outputs): "yaxis": { "title": objective_name, }, + "title": title } ) else: @@ -349,6 +344,7 @@ def load_outputs(run, inputs, outputs): xaxis=dict(tickvals=x_tickvals, ticktext=x_ticktext, title=hp1_name), yaxis=dict(tickvals=y_tickvals, ticktext=y_ticktext, title=hp2_name), margin=Config.FIGURE_MARGIN, + title=title ) ) @@ -356,3 +352,12 @@ def load_outputs(run, inputs, outputs): save_image(figure, "pdp.pdf") return figure + + @staticmethod + def load_outputs(run, inputs, outputs): + show_confidence = inputs["show_confidence"] + show_ice = inputs["show_ice"] + + figure = PartialDependencies.get_pdp_figure(run, inputs, outputs, show_confidence, show_ice) + + return figure diff --git a/deepcave/plugins/hyperparameter/symbolic_explanations.py b/deepcave/plugins/hyperparameter/symbolic_explanations.py new file mode 100644 index 00000000..d9ba94f9 --- /dev/null +++ b/deepcave/plugins/hyperparameter/symbolic_explanations.py @@ -0,0 +1,611 @@ +# noqa: D400 +""" +# SymbolicExplanations + +This module provides utilities for generating Symbolic Explanations. + +Provided utilities include getting input and output layout, +processing the data and loading the outputs. + +## Classes + - SymbolicExplanations: Leverage Symbolic Explanations to obtain a formula and plot it. + +## Constants + GRID_POINTS_PER_AXIS : int + SAMPLES_PER_HP : int + MAX_SAMPLES : int + MAX_SHOWN_SAMPLES : int +""" + +from typing import Any, Callable, Dict, List, Union + +import dash_bootstrap_components as dbc +import numpy as np +import plotly.graph_objs as go +from ConfigSpace.hyperparameters import CategoricalHyperparameter +from dash import dcc, html +from gplearn.genetic import SymbolicRegressor +from pyPDP.algorithms.pdp import PDP + +from deepcave.config import Config +from deepcave.evaluators.epm.random_forest_surrogate import RandomForestSurrogate +from deepcave.plugins.hyperparameter.pdp import PartialDependencies +from deepcave.plugins.static import StaticPlugin +from deepcave.runs import Status +from deepcave.utils.layout import get_checklist_options, get_select_options, help_button +from deepcave.utils.styled_plotty import get_color, get_hyperparameter_ticks, save_image +from deepcave.utils.symbolic_regression import convert_symb, get_function_set + +SR_TRAIN_POINTS_PER_AXIS = 20 +SAMPLES_PER_HP = 10 +MAX_SAMPLES = 10000 +MAX_SHOWN_SAMPLES = 100 + + +class SymbolicExplanations(StaticPlugin): + """ + Generate Symbolic Explanations. + + Provided utilities include getting input and output layout, + processing the data and loading the outputs. + """ + + id = "symbolic_explanations" + name = "Symbolic Explanations" + icon = "fas fa-subscript" + help = "docs/plugins/symbolic_explanations.rst" + activate_run_selection = True + + @staticmethod + def get_input_layout(register: Callable) -> List[Union[dbc.Row, html.Details]]: + """ + Get the layout for the input block. + + Parameters + ---------- + register : Callable + Method to register (user) variables. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[Union[dbc.Row, html.Details] + The layout for the input block. + """ + return [ + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Objective"), + dbc.Select( + id=register("objective_id", ["value", "options"], type=int), + placeholder="Select objective ...", + ), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Budget"), + help_button( + "Combined budget means that the trial on the highest" + " evaluated budget is used.\n\n" + "Note: Selecting combined budget might be misleading if" + " a time objective is used. Often, higher budget take " + " longer to evaluate, which might negatively influence " + " the results." + ), + dbc.Select( + id=register("budget_id", ["value", "options"], type=int), + placeholder="Select budget ...", + ), + ], + md=6, + ), + ], + className="mb-3", + ), + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Hyperparameter #1"), + dbc.Select( + id=register("hyperparameter_name_1", ["value", "options"]), + placeholder="Select hyperparameter ...", + ), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Hyperparameter #2"), + dbc.Select( + id=register("hyperparameter_name_2", ["value", "options"]), + placeholder="Select hyperparameter ...", + ), + ], + md=6, + ), + ], + className="mb-3", + ), + dbc.Row( + [ + dbc.Col( + [ + html.Div( + [ + dbc.Label("Parsimony coefficient"), + help_button( + "Penalizes the complexity of the resulting formulas. The " + "higher the value, the higher the penalty on the " + "complexity will be, resulting in simpler formulas." + ), + dcc.Slider( + id=register("parsimony", "value", type=int), + marks=dict([i, str(10**i)] for i in range(-8, 1)), + min=-8, + max=0, + step=1, + updatemode="drag", + ), + ], + ) + ], + ) + ], + className="mb-3", + ), + html.Details( + [ + html.Summary("Additional options for symbolic regression configuration"), + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Generations"), + help_button("The number of generations to evolve."), + dbc.Input( + id=register("generations", type=int), + type="number", + min=1, + ), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Population Size"), + help_button( + "The number of formulas competing in each generation." + ), + dbc.Input( + id=register("population_size", type=int), + type="number", + min=1, + ), + ], + md=6, + ), + ], + className="mb-3", + style={"marginTop": "0.8em"}, + ), + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Random seed"), + help_button( + "The random seed to be used in the symbolic regression." + ), + dbc.Input( + id=register("random_seed", type=int), type="number", min=0 + ), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Metric"), + help_button( + "The metric to evaluate the fitness of the formulas." + ), + dbc.Select(id=register("metric", ["value", "options"])), + ], + md=6, + ), + ], + className="mb-3", + ), + ] + ), + ] + + def load_inputs(self) -> Dict[str, Dict[str, Any]]: + """ + Load the content for the defined inputs in 'get_input_layout'. + + This method is necessary to pre-load contents for the inputs. + If the plugin is called for the first time, or there are no results in the cache, + the plugin gets its content from this method. + + Returns + ------- + Dict[str, Dict[str, Any]] + Content to be filled. + """ + return { + "parsimony": {"value": "-4"}, + "generations": {"value": "10"}, + "population_size": {"value": "5000"}, + "random_seed": {"value": "0"}, + "metric": { + "options": get_select_options(values=["rmse", "mse", "mean absolute error"]), + "value": "rmse", + }, + } + + def load_dependency_inputs(self, run, previous_inputs, inputs) -> Dict[str, Any]: # type: ignore # noqa: E501 + """ + Work like 'load_inputs' but called after inputs have changed. + + Note + ---- + Only the changes have to be returned. The returned dictionary + will be merged with the inputs. + + Parameters + ---------- + run + The selected run. + inputs + Current content of the inputs. + previous_inputs + Previous content of the inputs. + Not used in this specific function. + + Returns + ------- + Dict[str, Any] + Dictionary with the changes. + """ + objective_names = run.get_objective_names() + objective_ids = run.get_objective_ids() + objective_options = get_select_options(objective_names, objective_ids) + + budgets = run.get_budgets(human=True) + budget_ids = run.get_budget_ids() + budget_options = get_checklist_options(budgets, budget_ids) + + hp_dict = run.configspace.get_hyperparameters_dict() + hp_names_numerical = [] + for k, v in hp_dict.items(): + if not isinstance(v, CategoricalHyperparameter): + hp_names_numerical.append(k) + hp_names = hp_names_numerical + + # Get selected values + objective_value = inputs["objective_id"]["value"] + budget_value = inputs["budget_id"]["value"] + hp1_value = inputs["hyperparameter_name_1"]["value"] + hp2_value = inputs["hyperparameter_name_2"]["value"] + + if objective_value is None: + objective_value = objective_ids[0] + if budget_value is None: + budget_value = budget_ids[-1] + if hp1_value is None: + hp1_value = hp_names[0] + + return { + "objective_id": {"options": objective_options, "value": objective_value}, + "budget_id": {"options": budget_options, "value": budget_value}, + "hyperparameter_name_1": { + "options": get_checklist_options(hp_names), + "value": hp1_value, + }, + "hyperparameter_name_2": { + "options": get_checklist_options([None] + hp_names), + "value": hp2_value, + }, + } + + @staticmethod + def process(run, inputs) -> Dict[str, Any]: # type: ignore + """ + Return raw data based on a run and the input data. + + Warning + ------- + The returned data must be JSON serializable. + + Note + ---- + The passed inputs are cleaned and therefore differ + compared to 'load_inputs' or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run + The run to process. + inputs + The input data. + + Returns + ------- + Dict[str, Any] + A serialized dictionary. + + Raises + ------ + RuntimeError + If the objective is None. + """ + hp_names = run.configspace.get_hyperparameter_names() + objective = run.get_objective(inputs["objective_id"]) + budget = run.get_budget(inputs["budget_id"]) + hp1 = inputs["hyperparameter_name_1"] + hp2 = inputs["hyperparameter_name_2"] + parsimony = 10 ** inputs["parsimony"] + generations = inputs["generations"] + population_size = inputs["population_size"] + random_seed = inputs["random_seed"] + metric = inputs["metric"] + + if objective is None: + raise RuntimeError("Objective not found.") + + # Encode data + df = run.get_encoded_data( + objective, + budget, + specific=True, + statuses=Status.SUCCESS, + ) + + X = df[hp_names].to_numpy() + Y = df[objective.name].to_numpy() + + # Let's initialize the surrogate + surrogate_model = RandomForestSurrogate(run.configspace, seed=0) + surrogate_model.fit(X, Y) + + # Prepare the hyperparameters + selected_hyperparameters = [hp1] + idx1 = run.configspace.get_idx_by_hyperparameter_name(hp1) + idxs = [idx1] + if hp2 is not None and hp2 != "": + selected_hyperparameters += [hp2] + idx2 = run.configspace.get_idx_by_hyperparameter_name(hp2) + idxs += [idx2] + + num_samples = SAMPLES_PER_HP * len(X) + # The samples are limited to max 10k + if num_samples > MAX_SAMPLES: + num_samples = MAX_SAMPLES + + # And finally call PDP + pdp = PDP.from_random_points( + surrogate_model, + selected_hyperparameter=selected_hyperparameters, + seed=0, + num_grid_points_per_axis=SR_TRAIN_POINTS_PER_AXIS, + num_samples=num_samples, + ) + + x_pdp = pdp.x_pdp + y_pdp = pdp.y_pdp.tolist() + pdp_variances = pdp.y_variances.tolist() + + x_ice = pdp._ice.x_ice.tolist() + y_ice = pdp._ice.y_ice.tolist() + + # The ICE curves have to be cut because it's too much data + if len(x_ice) > MAX_SHOWN_SAMPLES: + x_ice = x_ice[:MAX_SHOWN_SAMPLES] + y_ice = y_ice[:MAX_SHOWN_SAMPLES] + + if len(selected_hyperparameters) < len(hp_names): + # If number of hyperparameters to explain is smaller than number of hyperparameters + # optimizes, use PDP to train the symbolic explanation + x_symbolic = x_pdp + y_train = y_pdp + else: + # Else, use random samples evaluated with the surrogate model to train the symbolic + # explanation + cs = surrogate_model.config_space + random_samples = np.asarray( + [ + config.get_array() + for config in cs.sample_configuration( + SR_TRAIN_POINTS_PER_AXIS ** len(selected_hyperparameters) + ) + ] + ) + x_symbolic = random_samples + y_train = surrogate_model.predict(random_samples)[0] + + symb_params = dict( + population_size=population_size, + generations=generations, + function_set=get_function_set(), + metric=metric, + parsimony_coefficient=parsimony, + random_state=random_seed, + verbose=1, + ) + + # run SR on samples + symb_model = SymbolicRegressor(**symb_params) + symb_model.fit(x_symbolic[:, idxs], y_train) + + try: + conv_expr = ( + f"{objective.name} = " + f"{convert_symb(symb_model, n_decimals=3, hp_names=selected_hyperparameters)}" + ) + except: + conv_expr = ( + "Conversion of the expression failed. Please try another seed or increase " + "the parsimony hyperparameter." + ) + + if len(conv_expr) > 150: + conv_expr = ( + "Expression is too long to display. Please try another seed or increase " + "the parsimony hyperparameter." + ) + + y_symbolic = symb_model.predict(x_symbolic[:, idxs]).tolist() + + return { + "x": x_pdp.tolist(), + "x_symbolic": x_symbolic.tolist(), + "y": y_pdp, + "y_symbolic": y_symbolic, + "expr": conv_expr, + "variances": pdp_variances, + "x_ice": x_ice, + "y_ice": y_ice, + } + + @staticmethod + def get_output_layout(register: Callable) -> List[dcc.Graph]: + """ + Get the layout for the output block. + + Parameters + ---------- + register : Callable + Method to register outputs. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[dcc.Graph] + Layout for the output block. + """ + return [ + dcc.Graph(register("symb_graph", "figure"), style={"height": Config.FIGURE_HEIGHT}), + dcc.Graph(register("pdp_graph", "figure"), style={"height": Config.FIGURE_HEIGHT}), + ] + + @staticmethod + def load_outputs(run, inputs, outputs) -> List[go.Figure]: # type: ignore + """ + Read the raw data and prepare it for the layout. + + Note + ---- + The passed inputs are cleaned and therefore differ + compared to 'load_inputs' or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run + The selected run. + inputs + Input and filter values from the user. + outputs + Raw output from the run. + + Returns + ------- + List[go.Figure] + The figure of the Symbolic Explanation and the Partial Dependency Plot (PDP) leveraged + for training in the case that the number of hyperparameters to be explained is smaller + than the number of hyperparameters that was optimized, else, a Partial Dependency Plot + (PDP) for comparison. + """ + hp1_name = inputs["hyperparameter_name_1"] + hp1_idx = run.configspace.get_idx_by_hyperparameter_name(hp1_name) + hp1 = run.configspace.get_hyperparameter(hp1_name) + selected_hyperparameters = [hp1] + + hp2_name = inputs["hyperparameter_name_2"] + hp2_idx = None + hp2 = None + if hp2_name is not None and hp2_name != "": + hp2_idx = run.configspace.get_idx_by_hyperparameter_name(hp2_name) + hp2 = run.configspace.get_hyperparameter(hp2_name) + selected_hyperparameters += [hp2] + + hp_names = run.configspace.get_hyperparameter_names() + objective = run.get_objective(inputs["objective_id"]) + objective_name = objective.name + + # Parse outputs + x_symbolic = np.asarray(outputs["x_symbolic"]) + y_symbolic = np.asarray(outputs["y_symbolic"]) + expr = outputs["expr"] + + traces1 = [] + if hp2 is None: # 1D + traces1 += [ + go.Scatter( + x=x_symbolic[:, hp1_idx], + y=y_symbolic, + line=dict(color=get_color(0, 1)), + hoverinfo="skip", + showlegend=False, + ) + ] + + tickvals, ticktext = get_hyperparameter_ticks(hp1) + layout1 = go.Layout( + { + "xaxis": { + "tickvals": tickvals, + "ticktext": ticktext, + "title": hp1_name, + }, + "yaxis": { + "title": objective_name, + }, + "title": expr, + } + ) + else: + z = y_symbolic + traces1 += [ + go.Contour( + z=z, + x=x_symbolic[:, hp1_idx], + y=x_symbolic[:, hp2_idx], + colorbar=dict( + title=objective_name, + ), + hoverinfo="skip", + ) + ] + + x_tickvals, x_ticktext = get_hyperparameter_ticks(hp1) + y_tickvals, y_ticktext = get_hyperparameter_ticks(hp2) + + layout1 = go.Layout( + dict( + xaxis=dict(tickvals=x_tickvals, ticktext=x_ticktext, title=hp1_name), + yaxis=dict(tickvals=y_tickvals, ticktext=y_ticktext, title=hp2_name), + margin=Config.FIGURE_MARGIN, + title=expr, + ) + ) + + figure1 = go.Figure(data=traces1, layout=layout1) + save_image(figure1, "symbolic_explanation.pdf") + + if len(selected_hyperparameters) < len(hp_names): + pdp_title = "Partial Dependency leveraged for training of Symbolic Explanation:" + else: + pdp_title = "Partial Dependency for comparison:" + + figure2 = PartialDependencies.get_pdp_figure( + run, inputs, outputs, show_confidence=False, show_ice=False, title=pdp_title + ) + + return [figure1, figure2] diff --git a/deepcave/utils/logging.yml b/deepcave/utils/logging.yml index 68d54734..338c05a2 100644 --- a/deepcave/utils/logging.yml +++ b/deepcave/utils/logging.yml @@ -5,12 +5,12 @@ formatters: handlers: console: class: logging.StreamHandler - level: DEBUG + level: INFO formatter: simple stream: ext://sys.stdout loggers: src.plugins: - level: DEBUG + level: INFO handlers: [ console ] propagate: no matplotlib: @@ -22,6 +22,6 @@ loggers: handlers: [ console ] propagate: no root: - level: DEBUG + level: INFO handlers: [console] -disable_existing_loggers: false +disable_existing_loggers: true diff --git a/deepcave/utils/symbolic_regression.py b/deepcave/utils/symbolic_regression.py new file mode 100644 index 00000000..33781f60 --- /dev/null +++ b/deepcave/utils/symbolic_regression.py @@ -0,0 +1,151 @@ +# noqa: D400 +""" +# Symbolic Regression + +This module provides utilities for running symbolic regression with gplearn. +""" + +from typing import List, Union + +import numpy as np +import sympy +from gplearn import functions +from gplearn.functions import _Function, make_function +from gplearn.genetic import SymbolicRegressor + +from deepcave.utils.logs import get_logger + +logger = get_logger(__name__) + + +def exp(x): + """ + Get a safe exp function with a maximum value of 100000 to avoid overflow. + + Parameters + ---------- + x : float + The value to calculate the exponential of. + + Returns + ------- + float + The safe exponential of x. + """ + with np.errstate(all="ignore"): + max_value = np.full(shape=x.shape, fill_value=100000) + return np.minimum(np.exp(x), max_value) + + +def get_function_set() -> List[Union[str, _Function]]: + """ + Get a function set for symbolic regression with gplearn. + + Returns + ------- + List[Union[str, _Function]] + List of functions to use in symbolic regression. + """ + exp_func = make_function(function=exp, arity=1, name="exp") + + function_set = ["add", "sub", "mul", "div", "sqrt", "log", "sin", "cos", "abs", exp_func] + + return function_set + + +def convert_symb( + symb: SymbolicRegressor, n_decimals: int = None, hp_names: list = None +) -> sympy.core.expr: + """ + Convert a fitted symbolic regression to a simplified and potentially rounded mathematical + expression. + + Warning: eval is used in this function, thus it should not be used on unsanitized input (see + https://docs.sympy.org/latest/modules/core.html?highlight=eval#module-sympy.core.sympify). + + Parameters + ---------- + symb: SymbolicRegressor + Fitted symbolic regressor to find a simplified expression for. + n_decimals: Optional[int] + If set, round floats in the expression to this number of decimals. + hp_names: Optional[List[str]] + If set, replace X0 and X1 in the expression by the names given. + + Returns + ------- + SymbolicRegressor + Converted mathematical expression. + """ + # sqrt is protected function in gplearn, always returning sqrt(abs(x)) + sqrt_pos = [] + prev_sqrt_inserts = 0 + for i, f in enumerate(symb._program.program): + if isinstance(f, functions._Function) and f.name == "sqrt": + sqrt_pos.append(i) + for i in sqrt_pos: + symb._program.program.insert(i + prev_sqrt_inserts + 1, functions.abs1) + prev_sqrt_inserts += 1 + + # log is protected function in gplearn, always returning sqrt(abs(x)) + log_pos = [] + prev_log_inserts = 0 + for i, f in enumerate(symb._program.program): + if isinstance(f, functions._Function) and f.name == "log": + log_pos.append(i) + for i in log_pos: + symb._program.program.insert(i + prev_log_inserts + 1, functions.abs1) + prev_log_inserts += 1 + + symb_str = str(symb._program) + + converter = { + "sub": lambda x, y: x - y, + "div": lambda x, y: x / y, + "mul": lambda x, y: x * y, + "add": lambda x, y: x + y, + "neg": lambda x: -x, + "pow": lambda x, y: x**y, + } + + # Abort conversion for very long programs, as they take too much time or do not finish at all. + if symb._program.length_ > 300: + return symb_str + + # Convert formula string to SymPy object + symb_conv = sympy.sympify(symb_str.replace("[", "").replace("]", ""), locals=converter) + + # Replace variable names in formula by hyperparameter names + if hp_names is not None: + if len(hp_names) == 1: + X0, hp0 = sympy.symbols(f"X0 {hp_names[0]}") + symb_conv = symb_conv.subs(X0, hp0) + elif len(hp_names) == 2: + X0, hp0, X1, hp1 = sympy.symbols(f"X0 {hp_names[0]} X1 {hp_names[1]}") + symb_conv = symb_conv.subs(X0, hp0) + symb_conv = symb_conv.subs(X1, hp1) + else: + raise ValueError( + f"Numer of hyperparameters to be explained by symbolic explanations must not " + f"be larger than 2" + ) + + # Simplify the formula + try: + # Simplification can fail in some cases. If so, use the unsimplified version. + symb_simpl = sympy.simplify(symb_conv) + except Exception as e: + logger.debug( + f"Simplification of symbolic regression failed, use unsimplified expression " + f"instead: {e}" + ) + symb_simpl = symb_conv + + # Round floats to n_decimals + if n_decimals: + # Make sure also floats deeper in the expression tree are rounded + for a in sympy.preorder_traversal(symb_simpl): + if isinstance(a, sympy.core.numbers.Float): + symb_simpl = symb_simpl.subs(a, round(a, n_decimals)) + + return symb_simpl diff --git a/docs/images/plugins/symbolic_explanations.png b/docs/images/plugins/symbolic_explanations.png new file mode 100644 index 00000000..109aaf1d Binary files /dev/null and b/docs/images/plugins/symbolic_explanations.png differ diff --git a/docs/plugins/configurations.rst b/docs/plugins/configurations.rst index 2f532f31..9d4d5a90 100644 --- a/docs/plugins/configurations.rst +++ b/docs/plugins/configurations.rst @@ -13,7 +13,7 @@ Since configurations are used throughout the application, you might find links a plugin. This plugin is capable of answering following questions: * Where is the configuration coming from? -* How are the objective values wrt the budgets? +* How are the objective values with respect to the budgets? * How is the status of a trial associated with the selected configuration? * Which values have been used for a certain configuration? * How can I access the configuration in python? @@ -43,7 +43,7 @@ graph view gives you a nice overview, the table displays the concrete values. Code ---- Often a configuration is selected for deployment, which makes it crucial to access it somehow. -The code block provides you the code to access the configuration code-wise. +The code block provides you the code to access the configuration code-wise. .. image:: ../images/plugins/configurations.png diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 1d37a667..07f3bb00 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -15,6 +15,7 @@ Plugins budget_correlation importances partial_dependencies + symbolic_explanations Plugins are used to display data in a specific way. There are plugins to analyse the performance, @@ -78,6 +79,6 @@ DeepCAVE was designed so that the plugins require minimal design. We recommend u provided plugins as a starting point and change it to your needs. After you have created your plugin, you need to register it in your config file. If you work -on the branch directly, you can adapt ``deepcave/config.py`` to your needs. +on the branch directly, you can adapt ``deepcave/config.py`` to your needs. We would be very happy to receive pull-requests! diff --git a/docs/plugins/parallel_coordinates.rst b/docs/plugins/parallel_coordinates.rst index 9abdd43b..da01db1b 100644 --- a/docs/plugins/parallel_coordinates.rst +++ b/docs/plugins/parallel_coordinates.rst @@ -1,7 +1,7 @@ Parallel Coordinates ==================== -With parallel coordinates, you can see configurations plotted as a line through their hyperparamter +With parallel coordinates, you can see configurations plotted as a line through their hyperparameter values and to which final score they reach. You can use this to identify trends in hyperparamter value ranges that achieve certain scores. For example, you may find that high performing configurations may all share the same value for a diff --git a/docs/plugins/partial_dependencies.rst b/docs/plugins/partial_dependencies.rst index 3371e6e8..1d91bd5b 100644 --- a/docs/plugins/partial_dependencies.rst +++ b/docs/plugins/partial_dependencies.rst @@ -3,12 +3,12 @@ Partial Dependencies This plugin is capable of answering following questions: -* How does the objective change wrt one or two hyperparameters? For example, does the accuracy - increase if the learning rate decreases? +* How does the objective change with respect to one or two hyperparameters? For example, does the + accuracy increase if the learning rate decreases? * Do multiple trials show similar behavior? -.. warning:: +.. warning:: This page is under construction. diff --git a/docs/plugins/symbolic_explanations.rst b/docs/plugins/symbolic_explanations.rst new file mode 100644 index 00000000..5e9f8478 --- /dev/null +++ b/docs/plugins/symbolic_explanations.rst @@ -0,0 +1,22 @@ +Symbolic Explanations +==================== + +Symbolic Explanations allow to obtain explicit formulas quantifying the relation between +hyperparameter values and model performance by applying symbolic regression to meta-data collected +during hyperparameter optimization. + +The plugin is capable of answering similar questions as the Partial Dependencies plugin, i.e.: + +* How does the objective change with respect to one or two hyperparameters? For example, does the + accuracy increase if the learning rate decreases? +* Do multiple trials show similar behavior? + +While the Partial Dependencies plugin provides a plot describing the effects of hyperparameters on +the model performance, the Symbolic Explanations plugin additionally allows to obtain an explicit +formula capturing these effects. + +.. image:: ../images/plugins/symbolic_explanations.png + +To learn more about Symbolic Explanations, please see the paper +`Symbolic Explanations for Hyperparameter Optimization +`_. diff --git a/requirements.txt b/requirements.txt index 0f116b2b..93bd6cc3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,9 @@ matplotlib>=3.5.1 seaborn>=0.13.0 pyyaml>=6.0.1 kaleido>=0.2.1 +gplearn>=0.4.2 +sympy>=1.12 +kaleido>=0.2.1 # AutoML packages ConfigSpace==0.6.1 diff --git a/tests/test_utils/test_cache.py b/tests/test_utils/test_cache.py index 2751504b..3d7c3939 100644 --- a/tests/test_utils/test_cache.py +++ b/tests/test_utils/test_cache.py @@ -303,7 +303,7 @@ def test_logging_config(self): self.assertFalse(mpl_logger.propagate) plugin_logger = get_logger("src.plugins") - self.assertEqual(logging.DEBUG, plugin_logger.level) + self.assertEqual(logging.INFO, plugin_logger.level) self.assertFalse(plugin_logger.propagate)