diff --git a/supervision/metrics/core.py b/supervision/metrics/core.py index def5999a0..203bb59ed 100644 --- a/supervision/metrics/core.py +++ b/supervision/metrics/core.py @@ -33,6 +33,36 @@ def compute(self, *args, **kwargs) -> Any: raise NotImplementedError +class MetricResult(ABC): + """ + Base class for all metric results. + """ + + @abstractmethod + def to_pandas(): + """ + Convert the result to a pandas DataFrame. + + Returns: + (pd.DataFrame): The result as a DataFrame. + """ + raise NotImplementedError() + + @abstractmethod + def plot(): + """ + Plot the results. + """ + raise NotImplementedError() + + @abstractmethod + def _get_plot_details(): + """ + Get the metric details to be plotted. + """ + raise NotImplementedError() + + class MetricTarget(Enum): """ Specifies what type of detection is used to compute the metric. diff --git a/supervision/metrics/f1_score.py b/supervision/metrics/f1_score.py index 98cb5f265..5184d5121 100644 --- a/supervision/metrics/f1_score.py +++ b/supervision/metrics/f1_score.py @@ -15,7 +15,7 @@ oriented_box_iou_batch, ) from supervision.draw.color import LEGACY_COLOR_PALETTE -from supervision.metrics.core import AveragingMethod, Metric, MetricTarget +from supervision.metrics.core import AveragingMethod, Metric, MetricResult, MetricTarget from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -455,7 +455,7 @@ def _filter_predictions_and_targets_by_size( @dataclass -class F1ScoreResult: +class F1ScoreResult(MetricResult): """ The results of the F1 score metric calculation. @@ -583,15 +583,15 @@ def to_pandas(self) -> "pd.DataFrame": return pd.DataFrame(pandas_data, index=[0]) - def plot(self): + def _get_plot_details(self) -> Tuple[List[str], List[float], str, List[str]]: """ - Plot the F1 results. + Obtain the metric details for plotting them. - ![example_plot](\ - https://media.roboflow.com/supervision-docs/metrics/f1_plot_example.png\ - ){ align=center width="800" } + Returns: + Tuple[List[str], List[float], str, List[str]]: The details for plotting the + metric. It is a tuple of four elements: a list of labels, a list of + values, the title of the plot and the bar colors. """ - labels = ["F1@50", "F1@75"] values = [self.f1_50, self.f1_75] colors = [LEGACY_COLOR_PALETTE[0]] * 2 @@ -614,16 +614,28 @@ def plot(self): values += [large_objects.f1_50, large_objects.f1_75] colors += [LEGACY_COLOR_PALETTE[4]] * 2 - plt.rcParams["font.family"] = "monospace" - - _, ax = plt.subplots(figsize=(10, 6)) - ax.set_ylim(0, 1) - ax.set_ylabel("Value", fontweight="bold") title = ( f"F1 Score, by Object Size" f"\n(target: {self.metric_target.value}," f" averaging: {self.averaging_method.value})" ) + return labels, values, title, colors + + def plot(self): + """ + Plot the F1 results. + + ![example_plot](\ + https://media.roboflow.com/supervision-docs/metrics/f1_plot_example.png\ + ){ align=center width="800" } + """ + labels, values, title, colors = self._get_plot_details() + + plt.rcParams["font.family"] = "monospace" + + _, ax = plt.subplots(figsize=(10, 6)) + ax.set_ylim(0, 1) + ax.set_ylabel("Value", fontweight="bold") ax.set_title(title, fontweight="bold") x_positions = range(len(labels)) diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 9e7a30d0e..1e0637a33 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -15,7 +15,7 @@ oriented_box_iou_batch, ) from supervision.draw.color import LEGACY_COLOR_PALETTE -from supervision.metrics.core import Metric, MetricTarget +from supervision.metrics.core import Metric, MetricResult, MetricTarget from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -418,7 +418,7 @@ def _filter_detections_by_size( @dataclass -class MeanAveragePrecisionResult: +class MeanAveragePrecisionResult(MetricResult): """ The result of the Mean Average Precision calculation. @@ -559,18 +559,19 @@ def to_pandas(self) -> "pd.DataFrame": index=[0], ) - def plot(self): + def _get_plot_details(self) -> Tuple[List[str], List[float], str, List[str]]: """ - Plot the mAP results. + Obtain the metric details for plotting them. - ![example_plot](\ - https://media.roboflow.com/supervision-docs/metrics/mAP_plot_example.png\ - ){ align=center width="800" } + Returns: + Tuple[List[str], List[float], str, List[str]]: The details for plotting the + metric. It is a tuple of four elements: a list of labels, a list of + values, the title of the plot and the bar colors. """ - labels = ["mAP@50:95", "mAP@50", "mAP@75"] values = [self.map50_95, self.map50, self.map75] colors = [LEGACY_COLOR_PALETTE[0]] * 3 + title = "Mean Average Precision" if self.small_objects is not None: labels += ["Small: mAP@50:95", "Small: mAP@50", "Small: mAP@75"] @@ -599,12 +600,25 @@ def plot(self): ] colors += [LEGACY_COLOR_PALETTE[4]] * 3 + return labels, values, title, colors + + def plot(self): + """ + Plot the mAP results. + + ![example_plot](\ + https://media.roboflow.com/supervision-docs/metrics/mAP_plot_example.png\ + ){ align=center width="800" } + """ + + labels, values, title, colors = self._get_plot_details() + plt.rcParams["font.family"] = "monospace" _, ax = plt.subplots(figsize=(10, 6)) ax.set_ylim(0, 1) ax.set_ylabel("Value", fontweight="bold") - ax.set_title("Mean Average Precision", fontweight="bold") + ax.set_title(title, fontweight="bold") x_positions = range(len(labels)) bars = ax.bar(x_positions, values, color=colors, align="center") diff --git a/supervision/metrics/mean_average_recall.py b/supervision/metrics/mean_average_recall.py index 9c3a40718..6e4d9c0d6 100644 --- a/supervision/metrics/mean_average_recall.py +++ b/supervision/metrics/mean_average_recall.py @@ -15,7 +15,7 @@ oriented_box_iou_batch, ) from supervision.draw.color import LEGACY_COLOR_PALETTE -from supervision.metrics.core import Metric, MetricTarget +from supervision.metrics.core import Metric, MetricResult, MetricTarget from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -460,7 +460,7 @@ def _filter_predictions_and_targets_by_size( @dataclass -class MeanAverageRecallResult: +class MeanAverageRecallResult(MetricResult): # """ # The results of the recall metric calculation. @@ -622,13 +622,14 @@ def to_pandas(self) -> "pd.DataFrame": return pd.DataFrame(pandas_data, index=[0]) - def plot(self): + def _get_plot_details(self) -> Tuple[List[str], List[float], str, List[str]]: """ - Plot the Mean Average Recall results. + Obtain the metric details for plotting them. - ![example_plot](\ - https://media.roboflow.com/supervision-docs/metrics/mAR_plot_example.png\ - ){ align=center width="800" } + Returns: + Tuple[List[str], List[float], str, List[str]]: The details for plotting the + metric. It is a tuple of four elements: a list of labels, a list of + values, the title of the plot and the bar colors. """ labels = ["mAR @ 1", "mAR @ 10", "mAR @ 100"] values = [self.mAR_at_1, self.mAR_at_10, self.mAR_at_100] @@ -664,15 +665,28 @@ def plot(self): ] colors += [LEGACY_COLOR_PALETTE[4]] * 3 + title = ( + f"Mean Average Recall, by Object Size" + f"\n(target: {self.metric_target.value})" + ) + return labels, values, title, colors + + def plot(self): + """ + Plot the Mean Average Recall results. + + ![example_plot](\ + https://media.roboflow.com/supervision-docs/metrics/mAR_plot_example.png\ + ){ align=center width="800" } + """ + + labels, values, title, colors = self._get_plot_details() + plt.rcParams["font.family"] = "monospace" _, ax = plt.subplots(figsize=(10, 6)) ax.set_ylim(0, 1) ax.set_ylabel("Value", fontweight="bold") - title = ( - f"Mean Average Recall, by Object Size" - f"\n(target: {self.metric_target.value})" - ) ax.set_title(title, fontweight="bold") x_positions = range(len(labels)) diff --git a/supervision/metrics/precision.py b/supervision/metrics/precision.py index a5d4011e8..49eae5adb 100644 --- a/supervision/metrics/precision.py +++ b/supervision/metrics/precision.py @@ -15,7 +15,7 @@ oriented_box_iou_batch, ) from supervision.draw.color import LEGACY_COLOR_PALETTE -from supervision.metrics.core import AveragingMethod, Metric, MetricTarget +from supervision.metrics.core import AveragingMethod, Metric, MetricResult, MetricTarget from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -458,7 +458,7 @@ def _filter_predictions_and_targets_by_size( @dataclass -class PrecisionResult: +class PrecisionResult(MetricResult): """ The results of the precision metric calculation. @@ -588,15 +588,15 @@ def to_pandas(self) -> "pd.DataFrame": return pd.DataFrame(pandas_data, index=[0]) - def plot(self): + def _get_plot_details(self) -> Tuple[List[str], List[float], str, List[str]]: """ - Plot the precision results. + Obtain the metric details for plotting them. - ![example_plot](\ - https://media.roboflow.com/supervision-docs/metrics/precision_plot_example.png\ - ){ align=center width="800" } + Returns: + Tuple[List[str], List[float], str, List[str]]: The details for plotting the + metric. It is a tuple of four elements: a list of labels, a list of + values, the title of the plot and the bar colors. """ - labels = ["Precision@50", "Precision@75"] values = [self.precision_at_50, self.precision_at_75] colors = [LEGACY_COLOR_PALETTE[0]] * 2 @@ -619,16 +619,29 @@ def plot(self): values += [large_objects.precision_at_50, large_objects.precision_at_75] colors += [LEGACY_COLOR_PALETTE[4]] * 2 - plt.rcParams["font.family"] = "monospace" - - _, ax = plt.subplots(figsize=(10, 6)) - ax.set_ylim(0, 1) - ax.set_ylabel("Value", fontweight="bold") title = ( f"Precision, by Object Size" f"\n(target: {self.metric_target.value}," f" averaging: {self.averaging_method.value})" ) + return labels, values, title, colors + + def plot(self): + """ + Plot the precision results. + + ![example_plot](\ + https://media.roboflow.com/supervision-docs/metrics/precision_plot_example.png\ + ){ align=center width="800" } + """ + + labels, values, title, colors = self._get_plot_details() + + plt.rcParams["font.family"] = "monospace" + + _, ax = plt.subplots(figsize=(10, 6)) + ax.set_ylim(0, 1) + ax.set_ylabel("Value", fontweight="bold") ax.set_title(title, fontweight="bold") x_positions = range(len(labels)) diff --git a/supervision/metrics/recall.py b/supervision/metrics/recall.py index b3586ff7d..b84658aec 100644 --- a/supervision/metrics/recall.py +++ b/supervision/metrics/recall.py @@ -15,7 +15,7 @@ oriented_box_iou_batch, ) from supervision.draw.color import LEGACY_COLOR_PALETTE -from supervision.metrics.core import AveragingMethod, Metric, MetricTarget +from supervision.metrics.core import AveragingMethod, Metric, MetricResult, MetricTarget from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -457,7 +457,7 @@ def _filter_predictions_and_targets_by_size( @dataclass -class RecallResult: +class RecallResult(MetricResult): """ The results of the recall metric calculation. @@ -587,15 +587,15 @@ def to_pandas(self) -> "pd.DataFrame": return pd.DataFrame(pandas_data, index=[0]) - def plot(self): + def _get_plot_details(self): """ - Plot the recall results. + Obtain the metric details for plotting them. - ![example_plot](\ - https://media.roboflow.com/supervision-docs/metrics/recall_plot_example.png\ - ){ align=center width="800" } + Returns: + Tuple[List[str], List[float], str, List[str]]: The details for plotting the + metric. It is a tuple of four elements: a list of labels, a list of + values, the title of the plot and the bar colors. """ - labels = ["Recall@50", "Recall@75"] values = [self.recall_at_50, self.recall_at_75] colors = [LEGACY_COLOR_PALETTE[0]] * 2 @@ -618,16 +618,30 @@ def plot(self): values += [large_objects.recall_at_50, large_objects.recall_at_75] colors += [LEGACY_COLOR_PALETTE[4]] * 2 - plt.rcParams["font.family"] = "monospace" - - _, ax = plt.subplots(figsize=(10, 6)) - ax.set_ylim(0, 1) - ax.set_ylabel("Value", fontweight="bold") title = ( f"Recall, by Object Size" f"\n(target: {self.metric_target.value}," f" averaging: {self.averaging_method.value})" ) + + return labels, values, title, colors + + def plot(self): + """ + Plot the recall results. + + ![example_plot](\ + https://media.roboflow.com/supervision-docs/metrics/recall_plot_example.png\ + ){ align=center width="800" } + """ + + labels, values, title, colors = self._get_plot_details() + + plt.rcParams["font.family"] = "monospace" + + _, ax = plt.subplots(figsize=(10, 6)) + ax.set_ylim(0, 1) + ax.set_ylabel("Value", fontweight="bold") ax.set_title(title, fontweight="bold") x_positions = range(len(labels)) diff --git a/supervision/metrics/utils/aggregate_metric_results.py b/supervision/metrics/utils/aggregate_metric_results.py new file mode 100644 index 000000000..57feac230 --- /dev/null +++ b/supervision/metrics/utils/aggregate_metric_results.py @@ -0,0 +1,163 @@ +from typing import TYPE_CHECKING, List + +import numpy as np +from matplotlib import pyplot as plt + +from supervision.draw.color import LEGACY_COLOR_PALETTE +from supervision.metrics.core import MetricResult +from supervision.metrics.utils.utils import ensure_pandas_installed + +if TYPE_CHECKING: + import pandas as pd + + +def aggregate_metric_results( + metrics_results: List[MetricResult], + model_names: List[str], + include_object_sizes=False, +) -> "pd.DataFrame": + """ + Convert a list of results to a pandas DataFrame. + + Args: + metrics_results (List[MetricResult]): List of results to be aggregated. + model_names (List[str]): List of model names corresponding to the results. + include_object_sizes (bool, optional): Whether to include object sizes in the + DataFrame. Defaults to False. + + Raises: + ValueError: List `metrics_results` can not be empty + ValueError: All elements of `metrics_results` must be of the same type + ValueError: Base class of elements in `metrics_results` must be of type + `MetricResult` + + Returns: + pd.DataFrame: The results as a DataFrame. + """ + ensure_pandas_installed() + import pandas as pd + + assert len(metrics_results) == len( + model_names + ), "Length of metrics_results and model_names must be equal" + + if len(metrics_results) == 0: + raise ValueError("List metrics_results must not be empty") + + first_elem_type = type(metrics_results[0]) + all_same_type = all(isinstance(x, first_elem_type) for x in metrics_results) + if not all_same_type: + raise ValueError("All metrics_results elements must be of the same type") + + if not isinstance(metrics_results[0], MetricResult): + raise ValueError("Base class of metrics_results must be of type MetricResult") + + pd_results = [] + for metric_result, model_name in zip(metrics_results, model_names): + pd_result = metric_result.to_pandas() + pd_result.insert(loc=0, column="Model Name", value=model_name) + pd_results.append(pd_result) + + df_merged = pd.concat(pd_results) + + if not include_object_sizes: + regex_pattern = "small|medium|large" + df_merged = df_merged.drop(columns=list(df_merged.filter(regex=regex_pattern))) + + return df_merged + + +def plot_aggregate_metric_results( + metrics_results: List[MetricResult], + model_names: List[str], + include_object_sizes=False, +): + """ + Plot a bar chart with the results of multiple metrics. + + Args: + metrics_results (List[MetricResult]): List of results to be plotted. + model_names (List[str]): List of model names corresponding to the results. + include_object_sizes (bool, optional): Whether to include object sizes in the + plot. Defaults to False. + + Raises: + ValueError: List `metrics_results` can not be empty + ValueError: All elements of `metrics_results` must be of the same type + ValueError: Base class of elements in `metrics_results` must be of type + `MetricResult` + """ + assert len(metrics_results) == len( + model_names + ), "Length of metrics_results and model_names must be equal" + + if len(metrics_results) == 0: + raise ValueError("List metrics_results must not be empty") + + first_elem_type = type(metrics_results[0]) + all_same_type = all(isinstance(x, first_elem_type) for x in metrics_results) + if not all_same_type: + raise ValueError("All metrics_results elements must be of the same type") + + if not isinstance(metrics_results[0], MetricResult): + raise ValueError("Base class of metrics_results must be of type MetricResult") + + model_values = [] + labels, values, title, _ = metrics_results[0]._get_plot_details() + model_values.append(values) + + for metric in metrics_results[1:]: + _, values, _, _ = metric._get_plot_details() + model_values.append(values) + + if not include_object_sizes: + labels_length = 3 if len(labels) % 3 == 0 else 2 + labels = labels[:labels_length] + aux_values = [] + for values in model_values: + aux_values.append(values[:labels_length]) + model_values = aux_values + + n = len(model_names) + x_positions = np.arange(len(labels)) + width = 0.8 / n + value_text_rotation = 90 if include_object_sizes else 0 + + plt.rcParams["font.family"] = "monospace" + + _, ax = plt.subplots(figsize=(10, 6)) + ax.set_ylim(0, 1) + ax.set_ylabel("Value", fontweight="bold") + ax.set_title(title, fontweight="bold") + + ax.set_xticks(x_positions) + ax.set_xticklabels(labels, rotation=45, ha="right") + + colors = LEGACY_COLOR_PALETTE[:n] + + for i, model_value in enumerate(model_values): + offset = (i - (n - 1) / 2) * width + bars = ax.bar( + x_positions + offset, + model_value, + width=width, + label=model_names[i], + color=colors[i % len(colors)], + ) + + for bar in bars: + y_value = bar.get_height() + ax.text( + bar.get_x() + bar.get_width() / 2, + y_value + 0.02, + f"{y_value:.2f}", + ha="center", + va="bottom", + rotation=value_text_rotation, + ) + + plt.rcParams["font.family"] = "sans-serif" + + plt.legend(loc="best") + plt.tight_layout() + plt.show()