From 730fd14ef632803aae2569b8d0251f58e7cfbffe Mon Sep 17 00:00:00 2001 From: Gian Pio Domiziani Date: Mon, 26 Apr 2021 20:50:41 +0200 Subject: [PATCH] micro/macro f1 metrics added. (#80) * micro/macro f1 metrics added. * format lines. --- flaml/automl.py | 4 ++-- flaml/ml.py | 8 ++++++-- test/test_automl.py | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/flaml/automl.py b/flaml/automl.py index ab52821f0..042380438 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -809,7 +809,7 @@ class AutoML: dataframe and label are ignored; If not, dataframe and label must be provided. metric: A string of the metric name or a function, - e.g., 'accuracy', 'roc_auc', 'f1', 'log_loss', 'mae', 'mse', 'r2' + e.g., 'accuracy', 'roc_auc', 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2' if passing a customized metric function, the function needs to have the follwing signature: @@ -902,7 +902,7 @@ class AutoML: else: metric = 'r2' self._state.metric = metric - if metric in ['r2', 'accuracy', 'roc_auc', 'f1', 'ap']: + if metric in ['r2', 'accuracy', 'roc_auc', 'f1', 'ap', 'micro_f1', 'macro_f1']: error_metric = f"1-{metric}" elif isinstance(metric, str): error_metric = metric diff --git a/flaml/ml.py b/flaml/ml.py index 18c0971c2..8b939518d 100644 --- a/flaml/ml.py +++ b/flaml/ml.py @@ -54,9 +54,9 @@ def sklearn_metric_loss_score( '''Loss using the specified metric Args: - metric_name: A string of the mtric name, one of + metric_name: A string of the metric name, one of 'r2', 'rmse', 'mae', 'mse', 'accuracy', 'roc_auc', 'log_loss', - 'f1', 'ap' + 'f1', 'ap', 'micro_f1', 'macro_f1' y_predict: A 1d or 2d numpy array of the predictions which can be used to calculate the metric. E.g., 2d for log_loss and 1d for others. @@ -88,6 +88,10 @@ def sklearn_metric_loss_score( elif 'log_loss' in metric_name: score = log_loss( y_true, y_predict, labels=labels, sample_weight=sample_weight) + elif 'micro_f1' in metric_name: + score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight, average='micro') + elif 'macro_f1' in metric_name: + score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight, average='macro') elif 'f1' in metric_name: score = 1 - f1_score(y_true, y_predict, sample_weight=sample_weight) elif 'ap' in metric_name: diff --git a/test/test_automl.py b/test/test_automl.py index 94a4452c6..5d8a17e2e 100644 --- a/test/test_automl.py +++ b/test/test_automl.py @@ -241,6 +241,23 @@ class TestAutoML(unittest.TestCase): y_pred = automl_experiment.predict(fake_df) + def test_micro_macro_f1(self): + automl_experiment = AutoML() + automl_experiment_macro = AutoML() + + automl_settings = { + "time_budget": 2, + "task": 'classification', + "log_file_name": "test/micro_macro_f1.log", + "log_training_metric": True, + "n_jobs": 1, + "model_history": True + } + + X_train, y_train = load_iris(return_X_y=True) + automl_experiment.fit(X_train=X_train, y_train=y_train, metric='micro_f1', **automl_settings) + automl_experiment_macro.fit(X_train=X_train, y_train=y_train, metric='macro_f1', **automl_settings) + def test_regression(self): automl_experiment = AutoML()