adding evaluation (#495)

* adding automl.score

* fixing the metric name in train_with_config

* adding pickle after score

* fixing a bug in automl.pickle

test/nlp/test_autohf.py

@@ -102,6 +102,8 @@ def test_hf_data():
             y_val=y_val,
             **automl_settings
         )
+        automl.score(X_val, y_val, **{"metric": "accuracy"})
+        automl.pickle("automl.pkl")
     except requests.exceptions.HTTPError:
         return
 
@@ -113,10 +115,6 @@ def test_hf_data():
         record_id=0,
         **automl_settings
     )
-    with open("automl.pkl", "wb") as f:
-        pickle.dump(automl, f, pickle.HIGHEST_PROTOCOL)
-    with open("automl.pkl", "rb") as f:
-        automl = pickle.load(f)
     automl.predict(X_test)
     automl.predict(["test test", "test test"])
     automl.predict(
@@ -183,8 +181,6 @@ def _test_custom_data():
         ]
     )
 
-    import pickle
-
     automl.pickle("automl.pkl")
 
     with open("automl.pkl", "rb") as f:

test/nlp/test_autohf_custom_metric.py

@@ -19,7 +19,7 @@ def custom_metric(
     from flaml.model import TransformersEstimator
 
     if estimator._trainer is None:
-        trainer, _, _ = estimator._init_model_for_predict(X_test)
+        trainer, _ = estimator._init_model_for_predict()
         estimator._trainer = None
     else:
         trainer = estimator._trainer
@@ -93,6 +93,14 @@ def test_custom_metric():
 
     # testing when max_iter=1 and do retrain only without hpo
 
+    try:
+        import ray
+
+        if not ray.is_initialized():
+            ray.init()
+    except ImportError:
+        return
+
     automl_settings = {
         "gpu_per_trial": 0,
         "max_iter": 1,
@@ -100,6 +108,7 @@ def test_custom_metric():
         "task": "seq-classification",
         "metric": custom_metric,
         "log_file_name": "seqclass.log",
+        "use_ray": {"local_dir": "data/outut/"},
     }
 
     automl_settings["hf_args"] = {
@@ -126,6 +135,8 @@ def test_custom_metric():
     automl.fit(
         X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings
     )
+    automl.score(X_val, y_val, **{"metric": custom_metric})
+    automl.pickle("automl.pkl")
 
     del automl