Class variables, cost_attr, and reproducibility (#587)

* test reproducibility

* cost_attr, _eps and penalty as instance variables

test/tune/test_reproducibility.py

@@ -0,0 +1,126 @@
+from functools import partial
+
+
+def _evaluation_fn(step, width, height):
+    return (0.1 + width * step / 100) ** (-1) + height * 0.1
+
+
+def _easy_objective(use_raytune, config):
+    if use_raytune:
+        from ray import tune
+    else:
+        from flaml import tune
+    # Hyperparameters
+    width, height = config["width"], config["height"]
+
+    for step in range(config["steps"]):
+        # Iterative training function - can be any arbitrary training procedure
+        intermediate_score = _evaluation_fn(step, width, height)
+        # Feed the score back back to Tune.
+        try:
+            tune.report(iterations=step, mean_loss=intermediate_score)
+        except StopIteration:
+            print("Trial stopped", step)
+            return
+
+
+def test_tune(
+    smoke_test=True, externally_setup_searcher=False, use_ray=False, use_raytune=False
+):
+    from flaml import tune
+    from flaml.searcher.blendsearch import BlendSearch
+
+    easy_objective_custom_tune = partial(_easy_objective, use_raytune)
+    search_space = {
+        "steps": 100,
+        "width": tune.uniform(0, 20),
+        "height": tune.uniform(-100, 100),
+    }
+    if externally_setup_searcher:
+
+        searcher = BlendSearch(
+            space=search_space,
+            time_budget_s=5,
+            metric="mean_loss",
+            mode="min",
+        )
+        assert (
+            searcher.cost_attr == "time_total_s"
+        ), "when time_budget_s is provided, cost_attr should be time_total_s"
+
+        searcher = BlendSearch(
+            space=search_space,
+            num_samples=10,
+            metric="mean_loss",
+            mode="min",
+        )
+        assert (
+            searcher.cost_attr is None
+        ), "when time_budget_s is not provided, cost_attr should be None."
+
+        searcher = BlendSearch(
+            space=search_space,
+            num_samples=10,
+            time_budget_s=5,
+            metric="mean_loss",
+            mode="min",
+        )
+        assert (
+            searcher.cost_attr == "time_total_s"
+        ), "As long as time_budget_s is provided and cost_attr not otherwise specified (i.e., using the default auto value), time_total_s is used as the cost_attr"
+
+        searcher = BlendSearch(
+            space=search_space,
+            num_samples=10,
+            time_budget_s=5,
+            metric="mean_loss",
+            mode="min",
+            cost_attr=None,
+        )
+        assert (
+            searcher.cost_attr is None
+        ), "When the cost_attr is explicitly specified to be None, BS should use None as the cost_attr."
+
+        searcher = BlendSearch(
+            space=search_space,
+            metric="mean_loss",
+            mode="min",
+        )
+    else:
+        searcher = None
+
+    analysis = tune.run(
+        easy_objective_custom_tune,
+        search_alg=searcher,
+        metric="mean_loss",
+        mode="min",
+        num_samples=10,
+        time_budget_s=5,
+        use_ray=use_ray,
+        config=search_space,
+    )
+
+    print("Best hyperparameters found were: ", analysis.best_config)
+    print("best results", analysis.best_result)
+    print("best results", analysis.results)
+    return analysis.best_config
+
+
+def test_reproducibility():
+    best_config_1 = test_tune(smoke_test=True)
+    best_config_2 = test_tune(smoke_test=True)
+    print(best_config_1)
+    print(best_config_2)
+    assert best_config_1 == best_config_2, "flaml.tune not reproducible"
+
+    best_config_1 = test_tune(smoke_test=True, externally_setup_searcher=True)
+    best_config_2 = test_tune(smoke_test=True, externally_setup_searcher=True)
+    print(best_config_1)
+    print(best_config_2)
+    assert (
+        best_config_1 == best_config_2
+    ), "flaml.tune not reproducible when the searcher is set up externally"
+
+
+if __name__ == "__main__":
+    test_reproducibility()