From b7c0c242697a13a4cceae8cde9eb5156188ebc29 Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sat, 7 Jan 2023 11:41:24 -0800 Subject: [PATCH 1/4] support percentage tolerance for lexicographic --- flaml/tune/searcher/blendsearch.py | 13 ++++- flaml/tune/searcher/flow2.py | 58 +++++++++++++++++-- flaml/tune/tune.py | 28 ++++++++- test/tune/test_lexiflow.py | 10 +++- .../Examples/Tune-Lexicographic-objectives.md | 5 ++ .../Use-Cases/Tune-User-Defined-Function.md | 8 ++- 6 files changed, 108 insertions(+), 14 deletions(-) diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py index f23d35c22..95f5d2fae 100644 --- a/flaml/tune/searcher/blendsearch.py +++ b/flaml/tune/searcher/blendsearch.py @@ -124,8 +124,7 @@ class BlendSearch(Searcher): objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives. - "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the metric names (provided in "metric"), and the values are the numerical target values. - - "tolerances" (optional): a dictionary to specify the optimality tolerances on objectives. The keys are the - metric names (provided in "metrics"), and the values are the numerical tolerances values. + - "tolerances" (optional): a dictionary to specify the optimality tolerances on objectives. The keys are the metric names (provided in "metrics"), and the values are the absolute/percentage tolerance in the form of numeric/string. E.g., ```python lexico_objectives = { @@ -135,6 +134,16 @@ class BlendSearch(Searcher): "targets": {"error_rate": 0.0}, } ``` + We also support percentage tolerance. + E.g., + ```python + lexico_objectives = { + "metrics": ["error_rate", "pred_time"], + "modes": ["min", "min"], + "tolerances": {"error_rate": "5%", "pred_time": "0%"}, + "targets": {"error_rate": 0.0}, + } + ``` experimental: A bool of whether to use experimental features. """ self._eps = SEARCH_THREAD_EPS diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py index ce097ba0b..e709ebae7 100644 --- a/flaml/tune/searcher/flow2.py +++ b/flaml/tune/searcher/flow2.py @@ -80,8 +80,7 @@ class FLOW2(Searcher): objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives - "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the metric names (provided in "metric"), and the values are the numerical target values. - - "tolerances" (optional): a dictionary to specify the optimality tolerances on objectives. The keys are the - metric names (provided in "metrics"), and the values are the numerical tolerances values. + - "tolerances" (optional): a dictionary to specify the optimality tolerances on objectives. The keys are the metric names (provided in "metrics"), and the values are the absolute/percentage tolerance in the form of numeric/string. E.g., ```python lexico_objectives = { @@ -91,6 +90,16 @@ class FLOW2(Searcher): "targets": {"error_rate": 0.0}, } ``` + We also support percentage tolerance. + E.g., + ```python + lexico_objectives = { + "metrics": ["error_rate", "pred_time"], + "modes": ["min", "min"], + "tolerances": {"error_rate": "5%", "pred_time": "0%"}, + "targets": {"error_rate": 0.0}, + } + ``` """ if mode: assert mode in ["min", "max"], "`mode` must be 'min' or 'max'." @@ -369,7 +378,20 @@ class FLOW2(Searcher): <= max( [ self._f_best[k_metric] - + self.lexico_objectives["tolerances"][k_metric], + + self.lexico_objectives["tolerances"][k_metric] + if not isinstance( + self.lexico_objectives["tolerances"][k_metric], str + ) + else self._f_best[k_metric] + * ( + 1 + + 0.01 + * float( + self.lexico_objectives["tolerances"][k_metric].replace( + "%", "" + ) + ) + ), self.lexico_objectives["targets"][k_metric], ] ) @@ -400,7 +422,20 @@ class FLOW2(Searcher): < max( [ self._f_best[k_metric] - + self.lexico_objectives["tolerances"][k_metric], + + self.lexico_objectives["tolerances"][k_metric] + if not isinstance( + self.lexico_objectives["tolerances"][k_metric], str + ) + else self._f_best[k_metric] + * ( + 1 + + 0.01 + * float( + self.lexico_objectives["tolerances"][ + k_metric + ].replace("%", "") + ) + ), k_target, ] ) @@ -409,7 +444,20 @@ class FLOW2(Searcher): < max( [ self._f_best[k_metric] - + self.lexico_objectives["tolerances"][k_metric], + + self.lexico_objectives["tolerances"][k_metric] + if not isinstance( + self.lexico_objectives["tolerances"][k_metric], str + ) + else self._f_best[k_metric] + * ( + 1 + + 0.01 + * float( + self.lexico_objectives["tolerances"][ + k_metric + ].replace("%", "") + ) + ), k_target, ] ) diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py index 755e1039c..dabdf1a7c 100644 --- a/flaml/tune/tune.py +++ b/flaml/tune/tune.py @@ -96,7 +96,20 @@ class ExperimentAnalysis(EA): <= max( [ f_best[k_metric] - + self.lexico_objectives["tolerances"][k_metric], + + self.lexico_objectives["tolerances"][k_metric] + if not isinstance( + self.lexico_objectives["tolerances"][k_metric], str + ) + else f_best[k_metric] + * ( + 1 + + 0.01 + * float( + self.lexico_objectives["tolerances"][k_metric].replace( + "%", "" + ) + ) + ), k_target, ] ) @@ -401,8 +414,7 @@ def run( objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives. - "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the metric names (provided in "metric"), and the values are the numerical target values. - - "tolerances" (optional): a dictionary to specify the optimality tolerances on objectives. The keys are the - metric names (provided in "metrics"), and the values are the numerical tolerances values. + - "tolerances" (optional): a dictionary to specify the optimality tolerances on objectives. The keys are the metric names (provided in "metrics"), and the values are the absolute/percentage tolerance in the form of numeric/string. E.g., ```python lexico_objectives = { @@ -411,6 +423,16 @@ def run( "tolerances": {"error_rate": 0.01, "pred_time": 0.0}, "targets": {"error_rate": 0.0}, } + ``` + We also support percentage tolerance. + E.g., + ```python + lexico_objectives = { + "metrics": ["error_rate", "pred_time"], + "modes": ["min", "min"], + "tolerances": {"error_rate": "5%", "pred_time": "0%"}, + "targets": {"error_rate": 0.0}, + } ``` **ray_args: keyword arguments to pass to ray.tune.run(). Only valid when use_ray=True. diff --git a/test/tune/test_lexiflow.py b/test/tune/test_lexiflow.py index 23d9d564f..57fc73daf 100644 --- a/test/tune/test_lexiflow.py +++ b/test/tune/test_lexiflow.py @@ -32,7 +32,7 @@ def _BraninCurrin(config): return {"brain": brain_result, "currin": currin_result} -def test_lexiflow(): +def test_lexiflow(mode="absolute"): train_dataset = torchvision.datasets.FashionMNIST( "test/data", train=True, @@ -105,7 +105,10 @@ def test_lexiflow(): lexico_objectives = {} lexico_objectives["metrics"] = ["error_rate", "flops"] - lexico_objectives["tolerances"] = {"error_rate": 0.02, "flops": 0.0} + if mode == "absolute": + lexico_objectives["tolerances"] = {"error_rate": 0.02, "flops": 0.0} + else: + lexico_objectives["tolerances"] = {"error_rate": "10%", "flops": "0%"} lexico_objectives["targets"] = {"error_rate": 0.0, "flops": 0.0} lexico_objectives["modes"] = ["min", "min"] @@ -188,5 +191,6 @@ def test_lexiflow_performance(): if __name__ == "__main__": - test_lexiflow() + test_lexiflow(mode="absolute") + test_lexiflow(mode="percentage") test_lexiflow_performance() diff --git a/website/docs/Examples/Tune-Lexicographic-objectives.md b/website/docs/Examples/Tune-Lexicographic-objectives.md index b215c3728..de323b2b4 100644 --- a/website/docs/Examples/Tune-Lexicographic-objectives.md +++ b/website/docs/Examples/Tune-Lexicographic-objectives.md @@ -162,5 +162,10 @@ analysis = tune.run( ) ``` +We also support providing percentage tolerance as shown below. + +```python +lexico_objectives["tolerances"] = {"error_rate": "5%", "flops": "0%"} +``` [Link to notebook](https://github.com/microsoft/FLAML/blob/main/notebook/tune_lexicographic.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/FLAML/blob/main/notebook/tune_lexicographic.ipynb) diff --git a/website/docs/Use-Cases/Tune-User-Defined-Function.md b/website/docs/Use-Cases/Tune-User-Defined-Function.md index 858d2d07c..d67ce5fe9 100644 --- a/website/docs/Use-Cases/Tune-User-Defined-Function.md +++ b/website/docs/Use-Cases/Tune-User-Defined-Function.md @@ -539,7 +539,7 @@ We support tuning multiple objectives with lexicographic preference by providing `lexico_objectives` is a dictionary that contains the following fields of key-value pairs: - `metrics`: a list of optimization objectives with the orders reflecting the priorities/preferences of the objectives. - `modes`: (optional) a list of optimization modes (each mode either "min" or "max") corresponding to the objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives. - - `tolerances`: (optional) a dictionary to specify the optimality tolerances on objectives. The keys are the metric names (provided in "metrics"), and the values are the numerical tolerances values. + - `tolerances`: (optional) a dictionary to specify the optimality tolerances on objectives. The keys are the metric names (provided in "metrics"), and the values are the absolute/percentage tolerance in the form of numeric/string. - `targets`: (optional) a dictionary to specify the optimization targets on the objectives. The keys are the metric names (provided in "metric"), and the values are the numerical target values. In the following example, we want to minimize `val_loss` and `pred_time` of the model where `val_loss` has high priority. The tolerances for `val_loss` and `pre_time` are 0.02 and 0 respectively. We do not set targets for these two objectives and we set them to -inf for both objectives. @@ -554,6 +554,12 @@ lexico_objectives["targets"] = {"val_loss": -float('inf'), "pred_time": -float(' # provide the lexico_objectives to tune.run tune.run(..., search_alg=None, lexico_objectives=lexico_objectives) ``` + +We also supports providing percentage tolerance as shown below. + +```python +lexico_objectives["tolerances"] = {"val_loss": "10%", "pred_time": "0%"} +``` NOTE: 1. When lexico_objectives is not None, the arguments metric, mode, will be invalid, and flaml's tune uses CFO as the `search_alg`, which makes the input (if provided) `search_alg` invalid. From 3a68da87742c040bbd1321bf435c57d4ccd3be43 Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Tue, 17 Jan 2023 06:49:59 -0800 Subject: [PATCH 2/4] update --- flaml/tune/searcher/flow2.py | 90 +++++++++++++++--------------------- test/tune/test_lexiflow.py | 38 +++++++++------ 2 files changed, 63 insertions(+), 65 deletions(-) diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py index e709ebae7..e656c317a 100644 --- a/flaml/tune/searcher/flow2.py +++ b/flaml/tune/searcher/flow2.py @@ -373,25 +373,27 @@ class FLOW2(Searcher): k_values = np.array(self._histories[k_metric]) feasible_value = k_values.take(feasible_index) self._f_best[k_metric] = np.min(feasible_value) + if not isinstance(self.lexico_objectives["tolerances"][k_metric], str): + tolerance_bound = ( + self._f_best[k_metric] + + self.lexico_objectives["tolerances"][k_metric] + ) + else: + assert ( + self.lexico_objectives["tolerances"][k_metric][-1] == "%" + ), "String tolerance of {} should use %% as the suffix".format(k_metric) + tolerance_bound = self._f_best[k_metric] * ( + 1 + + 0.01 + * float( + self.lexico_objectives["tolerances"][k_metric].replace("%", "") + ) + ) feasible_index_filter = np.where( feasible_value <= max( [ - self._f_best[k_metric] - + self.lexico_objectives["tolerances"][k_metric] - if not isinstance( - self.lexico_objectives["tolerances"][k_metric], str - ) - else self._f_best[k_metric] - * ( - 1 - + 0.01 - * float( - self.lexico_objectives["tolerances"][k_metric].replace( - "%", "" - ) - ) - ), + tolerance_bound, self.lexico_objectives["targets"][k_metric], ] ) @@ -417,47 +419,31 @@ class FLOW2(Searcher): if k_mode == "min" else -self.lexico_objectives["targets"][k_metric] ) - if ( - result[k_metric] - < max( - [ - self._f_best[k_metric] - + self.lexico_objectives["tolerances"][k_metric] - if not isinstance( - self.lexico_objectives["tolerances"][k_metric], str - ) - else self._f_best[k_metric] - * ( - 1 - + 0.01 - * float( - self.lexico_objectives["tolerances"][ - k_metric - ].replace("%", "") - ) - ), - k_target, - ] + if not isinstance(self.lexico_objectives["tolerances"][k_metric], str): + tolerance_bound = ( + self._f_best[k_metric] + + self.lexico_objectives["tolerances"][k_metric] ) - ) and ( + else: + assert ( + self.lexico_objectives["tolerances"][k_metric][-1] == "%" + ), "String tolerance of {} should use %% as the suffix".format( + k_metric + ) + tolerance_bound = self._f_best[k_metric] * ( + 1 + + 0.01 + * float( + self.lexico_objectives["tolerances"][k_metric].replace( + "%", "" + ) + ) + ) + if (result[k_metric] < max([tolerance_bound, k_target])) and ( self.best_obj[k_metric] < max( [ - self._f_best[k_metric] - + self.lexico_objectives["tolerances"][k_metric] - if not isinstance( - self.lexico_objectives["tolerances"][k_metric], str - ) - else self._f_best[k_metric] - * ( - 1 - + 0.01 - * float( - self.lexico_objectives["tolerances"][ - k_metric - ].replace("%", "") - ) - ), + tolerance_bound, k_target, ] ) diff --git a/test/tune/test_lexiflow.py b/test/tune/test_lexiflow.py index 57fc73daf..c366421ee 100644 --- a/test/tune/test_lexiflow.py +++ b/test/tune/test_lexiflow.py @@ -32,7 +32,7 @@ def _BraninCurrin(config): return {"brain": brain_result, "currin": currin_result} -def test_lexiflow(mode="absolute"): +def test_lexiflow(): train_dataset = torchvision.datasets.FashionMNIST( "test/data", train=True, @@ -105,12 +105,6 @@ def test_lexiflow(mode="absolute"): lexico_objectives = {} lexico_objectives["metrics"] = ["error_rate", "flops"] - if mode == "absolute": - lexico_objectives["tolerances"] = {"error_rate": 0.02, "flops": 0.0} - else: - lexico_objectives["tolerances"] = {"error_rate": "10%", "flops": "0%"} - lexico_objectives["targets"] = {"error_rate": 0.0, "flops": 0.0} - lexico_objectives["modes"] = ["min", "min"] search_space = { "n_layers": tune.randint(lower=1, upper=3), @@ -132,7 +126,27 @@ def test_lexiflow(mode="absolute"): "n_epoch": 1, } + # Non lexico tune + analysis = tune.run( + evaluate_function, + metric="error_rate", + mode="min", + num_samples=5, + config=search_space, + use_ray=False, + lexico_objectives=None, + low_cost_partial_config=low_cost_partial_config, + ) + print(analysis.best_trial) + print(analysis.best_config) + print(analysis.best_result) + # lexico tune + lexico_objectives["targets"] = {"error_rate": 0.0, "flops": 0.0} + lexico_objectives["modes"] = ["min", "min"] + + # 1. lexico tune: absoute tune + lexico_objectives["tolerances"] = {"error_rate": 0.02, "flops": 0.0} analysis = tune.run( evaluate_function, num_samples=5, @@ -145,15 +159,14 @@ def test_lexiflow(mode="absolute"): print(analysis.best_config) print(analysis.best_result) - # Non lexico tune + # 2. lexico tune: percentage tolerance + lexico_objectives["tolerances"] = {"error_rate": "10%", "flops": "0%"} analysis = tune.run( evaluate_function, - metric="error_rate", - mode="min", num_samples=5, config=search_space, use_ray=False, - lexico_objectives=None, + lexico_objectives=lexico_objectives, low_cost_partial_config=low_cost_partial_config, ) print(analysis.best_trial) @@ -191,6 +204,5 @@ def test_lexiflow_performance(): if __name__ == "__main__": - test_lexiflow(mode="absolute") - test_lexiflow(mode="percentage") + test_lexiflow() test_lexiflow_performance() From 184251a2a72ebe568040051ba88a4a3141660d7f Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sat, 28 Jan 2023 06:53:37 -0800 Subject: [PATCH 3/4] update --- flaml/tune/searcher/flow2.py | 14 +++++--------- test/tune/test_lexiflow.py | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py index e656c317a..035e3d868 100644 --- a/flaml/tune/searcher/flow2.py +++ b/flaml/tune/searcher/flow2.py @@ -392,10 +392,8 @@ class FLOW2(Searcher): feasible_index_filter = np.where( feasible_value <= max( - [ - tolerance_bound, - self.lexico_objectives["targets"][k_metric], - ] + tolerance_bound, + self.lexico_objectives["targets"][k_metric], ) )[0] feasible_index = feasible_index.take(feasible_index_filter) @@ -439,13 +437,11 @@ class FLOW2(Searcher): ) ) ) - if (result[k_metric] < max([tolerance_bound, k_target])) and ( + if (result[k_metric] < max(tolerance_bound, k_target)) and ( self.best_obj[k_metric] < max( - [ - tolerance_bound, - k_target, - ] + tolerance_bound, + k_target, ) ): continue diff --git a/test/tune/test_lexiflow.py b/test/tune/test_lexiflow.py index c366421ee..cad772af7 100644 --- a/test/tune/test_lexiflow.py +++ b/test/tune/test_lexiflow.py @@ -145,7 +145,7 @@ def test_lexiflow(): lexico_objectives["targets"] = {"error_rate": 0.0, "flops": 0.0} lexico_objectives["modes"] = ["min", "min"] - # 1. lexico tune: absoute tune + # 1. lexico tune: absolute tolerance lexico_objectives["tolerances"] = {"error_rate": 0.02, "flops": 0.0} analysis = tune.run( evaluate_function, From b8982f7cf2a1d95489bf99fae2c622b9838fcab8 Mon Sep 17 00:00:00 2001 From: skzhang1 Date: Sat, 28 Jan 2023 06:58:58 -0800 Subject: [PATCH 4/4] update --- flaml/tune/tune.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py index 47f6fa079..57b930d3d 100644 --- a/flaml/tune/tune.py +++ b/flaml/tune/tune.py @@ -91,27 +91,25 @@ class ExperimentAnalysis(EA): ) feasible_value = k_values.take(feasible_index) f_best[k_metric] = np.min(feasible_value) + feasible_index_filter = np.where( feasible_value <= max( - [ - f_best[k_metric] - + self.lexico_objectives["tolerances"][k_metric] - if not isinstance( - self.lexico_objectives["tolerances"][k_metric], str - ) - else f_best[k_metric] - * ( - 1 - + 0.01 - * float( - self.lexico_objectives["tolerances"][k_metric].replace( - "%", "" - ) + f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric] + if not isinstance( + self.lexico_objectives["tolerances"][k_metric], str + ) + else f_best[k_metric] + * ( + 1 + + 0.01 + * float( + self.lexico_objectives["tolerances"][k_metric].replace( + "%", "" ) - ), - k_target, - ] + ) + ), + k_target, ) )[0] feasible_index = feasible_index.take(feasible_index_filter)