diff --git a/flaml/automl.py b/flaml/automl.py index 74d60ea9a..c14772c06 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -685,7 +685,6 @@ class AutoML(BaseEstimator): fit_kwargs_by_estimator = { "transformer": { "output_dir": "test/data/output/", - "ckpt_per_epoch": 1, "fp16": False, } } @@ -1626,7 +1625,6 @@ class AutoML(BaseEstimator): fit_kwargs_by_estimator = { "transformer": { "output_dir": "test/data/output/", - "ckpt_per_epoch": 1, "fp16": False, } } @@ -2298,7 +2296,6 @@ class AutoML(BaseEstimator): fit_kwargs_by_estimator = { "transformer": { "output_dir": "test/data/output/", - "ckpt_per_epoch": 1, "fp16": False, } } diff --git a/flaml/model.py b/flaml/model.py index 30b9d0969..a11e6668c 100644 --- a/flaml/model.py +++ b/flaml/model.py @@ -417,32 +417,18 @@ class TransformersEstimator(BaseEstimator): def search_space(cls, data_size, task, **params): search_space_dict = { "learning_rate": { - "domain": tune.choice( - [1e-6, 2e-6, 4e-6, 8e-6, 16e-6, 32e-6, 64e-6, 128e-6] - ), - "init_value": 8e-6, + "domain": tune.loguniform(1e-6, 1e-4), + "init_value": 1e-5, }, "num_train_epochs": { - "domain": tune.choice([1, 3, 5, 7, 9]), + "domain": tune.choice([1, 2, 3, 4, 5]), "init_value": 3.0, # to be consistent with roberta }, "per_device_train_batch_size": { - "domain": tune.choice([4, 8, 16, 32]), + "domain": tune.choice([4, 8, 16, 32, 64]), "init_value": 32, }, - "warmup_ratio": { - "domain": tune.choice([0, 0.1, 0.2, 0.3]), - "init_value": 0.0, - }, - "weight_decay": { - "domain": tune.choice([0, 0.1, 0.2, 0.3]), - "init_value": 0.0, - }, - "adam_epsilon": { - "domain": tune.choice([1e-8, 1e-7, 1e-6]), - "init_value": 1e-6, - }, - "seed": {"domain": tune.randint(40, 45), "init_value": 42}, + "seed": {"domain": tune.randint(1, 40), "init_value": 20}, "global_max_steps": { "domain": sys.maxsize, "init_value": sys.maxsize, @@ -451,18 +437,6 @@ class TransformersEstimator(BaseEstimator): return search_space_dict - @property - def checkpoint_freq(self): - return ( - int( - min(self._training_args.num_train_epochs, 1) - * len(self._X_train) - / self._training_args.per_device_train_batch_size - / self._training_args.ckpt_per_epoch - ) - + 1 - ) - @property def fp16(self): return self._kwargs.get("gpu_per_trial") and self._training_args.fp16 @@ -512,9 +486,6 @@ class TransformersEstimator(BaseEstimator): local_dir, self.params, self.trial_id ) - self._training_args.eval_steps = ( - self._training_args.logging_steps - ) = self._training_args.saving_steps = self.checkpoint_freq self._training_args.fp16 = self.fp16 self._training_args.no_cuda = self.no_cuda @@ -762,7 +733,7 @@ class TransformersEstimator(BaseEstimator): if trainer.ckpt_to_metric: best_ckpt, _ = min( - trainer.ckpt_to_metric.items(), key=lambda x: x[1]["eval_loss"] + trainer.ckpt_to_metric.items(), key=lambda x: x[1]["eval_automl_metric"] ) best_ckpt_global_step = trainer.ckpt_to_global_step[best_ckpt] for each_ckpt in list(trainer.ckpt_to_metric): diff --git a/flaml/nlp/huggingface/trainer.py b/flaml/nlp/huggingface/trainer.py index b9ac0718a..96151c31c 100644 --- a/flaml/nlp/huggingface/trainer.py +++ b/flaml/nlp/huggingface/trainer.py @@ -19,9 +19,9 @@ class TrainerForAuto(Seq2SeqTrainer): return super().predict( test_dataset, ignore_keys, - metric_key_prefix, - max_length, - num_beams, + metric_key_prefix=metric_key_prefix, + max_length=max_length, + num_beams=num_beams, ) else: return super(Seq2SeqTrainer, self).predict( diff --git a/flaml/nlp/huggingface/training_args.py b/flaml/nlp/huggingface/training_args.py index 88533da0d..c3eb839b6 100644 --- a/flaml/nlp/huggingface/training_args.py +++ b/flaml/nlp/huggingface/training_args.py @@ -28,7 +28,6 @@ class TrainingArgumentsForAuto(TrainingArguments): pad_to_max_length (bool, optional, defaults to "False"): whether to pad all samples to model maximum sentence length. If False, will pad the samples dynamically when batching to the maximum length in the batch. - ckpt_per_epoch (int, optional, defaults to 1): An integer, the number of checkpoints per epoch. per_device_eval_batch_size (int, optional, defaults to 1): An integer, the per gpu evaluation batch size. label_list (List[str], optional, defaults to None): A list of string, the string list of the label names. When the task is sequence labeling/token classification, there are two formats of the labels: @@ -67,8 +66,6 @@ class TrainingArgumentsForAuto(TrainingArguments): }, ) - ckpt_per_epoch: int = field(default=1, metadata={"help": "checkpoint per epoch"}) - per_device_eval_batch_size: int = field( default=1, metadata={"help": "per gpu evaluation batch size"}, @@ -78,6 +75,18 @@ class TrainingArgumentsForAuto(TrainingArguments): default=None, metadata={"help": "The string list of the label names. "} ) + eval_steps: int = field( + default=500, metadata={"help": "Run an evaluation every X steps."} + ) + + save_steps: int = field( + default=500, metadata={"help": "Save checkpoint every X updates steps."} + ) + + logging_steps: int = field( + default=500, metadata={"help": "Log every X updates steps."} + ) + @staticmethod def load_args_from_console(): from dataclasses import fields diff --git a/notebook/automl_nlp.ipynb b/notebook/automl_nlp.ipynb index 3f1ef2356..60bc7e736 100644 --- a/notebook/automl_nlp.ipynb +++ b/notebook/automl_nlp.ipynb @@ -223,18 +223,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n", - "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n", - "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n" + "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n", + "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n", + "Reusing dataset glue (/home/xliu127/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n" ] } ], "source": [ "from datasets import load_dataset\n", "\n", - "train_dataset = load_dataset(\"glue\", \"sst2\", split=\"train\").to_pandas().iloc[:10000]\n", - "dev_dataset = load_dataset(\"glue\", \"sst2\", split=\"validation\").to_pandas().iloc[:10000]\n", - "test_dataset = load_dataset(\"glue\", \"sst2\", split=\"test\").to_pandas()" + "train_dataset = load_dataset(\"glue\", \"mrpc\", split=\"train\").to_pandas()\n", + "dev_dataset = load_dataset(\"glue\", \"mrpc\", split=\"validation\").to_pandas()\n", + "test_dataset = load_dataset(\"glue\", \"mrpc\", split=\"test\").to_pandas()" ] }, { @@ -341,7 +341,7 @@ "metadata": {}, "outputs": [], "source": [ - "custom_sent_keys = [\"sentence\"] # specify the column names of the input sentences\n", + "custom_sent_keys = [\"sentence1\", \"sentence2\"] # specify the column names of the input sentences\n", "label_key = \"label\" # specify the column name of the label\n", "\n", "X_train, y_train = train_dataset[custom_sent_keys], train_dataset[label_key]\n", @@ -358,26 +358,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/xgboost/compat.py:31: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n", - " from pandas import MultiIndex, Int64Index\n", - "2022-03-19 13:59:38,805\tERROR services.py:1421 -- Failed to start the dashboard: Failed to start the dashboard, return code 1\n", - "Failed to read dashboard log: [Errno 2] No such file or directory: '/tmp/ray/session_2022-03-19_13-59-37_048935_85869/logs/dashboard.log'\n", - "\u001b[2m\u001b[33m(raylet)\u001b[0m Traceback (most recent call last):\n", - "\u001b[2m\u001b[33m(raylet)\u001b[0m File \"/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/ray/dashboard/agent.py\", line 21, in \n", - "\u001b[2m\u001b[33m(raylet)\u001b[0m import ray.dashboard.utils as dashboard_utils\n", - "\u001b[2m\u001b[33m(raylet)\u001b[0m File \"/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/ray/dashboard/utils.py\", line 15, in \n", - "\u001b[2m\u001b[33m(raylet)\u001b[0m import aioredis # noqa: F401\n", - "\u001b[2m\u001b[33m(raylet)\u001b[0m ModuleNotFoundError: No module named 'aioredis'\n" - ] - } - ], + "outputs": [], "source": [ "''' import AutoML class from flaml package '''\n", "from flaml import AutoML\n", @@ -390,26 +373,24 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "\n", + "TIME_BUDGET=1200\n", "automl_settings = {\n", - " \"time_budget\": 500, # setting the time budget\n", + " \"time_budget\": TIME_BUDGET, # setting the time budget\n", " \"task\": \"seq-classification\", # setting the task as seq-classification\n", " \"fit_kwargs_by_estimator\": {\n", " \"transformer\": {\n", " \"output_dir\": \"data/output/\", # setting the output directory\n", - " \"ckpt_per_epoch\": 1, # setting the number of checkoints per epoch\n", - " \"model_path\": \"google/electra-base-discriminator\", # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base\n", - " \"per_device_eval_batch_size\": 16 # the batch size for validation (inference)\n", + " \"model_path\": \"bert-base-uncased\", # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base\n", " }\n", " },\n", " \"gpu_per_trial\": 1, # set to 0 if no GPU is available\n", " \"log_file_name\": \"seqclass.log\", # set the file to save the log for HPO\n", " \"log_type\": \"all\", # the log type for trials: \"all\" if logging all the trials, \"better\" if only keeping the better trials\n", - " \"use_ray\": True, # set whether to use Ray\n", + " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n", " \"n_concurrent_trials\": 4,\n", " \"keep_search_state\": True, # keeping the search state\n", "}" @@ -417,13 +398,13 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "== Status ==
Current time: 2022-03-19 14:08:10 (running for 00:08:23.62)
Memory usage on this node: 26.2/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/250.17 GiB heap, 0.0/111.21 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: 8275ca0a with val_loss=0.059633027522935755 and parameters={'learning_rate': 1.4736175808553141e-05, 'num_train_epochs': 7.623375372739029, 'per_device_train_batch_size': 16, 'warmup_ratio': 0.21605876280261357, 'weight_decay': 0.11938244526496489, 'adam_epsilon': 7.353322403647365e-07, 'seed': 42, 'global_max_steps': 9223372036854775807, 'learner': 'transformer'}
Result logdir: /home/xliu127/ray_results/train_2022-03-19_13-59-47
Number of trials: 14/1000000 (14 TERMINATED)

" + "== Status ==
Current time: 2022-07-21 13:27:54 (running for 00:20:14.52)
Memory usage on this node: 22.9/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/96 CPUs, 0/4 GPUs, 0.0/252.58 GiB heap, 0.0/112.24 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: a478b276 with val_loss=0.12009803921568629 and parameters={'learning_rate': 2.1872511767624938e-05, 'num_train_epochs': 3, 'per_device_train_batch_size': 4, 'seed': 7, 'global_max_steps': 9223372036854775807, 'learner': 'transformer'}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38
Number of trials: 33/1000000 (33 TERMINATED)

" ], "text/plain": [ "" @@ -436,164 +417,118 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86265)\u001b[0m {'loss': 0.688, 'learning_rate': 5.91472422187606e-05, 'epoch': 6.0}\n", - "\u001b[2m\u001b[36m(train pid=86265)\u001b[0m {'eval_loss': 0.6975926160812378, 'eval_automl_metric': 0.49082568807339455, 'eval_runtime': 0.9885, 'eval_samples_per_second': 882.13, 'eval_steps_per_second': 55.639, 'epoch': 6.0}\n", - "\u001b[2m\u001b[36m(train pid=86265)\u001b[0m {'eval_loss': 0.6959978938102722, 'eval_automl_metric': 0.49082568807339455, 'eval_runtime': 0.9612, 'eval_samples_per_second': 907.223, 'eval_steps_per_second': 57.222, 'epoch': 6.04}\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m {'loss': 0.133, 'learning_rate': 1.9966093201540806e-05, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m {'eval_loss': 0.29954516887664795, 'eval_automl_metric': 0.06880733944954132, 'eval_runtime': 0.9772, 'eval_samples_per_second': 892.372, 'eval_steps_per_second': 56.285, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=86265)\u001b[0m {'eval_loss': 0.6959978938102722, 'eval_automl_metric': 0.49082568807339455, 'eval_runtime': 0.9707, 'eval_samples_per_second': 898.292, 'eval_steps_per_second': 56.658, 'epoch': 6.04}\n", - "\u001b[2m\u001b[36m(train pid=86265)\u001b[0m {'train_runtime': 510.9986, 'train_samples_per_second': 142.692, 'train_steps_per_second': 17.838, 'train_loss': 0.6829553683756834, 'epoch': 6.04}\n" + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m {'loss': 0.3327, 'learning_rate': 1.0295123460382672e-05, 'epoch': 2.18}\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m {'loss': 0.4744, 'learning_rate': 3.8483947490148965e-06, 'epoch': 1.64}\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m {'loss': 0.1645, 'learning_rate': 4.8933330345560835e-06, 'epoch': 3.27}\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m {'loss': 0.5619, 'learning_rate': 1.3544721826857925e-05, 'epoch': 0.55}\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m {'eval_loss': 0.5415804386138916, 'eval_automl_metric': 0.13480392156862742, 'eval_runtime': 3.8874, 'eval_samples_per_second': 104.956, 'eval_steps_per_second': 104.956, 'epoch': 2.0}\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m {'train_runtime': 133.4439, 'train_samples_per_second': 54.974, 'train_steps_per_second': 13.744, 'train_loss': 0.4956582546754128, 'epoch': 2.0}\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m {'loss': 0.2012, 'learning_rate': 7.209055276814724e-06, 'epoch': 2.73}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86265)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86265)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86265)\u001b[0m Num examples = 872\n", - "\u001b[2m\u001b[36m(train pid=86265)\u001b[0m Batch size = 16\n" + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m Num examples = 408\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m Batch size = 1\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_5c51b244_32_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-26-12/checkpoint-1834/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_5c51b244_32_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-26-12/checkpoint-1834/vocab.txt\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_5c51b244_32_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-26-12/checkpoint-1834/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_5c51b244_32_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-26-12/checkpoint-1834/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=59407)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_5c51b244_32_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-26-12/checkpoint-1834/tokenizer_config.json\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'loss': 0.2158, 'learning_rate': 5.454224012822699e-06, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'eval_loss': 0.21694229543209076, 'eval_automl_metric': 0.06422018348623848, 'eval_runtime': 0.9902, 'eval_samples_per_second': 880.648, 'eval_steps_per_second': 55.545, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=86269)\u001b[0m {'eval_loss': 0.5855236649513245, 'eval_automl_metric': 0.18463302752293576, 'eval_runtime': 0.9877, 'eval_samples_per_second': 882.845, 'eval_steps_per_second': 55.684, 'epoch': 4.53}\n", - "\u001b[2m\u001b[36m(train pid=86269)\u001b[0m {'train_runtime': 373.6856, 'train_samples_per_second': 121.13, 'train_steps_per_second': 15.144, 'train_loss': 0.4671156674776214, 'epoch': 4.53}\n" + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m {'eval_loss': 0.5180040001869202, 'eval_automl_metric': 0.15931372549019607, 'eval_runtime': 3.7749, 'eval_samples_per_second': 108.082, 'eval_steps_per_second': 108.082, 'epoch': 1.0}\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m {'loss': 0.0951, 'learning_rate': 1.2306586074931466e-06, 'epoch': 3.82}\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m {'loss': 0.4703, 'learning_rate': 8.46798950794566e-06, 'epoch': 1.09}\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m {'eval_loss': 0.6077207326889038, 'eval_automl_metric': 0.11764705882352944, 'eval_runtime': 3.8401, 'eval_samples_per_second': 106.247, 'eval_steps_per_second': 106.247, 'epoch': 3.0}\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m {'eval_loss': 0.806081235408783, 'eval_automl_metric': 0.13480392156862742, 'eval_runtime': 3.7716, 'eval_samples_per_second': 108.177, 'eval_steps_per_second': 108.177, 'epoch': 4.0}\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m {'train_runtime': 256.0423, 'train_samples_per_second': 57.303, 'train_steps_per_second': 14.326, 'train_loss': 0.35705690737509027, 'epoch': 4.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86269)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86269)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86269)\u001b[0m Num examples = 872\n", - "\u001b[2m\u001b[36m(train pid=86269)\u001b[0m Batch size = 16\n" + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m Num examples = 408\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m Batch size = 1\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_227f7ab0_29_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-24-35/checkpoint-3668/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_227f7ab0_29_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-24-35/checkpoint-3668/vocab.txt\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_227f7ab0_29_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-24-35/checkpoint-3668/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_227f7ab0_29_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-24-35/checkpoint-3668/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=58929)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_227f7ab0_29_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-24-35/checkpoint-3668/tokenizer_config.json\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m {'loss': 0.0655, 'learning_rate': 1.1451941740938257e-05, 'epoch': 4.01}\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m {'eval_loss': 0.32505378127098083, 'eval_automl_metric': 0.06307339449541283, 'eval_runtime': 0.9623, 'eval_samples_per_second': 906.171, 'eval_steps_per_second': 57.155, 'epoch': 4.01}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'loss': 0.1585, 'learning_rate': 4.673980815375941e-06, 'epoch': 4.01}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'eval_loss': 0.26288139820098877, 'eval_automl_metric': 0.06422018348623848, 'eval_runtime': 0.9914, 'eval_samples_per_second': 879.599, 'eval_steps_per_second': 55.479, 'epoch': 4.01}\n" + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m {'loss': 0.1476, 'learning_rate': 4.122987093246777e-06, 'epoch': 3.27}\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m {'loss': 0.3574, 'learning_rate': 3.391257189033394e-06, 'epoch': 1.64}\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m {'eval_loss': 0.5781316757202148, 'eval_automl_metric': 0.1421568627450981, 'eval_runtime': 3.7277, 'eval_samples_per_second': 109.45, 'eval_steps_per_second': 109.45, 'epoch': 2.0}\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m {'train_runtime': 128.6158, 'train_samples_per_second': 57.038, 'train_steps_per_second': 14.26, 'train_loss': 0.4407762508745932, 'epoch': 2.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2022-03-19 14:09:46,075\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n" + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m Num examples = 408\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m Batch size = 1\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m {'eval_loss': 0.37855082750320435, 'eval_automl_metric': 0.06536697247706424, 'eval_runtime': 0.9622, 'eval_samples_per_second': 906.215, 'eval_steps_per_second': 57.158, 'epoch': 5.0}\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m {'loss': 0.0323, 'learning_rate': 2.937790280335705e-06, 'epoch': 5.01}\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m {'eval_loss': 0.38091975450515747, 'eval_automl_metric': 0.06536697247706424, 'eval_runtime': 0.9618, 'eval_samples_per_second': 906.667, 'eval_steps_per_second': 57.187, 'epoch': 5.01}\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m {'eval_loss': 0.4038548171520233, 'eval_automl_metric': 0.06995412844036697, 'eval_runtime': 0.9613, 'eval_samples_per_second': 907.15, 'eval_steps_per_second': 57.217, 'epoch': 5.35}\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m {'train_runtime': 240.3679, 'train_samples_per_second': 222.334, 'train_steps_per_second': 13.9, 'train_loss': 0.1605183750667532, 'epoch': 5.35}\n" + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m {'loss': 0.1029, 'learning_rate': 1.03691890967883e-06, 'epoch': 3.82}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m Num examples = 872\n", - "\u001b[2m\u001b[36m(train pid=86255)\u001b[0m Batch size = 16\n" + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_89b0b9a7_33_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-27-29/checkpoint-1834/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_89b0b9a7_33_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-27-29/checkpoint-1834/vocab.txt\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_89b0b9a7_33_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-27-29/checkpoint-1834/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_89b0b9a7_33_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-27-29/checkpoint-1834/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=59592)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_89b0b9a7_33_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=2,per_device_trai_2022-07-21_13-27-29/checkpoint-1834/tokenizer_config.json\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'loss': 0.1288, 'learning_rate': 3.893737617929184e-06, 'epoch': 5.01}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'eval_loss': 0.2783028781414032, 'eval_automl_metric': 0.06536697247706424, 'eval_runtime': 0.9904, 'eval_samples_per_second': 880.481, 'eval_steps_per_second': 55.535, 'epoch': 5.01}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'loss': 0.0921, 'learning_rate': 3.114740815366527e-06, 'epoch': 6.01}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'eval_loss': 0.2939322590827942, 'eval_automl_metric': 0.06422018348623848, 'eval_runtime': 0.9917, 'eval_samples_per_second': 879.284, 'eval_steps_per_second': 55.459, 'epoch': 6.01}\n" + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m {'eval_loss': 0.7132703065872192, 'eval_automl_metric': 0.12254901960784315, 'eval_runtime': 3.735, 'eval_samples_per_second': 109.237, 'eval_steps_per_second': 109.237, 'epoch': 4.0}\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m {'train_runtime': 264.9924, 'train_samples_per_second': 55.368, 'train_steps_per_second': 13.842, 'train_loss': 0.31339427247042223, 'epoch': 4.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2022-03-19 14:11:16,956\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'loss': 0.0749, 'learning_rate': 2.3357440128038706e-06, 'epoch': 7.01}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'eval_loss': 0.3375593423843384, 'eval_automl_metric': 0.0665137614678899, 'eval_runtime': 1.0158, 'eval_samples_per_second': 858.42, 'eval_steps_per_second': 54.143, 'epoch': 7.01}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 14:12:16,966\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'loss': 0.06, 'learning_rate': 1.5555008153571132e-06, 'epoch': 8.01}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'eval_loss': 0.33065176010131836, 'eval_automl_metric': 0.06307339449541283, 'eval_runtime': 0.9921, 'eval_samples_per_second': 878.905, 'eval_steps_per_second': 55.436, 'epoch': 8.01}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 14:13:16,976\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'eval_loss': 0.34692466259002686, 'eval_automl_metric': 0.06307339449541283, 'eval_runtime': 0.9923, 'eval_samples_per_second': 878.777, 'eval_steps_per_second': 55.427, 'epoch': 9.0}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'loss': 0.0547, 'learning_rate': 7.752576179103561e-07, 'epoch': 9.01}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'eval_loss': 0.3511528968811035, 'eval_automl_metric': 0.06307339449541283, 'eval_runtime': 1.0097, 'eval_samples_per_second': 863.616, 'eval_steps_per_second': 54.471, 'epoch': 9.01}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-03-19 14:14:16,986\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'eval_loss': 0.36141160130500793, 'eval_automl_metric': 0.06192660550458717, 'eval_runtime': 0.9929, 'eval_samples_per_second': 878.273, 'eval_steps_per_second': 55.396, 'epoch': 10.0}\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m {'train_runtime': 482.4828, 'train_samples_per_second': 207.261, 'train_steps_per_second': 12.954, 'train_loss': 0.1848659490966797, 'epoch': 10.0}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m Num examples = 872\n", - "\u001b[2m\u001b[36m(train pid=86211)\u001b[0m Batch size = 16\n", - "2022-03-19 14:14:22,379\tINFO tune.py:639 -- Total run time: 875.36 seconds (500.34 seconds for the tuning loop).\n", - "[flaml.automl: 03-19 14:14:25] {2837} INFO - selected model: None\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m Num examples = 408\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m Batch size = 1\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_45d8f8ce_31_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-25-34/checkpoint-2751/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_45d8f8ce_31_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-25-34/checkpoint-2751/vocab.txt\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_45d8f8ce_31_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-25-34/checkpoint-2751/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_45d8f8ce_31_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-25-34/checkpoint-2751/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=59250)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_13-07-38/train_45d8f8ce_31_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_13-25-34/checkpoint-2751/tokenizer_config.json\n", + "2022-07-21 13:30:17,073\tINFO tune.py:747 -- Total run time: 1358.29 seconds (1204.81 seconds for the tuning loop).\n", + "[flaml.automl: 07-21 13:30:20] {3314} INFO - selected model: None\n", "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] @@ -602,27 +537,25 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'loss': 0.528, 'learning_rate': 8.898933352349567e-06, 'epoch': 1.0}\n", - "{'loss': 0.2278, 'learning_rate': 1.3880017803076292e-05, 'epoch': 2.0}\n", - "{'loss': 0.1455, 'learning_rate': 1.1410179501562432e-05, 'epoch': 3.0}\n" + "{'loss': 0.405, 'learning_rate': 6.02286555920107e-06, 'epoch': 2.17}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[flaml.automl: 03-19 14:17:03] {2947} INFO - retrain transformer for 157.7s\n", - "[flaml.automl: 03-19 14:17:03] {2954} INFO - retrained model: None\n", - "[flaml.automl: 03-19 14:17:03] {2283} INFO - fit succeeded\n", - "[flaml.automl: 03-19 14:17:03] {2284} INFO - Time taken to find the best model: 370.53120946884155\n", - "[flaml.automl: 03-19 14:17:03] {2295} WARNING - Time taken to find the best model is 74% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n" + "[flaml.automl: 07-21 13:32:01] {3457} INFO - retrain transformer for 101.3s\n", + "[flaml.automl: 07-21 13:32:01] {3464} INFO - retrained model: None\n", + "[flaml.automl: 07-21 13:32:01] {2742} INFO - fit succeeded\n", + "[flaml.automl: 07-21 13:32:01] {2743} INFO - Time taken to find the best model: 1019.1050026416779\n", + "[flaml.automl: 07-21 13:32:01] {2754} WARNING - Time taken to find the best model is 85% of the provided time budget and not all estimators' hyperparameter search converged. Consider increasing the time budget.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "{'train_runtime': 146.8805, 'train_samples_per_second': 519.019, 'train_steps_per_second': 32.441, 'train_loss': 0.30043953600021217, 'epoch': 3.0}\n" + "{'train_runtime': 93.2817, 'train_samples_per_second': 117.965, 'train_steps_per_second': 7.397, 'train_loss': 0.3405460468236951, 'epoch': 3.0}\n" ] } ], @@ -631,6 +564,23 @@ "automl.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, **automl_settings)" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The best loss by FLAML: 0.12009803921568629\n" + ] + } + ], + "source": [ + "print(\"The best loss by FLAML: {}\".format(automl.best_loss))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -709,7 +659,9 @@ { "cell_type": "code", "execution_count": 9, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stderr", @@ -814,7 +766,464 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 3. Other Tasks" + "## 3. Spooky-author-identification example" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "== Status ==
Current time: 2022-07-22 07:20:52 (running for 00:30:13.14)
Memory usage on this node: 19.0/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/4 CPUs, 0/4 GPUs, 0.0/252.65 GiB heap, 0.0/112.27 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: 504afb96 with val_loss=0.11011235955056176 and parameters={'learning_rate': 4.567279255636039e-05, 'num_train_epochs': 5, 'per_device_train_batch_size': 32, 'seed': 37, 'global_max_steps': 9223372036854775807, 'learner': 'transformer'}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38
Number of trials: 12/1000000 (12 TERMINATED)

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m {'train_runtime': 675.882, 'train_samples_per_second': 108.628, 'train_steps_per_second': 6.791, 'train_loss': 0.15587145374491324, 'epoch': 5.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m Num examples = 4895\n", + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m Batch size = 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m {'eval_loss': 0.47973549365997314, 'eval_automl_metric': 0.12134831460674156, 'eval_runtime': 42.4421, 'eval_samples_per_second': 115.334, 'eval_steps_per_second': 115.334, 'epoch': 4.0}\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m {'train_runtime': 482.2078, 'train_samples_per_second': 121.806, 'train_steps_per_second': 1.908, 'train_loss': 0.23729169679724652, 'epoch': 4.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m Num examples = 4895\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m Batch size = 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'loss': 0.1844, 'learning_rate': 2.7020242630660653e-06, 'epoch': 3.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/vocab.txt\n", + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=78542)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_e041e7d6_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=5,per_device_trai_2022-07-22_07-09-19/checkpoint-4590/tokenizer_config.json\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/vocab.txt\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=78793)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_567b6878_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-22_07-12-37/checkpoint-920/tokenizer_config.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'eval_loss': 0.7813186049461365, 'eval_automl_metric': 0.13564862104187947, 'eval_runtime': 38.0461, 'eval_samples_per_second': 128.66, 'eval_steps_per_second': 128.66, 'epoch': 3.0}\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'loss': 0.0829, 'learning_rate': 2.3353000145500413e-06, 'epoch': 3.13}\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m {'eval_loss': 0.5877022743225098, 'eval_automl_metric': 0.11848825331971402, 'eval_runtime': 40.6459, 'eval_samples_per_second': 120.43, 'eval_steps_per_second': 120.43, 'epoch': 4.0}\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'loss': 0.0714, 'learning_rate': 1.9685757660340173e-06, 'epoch': 3.27}\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m {'loss': 0.0237, 'learning_rate': 3.91430499829805e-06, 'epoch': 4.36}\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'loss': 0.0584, 'learning_rate': 1.6018515175179931e-06, 'epoch': 3.41}\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'loss': 0.0836, 'learning_rate': 1.235127269001969e-06, 'epoch': 3.54}\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'loss': 0.0335, 'learning_rate': 8.684030204859451e-07, 'epoch': 3.68}\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m {'eval_loss': 0.6322397589683533, 'eval_automl_metric': 0.11664964249233911, 'eval_runtime': 39.775, 'eval_samples_per_second': 123.067, 'eval_steps_per_second': 123.067, 'epoch': 5.0}\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m {'train_runtime': 607.6324, 'train_samples_per_second': 120.83, 'train_steps_per_second': 3.777, 'train_loss': 0.17213530145699163, 'epoch': 5.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m Num examples = 4895\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m Batch size = 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'loss': 0.0562, 'learning_rate': 5.01678771969921e-07, 'epoch': 3.81}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/vocab.txt\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=79025)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_9a34cd4a_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-22_07-14-32/checkpoint-2295/tokenizer_config.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'loss': 0.0527, 'learning_rate': 1.3495452345389687e-07, 'epoch': 3.95}\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'eval_loss': 0.8104404211044312, 'eval_automl_metric': 0.12625127681307458, 'eval_runtime': 37.4885, 'eval_samples_per_second': 130.573, 'eval_steps_per_second': 130.573, 'epoch': 4.0}\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m {'train_runtime': 1051.1951, 'train_samples_per_second': 55.875, 'train_steps_per_second': 13.969, 'train_loss': 0.2927411694969607, 'epoch': 4.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m Num examples = 4895\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m Batch size = 1\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/vocab.txt\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=78225)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-22_06-50-38/train_d0e0b7d6_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_train_2022-07-22_07-08-52/checkpoint-14684/tokenizer_config.json\n", + "2022-07-22 07:27:20,126\tINFO tune.py:747 -- Total run time: 2201.85 seconds (1801.97 seconds for the tuning loop).\n", + "[flaml.automl: 07-22 07:27:25] {3314} INFO - selected model: None\n", + "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'loss': 0.2317, 'learning_rate': 5.95732076822092e-06, 'epoch': 4.35}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[flaml.automl: 07-22 07:35:19] {3457} INFO - retrain transformer for 474.4s\n", + "[flaml.automl: 07-22 07:35:19] {3464} INFO - retrained model: None\n", + "[flaml.automl: 07-22 07:35:19] {2742} INFO - fit succeeded\n", + "[flaml.automl: 07-22 07:35:19] {2743} INFO - Time taken to find the best model: 1118.247492313385\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'train_runtime': 463.5873, 'train_samples_per_second': 158.374, 'train_steps_per_second': 1.24, 'train_loss': 0.20362980179164722, 'epoch': 5.0}\n" + ] + } + ], + "source": [ + "from flaml import AutoML\n", + "import ray\n", + "import pandas as pd\n", + "from sklearn.model_selection import train_test_split\n", + "ray.init(num_cpus=4, num_gpus=4, ignore_reinit_error=True)\n", + "\n", + "df = pd.read_csv('/data/xliu127/projects/hyperopt/FLAML/data/spooky-author-identification.csv')\n", + "X, y = df.drop('author', axis=1), df['author']\n", + "\n", + "X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=123)\n", + "\n", + "\n", + "print(len(X_train), len(X_val))\n", + "automl_model = AutoML()\n", + "\n", + "automl_settings = {\n", + " \"time_budget\": 1800, \n", + " \"task\": \"seq-classification\", \n", + " \"fit_kwargs_by_estimator\": {\n", + " \"transformer\": {\n", + " \"output_dir\": \"data/output/\", \n", + " \"model_path\": \"bert-base-uncased\", \n", + " }\n", + " },\n", + " \"metric\": \"accuracy\",\n", + " \"gpu_per_trial\": 1, \n", + " \"log_file_name\": \"spooky_bert.log\", \n", + " \"log_type\": \"all\", \n", + " \"use_ray\": {\"local_dir\": \"data/output/\"}, # set whether to use Ray\n", + " \"n_concurrent_trials\": 4,\n", + " \"keep_search_state\": True, # keeping the search state\n", + "}\n", + "\n", + "automl_model.fit(X_train=X_train, y_train=y_train,X_val=X_val, y_val=y_val, **automl_settings)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "the best loss for spooky author identification: 0.11133810010214507\n" + ] + } + ], + "source": [ + "print(\"the best loss for spooky author identification: {}\".format(automl_model.best_loss))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "== Status ==
Current time: 2022-07-21 21:21:15 (running for 00:30:10.30)
Memory usage on this node: 20.5/376.6 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/4 CPUs, 0/4 GPUs, 0.0/252.62 GiB heap, 0.0/112.26 GiB objects (0.0/1.0 accelerator_type:V100)
Current best trial: 84d3be85 with val_loss=0.12951991828396325 and parameters={'learning_rate': 4.486769916716146e-05, 'num_train_epochs': 4, 'per_device_train_batch_size': 8, 'seed': 28, 'global_max_steps': 9223372036854775807, 'learner': 'transformer'}
Result logdir: /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05
Number of trials: 12/1000000 (12 TERMINATED)

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m {'eval_loss': 0.7418951392173767, 'eval_automl_metric': 0.1284984678243105, 'eval_runtime': 37.3935, 'eval_samples_per_second': 130.905, 'eval_steps_per_second': 130.905, 'epoch': 4.0}\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m {'train_runtime': 565.7729, 'train_samples_per_second': 103.816, 'train_steps_per_second': 6.49, 'train_loss': 0.2802804773409642, 'epoch': 4.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m Num examples = 4895\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m Batch size = 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m {'eval_loss': 1.0893423557281494, 'eval_automl_metric': 0.6024514811031665, 'eval_runtime': 39.7178, 'eval_samples_per_second': 123.245, 'eval_steps_per_second': 123.245, 'epoch': 3.0}\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'loss': 0.2369, 'learning_rate': 1.4090340380281214e-05, 'epoch': 2.72}\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m {'train_runtime': 566.9953, 'train_samples_per_second': 77.694, 'train_steps_per_second': 9.714, 'train_loss': 1.0928592581461545, 'epoch': 3.0}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'eval_loss': 1.092341661453247, 'eval_automl_metric': 0.6024514811031665, 'eval_runtime': 38.0057, 'eval_samples_per_second': 128.797, 'eval_steps_per_second': 128.797, 'epoch': 3.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m Num examples = 4895\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m Batch size = 1\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/vocab.json\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/merges.txt\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=50245)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_60247332_10_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=4,per_device_trai_2022-07-21_21-11-36/checkpoint-3672/tokenizer_config.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'loss': 1.0896, 'learning_rate': 1.5104688589428795e-05, 'epoch': 3.13}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/vocab.json\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/merges.txt\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=50412)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_6861ba34_11_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=3,per_device_trai_2022-07-21_21-11-51/checkpoint-3672/tokenizer_config.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'loss': 0.2195, 'learning_rate': 1.2404892966371977e-05, 'epoch': 3.0}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'loss': 1.0907, 'learning_rate': 1.2732721160184323e-05, 'epoch': 3.27}\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'loss': 0.1252, 'learning_rate': 1.0719445552462741e-05, 'epoch': 3.27}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'loss': 1.0926, 'learning_rate': 1.0360753730939852e-05, 'epoch': 3.41}\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'loss': 0.1093, 'learning_rate': 9.033998138553504e-06, 'epoch': 3.54}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'loss': 1.0908, 'learning_rate': 7.988786301695379e-06, 'epoch': 3.54}\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'loss': 0.1166, 'learning_rate': 7.348550724644269e-06, 'epoch': 3.81}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'loss': 1.0899, 'learning_rate': 5.616818872450909e-06, 'epoch': 3.68}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'loss': 1.0923, 'learning_rate': 3.244851443206437e-06, 'epoch': 3.81}\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'eval_loss': 0.7831101417541504, 'eval_automl_metric': 0.13462717058222673, 'eval_runtime': 37.9679, 'eval_samples_per_second': 128.925, 'eval_steps_per_second': 128.925, 'epoch': 4.0}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'loss': 1.0862, 'learning_rate': 8.728840139619655e-07, 'epoch': 3.95}\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'loss': 0.1164, 'learning_rate': 5.663103310735033e-06, 'epoch': 4.08}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'eval_loss': 1.0893481969833374, 'eval_automl_metric': 0.6024514811031665, 'eval_runtime': 36.2865, 'eval_samples_per_second': 134.899, 'eval_steps_per_second': 134.899, 'epoch': 4.0}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m {'train_runtime': 1069.9104, 'train_samples_per_second': 54.898, 'train_steps_per_second': 13.725, 'train_loss': 1.0960205283875493, 'epoch': 4.0}\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m Num examples = 4895\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m Batch size = 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'loss': 0.0542, 'learning_rate': 3.977655896825797e-06, 'epoch': 4.36}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/vocab.json\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/merges.txt\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=49988)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_ebe7d3ee_9_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0001,num_train_epochs=4,per_device_train_2022-07-21_21-08-22/checkpoint-11013/tokenizer_config.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'loss': 0.0618, 'learning_rate': 2.2922084829165607e-06, 'epoch': 4.63}\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'loss': 0.0494, 'learning_rate': 6.06761069007325e-07, 'epoch': 4.9}\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'eval_loss': 0.88468998670578, 'eval_automl_metric': 0.12972420837589382, 'eval_runtime': 37.9519, 'eval_samples_per_second': 128.979, 'eval_steps_per_second': 128.979, 'epoch': 5.0}\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m {'train_runtime': 873.0679, 'train_samples_per_second': 84.094, 'train_steps_per_second': 10.515, 'train_loss': 0.27977710040306475, 'epoch': 5.0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `RobertaForSequenceClassification.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m Num examples = 4895\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m Batch size = 1\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m Didn't find file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/added_tokens.json. We won't load it.\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/vocab.json\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/merges.txt\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/tokenizer.json\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m loading file None\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/special_tokens_map.json\n", + "\u001B[2m\u001B[36m(train pid=50658)\u001B[0m loading file /data/xliu127/projects/hyperopt/FLAML/notebook/data/output/train_2022-07-21_20-51-05/train_bd71ed64_12_global_max_steps=9223372036854775807,learner=transformer,learning_rate=0.0000,num_train_epochs=5,per_device_trai_2022-07-21_21-14-13/checkpoint-9180/tokenizer_config.json\n", + "2022-07-21 21:29:43,228\tINFO tune.py:747 -- Total run time: 2317.81 seconds (1801.93 seconds for the tuning loop).\n", + "[flaml.automl: 07-21 21:29:46] {3314} INFO - selected model: None\n", + "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'loss': 0.5742, 'learning_rate': 3.264882684494973e-05, 'epoch': 1.09}\n" + ] + } + ], + "source": [ + "automl_settings[\"fit_kwargs_by_estimator\"][\"transformer\"][\"model_path\"] = \"roberta-base\"\n", + "automl_settings[\"log_file_name\"] = \"spooky_roberta.log\"\n", + "automl_model.fit(X_train=X_train, y_train=y_train,X_val=X_val, y_val=y_val, **automl_settings)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8\n", + "8\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaA0lEQVR4nO3dfXRV9Z3v8ffXyEOKCEpShQRNdDEMj0Jv5KG0V1uHgq2AWBfC1XtLO63tdaF36QwtXLuQyXWqlVntHbvoVG+rtLN8gKEUgzorVdB22TKaAMpDMBIeCgl2jCj4FCQk3/vH2Qkn6Ulyguecfc7O57VWVvb+7c05v192+GSf3/7t3zZ3R0REct85YVdARERSQ4EuIhIRCnQRkYhQoIuIRIQCXUQkIs4N640LCgq8pKQkrLcXEclJ27Zte9vdCxNtCy3QS0pKqK6uDuvtRURykpn9qatt6nIREYmIpALdzGabWa2Z1ZnZsgTbLzWzzWa208xeNLPi1FdVRES602Ogm1kesBq4FhgLLDKzsZ12+yfgV+4+ESgH7kt1RUVEpHvJnKFPAerc/YC7nwKeBOZ12mcssCVYfiHBdhERSbNkAr0IOBK3Xh+UxXsNuCFYng8MNrNhnV/IzG41s2ozq25sbDyb+oqISBdSdVH074GrzGwHcBXQALR03sndH3b3MncvKyxMOOpGRLLQxh0NzLh/C6XLnmHG/VvYuKMh7CpJAskMW2wARsatFwdl7dz9KMEZupmdB3zV3Y+nqI4iEqKNOxpYvmEXTc2xc7SG400s37ALgOsnd/6wLmFKJtCrgFFmVkosyBcC/y1+BzMrAN5x91ZgOfBIqisqIuFYVVnbHuZtmppb+O76nTzxyuGQapXbxo44n3vmjEv56/bY5eLup4ElQCWwF1jn7nvMrNzM5ga7XQ3UmtkbwEXAP6a8piISiqPHmxKWn2ppzXBNpCdJ3Snq7s8Cz3YqWxG3vB5Yn9qqiUg2GDE0n4YEoV40NJ+1354eQo2kK7pTVES6tXTWaPL75XUoy++Xx9JZo0OqkXQltLlcRCQ3tF34/O76nZxqaaVoaD5LZ43WBdEspEAXkR5dP7mo/QKoulmyl7pcJPftXAc/Hg8rh8a+71wXdo1EQqEzdMltO9fBpjugObhod+JIbB1g4oLw6pVjNu5oYFVlLUePNzFCXSo5y9w9lDcuKytzzYcun9iPx8dCvLO8AVB8Zebrk4Pe/uBjDrz9Ia1xWXCOGZcVDKLgvAHtZXvePAHAuOFDMl7HrHfxBLj2/oy8lZltc/eyRNt0hi657UR94vKWjzNbjxx2+J2POoQ5QKs7+xs/4D/fP9le9tGpFj7VP6/zP5csokCX3DakOPEZ+pCR8PVnMl+fHPTVZc/Q1ef0qUUXdlifN6mIcVMvSX+l5Kwo0CW3XbOiYx86QL/8WLkkRTcORYdGuUhum7gA5jwY6zOH2Jn5nAd1QbQXdONQdOgMXXLfxAWw7ZexZXWz9JpuHIoOBbqI6MahiFCgZ4DG+IpIJijQ00wPB5CM2rkONpfHhnMOKY5dHNb1hD5DgZ5mejhAZqw4FrvppfyhrSHXJEQfNsKxI9D6tdj6W8DaI/B8BQzq+ZGPLx98h6mlF/a4n8TJsj+gGuWSZno4gGTMu4egtdPvVWtrrDwJU0svZN4kfWpMWtu0EyeOAH5m2okQ5xLSGXqaaYxvhjwaux197df78M905bWQl+gWIYNvH890baJvc3nH+x8gtr65PLSzdJ2hp1mPY3w1U6CkypDi3pXLJ9PVtBNdlWeAAj3Nrp9cxH03TKB/XuxHXTQ0n/tumBC7IJqFH9kkh12zInaXbDzdNZs+WfgHVF0uGdDlGN+uPrI9teTMjTKSnD/vis1415e1fczPoot0kZaF004o0MOkmQJT5+IJMOHGsGsRvokLFOCZkoV/QBXoYdJMgSK5Lcv+gKoPPUzq8xSRFFKgh0kzBYpICqnLJWyaKVBEUiS3ztA1ZltEpEu5c4aup7uLiHQrdwI9E2O2P3wL3v1TbNhg3gC44FIY9OmUvHTb5FFtt6h3oDHUIhkT5emscyfQ0z1m+8O34FgdeOuZ1z1WF1tOUah3SWOoRTIi6tNZ506gp3vM9o/HnwnzNt4a+xSQgtdvm9a1T08eJRKyrqazXlVZG4lAz52Loukes52FE+2ISGp1NZ11V+W5JqlAN7PZZlZrZnVmtizB9kvM7AUz22FmO83syymvaduY7SEjAUv9mO0snGhHRFJrxND8XpWn2sYdDcy4fwuly55hxv1b2LijIaWv32OXi5nlAauBmUA9UGVmFe5eE7fb94F17v4vZjYWeBYoSWlNIb232WbhRDsiklpLZ43u0IcOnaazTqNM9N8n04c+Bahz9wMAZvYkMA+ID3QHzg+WhwBHU1K7TMrCiXZEMiHKoz46a2tXGO3NRP99MoFeBMRfjawHpnbaZyXwWzO7HRgE/E2iFzKzW4FbAS655JLe1jX9smyiHZF0i/qoj0Sun1wUStsy0X+fqouii4A17l4MfBn4VzP7i9d294fdvczdywoLe35orYikV3dnjZJamei/TybQG4CRcevFQVm8vwXWAbj7VmAgUJCKCopI+kR91Ec26fFxlCmQTKBXAaPMrNTM+gMLgYpO+xwGrgEwszHEAr0xZbUUkbQIe9RHX9L2OMqiofkYnR5HmSI99qG7+2kzWwJUAnnAI+6+x8zKgWp3rwD+Dvh/ZnYnsQuki9090ePHRSSLhDnqoy9Kd/99UneKuvuzxIYixpetiFuuAWaktmoikm5hjvqQ1MudW/9FJC3CGvUhqZc7t/6LiEi3FOgiIhGhQBcRiQgFuohIRCjQRUQiQoEuIhIRCnQRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEQp0EZGI6FPzoW/c0aCJ/EUksvpMoG/c0dDhUVsNx5tYvmEXgEJdJNvsXAeby+FEPQwphmtWwMQFYdcq6/WZQF9VWdvhuYkATc0tfHf9Tp545XDa37/mzfcYO/z8tL+PSM7buQ423QHNTbH1E0di66BQ70Gf6UM/erwpYfmpltaMvP/Y4eczb5I+CYj0aHP5mTBv09wUK5du9Zkz9BFD82lIEOpFQ/NZ++3pIdRIRBI6Ud+7cmnXZ87Ql84aTX6/vA5l+f3yWDprdEg1EpGEhhT3rlza9ZlAv35yEffdMIGiofkYsTPz+26YoAuiItnmmhXQL79jWb/8WLl0q890uUAs1BXgIlmu7cKnRrn0Wp8KdBHJERMXKMDPQp/pchERiToFuohIRCjQRUQiQoEuIhIRCnQRkYhQoIuIRERSgW5ms82s1szqzGxZgu0/NrNXg683zOx4ymsqIiLd6nEcupnlAauBmUA9UGVmFe5e07aPu98Zt//twOQ01FVERLqRzBn6FKDO3Q+4+yngSWBeN/svAp5IReVERCR5yQR6EXAkbr0+KPsLZnYpUAps6WL7rWZWbWbVjY2Nva2riIh0I9UXRRcC6929JdFGd3/Y3cvcvaywsDDFby0i0rclE+gNwMi49eKgLJGFqLtFRCQUyQR6FTDKzErNrD+x0K7ovJOZ/TVwAbA1tVUUEZFk9Bjo7n4aWAJUAnuBde6+x8zKzWxu3K4LgSfd3dNTVRER6U5S0+e6+7PAs53KVnRaX5m6aomISG/pTlERkYhQoIuIRIQCXUQkIhToIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEUndWCS9t3FHA6sqazl6vIkRQ/NZOms0109OOEmliEhKKNDTYOOOBpZv2EVTc2zSyYbjTSzfsAtAoS4iaaMulzRYVVnbHuZtmppbWFVZG1KNRKQvUKCnwdHjTb0qFxFJBQV6GowYmt+rchGRVFCgp8HSWaPJ75fXoSy/Xx5LZ40OqUYi0hfoomgatF341CgXEckkBXqaXD+5SAEuIhmlLhcRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAFxGJCAW6iEhEKNBFRCIiqUA3s9lmVmtmdWa2rIt9FphZjZntMbPHU1tNERHpSY8PuDCzPGA1MBOoB6rMrMLda+L2GQUsB2a4+7tm9ul0VVhERBJL5gx9ClDn7gfc/RTwJDCv0z7fAla7+7sA7v5WaqspIiI9SSbQi4Ajcev1QVm8vwL+ysz+YGb/YWazU1VBERFJTqqeKXouMAq4GigGfm9mE9z9ePxOZnYrcCvAJZdckqK3FhERSO4MvQEYGbdeHJTFqwcq3L3Z3Q8CbxAL+A7c/WF3L3P3ssLCwrOts4iIJJBMoFcBo8ys1Mz6AwuBik77bCR2do6ZFRDrgjmQumqKiEhPegx0dz8NLAEqgb3AOnffY2blZjY32K0SOGZmNcALwFJ3P5auSouIyF8ydw/ljcvKyry6ujqU9xYRyVVmts3dyxJt052iIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAFxGJCAW6iEhEKNBFRCJCgS4iEhEKdBGRiFCgi4hEhAJdRCQiFOgiIhGhQBcRiQgFuohIRCjQRUQiQoEuIhIRCnQRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYkIBbqISEQo0EVEIkKBLiISEQp0EZGIUKCLiESEAl1EJCKSCnQzm21mtWZWZ2bLEmxfbGaNZvZq8PXN1FdVRES6c25PO5hZHrAamAnUA1VmVuHuNZ12XevuS9JQRxERSUIyZ+hTgDp3P+Dup4AngXnprZaIiPRWMoFeBByJW68Pyjr7qpntNLP1ZjYy0QuZ2a1mVm1m1Y2NjWdRXRER6UqqLopuAkrcfSLwHPDLRDu5+8PuXubuZYWFhSl6axERgeQCvQGIP+MuDsraufsxd/84WP058F9SUz0REUlWMoFeBYwys1Iz6w8sBCridzCz4XGrc4G9qauiiIgko8dRLu5+2syWAJVAHvCIu+8xs3Kg2t0rgDvMbC5wGngHWJzGOouISALm7qG8cVlZmVdXV4fy3iIiucrMtrl7WaJtulNURCQiFOgiIhGhQBcRiQgFuohIRCjQRUQiQoEuIhIRCnQRkYjo8caiTGpubqa+vp6TJ0+GXZWsM3DgQIqLi+nXr1/YVRGRLJVVgV5fX8/gwYMpKSnBzMKuTtZwd44dO0Z9fT2lpaVhV0dEslRWdbmcPHmSYcOGKcw7MTOGDRumTy4i0q2sCnRAYd4F/VxEpCdZF+giInJ2FOidHDp0iPHjx5/1v9+4cSM1NZ0ftyoikn4K9BQ6ffq0Al1EQpNVo1zi/cOmPdQcfS+lrzl2xPncM2dcj/udPn2am2++me3btzNu3Dh+9atfsXfvXu666y4++OADCgoKWLNmDcOHD+fqq69m0qRJvPTSS8yfP5+Kigp+97vfce+99/LrX/+ayy+/PKVtEBHpStYGephqa2v5xS9+wYwZM/jGN77B6tWr+c1vfsNTTz1FYWEha9eu5e677+aRRx4B4NSpU7TN7b5v3z6uu+46brzxxjCbICJ9UNYGejJn0ukycuRIZsyYAcAtt9zCD37wA3bv3s3MmTMBaGlpYfjwM0/du+mmm0Kpp4hIvKwN9DB1HiI4ePBgxo0bx9atWxPuP2jQoExUS0SkW7oomsDhw4fbw/vxxx9n2rRpNDY2tpc1NzezZ8+ehP928ODBvP/++xmrq4hIGwV6AqNHj2b16tWMGTOGd999l9tvv53169fzve99jyuuuIJJkybxxz/+MeG/XbhwIatWrWLy5Mns378/wzUXkb4sqx4SvXfvXsaMGRNKfXKBfj4ioodEi4j0AQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAFxGJCAX6WXjxxRe57rrrPtFrrFmzhqNHj6aoRiIiCvRuuTutra0pf92WlhYFuoikXPbO5fLvy+DPu1L7mhdPgGvv73aXQ4cOMWvWLKZOncq2bduYMmUKVVVVmBnf//732yfieu+99/jKV75CXV0dX/jCF/jpT3/KOeecw29/+1vuuecePv74Yy6//HIeffRRzjvvPEpKSrjpppt47rnnuOuuu6iurubmm28mPz+frVu3smrVKjZt2kRTUxOf/exneeihh/TYORHpFZ2hJ7Bv3z5uu+02ysvLqa+v57XXXuP5559n6dKlvPnmmwC88sor/OQnP6Gmpob9+/ezYcMG3n77be69916ef/55tm/fTllZGT/60Y/aX3fYsGFs376dW265hbKyMh577DFeffVV8vPzWbJkCVVVVezevZumpiaefvrpsJovIjkqqTN0M5sN/DOQB/zc3ROe5prZV4H1wJXuXp1on6T1cCadTpdeeinTpk3jzjvvZNGiReTl5XHRRRdx1VVXUVVVxfnnn8+UKVO47LLLAFi0aBEvvfQSAwcOpKampn3q3VOnTjF9+vT21+1umt0XXniBBx54gI8++oh33nmHcePGMWfOnPQ2VEQipcdAN7M8YDUwE6gHqsyswt1rOu03GPhfwMvpqGgmJTMdbufuEDPD3Zk5cyZPPPFEr1735MmT3HbbbVRXVzNy5EhWrlzJyZMne19xkajauQ42l8OJehhSDNesgIkLwq5V1kmmy2UKUOfuB9z9FPAkMC/Bfv8H+CEQmST6/Oc/z9q1a2lpaaGxsZHf//73TJkyBYh1uRw8eJDW1lbWrl3L5z73OaZNm8Yf/vAH6urqAPjwww954403Er52/DS7beFdUFDABx98wPr16zPQOpEcsXMdbLoDThwBPPZ90x2xcukgmUAvAo7ErdcHZe3M7DPASHd/prsXMrNbzazazKobGxt7XdlMmz9/PhMnTuSKK67gi1/8Ig888AAXX3wxAFdeeSVLlixhzJgxlJaWMn/+fAoLC1mzZg2LFi1i4sSJTJ8+nddffz3hay9evJjvfOc7TJo0iQEDBvCtb32L8ePHM2vWLK688spMNlMku20uh+amjmXNTbFy6aDH6XPN7EZgtrt/M1j/78BUd18SrJ8DbAEWu/shM3sR+Pue+tA1fW7v6ecjfdLKoUCinDJYeTyzdckCn3T63AZgZNx6cVDWZjAwHnjRzA4B04AKM0v4hiIivTKkuHflfVgygV4FjDKzUjPrDywEKto2uvsJdy9w9xJ3LwH+A5j7iUe5iIhA7AJov/yOZf3yY+XSQY+B7u6ngSVAJbAXWOfue8ys3MzmprpCYT1BKdvp5yJ91sQFMOdBGDISsNj3OQ9qlEsCWfUIuoMHDzJ48GCGDRumuyTjuDvHjh3j/fffp7S0NOzqiEiIuutDz6pb/4uLi6mvrycXRsBk2sCBAykuVp+hiHQtqwK9X79+OgMVETlLmstFRCQiFOgiIhGhQBcRiYjQRrmYWSPwp1DePHUKgLfDrkSKRa1NUWsPRK9Nak/vXOruhYk2hBboUWBm1V0NH8pVUWtT1NoD0WuT2pM66nIREYkIBbqISEQo0D+Zh8OuQBpErU1Raw9Er01qT4qoD11EJCJ0hi4iEhEKdBGRiFCg98DM8sxsh5k9HayXmtnLZlZnZmuDOeIxswHBel2wvSTUinfBzIaa2Xoze93M9prZdDO70MyeM7N9wfcLgn3NzB4M2rQzeNRgVjGzO81sj5ntNrMnzGxgrh0jM3vEzN4ys91xZb0+Jmb2tWD/fWb2tTDaEtQjUXtWBb9zO83sN2Y2NG7b8qA9tWY2K658dlBWZ2bLMtyMDhK1KW7b35mZm1lBsB7eMXJ3fXXzBdwFPA48HayvAxYGyz8D/mewfBvws2B5IbA27Lp30Z5fAt8MlvsDQ4EHgGVB2TLgh8Hyl4F/B4zYk6heDrv+ndpSBBwE8uOOzeJcO0bAfwU+A+yOK+vVMQEuBA4E3y8Ili/IovZ8CTg3WP5hXHvGAq8BA4BSYD+QF3ztBy4Lfk9fA8Zm0zEKykcSe1bEn4CCsI9R6L/M2fxF7HF7m4EvAk8HB+jtuF/M6UBlsFwJTA+Wzw32s7Db0Kk9Q4IAtE7ltcDwYHk4UBssPwQsSrRfNnxx5gHmFwY/86eBWbl4jICSTgHYq2MCLAIeiivvsF/Y7em0bT7wWLC8HFget60yOGbtxy3RftnSJmA9cAVwKC7QQztG6nLp3v8Fvgu0BuvDgOMee4oTQD2xUIEz4UKw/USwfzYpBRqBR4NupJ+b2SDgInd/M9jnz8BFwXJ7mwLx7Q2duzcA/wQcBt4k9jPfRm4foza9PSZZfaw6+QaxM1jI4faY2Tygwd1f67QptDYp0LtgZtcBb7n7trDrkkLnEvvY+C/uPhn4kNjH+XYeO3XIibGsQb/yPGJ/qEYAg4DZoVYqDXLpmPTEzO4GTgOPhV2XT8LMPgX8byCrHmyqQO/aDGCumR0CniTW7fLPwFAza3swSDHQECw3EOtPI9g+BDiWyQonoR6od/eXg/X1xAL+P81sOEDw/a1ge3ubAvHtzQZ/Axx090Z3bwY2EDtuuXyM2vT2mGT7scLMFgPXATcHf6Qgd9tzObETideCjCgGtpvZxYTYJgV6F9x9ubsXu3sJsQtoW9z9ZuAF4MZgt68BTwXLFcE6wfYtcb+0WcHd/wwcMbPRQdE1QA0d6965Tf8juGo/DTgR1w2QDQ4D08zsU2ZmnGlPzh6jOL09JpXAl8zsguCTy5eCsqxgZrOJdV/OdfeP4jZVAAuDEUilwCjgFaAKGBWMWOpP7P9gRabr3RV33+Xun3b3kiAj6oHPBP/HwjtGYV5kyJUv4GrOjHK5jNgvXB3wb8CAoHxgsF4XbL8s7Hp30ZZJQDWwE9hI7Gr7MGIXf/cBzwMXBvsasJrYaINdQFnY9U/Qnn8AXgd2A/9KbLRETh0j4Ali1wCaiQXD357NMSHWN10XfH09y9pTR6z/+NXg62dx+98dtKcWuDau/MvAG8G2u7PtGHXafogzF0VDO0a69V9EJCLU5SIiEhEKdBGRiFCgi4hEhAJdRCQiFOgiIhGhQBcRiQgFuohIRPx/cNXA2tMFSnEAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "from flaml.data import get_output_from_log\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "for each_file_name in ['bert', 'roberta']:\n", + " time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \\\n", + " get_output_from_log(filename='spooky_' + each_file_name + '.log', time_budget=3000)\n", + " print(len(valid_loss_history))\n", + " plt.scatter(time_history, 1 - np.array(valid_loss_history))\n", + " plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", + "\n", + "plt.legend(['bert', 'roberta'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Other Tasks" ] }, { @@ -835,7 +1244,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 3.1 Multiple Choice Example" + "### 4.1 Multiple Choice Example" ] }, { @@ -936,47 +1345,47 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m {'eval_loss': 0.6315866112709045, 'eval_automl_metric': 0.18779999999999997, 'eval_runtime': 15.4883, 'eval_samples_per_second': 645.648, 'eval_steps_per_second': 40.353, 'epoch': 1.66}\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m {'train_runtime': 190.7625, 'train_samples_per_second': 87.254, 'train_steps_per_second': 10.909, 'train_loss': 0.5091343906738046, 'epoch': 1.66}\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m {'eval_loss': 1.2118068933486938, 'eval_automl_metric': 0.2015, 'eval_runtime': 15.2585, 'eval_samples_per_second': 655.374, 'eval_steps_per_second': 40.961, 'epoch': 2.87}\n" + "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m {'eval_loss': 0.6315866112709045, 'eval_automl_metric': 0.18779999999999997, 'eval_runtime': 15.4883, 'eval_samples_per_second': 645.648, 'eval_steps_per_second': 40.353, 'epoch': 1.66}\n", + "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m {'train_runtime': 190.7625, 'train_samples_per_second': 87.254, 'train_steps_per_second': 10.909, 'train_loss': 0.5091343906738046, 'epoch': 1.66}\n", + "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m {'eval_loss': 1.2118068933486938, 'eval_automl_metric': 0.2015, 'eval_runtime': 15.2585, 'eval_samples_per_second': 655.374, 'eval_steps_per_second': 40.961, 'epoch': 2.87}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: ending3, ending1, video-id, sent1, ending0, sent2, fold-ind, ending2, startphrase, gold-source. If ending3, ending1, video-id, sent1, ending0, sent2, fold-ind, ending2, startphrase, gold-source are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m Num examples = 10000\n", - "\u001b[2m\u001b[36m(train pid=86157)\u001b[0m Batch size = 16\n" + "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m Using amp half precision backend\n", + "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: ending3, ending1, video-id, sent1, ending0, sent2, fold-ind, ending2, startphrase, gold-source. If ending3, ending1, video-id, sent1, ending0, sent2, fold-ind, ending2, startphrase, gold-source are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m Num examples = 10000\n", + "\u001B[2m\u001B[36m(train pid=86157)\u001B[0m Batch size = 16\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m {'eval_loss': 1.2118068933486938, 'eval_automl_metric': 0.2015, 'eval_runtime': 15.1369, 'eval_samples_per_second': 660.639, 'eval_steps_per_second': 41.29, 'epoch': 2.87}\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m {'train_runtime': 546.3809, 'train_samples_per_second': 156.658, 'train_steps_per_second': 39.165, 'train_loss': 0.5030154804349909, 'epoch': 2.87}\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'loss': 0.4854, 'learning_rate': 1.3592147782116173e-06, 'epoch': 2.0}\n" + "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m {'eval_loss': 1.2118068933486938, 'eval_automl_metric': 0.2015, 'eval_runtime': 15.1369, 'eval_samples_per_second': 660.639, 'eval_steps_per_second': 41.29, 'epoch': 2.87}\n", + "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m {'train_runtime': 546.3809, 'train_samples_per_second': 156.658, 'train_steps_per_second': 39.165, 'train_loss': 0.5030154804349909, 'epoch': 2.87}\n", + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'loss': 0.4854, 'learning_rate': 1.3592147782116173e-06, 'epoch': 2.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: fold-ind, sent2, gold-source, ending1, startphrase, sent1, ending0, video-id, ending2, ending3. If fold-ind, sent2, gold-source, ending1, startphrase, sent1, ending0, video-id, ending2, ending3 are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m Num examples = 10000\n", - "\u001b[2m\u001b[36m(train pid=86249)\u001b[0m Batch size = 16\n" + "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m Using amp half precision backend\n", + "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: fold-ind, sent2, gold-source, ending1, startphrase, sent1, ending0, video-id, ending2, ending3. If fold-ind, sent2, gold-source, ending1, startphrase, sent1, ending0, video-id, ending2, ending3 are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m Num examples = 10000\n", + "\u001B[2m\u001B[36m(train pid=86249)\u001B[0m Batch size = 16\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'eval_loss': 0.49709731340408325, 'eval_automl_metric': 0.17600000000000005, 'eval_runtime': 15.4983, 'eval_samples_per_second': 645.232, 'eval_steps_per_second': 40.327, 'epoch': 2.0}\n" + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'eval_loss': 0.49709731340408325, 'eval_automl_metric': 0.17600000000000005, 'eval_runtime': 15.4983, 'eval_samples_per_second': 645.232, 'eval_steps_per_second': 40.327, 'epoch': 2.0}\n" ] }, { @@ -990,8 +1399,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'eval_loss': 0.5254333019256592, 'eval_automl_metric': 0.17800000000000005, 'eval_runtime': 15.45, 'eval_samples_per_second': 647.251, 'eval_steps_per_second': 40.453, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'loss': 0.3989, 'learning_rate': 3.8051750127352887e-07, 'epoch': 3.0}\n" + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'eval_loss': 0.5254333019256592, 'eval_automl_metric': 0.17800000000000005, 'eval_runtime': 15.45, 'eval_samples_per_second': 647.251, 'eval_steps_per_second': 40.453, 'epoch': 3.0}\n", + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'loss': 0.3989, 'learning_rate': 3.8051750127352887e-07, 'epoch': 3.0}\n" ] }, { @@ -1005,9 +1414,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'eval_loss': 0.5254867076873779, 'eval_automl_metric': 0.17789999999999995, 'eval_runtime': 15.424, 'eval_samples_per_second': 648.341, 'eval_steps_per_second': 40.521, 'epoch': 3.0}\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'eval_loss': 0.5332269072532654, 'eval_automl_metric': 0.17830000000000001, 'eval_runtime': 15.4452, 'eval_samples_per_second': 647.45, 'eval_steps_per_second': 40.466, 'epoch': 3.39}\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m {'train_runtime': 382.2827, 'train_samples_per_second': 88.597, 'train_steps_per_second': 11.076, 'train_loss': 0.5299136270370808, 'epoch': 3.39}\n" + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'eval_loss': 0.5254867076873779, 'eval_automl_metric': 0.17789999999999995, 'eval_runtime': 15.424, 'eval_samples_per_second': 648.341, 'eval_steps_per_second': 40.521, 'epoch': 3.0}\n", + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'eval_loss': 0.5332269072532654, 'eval_automl_metric': 0.17830000000000001, 'eval_runtime': 15.4452, 'eval_samples_per_second': 647.45, 'eval_steps_per_second': 40.466, 'epoch': 3.39}\n", + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m {'train_runtime': 382.2827, 'train_samples_per_second': 88.597, 'train_steps_per_second': 11.076, 'train_loss': 0.5299136270370808, 'epoch': 3.39}\n" ] }, { @@ -1015,11 +1424,11 @@ "output_type": "stream", "text": [ "2022-03-19 14:43:56,739\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: ending2, sent1, ending0, sent2, ending3, video-id, gold-source, ending1, startphrase, fold-ind. If ending2, sent1, ending0, sent2, ending3, video-id, gold-source, ending1, startphrase, fold-ind are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m Num examples = 10000\n", - "\u001b[2m\u001b[36m(train pid=86195)\u001b[0m Batch size = 16\n", + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m Using amp half precision backend\n", + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m The following columns in the test set don't have a corresponding argument in `RobertaForMultipleChoice.forward` and have been ignored: ending2, sent1, ending0, sent2, ending3, video-id, gold-source, ending1, startphrase, fold-ind. If ending2, sent1, ending0, sent2, ending3, video-id, gold-source, ending1, startphrase, fold-ind are not expected by `RobertaForMultipleChoice.forward`, you can safely ignore this message.\n", + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m Num examples = 10000\n", + "\u001B[2m\u001B[36m(train pid=86195)\u001B[0m Batch size = 16\n", "2022-03-19 14:44:14,271\tINFO tune.py:639 -- Total run time: 795.18 seconds (504.18 seconds for the tuning loop).\n", "[flaml.automl: 03-19 14:44:19] {2837} INFO - selected model: None\n", "/data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", @@ -1065,7 +1474,6 @@ " \"fit_kwargs_by_estimator\": { # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base\n", " \"transformer\": {\n", " \"output_dir\": \"data/output/\", # setting the output directory\n", - " \"ckpt_per_epoch\": 1, # setting the number of checkoints per epoch\n", " \"per_device_eval_batch_size\": 16, # the batch size for validation (inference)\n", " }\n", " },\n", @@ -1133,7 +1541,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 3.2 Text Summarization Example" + "### 4.2 Text Summarization Example" ] }, { @@ -1217,37 +1625,37 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m warnings.warn(\n" + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m {'loss': 8.7635, 'learning_rate': 1.2308416834153697e-05, 'epoch': 0.11}\n" + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m {'loss': 8.7635, 'learning_rate': 1.2308416834153697e-05, 'epoch': 0.11}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m warnings.warn(\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m warnings.warn(\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m warnings.warn(\n", + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m warnings.warn(\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m warnings.warn(\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m /data/installation/anaconda3/envs/tmp/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m warnings.warn(\n", "2022-03-19 14:56:00,679\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m {'eval_loss': 6.893245697021484, 'eval_automl_metric': 0.8537338408275918, 'eval_runtime': 102.2734, 'eval_samples_per_second': 110.801, 'eval_steps_per_second': 6.932, 'epoch': 0.11}\n" + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m {'eval_loss': 6.893245697021484, 'eval_automl_metric': 0.8537338408275918, 'eval_runtime': 102.2734, 'eval_samples_per_second': 110.801, 'eval_steps_per_second': 6.932, 'epoch': 0.11}\n" ] }, { @@ -1255,32 +1663,32 @@ "output_type": "stream", "text": [ "2022-03-19 14:57:00,687\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m {'eval_loss': 7.381210803985596, 'eval_automl_metric': 0.8475751825208984, 'eval_runtime': 107.4032, 'eval_samples_per_second': 105.509, 'eval_steps_per_second': 6.601, 'epoch': 0.16}\n" + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m {'eval_loss': 7.381210803985596, 'eval_automl_metric': 0.8475751825208984, 'eval_runtime': 107.4032, 'eval_samples_per_second': 105.509, 'eval_steps_per_second': 6.601, 'epoch': 0.16}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m {'eval_loss': 10.150897979736328, 'eval_automl_metric': 0.8566791839938478, 'eval_runtime': 108.2143, 'eval_samples_per_second': 104.718, 'eval_steps_per_second': 6.552, 'epoch': 0.36}\n" + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m {'eval_loss': 10.150897979736328, 'eval_automl_metric': 0.8566791839938478, 'eval_runtime': 108.2143, 'eval_samples_per_second': 104.718, 'eval_steps_per_second': 6.552, 'epoch': 0.36}\n" ] }, { @@ -1294,23 +1702,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m {'eval_loss': 11.665904998779297, 'eval_automl_metric': 0.858011676038827, 'eval_runtime': 109.4667, 'eval_samples_per_second': 103.52, 'eval_steps_per_second': 6.477, 'epoch': 0.38}\n" + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m {'eval_loss': 11.665904998779297, 'eval_automl_metric': 0.858011676038827, 'eval_runtime': 109.4667, 'eval_samples_per_second': 103.52, 'eval_steps_per_second': 6.477, 'epoch': 0.38}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m {'eval_loss': 6.893245697021484, 'eval_automl_metric': 0.8537338408275918, 'eval_runtime': 110.7246, 'eval_samples_per_second': 102.344, 'eval_steps_per_second': 6.403, 'epoch': 0.11}\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m {'train_runtime': 220.8946, 'train_samples_per_second': 4.648, 'train_steps_per_second': 0.149, 'train_loss': 8.763471198804451, 'epoch': 0.11}\n" + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m {'eval_loss': 6.893245697021484, 'eval_automl_metric': 0.8537338408275918, 'eval_runtime': 110.7246, 'eval_samples_per_second': 102.344, 'eval_steps_per_second': 6.403, 'epoch': 0.11}\n", + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m {'train_runtime': 220.8946, 'train_samples_per_second': 4.648, 'train_steps_per_second': 0.149, 'train_loss': 8.763471198804451, 'epoch': 0.11}\n" ] }, { @@ -1318,40 +1726,40 @@ "output_type": "stream", "text": [ "2022-03-19 14:59:00,706\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m Num examples = 11332\n", - "\u001b[2m\u001b[36m(train pid=86232)\u001b[0m Batch size = 16\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m Using amp half precision backend\n", + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m Num examples = 11332\n", + "\u001B[2m\u001B[36m(train pid=86232)\u001B[0m Batch size = 16\n", + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m {'eval_loss': 7.381210803985596, 'eval_automl_metric': 0.8475751825208984, 'eval_runtime': 109.1975, 'eval_samples_per_second': 103.775, 'eval_steps_per_second': 6.493, 'epoch': 0.16}\n" + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m {'eval_loss': 7.381210803985596, 'eval_automl_metric': 0.8475751825208984, 'eval_runtime': 109.1975, 'eval_samples_per_second': 103.775, 'eval_steps_per_second': 6.493, 'epoch': 0.16}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m [nltk_data] Package punkt is already up-to-date!\n" + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m [nltk_data] Downloading package punkt to /home/xliu127/nltk_data...\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m [nltk_data] Package punkt is already up-to-date!\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m {'train_runtime': 232.9303, 'train_samples_per_second': 10.067, 'train_steps_per_second': 1.262, 'train_loss': 9.880440506280637, 'epoch': 0.16}\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m {'eval_loss': 10.150897979736328, 'eval_automl_metric': 0.8566791839938478, 'eval_runtime': 108.3182, 'eval_samples_per_second': 104.618, 'eval_steps_per_second': 6.546, 'epoch': 0.36}\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m {'train_runtime': 232.4568, 'train_samples_per_second': 92.218, 'train_steps_per_second': 2.887, 'train_loss': 11.215172903878349, 'epoch': 0.36}\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m {'eval_loss': 11.665904998779297, 'eval_automl_metric': 0.858011676038827, 'eval_runtime': 110.526, 'eval_samples_per_second': 102.528, 'eval_steps_per_second': 6.415, 'epoch': 0.38}\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m {'train_runtime': 236.6253, 'train_samples_per_second': 19.714, 'train_steps_per_second': 0.621, 'train_loss': 11.549961930614407, 'epoch': 0.38}\n" + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m {'train_runtime': 232.9303, 'train_samples_per_second': 10.067, 'train_steps_per_second': 1.262, 'train_loss': 9.880440506280637, 'epoch': 0.16}\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m {'eval_loss': 10.150897979736328, 'eval_automl_metric': 0.8566791839938478, 'eval_runtime': 108.3182, 'eval_samples_per_second': 104.618, 'eval_steps_per_second': 6.546, 'epoch': 0.36}\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m {'train_runtime': 232.4568, 'train_samples_per_second': 92.218, 'train_steps_per_second': 2.887, 'train_loss': 11.215172903878349, 'epoch': 0.36}\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m {'eval_loss': 11.665904998779297, 'eval_automl_metric': 0.858011676038827, 'eval_runtime': 110.526, 'eval_samples_per_second': 102.528, 'eval_steps_per_second': 6.415, 'epoch': 0.38}\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m {'train_runtime': 236.6253, 'train_samples_per_second': 19.714, 'train_steps_per_second': 0.621, 'train_loss': 11.549961930614407, 'epoch': 0.38}\n" ] }, { @@ -1359,18 +1767,18 @@ "output_type": "stream", "text": [ "2022-03-19 15:00:00,942\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m Num examples = 11332\n", - "\u001b[2m\u001b[36m(train pid=86184)\u001b[0m Batch size = 16\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m Num examples = 11332\n", - "\u001b[2m\u001b[36m(train pid=86160)\u001b[0m Batch size = 16\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m Using amp half precision backend\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m ***** Running Prediction *****\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m Num examples = 11332\n", - "\u001b[2m\u001b[36m(train pid=86225)\u001b[0m Batch size = 16\n", + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m Using amp half precision backend\n", + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m Num examples = 11332\n", + "\u001B[2m\u001B[36m(train pid=86184)\u001B[0m Batch size = 16\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m Using amp half precision backend\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m Num examples = 11332\n", + "\u001B[2m\u001B[36m(train pid=86160)\u001B[0m Batch size = 16\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m Using amp half precision backend\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m ***** Running Prediction *****\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m Num examples = 11332\n", + "\u001B[2m\u001B[36m(train pid=86225)\u001B[0m Batch size = 16\n", "2022-03-19 15:01:00,948\tWARNING ray_trial_executor.py:146 -- Skipping cleanup - trainable.stop did not return in time. Consider making `stop` a faster operation.\n", "2022-03-19 15:02:20,150\tINFO tune.py:639 -- Total run time: 950.87 seconds (500.36 seconds for the tuning loop).\n", "[flaml.automl: 03-19 15:02:25] {2837} INFO - selected model: None\n", @@ -1406,7 +1814,6 @@ " \"fit_kwargs_by_estimator\": { # if model_path is not set, the default model is t5-small: https://huggingface.co/t5-small\n", " \"transformer\": {\n", " \"output_dir\": \"data/output/\", # setting the output directory\n", - " \"ckpt_per_epoch\": 1, # setting the number of checkoints per epoch\n", " \"model_path\": \"t5-small\",\n", " \"per_device_eval_batch_size\": 16, # the batch size for validation (inference)\n", " }\n", @@ -1470,35 +1877,6 @@ "plt.step(time_history, 1 - np.array(best_valid_loss_history), where='post')\n", "plt.show()" ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using amp half precision backend\n", - "***** Running Prediction *****\n", - " Num examples = 1\n", - " Batch size = 64\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Predicted labels ['prison link Cymru said some ex-offenders were living rough for up to ']\n" - ] - } - ], - "source": [ - "'''compute predictions of testing dataset''' \n", - "y_pred = automl.predict([\"Prison Link Cymru had 1,099 referrals in 2015-16 and said some ex-offenders were living rough for up to a year before finding suitable accommodation.\\nWorkers at the charity claim investment in housing would be cheaper than jailing homeless repeat offenders.\\nThe Welsh Government said more people than ever were getting help to address housing problems.\\nChanges to the Housing Act in Wales, introduced in 2015, removed the right for prison leavers to be given priority for accommodation.\\nPrison Link Cymru, which helps people find accommodation after their release, said things were generally good for women because issues such as children or domestic violence were now considered.\\nHowever, the same could not be said for men, the charity said, because issues which often affect them, such as post traumatic stress disorder or drug dependency, were often viewed as less of a priority.\\nAndrew Stevens, who works in Welsh prisons trying to secure housing for prison leavers, said the need for accommodation was 'chronic'.\\n'There\\'s a desperate need for it, finding suitable accommodation for those leaving prison there is just a lack of it everywhere,' he said.\\n'It could take six months to a year, without a lot of help they could be on the streets for six months.\\n'When you think of the consequences of either being on the street, especially with the cold weather at the moment or you may have a roof over your head, sometimes there is only one choice.'\\nMr Stevens believes building more one-bedroom flats could help ease the problem.\\n'The average price is a hundred pounds a week to keep someone in a rented flat, prison is a lot more than that so I would imagine it would save the public purse quite a few pounds,' he said.\\nOfficial figures show 830 one-bedroom properties were built in the year to March 2016, of an overall total of 6,900 new properties in Wales.\\nMarc, 50, who has been in and out of prison for the past 20 years for burglary offences, said he struggled to find accommodation each time he was released.\\nHe said he would ask himself: 'Where am I going to stay? Where am I going to live? Have I got somewhere where I can see my daughter.'\\n'You\\'re put out among the same sort of people doing the same sort of thing, and it\\'s difficult, it\\'s difficult to get away from it. It\\'s like every man for himself, there\\'s nothing.'\\nMarc has now found stable accommodation with homeless charity Emmaus and said it had been life changing.\\n'You feel safe, you got hot food, you\\'ve got company of people in similar situations to yourself but all dealing with different issues. It\\'s a constructive, helpful atmosphere,' he said.\\nTom Clarke, chief executive of Emmaus South Wales, agreed there was not enough support available.\\n'We do still see [people] homeless on the streets, so clearly they haven\\'t got accommodation and haven\\'t got provision,' he said.\\n'I think the key is connecting people with the services they need. I don\\'t delude myself that Emmaus can offer a one size fits all for everyone, we can\\'t.\\n'But there must be other opportunities and given suitable encouragement I believe that can and should happen.'\\nA Welsh Government spokesman said the national pathway for homeless services to children, young people and adults in the secure estate had prevented many people from losing their home whilst serving their prison sentence.\\nIt added there were already significant demands for one-bedroom flats across the public and private sector and it was providing 20,000 new affordable homes in the next five years.\"])\n", - "print('Predicted labels', y_pred)" - ] } ], "metadata": { @@ -1520,9 +1898,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.8.0" } }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/notebook/research/acl2021.ipynb b/notebook/research/acl2021.ipynb index 4069f61d1..f0862193d 100644 --- a/notebook/research/acl2021.ipynb +++ b/notebook/research/acl2021.ipynb @@ -171,7 +171,6 @@ " \"resources_per_trial\": {\"gpu\": 1, \"cpu\": 1},\n", " \"num_samples\": 1,\n", " \"time_budget\": 100000, # unlimited time budget\n", - " \"ckpt_per_epoch\": 5,\n", " \"fp16\": True,\n", " \"algo_mode\": \"grid\", # set the search algorithm to grid search\n", " \"space_mode\": \"grid\", # set the search space to the recommended grid space\n", @@ -326,7 +325,6 @@ " \"resources_per_trial\": {\"gpu\": 1, \"cpu\": 1},\n", " \"num_samples\": -1,\n", " \"time_budget\": time_budget,\n", - " \"ckpt_per_epoch\": 5,\n", " \"fp16\": True,\n", " \"algo_mode\": \"hpo\", # set the search algorithm mode to hpo\n", " \"algo_name\": \"rs\",\n", @@ -364,10 +362,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(pid=50964)\u001b[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n", - "\u001b[2m\u001b[36m(pid=50964)\u001b[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n", - "\u001b[2m\u001b[36m(pid=50948)\u001b[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n", - "\u001b[2m\u001b[36m(pid=50948)\u001b[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n" + "\u001B[2m\u001B[36m(pid=50964)\u001B[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n", + "\u001B[2m\u001B[36m(pid=50964)\u001B[0m {'eval_loss': 0.5942569971084595, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10434782608695652}\n", + "\u001B[2m\u001B[36m(pid=50948)\u001B[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n", + "\u001B[2m\u001B[36m(pid=50948)\u001B[0m {'eval_loss': 0.649192214012146, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.2}\n" ] }, { @@ -485,12 +483,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(pid=54411)\u001b[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=54411)\u001b[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=54411)\u001b[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=54417)\u001b[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=54417)\u001b[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=54417)\u001b[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n" + "\u001B[2m\u001B[36m(pid=54411)\u001B[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=54411)\u001B[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=54411)\u001B[0m {'eval_loss': 0.624100387096405, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=54417)\u001B[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=54417)\u001B[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=54417)\u001B[0m {'eval_loss': 0.5938675999641418, 'eval_accuracy': 0.7156862745098039, 'eval_f1': 0.8258258258258258, 'epoch': 0.5}\n" ] }, { @@ -590,18 +588,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n", - "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n", - "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n", - "\u001b[2m\u001b[36m(pid=57835)\u001b[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n", - "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n", - "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n", - "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n", - "\u001b[2m\u001b[36m(pid=57836)\u001b[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n", - "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=57839)\u001b[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n" + "\u001B[2m\u001B[36m(pid=57835)\u001B[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n", + "\u001B[2m\u001B[36m(pid=57835)\u001B[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n", + "\u001B[2m\u001B[36m(pid=57835)\u001B[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n", + "\u001B[2m\u001B[36m(pid=57835)\u001B[0m {'eval_loss': 0.5822290778160095, 'eval_accuracy': 0.7058823529411765, 'eval_f1': 0.8181818181818181, 'epoch': 0.5043478260869565}\n", + "\u001B[2m\u001B[36m(pid=57836)\u001B[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n", + "\u001B[2m\u001B[36m(pid=57836)\u001B[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n", + "\u001B[2m\u001B[36m(pid=57836)\u001B[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n", + "\u001B[2m\u001B[36m(pid=57836)\u001B[0m {'eval_loss': 0.6087244749069214, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.10344827586206896}\n", + "\u001B[2m\u001B[36m(pid=57839)\u001B[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=57839)\u001B[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=57839)\u001B[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=57839)\u001B[0m {'eval_loss': 0.5486209392547607, 'eval_accuracy': 0.7034313725490197, 'eval_f1': 0.8141321044546851, 'epoch': 0.5}\n" ] }, { @@ -701,21 +699,21 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=61251)\u001b[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", - "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", - "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", - "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", - "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", - "\u001b[2m\u001b[36m(pid=61255)\u001b[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", - "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n", - "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n", - "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n", - "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n", - "\u001b[2m\u001b[36m(pid=61236)\u001b[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n" + "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=61251)\u001B[0m {'eval_loss': 0.6236899495124817, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.5}\n", + "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", + "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", + "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", + "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", + "\u001B[2m\u001B[36m(pid=61255)\u001B[0m {'eval_loss': 0.6249027848243713, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.3}\n", + "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n", + "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n", + "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n", + "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n", + "\u001B[2m\u001B[36m(pid=61236)\u001B[0m {'eval_loss': 0.6138392686843872, 'eval_accuracy': 0.6838235294117647, 'eval_f1': 0.8122270742358079, 'epoch': 0.20689655172413793}\n" ] }, { @@ -806,4 +804,4 @@ }, "nbformat": 4, "nbformat_minor": 1 -} +} \ No newline at end of file diff --git a/test/automl/test_custom_hp.py b/test/automl/test_custom_hp.py index 4bead3f88..b1dde9dd2 100644 --- a/test/automl/test_custom_hp.py +++ b/test/automl/test_custom_hp.py @@ -24,7 +24,6 @@ def test_custom_hp_nlp(): automl_settings["fit_kwargs_by_estimator"] = { "transformer": { "output_dir": "test/data/output/", - "ckpt_per_epoch": 1, "fp16": False, } } diff --git a/test/nlp/default/transformer_ms/glue-cola-.json b/test/nlp/default/transformer_ms/glue-cola-.json index fb081e2c9..afb1fb251 100644 --- a/test/nlp/default/transformer_ms/glue-cola-.json +++ b/test/nlp/default/transformer_ms/glue-cola-.json @@ -1,5 +1,5 @@ {"class": "transformer_ms", "hyperparameters": {"learning_rate": 1e-5, "num_train_epochs": 1.0, "per_device_train_batch_size": 8, - "warmup_ratio": 0.0, "weight_decay": 0.0, "adam_epsilon": 1e-6, "seed": 44, "global_max_steps": 101, + "seed": 44, "global_max_steps": 101, "model_path": "google/electra-base-discriminator"} } \ No newline at end of file diff --git a/test/nlp/default/transformer_ms/glue-mrpc-.json b/test/nlp/default/transformer_ms/glue-mrpc-.json index 7f7dfa142..ea7551fc2 100644 --- a/test/nlp/default/transformer_ms/glue-mrpc-.json +++ b/test/nlp/default/transformer_ms/glue-mrpc-.json @@ -1,5 +1,5 @@ {"class": "transformer_ms", "hyperparameters": {"learning_rate": 1e-5, "num_train_epochs": 1.0, "per_device_train_batch_size": 8, - "warmup_ratio": 0.0, "weight_decay": 0.0, "adam_epsilon": 1e-6, "seed": 43, "global_max_steps": 100, + "seed": 43, "global_max_steps": 100, "model_path": "google/electra-base-discriminator"} } \ No newline at end of file diff --git a/test/nlp/default/transformer_ms/glue-qnli-.json b/test/nlp/default/transformer_ms/glue-qnli-.json index 1b34071f2..63764fed0 100644 --- a/test/nlp/default/transformer_ms/glue-qnli-.json +++ b/test/nlp/default/transformer_ms/glue-qnli-.json @@ -1,5 +1,5 @@ {"class": "transformer_ms", "hyperparameters": {"learning_rate": 1e-5, "num_train_epochs": 1.0, "per_device_train_batch_size": 8, - "warmup_ratio": 0.0, "weight_decay": 0.0, "adam_epsilon": 1e-6, "seed": 41, "global_max_steps": 102, + "seed": 41, "global_max_steps": 102, "model_path": "google/electra-base-discriminator" } } \ No newline at end of file diff --git a/test/nlp/default/transformer_ms/glue-rte-.json b/test/nlp/default/transformer_ms/glue-rte-.json index dd15e3c51..16226f3b5 100644 --- a/test/nlp/default/transformer_ms/glue-rte-.json +++ b/test/nlp/default/transformer_ms/glue-rte-.json @@ -1,5 +1,5 @@ {"class": "transformer_ms", "hyperparameters": {"learning_rate": 1e-5, "num_train_epochs": 1.0, "per_device_train_batch_size": 8, - "warmup_ratio": 0.0, "weight_decay": 0.0, "adam_epsilon": 1e-6, "seed": 42, "global_max_steps": 103, + "seed": 42, "global_max_steps": 103, "model_path": "google/electra-base-discriminator" } } \ No newline at end of file diff --git a/test/nlp/default/transformer_ms/glue-sst2-.json b/test/nlp/default/transformer_ms/glue-sst2-.json index 456df0bf4..72f528216 100644 --- a/test/nlp/default/transformer_ms/glue-sst2-.json +++ b/test/nlp/default/transformer_ms/glue-sst2-.json @@ -1,5 +1,5 @@ {"class": "transformer_ms", "hyperparameters": {"learning_rate": 1e-5, "num_train_epochs": 1.0, "per_device_train_batch_size": 8, - "warmup_ratio": 0.0, "weight_decay": 0.0, "adam_epsilon": 1e-6, "seed": 40, "global_max_steps": 105, + "seed": 40, "global_max_steps": 105, "model_path": "google/electra-base-discriminator"} } \ No newline at end of file diff --git a/test/nlp/test_autohf.py b/test/nlp/test_autohf.py index 5131021bc..6e59bcf2d 100644 --- a/test/nlp/test_autohf.py +++ b/test/nlp/test_autohf.py @@ -27,6 +27,21 @@ def test_hf_data(): except requests.exceptions.HTTPError: return + import json + + with open("seqclass.log", "r") as fin: + for line in fin: + each_log = json.loads(line.strip("\n")) + if "validation_loss" in each_log: + val_loss = each_log["validation_loss"] + min_inter_result = min( + each_dict.get("eval_automl_metric", sys.maxsize) + for each_dict in each_log["logged_metric"]["intermediate_results"] + ) + + if min_inter_result != sys.maxsize: + assert val_loss == min_inter_result + automl = AutoML() automl_settings.pop("max_iter", None) diff --git a/test/nlp/test_autohf_summarization.py b/test/nlp/test_autohf_summarization.py index 4f45ef573..8cde806a9 100644 --- a/test/nlp/test_autohf_summarization.py +++ b/test/nlp/test_autohf_summarization.py @@ -6,7 +6,7 @@ from utils import get_toy_data_summarization, get_automl_settings @pytest.mark.skipif( sys.platform == "darwin" or sys.version < "3.7", - reason="do not run on mac os or py < 3.7", + reason="do not run on mac os or py3.6", ) def test_summarization(): # TODO: manual test for how effective postprocess_seq2seq_prediction_label is diff --git a/test/nlp/utils.py b/test/nlp/utils.py index 7835025f8..18bb92bc1 100644 --- a/test/nlp/utils.py +++ b/test/nlp/utils.py @@ -1518,7 +1518,6 @@ def get_automl_settings(estimator_name="transformer"): automl_settings["fit_kwargs_by_estimator"] = { estimator_name: { "output_dir": "test/data/output/", - "ckpt_per_epoch": 1, "fp16": False, } } @@ -1527,7 +1526,6 @@ def get_automl_settings(estimator_name="transformer"): estimator_name: { "model_path": "google/electra-small-discriminator", "output_dir": "test/data/output/", - "ckpt_per_epoch": 1, "fp16": False, } } diff --git a/test/test_gpu.py b/test/test_gpu.py index b9bf4760d..22a6aea65 100644 --- a/test/test_gpu.py +++ b/test/test_gpu.py @@ -85,7 +85,6 @@ def _test_hf_data(): "transformer": { "model_path": "facebook/muppet-roberta-base", "output_dir": "test/data/output/", - "ckpt_per_epoch": 5, "fp16": True, } } diff --git a/website/docs/Examples/AutoML-NLP.md b/website/docs/Examples/AutoML-NLP.md index df3b971aa..17181d786 100644 --- a/website/docs/Examples/AutoML-NLP.md +++ b/website/docs/Examples/AutoML-NLP.md @@ -86,7 +86,6 @@ automl_settings["fit_kwargs_by_estimator"] = { # setting the huggingface argume "transformer": { "model_path": "google/electra-small-discriminator", # if model_path is not set, the default model is facebook/muppet-roberta-base: https://huggingface.co/facebook/muppet-roberta-base "output_dir": "data/output/", # setting the output directory - "ckpt_per_epoch": 5, # setting the number of checkpoints per epoch "fp16": False, } # setting whether to use FP16 } @@ -138,7 +137,6 @@ automl_settings["fit_kwargs_by_estimator"] = { # setting the huggingface ar "transformer": { "model_path": "t5-small", # if model_path is not set, the default model is t5-small: https://huggingface.co/t5-small "output_dir": "data/output/", # setting the output directory - "ckpt_per_epoch": 5, # setting the number of checkpoints per epoch "fp16": False, } # setting whether to use FP16 } @@ -366,6 +364,6 @@ For tasks that are not currently supported, use `flaml.tune` for [customized tun ### Link to Jupyter notebook -To run these examples in our Jupyter notebook, please go to: +To run more examples, especially examples using Ray Tune, please go to: [Link to notebook](https://github.com/microsoft/FLAML/blob/main/notebook/automl_nlp.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/FLAML/blob/main/notebook/automl_nlp.ipynb) \ No newline at end of file