From b78f027a9f65f561772da9aa2892c153e4acdcdb Mon Sep 17 00:00:00 2001 From: Ningchen Wang Date: Sat, 14 Aug 2021 01:22:00 +0800 Subject: [PATCH 1/2] Use float for mlm_prob mlm_probability param of DataCollator accepts 0-1 float not integer https://huggingface.co/transformers/main_classes/data_collator.html?highlight=datacollator#datacollatorforlanguagemodeling --- examples/unsupervised_learning/MLM/train_mlm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/unsupervised_learning/MLM/train_mlm.py b/examples/unsupervised_learning/MLM/train_mlm.py index 20287bdf0..0bc055240 100644 --- a/examples/unsupervised_learning/MLM/train_mlm.py +++ b/examples/unsupervised_learning/MLM/train_mlm.py @@ -27,7 +27,7 @@ use_fp16 = False #Set to True, if your GPU supports FP16 operations max_length = 100 #Max length for a text input do_whole_word_mask = True #If set to true, whole words are masked -mlm_prob = 15 #Probability that a word is replaced by a [MASK] token +mlm_prob = 0.15 #Probability that a word is replaced by a [MASK] token # Load the model model = AutoModelForMaskedLM.from_pretrained(model_name) @@ -121,4 +121,4 @@ def __len__(self): print("Save model to:", output_dir) model.save_pretrained(output_dir) -print("Training done") \ No newline at end of file +print("Training done") From e34e8febff1858abe897dced33c9d5c975b66bc1 Mon Sep 17 00:00:00 2001 From: Ningchen Wang Date: Sat, 14 Aug 2021 01:34:22 +0800 Subject: [PATCH 2/2] Format spaces --- examples/unsupervised_learning/MLM/train_mlm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/unsupervised_learning/MLM/train_mlm.py b/examples/unsupervised_learning/MLM/train_mlm.py index 0bc055240..cebc72013 100644 --- a/examples/unsupervised_learning/MLM/train_mlm.py +++ b/examples/unsupervised_learning/MLM/train_mlm.py @@ -27,7 +27,7 @@ use_fp16 = False #Set to True, if your GPU supports FP16 operations max_length = 100 #Max length for a text input do_whole_word_mask = True #If set to true, whole words are masked -mlm_prob = 0.15 #Probability that a word is replaced by a [MASK] token +mlm_prob = 0.15 #Probability that a word is replaced by a [MASK] token # Load the model model = AutoModelForMaskedLM.from_pretrained(model_name)