Update hf-training-example.py

3 years ago · c673bd2fc9
parent 9cea8e599f
commit c673bd2fc9
1 changed files with 1 additions and 1 deletions
--- a/hf-training-example.py
+++ b/hf-training-example.py
@ -17,7 +17,7 @@ tokenizer = llamahf.LLaMATokenizer.from_pretrained(MODEL)
 model = llamahf.LLaMAForCausalLM.from_pretrained(MODEL).cpu()

 if tokenizer.pad_token is None:
-    tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+    tokenizer.add_special_tokens({'pad_token': '<|endoftext|>'})
    model.resize_token_embeddings(len(tokenizer))