Upload tokenization_codesage.py

tokenization_codesage.py

@@ -76,7 +76,7 @@ class CodeSageTokenizer(PreTrainedTokenizer):
         pad_token=None,
         add_prefix_space=False,
         add_bos_token=False,
-        add_eos_token=False,
+        add_eos_token=True,
         **kwargs,
     ):
         bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token