no message

3 years ago · 13bb641007
parent f79137b524
commit 13bb641007
1 changed files with 8 additions and 7 deletions
--- a/RWKV-v4/src/utils.py
+++ b/RWKV-v4/src/utils.py
@ -85,6 +85,7 @@ class TOKENIZER():
            self.UNKNOWN_CHAR = self.stoi[UNKNOWN_CHAR]

    def refine_context(self, context):
+        if self.charMode:
            context = context.strip().split('\n')
            for c in range(len(context)):
                context[c] = context[c].strip().strip('\u3000').strip('\r')