From 13bb64100768bf9a8f14cd24809314f3a0c75191 Mon Sep 17 00:00:00 2001 From: BlinkDL Date: Fri, 29 Jul 2022 21:37:54 +0800 Subject: [PATCH] no message --- RWKV-v4/src/utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/RWKV-v4/src/utils.py b/RWKV-v4/src/utils.py index 78bf604..db13dd0 100644 --- a/RWKV-v4/src/utils.py +++ b/RWKV-v4/src/utils.py @@ -85,13 +85,14 @@ class TOKENIZER(): self.UNKNOWN_CHAR = self.stoi[UNKNOWN_CHAR] def refine_context(self, context): - context = context.strip().split('\n') - for c in range(len(context)): - context[c] = context[c].strip().strip('\u3000').strip('\r') - context = list(filter(lambda c: c != '', context)) - context = '\n' + ('\n'.join(context)).strip() - if context == '': - context = '\n' + if self.charMode: + context = context.strip().split('\n') + for c in range(len(context)): + context[c] = context[c].strip().strip('\u3000').strip('\r') + context = list(filter(lambda c: c != '', context)) + context = '\n' + ('\n'.join(context)).strip() + if context == '': + context = '\n' return context