fix

3 years ago · 0c7cd08255
parent 3d43eaa1c8
commit 0c7cd08255
1 changed files with 3 additions and 0 deletions
--- a/RWKV-v4neo/src/dataset.py
+++ b/RWKV-v4neo/src/dataset.py
@ -189,6 +189,9 @@ class MyDataset(Dataset):
                        i = np.random.randint(0, self.data_size - req_len)
                    else:
                        i = np.random.randint(0, self.data_size)
+                else:
+                    # cheat: pick a random spot in dataset
+                    i = np.random.randint(0, self.data_size - req_len)

                if args.data_type == "binidx":
                    if args.my_pile_version == 1: