From 5817d265c3ed0444dce2d0c317c2839c36fef23f Mon Sep 17 00:00:00 2001 From: BlinkDL Date: Fri, 25 Mar 2022 21:40:34 +0800 Subject: [PATCH] no message --- RWKV-v2-RNN/enwik8-vocab.json | Bin 0 -> 155488 bytes RWKV-v2-RNN/run.py | 27 +++++++++++++++------------ RWKV-v2-RNN/train.py | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 RWKV-v2-RNN/enwik8-vocab.json diff --git a/RWKV-v2-RNN/enwik8-vocab.json b/RWKV-v2-RNN/enwik8-vocab.json new file mode 100644 index 0000000000000000000000000000000000000000..181cd096e41ca209989f4b7f7d0cfd2bf82e38a4 GIT binary patch literal 155488 zcmZ792lyBAzxVNmw1kkNsAv!EX(de+O;JWeX(~h{BeZOVL`IbMP)2siPDb{g4I^2{ z9=G59zv}sZKd<|`&vmZX z{{Mg9D6D7gS{v_L3;yT-_dnTW*H(4c)^yi)%76de&i~0~yVgT@ZOwOW)pl(yc5T&n zZ7p|gHFj;Sc5O9xZH>Y`wZh6hwZqCib;8O;b;HU{^}@vb%6G2ZR`6H8bLF;zzw(_cw-x-A z?_9a9;IDk=%5AN~%58`5TAPI1jtDEa9T`?`J1VT)c63;|txZ_D?U=A~TidX5+p%Hg zwsv9Vw)VT$rs1{@Vdb`tVdb_?Vdb{YVdb{t!pd!3!pd!3!^&;l!pd#Ohn3q-*tIqb zw{;IIxAh1sxAhDwxAh7uxAhJyxAh4tx1AVPZtELXZaXQg+;;M=wRyO$Us$=Ve^|Ng zl>fey?)(~PUWn^X4J+537FMo1J*-?eAgo+BFsxiRD6CvJc-Pt@Tz5uTxo${Uxo&7! zx$Vrba@(-5a@+8*a@&Zoa@)wTa@(k|a@*)#Ys+vOKYO(-#BF23|17tS4J)^u9ae5T zC#>9dZdkc(Tv)m7ys&cH`C;X@@w?Vm;kFCH%54{hmD?tSmD>tl0RQ`L-Kr3`O$;ly zT^v?!yCkgKHY2Ruc2iinZPu=};Ad~%BX_5r`{t&d3-3ugH{P3euAG;4?wp@?F1;`9 z+`1s`T)Qys+`B03@x8J*?cBR0?cBRG?cBR8?cBRO?cBQ}?cBRE?c95R+PQaC+PQai z*y9`KfwXh)gK6j9htkfy52u}b*QA|$A4xm+KALv!eJt(V`*_;9_ldB_cg>S&=iaB% z&b@2X&b{l>&b?2koqN}(oqL~2JNG`DcJ6&H?cDo(*yCI0g|u_;i)rWHm(tF?8`93b zFQ=V*H>RC?Ur9UnzM6LKeJ$UsHdioqPXGJNN#TcJBQ< z?cDoM+PU}Nv~zEv!aLt)jS96Y#Jz>n&b>v_&b>u+PSxM+PQa+v~zEnu*dh+o@wXaz0%ITd#9ay%ch-s_eneV?wfY* z-7oFjTQ2R~yMNlbw|v;+`>I0Px%YsybMJv^=iZ8G=iW+b=iY%oqMaK zoqMZ>J-)9FNjvu*ns)B3mUix~o_6l7k#_E_nRf22m3Ho}op$c6lXmW{8}|6Vs-Jf5 zJuL0q+aT@S+c53i+bHea+c@pq+a&GW+cfRm+br$e+dS;?ebplE+}kqk+}kSc+}k?s z+ zN;~&Y8@$?Ur`#Jt6Jf+db{v+avAV+cWLl+biwd+dJ*t+b8YZ zdt%t*`>Jo+x%Z^BbMMJ%=iYv4=idHl=iXD&&b_CmoqJD9JNKTRcJ3Vz_V~UUn0D?R zly>eNoObR#BkkNfB<ix%d3EbMN@HbMFOd=iUp`&b<@T z&b=33Ogs19m3Ho(n|AJf2oObSA688AMT9$V1U7mLCU72?7 zy+7^TyDIJ6`#{>c_u;g2@1tqw-pA6;y-$WczOUA%oqN}%oqL~7JNK?nJNLelcJ6&G z?cBR5?cDoD+PU}5v~%x=VUO>tkJ8S)U!aI@x1^nWx2BzYzezjy?npcLexG*k z{XOm6Tdc^=Ungl&D88@Crk#7sr=5E%q@8;YNIUl)n0D^1n0D^1ns)AOmUixKo_6l- z9`^XY>XCNt?U{D&?Ui=!?VWb+?UQ!yJu&Uv+c)jpds5oD_vEy5Z@*o8!FOlgSN+q@ zy{Dv|drwU}_nww^?ma#2+&duc+&eJs+&d`k+&eh!+v~%ysv~%yMv~%z1v~%xSVUO>tF=^-Cv1#Ytv(wJK=cJu`&rLh`j!QfD zo|ks+JwNT-J3j5)dqLRa`|859bMJ(-bMHlI=iZ5F=iZCc&b^nUoqI1$JNI6ecJ7^& zcJ94A?D2hdMcTRd%CvLuI6?p>62?p+-A_`X__cJ5u8cJ5u4cJ5uCcJ5t~cJ5u7cJ94D?cBR6?cDoN z+PU}Pu*dh+BWdT}$I{NdkEfk`pGZ6RKACpzeJbtTyEg6IyDshA`&`<&_xW9W!FMa( zS1+WUdtXjF_r8{P?tMM&-1}zQx%aKKbML!p=ic|y&b{xaoqInBdwgGgly>g@IPKi~ zY1+B>v$S*X=V|BOFVfDvU#6XVH>aI@x1^nWw}w5wuePO~d%sFM_ZBL;^Z(9jR;Y3K z=UNM=oqLOoqKmnJNFh#JNNFMcJ3_^_V~Ukm3Hneop$csBklb7mPtGJmPJBvxwlfANIUm7Ogr~B3VVEC zHBCGBHcvbEwoE(swn{tqwoW_u9-emYJtFPgdt}JEwxwli=xwmuL zx%arVb8nZlb8pwQb8ok_bMNtK=iY&7=iWhKkMFAqY3JT6)6Tt<)6Tt9($2lp($2lp z)6TtDrJZ}PNjvvmmv-*GKJ4**bwk>@_r|ny?~Jr_@65Dw?@ejv-ka0Ty|<*DduOGc zdv8rU_s-t67ku^dzPc^#+f2oObSAl6LN0ns)A8mUix49`^XYT9J0{U72?7 zy+7^TyDIJ6yE^UM`#{>c_rbJt??Y+l-iOo9y=%fA-&c>MoqHcmJNG`8cJ6&V?cDoB z+PU}1v~%xMY3JUxY3JT`Y3JUj!yeyP>(kD?&!nAupG`aWK9_dxeLn5n`$F2e_rQd#0Ux z_X>M_U+tZC?k$^k?%gNt+`Dhuxp%*`b8oq{bMO9X=ic&Z=iUlw=iURt9^Y37rk#5$ zrk#5$rJZ{ZN;~%+oObT5oObT5l6LN`ns)9zB<aetPZ-caRZ^N{6Z=Z}YTsZ;P#boqLZ;JNF))cJ6JH zcJ4hU?D2ioHtpPdY}&cEUD~;~ecHLVL)y8wW7@g5Q`)(=bK1H0xU_R`m$1k8RoAq0 zZ@08_@9}Bp-V@T!z1`E!y*<*-y*<;;y}i=Ty}i@Uy?w$S-&ZH5oqPMHoqJD8JNKTP zcJA$$cJA$;cJ4hT?c956+PU|%v~%z2VUO>t0cq#nfobR7L22jS!D;8-Gt$nzL(($2j%?%E4}ba`LRNIUn=Ogs19ly>gDIqlqgOWL`2 zR@%Au*0gi)?6h<5ZE5G;+ru8;S98+Ny?3OYd+$s;_uiFu?!7zh+&eez+Y3JT0Y3JUhY3JT$Y3JVMY3JS*VUO>t zm1*bR`_s<7tJ2QBtJBWC52T%YA51&+feL3yiyD{wXef3J(x%btybMI?u=ib-T&b^z`&b@D>oqOL*JNLeo zcJ6&U?cDoL*yH=^-L!M>duiw1_tVb3AEcdoKTJFKew23Z{W$I1`$^il_tUg{Wa~}`&-(%_xH4O z?;mOB-apgMy?>>hd;d;5_x_W1?)^9H@qJaO*v?=7Z}s1w)W*Gq)6Ts`($2j_)6TuS zrJZ|=rJZ|sPdoP(PdoRPNIUnI410WEl}bDJmQFkO?vZxxEt7Wc-81dnyI0z|cki@w zZ`rhS?>=eg-hIO!-&gykoqNlroqP9BJNK4PJNH&dJNF)tcJ4he?c7^2?c7@_?c94% z*yH=^;IwmZ<+O8em9%qj)wFZ(A!+B{L(|T^)zZ$r)zi+sHPX(#HNzg?SGCg4y|vTM zy>-&gy>-*hz4g-0z4g=1y@#cpdmE&kdmE*ldmD#6zOS03oqL<6oqL<5oqL<7oqJoP zoqJoRoqJoQoqJoSoqG>YJNF(D_V~U!GVR=ZRNA@s=(KZho3wN9F=^-CwrS_yW7E#P z?b6P@?bFV^9l{>pR~^&Ny`9p|y`9s}y~m}Ud%L8ad%LEcd%LBbdyh{$_nwe;?(H7- z_`d3qcJA$&cJA$!cJA$+cJA$ycJ4hf?c95E+PSx1+PSxX+PU|XU3I@lf+loFbNHYOK31cLW7YK8jOJ3<(WpN@#GCga$V!iLba@Bs7>Mp~0;Z8qAi^;5G>jZkNztj)VqxNN8}Uga&sd zi7&gmB{Y~Tp}{>88r&2CI|A7vKXD8aybW!9x-nJS?HX8VL;^kLW8`mC)c!2@QrxXfRwt zg9{}zxI{vODH0k?lh9zgga)@JiSNWk5*nP(nh3k`fw}lF*>Ega&&^Xi!E%gFPiQ z*egkVC+;nwL0Jh6_L0zFUkMHNlhB}?ga-RdXi#24g9;KF9FQcw6AzToprV8Zl_WGc zNJ4{yB{Zlkp+OZ14XR3LaEOEkhbD>d#A*^6RF}}8hJ*$+B{Zldp+Rj44eCf}P**~O zdJ-Dc4^r?mBk#n+Bs6Fsp+Q3l4H`*k&{#r)CK4JnmC&G>ga*weG-#0|z7tzYXwXVR zgCis~I8s7`qa-vqT0(<15*i#Mp+Q>-4USC`--+!cG-xlOK?ey9I!b8JNkW6p5*i#Q zp+Of34Z2Eb&@D-PCw7<6pofG8JtZ{gC80rY2@M8FXfRMhgFzA+43^N~j3n`$I7C8& zp%NOLDWSnI2@QrzXfQ%TgOL&%jFQk`w1fs{C5i9EF%lY#mC#_4ga(&OXfRnqgDDak zOqI}JnuG?^B{aA`Nqi^Hl+fU22@P(M&|sE?2DeIRFk3={+axr&T|$F75*pl*B)${x zl+fTV2@UR+&|t2F2KPv4aIb_0^CUEwFQLJG5*jQ>65ojnB{Wzpp}`Uf4VFr1uuMXO z9uam@g;x`f+d@G^Bb_or> zlh9y?ga*5p;@1zF{`dEjXi!{2gZ(8mC@-Nwr6lp4SXn}YLnSn*CZRzs2@UE?XmFT> z1`Q=NXeOaSO9>5*N)q3RM@wkXMnZ#QBsAzGp}{E<8Vr@t;7kb(hDm5JTtb5pN#Z+k zq=W{eBs3T;p}|=a8jO+9V621&lO!~_Ttb7%5*kcN65okaB{Y~Op}}+s4X%~Y;CcxS zZkEtsmV^ekN@#GKga&hx#CPId5*o~v(BNJP4dzK`FkeE0`y@12AfdrR2@MuYXs{$n zd?zlI&|sN_2FoQhSRtXoN(l{CNocTILW2h-G-k zNqi@MC!xU(2@Q(u5r1xM5q@smT|$H65*q9!p+Pwb4a!SsaG-<+m6ODGVpRzZs!3>2 zL&E2vri2EyBs8cap+Q{<4eCi~P+vlW!;-{zVgm^c8cJx;SVDs)5*jp<(4e`51}!8s zXepsVD+vvb3R3XhfOq225*oCT(BK#e4cba*&_P0jjuIMlkB2G>bwaD#*fH%n-6i-ZQVBs929LW4OH8r+p6z7yw4XmGED2J<8|m@lEheG(ch zkkDYEga(TxG*}{`!O|r0ow!UwgXIz$tdP)PrGy5nBs5qpp}~U^8ayPS!NU?7tVt5z ziBC#s@RWoGYb7*TC!xX95*j=wp~3SK8oVT-!3GHpHYSPh#4QpUY?aVpn}h~mNoer3 zga+S8Xz;Cs2HPbx_)bED9ZBLlu}B%-JI(+5`x`VUCZWOZ5*iej(4eG*2IV9)C@-PG zff5>2lF*=XlK4)nDxtxl5*k#K(4e}61~nu!s3oC69SIHUN@!3|LW9GS#CKu?2@M)b zXwXPPgT@jXG?CDtnS=(-B{XOup+QRt4O%6M@5G}dG&ov9gEkTx93!DY2MG;2N@&nY zLW3?68g!G;U}%tnuLRzSXG&-=OhSX<5*mz<&|sv52BRc27%ic}SrQtIkOWh6A%Q$mBiBsADtLW8mr8tjuKz7zMA&|p6a4a!Mqu)l-`bO#49BUp~3AE8r&(N!CVOq?nx5g ziSs2iSRkRnVhIhFNocS_LW7kO8r(0T!72$29+1%B!6fmWxJE*QwGtYvlhEL42@PJ5 z(4h2Qyu4c$Y8?I<%_@A@|SqTmHOA_CSwyD4{_m2@MXC(4ew} z22~||4(ds0P+vlW!z46lkR-kn8%k)dV??`Cyu7n2fNoerCga#i-Xz-zg1|Lai@UesjpGau%X_EL({7gcF&m}bYLPCQt zB{bM9p}`gj4Yo>XuuVdPuOu}1I!HkQ-ihBxXz;Cs2HPbx_)bED9TFORFQGx{vV6BJ z_^YPD5fU04DWSnpN#Z;4XbBCDm(bt@2@SeSXwXAKgWeJv^pVhDk%R`1NoeqRlK4*i zTtb5_5*mCjp~24*8Wh?m{+Kq7|1Nn64T?x;P*g&LViFpZND|+Pr6e>cEuldf2@T3g zXi!1I=b)m529+cdVbtE*XE1^L>2@UE?XwXnXgJu#MG?&oeNC^#& zmC&GllK4&>D51e12@QrwXfRYlgJBXHjFr&fJP8fXm(buM2@NJEiSNXj5*pkrp~3AE z8r&_R!8{2K=1XX>P(p*n5*jR#&|qnj_)c6Vp}__T4K_+>@P>p2?@MU#k%R`@BsBO& zLWAuR8hoE5z7v0v(BNkY4gOC;gF^ep@BS?d(O@?T4T?!fEuldl2@USvB?WJT_)eTFp}{>88qAZ>;64cr z7D;HZT0(<|Bs6$fLW4CD8a$aKz7y9;Xz;Xz2G2@p@SKDO&r4|VqJ#!7OK7lBLW4~b z8oZMvz7yY<(BKOR4K_|fx=3iyT|$EsB{aA~LW9W?8a%##Y|uP3*d(FBW#!|4CcYCVNoa7n zga%hiXfRnqgDDakOqI}Jx`YN-N%&g0T0(K|5*n0B65olXB{V1_p~0RK8tf&ZK}89l zgM%eBs4SsDRS6BMNoY_#Nqi^PlF*>8ga(ZzG-x8BK{E*rT1aTnN?1kW0Lqzd{sh&*CjM~LqdZ$B{XK@Bs3@{p+N}=4N4}7@5E9P8tfsVK^Y03gS{m* zC@Z1Cei9m#lhB}oga!vlXmC)H_)e@Mp+Qv%4Gxvipt^(xH6%2sC80rW2@M)bXwXPP zgC5HNodeULW5%@G&oj5gLV=cv`-S>iJc@g=q#Z@cL@!8 zNNCVoLW4dM8k{Jh!2k&j21;l!BuRWHj*`$|tb_*VNN6xlLWA=qG`K)Qg9#EETq~i$ zN(l|_PZHmW4@qe7u!IKdB{XNG}t7e!P^oVd?}&9W(f_pB#H0DZzVMNNkW5P zBsBO(LW9y(;?Jio!#|I-hlB=YBsADlLW8{|G}t>yd?%Kb&|qH)4fd1JpqzvT`%Cy7 zl$X$;f`kSKNN8}Nga#FZ6#UG{uP0WL(BL2m4Gxyjpo)YBRV6eyL_&ia5*pN!(4d}# z28Shy@5IIu8Z?p6pp}FMttB)#LPCS1B{XO!p}}|w4JJxxaA}hGPMjp6!LgQ5}|6qC?k_ayP1SVBUBvJx8XBcZ_o z5*i#Rp~1ltJ_nU0G^i<|K`jXlYA1>B#5xih)RoYnse}gYB{b+Hp}_zN4F*bRaE62i zLnJhqk|e$pr%GsWpM(bANocS`LWA!mH26V6gCd8JQ5*j=z zp~2%48ayGP!7~yXyp$xq6E{d`@T!Cc?@DO!o`eRw)!=8DrvLp43JrFb(4eM-2CXGD zI9x)5Ba*~-VjBq!+Dd59NkW6;BsAzDp+Qdx4F*VPFjzu^Arcx4O%mUUBP28!DWSn= z2@S?bXi%tO{2a6hpM$~@8WfSxptytvB_uQ`nIygwOG#)@T0(<8Bs3@^p+Qv%pMygr zG&od3gK82QRF}}8Mw0kWtSO;EEeQ>3OK5P6ga&OTG&oj5gLV=cbdb=Xql5;Xf)sp5 zX5(y11mC)ca2@NJmXmGiN23JUE zaHWI>las`E;uHxDrb=iqO+tg|5*l13p~2M>8eAix!Lo-X!szI8Q=@`4Sr3C!xUt2@MuXXs}2^gT)dW zERoP)se}g0lEio7atRGqNNBK9LWBDyG*~5}!D zMF|aFlF(p-ga$85Xs|I!d?&slp~0&X8oVZga)4_iSNYEB{cX#LW3_QG}tVm z!4?S(wn}KQO+tgOBsBP1LW6IT#CPJi5*lom&|rsz20uw?@Uw&le@bZZmxKmo8u6=A z&Hwu~DH`l6p+VIo@ts&rLW5co8Z?s7ps|DoO(ZmEDxpC$2@RS{XwX7JgO*9+JF%67 z2CXGDI9x)5BP299QbL2HBs4f$LW4FE8XO~`LE9wpop`K-2JIv?XfL5b2MG;2N@&nY zLW9l{8XPB~K^F-Px+aP5#BLHA951242@)D~m(ZYxga$n&H0ULvL2n5S`bcPSVvvHb z1m1~#B{VomLW7efH0URxL4OGiPLa^yR0$1ElhELF2@M7$iSNXL5*iGW&|t8H216t? z7%HK`nGzZdlh9zega#udG#HsAz7t1DXfRqrgR>+w7$c#Jl2%ND|+PbtE*XFQGw02@M)cXwY0j zgO(B+w35)EwS)$TOK5OplK4(MRzidJ5*l=p(4ez~23;gH=qjN>Hwg{8OK8wjLW5pG z3O)tA6MIW&&{sl(lO!}aSwe$TB{Vo)LW6-48Vr@t;A{yE&P@{EiI+%daJhsAQzSH) zCZWM}2@S56(BK*g4Q`asV1|SSHzkSh#G55FxJ^QXIT9M&DWSn!2@U2+XfR(wgZm^j zSRkRn!X)vXxJW{SB@!AemC#_hga)f5GazBB8+@5*jR(&|rmx22V+7 z@N|%Z0=yHSkPC|qB5*l=n(4doq&p~Gi4Z28Z&@D-PCw7<6pr?cey(Kj0 zC!xU*2@TGa&|sK^2E!#Z7%8E_s3h^7I9fu3u@V}bEuq1=5*mz?(BM1?4aQ4oaDjvd z7fNU_F-d$UPL|MMii8GNNoa75ga+42XfQ)UgPSEZm@T2f?GhTyNfO_QOC>Z|A)&!a z2@O_BXz-wf1`kPS@UVmi&q-*oK|+I#N#Z;4RS69~lF;B&2@SSLXs}g6gB=nY{2-yh zZxR~(A)&#ayQJXf#rRJATS9|EJ$L@h^uPZM4R)8%ptytvWhFG&M?!=05*k#H(BMD` z4Gu~Y--*>EG^j42L2U^Q>Pl#En1lumB{XO*p+Rd24cba*&@oATCw7z2pofG8CrW70 zUqXYy5*nN#p~0CF8Vr}vV2p$YW0S;p;yDr;oF}2d6%raumC#_iga)@uXmFQ=26szn zFjqo@g-PN&agl@uizPHzBB8-*2@M{Q(BKgX4W5+H;3)|W)=Oyce3JN1+$5pFTM`<) zC!xW|5*mCip~06D8vG`q!9Nlj{41eBv0m|ee0(RCkkDX%2@T3iXwX1HgH{q6w3g7| za0v~Lme8P$ga*eXiSNW?B{XO!p+S2I4LV9_&`Cmr<0LfbDxtyg5*nN!p+WZ~@txR1 zLW5os8uXUXppS$GeI+zFNkW5u5*nN$p}_zN4F)EO@5Dh88k`}a!B7beMoMTfMnZ$J z5*nN>p}{x_4aQ4oa8Zzg7ZSgoc%_5}QzSIFT0(=HB{aBILW8>`G?*`;!F>`Mtd`K= zu_W=G_>_bO>m)RIT0( zOK9+}ga)5TXz;0o2A@f2@P&j1UnYs~#LW^KY?IL7YY7d$kETO?92@S51(BKvc4c16#uvS8YbrKppC!xU$5*lnw z65olhNNDhxga(@>H26wFgRdnt_)$WG;(g+Gr1a+}N@%d3ga$Q}#CKu?2@M)aXwX4I zgX1JLI6*>#UJ@FdF5z=9P(p(Hxe3rE1|)65*mCjp~3G#{`Ys0cmBEIe@JLh|rog_3kK|+Hb5*qZD(BMP~4f;xGFhoLwGbJ<_ktDto z&y~<%oP-AFOK5Phga(&OXfRDegR3Mom@lEhatRF{OA_CS&q-*oK|+H!Bs6$aLWB1u zG}t1c!M73`6z&(lS2YcPCQMNY4Jsyy@5D+H8dQs_)a`s zLW98)8k`}a!B7beMoMUKfrJK=Bs7>Rp}{l>4W=iF@5HMmG`L1WgIN+9+$y2LYzYl+ zlh9y}ga&s=XmF>526rWi@5Fl~G`LqngLx7fESAtC80stB=McNpM(b0Bs8cmp}`Rn8XPI1!BG+#94(NN7-3!snoVlK4(MOhSVO5*oCW(4f7927M$n z7%ZW|IT9M2E1|(S2@Ng`Qt%y-cj5#I4K9+-kiMETO>@ zN#Z+ky@UqON@%c2LW6fCH26?LgHI(i_(np5|C7+*ZwUk=A#BB8;j5*mCap}|%O4R%Oq@RNiFdky3hU+^mdG^m&) zz7s1;XwXSQgDw&pbd}JckAwz;B{Wzip}|8E8oVf>!7oYDJ5fS|LW5$_CgGpKC?TOi zDG3eskkDW+2@Up^(4ee@1{EYUsF5VT6YEH5&``qXps|DoO(isFE}=mS2@P6GXmGTI z25lrX=$s_J6T3)g&`m;v<0UlcA)!Gp2@Ot?&|tWP2BRc2I7>o(Ya}$7A)&!NK?=SScqh)6&|rat1`8!L zSS6vss}dT#CZWM52@T$t(BK0J4fa2SPhyMzUQaZrB%#5<5*oCY(BNnZ4cba*&`v^w z4iXv+m(bt}2@Pf>iSNWaB{Wzjp}|WM8f=u%V2gwX-%4muXh{5#YaIUUzy=Z;G?UPv zd6M`}JX}J9qa-vqT0(>N5*l=p(BK3K4NjKuIT$RV!B7be#wLmH#BmZDjF-^h0tpSK zN@y@$LW8R%G`LPegBv6?m?fdX>?HA>xJ*KW6%rb(l+fT|2@M{R(BKIP4W5gkF8JR@gQ5}|l#tM%q=W{gB{V1_p+R{G4Jsyy@5F;7G^i}0 zK`jXl>PTo%S3-k&5*pN((4c{Y28|>%Xq_a!6Wd5=&{jf&Vp}|B64K9(;;8F<;Ws>+#yg@>P zSrQt|kdV-%4n(LqdZeBsBO_LWA9g#qXod z;;*4dXi!!{gZ(8ms4AgBoh0#{*iu4+RuUQL-cs#N#A1=p&)QXbBC*NN8}Lga+d!G?*yi zb8xYQ2A4}{@Ia7)0=yG9NNBK8LW6fDH26|NgCZmNF)jEjpuqtW8dR0g;7|z-YD;L) zDoK1Nwv*7Hi-ZOzNoa7oga#ueG`LhkgUci|xJg2Tdn7a{H8TDv#CPKU5*k#I(4eY> z28T#!P*XyKBPBHGD4{`D2@SeSXfP;Ad?yZ;&|rv!216w@I8#D{;Sw5*kJ@3%}=kZwU>`CW-IF{UkIfFQGvN2@NVsXmE&x z28T*$P(wn4ni4(-wInpCog}^!50lWKfrJJPB{XO%p+O4?4O&WQ&_+UojuIMlmeAmY zB=MctS3-l+B{Ucyp}`pv8jO?B;Cu-UCP-*-v4jSfNN8|HlK4)XCZWM}2@S51(BKvc z4Q5MdaF>Jzb0svmM?!=9B{W#QOA3BYi0{NT5*j=rp}}Jk8oVf>!AlYvyey%?TM`<4 zAfdr$5*mD-B)${Bm(buR2@U?1(4h3_o!_quTE{;>vAl!^6(uw{SVDv95*pNz(4cXW z_)a`rLW83uG&n{=gANit2VEpII9@`7UJ@FdETO@02@OUhiSNX-Bs3T=p}_?b8cdeZ zV48#m(jP;h)K>FQGvr2@RS_XwW=Kd?y|yp+N@; z4LV9_&`Cmrt`ZuYEa7v|UqXX3B{Uc=p}{#x;ydv|2@UR+&|rat21_M0SRtXoLlPQ1 zBB8-E5*oZFp~1IF;ydv-2@NWp8$YAX!|&;;EuleO2@RS{XmGTI2JIv?XfL6`@e&%G zkR-kn2TN!$Ov2}2s)Pp9B{aBNLW5f*G`L+tgB21Qtd!8;u_W=GxK=`gmn1aUD51fp z5*lol(BNwc4Yo^Y@Q;KBMaRWYZ_Ds|Ja#498;xLQJkl@c0!D51d?2@QUc(4fM3{FoN}Rny=A2@MXE z(4bP1_)e@Wp}`>%8XPL2K`jXl8cJx;NJ4|g5*jp-(4d)w1}&1rcVbHk4O&ZRaJYm9 zT_rR)Swe$B5*iGV&|rjw1|uaj7@H)%6VH{o=SyfXMM8sF5*pklp~1Zp8Z42} z;7JJ$o=pwFFjd0mV7Y__>m@XJNkW5HBs6$4Nqi^1Euq1?5*mCUp}{8-8hjz4 z!Iu&md@Z5DcM=->D51g6N#Z;4Hwg_2kB^_;7U8YDhlB=|B{Zlap+Or74F*eSuvtQb zpCmLWd_lY?z7vZ`Xt1Az&p~4e4Vp@5&_Y6kRuUSFlF(qHga$WCXmD$i_)c6Pp~3wU z8myMk;86(;9+%KyorDJOOK7l7LW3O=8vGih;AckOiLEc>6H*{FI6^{$qa-xwB%wif z2@Ot^(BM=F4NjBLV6=n=IHb`i& zF-d$UzAmA`76}cuOK7k|LWA!mH270OgVGb?XEgqP=RGAfI8Z`^!;-{zVq*yn+DK^7 zRzicW5*l=q(4en`24f|B4#r7naK3~F8LqdbQBs6$lLW2zw8oVc= z!G{tWd@P~Cr%B>FafgHkze{LP^rH9~E%@^?cm5l%%1UUkpM(Y#B{Zlap+OA^4Qfef zP%lY*Cw7p~ptFPqT_iN!FLiGRGk<OU;E^Qpo%n==22V<8@V0~oA4_PkRYHUBB{cX`LW4pV#~RWfB_PFQLH$ z5*j=up~0(3;ydwm2@N($Xz;#-2A@i3@P&j1+axsDA)!IxOZa}!^uPBB4R({z;Ls%T zop_jp28|>%I8s7`V^!6FF_R!C^@ zgoFm`Bs6$dLWA#v6nsbIo%ol828A!<^H%Vmp+Qjz4T?!OdrN3gPC|nw5*jp@ z(4a+<_)hFCp}~m~8uXLU;4}#h#z<%|RzibwBs3T=p~1xx8eEzrz7wyI&|tEJ22&(7 zm@1*cbO{ZvkhyFh@dz zyCpPuLPCRgB{cX#LW9jo;yZDhga%(rXs}&EgP$Zc_*p`OKO{6LetG=Fw+R1CRtX6W zN=j(3N0RtXJXk`5IuaVxlhB}{ga(ZyG-xHE!B7dGgR>+w7$c#ae{;fmq=)E zxr7EYBs91~LWBDxG*~L3!7>RA?w8PDb&!JZ2D}s3NNBJ@LW5T&Gg;On&D-+4lVViFqElF*>Bga*wdG-xiNL1zgK zdM1hQ#M31-7%ZW|nGza|kga$uKXt3Lr@$=Rs{Jc;~LW4aeG^m~=z7uOoXi!T+gL)Dg94VneX9*4ZN@&nu z!slS1ga)G|G`KuTd?#Klp}}kk4OU5Lutq|I4H6o>E}_9k5*mCWp~1Hj8hn=|z7r3f z96xXI-|t*qLW3(MG`LDagWDxESR|pr5(y2KN@%c7LW7M-;ydwG37>;6B{bM9p~0UL z8Wf)rTgTt;Twg+i<0UlcBcVZm2@Ot965oj;&B{XPu+QR6>L1N#Z-PwS)#oNNCVoLW4dM8k{Df!Egx;&X&;N90?6B zl+a*$lK4)%MnZ#YB{aBMLW4UcG`L$rgLx7fESAt z&|s*924f|B4#rDpa9NOoPXX`5t0Xj-E1|(W2@MuWXs}#DgOw5*tdh`Rjf4h|N@(z8 zlK4(sC!xW52@Rf;&|ss42CqwK@R5WDA4_PkSwe&D5*qxRB)$`WmC)c12@Q%~!&hU= z|Nd_&8kCmMpp1kD`$=d}Q9^^t5*i#Lp+Tc0@txR0LW2$x8XPB~K|cu%21;l!NJ4`V z5*mz>(BMJ|4JIUs@5GrB8r&hF!JQHs%$3k!p@asjB{X*Ci>{Bv>IC5i9EP7)dn zmC#^`ga$83Xs}U2gHI$h_*_DRza=y%d3|iqD*PHs{U8MecqcZH(4eJ+28T;%aFm1w zoh39lT|$GQ5*iGX&|s8=uZ6Rc#CPHt2@S?dXfRGfgUJ#a+#sRBjS?Enl+fT72@P(S z(BRG_@tt^=ga-FYXs}2^gJlvLtd!7Tm4pV5N@(!7ga%JYXz*l`_)dIYLW371G5*lhB}xga*e;XwXYSgWeJv z^i2}qi321w7%ZW|a0v~@OK5O`ga#8NG?*x%!KD%!Op?&xswDB9c)f%MH%e$QQ$mA# zB{aBCLW9K;8Z4L4;C=}W)=6maN|N|ad|g6=k0dnsOhSWg5*qw1p+Wr{<9C;q;qNzg zme8P^ga$n&H0YZoz7vN?XfQ%TgYzUbIA2183nVm{D&cc5OG1O&Bs5qep~2E!Qt^(F8Rny=Q2@PsVXizUnd?z-L(4et|2K^;8m?)vaZ4w$Rm(bud z2@NXT6zen&e}AEwga$p5#CPHp2@R%6XfRtsgS#X&m@A>deG(chk} zB)${ZNoeqbga#WVG}tJi!6pd}-jdMZLkSH&lF;B22@SqZ65om6N@%cMLW3O=8vG!k z!7maTl)gEBuA7CQ8~2pZpsa)j`$=d}E=hbRmY2}rKnV@1N@!3`LWAlO8q}5WIXFy0 zgQF!hXeXgT`y}z5*ik}*&Jr5*lF;B32@Ot_&|tiT2A4``aG8V#(rYiAPIl&_+Uo_7WO&kkFu;gwH{D2@U#4XwY9m zgHt6m7?dQw6VH&)V7P<^V05PiBlvrm?ojY zRT3IpEuq165*l1Dp}`Fj8r&qI!R--kDLoWNocUQga)-FG-xlOL1zgKdPrz6L_&iR z5*mz_(BP~j@trtULW2ns8eA%&!4wG%rb%dUy@UogOK5P5ga)%EG+3G>z7wC5(BL%* z4feb(eyW;;_rbmr8Z?y9pr3>Wqa-vqTS9}05*l2dB)$`;NN8}4ga$WC_#7;h&|sy6 z29HZ<@Qj28Z%SzJu7n0#lEio7b_or>lhC00?eTLE--(SRG&n*+gJUE#I6*>#-Vz%0 zlh9yHlK4)%Q9^@l5*q9~CpKsvz830BXfRPigGmw^Tq~i$90?89NNBJ*Nqi@kyd&Pz zBK$ij4wum2XbE2nr%GsWy@UoUBsAC}p}`Ib4Stu8oVl@!RrzlY?9F6 zT?q}|3sUeiBk#nIB{bM3p~1d)@u@2K-$sKf5*i#Xp}}zy8g!A+pu2F@iPexzLd~ln}i16OK9+ega-d7p}}7g8WdUi zXcl-K$MMHlG(kB{rGb+(@LCo*=^lc^V)J(vt~VlE2RmGCe7N>q?z3l*!%l?bAP@#Z+k!A=edt}&mQAW zTrHtNse}dxB{VoGp}{E$4eBH`sF%>7Rl*&#NoX*fNZg5I5*pl>&|pSFgGC7qo=9l0 zB%#4G2@NtojCZgjybmQ`LWAXr#GP0op+TvH24xZ&R7hxWTtb7aga)tl#qTtJeqy79 zJ7|~C;8G%SCw53^&?%uow}b}05*l2S(4b#Jg8>P5FfO6Nq=W_!6Nx)9|5|LlEBt<8 zg@gvHBs3_H&|s5<2E`H@?3B=;QbL1+5*kzo$^Bb_o!B7Z4!$L!L8F8QO%fWkNN8|Q zLW2nj4Q3@Y$onymY3@6vL46`|CpJrH&?ce51qlr2GbH6%t&Z3E1|($ zB5@}!NNDgxLW5@#8f4yzkHXgQ|5(;ZXiz1gLA8ViwGtW}lhEKqB5^0yNoY_np+SR$ z2BQ-0;HHEIV-gzNlF(p6LW3y@4ellqcVa<*eB?^QYhs~<1_vcHsFBd%YZ4lCNoX)B zp}{=~4Q3=Xc$`SwiEFRN2HQh}0tpQ^OK7k~LW7rnlKyjO(D>6-XmCbCgC+?Lnk6)7 zO(gEba}u5luS;lfUP6O*2@NhuXwV^{LAQhkS0yy)mC#^3k+>7*Bs7?p&|pDAgCz+K zo=RwtKM?O=XLz6LatRGqNNBK1LW50-#GUw(ga$5E`Of7M8dOPWP@PEJ zi8T@$)JkY@OhSWt2@M(~G&n1vLAQiExFVrJkAwz&iNu{aAfdtA5*iFiXi)lgeB?^Q z?-#a9Xiz4hLAitmEfN}ZNodfMNZg725*mz2Xs{@u!IFeK$PC6h+r#_TS4wE`f`kTZ zB{Wzkp+P|+aVM^q(4bI4gBK+<*eszziG(}YBB4R4ga&028dOMVus4yo6PqP8XpzvM zRYHS{5*l<$XwW60LAQhkgAy7HNoX*fNZg5|5*pl=&|pDAgZCvgSd`FUNkW6Xq4@kM z3!f3STtb5;2@SrTNZg6d5*oBfXmC+NgUrw4SDpKOi0AK9T_K^tS_us*CEUSY2@Up1 zXs|y>?$v;ucvwP%BN7_qzr%N$`_*XBGn@(ydL=aIlhB}FLW4mG4Mrq17)vDX#Bm7? zrX@6(ku z2Va%Y;FyF4$0am4ok-k?jS?E1kJ278E zgH;k5te4PWlY|CE5*ic-$vqR;i901U*d?Jsg@gu`67HZ{LW5cf4H_ggxFDgy6$uUc z5{WypUqXWc2@QrNG?ga!vBH2A!P28Sdx_=1E6uSjT6C80sJ zga%)f(BMmn#GP0pp+UWb2B#%7XpqpLQ9^@e2@TFlXz;3p2CqqI(3VKtiRUFWxFDfH zr-TNVB{b-f(4bdBgAoZ0GGp<%y*<1)Z<&M!d5OfGm@lEhDhUl%OK7l0LW6Y@8Wc)s zuu(#TO%m>)PC|okBocRGy@Up5B{XQ2&|pbIgUon**(mnWgYY6%SvN@&m`p+S2h zaVNeZp~0|(1|t$0j7n%QCgBclOK5OMLW4yK4TgRlTgT7Yp1Kt!?!=;rRA^8vp+Sj+ z2D>FRsFcv4T|$FP5*l<;xs)Po;5*qYNXfPn5!Jvc&`S0;49piM%93lbW%ClYt!4J5lF*-(l`nAzuMIv!0mP=@`TS9{h2@NVGG&m%oL5+k4hb1(qmC)dbga*eGi97Lxga)T1 zG^m%*;IxDW4H6nOOK9+_ga)rkXmD9VgRw;7PMna?AS|&|p?VgEo{d;K8^q*8{@Er*a&PizST?q|d zm(bvQ5*l1gB<{pZ5}pfhNNDh;ga(%-G`J$6!Bq(jdL=aYk%R`j!^5e^=#4Bs9pE&|tZQ1}h{qSSg{w zY6%TKDxtxeMB+|dE1|(U2@TduXs|&-gBK+<*e0RD4hao*N@!3ap~0R+;!fNrp}|)q zH29{322Bzgd|yI?zes5CR6>JHw&c(8O5-QvWU^ajbjW14B^K9WCR-|_MJBsLMvY8% zmy8~n?5AZE$z)%a(Ik`IEu%^%`x#m6@>v literal 0 HcmV?d00001 diff --git a/RWKV-v2-RNN/run.py b/RWKV-v2-RNN/run.py index 6a1c462..a9bfb59 100644 --- a/RWKV-v2-RNN/run.py +++ b/RWKV-v2-RNN/run.py @@ -6,12 +6,13 @@ import numpy as np import types import copy +import time import torch from torch.nn import functional as F from src.utils import TOKENIZER torch.backends.cudnn.benchmark = True torch.backends.cudnn.allow_tf32 = True -torch.backends.cuda.matmul.allow_tf32 = True +torch.backends.cuda.matmul.allow_tf32 = True ### Step 1: set model ################################################################################## @@ -20,8 +21,9 @@ n_layer = 6 n_embd = 512 model_type = 'RWKV' # 'RWKV' or 'RWKV-ffnPre' -MODEL_NAME = 'trained-31' # your trained model -WORD_NAME = 'vocab' # the .json vocab (generated by train.py) +# your trained model +MODEL_NAME = 'enwik8-ppl1.65-6064-1024-RWKV-6-512-2022-03-25-21-05-13' +WORD_NAME = 'enwik8-vocab' # the .json vocab (generated by train.py) # --> set UNKNOWN_CHAR to the rarest token in your vocab.json <-- # --> unknown tokens in your context will be denoted by it <-- @@ -44,12 +46,13 @@ top_p_newline = 0.9 ######################################################################################################## - np.set_printoptions(precision=4, suppress=True, linewidth=200) tokenizer = TOKENIZER(WORD_NAME, UNKNOWN_CHAR=UNKNOWN_CHAR) context = tokenizer.refine_context(context) -print('Your context has ' + str(len(context)) + ' tokens') +print('\nYour prompt has ' + str(len(context)) + ' tokens.') +print('\n--> Currently the first run takes a while if your prompt is long, as we are using RNN to process the prompt. This will be much faster in future versions. <--\n') + print(f'Loading {MODEL_NAME}...') ############################################################################################################## @@ -65,7 +68,7 @@ class RWKV_RNN(): self.w = types.SimpleNamespace() w = torch.load(MODEL_NAME + '.pth', - map_location=torch.device(RUN_DEVICE)) # .state_dict() + map_location=torch.device(RUN_DEVICE)) for x in w.keys(): if '.time_' in x: w[x] = w[x].squeeze() @@ -195,12 +198,12 @@ class RWKV_RNN(): model = RWKV_RNN(MODEL_NAME) -print('\n\n--> Currently the first run takes a while if your prompt is long, as we are using RNN to process the prompt. This will be much faster in future versions. <--\n') - for TRIAL in range(1 if DEBUG_DEBUG else NUM_TRIALS): + t_begin = time.time_ns() + src_len = len(context) ctx = [tokenizer.stoi.get(s, tokenizer.UNKNOWN_CHAR) for s in context] - print(context.replace('\n', '\n '), end='') + print(('-' * 30) + context, end='') model.clear() if TRIAL == 0: @@ -230,7 +233,7 @@ for TRIAL in range(1 if DEBUG_DEBUG else NUM_TRIALS): char = tokenizer.sample_logits(out, x, ctx_len, temperature=TEMPERATURE, top_p_usual=top_p, top_p_newline=top_p_newline) char = char.item() - print(tokenizer.itos[int(char)].replace( - '\n', '\n '), end='', flush=True) + print(tokenizer.itos[int(char)], end='', flush=True) ctx += [char] - print('\n' + '-' * 40, end='') + t_end = time.time_ns() + print("\n----------", round((t_end - t_begin) / (10 ** 9), 2), end='s ') diff --git a/RWKV-v2-RNN/train.py b/RWKV-v2-RNN/train.py index 64291a8..0644ecd 100644 --- a/RWKV-v2-RNN/train.py +++ b/RWKV-v2-RNN/train.py @@ -137,5 +137,5 @@ if __name__ == '__main__': trainer.train() - torch.save(model, 'trained-' + str(n_epoch) + '-' + trainer.get_run_name() + + torch.save(model.state_dict(), 'trained-' + str(n_epoch) + '-' + trainer.get_run_name() + '-' + datetime.datetime.today().strftime('%Y-%m-%d-%H-%M-%S') + '.pth')