novarobot
/
RWKV-LM
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update model.py

Browse Source
main
PENG Bo 4 years ago committed by GitHub
parent c8a751ed8b
commit b4fd1a7209
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File

2
src/model.py
Unescape Escape View File

@ -69,7 +69,7 @@ class RWKV_TimeMix(nn.Module):
if h < config.n_head - 1: if h < config.n_head - 1:
decay_speed = math.pow(config.ctx_len, -(h+1)/(config.n_head-1)) decay_speed = math.pow(config.ctx_len, -(h+1)/(config.n_head-1))
else: else:
decay_speed = 0 decay_speed = 0.0
ww[h] = torch.exp(curve * decay_speed) ww[h] = torch.exp(curve * decay_speed)
# print('layer', layer_id, 'head', h, 'decay_speed', round(decay_speed, 4), ww[h][:5].numpy(), '...', ww[h][-5:].numpy()) # print('layer', layer_id, 'head', h, 'decay_speed', round(decay_speed, 4), ww[h][:5].numpy(), '...', ww[h][-5:].numpy())
self.time_w = nn.Parameter(ww) self.time_w = nn.Parameter(ww)

Write Preview
Loading…
Cancel
Save