human
I am working on the file "FADL1/lesson6-rnn_codealong.ipynb".
The first task is:
The input and hidden states represent qualitatively different types of information, so adding them together can potentially lose information. Instead we can concatenate them together.
Can you write Python program for it?
 ... What is a the Python program you requested?
assistant
#Here is the Python program you requested.

class CharLoopConcatModel(nn.Module):
    def __init__(self, vocab_size, n_fac):
        super().__init__()
        self.e = nn.Embedding(vocab_size, n_fac)
        self.l_in = nn.Linear(n_fac + n_hidden, n_hidden)
        self.l_hidden = nn.Linear(n_hidden, n_hidden)
        self.l_out = nn.Linear(n_hidden, vocab_size)
        
    def forward(self, *cs):
        bs = cs[0].size(0)
#         h  = torch.zeros(bs, n_hidden).cuda()
        h = V(torch.zeros(bs, n_hidden).cuda())
        for c in cs:
            inp = torch.cat((h, self.e(c)), 1)
            inp = F.relu(self.l_in(inp))
#             h   = torch.tanh(self.l_hidden(inp))
            h   = F.tanh(self.l_hidden(inp))
            
        return F.log_softmax(self.l_out(inp), dim=-1)

model = CharLoopConcatModel(vocab_size, n_fac).cuda()
optimizer = optim.Adam(model.parameters(), 1e-3)

it = iter(mdata.trn_dl)
*xs,yt = next(it)
# t = model(*xs)
t = model(*V(xs))

xs[0].size(0)

t

fit(model, mdata, 1, optimizer, F.nll_loss)

set_lrs(optimizer, 1e-4)
fit(model, mdata, 1, optimizer, F.nll_loss)