human
2.3.2 Uncertainty with reverse dynamics
The paper
 Give me a the Python program you requested?
assistant
#Here is the Python program you requested.
class InverseDynamics(BaseIntrinsicRewardModule):
    def __init__(self, states_size, n_actions, hidden_size, alpha=0.1):
        super().__init__()
        self.module = MLP(
            2 * states_size,
            hidden_size,
            n_actions
        )
        self.alpha = alpha
        self.mean_reward = 0
        self.n_actions = n_actions
    
    def forward(self, s, s_next):
        # <YOUR CODE>
    
    
    def get_intrinsic_reward(self, state, action, next_state):
        with torch.no_grad():
            r = # <YOUR CODE>
            
            r_centered = r - self.mean_reward
            self.mean_reward = self.alpha * (r) + (1 - self.alpha) * self.mean_reward
            return r_centered

    def get_loss(self, state_batch, action_batch, next_state_batch):        
        a_pred_proba = self.forward(state_batch, next_state_batch)
        a_one_hot = to_one_hot(action_batch, self.n_actions)
        return -(a_pred_proba * a_one_hot).sum(dim=-1).mean()

agent = QLearningAgent(
    epsilon=.1, 
    alpha=0.5, 
    discount=.9, 
    get_legal_actions=lambda s: range(env.action_space.n)
)

inverse_dynamics = InverseDynamics(
    np.prod(env.observation_space.shape), 
    env.action_space.n, 
    16
)

train_with_reward(env, agent, inverse_dynamics, n_episodes=3000, 
                  update_reward_period=100, batch_size=100, n_iter=25)

test_agent(agent)