add advantage normalization for ppo gae

Author: quantumiracle

ppo_gae_discrete.py

@@ -75,7 +75,9 @@ def train_net(self):
                 advantage_lst.append([advantage])
             advantage_lst.reverse()
             advantage = torch.tensor(advantage_lst, dtype=torch.float)
-
+            # this can have significant improvement (efficiency, stability) on performance
+            advantage = (advantage - advantage.mean()) / (advantage.std() + 1e-5) 
+            
             pi = self.pi(s, softmax_dim=-1)
             dist_entropy = Categorical(pi).entropy()
             pi_a = pi.gather(1,a)
@@ -126,4 +128,4 @@ def main():
     env.close()
 
 if __name__ == '__main__':
-    main()
+    main()