Remove weight_decay parameter in GPT-2

This parameter appears to be non working or incorrectly tuned.
TingxunShi · Apr 10, 2019 · ac26785 · ac26785
1 parent c422d94
commit ac26785
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/config/models/gpt_2.py b/config/models/gpt_2.py
@@ -36,7 +36,6 @@ def auto_config(self, num_devices=1):
         "params": {
             "average_loss_in_time": True,
             "optimizer": "AdamOptimizer",
-            "weight_decay": 0.01,
             "learning_rate": 2.5e-4,
             "decay_type": "cosine_annealing",
             "decay_params": {