[Sharding] Fix gpt3 sharding hybrid parallel (PaddlePaddle#1648)

MissPenguin · Jan 27, 2022 · c728cab · c728cab
1 parent c720cf6
commit c728cab
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/examples/language_model/gpt-3/dygraph/run_pretrain.py b/examples/language_model/gpt-3/dygraph/run_pretrain.py
@@ -234,7 +234,7 @@ def do_train(args):
         if not any(nd in n for nd in ["bias", "norm"])
     ]
 
-    if args.sharding_stage == 1:
+    if args.sharding_stage == 1 and args.sharding_degree > 1:
         optimizer = DygraphShardingOptimizer(
             hcg=fleet.get_hybrid_communicate_group(),
             user_defined_strategy=strategy,