Adding Deepspeed zero1 config (#1675)

Change-Id: Ide19261f0830ba32e6d3ddf1ec662bfc097ff4ba Co-authored-by: Chetan Kumar Verma <[email protected]>
huggingface · Jan 17, 2025 · f10d5b0 · f10d5b0
1 parent aeaa9ca
commit f10d5b0
Showing 1 changed file with 13 additions and 0 deletions.
diff --git a/examples/language-modeling/llama3_ds_zero1_config.json b/examples/language-modeling/llama3_ds_zero1_config.json
@@ -0,0 +1,13 @@
+{
+    "steps_per_print": 64,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "gradient_accumulation_steps": "auto",
+    "bf16": {
+        "enabled": true
+    },
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": 1
+    }
+}