{
 "slug": "Distributed_training",
 "title": "Distributed training",
 "type": "concept",
 "short_desc": "Techniques for training models across thousands of accelerators: data, tensor, pipeline, and expert parallelism, plus sharding schemes like ZeRO/FSDP.",
 "categories": [
  "Training methods",
  "Hardware and compute"
 ],
 "infobox": {
  "Field": "Machine learning systems",
  "Introduced": "Modern LLM-scale stack assembled 2019-2021 (Megatron-LM, ZeRO, GShard)",
  "Used in": "Every frontier-scale training run",
  "Key papers": "arXiv:1909.08053, arXiv:1910.02054"
 },
 "infobox_links": {
  "Introduced": [
   "Megatron-LM",
   "Mixture_of_experts"
  ]
 },
 "aliases": [
  "Data parallelism",
  "Tensor parallelism",
  "Pipeline parallelism",
  "Expert parallelism",
  "ZeRO",
  "FSDP"
 ],
 "family": "",
 "org": "",
 "registry_status": "stub",
 "words": 313,
 "references": 3,
 "outbound": [
  "Anthropic",
  "DeepSeek",
  "GPT-3",
  "GPT-4",
  "Google_DeepMind",
  "Large_language_model",
  "Megatron-LM",
  "Meta_AI",
  "Mixture_of_experts",
  "NVIDIA",
  "OpenAI",
  "Transformer_(architecture)"
 ],
 "inbound": [
  "Adam_(optimizer)",
  "Alex_Krizhevsky",
  "Andrej_Karpathy",
  "Arcee_AI",
  "BLOOM",
  "Backpropagation",
  "Cerebras-GPT",
  "Cerebras_Systems",
  "DeepSeek-V3",
  "FlashAttention",
  "GPT-Neo",
  "GPT-NeoX-20B",
  "Gemini_2.5",
  "Grok-1.5",
  "Huawei",
  "LSTM",
  "Llama_3",
  "Llama_3.1",
  "Megatron-LM",
  "Megatron-Turing_NLG",
  "Mixture_of_experts",
  "NVIDIA",
  "Nous_Research",
  "PaLM",
  "Pretraining",
  "Thinking_Machines_Lab",
  "Transformer_(architecture)",
  "Trinity_(model_family)",
  "Trinity_Large",
  "Trinity_Large_Thinking",
  "YaLM-100B"
 ],
 "url": "wiki/Distributed_training.html",
 "built": "2026-07-24"
}
