{
 "slug": "Self-attention",
 "title": "Self-attention",
 "type": "concept",
 "short_desc": "Mechanism by which each position in a sequence computes its representation as an attention-weighted sum over all positions; the core operation of the Transformer.",
 "categories": [
  "Architectures",
  "Concepts"
 ],
 "infobox": {
  "Field": "Deep learning",
  "Introduced": "Intra-attention in reading models (2016); popularized by \"Attention Is All You Need\" (2017); antecedents in attention for translation (2014)",
  "Used in": "Transformer models generally",
  "Key papers": "arXiv:1706.03762, arXiv:1409.0473, arXiv:1601.06733"
 },
 "infobox_links": {
  "Introduced": [
   "Transformer_(architecture)"
  ],
  "Used in": [
   "Transformer_(architecture)"
  ]
 },
 "aliases": [
  "Intra-attention"
 ],
 "family": "",
 "org": "",
 "registry_status": "complete",
 "words": 3065,
 "references": 25,
 "outbound": [
  "Attention_sink",
  "BERT",
  "CLIP",
  "Claude_(model_family)",
  "DeepSeek",
  "DeepSeek-V2",
  "DeepSeek-V3",
  "Diffusion_language_model",
  "Feed-forward_network",
  "FlashAttention",
  "GPT-2",
  "GPT-3",
  "GPT_(model_family)",
  "Gemini_1.5",
  "Hybrid_architecture_(LLM)",
  "Kimi_K2",
  "LSTM",
  "Large_language_model",
  "Layer_normalization",
  "Llama_(model_family)",
  "Llama_2",
  "Llama_3",
  "Llama_3.1",
  "MMLU",
  "MiniMax-Text-01",
  "Mistral_7B",
  "Mixtral_8x7B",
  "Mixture_of_experts",
  "Multi-head_attention",
  "Multi-head_latent_attention",
  "OpenAI",
  "PagedAttention",
  "Positional_encoding",
  "Scaling_laws",
  "Sliding-window_attention",
  "Softmax",
  "Sparse_attention",
  "Stable_Diffusion",
  "State-space_model",
  "T5",
  "Transformer_(architecture)",
  "Vision_Transformer",
  "Whisper",
  "gpt-oss"
 ],
 "inbound": [
  "ALBERT",
  "Activation_steering",
  "AlphaFold",
  "Andrej_Karpathy",
  "Attention_sink",
  "BERT",
  "CLIP",
  "Circuits_(interpretability)",
  "Diffusion_language_model",
  "DistilBERT",
  "Embedding_(machine_learning)",
  "Falcon_(models)",
  "Feed-forward_network",
  "FlashAttention",
  "GPT-1",
  "GPT-3",
  "GloVe",
  "Google_DeepMind",
  "Hopfield_network",
  "Hybrid_architecture_(LLM)",
  "Jamba",
  "Kimi_K3",
  "LSTM",
  "Large_language_model",
  "Layer_normalization",
  "Logit_lens",
  "Mechanistic_interpretability",
  "Megatron-LM",
  "MiniMax-M1",
  "Mistral_(model_family)",
  "Mixture_of_experts",
  "Multi-head_attention",
  "Multi-head_latent_attention",
  "Next-token_prediction",
  "PagedAttention",
  "Positional_encoding",
  "RMSNorm",
  "RWKV",
  "Segment_Anything",
  "Seq2seq",
  "Sliding-window_attention",
  "Softmax",
  "Sparse_attention",
  "Sparse_autoencoder",
  "State-space_model",
  "The_Illustrated_Transformer",
  "Transformer_(architecture)",
  "Vision_Transformer",
  "Word2vec",
  "XLNet",
  "Yoshua_Bengio"
 ],
 "url": "wiki/Self-attention.html",
 "built": "2026-07-24"
}
