GLM-5 · MoE transformer block

in: $x_n$ [B batch, T positions, 6144 d] $R$ [T positions, 32 pairs] $M$ [T queries, T keys]
out: $x_{n+1}$ [B batch, T positions, 6144 d]
parameters: 9,877,404,672 · × 75 MoE blocks = 740,805,350,400

Parameters

9,877,404,672 parameters per MoE block, across its learned components

Exact per-instance accounting
Component	Calculation	Parameters
Attention RMSNorm	$6{,}144$	6,144
Multi-head latent attention	174,394,112 across its components	174,394,112
MoE RMSNorm	$6{,}144$	6,144
Mixture of experts	9,702,998,272 across its components	9,702,998,272
Per-instance total		9,877,404,672

Across 75 independent MoE blocks: $9{,}877{,}404{,}672 \times 75 = 740{,}805{,}350{,}400$ parameters.

Derived from the pinned configuration and transformers implementation, and equal to the count of the checkpoint’s tensors read from its safetensors headers, less the multi-token-prediction layer the implementation does not load. The total includes the routers’ 19,200 correction-bias values, which the implementation holds as buffers.

In numbers

Weights · one block, by sublayer

MoE block · bfloat1618.4 GiB

Attention · bfloat16333 MiB

MoE · bfloat1618.1 GiB

Two RMSNorms · bfloat1624 KiB

9,877,404,672 parameters per block: attention 174,394,112 (1.8%), the MoE 9,702,998,272 (98.2%), the two RMSNorms 12,288; × 75 MoE blocks = 740,805,350,400.

config.json 24–26 config.json 33 modeling_glm_moe_dsa.py 586–627 config.json 7 modeling_glm_moe_dsa.py 192 modeling_glm_moe_dsa.py 649 modeling_glm_moe_dsa.py 487–492 modeling_glm_moe_dsa.py 647

Codemodeling_glm_moe_dsa.py 586–627

modeling_glm_moe_dsa.py 586–627

class GlmMoeDsaDecoderLayer(GradientCheckpointingLayer):
    def __init__(self, config: GlmMoeDsaConfig, layer_idx: int):
        super().__init__()
        self.hidden_size = config.hidden_size
        self.self_attn = GlmMoeDsaAttention(config, layer_idx)

        self.mlp = GlmMoeDsaMoE(config) if config.mlp_layer_types[layer_idx] == "sparse" else GlmMoeDsaMLP(config)

        self.input_layernorm = GlmMoeDsaRMSNorm(config.hidden_size, config.rms_norm_eps)
        self.post_attention_layernorm = GlmMoeDsaRMSNorm(config.hidden_size, config.rms_norm_eps)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        use_cache: bool | None = False,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
        prev_topk_indices: torch.Tensor | None = None,  # MAIN DIFF with DSV3.2
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple[torch.Tensor, torch.Tensor | None]:
        residual = hidden_states
        hidden_states = self.input_layernorm(hidden_states)
        # Self Attention
        hidden_states, _, topk_indices = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            use_cache=use_cache,
            position_embeddings=position_embeddings,
            prev_topk_indices=prev_topk_indices,  # MAIN DIFF with DSV3.2
            **kwargs,
        )
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.post_attention_layernorm(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states
        return hidden_states, topk_indices

Sources1

Direct evidence

A block applies input RMSNorm, attention and a residual addition, then post-attention RMSNorm, the feed-forward network and a second residual addition; the first first_k_dense_replace blocks use the dense MLP, the rest MoE.

block · huggingface/transformers / modeling_glm_moe_dsa.py
Pinned revision 5e4d6304de55 ↗ · lines 586–627

Read code excerpt

class GlmMoeDsaDecoderLayer(GradientCheckpointingLayer):
    def __init__(self, config: GlmMoeDsaConfig, layer_idx: int):
        super().__init__()
        self.hidden_size = config.hidden_size
        self.self_attn = GlmMoeDsaAttention(config, layer_idx)

        self.mlp = GlmMoeDsaMoE(config) if config.mlp_layer_types[layer_idx] == "sparse" else GlmMoeDsaMLP(config)

        self.input_layernorm = GlmMoeDsaRMSNorm(config.hidden_size, config.rms_norm_eps)
        self.post_attention_layernorm = GlmMoeDsaRMSNorm(config.hidden_size, config.rms_norm_eps)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.LongTensor | None = None,
        past_key_values: Cache | None = None,
        use_cache: bool | None = False,
        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
        prev_topk_indices: torch.Tensor | None = None,  # MAIN DIFF with DSV3.2
        **kwargs: Unpack[TransformersKwargs],
    ) -> tuple[torch.Tensor, torch.Tensor | None]:
        residual = hidden_states
        hidden_states = self.input_layernorm(hidden_states)
        # Self Attention
        hidden_states, _, topk_indices = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            use_cache=use_cache,
            position_embeddings=position_embeddings,
            prev_topk_indices=prev_topk_indices,  # MAIN DIFF with DSV3.2
            **kwargs,
        )
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.post_attention_layernorm(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states
        return hidden_states, topk_indices