nano-decoder

nano-vllm/nanovllm/models/qwen3.py · bb823b3 · L120–169 · 固定 commit 上游

这是固定版本源码的只读摘录；可在线阅读，也可通过上游链接查看原文件。

120class Qwen3DecoderLayer(nn.Module):
121
122    def __init__(
123        self,
124        config: Qwen3Config,
125    ) -> None:
126        super().__init__()
127        self.self_attn = Qwen3Attention(
128            hidden_size=config.hidden_size,
129            num_heads=config.num_attention_heads,
130            num_kv_heads=config.num_key_value_heads,
131            max_position=config.max_position_embeddings,
132            rms_norm_eps=config.rms_norm_eps,
133            qkv_bias=getattr(config, 'attention_bias', True),
134            head_dim=getattr(config, 'head_dim', None),
135            rope_theta=getattr(config, "rope_theta", 1000000),
136            rope_scaling=getattr(config, "rope_scaling", None),
137        )
138        self.mlp = Qwen3MLP(
139            hidden_size=config.hidden_size,
140            intermediate_size=config.intermediate_size,
141            hidden_act=config.hidden_act,
142        )
143        self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
144        self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
145
146    def forward(
147        self,
148        positions: torch.Tensor,
149        hidden_states: torch.Tensor,
150        residual: torch.Tensor | None,
151    ) -> tuple[torch.Tensor, torch.Tensor]:
152        if residual is None:
153            hidden_states, residual = self.input_layernorm(hidden_states), hidden_states
154        else:
155            hidden_states, residual = self.input_layernorm(hidden_states, residual)
156        hidden_states = self.self_attn(positions, hidden_states)
157        hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
158        hidden_states = self.mlp(hidden_states)
159        return hidden_states, residual
160
161
162class Qwen3Model(nn.Module):
163
164    def __init__(
165        self,
166        config: Qwen3Config,
167    ) -> None:
168        super().__init__()
169        self.embed_tokens = VocabParallelEmbedding(config.vocab_size, config.hidden_size)