Network-level configuration for Qwen 3.8 transformer networks. More...

Inheritance diagram for Mila::Dnn::QwenConfig:

Public Member Functions
	QwenConfig (dim_t embedding_dim, dim_t num_layers)
	Construct a Qwen network configuration.
void	fromMetadata (const SerializationMetadata &meta) override
	Populate configuration from provided metadata.
dim_t	getAttentionPackedQKVWidth () const noexcept
	The width the GQA operation validates its build context against (no gate half).
dim_t	getDeltaNetGatingWidth () const noexcept
	DeltaNet gating width – one scalar per value head, for each of a and b.
dim_t	getDeltaNetKeyWidth () const noexcept
	DeltaNet key (and query) stream width = linear_num_key_heads * linear_head_dim.
dim_t	getDeltaNetQueryKeyWidth () const noexcept
	Fused DeltaNet [query\|key] projection width – twice the key width.
dim_t	getDeltaNetValueWidth () const noexcept
	DeltaNet value stream width = linear_num_value_heads * linear_head_dim.
dim_t	getFullAttentionInterval () const noexcept
dim_t	getGatedQProjectionWidth () const noexcept
	Query-projection OUTPUT width, doubled when the output gate is present.
dim_t	getHeadDim () const noexcept
dim_t	getHiddenDimension () const noexcept
dim_t	getKVProjectionWidth () const noexcept
	K and V projection width each = num_kv_heads * head_dim.
dim_t	getLanguageModelHeadPositions () const noexcept
dim_t	getLinearConvKernelDim () const noexcept
dim_t	getLinearHeadDim () const noexcept
dim_t	getLinearNumKeyHeads () const noexcept
dim_t	getLinearNumValueHeads () const noexcept
dim_t	getMaxSequenceLength () const noexcept
dim_t	getModelDim () const noexcept
	Residual-stream dimension.
dim_t	getNumDeltaNetLayers () const noexcept
	Count of Gated DeltaNet layers, i.e. of recurrent-state-holding layers.
dim_t	getNumFullAttentionLayers () const noexcept
	Count of full-attention layers, i.e. of KV-cache-holding layers.
dim_t	getNumHeads () const noexcept
dim_t	getNumKVHeads () const noexcept
dim_t	getNumLayers () const noexcept
dim_t	getPackedQKVWidth () const noexcept
	Fused QKV projection width = gated Q width + 2 * KV width.
float	getPartialRotaryFactor () const noexcept
dim_t	getQProjectionWidth () const noexcept
	Attention output width = num_heads * head_dim. What o_proj reads.
float	getRMSNormEpsilon () const noexcept
float	getRoPETheta () const noexcept
dim_t	getRotaryDim () const noexcept
	Rotated dimension count = partial_rotary_factor * head_dim (27B: 64).
bool	getTieWordEmbeddings () const noexcept
dim_t	getVocabSize () const noexcept
bool	hasAttentionOutputGate () const noexcept
bool	isFullAttentionLayer (dim_t layer_index) const noexcept
	True when layer_index holds full attention (and therefore a KV cache).
SerializationMetadata	toMetadata () const override
	Convert configuration into a SerializationMetadata object.
std::string	toString () const override
	Produce a short, human-readable summary of the configuration.
void	validate () const override
	Validate configuration parameters.
template<typename Self>
decltype(auto)	withAttentionOutputGate (this Self &&self, bool attention_output_gate)
	Whether the query projection is double width and emits [query \| gate].
template<typename Self>
decltype(auto)	withFullAttentionInterval (this Self &&self, dim_t interval)
	Layer period at which a full-attention layer appears (27B: 4).
template<typename Self>
decltype(auto)	withHeadDim (this Self &&self, dim_t head_dim)
	Per-head dimension, decoupled from the residual stream (5120 / 24 != 256).
template<typename Self>
decltype(auto)	withHiddenDimension (this Self &&self, dim_t hidden_dim)
	SwiGLU FFN intermediate width, uniform across both layer kinds.
template<typename Self>
decltype(auto)	withLanguageModelHeadPositions (this Self &&self, dim_t positions)
	Positions the language-model head evaluates per pass.
template<typename Self>
decltype(auto)	withLinearConvKernelDim (this Self &&self, dim_t kernel_dim)
template<typename Self>
decltype(auto)	withLinearHeadDim (this Self &&self, dim_t head_dim)
template<typename Self>
decltype(auto)	withLinearNumKeyHeads (this Self &&self, dim_t num_key_heads)
template<typename Self>
decltype(auto)	withLinearNumValueHeads (this Self &&self, dim_t num_value_heads)
template<typename Self>
decltype(auto)	withMaxSequenceLength (this Self &&self, dim_t max_seq_len)
template<typename Self>
decltype(auto)	withNumHeads (this Self &&self, dim_t num_heads)
template<typename Self>
decltype(auto)	withNumKVHeads (this Self &&self, dim_t num_kv_heads)
template<typename Self>
decltype(auto)	withPartialRotaryFactor (this Self &&self, float factor)
	Fraction of head_dim that rotates (0.25 on the 27B: 64 of 256).
template<typename Self>
decltype(auto)	withRMSNormEpsilon (this Self &&self, float eps)
template<typename Self>
decltype(auto)	withRoPETheta (this Self &&self, float theta)
template<typename Self>
decltype(auto)	withTieWordEmbeddings (this Self &&self, bool tie)
	Whether lm_head shares the token embedding table.
template<typename Self>
decltype(auto)	withVocabularyLength (this Self &&self, dim_t vocab_size)
Public Member Functions inherited from Mila::Dnn::ComponentConfig
virtual	~ComponentConfig ()=default
	Virtual destructor for polymorphic base.

Detailed Description

Network-level configuration for Qwen 3.8 transformer networks.

Fluent setters follow the C++23 explicit-object-parameter pattern used throughout the codebase. Defaults are the published 27B values, so a default-constructed config is the real model rather than a placeholder.

Constructor & Destructor Documentation

◆ QwenConfig()

Mila::Dnn::QwenConfig::QwenConfig	(	dim_t	embedding_dim,
		dim_t	num_layers )

inline

Construct a Qwen network configuration.

Parameters

embedding_dim	Residual-stream dimension (HuggingFace hidden_size). Must be > 0.
num_layers	Total transformer layers, full-attention and DeltaNet. Must be > 0.

Member Function Documentation

◆ fromMetadata()

void Mila::Dnn::QwenConfig::fromMetadata ( const SerializationMetadata & meta )

inlineoverridevirtual

Populate configuration from provided metadata.

Implementations should read available keys and leave missing keys at their current/default values to preserve forward/backward compatibility.

Parameters

meta	Metadata to read configuration values from.

Implements Mila::Dnn::ComponentConfig.

◆ getDeltaNetValueWidth()

dim_t Mila::Dnn::QwenConfig::getDeltaNetValueWidth ( ) const

inlinenoexcept

DeltaNet value stream width = linear_num_value_heads * linear_head_dim.

Also the width of z, of the mixer output, and of the gated result.

◆ getGatedQProjectionWidth()

dim_t Mila::Dnn::QwenConfig::getGatedQProjectionWidth ( ) const

inlinenoexcept

Query-projection OUTPUT width, doubled when the output gate is present.

The distinction from getQProjectionWidth() is the whole of attn_output_gate: the projection emits this many columns, the attention consumes half of them, and the other half is the gate.

◆ getModelDim()

dim_t Mila::Dnn::QwenConfig::getModelDim ( ) const

inlinenoexcept

Residual-stream dimension.

NOT num_heads * head_dim (5120 vs 6144).

◆ getPackedQKVWidth()

dim_t Mila::Dnn::QwenConfig::getPackedQKVWidth ( ) const

inlinenoexcept

Fused QKV projection width = gated Q width + 2 * KV width.

The block splits this into [query|gate], key, value. The GQA operation's own packed convention is the ungated (num_heads + 2 * num_kv_heads) * head_dim, which is what getAttentionPackedQKVWidth() reports – the two differ exactly by the gate half, and confusing them is how the gate goes missing.

◆ getRotaryDim()

dim_t Mila::Dnn::QwenConfig::getRotaryDim ( ) const

inlinenoexcept

Rotated dimension count = partial_rotary_factor * head_dim (27B: 64).

Truncated to an even width because RoPE pairs dimensions.

◆ isFullAttentionLayer()

bool Mila::Dnn::QwenConfig::isFullAttentionLayer ( dim_t layer_index ) const

inlinenoexcept

True when layer_index holds full attention (and therefore a KV cache).

Period N makes every Nth layer full attention; the 27B's 4 places them at layers 3, 7, ... 63, so the final layer is full attention and 16 of 64 hold a cache – the property Qwen3.8.md section 3 turns into a 12 GiB budget.

◆ toMetadata()

SerializationMetadata Mila::Dnn::QwenConfig::toMetadata ( ) const

inlineoverridevirtual

Convert configuration into a SerializationMetadata object.

Implementations should include any fields required to fully reconstruct the configuration via fromMetadata.

Returns: SerializationMetadata Metadata representation of the config.

Implements Mila::Dnn::ComponentConfig.

◆ toString()

std::string Mila::Dnn::QwenConfig::toString ( ) const

inlineoverridevirtual

Produce a short, human-readable summary of the configuration.

Implementations should return a compact, single-line description suitable for logging and debugging.

Returns: std::string Human-readable summary of the configuration.

Implements Mila::Dnn::ComponentConfig.

◆ validate()

void Mila::Dnn::QwenConfig::validate ( ) const

inlineoverridevirtual

Validate configuration parameters.

Called by callers to ensure the configuration represents a valid, constructible component. Implementations must throw std::invalid_argument (or a derived exception) when validation fails.

Exceptions

std::invalid_argument If the configuration is invalid.

Implements Mila::Dnn::ComponentConfig.

◆ withAttentionOutputGate()

template<typename Self>

decltype(auto) Mila::Dnn::QwenConfig::withAttentionOutputGate	(	this Self &&	self,
		bool	attention_output_gate )

inline

Whether the query projection is double width and emits [query | gate].

The published 27B sets this true. False is the plain-GQA geometry and exists so a test can pin the packed width against the ungated case.

◆ withFullAttentionInterval()

template<typename Self>

decltype(auto) Mila::Dnn::QwenConfig::withFullAttentionInterval	(	this Self &&	self,
		dim_t	interval )

inline

Layer period at which a full-attention layer appears (27B: 4).

Period N gives (N-1) DeltaNet layers per full-attention layer; the 27B's 4 yields the 3:1 interleave and 16 cache-holding layers of 64.

◆ withLanguageModelHeadPositions()

template<typename Self>

decltype(auto) Mila::Dnn::QwenConfig::withLanguageModelHeadPositions	(	this Self &&	self,
		dim_t	positions )

inline

Positions the language-model head evaluates per pass.

Default 1.

Generation needs a logit only at the last position, and at 248,320 vocabulary a BF16 logit row costs 0.474 MiB, so the head is built one row wide and the default pays for exactly that row. Teacher-forced scoring needs a logit at EVERY position, which cannot be materialized for a whole prefill chunk at once, so a scoring caller raises this to the number of rows it can afford and the head is evaluated in sub-chunks of that width.

This is a run capacity, not published geometry – it sizes buffers the way withMaxSequenceLength does, and unlike that bound it describes no property of the checkpoint, so it is deliberately absent from toMetadata().

◆ withPartialRotaryFactor()

template<typename Self>

decltype(auto) Mila::Dnn::QwenConfig::withPartialRotaryFactor	(	this Self &&	self,
		float	factor )

inline

Fraction of head_dim that rotates (0.25 on the 27B: 64 of 256).

Stored as the published fraction rather than the resolved width so the config reads as the checkpoint does; getRotaryDim() resolves it.

◆ withTieWordEmbeddings()

template<typename Self>

decltype(auto) Mila::Dnn::QwenConfig::withTieWordEmbeddings	(	this Self &&	self,
		bool	tie )

inline

Whether lm_head shares the token embedding table.

The 27B is UNTIED.

The documentation for this class was generated from the following file:

Mila/Src/Dnn/Components/Transformers/Qwen/Qwen.Config.ixx

Public Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ QwenConfig()

Member Function Documentation

◆ fromMetadata()

◆ getDeltaNetValueWidth()

◆ getGatedQProjectionWidth()

◆ getModelDim()

◆ getPackedQKVWidth()

◆ getRotaryDim()

◆ isFullAttentionLayer()

◆ toMetadata()

◆ toString()

◆ validate()

◆ withAttentionOutputGate()

◆ withFullAttentionInterval()

◆ withLanguageModelHeadPositions()

◆ withPartialRotaryFactor()

◆ withTieWordEmbeddings()