Qwen 3.8 compatible inference model. More...

Inheritance diagram for Mila::Dnn::QwenModel< TDeviceType, TPrecision >:

Public Types
using	ModelBase = LanguageModel<TDeviceType, TPrecision>
using	MR = typename DeviceTypeTraits<TDeviceType>::memory_resource
using	QwenKvPolicy = NoKvCompression
	KV policy for the full-attention layers.
using	TensorType = Tensor<TPrecision, MR>
using	TokenIndexType = Tensor<dtype_t::INT32, MR>
Public Types inherited from Mila::Dnn::LanguageModel< TDeviceType, TPrecision >
using	Base = Model<TDeviceType, TPrecision>
Public Types inherited from Mila::Dnn::Model< TDeviceType, TPrecision >
using	NetworkType = Network<TDeviceType, TPrecision>

Public Member Functions
	QwenModel (const QwenModel &)=delete
	QwenModel (QwenModel &&)=default
dim_t	contextLength () const noexcept
	Deployment context length: the KV-cache depth the network was built with.
const QwenModelConfig &	getModelConfig () const noexcept
	Deployment configuration this model was loaded with.
const QwenConfig &	getNetworkConfig () const noexcept
	Architecture/network configuration (read from the checkpoint metadata).
dim_t	maxSequenceLength () const noexcept override
QwenModel &	operator= (const QwenModel &)=delete
QwenModel &	operator= (QwenModel &&)=default
SequenceLogLikelihood	scoreTokens (const std::vector< int32_t > &tokens)
	Teacher-forced log-likelihood of a token sequence – the perplexity path.
bool	supportsPromptPrefixReuse () const noexcept
	False, always: this stack cannot reuse a prompt prefix.
std::string	toString () const override
	Human-readable summary of this model's configuration.
dim_t	vocabSize () const noexcept override
Public Member Functions inherited from Mila::Dnn::LanguageModel< TDeviceType, TPrecision >
	LanguageModel (const LanguageModel &)=delete
	LanguageModel (LanguageModel &&)=default
std::vector< std::string >	componentPaths () const
	Every observable component path, for choosing a pattern or diagnosing a zero.
GenerateStatus	generate (std::span< const int32_t > prompt_tokens, const std::function< void(int32_t)> &on_token, const GenerateParams &params={}, std::stop_token stop={})
	Generate tokens from a prompt, streaming each through on_token.
size_t	observe (std::string_view pattern, Compute::ComputePassMask passes, Compute::ActivationObserver sink)
	Watch activations flowing through matching components on matching passes.
LanguageModel &	operator= (const LanguageModel &)=delete
LanguageModel &	operator= (LanguageModel &&)=default
void	save (const std::filesystem::path &path) const
	Write this model's live weights as a safetensors weights file.
void	seedSampler (uint64_t seed)
	Seed the sampler's RNG for reproducible generation.
void	stopObserving ()
	Detach every observer and clear the sink. A probe that outlives its question costs.
std::string	weightQuantizationScheme () const
	The scheme name save records for these weights, e.g.
Public Member Functions inherited from Mila::Dnn::Model< TDeviceType, TPrecision >
	Model (const Model &)=delete
	Model (Model &&)=default
DeviceId	getDeviceId () const noexcept
	The device this model runs on.
MemoryStats	getMemoryStats () const
	Current memory allocation breakdown for this model.
RuntimeMode	getRuntimeMode () const noexcept
	The runtime mode this model was constructed for.
bool	isEval () const noexcept
	True if this model is currently in eval sub-state.
bool	isInferenceMode () const noexcept
	True if this model was constructed for inference.
bool	isTrainingMode () const noexcept
	True if this model was constructed for training.
Model &	operator= (const Model &)=delete
Model &	operator= (Model &&)=default
void	setEval (bool eval)
	Toggle eval sub-state for this model.
void	train ()
	Run the training loop for this model.

Static Public Member Functions
static QwenConfig	configFromMetadata (const WeightsMetadata &metadata)
	The network geometry an artifact's metadata declares.
static DeploymentFootprint	getDeploymentFootprint (const std::filesystem::path &path, const QwenModelConfig &model_config, DeviceId device_id=DeviceId{ TDeviceType, 0 })
	The same prediction, plus how this deployment would chunk its prefill.
static MemoryStats	getRequiredMemory (const std::filesystem::path &path, const QwenModelConfig &model_config, DeviceId device_id=DeviceId{ TDeviceType, 0 })
	What loading this checkpoint at this context length would cost in VRAM.
static std::unique_ptr< QwenModel< TDeviceType, TPrecision > >	load (const std::filesystem::path &path, const QwenModelConfig &model_config, DeviceId device_id=DeviceId{ TDeviceType, 0 })
	Load Qwen 3.8 from a Mila weights file.

Protected Member Functions
int32_t	eosToken () const noexcept override
GenerateStatus	onGenerating (std::span< const int32_t > prompt_tokens, const std::function< void(int32_t)> &on_token, const GenerateParams &params, std::stop_token stop) override
	Prefill + decode implementation hook.
void	onTraining () override
	Training loop hook – derived class owns the implementation.
std::unordered_set< int32_t >	stopTokens () const override
Protected Member Functions inherited from Mila::Dnn::LanguageModel< TDeviceType, TPrecision >
	LanguageModel (std::unique_ptr< LanguageModelNetwork< TDeviceType, TPrecision > > network, RuntimeMode runtime_mode, Serialization::WeightsMetadata source_metadata={}, WeightQuantization weight_quantization=WeightQuantization::None, int fp4_group_size=128)
int32_t	awaitSampledToken ()
	Block until the last enqueueSampleNext()'s token id is host-visible.
void	enqueueSampleNext (const TensorType &logits, TokenTensor &token_out, const SamplingParams &params)
	Enqueue a sampling step without waiting for the host readback.
virtual float	finalLogitSoftcap () const noexcept
	Optional final-logit softcap the sampler applies (0 disables).
const LanguageModelNetwork< TDeviceType, TPrecision > &	getNetwork () const noexcept
LanguageModelNetwork< TDeviceType, TPrecision > &	getNetwork () noexcept
int32_t	sampleNext (const TensorType &logits, TokenTensor &token_out, const SamplingParams &params)
	Sample the next token from a logits row on the device.
Protected Member Functions inherited from Mila::Dnn::Model< TDeviceType, TPrecision >
	Model (std::unique_ptr< NetworkType > network, RuntimeMode runtime_mode)
	Construct with a fully built network and runtime mode.

Additional Inherited Members
Protected Types inherited from Mila::Dnn::LanguageModel< TDeviceType, TPrecision >
using	MR = typename DeviceTypeTraits<TDeviceType>::memory_resource
using	TensorType = Tensor<TPrecision, MR>
using	TokenTensor = Tensor<TensorDataType::INT32, MR>
Protected Attributes inherited from Mila::Dnn::LanguageModel< TDeviceType, TPrecision >
int	fp4_group_size_ { 128 }
	The FP4 group the live weights were built at; meaningful only when they are FP4.
Serialization::WeightsMetadata	source_metadata_
	The loaded weights file's metadata, carried so save can write it back verbatim.
WeightQuantization	weight_quantization_ { WeightQuantization::None }
	What the live weights are, not what the source file was.
Protected Attributes inherited from Mila::Dnn::Model< TDeviceType, TPrecision >
std::unique_ptr< NetworkType >	network_
	The owned Network instance.

Detailed Description

template<DeviceType TDeviceType, TensorDataType TPrecision>
requires PrecisionSupportedOnDevice<TPrecision, TDeviceType>
class Mila::Dnn::QwenModel< TDeviceType, TPrecision >

Qwen 3.8 compatible inference model.

Owns a loaded, built QwenTransformer and drives the prefill + decode loop. Construction is only possible via load().

Thread safety: not thread-safe; external synchronization required if shared.

Member Typedef Documentation

◆ QwenKvPolicy

template<DeviceType TDeviceType, TensorDataType TPrecision>

using Mila::Dnn::QwenModel< TDeviceType, TPrecision >::QwenKvPolicy = NoKvCompression

KV policy for the full-attention layers.

Qwen's cache-holding layers attend the FULL context – there is no sliding window anywhere in this stack – so a bounded ring has nothing to bound and the policy is NoKvCompression. The 48 DeltaNet layers hold no cache at all, which is the property that makes a 27B model plausible on 12 GiB (Qwen3.8.md section 3).

Member Function Documentation

◆ configFromMetadata()

template<DeviceType TDeviceType, TensorDataType TPrecision>

QwenConfig Mila::Dnn::QwenModel< TDeviceType, TPrecision >::configFromMetadata ( const WeightsMetadata & metadata )

inlinestatic

The network geometry an artifact's metadata declares.

Public because loading the model is not the only way to drive it. The Phase 4 parity harness streams the 50 GiB artifact one layer at a time and never constructs a QwenModel, but it must build its blocks from the SAME geometry a real load would – a second reading of these twenty fields would agree until one of them changed, and then measure a different model than the one it is checking.

◆ eosToken()

template<DeviceType TDeviceType, TensorDataType TPrecision>

int32_t Mila::Dnn::QwenModel< TDeviceType, TPrecision >::eosToken ( ) const

inlineoverrideprotectedvirtualnoexcept

Implements Mila::Dnn::LanguageModel< TDeviceType, TPrecision >.

◆ getDeploymentFootprint()

template<DeviceType TDeviceType, TensorDataType TPrecision>

DeploymentFootprint Mila::Dnn::QwenModel< TDeviceType, TPrecision >::getDeploymentFootprint	(	const std::filesystem::path &	path,
		const QwenModelConfig &	model_config,
		DeviceId	device_id = DeviceId{ TDeviceType, 0 } )

inlinestatic

The same prediction, plus how this deployment would chunk its prefill.

Everything before the build is shared with load deliberately: the artifact check, the geometry and the context-length validation must be the ones a real load would apply, or the reported figure describes a model that would not load.

Exceptions

std::invalid_argument	on device type mismatch or zero context length.
std::runtime_error	on an unreadable artifact or an unsupported mode.

◆ getRequiredMemory()

template<DeviceType TDeviceType, TensorDataType TPrecision>

MemoryStats Mila::Dnn::QwenModel< TDeviceType, TPrecision >::getRequiredMemory	(	const std::filesystem::path &	path,
		const QwenModelConfig &	model_config,
		DeviceId	device_id = DeviceId{ TDeviceType, 0 } )

inlinestatic

What loading this checkpoint at this context length would cost in VRAM.

Reads the artifact header for geometry and constructs the graph, then reports what build() would allocate – without building it, without reading a weight, and therefore without needing the device to have room.

Exceptions

std::invalid_argument	on device type mismatch or zero context length.
std::runtime_error	on an unreadable artifact or an unsupported mode.

◆ load()

template<DeviceType TDeviceType, TensorDataType TPrecision>

std::unique_ptr< QwenModel< TDeviceType, TPrecision > > Mila::Dnn::QwenModel< TDeviceType, TPrecision >::load	(	const std::filesystem::path &	path,
		const QwenModelConfig &	model_config,
		DeviceId	device_id = DeviceId{ TDeviceType, 0 } )

inlinestatic

Load Qwen 3.8 from a Mila weights file.

The model_config carries the deployment decisions (context length, weight quantization, KV-cache compression); every architectural parameter is read from the checkpoint metadata.

Parameters

path	Path to the Qwen weights file.
model_config	Deployment configuration for this load.
device_id	Target device; must match TDeviceType.

Returns: Inference-ready QwenModel.

Exceptions

std::invalid_argument	on device type mismatch or zero context length.
std::runtime_error	on load failure or a quantization mode this chassis does not yet carry an artifact for.

◆ maxSequenceLength()

template<DeviceType TDeviceType, TensorDataType TPrecision>

dim_t Mila::Dnn::QwenModel< TDeviceType, TPrecision >::maxSequenceLength ( ) const

inlineoverridevirtualnoexcept

Implements Mila::Dnn::LanguageModel< TDeviceType, TPrecision >.

◆ onGenerating()

template<DeviceType TDeviceType, TensorDataType TPrecision>

GenerateStatus Mila::Dnn::QwenModel< TDeviceType, TPrecision >::onGenerating	(	std::span< const int32_t >	prompt_tokens,
		const std::function< void(int32_t)> &	on_token,
		const GenerateParams &	params,
		std::stop_token	stop )

inlineoverrideprotectedvirtual

Prefill + decode implementation hook.

Derived classes own the full autoregressive generation loop. on_token must be called for every generated token except EOS. stop.stop_requested() must be checked on each decode step and generation must abort early when signalled, returning the GenerateStatus that reflects why the loop stopped.

Parameters

prompt_tokens	Input token ids.
on_token	Per-token callback.
params	Per-call generation parameters (loop bound + sampling).
stop	Stop token for cooperative cancellation.

Returns: Why generation stopped.

Implements Mila::Dnn::LanguageModel< TDeviceType, TPrecision >.

◆ onTraining()

template<DeviceType TDeviceType, TensorDataType TPrecision>

void Mila::Dnn::QwenModel< TDeviceType, TPrecision >::onTraining ( )

inlineoverrideprotectedvirtual

Training loop hook – derived class owns the implementation.

Called by train() after precondition enforcement. The derived class has total control over data loading, optimizer construction, loss computation, backward pass, checkpointing, and sampling.

Pure virtual – a model declaring RuntimeMode::Training must provide a training loop.

Implements Mila::Dnn::Model< TDeviceType, TPrecision >.

◆ scoreTokens()

template<DeviceType TDeviceType, TensorDataType TPrecision>

SequenceLogLikelihood Mila::Dnn::QwenModel< TDeviceType, TPrecision >::scoreTokens ( const std::vector< int32_t > & tokens )

inline

Teacher-forced log-likelihood of a token sequence – the perplexity path.

Reports the probability the model assigned to each token given the ones before it. Nothing is sampled, so the result depends only on the model and the text, which is what makes it comparable between two quantizations of the same weights.

Perplexity over a corpus is exp( -total_log_probability / scored_positions ) after summing several of these. Segment the corpus and sum, rather than averaging each segment, or a short segment weighs as much as a long one.

Needs a deployment built with withLanguageModelHeadPositions() above 1 to run at a sensible rate; at the default of 1 it works and costs one head pass per token.

Parameters

tokens A single sequence, at least 2 tokens, no longer than contextLength().

◆ stopTokens()

template<DeviceType TDeviceType, TensorDataType TPrecision>

std::unordered_set< int32_t > Mila::Dnn::QwenModel< TDeviceType, TPrecision >::stopTokens ( ) const

inlineoverrideprotectedvirtual

Reimplemented from Mila::Dnn::LanguageModel< TDeviceType, TPrecision >.

◆ supportsPromptPrefixReuse()

template<DeviceType TDeviceType, TensorDataType TPrecision>

bool Mila::Dnn::QwenModel< TDeviceType, TPrecision >::supportsPromptPrefixReuse ( ) const

inlinenoexcept

False, always: this stack cannot reuse a prompt prefix.

Stated as a model property rather than left for a caller to discover through a refused rewind, because the reason is architectural and permanent – 48 of the 64 layers hold a recurrent state, and a lossy summary cannot be rolled back.

◆ toString()

template<DeviceType TDeviceType, TensorDataType TPrecision>

std::string Mila::Dnn::QwenModel< TDeviceType, TPrecision >::toString ( ) const

inlineoverridevirtual

Human-readable summary of this model's configuration.

Implements Mila::Dnn::Model< TDeviceType, TPrecision >.

◆ vocabSize()

template<DeviceType TDeviceType, TensorDataType TPrecision>

dim_t Mila::Dnn::QwenModel< TDeviceType, TPrecision >::vocabSize ( ) const

inlineoverridevirtualnoexcept

Implements Mila::Dnn::LanguageModel< TDeviceType, TPrecision >.

The documentation for this class was generated from the following file:

Mila/Src/Dnn/Models/Qwen/QwenModel.ixx

Public Types

Public Member Functions

Static Public Member Functions

Protected Member Functions

Additional Inherited Members

Detailed Description

Member Typedef Documentation

◆ QwenKvPolicy

Member Function Documentation

◆ configFromMetadata()

◆ eosToken()

◆ getDeploymentFootprint()

◆ getRequiredMemory()

◆ load()

◆ maxSequenceLength()

◆ onGenerating()

◆ onTraining()

◆ scoreTokens()

◆ stopTokens()

◆ supportsPromptPrefixReuse()

◆ toString()

◆ vocabSize()