BpeVocabulary.ixx File Reference

BPE vocabulary for GPT-2, Llama 3.x, and Mistral model families. More...

#include <string>
#include <string_view>
#include <vector>
#include <unordered_map>
#include <unordered_set>
#include <optional>
#include <filesystem>
#include <fstream>
#include <sstream>
#include <algorithm>
#include <stdexcept>
#include <cstdint>
#include <chrono>
#include <iostream>
#include <iomanip>
#include <format>
import Logging.Logger;
import Serialization.Metadata;
import Data.BpePreTokenizationMode;
import Data.FileHeader;
import Data.BpeVocabularyConfig;
import Data.TokenizerVocabulary;
import Data.Tokenizer;
import Data.SpecialTokens;

Classes
class	Mila::Data::BpeVocabulary
	Unified Byte Pair Encoding (BPE) vocabulary. More...

Namespaces
namespace	Mila
	Mila main API namespace.

Typedefs
using	Mila::Data::TokenId

Detailed Description

BPE vocabulary for GPT-2, Llama 3.x, and Mistral model families.