Mila::Data::Tokenizer Class Reference

Inheritance diagram for Mila::Data::Tokenizer:

Public Member Functions
virtual std::string	decode (std::span< const TokenId > tokens)=0
virtual std::vector< TokenId >	encode (const std::string &text)=0
virtual std::optional< TokenId >	getBosTokenId () const =0
virtual std::optional< TokenId >	getEosTokenId () const =0
virtual std::optional< TokenId >	getPadTokenId () const =0
virtual size_t	getVocabSize () const =0
virtual bool	isValidToken (TokenId tokenId) const =0
virtual std::string	tokenToString (TokenId tokenId) const =0

Member Function Documentation

decode()

virtual std::string Mila::Data::Tokenizer::decode ( std::span< const TokenId > tokens )

pure virtual

Implemented in Mila::Data::BpeTokenizer, and Mila::Data::CharTokenizer.

encode()

virtual std::vector< TokenId > Mila::Data::Tokenizer::encode ( const std::string & text )

pure virtual

Implemented in Mila::Data::BpeTokenizer, and Mila::Data::CharTokenizer.

getBosTokenId()

virtual std::optional< TokenId > Mila::Data::Tokenizer::getBosTokenId ( ) const

pure virtual

Implemented in Mila::Data::CharTokenizer.

getEosTokenId()

virtual std::optional< TokenId > Mila::Data::Tokenizer::getEosTokenId ( ) const

pure virtual

Implemented in Mila::Data::CharTokenizer.

getPadTokenId()

virtual std::optional< TokenId > Mila::Data::Tokenizer::getPadTokenId ( ) const

pure virtual

Implemented in Mila::Data::CharTokenizer.

getVocabSize()

virtual size_t Mila::Data::Tokenizer::getVocabSize ( ) const

pure virtual

Implemented in Mila::Data::CharTokenizer.

isValidToken()

virtual bool Mila::Data::Tokenizer::isValidToken ( TokenId tokenId ) const

pure virtual

Implemented in Mila::Data::CharTokenizer.

tokenToString()

virtual std::string Mila::Data::Tokenizer::tokenToString ( TokenId tokenId ) const

pure virtual

Implemented in Mila::Data::CharTokenizer.

---

The documentation for this class was generated from the following file:

• Mila/Src/Data/Tokenizers/Tokenizer.ixx