bytes

Functions to get the byte vocabulary from a HuggingFace tokenizer

get_byte_vocab(tokenizer)

Extract byte vocabulary from a tokenizer.

The byte encoding is chosen by the tokenizer's decoder: ByteLevel uses the GPT-2 byte alphabet, ByteFallback or Metaspace uses SentencePiece pieces. Any other declared decoder is rejected.

Parameters:

Name	Type	Description	Default
tokenizer		A Hugging Face tokenizer instance.	required

Returns:

Type	Description
list[bytes]	List of byte representations of tokens.

Raises:

Type	Description
ByteVocabError	If vocabulary cannot be decoded.

Source code in genlm/backend/tokenization/bytes.py

def get_byte_vocab(tokenizer):
    """Extract byte vocabulary from a tokenizer.

    The byte encoding is chosen by the tokenizer's decoder: ``ByteLevel`` uses the GPT-2
    byte alphabet, ``ByteFallback`` or ``Metaspace`` uses SentencePiece pieces. Any other
    declared decoder is rejected.

    Args:
        tokenizer: A Hugging Face tokenizer instance.

    Returns:
        (list[bytes]): List of byte representations of tokens.

    Raises:
        ByteVocabError: If vocabulary cannot be decoded.
    """
    kinds = _decoder_kinds(tokenizer)

    if kinds & {"ByteFallback", "Metaspace"}:
        return get_byte_tokens_from_sp(tokenizer)

    if "ByteLevel" in kinds:
        byte_decoder = _get_default_byte_decoder()
        try:
            check_byte_decoder(tokenizer, byte_decoder)
        except ByteDecoderError as e:
            raise ByteVocabError(
                "Tokenizer declares a ByteLevel decoder but does not round-trip under "
                "the GPT-2 byte alphabet."
            ) from e
        return get_byte_tokens_from_byte_decoder(tokenizer, byte_decoder)

    if kinds:
        raise ByteVocabError(
            f"Cannot determine the byte encoding of {tokenizer.name_or_path!r}: its "
            f"decoder is {sorted(kinds)}, which is neither ByteLevel nor SentencePiece."
        )

    # No backend decoder to inspect; try SentencePiece model.
    if hasattr(tokenizer, "sp_model"):
        return get_byte_tokens_from_sp(tokenizer)

    byte_decoder = (
        getattr(tokenizer, "byte_decoder", None) or _get_default_byte_decoder()
    )
    try:
        check_byte_decoder(tokenizer, byte_decoder)
    except ByteDecoderError as e:
        raise ByteVocabError(
            f"Tokenizer {tokenizer.name_or_path!r} exposes no decoder to inspect and "
            "does not round-trip under the GPT-2 byte alphabet."
        ) from e
    return get_byte_tokens_from_byte_decoder(tokenizer, byte_decoder)

get_byte_tokens_from_byte_decoder(tokenizer, byte_decoder)

Convert tokens to bytes using a byte decoder mapping.

Special tokens are handled by directly encoding their string representation.

Parameters:

Name	Type	Description	Default
tokenizer		A Hugging Face tokenizer instance	required
byte_decoder	dict	Dictionary mapping characters to bytes	required

Returns:

Name	Type	Description
byte_tokens	list[byte]	List of byte representations for each token

Source code in genlm/backend/tokenization/bytes.py

def get_byte_tokens_from_byte_decoder(tokenizer, byte_decoder):
    """Convert tokens to bytes using a byte decoder mapping.

    Special tokens are handled by directly encoding their string representation.

    Args:
        tokenizer: A Hugging Face tokenizer instance
        byte_decoder (dict): Dictionary mapping characters to bytes

    Returns:
        byte_tokens (list[byte]): List of byte representations for each token
    """
    special_tokens_map = {v: k for k, v in tokenizer.get_added_vocab().items()}
    byte_tokens = [
        bytes([byte_decoder[b] for b in tokenizer.convert_ids_to_tokens(i)])
        if i not in special_tokens_map
        else special_tokens_map[i].encode()
        for i in range(len(tokenizer))
    ]
    return byte_tokens

get_byte_tokens_from_sp(tokenizer)

Convert SentencePiece token strings to bytes.

<0xXX> escapes become the byte they name and ▁ becomes a space; special tokens are encoded directly.

Parameters:

Name	Type	Description	Default
tokenizer		A Hugging Face tokenizer instance	required

Returns:

Name	Type	Description
byte_tokens	list[bytes]	List of byte representations for each token in the vocabulary

Source code in genlm/backend/tokenization/bytes.py

def get_byte_tokens_from_sp(tokenizer):
    """Convert SentencePiece token strings to bytes.

    ``<0xXX>`` escapes become the byte they name and ``▁`` becomes a space; special
    tokens are encoded directly.

    Args:
        tokenizer: A Hugging Face tokenizer instance

    Returns:
        byte_tokens (list[bytes]): List of byte representations for each token in the vocabulary
    """
    special_tokens_map = {
        token_id: token for token, token_id in tokenizer.get_added_vocab().items()
    }
    byte_tokens = [b""] * len(tokenizer)
    prefix_space = "▁".encode()
    for i in range(len(tokenizer)):
        if i in special_tokens_map:
            byte_coded = special_tokens_map[i].encode()
        else:
            byte_coded = re.sub(
                rb"<0x(..)>",
                lambda x: bytes.fromhex(x[1].decode()),
                tokenizer.convert_ids_to_tokens(i).encode(),
            )
        byte_tokens[i] = byte_coded.replace(prefix_space, b" ")
    return byte_tokens

check_byte_decoder(tokenizer, byte_decoder)

Verify that a byte decoder can properly handle all tokens.

Parameters:

Name	Type	Description	Default
tokenizer		A Hugging Face tokenizer instance	required
byte_decoder	dict	Dictionary mapping characters to bytes	required

Raises:

Type	Description
ByteDecoderError	If byte decoder fails validation checks

Source code in genlm/backend/tokenization/bytes.py

def check_byte_decoder(tokenizer, byte_decoder):
    """Verify that a byte decoder can properly handle all tokens.

    Args:
        tokenizer: A Hugging Face tokenizer instance
        byte_decoder (dict): Dictionary mapping characters to bytes

    Raises:
        ByteDecoderError: If byte decoder fails validation checks
    """
    _check_byte_decoder_has_all_bytes(tokenizer, byte_decoder)
    _check_complex_roundtrip(tokenizer, byte_decoder)