cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.tokenizer.GenericTokenizer#

class cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.tokenizer.GenericTokenizer(processing_params, filepath)[source]#

Bases: object

Methods

convert_ids_to_tokens

decode

initialize_customtokenizer

initialize_gpt2tokenizer

initialize_huggingfacetokenizer

initialize_neoxtokenizer

previous

cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.tokenizer

next

cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.utils