cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.TokenFlowDataProcessor#

class cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.TokenFlowDataProcessor(filepath, data_params)[source]#

Bases: object

Methods

get_datadict

get_stats

load_data

previous

cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.save_sequence_distribution

next

cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.tokenizer