cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.save_sequence_distribution#

cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.save_sequence_distribution(file_directory, data_params_path)[source]#

previous

cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.process_file_for_sequence_distribution

next

cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.TokenFlowDataProcessor