cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset.tokenize_dataset#

cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset.tokenize_dataset(input_dir, output_dir, sp_model_path, input_file_name)[source]#

previous

cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset

next

cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset.tokenize_text