cerebras.modelzoo.data_preparation.data_preprocessing.utils.check_fim_special_tokens

cerebras.modelzoo.data_preparation.data_preprocessing.utils.check_fim_special_tokens(params, tokenizer)

previous

cerebras.modelzoo.data_preparation.data_preprocessing.utils.args_to_params

next

cerebras.modelzoo.data_preparation.data_preprocessing.utils.chunk