cerebras.modelzoo.data_preparation.data_preprocessing.data_dedup.generate_duplicate_pairs.split_files#

cerebras.modelzoo.data_preparation.data_preprocessing.data_dedup.generate_duplicate_pairs.split_files(input_dir, n_proc)[source]#

previous

cerebras.modelzoo.data_preparation.data_preprocessing.data_dedup.generate_duplicate_pairs.optimal_param

next

cerebras.modelzoo.data_preparation.data_preprocessing.data_dedup.to_hash