cerebras.modelzoo.data_preparation.nlp.data_dedup.to_hash.get_documents

cerebras.modelzoo.data_preparation.nlp.data_dedup.to_hash.get_documents(input_dir, jsonl_key, format, threshold, job_id, n_jobs)

previous

cerebras.modelzoo.data_preparation.nlp.data_dedup.to_hash.generate_hashes

next

cerebras.modelzoo.data_preparation.nlp.data_dedup.to_hash.get_features