Update nemo_curator/modules/exact_dedup.py

Co-authored-by: Sarah Yurick <[email protected]> Signed-off-by: Praateek Mahajan <[email protected]>
NVIDIA · Feb 7, 2025 · 2beca67 · 2beca67
1 parent f172c72
commit 2beca67
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/nemo_curator/modules/exact_dedup.py b/nemo_curator/modules/exact_dedup.py
@@ -181,7 +181,7 @@ def identify_duplicates(self, dataset: DocumentDataset) -> DocumentDataset:
         return DocumentDataset.read_parquet(
             write_path,
             backend=backend,
-            # we read with FPP=1 so that groups are read in whole (and don't exist across partitions)
+            # We read with files_per_partition=1 so that groups are read in whole (and do not exist across partitions)
             files_per_partition=1,
             blocksize=None,
         )