Removal logic for fuzzy / exact (no class abstraction) (NVIDIA#509)

Signed-off-by: Phillip Mobley <[email protected]>
philm001 · Feb 10, 2025 · 39db13d · 39db13d
1 parent 9638f2c
commit 39db13d
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/nemo_curator/modules/exact_dedup.py b/nemo_curator/modules/exact_dedup.py
@@ -146,7 +146,7 @@ def hash_documents(
             # TODO: Generalize ty using self.hash_method
             return df.apply(lambda x: md5(x.encode()).hexdigest())
 
-    def call(self, dataset: DocumentDataset) -> Union[DocumentDataset, str]:
+    def identify_duplicates(self, dataset: DocumentDataset) -> DocumentDataset:
         """
         Find document ID's for exact duplicates in a given DocumentDataset
         Parameters

diff --git a/nemo_curator/modules/fuzzy_dedup/fuzzyduplicates.py b/nemo_curator/modules/fuzzy_dedup/fuzzyduplicates.py
@@ -131,7 +131,9 @@ def __init__(
             profile_dir=self.config.profile_dir,
         )
 
-    def call(self, dataset: DocumentDataset):
+    def identify_duplicates(
+        self, dataset: DocumentDataset
+    ) -> Optional[DocumentDataset]:
         """
         Parameters
         ----------