Export libcudf env for long string support

Signed-off-by: Ayush Dattagupta <[email protected]>
NVIDIA · Jul 10, 2024 · 19777e7 · 19777e7
1 parent ba7e8a1
commit 19777e7
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/nemo_curator/__init__.py b/nemo_curator/__init__.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import sys
 
 import dask
@@ -48,3 +49,6 @@
 # See https://github.com/NVIDIA/NeMo-Curator/issues/33
 # This also happens when reading and writing to files
 dask.config.set({"dataframe.convert-string": False})
+
+# Enable libcudf large string support
+os.environ["LIBCUDF_LARGE_STRINGS_ENABLED"] = "1"