Export libcudf env for long string support

Signed-off-by: Ayush Dattagupta <[email protected]>
NVIDIA · May 21, 2024 · 3849f25 · 3849f25
1 parent fb63641
commit 3849f25
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/nemo_curator/__init__.py b/nemo_curator/__init__.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+
 import dask
 
 from .modules import *
@@ -21,3 +23,6 @@
 # See https://github.com/NVIDIA/NeMo-Curator/issues/33
 # This also happens when reading and writing to files
 dask.config.set({"dataframe.convert-string": False})
+
+# Enable libcudf large string support
+os.environ["LIBCUDF_LARGE_STRINGS_ENABLED"] = "1"