pyproject.toml

[project]
name = "data_prep_toolkit_transforms"
version = "1.0.0a2"
requires-python = ">=3.10,<3.13"
keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
description = "Data Preparation Toolkit Transforms using Ray"
license = {text = "Apache-2.0"}
readme = {file = "README-list.md", content-type = "text/markdown"}
authors = [
    { name = "Maroun Touma", email = "touma@us.ibm.com" },
]
dynamic = ["dependencies","optional-dependencies"]

[build-system]
requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
build-backend = "setuptools.build_meta"


[tool.setuptools.dynamic.dependencies]
file = ["requirements.txt"]

[tool.setuptools.dynamic.optional-dependencies]
dev = { file = ["requirements-dev.txt"]}
ray = { file = ["requirements-ray.txt"]}
all = { file = [
"code/proglang_select/python/requirements.txt",
"code/header_cleanser/python/requirements.txt",
"code/license_select/python/requirements.txt",
"code/code_quality/python/requirements.txt",
"code/code2parquet/python/requirements.txt",

##### pii_redactor seem to be failing UT
## "language/pii_redactor/python/requirements.txt",

"universal/profiler/python/requirements.txt",
"universal/resize/python/requirements.txt",

"language/lang_id/requirements.txt",
"language/doc_quality/requirements.txt",
"language/pdf2parquet/requirements.txt",
"language/doc_chunk/requirements.txt",
"language/text_encoder/requirements.txt",
"language/similarity/requirements.txt",

##### Cannot have html2parquet until we solve
## docling-ibm-models 1.1.7 depends on lxml<5.0.0 and >=4.9.1
## trafilatura 1.12.0 depends on lxml>=5.2.2; platform_system != "Darwin" or python_version > "3.8"
## "language/html2parquet/requirements.txt",

"universal/filter/requirements.txt",
"universal/doc_id/requirements.txt",
"universal/ededup/requirements.txt",
"universal/fdedup/requirements.txt",
"universal/hap/requirements.txt",
"universal/tokenization/requirements.txt",
"universal/web2parquet/requirements.txt"
]}

language = { file = [
##### pii_redactor seem to be failing UT
## "language/pii_redactor/python/requirements.txt",

"language/lang_id/requirements.txt",
"language/doc_quality/requirements.txt",
"language/pdf2parquet/requirements.txt",
"language/doc_chunk/requirements.txt",
"language/text_encoder/requirements.txt",
"language/similarity/requirements.txt",

##### Cannot have html2parquet until we solve
## docling-ibm-models 1.1.7 depends on lxml<5.0.0 and >=4.9.1
## trafilatura 1.12.0 depends on lxml>=5.2.2; platform_system != "Darwin" or python_version > "3.8"
## "language/html2parquet/requirements.txt",

"universal/filter/requirements.txt",
"universal/doc_id/requirements.txt",
"universal/ededup/requirements.txt",
"universal/fdedup/requirements.txt",
"universal/hap/requirements.txt",
"universal/tokenization/requirements.txt",
"universal/web2parquet/requirements.txt"
]}

# pyproject.toml must be in a parent and cannot be in sibling
# i.e. Cannot access '../code/proglang_select/python/..

proglang_select = { file = ["code/proglang_select/python/requirements.txt"]} 
header_cleanser = {file =  ["code/header_cleanser/python/requirements.txt"]}
license_select = { file = ["code/license_select/python/requirements.txt"]} 
code_quality = { file = ["code/code_quality/python/requirements.txt"]} 
code2parquet = {file =  ["code/code2parquet/python/requirements.txt"]}
code_profiler = { file = ["code/code_profiler/python/requirements.txt"]} 

pii_redactor = { file = ["language/pii_redactor/python/requirements.txt"]} 

profiler = { file = ["universal/profiler/python/requirements.txt"]} 
resize = { file = ["universal/resize/python/requirements.txt"]} 

######## Named transforms
doc_chunk = { file = ["language/doc_chunk/requirements.txt"]}
doc_quality = { file = ["language/doc_quality/requirements.txt"]}
html2parquet = { file = ["language/html2parquet/requirements.txt"]}
lang_id = { file = ["language/lang_id/requirements.txt"]}
pdf2parquet = { file = ["language/pdf2parquet/requirements.txt"]}
text_encoder = { file = ["language/text_encoder/requirements.txt"]}

filter = { file = ["universal/filter/requirements.txt"]}
doc_id = { file = ["universal/doc_id/requirements.txt"]}
hap = { file = ["universal/hap/requirements.txt"]}
ededup = { file = ["universal/ededup/requirements.txt"]} 
fdedup = { file = ["universal/fdedup/requirements.txt"]} 
tokenization = { file = ["universal/tokenization/requirements.txt"]} 

web2parquet = { file = ["universal/web2parquet/requirements.txt"]}

similarity = { file = ["language/similarity/requirements.txt"]}

# Does not seem to work for our custom layout
# copy all files to a single src and let automatic discovery find them

# When combing named modules with flat modules, need to run
# the build twice, once with the block below commented out
# and once after adding the lines below
[tool.setuptools.package-dir]
dpk_web2parquet = "universal/web2parquet/dpk_web2parquet"
dpk_doc_chunk = "language/doc_chunk/dpk_doc_chunk"
dpk_doc_quality = "language/doc_quality/dpk_doc_quality"
dpk_html2parquet = "language/html2parquet/dpk_html2parquet"
dpk_lang_id = "language/lang_id/dpk_lang_id"
dpk_pdf2parquet = "language/pdf2parquet/dpk_pdf2parquet"
dpk_text_encoder = "language/text_encoder/dpk_text_encoder"
dpk_doc_id = "universal/doc_id/dpk_doc_id"
dpk_hap = "universal/hap/dpk_hap"
dpk_ededup = "universal/ededup/dpk_ededup"
dpk_fdedup = "universal/fdedup/dpk_fdedup"
dpk_tokenization = "universal/tokenization/dpk_tokenization"
dpk_similarity = "language/similarity/dpk_similarity"
dpk_filter = "universal/filter/dpk_filter"


#[tool.setuptools.package-data]
#"*" = ["*.txt"]

[options]
package_dir = ["src","test"]

[options.packages.find]
where = ["src"]

[tool.pytest.ini_options]
# Currently we use low coverage since we have to run tests separately (see makefile)
#addopts = "--cov --cov-report term-missing --cov-fail-under 25"
markers = ["unit: unit tests", "integration: integration tests"]