Move FilterList into pica-utils crate (#725)

Cargo.toml

@@ -28,17 +28,17 @@ pica-select = { version = "0.21", path = "./crates/pica-select" }
 pica-utils = { version = "0.21", path = "./crates/pica-utils" }
 
 anyhow = "1.0"
-arrow2 = "0.18"
 bstr = "1.8"
 chrono = { version = "0.4", default-features = false }
 clap = "4.4"
 clap_complete = "4.4"
 csv = "1.3"
 flate2 = "1.0"
+polars = { version = "0.35.4", features = ["ipc", "decompress", "performant"] }
+quickcheck = "1.0"
 rand = "0.8"
 regex = "1.10"
 serde = "1.0"
-quickcheck = "1.0"
 serde_json = "1.0"
 sha2 = "0.10"
 thiserror = "1.0"

crates/pica-toolkit/Cargo.toml

@@ -8,7 +8,6 @@ keywords.workspace = true
 edition.workspace = true
 
 [dependencies]
-arrow2 = { workspace = true, features = ["io_ipc", "io_ipc_compression"] }
 bstr = { workspace = true }
 clap = { workspace = true, features = ["cargo", "derive", "wrap_help"] }
 clap_complete = { workspace = true }

crates/pica-toolkit/src/commands/filter.rs

@@ -8,11 +8,11 @@ use pica_matcher::{
     MatcherOptions, OccurrenceMatcher, ParseMatcherError,
     RecordMatcher, TagMatcher,
 };
+use pica_path::PathExt;
 use pica_record::io::{ReaderBuilder, RecordsIterator, WriterBuilder};
-use pica_utils::NormalizationForm;
+use pica_utils::{FilterList, NormalizationForm};
 use serde::{Deserialize, Serialize};
 
-use crate::common::FilterList;
 use crate::progress::Progress;
 use crate::util::{CliError, CliResult};
 use crate::{gzip_flag, skip_invalid_flag, Config};
@@ -69,8 +69,8 @@ pub(crate) struct Filter {
     /// If the file extension is `.feather`, `.arrow`, or `.ipc` the
     /// file is automatically interpreted as Apache Arrow;
     /// otherwise the file is read as CSV.
-    #[arg(long, short = 'A')]
-    allow_list: Vec<PathBuf>,
+    #[arg(long = "allow-lists", short = 'A')]
+    allow_lists: Vec<PathBuf>,
 
     /// Ignore records which are explicitly listed in one of the
     /// given deny-lists.
@@ -80,8 +80,8 @@ pub(crate) struct Filter {
     /// If the file extension is `.feather`, `.arrow`, or `.ipc` the
     /// file is automatically interpreted as Apache Arrow;
     /// otherwise the file is read as CSV.
-    #[arg(long, short = 'D')]
-    deny_list: Vec<PathBuf>,
+    #[arg(long = "deny-lists", short = 'D')]
+    deny_lists: Vec<PathBuf>,
 
     /// Limit the result to first <n> records
     ///
@@ -247,17 +247,11 @@ impl Filter {
             }
         }
 
-        let allow_list = if !self.allow_list.is_empty() {
-            FilterList::new(self.allow_list)?
-        } else {
-            FilterList::default()
-        };
-
-        let deny_list = if !self.deny_list.is_empty() {
-            FilterList::new(self.deny_list)?
-        } else {
-            FilterList::default()
-        };
+        let filter_list = FilterList::new()
+            .allow(self.allow_lists)
+            .unwrap()
+            .deny(self.deny_lists)
+            .unwrap();
 
         let mut progress = Progress::new(self.progress);
 
@@ -283,15 +277,7 @@ impl Filter {
                     Ok(mut record) => {
                         progress.record();
 
-                        if !allow_list.is_empty()
-                            && !allow_list.check(&record)
-                        {
-                            continue;
-                        }
-
-                        if !deny_list.is_empty()
-                            && deny_list.check(&record)
-                        {
+                        if !filter_list.check(record.idn()) {
                             continue;
                         }
 

crates/pica-toolkit/src/commands/select.rs

@@ -9,12 +9,12 @@ use std::str::FromStr;
 
 use clap::Parser;
 use pica_matcher::{MatcherOptions, RecordMatcher};
+use pica_path::PathExt;
 use pica_record::io::{ReaderBuilder, RecordsIterator};
 use pica_select::{Query, QueryExt, QueryOptions};
-use pica_utils::NormalizationForm;
+use pica_utils::{FilterList, NormalizationForm};
 use serde::{Deserialize, Serialize};
 
-use crate::common::FilterList;
 use crate::config::Config;
 use crate::progress::Progress;
 use crate::skip_invalid_flag;
@@ -119,8 +119,8 @@ pub(crate) struct Select {
     /// If the file extension is `.feather`, `.arrow`, or `.ipc` the
     /// file is automatically interpreted as Apache Arrow;
     /// otherwise the file is read as CSV.
-    #[arg(long, short = 'A')]
-    allow_list: Vec<PathBuf>,
+    #[arg(long = "allow-lists", short = 'A')]
+    allow_lists: Vec<PathBuf>,
 
     /// Ignore records which are explicitly listed in one of the
     /// given deny-lists.
@@ -130,8 +130,8 @@ pub(crate) struct Select {
     /// If the file extension is `.feather`, `.arrow`, or `.ipc` the
     /// file is automatically interpreted as Apache Arrow;
     /// otherwise the file is read as CSV.
-    #[arg(long, short = 'D')]
-    deny_list: Vec<PathBuf>,
+    #[arg(long = "deny-lists", short = 'D')]
+    deny_lists: Vec<PathBuf>,
 
     /// Show progress bar (requires `-o`/`--output`).
     #[arg(short, long, requires = "output")]
@@ -235,17 +235,11 @@ impl Select {
             None
         };
 
-        let allow_list = if !self.allow_list.is_empty() {
-            FilterList::new(self.allow_list)?
-        } else {
-            FilterList::default()
-        };
-
-        let deny_list = if !self.deny_list.is_empty() {
-            FilterList::new(self.deny_list)?
-        } else {
-            FilterList::default()
-        };
+        let filter_list = FilterList::new()
+            .allow(self.allow_lists)
+            .unwrap()
+            .deny(self.deny_lists)
+            .unwrap();
 
         let query = NormalizationForm::translit_opt(&self.query, nf);
         let query = Query::from_str(&query)?;
@@ -277,15 +271,7 @@ impl Select {
                     Ok(record) => {
                         progess.record();
 
-                        if !allow_list.is_empty()
-                            && !allow_list.check(&record)
-                        {
-                            continue;
-                        }
-
-                        if !deny_list.is_empty()
-                            && deny_list.check(&record)
-                        {
+                        if !filter_list.check(record.idn()) {
                             continue;
                         }
 

crates/pica-toolkit/src/main.rs

@@ -5,7 +5,6 @@ extern crate serde;
 extern crate termcolor;
 
 mod commands;
-mod common;
 mod config;
 mod macros;
 mod progress;

crates/pica-toolkit/tests/snapshot/select/040-select-allow-list.in/allow.csv

@@ -1 +1,2 @@
+idn
 040011569

crates/pica-toolkit/tests/snapshot/select/041-select-deny-list.in/deny.csv

@@ -1,2 +1,3 @@
+idn
 118515551
 1012852784

crates/pica-utils/Cargo.toml

@@ -8,5 +8,8 @@ keywords.workspace = true
 edition.workspace = true
 
 [dependencies]
-unicode-normalization = { version =  "0.1.22" }
+bstr = { workspace = true }
+polars = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
+unicode-normalization = { version =  "0.1.22" }
+thiserror = { workspace = true }