From 1dd97d199f6ba5ae220a602398518cc35e898455 Mon Sep 17 00:00:00 2001 From: Alex Langberg Date: Mon, 8 Jun 2015 19:27:38 +0200 Subject: [PATCH] Attr. --- lib/natural/tokenizers/tokenizer_case.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/natural/tokenizers/tokenizer_case.js b/lib/natural/tokenizers/tokenizer_case.js index d16e2cef2..d7677add4 100644 --- a/lib/natural/tokenizers/tokenizer_case.js +++ b/lib/natural/tokenizers/tokenizer_case.js @@ -35,6 +35,7 @@ CaseTokenizer.prototype.attach = function() { } }; +// Idea from Seagull: http://stackoverflow.com/a/26482650 CaseTokenizer.prototype.tokenize = function(text, preserveApostrophe) { var whitelist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']; var lower = text.toLowerCase();