diff --git a/lib/natural/tokenizers/tokenizer_case.js b/lib/natural/tokenizers/tokenizer_case.js index d16e2cef2..d7677add4 100644 --- a/lib/natural/tokenizers/tokenizer_case.js +++ b/lib/natural/tokenizers/tokenizer_case.js @@ -35,6 +35,7 @@ CaseTokenizer.prototype.attach = function() { } }; +// Idea from Seagull: http://stackoverflow.com/a/26482650 CaseTokenizer.prototype.tokenize = function(text, preserveApostrophe) { var whitelist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']; var lower = text.toLowerCase();