Attr.

CheeseTurtle · Jun 8, 2015 · 1dd97d1 · 1dd97d1
1 parent 52c900c
commit 1dd97d1
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/lib/natural/tokenizers/tokenizer_case.js b/lib/natural/tokenizers/tokenizer_case.js
@@ -35,6 +35,7 @@ CaseTokenizer.prototype.attach = function() {
   }
 };
 
+// Idea from Seagull: http://stackoverflow.com/a/26482650
 CaseTokenizer.prototype.tokenize = function(text, preserveApostrophe) {
   var whitelist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'];
   var lower = text.toLowerCase();