Skip to content

Commit

Permalink
Add to natural.
Browse files Browse the repository at this point in the history
  • Loading branch information
alexlangberg committed Jun 8, 2015
1 parent 5cc1ae0 commit 52c900c
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 5 deletions.
1 change: 1 addition & 0 deletions lib/natural/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ exports.AggressiveTokenizer = require('./tokenizers/aggressive_tokenizer');
exports.RegexpTokenizer = require('./tokenizers/regexp_tokenizer').RegexpTokenizer;
exports.WordTokenizer = require('./tokenizers/regexp_tokenizer').WordTokenizer;
exports.WordPunctTokenizer = require('./tokenizers/regexp_tokenizer').WordPunctTokenizer;
exports.CaseTokenizer = require('./tokenizers/tokenizer_case').CaseTokenizer;
exports.TreebankWordTokenizer = require('./tokenizers/treebank_word_tokenizer');
exports.TokenizerJa = require('./tokenizers/tokenizer_ja');
exports.BayesClassifier = require('./classifiers/bayes_classifier');
Expand Down
10 changes: 5 additions & 5 deletions lib/natural/tokenizers/tokenizer_case.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,20 @@

var Tokenizer = require('./tokenizer'),
util = require('util'),
TokenizerCase = function() {
CaseTokenizer = function() {
};

util.inherits(TokenizerCase, Tokenizer);
util.inherits(CaseTokenizer, Tokenizer);

TokenizerCase.prototype.attach = function() {
CaseTokenizer.prototype.attach = function() {
var self = this;

String.prototype.tokenize = function(preserveApostrophe) {
return self.tokenize(this, preserveApostrophe);
}
};

TokenizerCase.prototype.tokenize = function(text, preserveApostrophe) {
CaseTokenizer.prototype.tokenize = function(text, preserveApostrophe) {
var whitelist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'];
var lower = text.toLowerCase();
var upper = text.toUpperCase();
Expand All @@ -53,4 +53,4 @@ TokenizerCase.prototype.tokenize = function(text, preserveApostrophe) {
return this.trim(result.replace(/\s+/g, ' ').split(' '));
};

module.exports = TokenizerCase;
exports.CaseTokenizer = CaseTokenizer;

0 comments on commit 52c900c

Please sign in to comment.