Merge pull request NaturalNode#1 from NaturalNode/master

Update my fork.
CheeseTurtle · Apr 30, 2015 · 3a5181c · 3a5181c
2 parents fd0340c + a792545
commit 3a5181c
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 12 deletions.
diff --git a/lib/natural/classifiers/bayes_classifier.js b/lib/natural/classifiers/bayes_classifier.js
@@ -20,10 +20,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-var PorterStemmer = require('../stemmers/porter_stemmer'),
-util = require('util'),
-Classifier = require('./classifier'),
-ApparatusBayesClassifier = require('apparatus').BayesClassifier;
+var util = require('util'),
+    Classifier = require('./classifier'),
+    ApparatusBayesClassifier = require('apparatus').BayesClassifier;
 
 var BayesClassifier = function(stemmer, smoothing) {
     var abc = new ApparatusBayesClassifier();

diff --git a/lib/natural/classifiers/logistic_regression_classifier.js b/lib/natural/classifiers/logistic_regression_classifier.js
@@ -20,10 +20,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-var PorterStemmer = require('../stemmers/porter_stemmer'),
-util = require('util'),
-Classifier = require('./classifier'),
-ApparatusLogisticRegressionClassifier = require('apparatus').LogisticRegressionClassifier;
+var util = require('util'),
+    Classifier = require('./classifier'),
+    ApparatusLogisticRegressionClassifier = require('apparatus').LogisticRegressionClassifier;
 
 var LogisticRegressionClassifier = function(stemmer) {
     Classifier.call(this, new ApparatusLogisticRegressionClassifier(), stemmer);

diff --git a/lib/natural/stemmers/stemmer.js b/lib/natural/stemmers/stemmer.js
@@ -40,11 +40,21 @@ module.exports = function() {
 
     stemmer.tokenizeAndStem = function(text, keepStops) {
         var stemmedTokens = [];
-
-        new Tokenizer().tokenize(text).forEach(function(token) {
-            if(keepStops || stopwords.words.indexOf(token) == -1)
+        var lowercaseText = text.toLowerCase();
+        var tokens = new Tokenizer().tokenize(lowercaseText);
+
+        if (keepStops) {
+            tokens.forEach(function(token) {
                 stemmedTokens.push(stemmer.stem(token));
-        });
+            });
+        }
+
+        else {
+            tokens.forEach(function(token) {
+                if (stopwords.words.indexOf(token) == -1)
+                    stemmedTokens.push(stemmer.stem(token));
+            });
+        }
 
         return stemmedTokens;
     };

diff --git a/spec/porter_stemmer_spec.js b/spec/porter_stemmer_spec.js
@@ -21,6 +21,7 @@ THE SOFTWARE.
 */
 
 var stemmer = require('../lib/natural/stemmers/porter_stemmer');
+var stopwords = require('../lib/natural/util/stopwords');
 
 describe('porter_stemmer', function() {
 	it('should categorizeGroups', function() {
@@ -179,4 +180,17 @@ describe('porter_stemmer', function() {
 		expect('scoring stinks'.tokenizeAndStem()).toEqual(['score', 'stink']);
 		expect('SCORING STINKS'.tokenizeAndStem()).toEqual(['score', 'stink']);
 	});
+
+	it('should tokenize and stem ignoring stopwords', function() {
+		expect('My dog is very fun TO play with And another thing, he is A poodle.'.tokenizeAndStem()).toEqual(['dog', 'fun', 'plai', 'thing', 'poodl']);
+	});
+
+	it('should tokenize and stem ignoring all capital stopwords', function() {
+		var allCapitalStopwords = stopwords.words.join(' ').toUpperCase();
+		expect(allCapitalStopwords.tokenizeAndStem()).toEqual([]);
+	});
+
+	it('should tokenize and stem including stopwords', function() {
+		expect('My dog is very fun TO play with And another thing, he is A poodle.'.tokenizeAndStem(true)).toEqual(['my', 'dog', 'is', 'veri', 'fun', 'to', 'plai', 'with', 'and', 'anoth', 'thing', 'he', 'is', 'a', 'poodl']);
+	});
 });