Local search (#220)

* Version Bump

* local search skeleton

* Added WaitService middleware

* Return matching queries

* wait action middleware rename/refactor

* bigger debounce time

* Fix: Initialize state using Immutable

* Local Search without integrations

* Local Search refactor: Keep state in closure, recurse

* “string” should be treated as the default widget by the inference. Closes #199

src/actions/search.js

@@ -1,6 +1,10 @@
+import fuzzy from 'fuzzy';
 import { currentBackend } from '../backends/backend';
 import { getIntegrationProvider } from '../integrations';
-import { selectIntegration } from '../reducers';
+import { selectIntegration, selectEntries } from '../reducers';
+import { selectInferedField } from '../reducers/collections';
+import { WAIT_UNTIL_ACTION } from '../redux/middleware/waitUntilAction';
+import { loadEntries, ENTRIES_SUCCESS } from './entries';
 
 /*
  * Contant Declarations
@@ -102,20 +106,19 @@ export function clearSearch() {
 export function searchEntries(searchTerm, page = 0) {
   return (dispatch, getState) => {
     const state = getState();
-    let collections = state.collections.keySeq().toArray();
-    collections = collections.filter(collection => selectIntegration(state, collection, 'search'));
+    const allCollections = state.collections.keySeq().toArray();
+    const collections = allCollections.filter(collection => selectIntegration(state, collection, 'search'));
     const integration = selectIntegration(state, collections[0], 'search');
     if (!integration) {
-      dispatch(searchFailure(searchTerm, 'Search integration is not configured.'));
+      localSearch(searchTerm, getState, dispatch);
+    } else {
+      const provider = getIntegrationProvider(state.integrations, currentBackend(state.config).getToken, integration);
+      dispatch(searchingEntries(searchTerm));
+      provider.search(collections, searchTerm, page).then(
+        response => dispatch(searchSuccess(searchTerm, response.entries, response.pagination)),
+        error => dispatch(searchFailure(searchTerm, error))
+      );
     }
-    const provider = integration ?
-      getIntegrationProvider(state.integrations, currentBackend(state.config).getToken, integration)
-      : currentBackend(state.config);
-    dispatch(searchingEntries(searchTerm));
-    provider.search(collections, searchTerm, page).then(
-      response => dispatch(searchSuccess(searchTerm, response.entries, response.pagination)),
-      error => dispatch(searchFailure(searchTerm, error))
-    );
   };
 }
 
@@ -125,16 +128,98 @@ export function query(namespace, collection, searchFields, searchTerm) {
   return (dispatch, getState) => {
     const state = getState();
     const integration = selectIntegration(state, collection, 'search');
+    dispatch(querying(namespace, collection, searchFields, searchTerm));
     if (!integration) {
-      dispatch(searchFailure(namespace, searchTerm, 'Search integration is not configured.'));
+      localQuery(namespace, collection, searchFields, searchTerm, state, dispatch);
+    } else {
+      const provider = getIntegrationProvider(state.integrations, currentBackend(state.config).getToken, integration);
+      provider.searchBy(searchFields.map(f => `data.${ f }`), collection, searchTerm).then(
+        response => dispatch(querySuccess(namespace, collection, searchFields, searchTerm, response)),
+        error => dispatch(queryFailure(namespace, collection, searchFields, searchTerm, error))
+      );
     }
-    const provider = integration ?
-      getIntegrationProvider(state.integrations, currentBackend(state.config).getToken, integration)
-      : currentBackend(state.config);
-    dispatch(querying(namespace, collection, searchFields, searchTerm));
-    provider.searchBy(searchFields, collection, searchTerm).then(
-      response => dispatch(querySuccess(namespace, collection, searchFields, searchTerm, response)),
-      error => dispatch(queryFailure(namespace, collection, searchFields, searchTerm, error))
-    );
   };
 }
+
+// Local Query & Search functions
+
+function localSearch(searchTerm, getState, dispatch) {
+  return (function acc(localResults = { entries: [] }) {
+    function processCollection(collection, collectionKey) {
+      const state = getState();
+      if (state.entries.hasIn(['pages', collectionKey, 'ids'])) {
+        const searchFields = [
+          selectInferedField(collection, 'title'),
+          selectInferedField(collection, 'shortTitle'),
+          selectInferedField(collection, 'author'),
+        ];
+        const collectionEntries = selectEntries(state, collectionKey).toJS();
+        const filteredEntries = fuzzy.filter(searchTerm, collectionEntries, {
+          extract: entry => searchFields.reduce((acc, field) => {
+            const f = entry.data[field];
+            return f ? `${ acc } ${ f }` : acc;
+          }, ""),
+        }).filter(entry => entry.score > 5);
+        localResults[collectionKey] = true;
+        localResults.entries = localResults.entries.concat(filteredEntries);
+
+        const returnedKeys = Object.keys(localResults);
+        const allCollections = state.collections.keySeq().toArray();
+        if (allCollections.every(v => returnedKeys.indexOf(v) !== -1)) {
+          const sortedResults = localResults.entries.sort((a, b) => {
+            if (a.score > b.score) return -1;
+            if (a.score < b.score) return 1;
+            return 0;
+          }).map(f => f.original);
+          if (allCollections.size > 3 || localResults.entries.length > 30) {
+            console.warn('The Netlify CMS is currently using a Built-in search.' +
+            '\nWhile this works great for small sites, bigger projects might benefit from a separate search integration.' + 
+            '\nPlease refer to the documentation for more information');
+          }
+          dispatch(searchSuccess(searchTerm, sortedResults, 0));
+        }
+      } else {
+        // Collection entries aren't loaded yet.
+        // Dispatch loadEntries and wait before redispatching this action again.
+        dispatch({
+          type: WAIT_UNTIL_ACTION,
+          predicate: action => (action.type === ENTRIES_SUCCESS && action.payload.collection === collectionKey),
+          run: () => processCollection(collection, collectionKey),
+        });
+        dispatch(loadEntries(collection));
+      }
+    }
+    getState().collections.forEach(processCollection);
+  }());
+}
+
+
+function localQuery(namespace, collection, searchFields, searchTerm, state, dispatch) {
+  // Check if entries in this collection were already loaded
+  if (state.entries.hasIn(['pages', collection, 'ids'])) {
+    const entries = selectEntries(state, collection).toJS();
+    const filteredEntries = fuzzy.filter(searchTerm, entries, {
+      extract: entry => searchFields.reduce((acc, field) => {
+        const f = entry.data[field];
+        return f ? `${ acc } ${ f }` : acc;
+      }, ""),
+    }).filter(entry => entry.score > 5);
+
+    const resultObj = {
+      query: searchTerm,
+      hits: [],
+    };
+
+    resultObj.hits = filteredEntries.map(f => f.original);
+    dispatch(querySuccess(namespace, collection, searchFields, searchTerm, resultObj));
+  } else {
+    // Collection entries aren't loaded yet.
+    // Dispatch loadEntries and wait before redispatching this action again.
+    dispatch({
+      type: WAIT_UNTIL_ACTION,
+      predicate: action => (action.type === ENTRIES_SUCCESS && action.payload.collection === collection),
+      run: dispatch => dispatch(query(namespace, collection, searchFields, searchTerm)),
+    });
+    dispatch(loadEntries(state.collections.get(collection)));
+  }
+}

src/components/Widgets/RelationControl.js

@@ -37,7 +37,7 @@ class RelationControl extends Component {
     const { value, field } = this.props;
     if (value) {
       const collection = field.get('collection');
-      const searchFields = field.get('searchFields').map(f => `data.${ f }`).toJS();
+      const searchFields = field.get('searchFields').toJS();
       this.props.query(this.controlID, collection, searchFields, value);
     }
   }
@@ -67,32 +67,14 @@ class RelationControl extends Component {
     if (value.length < 2) return;
     const { field } = this.props;
     const collection = field.get('collection');
-    const searchFields = field.get('searchFields').map(f => `data.${ f }`).toJS();
+    const searchFields = field.get('searchFields').toJS();
     this.props.query(this.controlID, collection, searchFields, value);
-  }, 80);
+  }, 100);
 
   onSuggestionsClearRequested = () => {
     this.props.clearSearch();
   };
 
-  getMatchingHits = (value) => {
-    const { field, queryHits } = this.props;
-    const searchFields = field.get('searchFields').toJS();
-    const escapedValue = escapeRegexCharacters(value.trim());
-    const regex = new RegExp(`^ ${ escapedValue }`, 'i');
-
-    if (escapedValue === '') {
-      return [];
-    }
-    return queryHits.get(this.controlID).filter((hit) => {
-      let testResult = false;
-      searchFields.forEach((f) => {
-        testResult = testResult || regex.test(hit.data[f]);
-      });
-      return testResult;
-    });
-  };
-
   getSuggestionValue = (suggestion) => {
     const { field } = this.props;
     const valueField = field.get('valueField');

src/reducers/collections.js

@@ -100,14 +100,13 @@ export const selectInferedField = (collection, fieldName) => {
 
   // If colllection has no fields or fieldName is not defined within inferables list, return null
   if (!fields || !inferableField) return null;
-
   // Try to return a field of the specified type with one of the synonyms
-  const mainTypeFields = fields.filter(f => f.get('widget') === inferableField.type).map(f => f.get('name'));
+  const mainTypeFields = fields.filter(f => f.get('widget', 'string') === inferableField.type).map(f => f.get('name'));
   field = mainTypeFields.filter(f => inferableField.synonyms.indexOf(f) !== -1);
   if (field && field.size > 0) return field.first();
 
   // Try to return a field for each of the specified secondary types
-  const secondaryTypeFields = fields.filter(f => inferableField.secondaryTypes.indexOf(f.get('widget')) !== -1).map(f => f.get('name'));
+  const secondaryTypeFields = fields.filter(f => inferableField.secondaryTypes.indexOf(f.get('widget', 'string')) !== -1).map(f => f.get('name'));
   field = secondaryTypeFields.filter(f => inferableField.synonyms.indexOf(f) !== -1);
   if (field && field.size > 0) return field.first();
 

src/reducers/search.js

@@ -13,7 +13,7 @@ let response;
 let page;
 let searchTerm;
 
-const defaultState = Map({ isFetching: false, term: null, page: 0, entryIds: [], queryHits: [] });
+const defaultState = Map({ isFetching: false, term: null, page: 0, entryIds: List([]), queryHits: Map({}) });
 
 const entries = (state = defaultState, action) => {
   switch (action.type) {

src/store/configureStore.js → src/redux/configureStore.js

@@ -1,10 +1,11 @@
 import { createStore, applyMiddleware, compose } from 'redux';
 import thunkMiddleware from 'redux-thunk';
+import waitUntilAction from './middleware/waitUntilAction';
 import reducer from '../reducers/combinedReducer';
 
 export default function configureStore(initialState) {
   const store = createStore(reducer, initialState, compose(
-    applyMiddleware(thunkMiddleware),
+    applyMiddleware(thunkMiddleware, waitUntilAction),
     window.devToolsExtension ? window.devToolsExtension() : f => f
   ));
 

src/redux/middleware/waitUntilAction.js

@@ -0,0 +1,47 @@
+// Based on wait-service by Mozilla: 
+// https://github.com/mozilla/gecko-dev/blob/master/devtools/client/shared/redux/middleware/wait-service.js
+
+/**
+ * A middleware that provides the ability for actions to install a 
+ * function to be run once when a specific condition is met by an
+ * action coming through the system. Think of it as a thunk that
+ * blocks until the condition is met.
+ */
+export const WAIT_UNTIL_ACTION = 'WAIT_UNTIL_ACTION';
+
+export default function waitUntilAction({ dispatch, getState }) {
+  let pending = [];
+
+  function checkPending(action) {
+    const readyRequests = [];
+    const stillPending = [];
+
+    // Find the pending requests whose predicates are satisfied with
+    // this action. Wait to run the requests until after we update the
+    // pending queue because the request handler may synchronously
+    // dispatch again and run this service (that use case is
+    // completely valid).
+    for (const request of pending) {
+      if (request.predicate(action)) {
+        readyRequests.push(request);
+      } else {
+        stillPending.push(request);
+      }
+    }
+
+    pending = stillPending;
+    for (const request of readyRequests) {
+      request.run(dispatch, getState, action);
+    }
+  }
+
+  return next => (action) => {
+    if (action.type === WAIT_UNTIL_ACTION) {
+      pending.push(action);
+      return null;
+    }
+    const result = next(action);
+    checkPending(action);
+    return result;
+  };
+}

src/root.js

@@ -3,7 +3,7 @@ import { Provider } from 'react-redux';
 import { Router } from 'react-router';
 import routes from './routing/routes';
 import history, { syncHistory } from './routing/history';
-import configureStore from './store/configureStore';
+import configureStore from './redux/configureStore';
 import { setStore } from './valueObjects/AssetProxy';
 
 const store = configureStore();