From 0dffe1805f2eb803372e4ebdec1ca268b063146e Mon Sep 17 00:00:00 2001 From: "mansoor.sajjad" Date: Tue, 22 Nov 2022 12:41:00 +0100 Subject: [PATCH 1/8] Filtering on addendum namespaces. --- query/autocomplete.js | 35 +++-- query/view/addendum_namespace_filter.js | 13 ++ sanitizer/_addendum.js | 49 +++++++ sanitizer/autocomplete.js | 7 +- sanitizer/sanitizeAll.js | 8 +- ...ocomplete_configured_addendum_namespace.js | 69 ++++++++++ ..._multiple_configured_addendum_namespace.js | 72 ++++++++++ test/unit/query/autocomplete.js | 130 +++++++++--------- ...ete_with_configured_addendum_namespaces.js | 79 +++++++++++ 9 files changed, 381 insertions(+), 81 deletions(-) create mode 100644 query/view/addendum_namespace_filter.js create mode 100644 sanitizer/_addendum.js create mode 100644 test/unit/fixture/autocomplete_configured_addendum_namespace.js create mode 100644 test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js create mode 100644 test/unit/query/autocomplete_with_configured_addendum_namespaces.js diff --git a/query/autocomplete.js b/query/autocomplete.js index 6e980a75c..88fcabfbd 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -5,19 +5,21 @@ const textParser = require('./text_parser_pelias'); const config = require('pelias-config').generate(); const placeTypes = require('../helper/placeTypes'); const toSingleField = require('./view/helper').toSingleField; +const peliasConfig = require('pelias-config').generate(); // additional views (these may be merged in to pelias/query at a later date) var views = { - custom_boosts: require('./view/boost_sources_and_layers'), - ngrams_strict: require('./view/ngrams_strict'), - ngrams_last_token_only: require('./view/ngrams_last_token_only'), - ngrams_last_token_only_multi: require('./view/ngrams_last_token_only_multi'), - admin_multi_match_first: require('./view/admin_multi_match_first'), - admin_multi_match_last: require('./view/admin_multi_match_last'), - phrase_first_tokens_only: require('./view/phrase_first_tokens_only'), - boost_exact_matches: require('./view/boost_exact_matches'), + custom_boosts: require('./view/boost_sources_and_layers'), + ngrams_strict: require('./view/ngrams_strict'), + ngrams_last_token_only: require('./view/ngrams_last_token_only'), + ngrams_last_token_only_multi: require('./view/ngrams_last_token_only_multi'), + admin_multi_match_first: require('./view/admin_multi_match_first'), + admin_multi_match_last: require('./view/admin_multi_match_last'), + phrase_first_tokens_only: require('./view/phrase_first_tokens_only'), + boost_exact_matches: require('./view/boost_exact_matches'), max_character_count_layer_filter: require('./view/max_character_count_layer_filter'), - focus_point_filter: require('./view/focus_point_distance_filter') + focus_point_filter: require('./view/focus_point_distance_filter'), + addendum_namespace_filter: require('./view/addendum_namespace_filter') }; // add abbrevations for the fields pelias/parser is able to detect. @@ -65,6 +67,11 @@ query.filter( peliasQuery.view.categories ); query.filter( peliasQuery.view.boundary_gid ); query.filter( views.focus_point_filter ); +const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces'); +Object.keys(configuredAddendumNamespaces).forEach(namespace => { + query.filter( views.addendum_namespace_filter(namespace) ); +}); + // -------------------------------- /** @@ -75,6 +82,14 @@ function generateQuery( clean ){ const vs = new peliasQuery.Vars( defaults ); + //addendum + const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces'); + Object.keys(configuredAddendumNamespaces) + .filter(namespace => clean[namespace]) + .forEach(namespace => { + vs.var( namespace, clean[namespace] ); + }); + // sources if( _.isArray(clean.sources) && !_.isEmpty(clean.sources) ){ vs.var( 'sources', clean.sources ); @@ -104,7 +119,7 @@ function generateQuery( clean ){ vs.var( 'input:name', clean.text ); // if the tokenizer has run then we set 'input:name' to as the combination of the - // 'complete' tokens with the 'incomplete' tokens, the resuting array differs + // 'complete' tokens with the 'incomplete' tokens, the resulting array differs // slightly from the 'input:name:tokens' array as some tokens might have been // removed in the process; such as single grams which are not present in then // ngrams index. diff --git a/query/view/addendum_namespace_filter.js b/query/view/addendum_namespace_filter.js new file mode 100644 index 000000000..6ad6f646a --- /dev/null +++ b/query/view/addendum_namespace_filter.js @@ -0,0 +1,13 @@ +module.exports = function (addendumParameter) { + return function (vs) { + if (!vs.isset(addendumParameter)) { + return null; + } + + return { + terms: { + [`addendum.${addendumParameter}`]: vs.var(addendumParameter) + } + }; + }; +}; \ No newline at end of file diff --git a/sanitizer/_addendum.js b/sanitizer/_addendum.js new file mode 100644 index 000000000..05732fad5 --- /dev/null +++ b/sanitizer/_addendum.js @@ -0,0 +1,49 @@ +const _ = require('lodash'); +const nonEmptyString = (v) => _.isString(v) && !_.isEmpty(v); +const peliasConfig = require('pelias-config').generate(); + +function _sanitize(raw, clean) { + + // error & warning messages + const messages = { errors: [], warnings: [] }; + + // target input params + const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces'); + + Object.keys(raw) + .filter(field => configuredAddendumNamespaces.hasOwnProperty(field)) + .forEach(field => { + const rawValue = raw[field]; + + if (!nonEmptyString(rawValue)) { + messages.errors.push(field + ' should not be empty'); + } else { + const validationType = configuredAddendumNamespaces[field].type; + if (validationType.toLowerCase() === 'array') { + const valuesArray = rawValue.split(',').filter(nonEmptyString); + if (_.isArray(valuesArray) && _.isEmpty(valuesArray)) { + messages.errors.push(field + ' should not be empty'); + } + } else { + if (typeof rawValue !== validationType) { + messages.errors.push(field + ': Invalid parameter type, expecting: ' + validationType + ', got: ' + rawValue); + } + if (validationType === 'number' && isNaN(rawValue)) { + messages.errors.push(field + ': Invalid parameter type, expecting: ' + validationType + ', got NaN: ' + rawValue); + } + if (validationType === 'object' && !_.isObject(rawValue)) { + messages.errors.push(field + ': Invalid parameter type, expecting: ' + validationType + ', got: ' + rawValue); + } + if (validationType === 'object' && _.isArray(rawValue)) { + messages.errors.push(field + ': Invalid parameter type, expecting: ' + validationType + ', got array: ' + rawValue); + } + } + } + }); + + return messages; +} + +module.exports = () => ({ + sanitize: _sanitize, +}); diff --git a/sanitizer/autocomplete.js b/sanitizer/autocomplete.js index 34351f072..710f2de93 100644 --- a/sanitizer/autocomplete.js +++ b/sanitizer/autocomplete.js @@ -1,5 +1,5 @@ -var type_mapping = require('../helper/type_mapping'); -var sanitizeAll = require('../sanitizer/sanitizeAll'); +const type_mapping = require('../helper/type_mapping'); +const sanitizeAll = require('../sanitizer/sanitizeAll'); /** * a list of query-string parameters groups for this endpoint @@ -44,7 +44,8 @@ module.exports.middleware = (_api_pelias_config) => { boundary_country: require('../sanitizer/_boundary_country')(), categories: require('../sanitizer/_categories')(), request_language: require('../sanitizer/_request_language')(), - boundary_gid: require('../sanitizer/_boundary_gid')() + boundary_gid: require('../sanitizer/_boundary_gid')(), + addendum: require('../sanitizer/_addendum')() }; return ( req, res, next ) => { diff --git a/sanitizer/sanitizeAll.js b/sanitizer/sanitizeAll.js index 50bff0146..64f1655e7 100644 --- a/sanitizer/sanitizeAll.js +++ b/sanitizer/sanitizeAll.js @@ -1,5 +1,6 @@ const PeliasParameterError = require('./PeliasParameterError'); const PeliasTimeoutError = require('../sanitizer/PeliasTimeoutError'); +const peliasConfig = require('pelias-config').generate(); function getCorrectErrorType(message) { if (message.includes( 'Timeout')) { @@ -37,7 +38,7 @@ function sanitize( req, sanitizers ){ const params = req.query || {}; for (let s in sanitizers) { - var sanity = sanitizers[s].sanitize( params, req.clean, req ); + const sanity = sanitizers[s].sanitize(params, req.clean, req); // if errors occurred then set them // on the req object. @@ -63,6 +64,7 @@ function checkParameters( req, sanitizers ) { // (in this case from the GET querystring params) const params = req.query || {}; const goodParameters = {}; + const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces'); for (let s in sanitizers) { @@ -74,7 +76,7 @@ function checkParameters( req, sanitizers ) { const prop = sanitizers[s].expected()[t]; if (prop.hasOwnProperty('name')){ // adds name of valid parameter - goodParameters[prop.name] = prop.name; + goodParameters[prop.name] = prop; } } } @@ -83,7 +85,7 @@ function checkParameters( req, sanitizers ) { // add a warning message if (Object.keys(goodParameters).length !== 0) { for (let p in params) { - if (!goodParameters.hasOwnProperty(p)){ + if (!goodParameters.hasOwnProperty(p) && !configuredAddendumNamespaces.hasOwnProperty(p)) { req.warnings = req.warnings.concat('Invalid Parameter: ' + p); } } diff --git a/test/unit/fixture/autocomplete_configured_addendum_namespace.js b/test/unit/fixture/autocomplete_configured_addendum_namespace.js new file mode 100644 index 000000000..e2c7267e4 --- /dev/null +++ b/test/unit/fixture/autocomplete_configured_addendum_namespace.js @@ -0,0 +1,69 @@ +module.exports = { + 'query': { + 'bool': { + 'must': [ + { + 'constant_score': { + 'filter': { + 'multi_match': { + 'fields': ['name.default', 'name.en'], + 'analyzer': 'peliasQuery', + 'query': 'something', + 'boost': 100, + 'type': 'phrase', + 'slop': 3 + } + } + } + }], + 'should': [ + { + 'function_score': { + 'query': { + 'match_all': {} + }, + 'max_boost': 20, + 'score_mode': 'first', + 'boost_mode': 'replace', + 'functions': [{ + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'popularity', + 'missing': 1 + }, + 'weight': 1 + }] + } + }, { + 'function_score': { + 'query': { + 'match_all': {} + }, + 'max_boost': 20, + 'score_mode': 'first', + 'boost_mode': 'replace', + 'functions': [{ + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'population', + 'missing': 1 + }, + 'weight': 3 + }] + } + }], + 'filter': [ + { + 'terms': { + 'addendum.tariff_zone_ids': [ + 'TAR-123', 'TAR-345' + ] + } + } + ] + } + }, + 'sort': ['_score'], + 'size': 20, + 'track_scores': true +}; diff --git a/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js b/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js new file mode 100644 index 000000000..ff28a17e5 --- /dev/null +++ b/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js @@ -0,0 +1,72 @@ +module.exports = { + 'query': { + 'bool': { + 'must': [ + { + 'constant_score': { + 'filter': { + 'multi_match': { + 'fields': ['name.default', 'name.en'], + 'analyzer': 'peliasQuery', + 'query': 'something', + 'boost': 100, + 'type': 'phrase', + 'slop': 3 + } + } + } + }], + 'should': [ + { + 'function_score': { + 'query': { + 'match_all': {} + }, + 'max_boost': 20, + 'score_mode': 'first', + 'boost_mode': 'replace', + 'functions': [{ + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'popularity', + 'missing': 1 + }, + 'weight': 1 + }] + } + }, { + 'function_score': { + 'query': { + 'match_all': {} + }, + 'max_boost': 20, + 'score_mode': 'first', + 'boost_mode': 'replace', + 'functions': [{ + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'population', + 'missing': 1 + }, + 'weight': 3 + }] + } + }], + 'filter': [ + { + 'terms': { + 'addendum.tariff_zone_ids': [ + 'TAR-123', 'TAR-345' + ], + 'addendum.tariff_zone_authorities': [ + 'TAR' + ] + } + } + ] + } + }, + 'sort': ['_score'], + 'size': 20, + 'track_scores': true +}; diff --git a/test/unit/query/autocomplete.js b/test/unit/query/autocomplete.js index 4651502f1..f30936668 100644 --- a/test/unit/query/autocomplete.js +++ b/test/unit/query/autocomplete.js @@ -6,7 +6,7 @@ const defaultPeliasConfig = { } }; -var generate = proxyquire('../../../query/autocomplete', { +const generate = proxyquire('../../../query/autocomplete', { 'pelias-config': defaultPeliasConfig }); @@ -20,32 +20,32 @@ module.exports.tests.interface = function(test, common) { }; module.exports.tests.query = function(test, common) { - test('valid lingustic-only autocomplete', function(t) { - var query = generate({ + test('valid linguistic-only autocomplete', function(t) { + const query = generate({ text: 'test', tokens: ['test'], tokens_complete: [], tokens_incomplete: ['test'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_only'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_only'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_only'); t.end(); }); - test('valid lingustic autocomplete with 3 tokens', function(t) { - var query = generate({ + test('valid linguistic autocomplete with 3 tokens', function(t) { + const query = generate({ text: 'one two three', tokens: ['one','two','three'], tokens_complete: ['one','two'], tokens_incomplete: ['three'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_multiple_tokens'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_multiple_tokens'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_multiple_tokens'); @@ -53,24 +53,24 @@ module.exports.tests.query = function(test, common) { }); // This is to prevent a query like '30 west' from considering the 'west' part as an admin component - test('valid lingustic autocomplete with 3 tokens - first two are numeric', function (t) { - var query = generate({ + test('valid linguistic autocomplete with 3 tokens - first two are numeric', function (t) { + const query = generate({ text: '1 1 three', tokens: ['1', '2', 'three'], tokens_complete: ['1', '2'], tokens_incomplete: ['three'] }); - var compiled = JSON.parse(JSON.stringify(query)); - var expected = require('../fixture/autocomplete_linguistic_multiple_tokens_complete_numeric'); + const compiled = JSON.parse(JSON.stringify(query)); + const expected = require('../fixture/autocomplete_linguistic_multiple_tokens_complete_numeric'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_multiple_tokens_complete_numeric'); t.end(); }); - test('valid lingustic autocomplete with comma delimited admin section', function(t) { - var query = generate({ + test('valid linguistic autocomplete with comma delimited admin section', function(t) { + const query = generate({ text: 'one two, three', parsed_text: { subject: 'one two', @@ -82,8 +82,8 @@ module.exports.tests.query = function(test, common) { tokens_incomplete: [] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_with_admin'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_with_admin'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_with_admin'); @@ -93,16 +93,16 @@ module.exports.tests.query = function(test, common) { // if the final token is less than 2 chars we need to remove it from the string. // note: this behaviour is tied to having a min_gram size of 2. // note: if 1 grams are enabled at a later date, remove this behaviour. - test('valid lingustic autocomplete final token', function(t) { - var query = generate({ + test('valid linguistic autocomplete final token', function(t) { + const query = generate({ text: 'one t', tokens: ['one','t'], tokens_complete: ['one'], tokens_incomplete: [] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_final_token'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_final_token'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_final_token'); @@ -131,49 +131,49 @@ module.exports.tests.query = function(test, common) { 'pelias-config': configWithCustomSettings }); - test('valid lingustic autocomplete one character token', function(t) { - var query = generate_custom({ + test('valid linguistic autocomplete one character token', function(t) { + const query = generate_custom({ text: 't', tokens: ['t'], tokens_complete: [], tokens_incomplete: ['t'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_one_char_token'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_one_char_token'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_one_char_token'); t.end(); }); - test('valid lingustic autocomplete two character token', function(t) { + test('valid linguistic autocomplete two character token', function(t) { console.log(`config value: ${configWithCustomSettings.generate().get('api.autocomplete.exclude_address_length')}`); - var query = generate_custom({ + const query = generate_custom({ text: 'te', tokens: ['te'], tokens_complete: [], tokens_incomplete: ['te'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_two_char_token'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_two_char_token'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_two_char_token'); t.end(); }); - test('valid lingustic autocomplete three character token', function(t) { - var query = generate_custom({ + test('valid linguistic autocomplete three character token', function(t) { + const query = generate_custom({ text: 'tes', tokens: ['tes'], tokens_complete: [], tokens_incomplete: ['tes'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_three_char_token'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_three_char_token'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_three_char_token'); @@ -183,7 +183,7 @@ module.exports.tests.query = function(test, common) { // end tests with custom pelias.json settings test('autocomplete + focus', function(t) { - var query = generate({ + const query = generate({ text: 'test', 'focus.point.lat': 29.49136, 'focus.point.lon': -82.50622, @@ -192,8 +192,8 @@ module.exports.tests.query = function(test, common) { tokens_incomplete: ['test'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_focus'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_focus'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_focus'); @@ -201,7 +201,7 @@ module.exports.tests.query = function(test, common) { }); test('autocomplete + focus on null island', function(t) { - var query = generate({ + const query = generate({ text: 'test', 'focus.point.lat': 0, 'focus.point.lon': 0, @@ -210,8 +210,8 @@ module.exports.tests.query = function(test, common) { tokens_incomplete: ['test'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_focus_null_island'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_focus_null_island'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_focus_null_island'); @@ -219,7 +219,7 @@ module.exports.tests.query = function(test, common) { }); test('valid sources filter', function(t) { - var query = generate({ + const query = generate({ 'text': 'test', 'sources': ['test_source'], tokens: ['test'], @@ -227,8 +227,8 @@ module.exports.tests.query = function(test, common) { tokens_incomplete: ['test'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_with_source_filtering'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_with_source_filtering'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'valid autocomplete query with source filtering'); @@ -236,7 +236,7 @@ module.exports.tests.query = function(test, common) { }); test('valid layers filter', function(t) { - var query = generate({ + const query = generate({ 'text': 'test', 'layers': ['country'], tokens: ['test'], @@ -244,8 +244,8 @@ module.exports.tests.query = function(test, common) { tokens_incomplete: ['test'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_with_layer_filtering'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_with_layer_filtering'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'valid autocomplete query with layer filtering'); @@ -253,7 +253,7 @@ module.exports.tests.query = function(test, common) { }); test('valid categories filter', function (t) { - var clean = { + const clean = { text: 'test', tokens: ['test'], tokens_complete: [], @@ -261,10 +261,10 @@ module.exports.tests.query = function(test, common) { categories: ['retail', 'food'] }; - var query = generate(clean); + const query = generate(clean); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_with_category_filtering'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_with_category_filtering'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'valid autocomplete query with category filtering'); @@ -272,7 +272,7 @@ module.exports.tests.query = function(test, common) { }); test('single character street address', function(t) { - var query = generate({ + const query = generate({ text: 'k road, laird', parsed_text: { subject: 'k road', @@ -285,8 +285,8 @@ module.exports.tests.query = function(test, common) { tokens_incomplete: [] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_single_character_street'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_single_character_street'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_single_character_street'); @@ -294,7 +294,7 @@ module.exports.tests.query = function(test, common) { }); test('valid boundary.country search', function(t) { - var query = generate({ + const query = generate({ text: 'test', tokens: ['test'], tokens_complete: [], @@ -302,8 +302,8 @@ module.exports.tests.query = function(test, common) { 'boundary.country': ['ABC'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_boundary_country'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_boundary_country'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete: valid boundary.country query'); @@ -311,7 +311,7 @@ module.exports.tests.query = function(test, common) { }); test('autocomplete + bbox around San Francisco', function(t) { - var query = generate({ + const query = generate({ text: 'test', 'boundary.rect.max_lat': 37.83239, 'boundary.rect.max_lon': -122.35698, @@ -322,8 +322,8 @@ module.exports.tests.query = function(test, common) { tokens_incomplete: ['test'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_bbox_san_francisco'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_bbox_san_francisco'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_linguistic_bbox_san_francisco'); @@ -331,7 +331,7 @@ module.exports.tests.query = function(test, common) { }); test('autocomplete + circle around San Francisco', function(t) { - var query = generate({ + const query = generate({ text: 'test', 'boundary.circle.lat': 37.83239, 'boundary.circle.lon': -122.35698, @@ -341,8 +341,8 @@ module.exports.tests.query = function(test, common) { tokens_incomplete: ['test'] }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_linguistic_circle_san_francisco'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_linguistic_circle_san_francisco'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'query matches autocomplete_linguistic_circle_san_francisco fixture'); @@ -350,7 +350,7 @@ module.exports.tests.query = function(test, common) { }); test('valid boundary.gid search', function(t) { - var query = generate({ + const query = generate({ text: 'test', tokens: ['test'], tokens_complete: [], @@ -358,8 +358,8 @@ module.exports.tests.query = function(test, common) { 'boundary.gid': '123' }); - var compiled = JSON.parse( JSON.stringify( query ) ); - var expected = require('../fixture/autocomplete_boundary_gid'); + const compiled = JSON.parse( JSON.stringify( query ) ); + const expected = require('../fixture/autocomplete_boundary_gid'); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete: valid boundary.gid query'); @@ -373,7 +373,7 @@ module.exports.all = function (tape, common) { return tape('autocomplete query ' + name, testFunction); } - for( var testCase in module.exports.tests ){ + for( const testCase in module.exports.tests ){ module.exports.tests[testCase](test, common); } }; diff --git a/test/unit/query/autocomplete_with_configured_addendum_namespaces.js b/test/unit/query/autocomplete_with_configured_addendum_namespaces.js new file mode 100644 index 000000000..04e4d7a93 --- /dev/null +++ b/test/unit/query/autocomplete_with_configured_addendum_namespaces.js @@ -0,0 +1,79 @@ +const proxyquire = require('proxyquire').noCallThru(); +const realPeliasConfig = require('pelias-config'); +const peliasConfig = { + generate: function() { + const config = realPeliasConfig.generateDefaults(); + config.addendum_namespaces = { + tariff_zone_ids: { + type: 'array' + } + }; + return config; + } +}; + +const generate = proxyquire('../../../query/autocomplete', { + 'pelias-config': peliasConfig +}); + +module.exports.tests = {}; + +module.exports.tests.interface = function(test, common) { + test('valid interface', function(t) { + t.equal(typeof generate, 'function', 'valid function'); + t.end(); + }); +}; + +module.exports.tests.query = function(test, common) { + test('single configured addendum namespace', function(t) { + const query = generate({ + text: 'something', + tokens: ['something'], + tokens_complete: [], + tokens_incomplete: ['something'], + tariff_zone_ids: ['TAR-123', 'TAR-345'] + }); + + const compiled = JSON.parse( JSON.stringify( query ) ); + + const expected = require('../fixture/autocomplete_configured_addendum_namespace.js'); + common.diff(compiled.body, expected); + + t.deepEqual(compiled.type, 'autocomplete', 'query type set'); + t.deepEqual(compiled.body, expected, 'autocomplete_configured_addendum_namespace'); + t.end(); + }); + + test('Multiple configured addendum namespace', function(t) { + const query = generate({ + text: 'something', + tokens: ['something'], + tokens_complete: [], + tokens_incomplete: ['something'], + tariff_zone_ids: ['TAR-123', 'TAR-345'], + tariff_zone_authorities: ['TAR'] + }); + + const compiled = JSON.parse( JSON.stringify( query ) ); + + const expected = require('../fixture/autocomplete_configured_addendum_namespace.js'); + common.diff(compiled.body, expected); + + t.deepEqual(compiled.type, 'autocomplete', 'query type set'); + t.deepEqual(compiled.body, expected, 'autocomplete_configured_addendum_namespace'); + t.end(); + }); + +}; + +module.exports.all = function (tape, common) { + + function test(name, testFunction) { + return tape('autocomplete query ' + name, testFunction); + } + + for( const testCase in module.exports.tests ){ + module.exports.tests[testCase](test, common); + } +}; From 582217480997e0d2f28e8f4e37b93e3b8b800059 Mon Sep 17 00:00:00 2001 From: "mansoor.sajjad" Date: Wed, 23 Nov 2022 14:06:34 +0100 Subject: [PATCH 2/8] Fixing the sanitizer and adding unit tests. --- sanitizer/_addendum.js | 81 +++-- ..._multiple_configured_addendum_namespace.js | 97 +++--- ...ete_with_configured_addendum_namespaces.js | 53 ++- test/unit/run.js | 16 +- test/unit/sanitizer/_addendum.js | 314 ++++++++++++++++++ 5 files changed, 471 insertions(+), 90 deletions(-) create mode 100644 test/unit/sanitizer/_addendum.js diff --git a/sanitizer/_addendum.js b/sanitizer/_addendum.js index 05732fad5..22b91e219 100644 --- a/sanitizer/_addendum.js +++ b/sanitizer/_addendum.js @@ -11,32 +11,65 @@ function _sanitize(raw, clean) { const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces'); Object.keys(raw) - .filter(field => configuredAddendumNamespaces.hasOwnProperty(field)) - .forEach(field => { - const rawValue = raw[field]; + .filter(namespace => configuredAddendumNamespaces.hasOwnProperty(namespace)) + .forEach(namespace => { - if (!nonEmptyString(rawValue)) { - messages.errors.push(field + ' should not be empty'); + if (!nonEmptyString(raw[namespace])) { + messages.errors.push(namespace + ' should be a non empty string'); } else { - const validationType = configuredAddendumNamespaces[field].type; - if (validationType.toLowerCase() === 'array') { - const valuesArray = rawValue.split(',').filter(nonEmptyString); - if (_.isArray(valuesArray) && _.isEmpty(valuesArray)) { - messages.errors.push(field + ' should not be empty'); - } - } else { - if (typeof rawValue !== validationType) { - messages.errors.push(field + ': Invalid parameter type, expecting: ' + validationType + ', got: ' + rawValue); - } - if (validationType === 'number' && isNaN(rawValue)) { - messages.errors.push(field + ': Invalid parameter type, expecting: ' + validationType + ', got NaN: ' + rawValue); - } - if (validationType === 'object' && !_.isObject(rawValue)) { - messages.errors.push(field + ': Invalid parameter type, expecting: ' + validationType + ', got: ' + rawValue); - } - if (validationType === 'object' && _.isArray(rawValue)) { - messages.errors.push(field + ': Invalid parameter type, expecting: ' + validationType + ', got array: ' + rawValue); - } + const rawValue = raw[namespace].trim(); + const validationType = configuredAddendumNamespaces[namespace].type; + switch (validationType.toLowerCase()) { + case 'array': + const valuesArray = rawValue.split(',').filter(nonEmptyString); + if (_.isArray(valuesArray) && _.isEmpty(valuesArray)) { + messages.errors.push(namespace + ' should not be empty'); + } else { + clean[namespace] = valuesArray; + } + break; + + case 'string': + clean[namespace] = rawValue; + break; + + case 'number': + if (isNaN(rawValue)) { + messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got NaN: ' + rawValue); + } else { + clean[namespace] = Number(rawValue); + } + break; + + case 'boolean': + if ('true' !== rawValue && 'false' !== rawValue) { + messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got: ' + rawValue); + } else { + clean[namespace] = 'true' === rawValue; + } + break; + + case 'object': + try { + const parsed = JSON.parse(rawValue); + if (!_.isObject(parsed)) { + messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got ' + rawValue); + } else if (_.isArray(parsed)) { + messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got array: ' + rawValue); + } else { + clean[namespace] = parsed; + } + } catch (e) { + messages.errors.push( + namespace + ': Invalid parameter type, expecting: ' + validationType + ', got invalid JSON: ' + rawValue + ); + } + break; + + default: + if (typeof rawValue !== validationType) { + messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got: ' + rawValue); + } } } }); diff --git a/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js b/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js index ff28a17e5..4547b3e55 100644 --- a/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js +++ b/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js @@ -3,65 +3,74 @@ module.exports = { 'bool': { 'must': [ { - 'constant_score': { - 'filter': { - 'multi_match': { - 'fields': ['name.default', 'name.en'], - 'analyzer': 'peliasQuery', - 'query': 'something', - 'boost': 100, - 'type': 'phrase', - 'slop': 3 + 'constant_score': { + 'filter': { + 'multi_match': { + 'fields': ['name.default', 'name.en'], + 'analyzer': 'peliasQuery', + 'query': 'something', + 'boost': 100, + 'type': 'phrase', + 'slop': 3 + } } } - } - }], + }], 'should': [ { - 'function_score': { - 'query': { - 'match_all': {} - }, - 'max_boost': 20, - 'score_mode': 'first', - 'boost_mode': 'replace', - 'functions': [{ - 'field_value_factor': { - 'modifier': 'log1p', - 'field': 'popularity', - 'missing': 1 + 'function_score': { + 'query': { + 'match_all': {} }, - 'weight': 1 - }] - } - }, { - 'function_score': { - 'query': { - 'match_all': {} - }, - 'max_boost': 20, - 'score_mode': 'first', - 'boost_mode': 'replace', - 'functions': [{ - 'field_value_factor': { - 'modifier': 'log1p', - 'field': 'population', - 'missing': 1 + 'max_boost': 20, + 'score_mode': 'first', + 'boost_mode': 'replace', + 'functions': [{ + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'popularity', + 'missing': 1 + }, + 'weight': 1 + }] + } + }, { + 'function_score': { + 'query': { + 'match_all': {} }, - 'weight': 3 - }] - } - }], + 'max_boost': 20, + 'score_mode': 'first', + 'boost_mode': 'replace', + 'functions': [{ + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'population', + 'missing': 1 + }, + 'weight': 3 + }] + } + }], 'filter': [ { 'terms': { 'addendum.tariff_zone_ids': [ 'TAR-123', 'TAR-345' - ], + ] + } + }, + { + 'terms': { 'addendum.tariff_zone_authorities': [ 'TAR' ] } + }, + { + 'terms': { + 'addendum.public_id': '12345' + } } ] } diff --git a/test/unit/query/autocomplete_with_configured_addendum_namespaces.js b/test/unit/query/autocomplete_with_configured_addendum_namespaces.js index 04e4d7a93..4e16d40c7 100644 --- a/test/unit/query/autocomplete_with_configured_addendum_namespaces.js +++ b/test/unit/query/autocomplete_with_configured_addendum_namespaces.js @@ -1,12 +1,18 @@ const proxyquire = require('proxyquire').noCallThru(); const realPeliasConfig = require('pelias-config'); const peliasConfig = { - generate: function() { - const config = realPeliasConfig.generateDefaults(); + generate: function () { + const config = realPeliasConfig.generateDefaults(); config.addendum_namespaces = { tariff_zone_ids: { type: 'array' - } + }, + tariff_zone_authorities: { + type: 'array' + }, + public_id: { + type: 'string' + }, }; return config; } @@ -18,15 +24,15 @@ const generate = proxyquire('../../../query/autocomplete', { module.exports.tests = {}; -module.exports.tests.interface = function(test, common) { - test('valid interface', function(t) { +module.exports.tests.interface = function (test, common) { + test('valid interface', function (t) { t.equal(typeof generate, 'function', 'valid function'); t.end(); }); }; -module.exports.tests.query = function(test, common) { - test('single configured addendum namespace', function(t) { +module.exports.tests.query = function (test, common) { + test('single configured addendum namespace', function (t) { const query = generate({ text: 'something', tokens: ['something'], @@ -35,36 +41,53 @@ module.exports.tests.query = function(test, common) { tariff_zone_ids: ['TAR-123', 'TAR-345'] }); - const compiled = JSON.parse( JSON.stringify( query ) ); + const compiled = JSON.parse(JSON.stringify(query)); const expected = require('../fixture/autocomplete_configured_addendum_namespace.js'); - common.diff(compiled.body, expected); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_configured_addendum_namespace'); t.end(); }); - test('Multiple configured addendum namespace', function(t) { + test('Multiple configured addendum namespace', function (t) { + const query = generate({ + text: 'something', + tokens: ['something'], + tokens_complete: [], + tokens_incomplete: ['something'], + tariff_zone_ids: ['TAR-123', 'TAR-345'], + tariff_zone_authorities: ['TAR'], + public_id: '12345' + }); + + const compiled = JSON.parse(JSON.stringify(query)); + + const expected = require('../fixture/autocomplete_multiple_configured_addendum_namespace.js'); + + t.deepEqual(compiled.type, 'autocomplete', 'query type set'); + t.deepEqual(compiled.body, expected, 'autocomplete_multiple_configured_addendum_namespace'); + t.end(); + }); + + test('Non-configured namespace will be ignored', function (t) { const query = generate({ text: 'something', tokens: ['something'], tokens_complete: [], tokens_incomplete: ['something'], tariff_zone_ids: ['TAR-123', 'TAR-345'], - tariff_zone_authorities: ['TAR'] + something_id: 'TAR-345', }); - const compiled = JSON.parse( JSON.stringify( query ) ); + const compiled = JSON.parse(JSON.stringify(query)); const expected = require('../fixture/autocomplete_configured_addendum_namespace.js'); - common.diff(compiled.body, expected); t.deepEqual(compiled.type, 'autocomplete', 'query type set'); t.deepEqual(compiled.body, expected, 'autocomplete_configured_addendum_namespace'); t.end(); }); - }; module.exports.all = function (tape, common) { @@ -73,7 +96,7 @@ module.exports.all = function (tape, common) { return tape('autocomplete query ' + name, testFunction); } - for( const testCase in module.exports.tests ){ + for (const testCase in module.exports.tests) { module.exports.tests[testCase](test, common); } }; diff --git a/test/unit/run.js b/test/unit/run.js index 2672804d4..5d89643db 100644 --- a/test/unit/run.js +++ b/test/unit/run.js @@ -1,14 +1,14 @@ -var tape = require('tape'), - diff = require('difflet')({ indent : 2, comment : true }); +const tape = require('tape'), + diff = require('difflet')({ indent: 2, comment: true }); -var common = { +const common = { // a visual deep diff rendered using console.error() - diff: function( actual, expected ){ - console.error( diff.compare( actual, expected ) ); + diff: function (actual, expected) { + console.error(diff.compare(actual, expected)); } }; -var tests = [ +const tests = [ require('./app'), require('./schema'), require('./controller/coarse_reverse'), @@ -68,6 +68,7 @@ var tests = [ require('./query/autocomplete'), require('./query/autocomplete_token_matching_permutations'), require('./query/autocomplete_defaults'), + require('./query/autocomplete_with_configured_addendum_namespaces'), require('./query/autocomplete_with_custom_boosts'), require('./query/reverse'), require('./query/reverse_defaults'), @@ -113,6 +114,7 @@ var tests = [ require('./sanitizer/search'), require('./sanitizer/defer_to_pelias_parser'), require('./sanitizer/wrap'), + require('./sanitizer/_addendum'), require('./service/configurations/Interpolation'), require('./service/configurations/Language'), require('./service/configurations/Libpostal'), @@ -122,6 +124,6 @@ var tests = [ require('./service/search') ]; -tests.map(function(t) { +tests.map(function (t) { t.all(tape, common); }); diff --git a/test/unit/sanitizer/_addendum.js b/test/unit/sanitizer/_addendum.js new file mode 100644 index 000000000..2093b918c --- /dev/null +++ b/test/unit/sanitizer/_addendum.js @@ -0,0 +1,314 @@ +const proxyquire = require('proxyquire').noCallThru(); +const realPeliasConfig = require('pelias-config'); + +const makePeliasConfig = (addendum_namespaces) => { + const config = realPeliasConfig.generateDefaults(); + + return addendum_namespaces ? { + generate: () => ({ + ...config, + addendum_namespaces: addendum_namespaces, + get: (name) => name === 'addendum_namespaces' ? addendum_namespaces : undefined + }), + } : { + generate: () => ({ + ...config, + get: () => undefined + }), + }; +}; + +module.exports.tests = {}; + +module.exports.tests.sanitize_boundary_country = function (test, common) { + test('no addendum_namespaces config, should return no errors and warnings', function (t) { + const raw = {}; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { 'pelias-config': makePeliasConfig() }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean, {}, 'should be empty object'); + t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of any type with empty value, should return error', function (t) { + const raw = { tariff_zone_id: '' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone_id: { + type: 'string' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone_id, undefined, 'should be undefined'); + t.deepEquals( + errorsAndWarnings, + { errors: ['tariff_zone_id should be a non empty string'], warnings: [] }, + 'Empty string should not be excepted'); + t.end(); + }); + + test('configured addendum_namespace of type array, with correct raw data, should return no errors and warnings', function (t) { + const raw = { tariff_zone_ids: '1,2,3' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone_ids: { + type: 'array' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone_ids, ['1', '2', '3'], 'should be an array'); + t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type array, with only commas, should and sanitized with errors', function (t) { + const raw = { tariff_zone_ids: ',,,' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone_ids: { + type: 'array' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone_ids, undefined, 'should be undefined'); + t.deepEquals(errorsAndWarnings, { + errors: ['tariff_zone_ids should not be empty'], + warnings: [] + }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type array should be sanitized, and should return no errors and warnings', function (t) { + const raw = { tariff_zone_ids: ',1,,3' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone_ids: { + type: 'array' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone_ids, ['1', '3'], 'should be an array'); + t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type array, with the single string, sanitized and should return no errors and warnings', + function (t) { + const raw = { tariff_zone_ids: '1' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone_ids: { + type: 'array' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone_ids, ['1'], 'should be an array'); + t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type string with correct raw data, should return no errors and warnings', function (t) { + const raw = { tariff_zone_id: 'TAR-1' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone_id: { + type: 'string' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone_id, 'TAR-1', 'should be valid string'); + t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type string with extra spaces, should be trimmed', function (t) { + const raw = { tariff_zone_id: ' TAR-1 ' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone_id: { + type: 'string' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone_id, 'TAR-1', 'should be valid string'); + t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type number with correct value, should return no errors and warnings', function (t) { + const raw = { tariff_zone_id: '123' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone_id: { + type: 'number' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone_id, 123, 'should be valid number'); + t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type number with incorrect value, should return error', function (t) { + const raw = { tariff_zone_id: '123b' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone_id: { + type: 'number' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone_id, undefined, 'should be undefined'); + t.deepEquals(errorsAndWarnings, { + errors: ['tariff_zone_id: Invalid parameter type, expecting: number, got NaN: 123b'], + warnings: [] + }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type boolean with correct value, should return no errors and warnings', function (t) { + const raw = { tariff_zone: 'true' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone: { + type: 'boolean' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone, true, 'should be a valid boolean value'); + t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type boolean with incorrect value, should return error', function (t) { + const raw = { tariff_zone: 'true123' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone: { + type: 'boolean' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone, undefined, 'should be undefined'); + t.deepEquals( + errorsAndWarnings, + { errors: ['tariff_zone: Invalid parameter type, expecting: boolean, got: true123'], warnings: [] }, + 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type object with correct value, should return no errors and warnings', function (t) { + const raw = { tariff_zone: '{\"a\":\"b\"}' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone: { + type: 'object' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone, { 'a': 'b' }, 'should be valid object'); + t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type object with incorrect value, should return error', function (t) { + const raw = { tariff_zone: '{\"a\":\"b\"' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone: { + type: 'object' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone, undefined, 'should be undefined'); + t.deepEquals( + errorsAndWarnings, + { + errors: ['tariff_zone: Invalid parameter type, expecting: object, got invalid JSON: {"a":"b"'], + warnings: [] + }, + 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type object with array value, should return error', function (t) { + const raw = { tariff_zone: '[1,2,3]' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone: { + type: 'object' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone, undefined, 'should be undefined'); + t.deepEquals( + errorsAndWarnings, + { + errors: ['tariff_zone: Invalid parameter type, expecting: object, got array: [1,2,3]'], + warnings: [] + }, + 'no warnings or errors'); + t.end(); + }); + + test('configured addendum_namespace of type object with non object value, should return error', function (t) { + const raw = { tariff_zone: 'anyStringOrNumber' }; + const clean = {}; + const sanitizer = proxyquire('../../../sanitizer/_addendum', { + 'pelias-config': makePeliasConfig({ + tariff_zone: { + type: 'object' + } + }) + }); + const errorsAndWarnings = sanitizer().sanitize(raw, clean); + t.deepEquals(clean.tariff_zone, undefined, 'should be undefined'); + t.deepEquals( + errorsAndWarnings, + { + errors: ['tariff_zone: Invalid parameter type, expecting: object, got invalid JSON: anyStringOrNumber'], + warnings: [] + }, + 'no warnings or errors'); + t.end(); + }); +}; + +module.exports.all = function (tape, common) { + function test(name, testFunction) { + return tape('SANITIZE _addendum ' + name, testFunction); + } + + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common); + } +}; From a83aa62c7ca68afa516407c82ee1d508501818d6 Mon Sep 17 00:00:00 2001 From: "mansoor.sajjad" Date: Thu, 24 Nov 2022 11:38:29 +0100 Subject: [PATCH 3/8] Fixing the geojsonify and unit tests. --- helper/geojsonify.js | 54 +++++--- sanitizer/_addendum.js | 8 +- test/unit/helper/geojsonify.js | 203 +++++++++++++++++++++++++------ test/unit/sanitizer/_addendum.js | 4 +- 4 files changed, 204 insertions(+), 65 deletions(-) diff --git a/helper/geojsonify.js b/helper/geojsonify.js index ef69a6f47..14fddd05e 100644 --- a/helper/geojsonify.js +++ b/helper/geojsonify.js @@ -8,8 +8,9 @@ const codec = require('pelias-model').codec; const field = require('./fieldValue'); const decode_gid = require('./decode_gid'); const iso3166 = require('./iso3166'); +const peliasConfig = require('pelias-config').generate(); -function geojsonifyPlaces( params, docs ){ +function geojsonifyPlaces(params, docs) { // flatten & expand data for geojson conversion const geodata = docs @@ -28,8 +29,8 @@ function geojsonifyPlaces( params, docs ){ const extentPoints = extractExtentPoints(geodata); // convert to geojson - const geojson = GeoJSON.parse( geodata, { Point: ['lat', 'lng'] }); - const geojsonExtentPoints = GeoJSON.parse( extentPoints, { Point: ['lat', 'lng'] }); + const geojson = GeoJSON.parse(geodata, { Point: ['lat', 'lng'] }); + const geojsonExtentPoints = GeoJSON.parse(extentPoints, { Point: ['lat', 'lng'] }); // to insert the bbox property at the top level of each feature, it must be done separately after // initial geojson construction is finished @@ -72,16 +73,26 @@ function geojsonifyPlace(params, place) { // add addendum data if available // note: this should be the last assigned property, for aesthetic reasons. if (_.has(place, 'addendum')) { - let addendum = {}; - for(let namespace in place.addendum){ - try { - addendum[namespace] = codec.decode(place.addendum[namespace]); - } catch( e ){ - logger.warn(`doc ${doc.gid} failed to decode addendum namespace ${namespace}`); + const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces'); + let nonConfiguredNamespaces = {}; + let configuredNamespaces = {}; + for (const namespace in place.addendum) { + const isConfigured = configuredAddendumNamespaces[namespace]; + if (isConfigured) { + configuredNamespaces[namespace] = place.addendum[namespace]; + } else { + try { + nonConfiguredNamespaces[namespace] = codec.decode(place.addendum[namespace]); + } catch (e) { + logger.warn(`doc ${doc.gid} failed to decode addendum namespace ${namespace}`); + } } } - if( Object.keys(addendum).length ){ - doc.addendum = addendum; + if (Object.keys(nonConfiguredNamespaces).length) { + doc.addendum = nonConfiguredNamespaces; + } + for (let namespace in configuredNamespaces) { + doc[namespace] = configuredNamespaces[namespace]; } } @@ -133,8 +144,7 @@ function extractExtentPoints(geodata) { lat: place.bounding_box.max_lat }); - } - else { + } else { // otherwise, use the point for the extent extentPoints.push({ lng: place.lng, @@ -158,13 +168,13 @@ function computeBBox(geojson, geojsonExtentPoints) { // @note: extent() sometimes throws Errors for unusual data // eg: https://github.com/pelias/pelias/issues/84 try { - var bbox = extent( geojsonExtentPoints ); - if( !!bbox ){ + const bbox = extent(geojsonExtentPoints); + if (!!bbox) { geojson.bbox = bbox; } - } catch( e ){ - logger.error( 'bbox error', e.message, e.stack ); - logger.error( 'geojson', geojsonExtentPoints ); + } catch (e) { + logger.error('bbox error', e.message, e.stack); + logger.error('geojson', geojsonExtentPoints); } } @@ -176,11 +186,15 @@ function computeBBox(geojson, geojsonExtentPoints) { function addISO3166PropsPerFeature(geojson) { geojson.features.forEach(feature => { let code = _.get(feature, 'properties.country_a') || _.get(feature, 'properties.dependency_a') || ''; - if (!_.isString(code) || _.isEmpty(code)){ return; } + if (!_.isString(code) || _.isEmpty(code)) { + return; + } let info = iso3166.info(code); let alpha2 = _.get(info, 'alpha2'); - if (!_.isString(alpha2) || _.size(alpha2) !== 2) { return; } + if (!_.isString(alpha2) || _.size(alpha2) !== 2) { + return; + } _.set(feature, 'properties.country_code', alpha2); }); diff --git a/sanitizer/_addendum.js b/sanitizer/_addendum.js index 22b91e219..376137aa5 100644 --- a/sanitizer/_addendum.js +++ b/sanitizer/_addendum.js @@ -1,7 +1,8 @@ const _ = require('lodash'); -const nonEmptyString = (v) => _.isString(v) && !_.isEmpty(v); const peliasConfig = require('pelias-config').generate(); +const nonEmptyString = (v) => _.isString(v) && !_.isEmpty(v); + function _sanitize(raw, clean) { // error & warning messages @@ -67,9 +68,8 @@ function _sanitize(raw, clean) { break; default: - if (typeof rawValue !== validationType) { - messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got: ' + rawValue); - } + messages.errors.push(namespace + ': Unsupported configured type ' + validationType + ', ' + + 'valid types are array, string, number, boolean and object'); } } }); diff --git a/test/unit/helper/geojsonify.js b/test/unit/helper/geojsonify.js index 69e08999f..903c869bf 100644 --- a/test/unit/helper/geojsonify.js +++ b/test/unit/helper/geojsonify.js @@ -1,11 +1,29 @@ const geojsonify = require('../../../helper/geojsonify'); const proxyquire = require('proxyquire').noCallThru(); const codec = require('pelias-model').codec; +const peliasConfig = require('pelias-config'); + +const makePeliasConfig = (addendum_namespaces) => { + const config = peliasConfig.generateDefaults(); + + return addendum_namespaces ? { + generate: () => ({ + ...config, + addendum_namespaces: addendum_namespaces, + get: (name) => name === 'addendum_namespaces' ? addendum_namespaces : undefined + }), + } : { + generate: () => ({ + ...config, + get: () => undefined + }), + }; +}; module.exports.tests = {}; -module.exports.tests.interface = function(test, common) { - test('valid interface', function(t) { +module.exports.tests.interface = function (test, common) { + test('valid interface', function (t) { t.equal(typeof geojsonify, 'function', 'geojsonify is a function'); t.equal(geojsonify.length, 2, 'accepts x arguments'); t.end(); @@ -14,9 +32,9 @@ module.exports.tests.interface = function(test, common) { // ensure null island coordinates work // ref: https://github.com/pelias/pelias/issues/84 -module.exports.tests.earth = function(test, common) { - test('earth', function(t) { - var earth = [{ +module.exports.tests.earth = function (test, common) { + test('earth', function (t) { + const earth = [{ '_type': 'doc', '_id': 'whosonfirst:continent:6295630', 'source': 'whosonfirst', @@ -30,8 +48,8 @@ module.exports.tests.earth = function(test, common) { } }]; - t.doesNotThrow(function(){ - geojsonify( {}, earth ); + t.doesNotThrow(function () { + geojsonify({}, earth); }); t.end(); }); @@ -93,7 +111,7 @@ module.exports.tests.bounding_box = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 21.212121, 12.121212 ] + coordinates: [21.212121, 12.121212] }, properties: { id: 'id 1', @@ -111,7 +129,7 @@ module.exports.tests.bounding_box = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 31.313131, 13.131313 ] + coordinates: [31.313131, 13.131313] }, properties: { id: 'id 2', @@ -200,7 +218,7 @@ module.exports.tests.bounding_box = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 21.212121, 12.121212 ] + coordinates: [21.212121, 12.121212] }, properties: { id: 'id 1', @@ -213,13 +231,13 @@ module.exports.tests.bounding_box = (test, common) => { property1: 'property 1', property2: 'property 2' }, - bbox: [ 1, 1, 2, 2 ] + bbox: [1, 1, 2, 2] }, { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 31.313131, 13.131313 ] + coordinates: [31.313131, 13.131313] }, properties: { id: 'id 2', @@ -232,10 +250,10 @@ module.exports.tests.bounding_box = (test, common) => { property3: 'property 3', property4: 'property 4' }, - bbox: [ -3, -3, -1, -1 ] + bbox: [-3, -3, -1, -1] } ], - bbox: [ -3, -3, 2, 2 ] + bbox: [-3, -3, 2, 2] }; t.deepEquals(actual, expected); @@ -304,7 +322,7 @@ module.exports.tests.bounding_box = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 21.212121, 12.121212 ] + coordinates: [21.212121, 12.121212] }, properties: { id: 'id 1', @@ -322,7 +340,7 @@ module.exports.tests.bounding_box = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 31.313131, 13.131313 ] + coordinates: [31.313131, 13.131313] }, properties: { id: 'id 2', @@ -335,10 +353,10 @@ module.exports.tests.bounding_box = (test, common) => { property3: 'property 3', property4: 'property 4' }, - bbox: [ -3, -3, -1, -1 ] + bbox: [-3, -3, -1, -1] } ], - bbox: [ -3, -3, 21.212121, 12.121212 ] + bbox: [-3, -3, 21.212121, 12.121212] }; t.deepEquals(actual, expected); @@ -390,7 +408,7 @@ module.exports.tests.non_optimal_conditions = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 21.212121, 12.121212 ] + coordinates: [21.212121, 12.121212] }, properties: { id: 'id 1', @@ -461,7 +479,7 @@ module.exports.tests.non_optimal_conditions = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 31.313131, 13.131313 ] + coordinates: [31.313131, 13.131313] }, properties: { id: 'id 2', @@ -554,7 +572,7 @@ module.exports.tests.non_optimal_conditions = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 21.212121, 12.121212 ] + coordinates: [21.212121, 12.121212] }, properties: { id: 'id 1', @@ -571,7 +589,7 @@ module.exports.tests.non_optimal_conditions = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 31.313131, 13.131313 ] + coordinates: [31.313131, 13.131313] }, properties: { id: 'id 2', @@ -588,7 +606,7 @@ module.exports.tests.non_optimal_conditions = (test, common) => { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 41.414141, 14.141414 ] + coordinates: [41.414141, 14.141414] }, properties: { id: 'id 3', @@ -641,9 +659,9 @@ module.exports.tests.non_optimal_conditions = (test, common) => { // ensure that if elasticsearch returns an array of values for name.default // .. that we handle this case and select the first element for the label. -module.exports.tests.nameAliases = function(test, common) { - test('name aliases', function(t) { - var aliases = [{ +module.exports.tests.nameAliases = function (test, common) { + test('name aliases', function (t) { + const aliases = [{ '_id': 'example:example:1', 'source': 'example', 'layer': 'example', @@ -662,7 +680,7 @@ module.exports.tests.nameAliases = function(test, common) { type: 'Feature', geometry: { type: 'Point', - coordinates: [ 0, 0 ] + coordinates: [0, 0] }, properties: { id: '1', @@ -674,10 +692,10 @@ module.exports.tests.nameAliases = function(test, common) { name: 'Example1' } }], - bbox: [ 0, 0, 0, 0 ] + bbox: [0, 0, 0, 0] }; - var actual = geojsonify( {}, aliases ); + const actual = geojsonify({}, aliases); t.deepEquals(actual, expected); t.end(); }); @@ -685,9 +703,9 @@ module.exports.tests.nameAliases = function(test, common) { }; // ensure addendums aree decoded and printed properly -module.exports.tests.addendum = function(test, common) { - test('addendum: not set in source', function(t) { - var example = [{ +module.exports.tests.addendum = function (test, common) { + test('addendum: not set in source', function (t) { + const example = [{ '_id': 'whosonfirst:continent:6295630', 'source': 'whosonfirst', 'layer': 'continent', @@ -705,8 +723,8 @@ module.exports.tests.addendum = function(test, common) { t.end(); }); - test('addendum: set in source', function(t) { - var example = [{ + test('addendum: set in source', function (t) { + const example = [{ '_id': 'whosonfirst:continent:6295630', 'source': 'whosonfirst', 'layer': 'continent', @@ -731,8 +749,8 @@ module.exports.tests.addendum = function(test, common) { t.end(); }); - test('addendum: partially corrupted', function(t) { - var example = [{ + test('addendum: partially corrupted', function (t) { + const example = [{ '_id': 'whosonfirst:continent:6295630', 'source': 'whosonfirst', 'layer': 'continent', @@ -755,8 +773,9 @@ module.exports.tests.addendum = function(test, common) { }); t.end(); }); - test('addendum: all corrupted', function(t) { - var example = [{ + + test('addendum: all corrupted', function (t) { + const example = [{ '_id': 'whosonfirst:continent:6295630', 'source': 'whosonfirst', 'layer': 'continent', @@ -778,6 +797,112 @@ module.exports.tests.addendum = function(test, common) { t.false(collection.features[0].properties.addendum); t.end(); }); + + test('addendum: only configured addendum namespaces', function (t) { + const example = [{ + '_id': 'whosonfirst:continent:6295630', + 'source': 'whosonfirst', + 'layer': 'continent', + 'name': { + 'default': 'Earth' + }, + 'center_point': { + 'lon': 0, + 'lat': 0 + }, + 'addendum': { + 'wikipedia': { slug: 'HackneyCityFarm' }, + 'geonames': ['name1', 'name2'] + } + }]; + + const geojsonify = proxyquire('../../../helper/geojsonify', { + 'pelias-config': makePeliasConfig({ + wikipedia: { + type: 'object' + }, + geonames: { + type: 'array' + } + }) + }); + + let collection = geojsonify({}, example); + t.deepEqual(collection.features[0].properties.wikipedia, { slug: 'HackneyCityFarm' }); + t.deepEqual(collection.features[0].properties.geonames, ['name1', 'name2']); + t.end(); + }); + + test('addendum: mix of configured and non-configured addendum namespaces', function (t) { + const example = [{ + '_id': 'whosonfirst:continent:6295630', + 'source': 'whosonfirst', + 'layer': 'continent', + 'name': { + 'default': 'Earth' + }, + 'center_point': { + 'lon': 0, + 'lat': 0 + }, + 'addendum': { + 'description': '{\"nor\":\"i Bogstadveien\"}', + 'wikipedia': { slug: 'HackneyCityFarm' }, + 'geonames': ['name1', 'name2'] + } + }]; + + const geojsonify = proxyquire('../../../helper/geojsonify', { + 'pelias-config': makePeliasConfig({ + wikipedia: { + type: 'object' + }, + geonames: { + type: 'array' + } + }) + }); + + let collection = geojsonify({}, example); + t.deepEqual(collection.features[0].properties.wikipedia, { slug: 'HackneyCityFarm' }); + t.deepEqual(collection.features[0].properties.geonames, ['name1', 'name2']); + t.deepEqual(collection.features[0].properties.addendum, { + description: { nor: 'i Bogstadveien' } + }); + t.end(); + }); + + test('addendum: removing the configuration for the previously configured namespace should be ignored with warning', function (t) { + const example = [{ + '_id': 'whosonfirst:continent:6295630', + 'source': 'whosonfirst', + 'layer': 'continent', + 'name': { + 'default': 'Earth' + }, + 'center_point': { + 'lon': 0, + 'lat': 0 + }, + 'addendum': { + 'wikipedia': { slug: 'HackneyCityFarm' }, + 'geonames': ['name1', 'name2'] + } + }]; + + const geojsonify = proxyquire('../../../helper/geojsonify', { + 'pelias-config': makePeliasConfig({ + geonames: { + type: 'array' + } + }) + }); + + let collection = geojsonify({}, example); + t.deepEqual(collection.features[0].properties.wikipedia, undefined); + t.deepEqual(collection.features[0].properties.geonames, ['name1', 'name2']); + t.end(); + }); }; // iso3166 country code info should be calculated when avaiable @@ -857,7 +982,7 @@ module.exports.all = (tape, common) => { return tape(`geojsonify: ${name}`, testFunction); } - for( var testCase in module.exports.tests ){ + for (const testCase in module.exports.tests) { module.exports.tests[testCase](test, common); } }; diff --git a/test/unit/sanitizer/_addendum.js b/test/unit/sanitizer/_addendum.js index 2093b918c..e6c28c7bb 100644 --- a/test/unit/sanitizer/_addendum.js +++ b/test/unit/sanitizer/_addendum.js @@ -1,8 +1,8 @@ const proxyquire = require('proxyquire').noCallThru(); -const realPeliasConfig = require('pelias-config'); +const peliasConfig = require('pelias-config'); const makePeliasConfig = (addendum_namespaces) => { - const config = realPeliasConfig.generateDefaults(); + const config = peliasConfig.generateDefaults(); return addendum_namespaces ? { generate: () => ({ From ffc8fd7a0307f01ccb4676f166bb8793c2bb0c0b Mon Sep 17 00:00:00 2001 From: "mansoor.sajjad" Date: Mon, 28 Nov 2022 18:41:20 +0100 Subject: [PATCH 4/8] Adding the config validation for addendum_namespaces --- query/autocomplete.js | 5 +- sanitizer/_addendum.js | 19 +--- sanitizer/nearby.js | 3 +- sanitizer/reverse.js | 3 +- sanitizer/search.js | 3 +- sanitizer/structured_geocoding.js | 3 +- schema.js | 7 +- test/unit/sanitizer/_addendum.js | 75 +------------ test/unit/schema.js | 180 ++++++++++++++++++++++++++++-- 9 files changed, 196 insertions(+), 102 deletions(-) diff --git a/query/autocomplete.js b/query/autocomplete.js index 88fcabfbd..88da96c16 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -5,7 +5,6 @@ const textParser = require('./text_parser_pelias'); const config = require('pelias-config').generate(); const placeTypes = require('../helper/placeTypes'); const toSingleField = require('./view/helper').toSingleField; -const peliasConfig = require('pelias-config').generate(); // additional views (these may be merged in to pelias/query at a later date) var views = { @@ -67,7 +66,7 @@ query.filter( peliasQuery.view.categories ); query.filter( peliasQuery.view.boundary_gid ); query.filter( views.focus_point_filter ); -const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces'); +const configuredAddendumNamespaces = config.get('addendum_namespaces'); Object.keys(configuredAddendumNamespaces).forEach(namespace => { query.filter( views.addendum_namespace_filter(namespace) ); }); @@ -83,7 +82,7 @@ function generateQuery( clean ){ const vs = new peliasQuery.Vars( defaults ); //addendum - const configuredAddendumNamespaces = peliasConfig.get('addendum_namespaces'); + const configuredAddendumNamespaces = config.get('addendum_namespaces'); Object.keys(configuredAddendumNamespaces) .filter(namespace => clean[namespace]) .forEach(namespace => { diff --git a/sanitizer/_addendum.js b/sanitizer/_addendum.js index 376137aa5..13eaa96a5 100644 --- a/sanitizer/_addendum.js +++ b/sanitizer/_addendum.js @@ -50,26 +50,9 @@ function _sanitize(raw, clean) { } break; - case 'object': - try { - const parsed = JSON.parse(rawValue); - if (!_.isObject(parsed)) { - messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got ' + rawValue); - } else if (_.isArray(parsed)) { - messages.errors.push(namespace + ': Invalid parameter type, expecting: ' + validationType + ', got array: ' + rawValue); - } else { - clean[namespace] = parsed; - } - } catch (e) { - messages.errors.push( - namespace + ': Invalid parameter type, expecting: ' + validationType + ', got invalid JSON: ' + rawValue - ); - } - break; - default: messages.errors.push(namespace + ': Unsupported configured type ' + validationType + ', ' + - 'valid types are array, string, number, boolean and object'); + 'valid types are array, string, number and boolean'); } } }); diff --git a/sanitizer/nearby.js b/sanitizer/nearby.js index 182c40f8c..ffe33e7ba 100644 --- a/sanitizer/nearby.js +++ b/sanitizer/nearby.js @@ -37,7 +37,8 @@ module.exports.middleware = (_api_pelias_config) => { geo_reverse: require('../sanitizer/_geo_reverse')(), boundary_country: require('../sanitizer/_boundary_country')(), categories: require('../sanitizer/_categories')(), - request_language: require('../sanitizer/_request_language')() + request_language: require('../sanitizer/_request_language')(), + addendum: require('../sanitizer/_addendum')() }; return function( req, res, next ){ diff --git a/sanitizer/reverse.js b/sanitizer/reverse.js index 33cfe28ba..342608370 100644 --- a/sanitizer/reverse.js +++ b/sanitizer/reverse.js @@ -37,7 +37,8 @@ module.exports.middleware = (_api_pelias_config) => { geo_reverse: require('../sanitizer/_geo_reverse')(), boundary_country: require('../sanitizer/_boundary_country')(), request_language: require('../sanitizer/_request_language')(), - boundary_gid: require('../sanitizer/_boundary_gid')() + boundary_gid: require('../sanitizer/_boundary_gid')(), + addendum: require('../sanitizer/_addendum')() }; // middleware diff --git a/sanitizer/search.js b/sanitizer/search.js index 9faf84b98..42fecea24 100644 --- a/sanitizer/search.js +++ b/sanitizer/search.js @@ -45,7 +45,8 @@ module.exports.middleware = (_api_pelias_config) => { // this can go away once geonames has been abrogated geonames_warnings: require('../sanitizer/_geonames_warnings')(), request_language: require('../sanitizer/_request_language')(), - boundary_gid: require('../sanitizer/_boundary_gid')() + boundary_gid: require('../sanitizer/_boundary_gid')(), + addendum: require('../sanitizer/_addendum')() }; return ( req, res, next ) => { diff --git a/sanitizer/structured_geocoding.js b/sanitizer/structured_geocoding.js index 4a8ddd640..6d025bca4 100644 --- a/sanitizer/structured_geocoding.js +++ b/sanitizer/structured_geocoding.js @@ -51,7 +51,8 @@ module.exports.middleware = (_api_pelias_config) => { boundary_country: require('../sanitizer/_boundary_country')(), categories: require('../sanitizer/_categories')(), request_language: require('../sanitizer/_request_language')(), - boundary_gid: require('../sanitizer/_boundary_gid')() + boundary_gid: require('../sanitizer/_boundary_gid')(), + addendum: require('../sanitizer/_addendum')() }; return ( req, res, next ) => { diff --git a/schema.js b/schema.js index ea0f373ed..3cdb52351 100644 --- a/schema.js +++ b/schema.js @@ -61,5 +61,10 @@ module.exports = Joi.object().keys({ }).unknown(true), esclient: Joi.object().required().keys({ requestTimeout: Joi.number().integer().min(0) - }).unknown(true) + }).unknown(true), + addendum_namespaces: Joi.object().pattern( + Joi.string().min(2), Joi.object().required().keys({ + type: Joi.string().valid('array', 'number', 'string', 'boolean').required() + }) + ) }).unknown(true); diff --git a/test/unit/sanitizer/_addendum.js b/test/unit/sanitizer/_addendum.js index e6c28c7bb..d25b247da 100644 --- a/test/unit/sanitizer/_addendum.js +++ b/test/unit/sanitizer/_addendum.js @@ -20,7 +20,7 @@ const makePeliasConfig = (addendum_namespaces) => { module.exports.tests = {}; -module.exports.tests.sanitize_boundary_country = function (test, common) { +module.exports.tests.sanitize_boundary_country = function (test) { test('no addendum_namespaces config, should return no errors and warnings', function (t) { const raw = {}; const clean = {}; @@ -181,7 +181,7 @@ module.exports.tests.sanitize_boundary_country = function (test, common) { t.deepEquals(errorsAndWarnings, { errors: ['tariff_zone_id: Invalid parameter type, expecting: number, got NaN: 123b'], warnings: [] - }, 'no warnings or errors'); + }, 'should be a error'); t.end(); }); @@ -216,11 +216,11 @@ module.exports.tests.sanitize_boundary_country = function (test, common) { t.deepEquals( errorsAndWarnings, { errors: ['tariff_zone: Invalid parameter type, expecting: boolean, got: true123'], warnings: [] }, - 'no warnings or errors'); + 'should be a error'); t.end(); }); - test('configured addendum_namespace of type object with correct value, should return no errors and warnings', function (t) { + test('configured addendum_namespace of any type other than array, string, number and boolean, should return error', function (t) { const raw = { tariff_zone: '{\"a\":\"b\"}' }; const clean = {}; const sanitizer = proxyquire('../../../sanitizer/_addendum', { @@ -231,74 +231,11 @@ module.exports.tests.sanitize_boundary_country = function (test, common) { }) }); const errorsAndWarnings = sanitizer().sanitize(raw, clean); - t.deepEquals(clean.tariff_zone, { 'a': 'b' }, 'should be valid object'); - t.deepEquals(errorsAndWarnings, { errors: [], warnings: [] }, 'no warnings or errors'); - t.end(); - }); - - test('configured addendum_namespace of type object with incorrect value, should return error', function (t) { - const raw = { tariff_zone: '{\"a\":\"b\"' }; - const clean = {}; - const sanitizer = proxyquire('../../../sanitizer/_addendum', { - 'pelias-config': makePeliasConfig({ - tariff_zone: { - type: 'object' - } - }) - }); - const errorsAndWarnings = sanitizer().sanitize(raw, clean); - t.deepEquals(clean.tariff_zone, undefined, 'should be undefined'); - t.deepEquals( - errorsAndWarnings, - { - errors: ['tariff_zone: Invalid parameter type, expecting: object, got invalid JSON: {"a":"b"'], - warnings: [] - }, - 'no warnings or errors'); - t.end(); - }); - - test('configured addendum_namespace of type object with array value, should return error', function (t) { - const raw = { tariff_zone: '[1,2,3]' }; - const clean = {}; - const sanitizer = proxyquire('../../../sanitizer/_addendum', { - 'pelias-config': makePeliasConfig({ - tariff_zone: { - type: 'object' - } - }) - }); - const errorsAndWarnings = sanitizer().sanitize(raw, clean); - t.deepEquals(clean.tariff_zone, undefined, 'should be undefined'); - t.deepEquals( - errorsAndWarnings, - { - errors: ['tariff_zone: Invalid parameter type, expecting: object, got array: [1,2,3]'], - warnings: [] - }, - 'no warnings or errors'); - t.end(); - }); - - test('configured addendum_namespace of type object with non object value, should return error', function (t) { - const raw = { tariff_zone: 'anyStringOrNumber' }; - const clean = {}; - const sanitizer = proxyquire('../../../sanitizer/_addendum', { - 'pelias-config': makePeliasConfig({ - tariff_zone: { - type: 'object' - } - }) - }); - const errorsAndWarnings = sanitizer().sanitize(raw, clean); t.deepEquals(clean.tariff_zone, undefined, 'should be undefined'); t.deepEquals( errorsAndWarnings, - { - errors: ['tariff_zone: Invalid parameter type, expecting: object, got invalid JSON: anyStringOrNumber'], - warnings: [] - }, - 'no warnings or errors'); + { errors: ['tariff_zone: Unsupported configured type object, valid types are array, string, number and boolean'], warnings: [] }, + 'should be a errors'); t.end(); }); }; diff --git a/test/unit/schema.js b/test/unit/schema.js index 2ea9a38a1..420fd002b 100644 --- a/test/unit/schema.js +++ b/test/unit/schema.js @@ -1,10 +1,8 @@ -const Joi = require('@hapi/joi'); const schema = require('../../schema'); -const _ = require('lodash'); module.exports.tests = {}; -module.exports.tests.completely_valid = (test, common) => { +module.exports.tests.completely_valid = (test) => { test('all valid configuration elements should not throw error', (t) => { var config = { api: { @@ -70,7 +68,7 @@ module.exports.tests.completely_valid = (test, common) => { }; -module.exports.tests.api_validation = (test, common) => { +module.exports.tests.api_validation = (test) => { test('config without api should throw error', (t) => { var config = { esclient: {} @@ -520,7 +518,7 @@ module.exports.tests.api_validation = (test, common) => { }; -module.exports.tests.api_services_validation = (test, common) => { +module.exports.tests.api_services_validation = (test) => { test('unsupported children of api.services should be disallowed', (t) => { var config = { api: { @@ -544,7 +542,7 @@ module.exports.tests.api_services_validation = (test, common) => { }; -module.exports.tests.service_validation = (test, common) => { +module.exports.tests.service_validation = (test) => { // these tests apply for all the individual service definitions const services = ['pip', 'placeholder', 'interpolation', 'libpostal']; @@ -865,7 +863,7 @@ module.exports.tests.service_validation = (test, common) => { }; -module.exports.tests.esclient_validation = (test, common) => { +module.exports.tests.esclient_validation = (test) => { test('config without esclient should throw error', (t) => { var config = { api: { @@ -971,6 +969,174 @@ module.exports.tests.esclient_validation = (test, common) => { }; +module.exports.tests.addendum_namespaces_validation = (test) => { + test( 'addendum_namespaces should be of object type', (t) => { + const config = { + api: { + version: 'version value', + indexName: 'index name value', + host: 'host value' + }, + addendum_namespaces: { + tariff_zone_ids: '123' + }, + esclient: { + requestTimeout: 1 + } + }; + + const result = schema.validate(config); + + t.equals(result.error.details.length, 1); + t.equals(result.error.details[0].message, '"addendum_namespaces.tariff_zone_ids" must be of type object'); + t.end(); + + }); + + test( 'addendum_namespaces of type other than array, string , boolean and number, should not be acceptable', (t) => { + const config = { + api: { + version: 'version value', + indexName: 'index name value', + host: 'host value' + }, + addendum_namespaces: { + tariff_zone_ids: { + type: 'object' + } + }, + esclient: { + requestTimeout: 1 + } + }; + + const result = schema.validate(config); + + t.equals(result.error.details.length, 1); + t.equals(result.error.details[0].message, '"addendum_namespaces.tariff_zone_ids.type" must be one of [array, number, string, boolean]'); + t.end(); + + }); + + test( 'addendum_namespaces name should be at least 2 characters', (t) => { + const config = { + api: { + version: 'version value', + indexName: 'index name value', + host: 'host value' + }, + addendum_namespaces: { + t: { + type: 'object' + } + }, + esclient: { + requestTimeout: 1 + } + }; + + const result = schema.validate(config); + + t.equals(result.error.details.length, 1); + t.equals(result.error.details[0].message, '"addendum_namespaces.t" is not allowed'); + t.end(); + }); + + test( 'addendum_namespaces of type array should be acceptable', (t) => { + const config = { + api: { + version: 'version value', + indexName: 'index name value', + host: 'host value' + }, + addendum_namespaces: { + tariff_zone_ids: { + type: 'array' + } + }, + esclient: { + requestTimeout: 1 + } + }; + + const result = schema.validate(config); + + t.notOk(result.error); + t.end(); + + }); + + test( 'addendum_namespaces of type string should be acceptable', (t) => { + const config = { + api: { + version: 'version value', + indexName: 'index name value', + host: 'host value' + }, + addendum_namespaces: { + tariff_zone_ids: { + type: 'string' + } + }, + esclient: { + requestTimeout: 1 + } + }; + + const result = schema.validate(config); + + t.notOk(result.error); + t.end(); + }); + + test( 'addendum_namespaces of type number should be acceptable', function(t) { + const config = { + api: { + version: 'version value', + indexName: 'index name value', + host: 'host value' + }, + addendum_namespaces: { + tariff_zone_ids: { + type: 'number' + } + }, + esclient: { + requestTimeout: 1 + } + }; + + const result = schema.validate(config); + + t.notOk(result.error); + t.end(); + }); + + test( 'addendum_namespaces of type boolean should be acceptable', function(t) { + const config = { + api: { + version: 'version value', + indexName: 'index name value', + host: 'host value' + }, + addendum_namespaces: { + tariff_zone_ids: { + type: 'boolean' + } + }, + esclient: { + requestTimeout: 1 + } + }; + + const result = schema.validate(config); + + t.notOk(result.error); + t.end(); + }); + +}; + module.exports.all = (tape, common) => { function test(name, testFunction) { From 1ce17ba0aa9b2e4fcb183d2b9849c7df340071fe Mon Sep 17 00:00:00 2001 From: "mansoor.sajjad" Date: Tue, 29 Nov 2022 11:22:16 +0100 Subject: [PATCH 5/8] Moving the config validation for addendum_namespaces into pelias-config --- schema.js | 7 +- test/unit/schema.js | 169 -------------------------------------------- 2 files changed, 1 insertion(+), 175 deletions(-) diff --git a/schema.js b/schema.js index 3cdb52351..ea0f373ed 100644 --- a/schema.js +++ b/schema.js @@ -61,10 +61,5 @@ module.exports = Joi.object().keys({ }).unknown(true), esclient: Joi.object().required().keys({ requestTimeout: Joi.number().integer().min(0) - }).unknown(true), - addendum_namespaces: Joi.object().pattern( - Joi.string().min(2), Joi.object().required().keys({ - type: Joi.string().valid('array', 'number', 'string', 'boolean').required() - }) - ) + }).unknown(true) }).unknown(true); diff --git a/test/unit/schema.js b/test/unit/schema.js index 420fd002b..23a5bb0ec 100644 --- a/test/unit/schema.js +++ b/test/unit/schema.js @@ -968,175 +968,6 @@ module.exports.tests.esclient_validation = (test) => { }); }; - -module.exports.tests.addendum_namespaces_validation = (test) => { - test( 'addendum_namespaces should be of object type', (t) => { - const config = { - api: { - version: 'version value', - indexName: 'index name value', - host: 'host value' - }, - addendum_namespaces: { - tariff_zone_ids: '123' - }, - esclient: { - requestTimeout: 1 - } - }; - - const result = schema.validate(config); - - t.equals(result.error.details.length, 1); - t.equals(result.error.details[0].message, '"addendum_namespaces.tariff_zone_ids" must be of type object'); - t.end(); - - }); - - test( 'addendum_namespaces of type other than array, string , boolean and number, should not be acceptable', (t) => { - const config = { - api: { - version: 'version value', - indexName: 'index name value', - host: 'host value' - }, - addendum_namespaces: { - tariff_zone_ids: { - type: 'object' - } - }, - esclient: { - requestTimeout: 1 - } - }; - - const result = schema.validate(config); - - t.equals(result.error.details.length, 1); - t.equals(result.error.details[0].message, '"addendum_namespaces.tariff_zone_ids.type" must be one of [array, number, string, boolean]'); - t.end(); - - }); - - test( 'addendum_namespaces name should be at least 2 characters', (t) => { - const config = { - api: { - version: 'version value', - indexName: 'index name value', - host: 'host value' - }, - addendum_namespaces: { - t: { - type: 'object' - } - }, - esclient: { - requestTimeout: 1 - } - }; - - const result = schema.validate(config); - - t.equals(result.error.details.length, 1); - t.equals(result.error.details[0].message, '"addendum_namespaces.t" is not allowed'); - t.end(); - }); - - test( 'addendum_namespaces of type array should be acceptable', (t) => { - const config = { - api: { - version: 'version value', - indexName: 'index name value', - host: 'host value' - }, - addendum_namespaces: { - tariff_zone_ids: { - type: 'array' - } - }, - esclient: { - requestTimeout: 1 - } - }; - - const result = schema.validate(config); - - t.notOk(result.error); - t.end(); - - }); - - test( 'addendum_namespaces of type string should be acceptable', (t) => { - const config = { - api: { - version: 'version value', - indexName: 'index name value', - host: 'host value' - }, - addendum_namespaces: { - tariff_zone_ids: { - type: 'string' - } - }, - esclient: { - requestTimeout: 1 - } - }; - - const result = schema.validate(config); - - t.notOk(result.error); - t.end(); - }); - - test( 'addendum_namespaces of type number should be acceptable', function(t) { - const config = { - api: { - version: 'version value', - indexName: 'index name value', - host: 'host value' - }, - addendum_namespaces: { - tariff_zone_ids: { - type: 'number' - } - }, - esclient: { - requestTimeout: 1 - } - }; - - const result = schema.validate(config); - - t.notOk(result.error); - t.end(); - }); - - test( 'addendum_namespaces of type boolean should be acceptable', function(t) { - const config = { - api: { - version: 'version value', - indexName: 'index name value', - host: 'host value' - }, - addendum_namespaces: { - tariff_zone_ids: { - type: 'boolean' - } - }, - esclient: { - requestTimeout: 1 - } - }; - - const result = schema.validate(config); - - t.notOk(result.error); - t.end(); - }); - -}; - module.exports.all = (tape, common) => { function test(name, testFunction) { From 976ce17129b7bb51bf13b3cfa14348027d5f0ba0 Mon Sep 17 00:00:00 2001 From: "mansoor.sajjad" Date: Tue, 29 Nov 2022 14:11:14 +0100 Subject: [PATCH 6/8] Updating the queries to use the addendum filter. --- query/reverse.js | 17 ++++++++++++++++- query/search.js | 22 +++++++++++++++++----- query/structured_geocoding.js | 21 ++++++++++++++++----- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/query/reverse.js b/query/reverse.js index 6f145680b..6909e9906 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -1,6 +1,8 @@ const _ = require('lodash'); const peliasQuery = require('pelias-query'); const defaults = require('./reverse_defaults'); +const config = require('pelias-config').generate(); +const addendum_namespace_filter = require('./view/addendum_namespace_filter'); //------------------------------ // reverse geocode query @@ -21,12 +23,25 @@ query.filter( peliasQuery.view.categories ); query.filter( peliasQuery.view.boundary_country ); query.filter( peliasQuery.view.boundary_gid ); +const configuredAddendumNamespaces = config.get('addendum_namespaces'); +Object.keys(configuredAddendumNamespaces).forEach(namespace => { + query.filter( addendum_namespace_filter(namespace) ); +}); + // -------------------------------- function generateQuery( clean ){ const vs = new peliasQuery.Vars( defaults ); + //addendum + const configuredAddendumNamespaces = config.get('addendum_namespaces'); + Object.keys(configuredAddendumNamespaces) + .filter(namespace => clean[namespace]) + .forEach(namespace => { + vs.var( namespace, clean[namespace] ); + }); + // set size if( clean.querySize ){ vs.var( 'size', clean.querySize); @@ -55,7 +70,7 @@ function generateQuery( clean ){ // bounding circle // note: the sanitizers will take care of the case // where point.lan/point.lon are provided in the - // absense of boundary.circle.lat/boundary.circle.lon + // absence of boundary.circle.lat/boundary.circle.lon if( _.isFinite(clean['boundary.circle.lat']) && _.isFinite(clean['boundary.circle.lon']) ){ diff --git a/query/search.js b/query/search.js index fd1a048e7..84e2b6061 100644 --- a/query/search.js +++ b/query/search.js @@ -2,6 +2,8 @@ const _ = require('lodash'); const peliasQuery = require('pelias-query'); const defaults = require('./search_defaults'); const textParser = require('./text_parser'); +const config = require('pelias-config').generate(); +const addendum_namespace_filter = require('./view/addendum_namespace_filter'); //------------------------------ // general-purpose search query @@ -22,6 +24,12 @@ fallbackQuery.filter( peliasQuery.view.sources ); fallbackQuery.filter( peliasQuery.view.layers ); fallbackQuery.filter( peliasQuery.view.categories ); fallbackQuery.filter( peliasQuery.view.boundary_gid ); + +const configuredAddendumNamespaces = config.get('addendum_namespaces'); +Object.keys(configuredAddendumNamespaces).forEach(namespace => { + fallbackQuery.filter( addendum_namespace_filter(namespace) ); +}); + // -------------------------------- /** @@ -36,6 +44,14 @@ function generateQuery( clean ){ // input text vs.var( 'input:name', clean.text ); + //addendum + const configuredAddendumNamespaces = config.get('addendum_namespaces'); + Object.keys(configuredAddendumNamespaces) + .filter(namespace => clean[namespace]) + .forEach(namespace => { + vs.var( namespace, clean[namespace] ); + }); + // sources if( _.isArray(clean.sources) && !_.isEmpty(clean.sources) ) { vs.var( 'sources', clean.sources); @@ -113,11 +129,7 @@ function generateQuery( clean ){ textParser( clean.parsed_text, vs ); } - var q = getQuery(vs); - - //console.log(JSON.stringify(q, null, 2)); - - return q; + return getQuery(vs); } function getQuery(vs) { diff --git a/query/structured_geocoding.js b/query/structured_geocoding.js index 5ceaab2e2..2ef9183a1 100644 --- a/query/structured_geocoding.js +++ b/query/structured_geocoding.js @@ -2,6 +2,8 @@ const _ = require('lodash'); const peliasQuery = require('pelias-query'); const defaults = require('./search_defaults'); const textParser = require('./text_parser'); +const config = require('pelias-config').generate(); +const addendum_namespace_filter = require('./view/addendum_namespace_filter'); //------------------------------ // general-purpose search query @@ -22,6 +24,11 @@ structuredQuery.filter( peliasQuery.view.sources ); structuredQuery.filter( peliasQuery.view.layers ); structuredQuery.filter( peliasQuery.view.categories ); structuredQuery.filter( peliasQuery.view.boundary_gid ); + +const configuredAddendumNamespaces = config.get('addendum_namespaces'); +Object.keys(configuredAddendumNamespaces).forEach(namespace => { + structuredQuery.filter( addendum_namespace_filter(namespace) ); +}); // -------------------------------- /** @@ -35,6 +42,14 @@ function generateQuery( clean ){ // input text vs.var( 'input:name', clean.text ); + //addendum + const configuredAddendumNamespaces = config.get('addendum_namespaces'); + Object.keys(configuredAddendumNamespaces) + .filter(namespace => clean[namespace]) + .forEach(namespace => { + vs.var( namespace, clean[namespace] ); + }); + // sources vs.var( 'sources', clean.sources); @@ -103,11 +118,7 @@ function generateQuery( clean ){ textParser( clean.parsed_text, vs ); } - const q = getQuery(vs); - - // console.log(JSON.stringify(q.body, null, 2)); - - return q; + return getQuery(vs); } function getQuery(vs) { From 2d7f77099a013234933aac3866a4eaa4ef93f033 Mon Sep 17 00:00:00 2001 From: "mansoor.sajjad" Date: Tue, 29 Nov 2022 15:25:02 +0100 Subject: [PATCH 7/8] Updating the filter to use the correct query based on type. --- query/autocomplete.js | 2 +- query/reverse.js | 2 +- query/search.js | 2 +- query/structured_geocoding.js | 2 +- query/view/addendum_namespace_filter.js | 21 +++++++++++++++------ test/unit/schema.js | 13 ++++++++----- 6 files changed, 27 insertions(+), 15 deletions(-) diff --git a/query/autocomplete.js b/query/autocomplete.js index 88da96c16..6e2289816 100644 --- a/query/autocomplete.js +++ b/query/autocomplete.js @@ -68,7 +68,7 @@ query.filter( views.focus_point_filter ); const configuredAddendumNamespaces = config.get('addendum_namespaces'); Object.keys(configuredAddendumNamespaces).forEach(namespace => { - query.filter( views.addendum_namespace_filter(namespace) ); + query.filter( views.addendum_namespace_filter(namespace, configuredAddendumNamespaces[namespace].type) ); }); // -------------------------------- diff --git a/query/reverse.js b/query/reverse.js index 6909e9906..fc9574f62 100644 --- a/query/reverse.js +++ b/query/reverse.js @@ -25,7 +25,7 @@ query.filter( peliasQuery.view.boundary_gid ); const configuredAddendumNamespaces = config.get('addendum_namespaces'); Object.keys(configuredAddendumNamespaces).forEach(namespace => { - query.filter( addendum_namespace_filter(namespace) ); + query.filter( addendum_namespace_filter(namespace, configuredAddendumNamespaces[namespace].type) ); }); // -------------------------------- diff --git a/query/search.js b/query/search.js index 84e2b6061..d8e390582 100644 --- a/query/search.js +++ b/query/search.js @@ -27,7 +27,7 @@ fallbackQuery.filter( peliasQuery.view.boundary_gid ); const configuredAddendumNamespaces = config.get('addendum_namespaces'); Object.keys(configuredAddendumNamespaces).forEach(namespace => { - fallbackQuery.filter( addendum_namespace_filter(namespace) ); + fallbackQuery.filter( addendum_namespace_filter(namespace, configuredAddendumNamespaces[namespace].type) ); }); // -------------------------------- diff --git a/query/structured_geocoding.js b/query/structured_geocoding.js index 2ef9183a1..2fca8e8a5 100644 --- a/query/structured_geocoding.js +++ b/query/structured_geocoding.js @@ -27,7 +27,7 @@ structuredQuery.filter( peliasQuery.view.boundary_gid ); const configuredAddendumNamespaces = config.get('addendum_namespaces'); Object.keys(configuredAddendumNamespaces).forEach(namespace => { - structuredQuery.filter( addendum_namespace_filter(namespace) ); + structuredQuery.filter( addendum_namespace_filter(namespace, configuredAddendumNamespaces[namespace].type) ); }); // -------------------------------- diff --git a/query/view/addendum_namespace_filter.js b/query/view/addendum_namespace_filter.js index 6ad6f646a..7b55b22ef 100644 --- a/query/view/addendum_namespace_filter.js +++ b/query/view/addendum_namespace_filter.js @@ -1,13 +1,22 @@ -module.exports = function (addendumParameter) { +module.exports = function (addendumParameter, type) { return function (vs) { if (!vs.isset(addendumParameter)) { return null; } - return { - terms: { - [`addendum.${addendumParameter}`]: vs.var(addendumParameter) - } - }; + switch (type) { + case 'array': + return { + terms: { + [`addendum.${addendumParameter}`]: vs.var(addendumParameter) + } + }; + default: + return { + term: { + [`addendum.${addendumParameter}`]: vs.var(addendumParameter) + } + }; + } }; }; \ No newline at end of file diff --git a/test/unit/schema.js b/test/unit/schema.js index 23a5bb0ec..2ea9a38a1 100644 --- a/test/unit/schema.js +++ b/test/unit/schema.js @@ -1,8 +1,10 @@ +const Joi = require('@hapi/joi'); const schema = require('../../schema'); +const _ = require('lodash'); module.exports.tests = {}; -module.exports.tests.completely_valid = (test) => { +module.exports.tests.completely_valid = (test, common) => { test('all valid configuration elements should not throw error', (t) => { var config = { api: { @@ -68,7 +70,7 @@ module.exports.tests.completely_valid = (test) => { }; -module.exports.tests.api_validation = (test) => { +module.exports.tests.api_validation = (test, common) => { test('config without api should throw error', (t) => { var config = { esclient: {} @@ -518,7 +520,7 @@ module.exports.tests.api_validation = (test) => { }; -module.exports.tests.api_services_validation = (test) => { +module.exports.tests.api_services_validation = (test, common) => { test('unsupported children of api.services should be disallowed', (t) => { var config = { api: { @@ -542,7 +544,7 @@ module.exports.tests.api_services_validation = (test) => { }; -module.exports.tests.service_validation = (test) => { +module.exports.tests.service_validation = (test, common) => { // these tests apply for all the individual service definitions const services = ['pip', 'placeholder', 'interpolation', 'libpostal']; @@ -863,7 +865,7 @@ module.exports.tests.service_validation = (test) => { }; -module.exports.tests.esclient_validation = (test) => { +module.exports.tests.esclient_validation = (test, common) => { test('config without esclient should throw error', (t) => { var config = { api: { @@ -968,6 +970,7 @@ module.exports.tests.esclient_validation = (test) => { }); }; + module.exports.all = (tape, common) => { function test(name, testFunction) { From 533ed396cdec30a0a09169031a4c7938a9431f7b Mon Sep 17 00:00:00 2001 From: "mansoor.sajjad" Date: Tue, 29 Nov 2022 15:29:14 +0100 Subject: [PATCH 8/8] Fixing the test. --- .../autocomplete_multiple_configured_addendum_namespace.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js b/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js index 4547b3e55..582b44122 100644 --- a/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js +++ b/test/unit/fixture/autocomplete_multiple_configured_addendum_namespace.js @@ -68,7 +68,7 @@ module.exports = { } }, { - 'terms': { + 'term': { 'addendum.public_id': '12345' } }