From 65924d490dfddd9956f30017d638ee9c3b144b50 Mon Sep 17 00:00:00 2001 From: Kyriakos Barbounakis Date: Fri, 11 Oct 2024 19:36:10 +0300 Subject: [PATCH] implement custom query expression for attributes (#166) * implement custom query expression for attributes * fix regex warnings * rm unused regexp group * rm unused regexp group * rm unused regexp group * rm unused regexp group * rm unused regexp group * 2.6.56 --- data-field-query-resolver.js | 194 +++++++++++++++ data-listeners.js | 55 ++++- data-model.js | 13 +- model-schema.json | 51 +++- package-lock.json | 4 +- package.json | 2 +- spec/CustomQueryExpr.spec.ts | 467 +++++++++++++++++++++++++++++++++++ types.d.ts | 21 ++ 8 files changed, 789 insertions(+), 18 deletions(-) create mode 100644 data-field-query-resolver.js create mode 100644 spec/CustomQueryExpr.spec.ts diff --git a/data-field-query-resolver.js b/data-field-query-resolver.js new file mode 100644 index 0000000..4ba4c26 --- /dev/null +++ b/data-field-query-resolver.js @@ -0,0 +1,194 @@ +const {Args, DataError} = require('@themost/common'); +const {hasOwnProperty} = require('./has-own-property'); +const {QueryEntity, QueryExpression, QueryField} = require('@themost/query'); +class DataFieldQueryResolver { + /** + * @param {import("./data-model").DataModel} target + */ + constructor(target) { + this.target = target; + } + + /** + * + * @param {string} value + * @returns {string} + */ + formatName(value) { + if (/^\$/.test(value)) { + return value.replace(/(\$?(\w+)?)/g, '$2').replace(/\.(\w+)/g, '.$1') + } + return value; + } + + nameReplacer(key, value) { + if (typeof value === 'string') { + if (/^\$\w+$/.test(value)) { + const baseModel = this.target.base(); + const name = value.replace(/^\$/, ''); + let field = null; + let collection = null; + // try to find if field belongs to base model + if (baseModel) { + field = baseModel.getAttribute(name); + collection = baseModel.viewAdapter; + } + if (field == null) { + collection = this.target.sourceAdapter; + field = this.target.getAttribute(name); + } + if (field) { + return { + $name: collection + '.' + name + } + } + throw new DataError('An expression contains an attribute that cannot be found', null, this.target.name, name); + } else { // noinspection RegExpUnnecessaryNonCapturingGroup + if (/^\$\w+\.\w+$/.test(value)) { + return { + $name: value.replace(/^\$/, '') + } + } + } + } + return value; + } + + /** + * @param {import("./types").DataField} field + * @returns {{$select?: import("@themost/query").QueryField, $expand?: import("@themost/query").QueryEntity[]}|null} + */ + resolve(field) { + Args.check(field != null, new DataError('E_FIELD','Field may not be null', null, this.target.name)); + if (Array.isArray(field.query) === false) { + return { + select: null, + expand: [] + }; + } + let expand = []; + let select = null; + const self = this; + // get base model + const baseModel = this.target.base(); + for (const stage of field.query) { + if (stage.$lookup) { + // get from model + const from = stage.$lookup.from; + const fromModel = this.target.context.model(from); + if (stage.$lookup.pipeline && stage.$lookup.pipeline.length) { + stage.$lookup.pipeline.forEach(function(pipelineStage) { + if (pipelineStage.$match && pipelineStage.$match.$expr) { + const q = new QueryExpression().select('*').from(self.target.sourceAdapter); + // get expression as string + const exprString = JSON.stringify(pipelineStage.$match.$expr, function(key, value) { + if (typeof value === 'string') { + if (/\$\$\w+/.test(value)) { + let localField = /\$\$(\w+)/.exec(value)[1]; + let localFieldAttribute = self.target.getAttribute(localField); + if (localFieldAttribute && localFieldAttribute.model === self.target.name) { + return { + $name: self.target.sourceAdapter + '.' + localField + } + } + if (baseModel) { + localFieldAttribute = baseModel.getAttribute(localField); + if (localFieldAttribute) { + return { + $name: baseModel.viewAdapter + '.' + localField + } + } + } + throw new DataError('E_FIELD', 'Data field cannot be found', null, self.target.name, localField); + } + } + return self.nameReplacer(key, value); + }); + const joinCollection = new QueryEntity(fromModel.viewAdapter).as(stage.$lookup.as).left(); + Object.defineProperty(joinCollection, 'model', { + configurable: true, + enumerable: false, + writable: true, + value: fromModel.name + }); + const joinExpression = Object.assign(new QueryExpression(), { + $where: JSON.parse(exprString) + }); + q.join(joinCollection).with(joinExpression); + const appendExpand = [].concat(q.$expand); + expand.push.apply(expand, appendExpand); + } + }); + } else { + let localField = this.formatName(stage.$lookup.localField); + if (/\./.test(localField) === false) { + // get local field expression + let localFieldAttribute = this.target.getAttribute(localField); + if (localFieldAttribute && localFieldAttribute.model === this.target.name) { + localField = `${this.target.sourceAdapter}.${localField}`; + } else { + // get base model + const baseModel = this.target.base(); + if (baseModel) { + localFieldAttribute = baseModel.getAttribute(localField); + if (localFieldAttribute) { + localField = `${baseModel.viewAdapter}.${localField}`; + } + } + } + } + const foreignField = this.formatName(stage.$lookup.foreignField); + const q = new QueryExpression().select('*').from(this.target.sourceAdapter); + Args.check(fromModel != null, new DataError('E_MODEL', 'Data model cannot be found', null, from)); + const joinCollection = new QueryEntity(fromModel.viewAdapter).as(stage.$lookup.as).left(); + Object.defineProperty(joinCollection, 'model', { + configurable: true, + enumerable: false, + writable: true, + value: fromModel.name + }); + q.join(joinCollection).with( + new QueryExpression().where(new QueryField(localField)) + .equal(new QueryField(foreignField).from(stage.$lookup.as)) + ); + const appendExpand = [].concat(q.$expand); + expand.push.apply(expand, appendExpand); + } + } + const name = field.property || field.name; + if (stage.$project) { + Args.check(hasOwnProperty(stage.$project, name), new DataError('E_QUERY', 'Field projection expression is missing.', null, this.target.name, field.name)); + const expr = Object.getOwnPropertyDescriptor(stage.$project, name).value; + if (typeof expr === 'string') { + select = new QueryField(this.formatName(expr)).as(name) + } else { + const expr1 = Object.defineProperty({}, name, { + configurable: true, + enumerable: true, + writable: true, + value: expr + }); + // Important note: Field references e.g. $customer.email + // are not supported by @themost/query@Formatter + // and should be replaced by name references e.g. { "$name": "customer.email" } + // A workaround is being used here is a regular expression replacer which + // will try to replace "$customer.email" with { "$name": "customer.email" } + // but this operation is definitely a feature request for @themost/query + const finalExpr = JSON.parse(JSON.stringify(expr1, function(key, value) { + return self.nameReplacer(key, value); + })); + select = Object.assign(new QueryField(), finalExpr); + } + } + } + return { + $select: select, + $expand: expand + } + } + +} + +module.exports = { + DataFieldQueryResolver +} diff --git a/data-listeners.js b/data-listeners.js index 00c0652..08f7b79 100644 --- a/data-listeners.js +++ b/data-listeners.js @@ -2,14 +2,16 @@ var async = require('async'); var {sprintf} = require('sprintf-js'); var _ = require('lodash'); +var {isEqual} = require('lodash'); var {QueryUtils} = require('@themost/query'); var {QueryField} = require('@themost/query'); var {QueryFieldRef} = require('@themost/query'); -var {NotNullError} = require('@themost/common'); +var {NotNullError, DataError} = require('@themost/common'); var {UniqueConstraintError} = require('@themost/common'); var {TraceUtils} = require('@themost/common'); var {TextUtils} = require('@themost/common'); var {DataCacheStrategy} = require('./data-cache'); +var {DataFieldQueryResolver} = require('./data-field-query-resolver'); /** * @classdesc Represents an event listener for validating not nullable fields. This listener is automatically registered in all data models. @@ -668,14 +670,32 @@ DataModelCreateViewListener.prototype.afterUpgrade = function(event, callback) { } // get base model var baseModel = self.base(); + var additionalExpand = []; // get array of fields - var fields = self.attributes.filter(function(x) { - return (self.name=== x.model) && (!x.many); - }).map(function(x) { - return QueryField.select(x.name).from(adapter); - }); + var fields = []; + try { + fields = self.attributes.filter(function(x) { + return (self.name=== x.model) && (!x.many); + }).map(function(x) { + if (x.readonly && x.query) { + // resolve field expression (and additional joins) + var expr = new DataFieldQueryResolver(self).resolve(x); + if (expr) { + // hold additional joins + additionalExpand.push.apply(additionalExpand, expr.$expand); + // and return the resolved query expression for this field + return expr.$select; + } + // throw error + throw new DataError('E_QUERY', 'The given field defines a custom query expression but it cannot be resolved', null, self.name, x.name); + } + return QueryField.select(x.name).from(adapter); + }); + } catch (error) { + return callback(error); + } /** - * @type {QueryExpression} + * @type {import("@themost/query").QueryExpression} */ var q = QueryUtils.query(adapter).select(fields); var baseAdapter = null; @@ -691,14 +711,23 @@ DataModelCreateViewListener.prototype.afterUpgrade = function(event, callback) { baseFields.push(QueryField.select(x.name).from(baseAdapter)) }); } - if (baseFields.length>0) - { + q.$expand = []; + if (baseFields.length > 0) { var from = new QueryFieldRef(adapter, self.key().name); var to = new QueryFieldRef(baseAdapter, self.base().key().name); - q.$expand = { $entity: { },$with:[] }; - q.$expand.$entity[baseAdapter]=baseFields; - q.$expand.$with.push(from); - q.$expand.$with.push(to); + var addExpand = { $entity: { },$with:[] } + addExpand.$entity[baseAdapter] = baseFields; + addExpand.$with.push(from); + addExpand.$with.push(to); + q.$expand.push(addExpand); + } + for (var expand of additionalExpand) { + var findIndex = q.$expand.findIndex(function(item) { + return isEqual(expand, item); + }); + if (findIndex < 0) { + q.$expand.push(expand); + } } //execute query return db.createView(view, q, function(err) { diff --git a/data-model.js b/data-model.js index 2578a34..167c5de 100644 --- a/data-model.js +++ b/data-model.js @@ -2286,8 +2286,19 @@ DataModel.prototype.migrate = function(callback) return callback(null, false); } var context = self.context; - //do migration + // do migration var fields = self.attributes.filter(function(x) { + if (x.insertable === false && x.editable === false && x.model === self.name) { + if (typeof x.query === 'undefined') { + throw new DataError('E_MODEL', 'A non-insertable and non-editable field should have a custom query defined.', null, self.name, x.name); + } + // validate source and view + if (self.sourceAdapter === self.viewAdapter) { + throw new DataError('E_MODEL', 'A data model with the same source and view data object cannot have virtual columns.', null, self.name, x.name); + } + // exclude virtual column + return false; + } return (self.name === x.model) && (!x.many); }); diff --git a/model-schema.json b/model-schema.json index 7f1b740..972238e 100644 --- a/model-schema.json +++ b/model-schema.json @@ -315,9 +315,58 @@ }, "validator": { "type": "string", - "description": "A string which represetns the module path that exports a custom validator e.g. ./validators/custom-validator.js" + "description": "A string which represents the module path that exports a custom validator e.g. ./validators/custom-validator.js" } } + }, + "query": { + "type": "array", + "description": "Defines a custom query expression to be used while selecting field.", + "items": { + "anyOf":[ + { + "type": "object", + "$lookup": { + "type": "object", + "properties": { + "from": { + "type": "string" + }, + "localField": { + "type": "string" + }, + "foreignField": { + "type": "string" + }, + "let": { + "type": "object", + "additionalProperties": true + }, + "pipeline": { + "type": "object", + "additionalProperties": true + }, + "as": { + "type": "string" + } + }, + "additionalProperties": true, + "required": [ + "from" + ], + "description": "A query expression for joining other data models" + } + }, + { + "type": "object", + "$project": { + "type": "object", + "additionalProperties": true, + "description": "A query expression for selecting field" + } + } + ] + } } }, "required": [ diff --git a/package-lock.json b/package-lock.json index 8268219..ef9d562 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@themost/data", - "version": "2.6.55", + "version": "2.6.56", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@themost/data", - "version": "2.6.55", + "version": "2.6.56", "license": "BSD-3-Clause", "dependencies": { "@themost/events": "^1.3.0", diff --git a/package.json b/package.json index 7923296..c448ef9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@themost/data", - "version": "2.6.55", + "version": "2.6.56", "description": "MOST Web Framework Codename Blueshift - Data module", "main": "index.js", "types": "index.d.ts", diff --git a/spec/CustomQueryExpr.spec.ts b/spec/CustomQueryExpr.spec.ts new file mode 100644 index 0000000..7e169a9 --- /dev/null +++ b/spec/CustomQueryExpr.spec.ts @@ -0,0 +1,467 @@ +import {TestUtils} from './adapter/TestUtils'; +import { TestApplication } from './TestApplication'; +import { DataContext } from '../types'; +import { DataConfigurationStrategy } from '../data-configuration'; +import {SqliteAdapter} from '@themost/sqlite'; +import { resolve } from 'path'; + +const TempOrderSchema = { + "name": "TempOrder", + "version": "3.0.0", + "fields": [ + { + "@id": "https://themost.io/schemas/id", + "name": "id", + "title": "ID", + "description": "The identifier of the item.", + "type": "Counter", + "primary": true + }, + { + "name": "acceptedOffer", + "title": "Accepted Offer", + "description": "The offer e.g. product included in the order.", + "type": "Offer" + }, + { + "name": "customer", + "title": "Customer", + "description": "Party placing the order.", + "type": "Person", + "editable": false, + "nullable": false + }, + { + "name": "orderDate", + "title": "Order Date", + "description": "Date order was placed.", + "type": "DateTime", + "value": "javascript:return new Date();" + }, + { + "name": "orderEmail", + "readonly": true, + "type": 'Text', + "nullable": true, + "query": [ + { + "$lookup": { + "from": "Person", + "foreignField": "id", + "localField": "customer", + "as": "customer" + } + }, + { + "$project": { + "orderEmail": "$customer.email" + } + } + ] + }, + { + "name": 'orderAddressLocality', + "readonly": true, + "type": 'Text', + "nullable": true, + "query": [ + { + "$lookup": { + "from": "Person", + "foreignField": "id", + "localField": "customer", + "as": "customer" + } + }, + { + "$lookup": { + "from": "PostalAddress", + "foreignField": "id", + "localField": "$customer.address", + "as": "address" + } + }, + { + "$project": { + "orderAddressLocality": "$address.addressLocality" + } + } + ] + }, + { + "name": "orderedItem", + "title": "Ordered Item", + "description": "The item ordered.", + "type": "Product", + "expandable": true, + "editable": true, + "nullable": false + } + ], + "privileges": [ + { + "mask": 15, + "type": "global" + }, + { + "mask": 15, + "type": "global", + "account": "Administrators" + }, + { + "mask": 1, + "type": "self", + "filter": "customer/user eq me()" + } + ] +}; + +const NewOrderSchema = { + "name": "NewOrder", + "version": "3.0.0", + "fields": [ + { + "@id": "https://themost.io/schemas/id", + "name": "id", + "type": "Counter", + "primary": true + }, + { + "name": "acceptedOffer", + "type": "Offer" + }, + { + "name": "customer", + "type": "Person", + "editable": false, + "nullable": false + }, + { + "name": "orderDate", + "type": "DateTime", + "value": "javascript:return new Date();" + }, + { + "name": "orderedItem", + "type": "Product", + "expandable": true, + "editable": true, + "nullable": false + }, + { + "name": "priceCategory", + "readonly": true, + "type": 'Text', + "nullable": true, + "query": [ + { + "$lookup": { + "from": "Person", + "foreignField": "id", + "localField": "customer", + "as": "customer" + } + }, + { + "$lookup": { + "from": "Product", + "foreignField": "id", + "localField": "orderedItem", + "as": "orderedItem" + } + }, + { + "$project": { + "priceCategory": { + "$cond": [ + { + "$gt": [ + "$orderedItem.price", + 1000 + ] + }, + 'Expensive', + 'Normal' + ] + } + } + } + ] + }, + ], + "privileges": [ + { + "mask": 15, + "type": "global" + }, + { + "mask": 15, + "type": "global", + "account": "Administrators" + }, + { + "mask": 1, + "type": "self", + "filter": "customer/user eq me()" + } + ] +}; + +const NewLocalOrderSchema = { + "name": "NewLocalOrder", + "version": "3.0.0", + "fields": [ + { + "@id": "https://themost.io/schemas/id", + "name": "id", + "title": "ID", + "description": "The identifier of the item.", + "type": "Counter", + "primary": true + }, + { + "name": "acceptedOffer", + "title": "Accepted Offer", + "description": "The offer e.g. product included in the order.", + "type": "Offer" + }, + { + "name": "customer", + "title": "Customer", + "description": "Party placing the order.", + "type": "Person", + "editable": false, + "nullable": false + }, + { + "name": "orderDate", + "title": "Order Date", + "description": "Date order was placed.", + "type": "DateTime", + "value": "javascript:return new Date();" + }, + { + "name": "orderEmail", + "readonly": true, + "type": 'Text', + "nullable": true, + "query": [ + { + "$lookup": { + "from": "Person", + "pipeline": [ + { + "$match": { + "$expr": { + "$eq": [ "$$customer", "$customer.id" ] + } + } + } + ], + "as": "customer" + } + }, + { + "$project": { + "orderEmail": "$customer.email" + } + } + ] + }, + { + "name": "orderedItem", + "title": "Ordered Item", + "description": "The item ordered.", + "type": "Product", + "expandable": true, + "editable": true, + "nullable": false + } + ], + "privileges": [ + { + "mask": 15, + "type": "global" + }, + { + "mask": 15, + "type": "global", + "account": "Administrators" + }, + { + "mask": 1, + "type": "self", + "filter": "customer/user eq me()" + } + ] +}; + +const ExtendedProductSchema = { + "name": "ExtendedProduct", + "version": "3.0.0", + "inherits": "Product", + "fields": [ + { + "name": "priceCategory", + "type": "Text", + "readonly": true, + "insertable": false, + "editable": false, + "nullable": true, + "query": [ + { + "$project": { + "priceCategory": { + "$cond": [ + { + "$gt": [ + "$price", + 1000 + ] + }, + 'Expensive', + 'Normal' + ] + } + } + } + ] + } + ], + "privileges": [ + { + "mask": 15, + "type": "global" + }, + { + "mask": 15, + "type": "global", + "account": "Administrators" + }, + { + "mask": 1, + "type": "self", + "filter": "customer/user eq me()" + } + ] +} + +describe('CustomQueryExpression', () => { + + let app: TestApplication; + let context: DataContext; + + beforeAll(async () => { + app = new TestApplication(resolve(process.cwd(), 'spec/test2')); + context = app.createContext(); + }); + + afterAll(async () => { + await context.finalizeAsync(); + await app.finalize(); + }); + + it('should change model definition', async () => { + await TestUtils.executeInTransaction(context, async () => { + const configuration = app.getConfiguration().getStrategy(DataConfigurationStrategy); + configuration.setModelDefinition(TempOrderSchema); + await context.model('TempOrder').migrateAsync(); + // insert a temporary object + const newOrder: any = { + orderDate: new Date(), + orderedItem: { + name: 'Samsung Galaxy S4' + }, + customer: { + email: 'luis.nash@example.com' + } + }; + await context.model('TempOrder').silent().save(newOrder); + const item = await context.model('TempOrder').where('id').equal(newOrder.id).silent().getItem(); + expect(item).toBeTruthy(); + expect(item.orderEmail).toEqual('luis.nash@example.com'); + const orderAddressLocality = await context.model('Person') + .where('email').equal('luis.nash@example.com') + .select('address/addressLocality').silent().value(); + expect(item.orderAddressLocality).toEqual(orderAddressLocality); + + }); + }); + + it('should use custom query with expression', async () => { + await TestUtils.executeInTransaction(context, async () => { + const configuration = app.getConfiguration().getStrategy(DataConfigurationStrategy); + configuration.setModelDefinition(NewOrderSchema); + await context.model('NewOrder').migrateAsync(); + // insert a temporary object + const newOrder: any = { + orderDate: new Date(), + orderedItem: { + name: 'Samsung Galaxy S4' + }, + customer: { + email: 'luis.nash@example.com' + } + }; + await context.model('NewOrder').silent().save(newOrder); + const item = await context.model('NewOrder').where('id').equal(newOrder.id).silent().getItem(); + expect(item).toBeTruthy(); + const price = await context.model('Product') + .where('name').equal('Samsung Galaxy S4') + .select('price').silent().value(); + expect(item.priceCategory).toEqual(price <= 1000 ? 'Normal' : 'Expensive'); + + }); + }); + + it('should use custom query with join expression', async () => { + await TestUtils.executeInTransaction(context, async () => { + const configuration = app.getConfiguration().getStrategy(DataConfigurationStrategy); + configuration.setModelDefinition(NewLocalOrderSchema); + await context.model('NewLocalOrder').migrateAsync(); + // insert a temporary object + const newOrder: any = { + orderDate: new Date(), + orderedItem: { + name: 'Samsung Galaxy S4' + }, + customer: { + email: 'luis.nash@example.com' + } + }; + await context.model('NewLocalOrder').silent().save(newOrder); + const item = await context.model('NewLocalOrder').where('id').equal(newOrder.id).silent().getItem(); + expect(item).toBeTruthy(); + const price = await context.model('Product') + .where('name').equal('Samsung Galaxy S4') + .select('price').silent().value(); + expect(item.orderEmail).toEqual('luis.nash@example.com'); + + }); + }); + + it('should use custom query to project a readonly attribute', async () => { + await TestUtils.executeInTransaction(context, async () => { + const configuration = app.getConfiguration().getStrategy(DataConfigurationStrategy); + configuration.setModelDefinition(ExtendedProductSchema); + const ExtendedProducts = context.model('ExtendedProduct'); + await ExtendedProducts.migrateAsync(); + // validate non-insertable columns + const db: SqliteAdapter = context.db as SqliteAdapter; + const columns = await db.table(ExtendedProducts.sourceAdapter).columnsAsync(); + expect(columns.find((item) => item.name === 'priceCategory')).toBeFalsy(); + // insert a temporary object + const newProduct: any = { + name: 'Samsung Galaxy S4 XL', + price: 560, + priceCategory: 'Normal' + }; + await context.model('ExtendedProduct').silent().save(newProduct); + const item = await context.model('ExtendedProduct').where('id').equal(newProduct.id).silent().getItem(); + expect(item).toBeTruthy(); + expect(item.priceCategory).toEqual('Normal'); + + }); + }); + +}); diff --git a/types.d.ts b/types.d.ts index 09eff4a..62c2dcd 100644 --- a/types.d.ts +++ b/types.d.ts @@ -146,6 +146,26 @@ export declare class DataAssociationMapping { } +export declare interface QueryPipelineLookup { + from: string; + localField?: string; + foreignField?: string; + let?: string; + pipeline?: { + $match: any; + } + as?: string +} + +export declare interface QueryPipelineProject { + [name: string]: string | (1 | 0) | any; +} + +export declare interface QueryPipelineStage { + $lookup?: QueryPipelineLookup; + $project?: QueryPipelineProject; +} + export declare class DataField { name: string; property?: string; @@ -169,6 +189,7 @@ export declare class DataField { multiplicity?: string; indexed?: boolean; size?: number; + query?: QueryPipelineStage[]; } export declare class DataEventArgs {