diff --git a/test/traceql_parser.test.js b/test/traceql_parser.test.js new file mode 100644 index 00000000..7042cc67 --- /dev/null +++ b/test/traceql_parser.test.js @@ -0,0 +1,46 @@ +const parser = require('../traceql/parser') + +it('traceql: one selector', () => { + const res = parser.ParseScript('{.testId="12345"}') + expect(res.rootToken.value).toEqual('{.testId="12345"}') +}) + +it('traceql: multiple selectors', () => { + const res = parser.ParseScript('{.testId="12345" &&.spanN=9}') + expect(res.rootToken.value).toEqual('{.testId="12345" &&.spanN=9}') +}) + +it('traceql: multiple selectors OR Brackets', () => { + const res = parser.ParseScript('{.testId="12345" && (.spanN=9 ||.spanN=8)}') + expect(res.rootToken.value).toEqual('{.testId="12345" && (.spanN=9 ||.spanN=8)}') +}) + +it('traceql: multiple selectors regexp', () => { + const res = parser.ParseScript('{.testId="12345" &&.spanN=~"(9|8)"}') + expect(res.rootToken.value).toEqual('{.testId="12345" &&.spanN=~"(9|8)"}') +}) + +it('traceql: duration', () => { + const res = parser.ParseScript('{.testId="12345" && duration>=9ms}') + expect(res.rootToken.value).toEqual('{.testId="12345" && duration>=9ms}') +}) + +it('traceql: float comparison', () => { + const res = parser.ParseScript('{.testId="12345" &&.spanN>=8.9}') + expect(res.rootToken.value).toEqual('{.testId="12345" &&.spanN>=8.9}') +}) + +it('traceql: count empty result', () => { + const res = parser.ParseScript('{.testId="12345" &&.spanN>=8.9} | count() > 1') + expect(res.rootToken.value).toEqual('{.testId="12345" &&.spanN>=8.9} | count() > 1') +}) + +it('traceql: max duration empty result', () => { + const res = parser.ParseScript('{.testId="12345" &&.spanN>=8.9} | max(duration) > 9ms') + expect(res.rootToken.value).toEqual('{.testId="12345" &&.spanN>=8.9} | max(duration) > 9ms') +}) + +it('traceql: max duration', () => { + const res = parser.ParseScript('{.testId="12345" &&.spanN>=8.9} | max(duration) > 8ms') + expect(res.rootToken.value).toEqual('{.testId="12345" &&.spanN>=8.9} | max(duration) > 8ms') +}) diff --git a/traceql/clickhouse_transpiler/aggregator.js b/traceql/clickhouse_transpiler/aggregator.js index 49d74396..5348b8fd 100644 --- a/traceql/clickhouse_transpiler/aggregator.js +++ b/traceql/clickhouse_transpiler/aggregator.js @@ -3,12 +3,23 @@ const { getCompareFn, durationToNs } = require('./shared') module.exports = class Builder { constructor () { + this.main = null this.fn = '' this.attr = '' this.compareFn = '' this.compareVal = '' } + /** + * + * @param main {BuiltProcessFn} + * @returns {Builder} + */ + withMain (main) { + this.main = main + return this + } + /** * * @param fn {string} @@ -54,12 +65,16 @@ module.exports = class Builder { */ build () { const self = this - /** @type {ProcessFn} */ - const res = (sel, ctx) => { + /** @type {BuiltProcessFn} */ + const res = (ctx) => { + const sel = this.main(ctx) const fCmpVal = self.cmpVal() const agg = self.aggregator() const compareFn = getCompareFn(self.compareFn) - return sel.having(compareFn(agg, Sql.val(fCmpVal))) + const comparreExp = compareFn(agg, Sql.val(fCmpVal)) + // .having is broken + sel.having_conditions = Sql.And([...sel.having_conditions.args, comparreExp]) + return sel } return res } diff --git a/traceql/clickhouse_transpiler/attr_condition.js b/traceql/clickhouse_transpiler/attr_condition.js index 12343c7f..33e45535 100644 --- a/traceql/clickhouse_transpiler/attr_condition.js +++ b/traceql/clickhouse_transpiler/attr_condition.js @@ -2,6 +2,7 @@ const { getCompareFn, durationToNs, unquote } = require('./shared') const Sql = require('@cloki/clickhouse-sql') module.exports = class Builder { constructor () { + this.main = null this.terms = [] this.conds = null this.aggregatedAttr = '' @@ -12,6 +13,16 @@ module.exports = class Builder { this.where = [] } + /** + * + * @param main {BuiltProcessFn} + * @returns {Builder} + */ + withMain (main) { + this.main = main + return this + } + /** * @param terms {[]} * @returns {Builder} @@ -45,8 +56,9 @@ module.exports = class Builder { */ build () { const self = this - /** @type {ProcessFn} */ - const res = (sel, ctx) => { + /** @type {BuiltProcessFn} */ + const res = (ctx) => { + const sel = this.main(ctx) self.alias = 'bsCond' for (const term of self.terms) { const sqlTerm = self.getTerm(term) @@ -59,14 +71,14 @@ module.exports = class Builder { const having = self.getCond(self.conds) self.aggregator(sel) sel.conditions = Sql.And(sel.conditions, Sql.Or(...self.where)) - sel.having_conditions = Sql.And(sel.having_conditions, having) + sel.having(having) return sel } return res } /** - * @typedef {{simpleIdx: number, op: string, comlex: [Condition]}} Condition + * @typedef {{simpleIdx: number, op: string, complex: [Condition]}} Condition */ /** * @param c {Condition} @@ -74,7 +86,7 @@ module.exports = class Builder { getCond (c) { if (c.simpleIdx === -1) { const subs = [] - for (const s of c.comlex) { + for (const s of c.complex) { subs.push(this.getCond(s)) } switch (c.op) { @@ -89,7 +101,7 @@ module.exports = class Builder { if (!this.isAliased) { left = groupBitOr(bitSet(this.sqlConditions), this.alias) } - return Sql.Ne(bitAnd(left, Sql.val(c.simpleIdx)), Sql.val(0)) + return Sql.Ne(bitAnd(left, new Sql.Raw((BigInt(1) << BigInt(c.simpleIdx)).toString())), Sql.val(0)) } /** @@ -100,10 +112,8 @@ module.exports = class Builder { return } - const s = sel.select() if (this.aggregatedAttr === 'duration') { - s.push([new Sql.Raw('toFloat64(duration)'), 'agg_val']) - sel.select(...s) + sel.select([new Sql.Raw('toFloat64(any(traces_idx.duration))'), 'agg_val']) return } @@ -113,11 +123,10 @@ module.exports = class Builder { if (this.aggregatedAttr.match(/^resource\./)) { this.aggregatedAttr = this.aggregatedAttr.substr(9) } - if (this.aggregatedAttr.match(/^\.*/)) { + if (this.aggregatedAttr.match(/^\./)) { this.aggregatedAttr = this.aggregatedAttr.substr(1) } - s.push([sqlAttrValue(this.aggregatedAttr), 'agg_val']) - sel.select(...s) + sel.select([sqlAttrValue(this.aggregatedAttr), 'agg_val']) this.where.push(Sql.Eq(new Sql.Raw('key'), Sql.val(this.aggregatedAttr))) } @@ -127,7 +136,7 @@ module.exports = class Builder { key = key.substr(5) } else if (key.match(/^resource\./)) { key = key.substr(9) - } else if (key.match(/^.*/)) { + } else if (key.match(/^\./)) { key = key.substr(1) } else { switch (key) { @@ -150,7 +159,7 @@ module.exports = class Builder { getDurationCondition (key, term) { const fVal = durationToNs(term.Child('value').value) - const fn = getCompareFn(term.Child('op')) + const fn = getCompareFn(term.Child('op').value) return fn(new Sql.Raw('traces_idx.duration'), Math.floor(fVal)) } @@ -182,7 +191,7 @@ module.exports = class Builder { } getNumberCondition (key, term) { - const fn = getCompareFn(term.Child('op')) + const fn = getCompareFn(term.Child('op').value) if (!term.Child('value').value.match(/^\d+.?\d*$/)) { throw new Error(`invalid value in ${term.value}`) } @@ -248,7 +257,7 @@ function bitSet (terms) { const res = new Sql.Raw('') res.terms = terms res.toString = () => { - return terms.map((t, i) => `bitShiftLeft(toUint64(${t.toString()}), ${i})`).join('+') + return res.terms.map((t, i) => `bitShiftLeft(toUInt64(${t.toString()}), ${i})`).join('+') } return res } diff --git a/traceql/clickhouse_transpiler/group_by.js b/traceql/clickhouse_transpiler/group_by.js index 65cad4b5..fab2da06 100644 --- a/traceql/clickhouse_transpiler/group_by.js +++ b/traceql/clickhouse_transpiler/group_by.js @@ -1,8 +1,7 @@ const Sql = require('@cloki/clickhouse-sql') -/** - * @type {ProcessFn} - */ -module.exports = (sel, ctx) => { +const { standardBuilder } = require('./shared') + +module.exports = standardBuilder((sel, ctx) => { const withMain = new Sql.With('index_search', sel) return (new Sql.Select()) .with(withMain) @@ -14,4 +13,4 @@ module.exports = (sel, ctx) => { ).from(new Sql.WithReference(withMain)) .groupBy('trace_id') .orderBy([new Sql.Raw('max(index_search.timestamp_ns)'), 'desc']) -} +}) diff --git a/traceql/clickhouse_transpiler/index.js b/traceql/clickhouse_transpiler/index.js new file mode 100644 index 00000000..e43373cb --- /dev/null +++ b/traceql/clickhouse_transpiler/index.js @@ -0,0 +1,109 @@ +const AttrConditionPlanner = require('./attr_condition') +const InitIndexPlanner = require('./init') +const IndexGroupByPlanner = require('./group_by') +const AggregatorPlanner = require('./aggregator') +const IndexLimitPlanner = require('./limit') +const TracesDataPlanner = require('./traces_data') + +/** + * @param script {Token} + */ +module.exports = (script) => { + return new Planner(script).plan() +} + +class Planner { + /** + * + * @param script {Token} + */ + constructor (script) { + this.script = script + this.termIdx = [] + this.cond = null + this.aggregatedAttr = '' + this.cmpVal = '' + this.terms = {} + this.aggFn = '' + } + + plan () { + this.check() + this.analyze() + let res = (new AttrConditionPlanner()) + .withTerms(this.termIdx) + .withConditions(this.cond) + .withAggregatedAttr(this.aggregatedAttr) + .withMain((new InitIndexPlanner()).build()) + .build() + res = (new IndexGroupByPlanner()).withMain(res).build() + if (this.aggFn) { + res = (new AggregatorPlanner()) + .withFn(this.aggFn) + .withAttr(this.aggregatedAttr) + .withCompareFn(this.script.Child('cmp').value) + .withCompareVal(this.script.Child('cmp_val').value) + .withMain(res) + .build() + } + res = (new IndexLimitPlanner()).withMain(res).build() + res = (new TracesDataPlanner()).withMain(res).build() + res = (new IndexLimitPlanner()).withMain(res).build() + + return res + } + + check () { + if (this.script.Children('SYNTAX').length > 1) { + throw new Error('more than one selector is not supported') + } + } + + analyze () { + this.terms = {} + this.cond = this.analyzeCond(this.script.Child('attr_selector_exp')) + this.analyzeAgg() + } + + /** + * + * @param token {Token} + */ + analyzeCond (token) { + let res = {} + const complexHead = token.tokens.find(x => x.name === 'complex_head') + const simpleHead = token.tokens.find(x => x.name === 'attr_selector') + if (complexHead) { + res = this.analyzeCond(complexHead.tokens.find(x => x.name === 'attr_selector_exp')) + } else if (simpleHead) { + const term = simpleHead.value + if (this.terms[term]) { + res = { simpleIdx: this.terms[term] - 1, op: '', complex: [] } + } else { + this.termIdx.push(simpleHead) + this.terms[term] = this.termIdx.length + res = { simpleIdx: this.termIdx.length - 1, op: '', complex: [] } + } + } + const tail = token.tokens.find(x => x.name === 'tail') + if (tail) { + res = { + simpleIdx: -1, + op: token.tokens.find(x => x.name === 'and_or').value, + complex: [res, this.analyzeCond(tail.tokens.find(x => x.name === 'attr_selector_exp'))] + } + } + return res + } + + analyzeAgg () { + const agg = this.script.Child('aggregator') + if (!agg) { + return + } + this.aggFn = agg.Child('fn').value + const labelName = agg.Child('attr').Child('label_name') + this.aggregatedAttr = labelName ? labelName.value : '' + this.cmpVal = agg.Child('cmp_val').value + } +} diff --git a/traceql/clickhouse_transpiler/init.js b/traceql/clickhouse_transpiler/init.js index d4788398..ff63f4fe 100644 --- a/traceql/clickhouse_transpiler/init.js +++ b/traceql/clickhouse_transpiler/init.js @@ -1,25 +1,35 @@ const Sql = require('@cloki/clickhouse-sql') const { format } = require('date-fns') +const { standardBuilder } = require('./shared') /** - * @typedef {function(Sql.Select, { + * @typedef {{ * from: Date, * to: Date, * tracesAttrsTable: string, - * limit: number - * }): Select} ProcessFn - * @type ProcessFn + * limit: number, + * isCluster: boolean, + * tracesTable: string, + * tracesDistTable: string + * }} Context */ -module.exports.process = (sel, ctx) => { +/** + * @typedef {function(Sql.Select, Context): Select} ProcessFn + */ + +/** + * @type {ProcessFn} + */ +module.exports = standardBuilder((sel, ctx) => { return (new Sql.Select()).select(['trace_id', 'trace_id'], [new Sql.Raw('lower(hex(span_id))'), 'span_id'], [new Sql.Raw('any(duration)'), 'duration'], - [new Sql.Raw('any(timestamp_ns)', 'timestamp_ns')]) + [new Sql.Raw('any(timestamp_ns)'), 'timestamp_ns']) .from([ctx.tracesAttrsTable, 'traces_idx']) .where(Sql.And( Sql.Gte('date', Sql.val(format(ctx.from, 'yyyy-MM-dd'))), - Sql.Lt('date', Sql.val(format(ctx.to, 'yyyy-MM-dd'))), + Sql.Lte('date', Sql.val(format(ctx.to, 'yyyy-MM-dd'))), Sql.Gte('traces_idx.timestamp_ns', new Sql.Raw(ctx.from.getTime() + '000000')), Sql.Lt('traces_idx.timestamp_ns', new Sql.Raw(ctx.to.getTime() + '000000')) )).groupBy('trace_id', 'span_id') .orderBy(['timestamp_ns', 'desc']) -} +}) diff --git a/traceql/clickhouse_transpiler/limit.js b/traceql/clickhouse_transpiler/limit.js index 722d9926..3ec1c224 100644 --- a/traceql/clickhouse_transpiler/limit.js +++ b/traceql/clickhouse_transpiler/limit.js @@ -1,10 +1,8 @@ -/** - * - * @type {ProcessFn} - */ -module.exports.process = (sel, ctx) => { +const { standardBuilder } = require('./shared') + +module.exports = standardBuilder((sel, ctx) => { if (!ctx.limit) { return sel } return sel.limit(ctx.limit) -} +}) diff --git a/traceql/clickhouse_transpiler/shared.js b/traceql/clickhouse_transpiler/shared.js index 9f5253d8..13dbc410 100644 --- a/traceql/clickhouse_transpiler/shared.js +++ b/traceql/clickhouse_transpiler/shared.js @@ -1,5 +1,4 @@ const Sql = require('@cloki/clickhouse-sql') -const { json } = require('../../parser/registry/parser_registry') /** * * @param op {string} @@ -41,10 +40,41 @@ module.exports.durationToNs = (duration) => { module.exports.unquote = (val) => { if (val[0] === '"') { - return json.parse(val) + return JSON.parse(val) } if (val[0] === '`') { return val.substr(1, val.length - 2) } throw new Error('unquote not supported') } + +/** + * @typedef {function(Context): Select} BuiltProcessFn + */ +/** + * @param fn {ProcessFn} + * @returns {{ + * new(): { + * withMain(BuiltProcessFn): this, + * build(): BuiltProcessFn + * }, + * prototype: { + * withMain(BuiltProcessFn): this, + * build(): BuiltProcessFn + * }}} + */ +module.exports.standardBuilder = (fn) => { + return class { + withMain (main) { + this.main = main + return this + } + + build () { + return (ctx) => { + const sel = this.main ? this.main(ctx) : null + return fn(sel, ctx) + } + } + } +} diff --git a/traceql/clickhouse_transpiler/traces_data.js b/traceql/clickhouse_transpiler/traces_data.js new file mode 100644 index 00000000..f99928ef --- /dev/null +++ b/traceql/clickhouse_transpiler/traces_data.js @@ -0,0 +1,34 @@ +const Sql = require('@cloki/clickhouse-sql') +const { standardBuilder } = require('./shared') +/** + * @type {ProcessFn} + */ +const processFn = (sel, ctx) => { + const table = !ctx.isCluster ? ctx.tracesTable : ctx.tracesDistTable + const withMain = new Sql.With('index_grouped', sel) + const withTraceIds = new Sql.With('trace_ids', (new Sql.Select()) + .select('trace_id') + .from(new Sql.WithReference(withMain))) + return (new Sql.Select()) + .with(withMain, withTraceIds) + .select( + [new Sql.Raw('lower(hex(traces.trace_id))'), 'trace_id'], + [new Sql.Raw('any(index_grouped.span_id)'), 'span_id'], + [new Sql.Raw('any(index_grouped.duration)'), 'duration'], + [new Sql.Raw('any(index_grouped.timestamp_ns)'), 'timestamp_ns'], + [new Sql.Raw('min(traces.timestamp_ns)'), 'start_time_unix_nano'], + [new Sql.Raw( + 'toFloat64(max(traces.timestamp_ns + traces.duration_ns) - min(traces.timestamp_ns)) / 1000000' + ), 'duration_ms'], + [new Sql.Raw('argMin(traces.name, traces.timestamp_ns)', 'root_service_name'), 'root_service_name'] + ).from([table, 'traces']).join( + new Sql.WithReference(withMain), + 'left any', + Sql.Eq(new Sql.Raw('traces.trace_id'), new Sql.Raw('index_grouped.trace_id')) + ).where(Sql.And( + new Sql.In(new Sql.Raw('traces.trace_id'), 'in', new Sql.WithReference(withTraceIds)) + )).groupBy('traces.trace_id') + .orderBy(['start_time_unix_nano', 'desc']) +} + +module.exports = standardBuilder(processFn) diff --git a/traceql/index.js b/traceql/index.js index d36e6578..c987dce5 100644 --- a/traceql/index.js +++ b/traceql/index.js @@ -1,9 +1,9 @@ -const { TranspileTraceQL } = require('../wasm_parts/main') -const { clusterName } = require('../common') +const parser = require('./parser') +const transpiler = require('./clickhouse_transpiler') +const logger = require('../lib/logger') const { DATABASE_NAME } = require('../lib/utils') -const dist = clusterName ? '_dist' : '' +const { clusterName } = require('../common') const { rawRequest } = require('../lib/db/clickhouse') -const logger = require('../lib/logger') /** * @@ -14,30 +14,20 @@ const logger = require('../lib/logger') * @returns {Promise<[]>} */ const search = async (query, limit, from, to) => { - const _dbname = '`' + DATABASE_NAME() + '`' - const request = { - Request: query, - Ctx: { - IsCluster: !!clusterName, - OrgID: '0', - FromS: Math.floor(from.getTime() / 1000) - 600, - ToS: Math.floor(to.getTime() / 1000), - Limit: parseInt(limit), - - TimeSeriesGinTableName: `${_dbname}.time_series_gin`, - SamplesTableName: `${_dbname}.samples_v3${dist}`, - TimeSeriesTableName: `${_dbname}.time_series`, - TimeSeriesDistTableName: `${_dbname}.time_series_dist`, - Metrics15sTableName: `${_dbname}.metrics_15s${dist}`, - - TracesAttrsTable: `${_dbname}.tempo_traces_attrs_gin`, - TracesAttrsDistTable: `${_dbname}.tempo_traces_attrs_gin_dist`, - TracesTable: `${_dbname}.tempo_traces`, - TracesDistTable: `${_dbname}.tempo_traces_dist` - } + const _dbname = DATABASE_NAME() + /** @type {Context} */ + const ctx = { + tracesDistTable: `${_dbname}.tempo_traces_dist`, + tracesTable: `${_dbname}.tempo_traces`, + isCluster: !!clusterName, + tracesAttrsTable: `${_dbname}.tempo_traces_attrs_gin`, + from: from, + to: to, + limit: limit } - logger.debug(JSON.stringify(request)) - const sql = TranspileTraceQL(request) + const scrpit = parser.ParseScript(query) + const planner = transpiler(scrpit.rootToken) + const sql = planner(ctx) const response = await rawRequest(sql + ' FORMAT JSON', null, DATABASE_NAME()) const traces = response.data.data.map(row => ({ traceID: row.trace_id, @@ -49,7 +39,7 @@ const search = async (query, limit, from, to) => { { spans: row.span_id.map((spanId, i) => ({ spanID: spanId, - startTimeUnixNano: row.timestamps_ns[i], + startTimeUnixNano: row.timestamp_ns[i], durationNanos: row.duration[i], attributes: [] })), diff --git a/traceql/traceql.bnf b/traceql/traceql.bnf index a2a1d549..529f687c 100644 --- a/traceql/traceql.bnf +++ b/traceql/traceql.bnf @@ -1,13 +1,16 @@ ::= *( ) selector ::= "{" "}" [ ] -attr_selector_exp ::= ( | "(" ")") [ ] +attr_selector_exp ::= ( | ) [ ] +complex_head ::= "(" ")" +tail ::= and_or ::= "&&" | "||" -aggregator ::= "|" [] +aggregator ::= "|" fn ::= "count"|"sum"|"min"|"max"|"avg" -attr ::= "(" [] ")" +attr ::= "(" [ ] ")" cmp ::= "="|"!="|"<"|"<="|">"|">=" +cmp_val ::= [] measurement ::= "ns"|"us"|"ms"|"s"|"m"|"h"|"d" label_name ::= ("." | | "_") *("." | | "_" | )