From b53e279f37e1fd5ffe601040d2ba0561087271e1 Mon Sep 17 00:00:00 2001 From: akvlad Date: Mon, 19 Feb 2024 14:13:29 +0200 Subject: [PATCH] fix: restrict more than 2M nodes; improve CH request; add dist support for pyroscope v0.2 --- lib/db/maintain/scripts.js | 17 ++++- pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm | Bin 84610 -> 84714 bytes pyroscope/pprof-bin/src/lib.rs | 25 ++++-- pyroscope/pyroscope.js | 88 +++++++++++++++++++--- 4 files changed, 111 insertions(+), 19 deletions(-) diff --git a/lib/db/maintain/scripts.js b/lib/db/maintain/scripts.js index fb506879..7c2aa097 100644 --- a/lib/db/maintain/scripts.js +++ b/lib/db/maintain/scripts.js @@ -400,7 +400,7 @@ module.exports.profiles = [ ADD COLUMN IF NOT EXISTS \`tree\` Array(Tuple(UInt64, UInt64, UInt64, Array(Tuple(String, Int64, Int64)))), ADD COLUMN IF NOT EXISTS \`functions\` Array(Tuple(UInt64, String))`, - 'RENAME TABLE IF EXISTS profiles_mv {{{OnCluster}}} TO profiles_mv_bak', + 'RENAME TABLE IF EXISTS profiles_mv TO profiles_mv_bak {{{OnCluster}}}', `CREATE MATERIALIZED VIEW IF NOT EXISTS profiles_mv {{{OnCluster}}} TO profiles AS SELECT @@ -462,5 +462,18 @@ module.exports.profiles_dist = [ key String, val String, val_id UInt64 - ) ENGINE = Distributed('{{CLUSTER}}','{{DB}}','profiles_series_keys', rand());` + ) ENGINE = Distributed('{{CLUSTER}}','{{DB}}','profiles_series_keys', rand());`, + + `ALTER TABLE profiles_dist {{{OnCluster}}} + ADD COLUMN IF NOT EXISTS \`tree\` Array(Tuple(UInt64, UInt64, UInt64, Array(Tuple(String, Int64, Int64)))), + ADD COLUMN IF NOT EXISTS \`functions\` Array(Tuple(UInt64, String))`, + + `ALTER TABLE profiles_dist {{{OnCluster}}} + ADD COLUMN IF NOT EXISTS \`sample_types_units\` Array(Tuple(String, String))`, + + `ALTER TABLE profiles_series_dist {{{OnCluster}}} + ADD COLUMN IF NOT EXISTS \`sample_types_units\` Array(Tuple(String, String))`, + + `ALTER TABLE profiles_series_gin_dist {{{OnCluster}}} + ADD COLUMN IF NOT EXISTS \`sample_types_units\` Array(Tuple(String, String))` ] diff --git a/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm b/pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm index 2d96df6859ddf431409c89c418b94ea557629ac4..4162f86ed61b7e8a4e3e33055c694b19638cde6c 100644 GIT binary patch delta 9673 zcmc&)4Uingb?%;@y}8+$y}7&H``4*wbdvTgA)P?JlVx07FGxrrA%So#6YKy^Tx3r% zh$t?_B^X%)zR2PHQT5r7BWs`~P+K>+gN@(`~^o1#msN%||g%hn;j#z96N<<)Z-ix*MdVV*3+go!OIOd1Vg?N;=$ zq-RaYMUVU&R*m*h=B<=f1#7IVM=)BMwbjfM4+)OpF{8wZX`gRkX?ZLyFF0*2PdoHP zUOvRD6=%XmgWQiSc1PLW$a*IG0jt-357}DZ^sRbzWjFod6ZSM0`Fi2Trug-9;QC6=#bkXt- zIXNXda1PN0sIn%ZO(Yk^Jn-_OW7J+3yMdJ#_DtmrzcqH%`46ARJJgL2f04WD+Yf(b zLk@?l*FH5<8QLe@U9l@H$u0{$!B(*Lq^)9I1--i>S6Q`hmuRb8uy2>yAdO0Ij$P>o z?a=59gg^4zy{RkhAZN>%@TcP!LUch@rR~-v%uU#3^v_)tTEbP62YP#QM*u-mQS3mU(Nq$RFe_Et z4-Bgt&uLTl9Vqjndf~uk&jo_)zBs444=(XAfhV}gskH}h^&*3qNd4u(-n1yc#j3Gg zJDU(^6AHeXZ4sfgRVl5FE!Hc082+LG&qh4zsitjUu zcQ94L)&6^mdFY-QrI*yw-}#Ny6`f$Dgi*psKb?X_pVyT!wu)K@Hf`^y(gCBFZjl_g z_h_NOhLPrD`+l}};X@elh(Bt3O&hmnfD8c`9GOZA!dmZ4rr5r(4P2V>pG;Wj&^QL% z|J+NfE7TH&OGB22smBZx7uJSWvr?`8JgwEiMtENSEonOmRE*wRdL28%!!J%`;S|e`|svoQ? zs*oYCAO_X?-y1y(R5&#<=Q*HC2vl1dpem3hXYe|0hUY*(hFb!V$@P z^fAbTlz7pWyePm8(!sMl&{0sw9zFf>F3>mh$8eCoTvZi)Kq3L(C)1u^FO;hy&I-WF z6Td&08nuHG1_h5T90g(ktquusI8cs51QrLRqupqNnjOh}(TBhD*AFI0rLz&Kba%w4 zCMO`)!}8*o@Icm7AcxgQ^jnY~a&n}Tn45H9FOtWL~FuVOX&6z$T}!^!EB z74#ANSsL2%tC(h|5gAcPfvAAALJAoa7J+||Izl~&J8-!0b)rkX^?pyj3*Dk8IF7SR zy>U2M2AdV6DMtDy-PQC1lcMriql{Dam3VA8_32a!B;A)ws}+m5*q1g%FuwfW1GAV| zAcr2H1mPo|p+x=(Bl^+VZ&H>9y}}UOAQjbU7!39WZNngDtQpD!2WztoV3wG&X3D*H zG}&bX2qgevUJB}7sre1Gdw#XTX>EZ-oU#(h2{6UDHHMr^ivkhL5iDdCty^@SZX z75}g_5!7%8@52&Kxa*{uL?>`ZYbV@c2isy)%eUV>F|=ME@yOk0Zlr2($sKNP;NLQ(82|zFJWbNuqir z3P~p+Kx!}$0y72!Az@$)O$Gx?kpl(-xOMWmbOe<9E7B7@HL`zg5E066AqIt}NDDQ^ z>nRWz0tb9e7>X=%ARN3tJyaQDaXm!9iX9#4`Nesehl>zyGX%`<|rw zi}ml9_3vX(E(=3>9x>`_?cdb1PZrhtPyStZP{$peCvjz{-hOK2@<3QR!~p%-d z2nOogGh49WB~O2tOF#xqWO6>v&H!l@at$(>)C}_P20Fwm?@^XWX2ZAJM5fG1GjdHRb%34Kcj3Z;*L!T_K!&OJT2%ZNT6u?9~?FlQU{W(?qyU z5lG+J5eTV(lnar=Xr)kDt$sHC*@KeFdwSKQ&vu;^nAk0?qmNXQ{*=H3-WB%JNA_ZI zOWR3b7Le$O4iR~TNR*gL#}*!W+^T$;l;p_Mlu!sro|P2F#Bfun0$*t~Ur0#7wL#D6 zv1U-i*T9JX@O0W7bij)u@`W?Sy{APzWsvbFU^zu@jODIsWa7Ck^=o#F^;fFoEkRZ|5lR)t37jrLkZX=la>fYl^xUXI~z6`(L*T4NID8KH+e{cQ}Bm1O8{AeYTQ-%_eXeHu$ z3?)(}DG_O=L>iP00dYL=$2Yf9;(M7A@yt+^dpRlbUG`qB%JX+k%;1Y$+F4PH7J?dDKD-U0L-+T82|fnXLfp8B`v#}IKKxP=>uev3u~ zYS{}b77#8_nNZkk@DU`~hHwzmaJurL-<}uFN#RyN$>x@4>F^a7J}nf=CPA={!$@x{y(gA`CQBtF)mPZBo4hYZX zj4|T8lDhuX1wAgk2wB3ZE@e!_+HA8_z<;e$Ob<1(Frb`t8fC4|01C6N{UtB!g0->L4 zAXM2e#Wn&xwfLpm(-vm^Uxdg2W{o#C4oo&~fF69cc7qWSP%YrQ{SI^ofd?=u>W&xH z+BLJSRrmX{?(~c%@Z1E;s`byc>i-D!_aUAh+aO*)3i;$t+*aL>Q1=45;~lPuOVehY zSdpBnhXhZU3?ibc_Oc{zP0{$8^Ua01#rD3Dt(PA1Fc_g({T7{dSMSdD_ZI1S6>a+E z!m6)L#V_~w6hX<^Fy3g|{BkLc5ZmJf>Dr7Itj&Z)9IqjG_LHw8`qI~>4!SN~gqr(t zJmUx)I_SEz3NM2lawS~JI-R3;>XoP&A~!~_r&-`mAZaBneWh94)mjK1p)+}9fWstzY1|H*vzit3X!KT6LAR=ru9axY=|sis@Duzh*3xe-x%7I z$D*ldZNz(2o-q6rv`*<^O2Kz`QThRtl#W~a0|Q9FLLetjH3Hy4{Q?{;-@o}PT5qnxxz9#8d>)=`yJ5~yt z0ur!=h`f^}I3->5jvK*wU?BMr41_7-t3}A+goy6Drectd<4T@igQ@Wi1;_$q=&dGZ zLO=tt0D$I^2by;=xl`iGfEcWYG#7VZPFwz}AWf}-+OWWt)Y^&KA+5eB>O&xl`Z8Q> zGzA}w{8FsZJ+d8hxRw6t${BWR<*R%4PVYZ7yPMwU2Z2LT|YQZVj*CH5el-z;2wUl9>-4bSBrL3Aac zXlr}YEu9@xsJ=pn2?&NH>}Cp;Rq0J(KNKG@efXlNp8eC_vZJwSr{$`|93W3*Gk9KP$mRzGF-T=Y$B7 ztb)*u)#Am645bJDFYY$0igk#j!;;DD7fMIqF=6kd4_O`^kZBPPZCWb}hX8yzWIs6M zE7ybtytkpr5zZF+c#s4c7}g~=fuW1~ZAkFN=p(_vF*m}r{}mY45QfZ-61%o(z2m47LQ_ZqAZfDQp@Rs(c>59OvkEd4*-gSzLnApLpQ9s({{ z{no+GSjNL(CafNb|aqBl#A=7aHr2_4AQ6y zR_5)3ea>o2(gLhn$F`5?Xe8o83&LM;444%3vmDrj$*hxS(&--L6C%y} zH#)s0m?OqgGqEV7(5Gv>spC0&_s+F&<~NZD`gRQ$t(WGI@dA@Y*yFs7VLRm`A?R5; zcJuO#EpU05!u)6K?dXC>4tvw!Ol4vp(TV68wI@Ff0X^iwCi9i{Y2odSb092C(`t#? z#+D>C6$(C>*w8;n8HpDM5?|uUl5E0wuogE7t=1GM7I-p;D+O~CU~@!Ng6WT{_4WHk zL05Z!-|=I%PJ0n}l#wb(1gzu)1%y_#ur5+8L=YKZ8JnPF+KTMMCtgFuM(3*c z^@R)FT@+Yp{#&P~SqHC2OW8^1HjT$xC{A7bdbx~u;Ee9(LHGyFh3Sm$=HWBMv#+m6 z^Ahjp0N1B+EV#^?3_?A`Mn3R`Bg|Ys#cn=*I!5e5jW`BXGm0{U%(xU(-LjSd(JWgHbyls4SFtWk$TYP~xqP z{I!hCZ9op-BJDjGgdQ4zIyj8lS@Rm`qw}2ze4vqO1Fp z@CJUO3Zof&iT-u8+l$x_(lIS&1T^JSx1-l@0&EMg7d+7CMgP}7`xozDm41OSHi&=Q zc*K09zdWaWGs-KQ+Nr#-seB9alXJ=|=9CvVm8X#(Zj@8@5DE(#73=`=7tbj#YbqZ^ zzPY>|<&QVDQyH;ReM3}kmj5`)MJD&(Hp}ww< z=JpPh&7pk8oO(AZnw8dr@+X@FZbiA&RK9*r`L;RbJC7e)!Z+F1UWWPv<3`%dJeyG- zZ7Od;c}Y|GnmPTio74Y>Iqf?h+`OEZyj`fK19))LGCu#oyU*p(@q^3xKMnj3qlI%N delta 9532 zcmc&)eUM#cd4JE>J?Gwg&fSyjcQz#NIlD{llCO;kn;3D}_d>#k5C{epDO&rXwRfXU zXoiVS;KY2fR17Hx1~5TWiY5XJL9wI|U{gsAGt{B2Gt|~HvGNC6XM~21l-fyuzxUj) zEUPm*&X`Q@`~CGizvuTn?|br-dz@YOILE4%`JL0>+-&_pnjH8u^Mb^=wO=SX9(N1~ z*`d6uG^81RT6UC8KbptHi{{G?X-S*4s-qektVQRtX@~m^j2<43XJhniId8=5Kr(;C z43UTt)0eZA#FUs9Lq;%awls!JvC`_1vlBG>)e&<5m9tfdTp5#1nd9*+&Zc^*u_&8i zWJn}_WXf6Uuu%=K39mFq`B%;=n}>}xHZtm{QC%!V6tV6hBbHb&wxo$g!@1bI8D12( ztqBa-rv61#!(GgrNkmnO=FZxr;LgnJvvs0AX<&FHnAj1|`7*Ya!`5=r;L&nCp(k_d zKBHQ(S6XOL2av_-nB9r2mu24;joR0dt@TaLG^#5*`NvpkO&BuQC~RrTUym8F++nt3 zfe^)VRS8uNCyhBMJfv5HfOYdm)P&WjgiW6j6XQAde_0SQA(dVxY3J&Agd?pPumYI^ss;@4cVWrJ+vF5)QBmj|9Gf7DXH&?YGl=P z<3cyCG**jE(pBba@c{taQ0I)T%3KR&xLVD$92W)wKrvTGh$)c=WW$y!5A%*z<4#Q5 z$@z*|Qowkx8Ws9QqFx3hJhXdixC*llGjqbaUfCQDP(Ng?ieqU5{(ycgCa=dvgp7}j zM?FAJPou@S#vWDLL}bs}Ag8A8m=dZt+#Ct2us(e@ttq{1;suZlCaa6yLcNwe1mBYh z-o{d~ksjFJVf3frv7U7Gm;WS*WdU3i3nqe+DM%51OjkxYBx-F~v$d^C)uNYfQ3eq2 zaf?7jraT7+cj~8>4kjamI761#a%6K7n2-R#k?}Y$&2{!@f`dESv{g3#eX$5^juSxq zZ@jgJC{3APABbX6ixY*lf!Qifv$x2-+Bk?&RNtqZL&_Po>G7{@O9FC{Edym&)!Ez8 znp6M7q!@0J0mjitAQXTYM_&{Ra-kmMVT{)pv*#}`#>vKbG^7GRmvJ93lZj5O7MDqf z(5e73LPIsfszb_d&-l*)|HQrZJlJ?{Ew7UdqajogW*80ApB`R3JTvCHAo{L`pz?Lh z+zu?xQMPuVF>1EOy|NvDd)M{FNR_<-sdBbO zs3!f(b+Ei>F7ZHX%F|W#ka+XTLym_-?-6q@a^vvIHUb6Rh~3Fv^y;XF@8>B!J)G`d z!2H_Dd!~lA`aG7|o=q86VYP?{C>08%OPU0K7dt{ch&Z5JcsUuSJAd9)4ACuo*nm32 z^yP=*C9qjZNeRlIva87lCdKk_vka{|sr>4^#GzJ7oZV52suhzg>?n(|i=$q@X9^2T zxsDLS7d-M3*b;{fp<AozpE zjWt0a>(K!<{?f#hP_BBFNiN*v`$bto3|G~Ly3?qWFye(j2Z%tzHv#7@@q&Yxa0Oz* z$jvwwem)QB=|I3suYUZOk!-=DN3FYjt4^{O#QezP?Qo@AOQ znHET^8C9{KlnuK_sfldR~buRS_&ttZVo-mLTSRyYrDBsu^KBvn}7 z7bAqwL|0I&Hb5C7|)n@2N>9E^GjChzY)7@XFA(5r!c6;b{tVJI#oh)Y=kj^kjotV+}nBV=*OL z84jnbpNM|_@MCm;clzKHv)>hhSTkBjJ4KQ-MRHyU0zV3SX{WuIhNB@*sm3I&DxiH*vy5(=&cBjGTzXLj-Hwd)V`P|Wbem<4E zM4Xe}`P4$gNuPOY34VW~e|wKJKX&~8Hl9b$4uyyxEkuZ%LPVm4h}$rP$Vv(kXq5kS6m)gqJRPW?4URfy$V% zZiWv(#xbOg0NUU^5B+vMvp9i=0Xt_`J5$H3xaE1V#X!(f?s#nI*vQ(@7li?@z1J+YuIQ1h#ruzkcCGDwG>T^i=q8{j_25JHgtg7j0*UfLD& z0mzhgb&(M2`Q*xMsr1=r--G9;H=Z4w)7UEnDzT@@Ju=?25{U8ArO#b;863ZM%A`WX zZ;X4PVZmFrp~(t#s!rSst6PfOG;dM_0As)`2S!Vudv3wJ84&u>CPGu|`N%@dmo9kz zwqypg{$C=}hgtI-2$*bM`#cG|p+_u|i)yjoZMBQL52IptJfPO@mhMd5@2Ypl4b_l% zU4muR`sZfq{{Z_Jup7RuVS4=rRG?>ZX6k-~-Am|>XSWh=N*kfE68)=(1TU5XQ-oFR za5>+aQs#Cp$!^Ir9PeB8{qo%ogAt`QuFm!D>6=r%-6g)JqD?@ACs3_OA0l~28dKulkmd|sI@N6p<(&j?}&pI4?;cnI{>3V4+< zo#V6g3e?P{N8@8@J%%SpW|}Q|p>?_&^(1(6uFqFS?s~n#wD~w&?M@K!`S=*lUc!Sn z8Il1GT~D8W)~!`?1AXwC&_EwSgqP^D`L>auUrznPpqcY}lT&-lYKXeBxyRIP6K12W zY#zw!?rzqpL+Fp^=CXMxYdhR*E1O5Nx<{LJ>Phg52)E)gy9=+=N!gq-NnbhjLQAKY zP3mFRkagIonAgH1&&rL`4IJ1}zYr`ys`SOAO_@vgeS7|md=PIggga~YbeLnv zGM3F3uCzX$m)FKNNAsz$TG zW)>n@91Nhqt-?_gdaHB->W(byBMX@7J!3AP-#>7c4S~fxHqD~70gq5QV)#)=o#{hN z!FP8seH%hbM$8H18Cl(p6bxx8e=`7hG%Fcd=`c$WE8j{FAkV-9zt|VBp5qDeTN9{D z5~ZZg?JKQM$+oh2BhC!0Y>sRMStx%zw$|FX(MPVCKq5c_j$qI`1=3~`^6{3=0|RX) z@F56$V=FM=guwhV9FIVKuA_|N8Z3=ZCO{S-!v~s3$T^TeG)IS z#oH%o46`yWk3Sa;Vs_&e8RK(`ECSa!$>0%0R}zA@x59(l+s0A-0Uajz5Dso_f)uWjNz;B5{CevYRFuQuO@cGc-LSV z09M^sE8t;5sMC9DEg;i0`A#QHt3I_wc#$%gy1La<~CMe+=#E1+H_)tI$tj|C&mVqE(zpG;Rw+Mpt z%cuRs{T0d&n#y;##^HI&r|QEN)#E9)25Fd}hI9Z(i(dk1eVRo$&BebxOs@}HF#Vez zE5me{{_w?)1U8bPkZK{eLkF=XK(;*^FcE10j-aD(DZl~Dj;V7K?}#huUDyK#%z%xg z)%YYv#-ll4D4WS*?G)Bw1Jc`xKik*Bncl=H@ToX4;3F#nGuirXho?c{rh7x6ffC9& zd<>3ad_lzDfxVR<|Ld9W??3s@&mKr_#iz}s6MJW>L><;{8?r_}2rRbItBe*~hiWRp z1{5iJ`tIvTUKsNv*uOOTVhiGlT#3Ys5Yt2kO#vn#WN~c(b0m*~Y^2MN?PX9ad1ZhU z!Y>3c)yBc#dMLU8+RoeG5o4O)K^k~5Hpv&ZUn(xib{0_NgaYgp+qeee=UG(G zINw$mN+zEm#z>h(Gkbz}0kTMH9) zChEprqxT<{B{@ECSA$8m{vkamzge|y}jX{%jugtK@;YVDARcN|SH)X`7o8Sa> z)oYr|vfg-wLr{g@n{`tbHq?4=XVtP=n38qJwY3>jVbp;vlDS(mGrE|gGP?J9)0mS; zg|}d6SSoy&h!QDEm(00J%kp9s=9W3Hq6e@d(twmOXmezg!)Z!vRhKbX0K ztI-rQUon6jJ@P0=Hv8^Dt9rTC7rR(8n{=(C)7gmCuq5_^Z2+(8(c3W7XED({9B@7l z*741g-+k+bM;KK$Ekgrb1-!_&%JP+rl}&hlVOc`Ndx)5^=Hl^3*>Ka2dlW;qeJ zqtM^15O*Md#kBI0mhu;oZ!P~Z$``k^v%Gj(xpn?8p{+xx1AC`L@oX+%KljmlPh4d5 zB_%_Ma?9vhD7UWGj`Bqq%tNd-0TgBs|{JzF~%Y|5niq1xb5FeRVzIj@C$5Y-S;~HztrKpcFZotie xq);AeDPNBA!j|&2)B0~>|9f{VHTt?gfovqy(L14{f6Lb={Pa`bU24qj`+t2;b%p={ diff --git a/pyroscope/pprof-bin/src/lib.rs b/pyroscope/pprof-bin/src/lib.rs index a87cbee7..da5af679 100644 --- a/pyroscope/pprof-bin/src/lib.rs +++ b/pyroscope/pprof-bin/src/lib.rs @@ -6,20 +6,18 @@ use lazy_static::lazy_static; use pprof_pb::google::v1::Function; use pprof_pb::google::v1::Location; use pprof_pb::google::v1::Profile; -use pprof_pb::querier::v1::FlameGraph; use pprof_pb::querier::v1::Level; +use pprof_pb::querier::v1::FlameGraph; use pprof_pb::querier::v1::SelectMergeStacktracesResponse; +use std::panic; use prost::Message; use std::collections::{HashMap, HashSet}; -use std::io::stderr; use std::slice::SliceIndex; use std::sync::Mutex; use std::vec::Vec; use wasm_bindgen::prelude::*; use ch64::city_hash_64; use ch64::read_uint64_le; -use ch64::hash_128_to_64; -use std::panic; pub mod pprof_pb { @@ -69,6 +67,21 @@ fn find_node(id: u64, nodes: &Vec) -> i32 { n } +fn get_node_id(parent_id: u64, name_hash: u64, level: u16) -> u64 { + let mut node_bytes: [u8; 16] = [0; 16]; + for i in 0..8 { + node_bytes[i] = ((parent_id >> (i * 8)) & 0xFF) as u8; + } + for i in 0..8 { + node_bytes[i+8] = ((name_hash >> (i * 8)) & 0xFF) as u8; + } + let mut _level = level; + if _level > 511 { + _level = 511; + } + (city_hash_64(&node_bytes[0..]) >> 9) | ((_level as u64) << 55) +} + fn merge(tree: &mut Tree, p: &Profile) { let mut functions: HashMap = HashMap::new(); for f in p.function.iter() { @@ -112,7 +125,9 @@ fn merge(tree: &mut Tree, p: &Profile) { let location = locations[&s.location_id[i]]; let name = &p.string_table[functions[&location.line[0].function_id].name as usize]; let name_hash = city_hash_64(name.as_bytes()); - let node_id = hash_128_to_64(parent_id, name_hash); + let node_id = get_node_id( + parent_id, name_hash,(s.location_id.len() - i) as u16 + ); if !tree.nodes.contains_key(&parent_id) && tree.nodes_num < 2000000{ tree.nodes.insert(parent_id, Vec::new()); } diff --git a/pyroscope/pyroscope.js b/pyroscope/pyroscope.js index a16fc4d9..1d4b8c1a 100644 --- a/pyroscope/pyroscope.js +++ b/pyroscope/pyroscope.js @@ -167,10 +167,53 @@ const labelSelectorQuery = (query, labelSelector) => { )) } +const serviceNameSelectorQuery = (labelSelector) => { + const empty = Sql.Eq(new Sql.Raw('1'), new Sql.Raw('1')) + if (!labelSelector || !labelSelector.length || labelSelector === '{}') { + return empty + } + const labelSelectorScript = compiler.ParseScript(labelSelector).rootToken + let conds = null + for (const rule of labelSelectorScript.Children('log_stream_selector_rule')) { + const label = rule.Child('label').value + if (label !== 'service_name') { + continue + } + const val = JSON.parse(rule.Child('quoted_str').value) + let valRul = null + switch (rule.Child('operator').value) { + case '=': + valRul = Sql.Eq(new Sql.Raw('service_name'), Sql.val(val)) + break + case '!=': + valRul = Sql.Ne(new Sql.Raw('service_name'), Sql.val(val)) + break + case '=~': + valRul = Sql.Eq(new Sql.Raw(`match(service_name, ${Sql.quoteVal(val)})`), 1) + break + case '!~': + valRul = Sql.Ne(new Sql.Raw(`match(service_name, ${Sql.quoteVal(val)})`), 1) + } + conds = valRul + } + return conds || empty +} + const selectMergeStacktraces = async (req, res) => { return await selectMergeStacktracesV2(req, res) } +const sqlWithReference = (ref) => { + const res = new Sql.WithReference(ref) + res.toString = function () { + if (this.ref.inline) { + return `(${this.ref.query.toString()}) as ${this.ref.alias}` + } + return this.ref.alias + } + return res +} + const selectMergeStacktracesV2 = async (req, res) => { const dist = clusterName ? '_dist' : '' const typeRegex = parseTypeId(req.body.getProfileTypeid()) @@ -182,15 +225,21 @@ const selectMergeStacktracesV2 = async (req, res) => { ? Math.floor(parseInt(req.body.getEnd()) / 1000) : Math.floor(Date.now() / 1000) const v2 = checkVersion('profiles_v2', (fromTimeSec - 3600) * 1000) + const serviceNameSelector = serviceNameSelectorQuery(sel) + const typeIdSelector = Sql.Eq( + 'type_id', + Sql.val(`${typeRegex.type}:${typeRegex.periodType}:${typeRegex.periodUnit}`) + ) const idxSelect = (new Sql.Select()) .select('fingerprint') .from(`${DATABASE_NAME()}.profiles_series_gin`) .where( Sql.And( Sql.Eq(new Sql.Raw(`has(sample_types_units, (${Sql.quoteVal(typeRegex.sampleType)},${Sql.quoteVal(typeRegex.sampleUnit)}))`), 1), - Sql.Eq('type_id', Sql.val(`${typeRegex.type}:${typeRegex.periodType}:${typeRegex.periodUnit}`)), + typeIdSelector, Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)), - Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)) + Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)), + serviceNameSelector ) ).groupBy('fingerprint') labelSelectorQuery(idxSelect, sel) @@ -205,7 +254,9 @@ const selectMergeStacktracesV2 = async (req, res) => { Sql.And( Sql.Gte('timestamp_ns', new Sql.Raw(Math.floor(fromTimeSec) + '000000000')), Sql.Lte('timestamp_ns', new Sql.Raw(Math.floor(toTimeSec) + '000000000')), - new Sql.In('fingerprint', 'IN', new Sql.WithReference(withIdxSelect)) + new Sql.In('fingerprint', 'IN', sqlWithReference(withIdxSelect)), + typeIdSelector, + serviceNameSelector )) if (process.env.ADVANCED_PROFILES_MERGE_LIMIT) { rawReq.orderBy(['timestamp_ns', 'desc']).limit(parseInt(process.env.ADVANCED_PROFILES_MERGE_LIMIT)) @@ -214,15 +265,16 @@ const selectMergeStacktracesV2 = async (req, res) => { const joinedReq = (new Sql.Select()).with(withRawReq).select([ new Sql.Raw('(raw.tree.1, raw.tree.2, raw.tree.3, sum(raw.tree.4), sum(raw.tree.5))'), 'tree2' - ]).from(new Sql.WithReference(withRawReq)) + ]).from(sqlWithReference(withRawReq)) .join('raw.tree', 'array') .groupBy(new Sql.Raw('raw.tree.1'), new Sql.Raw('raw.tree.2'), new Sql.Raw('raw.tree.3')) + .orderBy(new Sql.Raw('raw.tree.1')).limit(2000000) const withJoinedReq = new Sql.With('joined', joinedReq, !!clusterName) const joinedAggregatedReq = (new Sql.Select()).select( - [new Sql.Raw('groupArray(tree2)'), 'tree']).from(new Sql.WithReference(withJoinedReq)) + [new Sql.Raw('groupArray(tree2)'), 'tree']).from(sqlWithReference(withJoinedReq)) const functionsReq = (new Sql.Select()).select( [new Sql.Raw('groupUniqArray(raw.functions)'), 'functions2'] - ).from(new Sql.WithReference(withRawReq)).join('raw.functions', 'array') + ).from(sqlWithReference(withRawReq)).join('raw.functions', 'array') let brackLegacy = (new Sql.Select()).select( [new Sql.Raw('[]::Array(String)'), 'legacy'] @@ -236,8 +288,10 @@ const selectMergeStacktracesV2 = async (req, res) => { Sql.And( Sql.Gte('timestamp_ns', new Sql.Raw(Math.floor(fromTimeSec) + '000000000')), Sql.Lte('timestamp_ns', new Sql.Raw(Math.floor(toTimeSec) + '000000000')), - new Sql.In('fingerprint', 'IN', new Sql.WithReference(withIdxSelect)), - Sql.Eq(new Sql.Raw('empty(tree)'), 1) + new Sql.In('fingerprint', 'IN', sqlWithReference(withIdxSelect)), + Sql.Eq(new Sql.Raw('empty(tree)'), 1), + typeIdSelector, + serviceNameSelector )) if (process.env.ADVANCED_PROFILES_MERGE_LIMIT) { legacy.orderBy(['timestamp_ns', 'desc']).limit(parseInt(process.env.ADVANCED_PROFILES_MERGE_LIMIT)) @@ -245,7 +299,7 @@ const selectMergeStacktracesV2 = async (req, res) => { withLegacy = new Sql.With('legacy', legacy, !!clusterName) brackLegacy = (new Sql.Select()) .select([new Sql.Raw('groupArray(payload)'), 'payloads']) - .from(new Sql.WithReference(withLegacy)) + .from(sqlWithReference(withLegacy)) } brackLegacy = new Sql.Raw(`(${brackLegacy.toString()})`) const brack1 = new Sql.Raw(`(${joinedAggregatedReq.toString()})`) @@ -331,12 +385,18 @@ const selectSeries = async (req, res) => { } const aggregation = _req.getAggregation && _req.getAggregation() + const typeIdSelector = Sql.Eq( + 'type_id', + Sql.val(`${typeID.type}:${typeID.periodType}:${typeID.periodUnit}`)) + const serviceNameSelector = serviceNameSelectorQuery(labelSelector) + const idxReq = (new Sql.Select()) .select(new Sql.Raw('fingerprint')) .from(`${DATABASE_NAME()}.profiles_series_gin`) .where( Sql.And( - Sql.Eq('type_id', Sql.val(`${typeID.type}:${typeID.periodType}:${typeID.periodUnit}`)), + typeIdSelector, + serviceNameSelector, Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)), Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)), Sql.Eq(new Sql.Raw( @@ -361,7 +421,9 @@ const selectSeries = async (req, res) => { .where(Sql.And( new Sql.In('fingerprint', 'IN', new Sql.WithReference(withIdxReq)), Sql.Gte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(fromTimeSec)}))`)), - Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)) + Sql.Lte('date', new Sql.Raw(`toDate(FROM_UNIXTIME(${Math.floor(toTimeSec)}))`)), + typeIdSelector, + serviceNameSelector )) const withLabelsReq = new Sql.With('labels', labelsReq, !!clusterName) @@ -388,7 +450,9 @@ const selectSeries = async (req, res) => { Sql.And( new Sql.In('p.fingerprint', 'IN', new Sql.WithReference(withIdxReq)), Sql.Gte('p.timestamp_ns', new Sql.Raw(`${fromTimeSec}000000000`)), - Sql.Lt('p.timestamp_ns', new Sql.Raw(`${toTimeSec}000000000`)) + Sql.Lt('p.timestamp_ns', new Sql.Raw(`${toTimeSec}000000000`)), + typeIdSelector, + serviceNameSelector ) ).groupBy('timestamp_ns', 'fingerprint') .orderBy(['fingerprint', 'ASC'], ['timestamp_ns', 'ASC'])