From f59d7acee652591a0d9e04bc40659f71331e7361 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 24 Jun 2020 14:50:24 -0700 Subject: [PATCH] Arrow DataFrame: cast BigInt/UInt to numbers (#25811) --- .../src/dataframe/ArrayDataFrame.test.ts | 12 +++- .../src/dataframe/ArrowDataFrame.ts | 54 +++++++++------- .../__snapshots__/ArrowDataFrame.test.ts.snap | 61 ++++++++++-------- .../__snapshots__/all_types.golden.arrow | Bin 11114 -> 10578 bytes 4 files changed, 78 insertions(+), 49 deletions(-) diff --git a/packages/grafana-data/src/dataframe/ArrayDataFrame.test.ts b/packages/grafana-data/src/dataframe/ArrayDataFrame.test.ts index 00fea82311d..6af98789d15 100644 --- a/packages/grafana-data/src/dataframe/ArrayDataFrame.test.ts +++ b/packages/grafana-data/src/dataframe/ArrayDataFrame.test.ts @@ -7,13 +7,15 @@ describe('Array DataFrame', () => { { name: 'first', value: 1, time: 123 }, { name: 'second', value: 2, time: 456, extra: 'here' }, { name: 'third', value: 3, time: 789 }, + { name: '4th (NaN)', value: NaN, time: 1000 }, + { name: '5th (Null)', value: null, time: 1100 }, ]; const frame = new ArrayDataFrame(input); frame.name = 'Hello'; frame.refId = 'Z'; frame.setFieldType('phantom', FieldType.string, v => '🦥'); - const field = frame.fields.find(f => f.name == 'value'); + const field = frame.fields.find(f => f.name === 'value'); field!.config.unit = 'kwh'; test('Should support functional methods', () => { @@ -48,6 +50,8 @@ describe('Array DataFrame', () => { "first", "second", "third", + "4th (NaN)", + "5th (Null)", ], }, Object { @@ -61,6 +65,8 @@ describe('Array DataFrame', () => { 1, 2, 3, + NaN, + null, ], }, Object { @@ -72,6 +78,8 @@ describe('Array DataFrame', () => { 123, 456, 789, + 1000, + 1100, ], }, Object { @@ -83,6 +91,8 @@ describe('Array DataFrame', () => { "🦥", "🦥", "🦥", + "🦥", + "🦥", ], }, ], diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts index 20d5045a528..4aceeaf865e 100644 --- a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts +++ b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts @@ -1,4 +1,5 @@ import { DataFrame, FieldType, Field, Vector } from '../types'; +import { FunctionalVector } from '../vector/FunctionalVector'; import { Table, @@ -48,14 +49,18 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame { if (col) { const schema = table.schema.fields[i]; let type = FieldType.other; - const values: Vector = col; + let values: Vector = col; switch ((schema.typeId as unknown) as ArrowType) { case ArrowType.Decimal: - case ArrowType.Int: case ArrowType.FloatingPoint: { type = FieldType.number; break; } + case ArrowType.Int: { + type = FieldType.number; + values = new NumberColumn(col); // Cast to number + break; + } case ArrowType.Bool: { type = FieldType.boolean; break; @@ -73,7 +78,7 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame { } fields.push({ - name: stripFieldNamePrefix(col.name), + name: col.name, type, values, config: parseOptionalMeta(col.metadata.get('config')) || {}, @@ -92,17 +97,6 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame { }; } -// fieldNamePrefixSep is the delimiter used with fieldNamePrefix. -const fieldNamePrefixSep = '🦥: '; - -function stripFieldNamePrefix(name: string): string { - const idx = name.indexOf(fieldNamePrefixSep); - if (idx > 0) { - return name.substring(idx + fieldNamePrefixSep.length); - } - return name; -} - function toArrowVector(field: Field): ArrowVector { // OR: Float64Vector.from([1, 2, 3])); @@ -129,17 +123,10 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table { if (table instanceof Table) { return table as Table; } - // Make sure the names are unique - const names = new Set(); table = Table.new( data.fields.map((field, index) => { - let name = field.name; - if (names.has(field.name)) { - name = `${index}${fieldNamePrefixSep}${field.name}`; - } - names.add(name); - const column = Column.new(name, toArrowVector(field)); + const column = Column.new(field.name, toArrowVector(field)); if (field.labels) { column.metadata.set('labels', JSON.stringify(field.labels)); } @@ -161,3 +148,26 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table { } return table; } + +class NumberColumn extends FunctionalVector { + constructor(private col: Column) { + super(); + } + + get length() { + return this.col.length; + } + + get(index: number): number { + const v = this.col.get(index); + if (v === null || isNaN(v)) { + return v; + } + + // The conversion operations are always silent, never give errors, + // but if the bigint is too huge and won’t fit the number type, + // then extra bits will be cut off, so we should be careful doing such conversion. + // See https://javascript.info/bigint + return Number(v); + } +} diff --git a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap index 29d44ce7138..89c69fe8459 100644 --- a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap +++ b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap @@ -21,6 +21,7 @@ Object { Object { "config": Object { "decimals": 2, + "displayName": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)", "filterable": false, "links": Array [ Object { @@ -33,7 +34,6 @@ Object { "min": null, "noValue": "😤", "nullValueMode": "null", - "title": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)", }, "labels": Object { "aLabelKey": "aLabelValue", @@ -136,11 +136,11 @@ Object { "name": "int64_values", "type": "number", "values": Array [ - "\\"-9223372036854775808\\"", - "\\"-9007199254740991\\"", - "\\"1\\"", - "\\"9007199254740991\\"", - "\\"9223372036854775807\\"", + -9223372036854776000, + -9007199254740991, + 1, + 9007199254740991, + 9223372036854776000, ], }, Object { @@ -149,11 +149,11 @@ Object { "name": "nullable_int64_values", "type": "number", "values": Array [ - "\\"-9223372036854775808\\"", - "\\"-9007199254740991\\"", + -9223372036854776000, + -9007199254740991, null, - "\\"9007199254740991\\"", - "\\"9223372036854775807\\"", + 9007199254740991, + 9223372036854776000, ], }, Object { @@ -240,11 +240,11 @@ Object { "name": "uint64_values", "type": "number", "values": Array [ - "\\"0\\"", - "\\"0\\"", - "\\"1\\"", - "\\"9007199254740991\\"", - "\\"18446744073709551615\\"", + 0, + 0, + 1, + 9007199254740991, + 18446744073709552000, ], }, Object { @@ -253,11 +253,11 @@ Object { "name": "nullable_uint64_values", "type": "number", "values": Array [ - "\\"0\\"", - "\\"0\\"", + 0, + 0, null, - "\\"9007199254740991\\"", - "\\"18446744073709551615\\"", + 9007199254740991, + 18446744073709552000, ], }, Object { @@ -351,6 +351,19 @@ Object { 9223372036854.775, ], }, + Object { + "config": Object {}, + "labels": undefined, + "name": "timestamps", + "type": "time", + "values": Array [ + 0, + 1568039445000, + 1568039450000, + 9007199254.740992, + 9223372036854.775, + ], + }, Object { "config": Object {}, "labels": undefined, @@ -366,13 +379,9 @@ Object { }, ], "meta": Object { - "limit": 4242, - "searchWords": Array [ - "Grafana", - "❤️", - " 🦥 ", - "test", - ], + "custom": Object { + "Hi": "there", + }, }, "name": "many_types", "refId": "A", diff --git a/packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow b/packages/grafana-data/src/dataframe/__snapshots__/all_types.golden.arrow index 67f764eee6c3f3929578541172ad74ae2d91d495..045140591306943bb61cca04314e176eb28a832d 100644 GIT binary patch literal 10578 zcmeHNU2Ggz6~1e4U6YzB1UDF{#Ap;%uH?i{2vM*gc1YBQl0wu(#cGR^u|1m|vO8n# z&IVg1vL7lGMM#7wLU{;kg#u3%iclY-K1IMoB>I3V9xBv_sOm%1ml!A^l%M7M?%cD! zduMko5RZGEvvs$Y3S5T{na2|Ii0R7h4oB6 z4yqxV)w_mo7oI;0SoIeg^)#va)2sfm$e;Gpa##!f3l~76-D;QysMO~HXC6lzI&kbU zrG5(|Cjf7MLa8~xAJDlofcG9%Y7y|kkWwk&BXsgrKoM{i3P%Cg;oOseHwKiN0NjB? z#{nO}#`giVd;X(llfE1?0Iv#s2Aih8{-)!An5dHj5vzISM*l)bEaCY?;d0dSGn1d=Iv;vHV}yc`XnYwl_W?H>=)NuTR;0Z z$8n&i%r8wm9)(^4${=g8Jq0>_8v`Islw&d%A3HfWz-A}j+TVKndEMCZU?=bA7x#7Z z^OCk%=qd9P6Fc-Riybv+kMS7jIX^c|=KMtLVQw43A4_4=UrT2Gq3iEJ+k5??vFSm> zPC)CuHlo;itvTE8AO~l+kokd$DeJus*za{1bn*9-(B+uS*?a~$Nt>N`pZj}nKT~7N zgM7|UYrKk}+qv=!*l zo3qJ08%yRz6HDFveAn1!o=V>>@tn<_$Qjz~#QV8_^!D>8`XYWl0Ud2|d$8qZ z&L;Dea=l~@nOLHqCQk?KJiTpf)6WqAt|+{{<{1hZ{hDQ z#CbP=?R+Rgx1F!c#;%8U*I!%CH}TcY-y6oh<*zNC^LN9tYy55F{}A+E?{^ga>E>^* z94hPeRTY>=uD`b24l~}}{Jm=ITmIVOIe+h3c8$Mn{C@^Y-|uQ(8d73^f+HacIyM0FCd>Mut1rAn~!O&p4#_1Rib3gWb}TQZK9~-;JL3HeUP1&pUF}D@j^5Tl!ue6q;i5r?u$6rY3G%d37GrPSyjP2wO8 zz1`laa_F7ZrjI0XJ*=G%(kO|&XOc#|s2?Go2DP(c`n5_BpYx~FS_6G))GE+YPSd6O z^yH-OJEEf{zh@z-PL`AJ?M{>3<*>3exv1?;`WL=4;m1if)YJaPYu9f07jm&RiX7fB&FvX2b)RRoutM@3j5h$!1in4fr?f+*m}x8a9miVp4Ax6>83XMuFe=-_?8VmYwBC3mP&AuzikMD-*NEY zi=e3m-)```rWW~o1{eB6gG(j&nDBAqfgyR^jw{NQ)2VD!Ns2`hdyU; zQ6aeKUvlJ^4KDP{j=bP9{F{LRQL8x?fvo|wZ5*_)}V(m$0E2| z$DeAhHAzGfy4w!YI)QPAWpRyw$4!1iXPZqYj-ut1y4#ev1rN#>^!kQ}wU%9nKSQ&1 z(B{XE!L_-%jsk`{o?4(CzSZ6w0{_Anc zbgj=|IMxW{6FsDD+S;UVwo&8EZz!YG26V8@9LehYt?}*5Fa5c{mhsOVVeGMpeSD^G zFIR>R|M+*menP!Ea_$Q1Os?gR?z|D^lJIOMme|KVkqinw;>ffE1ds62jqlnOv zGc%*8e|OTlMpON}lWczN-<@>67w_Ml^zTmkcPH(4f&IG^{ui45-AVuMq{w@K&OPVMoOAA(Gc$L5@X05?^7N=ub(D#ZDYZwj+@(IE`qka44=ttk z>SwCZ3#=dQ-ST`l+E)R+fa8EcKp$Y{R=rLdc?*DpfI9${aN+TJ)T=^gzwEypBxg@# zXP3gNkxznhh{x(x;X8^p`#9%U!=N&^_;gyCull90`HxkCg&+z1A^)8ptle7wE*jp( z%Rl;|hej4wGym&DejJsf%rA`<$BO6C2lXq5S%6A?5-{^or9Ob7BY?HLVFTbYlo|kB z_z-LbWH9EBcPjM^;O$*X9R*y6L5~2gqQfD;8l12faJdI#0G@~8eSnJ?gJVAX*Lt1y zJ|O_h3v7c$(=TrrpX~?!DM0Zd8dCC-S{w(nad;w&%CIOXFI6?a^$}Rrw?3%X`4+a5 zd+EM#I0OjY{;9|si2j1IY>TgHB0#%{-7`I{M&@j*~(SCicrWz0cP zIBxMl$L`rQjZd5j;u@pF_$qJvE0Eub+zWqcv3=KJ^k;3~3%a%a9_mpvdgO=ad(4l$ z4>Ij|PyOAsySG65?RXbtJlcI9#~XF*UWn5m8y~Z#yJJ`W!*|}ZBiZju>~S6uPIyLY>%ec|SQK3bLKZp*UWWIB|vme%E z?1mrQd`KlaypvP^uGc;5hiznkE-~894@OuY+xcMv^d>(@j&xLL(Sdn_I%q$%k$K_$&VG1Z z#-<-W4H$LvC6$S{<_YS)6WL!&Y|;;sCwlUEVhVKYhrQHA@};9fiw-7F^yxf7 z`A)L0{j1d<(~z+_`z_Id{^0uOrp@Lo(YkI<=yh8argkX*dx_H{;3uFo<%;8=(hkl~}2^hAJ_rrA=*ZRRtmrx*=C0hM3g*l2&3m$%``*!m0 zNSwCw!!w}g{U9l`Ne$NzZ{>8LAGVVD1$a97;de5w^@E%8pg=B5wEE$QUdvU1^OSzr zPX1kq&vt&82R-ixNqw?0H^F-WfYS3Ee*hBz_Pq!m>jS+%Hp^#Tg{B;FCC87W7|uK8 zDDg`pLw-3}_DiEYzi^K+mnI9*Ni)viG?&=4Sb?!mkuVwr6KA-;F z2gM0EsTBrL^Gx?5;4px;j3NFw#vnlFP-k0G%_>oHa_juNB(hVGeF$*Q4~_=2Vf^Lr ztY4D$SGmK-v28X-F>OP!w&O4q##N;_XQk@IX{cCI7EMPDD=#c&>MWB5}t|D zTGgvArnPw9n+?5193wL?ET^ZU`^|xIE3RssW{biF%@(3K3oAT}`K5&*u7*SYd^i`C z(ez72yd%CV9@pe`nI?a=1;^5PYDXNM#ZhvaN7zlZ52s0xh28=0_+scirH4M0Ce^TV zCdi^R@xGYWl6l?a;Wnt846`r9L2}wJWtAFqsa0ZhznEo9)za{=)*W%ukbht+7aC!`#^Zy_@*r3gE`q>v&h|0F*n} zW_+yz9sw}EF3bAtq0M-dCDje9rulMBWcGuf^R<0ko!9fm#*UdYeR|F?hNvq;{~Qc= z=T{#_q<)=r0$G1Qna1}3^0D;e6u_C*4{7#4O@mQ{-pg+(1EAP%DSW;R9(2(tjQ+*<45^o;o|1Z9YFk;{%D*85i(hl_-&#SbExt$a8&X^On}S>V zZNY72@d3-n#ExR+2L!kD5y1@?;zsa6dBFU@V&!KX`g4xF#VtR-RDBSL!69Si1Whq*Vgr49ndL z0nJT*q4QauP8>zcDRrZ6;}kq7_v!Ttjg`i@4!=Rox|Zk1PFo+Ca&j6ce?j+ZBy)5; z;F#vQ{S|vi(3EC=!=;Hrw{o^_V^4jneyj{@(`Une?HAM6>LlE;LLi^*hqTSHHtCzs z*yH4{D5KN{`e2znWa?WD|8C81^|`fT_$LoB{#fkzxPE9Zmv$Ze-f!N#S6%2oeF?RZ zYuM8muOVdl<$=r3b*LY^5}pXS7vWxo&xQb;V@#;j{Rk%n%8VpFyPq*n`P;o4>fQ}C z-;Z?fhPJ*(>fQ~R+q~}G5U%)gUnjeFL*|mVdpBhK(7hWne(2r}