Live: handle influx input with incomplete/asymmetrical field set (#36664)

This commit is contained in:
Alexander Emelin 2021-07-13 19:17:59 +03:00 committed by GitHub
parent 2026a1dfb6
commit 607c5d2555
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 268 additions and 7 deletions

View File

@ -131,7 +131,18 @@ func (c *Converter) convertWithLabelsColumn(metrics []influx.Metric) ([]telemetr
frameWrappers := make([]telemetry.FrameWrapper, 0, len(metricFrames))
for _, key := range frameKeyOrder {
frameWrappers = append(frameWrappers, metricFrames[key])
frame := metricFrames[key]
// For all fields except labels and time fill columns with nulls in
// case of unequal length.
for i := 2; i < len(frame.fields); i++ {
if frame.fields[i].Len() < frame.fields[0].Len() {
numNulls := frame.fields[0].Len() - frame.fields[i].Len()
for j := 0; j < numNulls; j++ {
frame.fields[i].Append(nil)
}
}
}
frameWrappers = append(frameWrappers, frame)
}
return frameWrappers, nil
@ -233,11 +244,27 @@ func (s *metricFrame) append(m influx.Metric) error {
v = nil
}
}
// If field does not have a desired length till this moment
// we fill it with nulls up to the currently processed index.
if field.Len() < s.fields[0].Len()-1 {
numNulls := s.fields[0].Len() - 1 - field.Len()
for i := 0; i < numNulls; i++ {
field.Append(nil)
}
}
field.Append(v)
} else {
field := data.NewFieldFromFieldType(ft, 1)
field := data.NewFieldFromFieldType(ft, 0)
field.Name = f.Key
field.Set(0, v)
// If field appeared at the moment when we already filled some columns
// we fill it with nulls up to the currently processed index.
if field.Len() < s.fields[0].Len()-1 {
numNulls := s.fields[0].Len() - 1 - field.Len()
for i := 0; i < numNulls; i++ {
field.Append(nil)
}
}
field.Append(v)
s.fields = append(s.fields, field)
s.fieldCache[f.Key] = len(s.fields) - 1
}

View File

@ -3,6 +3,7 @@ package telegraf
import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"path/filepath"
"testing"
@ -40,7 +41,7 @@ func checkTestData(tb testing.TB, file string) *backend.DataResponse {
dr.Frames = append(dr.Frames, w.Frame())
}
err = experimental.CheckGoldenDataResponse(filepath.Join("testdata", file+".golden.txt"), dr, true)
err = experimental.CheckGoldenDataResponse(filepath.Join("testdata", file+".golden.txt"), dr, *update)
require.NoError(tb, err)
return dr
}
@ -92,27 +93,41 @@ func TestConverter_Convert_LabelsColumn(t *testing.T) {
{Name: "same_metrics_same_labels_different_time", NumFields: 7, FieldLength: 3, NumFrames: 1},
{Name: "same_metrics_different_labels_different_time", NumFields: 7, FieldLength: 2, NumFrames: 1},
{Name: "same_metrics_different_labels_same_time", NumFields: 12, FieldLength: 13, NumFrames: 1},
{Name: "incomplete_fields", NumFields: 4, FieldLength: 4, NumFrames: 1},
{Name: "incomplete_fields_2", NumFields: 4, FieldLength: 5, NumFrames: 1},
{Name: "incomplete_fields_full", NumFrames: 5},
}
for _, tt := range testCases {
t.Run(tt.Name, func(t *testing.T) {
testData := loadTestData(t, tt.Name)
if *pprint {
fmt.Println(string(testData))
}
converter := NewConverter(WithUseLabelsColumn(true))
frameWrappers, err := converter.Convert(testData)
require.NoError(t, err)
require.Len(t, frameWrappers, tt.NumFrames)
for _, fw := range frameWrappers {
frame := fw.Frame()
require.Len(t, frame.Fields, tt.NumFields)
require.Equal(t, tt.FieldLength, frame.Fields[0].Len())
if tt.NumFrames == 1 {
require.Len(t, frame.Fields, tt.NumFields)
require.Equal(t, tt.FieldLength, frame.Fields[0].Len())
}
_, err := data.FrameToJSON(frame, data.IncludeAll)
require.NoError(t, err)
if *pprint {
s, err := frame.StringTable(100, 100)
require.NoError(t, err)
fmt.Println(s)
}
}
})
}
}
var update = flag.Bool("update", false, "update golden files")
var pprint = flag.Bool("pprint", false, "pretty print test case")
func TestConverter_Convert_NumFrameFields(t *testing.T) {
testData := loadTestData(t, "same_metrics_different_labels_same_time")

View File

@ -1,6 +1,6 @@
🌟 This was machine generated. Do not edit. 🌟
Frame[0]
Frame[0]
Name: system
Dimensions: 5 Fields by 4 Rows
+----------------+-------------------------------+------------------+-----------------+-----------------+

View File

@ -0,0 +1,4 @@
node_cpu,cpu=7,mode=user seconds_total=6410.4799999999996 1625783151607273176
node_cpu,cpu=0,mode=user guest_seconds_total=0 1625783151607273176
node_cpu,cpu=0,mode=nice guest_seconds_total=0 1625783151607273176
node_cpu,cpu=1,mode=user guest_seconds_total=0 1625783151607273176

View File

@ -0,0 +1,5 @@
node_cpu,cpu=7,mode=user seconds_total=6410.4799999999996 1625783151607273170
node_cpu,cpu=0,mode=user guest_seconds_total=0 1625783151607273175
node_cpu,cpu=0,mode=nice guest_seconds_total=0 1625783151607273175
node_cpu,cpu=1,mode=user guest_seconds_total=0 1625783151607273175
node_cpu,cpu=7,mode=user seconds_total=2410.4799999999996 1625783151607273178

View File

@ -0,0 +1,210 @@
node_cpu,cpu=0,mode=idle seconds_total=99081.979999999996 1625842606118404128
node_cpu,cpu=0,mode=iowait seconds_total=53.490000000000002 1625842606118404128
node_cpu,cpu=0,mode=irq seconds_total=0 1625842606118404128
node_cpu,cpu=0,mode=nice seconds_total=7.2599999999999998 1625842606118404128
node_cpu,cpu=0,mode=softirq seconds_total=147.97 1625842606118404128
node_cpu,cpu=0,mode=steal seconds_total=0 1625842606118404128
node_cpu,cpu=0,mode=system seconds_total=1851.9000000000001 1625842606118404128
node_cpu,cpu=0,mode=user seconds_total=7192.9799999999996 1625842606118404128
node_cpu,cpu=1,mode=idle seconds_total=2119.0999999999999 1625842606118404128
node_cpu,cpu=1,mode=iowait seconds_total=1.3899999999999999 1625842606118404128
node_cpu,cpu=1,mode=irq seconds_total=0 1625842606118404128
node_cpu,cpu=1,mode=nice seconds_total=7.2400000000000002 1625842606118404128
node_cpu,cpu=1,mode=softirq seconds_total=13.390000000000001 1625842606118404128
node_cpu,cpu=1,mode=steal seconds_total=0 1625842606118404128
node_cpu,cpu=1,mode=system seconds_total=1717.53 1625842606118404128
node_cpu,cpu=1,mode=user seconds_total=7760.1400000000003 1625842606118404128
node_cpu,cpu=2,mode=idle seconds_total=2115.0100000000002 1625842606118404128
node_cpu,cpu=2,mode=iowait seconds_total=1.1499999999999999 1625842606118404128
node_cpu,cpu=2,mode=irq seconds_total=0 1625842606118404128
node_cpu,cpu=2,mode=nice seconds_total=7.3799999999999999 1625842606118404128
node_cpu,cpu=2,mode=softirq seconds_total=909.46000000000004 1625842606118404128
node_cpu,cpu=2,mode=steal seconds_total=0 1625842606118404128
node_cpu,cpu=2,mode=system seconds_total=1772.5 1625842606118404128
node_cpu,cpu=2,mode=user seconds_total=7481.4899999999998 1625842606118404128
node_cpu,cpu=3,mode=idle seconds_total=2136.0300000000002 1625842606118404128
node_cpu,cpu=3,mode=iowait seconds_total=1.47 1625842606118404128
node_cpu,cpu=3,mode=irq seconds_total=0 1625842606118404128
node_cpu,cpu=3,mode=nice seconds_total=6.5300000000000002 1625842606118404128
node_cpu,cpu=3,mode=softirq seconds_total=9.75 1625842606118404128
node_cpu,cpu=3,mode=steal seconds_total=0 1625842606118404128
node_cpu,cpu=3,mode=system seconds_total=1721.2 1625842606118404128
node_cpu,cpu=3,mode=user seconds_total=7675.8699999999999 1625842606118404128
node_cpu,cpu=4,mode=idle seconds_total=2135.77 1625842606118404128
node_cpu,cpu=4,mode=iowait seconds_total=1.1799999999999999 1625842606118404128
node_cpu,cpu=4,mode=irq seconds_total=0 1625842606118404128
node_cpu,cpu=4,mode=nice seconds_total=9.0600000000000005 1625842606118404128
node_cpu,cpu=4,mode=softirq seconds_total=8.4199999999999999 1625842606118404128
node_cpu,cpu=4,mode=steal seconds_total=0 1625842606118404128
node_cpu,cpu=4,mode=system seconds_total=1687.9400000000001 1625842606118404128
node_cpu,cpu=4,mode=user seconds_total=8106.6899999999996 1625842606118404128
node_cpu,cpu=5,mode=idle seconds_total=2135.7800000000002 1625842606118404128
node_cpu,cpu=5,mode=iowait seconds_total=1.3600000000000001 1625842606118404128
node_cpu,cpu=5,mode=irq seconds_total=0 1625842606118404128
node_cpu,cpu=5,mode=nice seconds_total=8.3000000000000007 1625842606118404128
node_cpu,cpu=5,mode=softirq seconds_total=7.9800000000000004 1625842606118404128
node_cpu,cpu=5,mode=steal seconds_total=0 1625842606118404128
node_cpu,cpu=5,mode=system seconds_total=1656.8199999999999 1625842606118404128
node_cpu,cpu=5,mode=user seconds_total=7809.5100000000002 1625842606118404128
node_cpu,cpu=6,mode=idle seconds_total=2142.21 1625842606118404128
node_cpu,cpu=6,mode=iowait seconds_total=1.5600000000000001 1625842606118404128
node_cpu,cpu=6,mode=irq seconds_total=0 1625842606118404128
node_cpu,cpu=6,mode=nice seconds_total=11.5 1625842606118404128
node_cpu,cpu=6,mode=softirq seconds_total=7.6100000000000003 1625842606118404128
node_cpu,cpu=6,mode=steal seconds_total=0 1625842606118404128
node_cpu,cpu=6,mode=system seconds_total=1655.5599999999999 1625842606118404128
node_cpu,cpu=6,mode=user seconds_total=7865.5200000000004 1625842606118404128
node_cpu,cpu=7,mode=idle seconds_total=2136.9899999999998 1625842606118404128
node_cpu,cpu=7,mode=iowait seconds_total=1.45 1625842606118404128
node_cpu,cpu=7,mode=irq seconds_total=0 1625842606118404128
node_cpu,cpu=7,mode=nice seconds_total=7.3200000000000003 1625842606118404128
node_cpu,cpu=7,mode=softirq seconds_total=6.9900000000000002 1625842606118404128
node_cpu,cpu=7,mode=steal seconds_total=0 1625842606118404128
node_cpu,cpu=7,mode=system seconds_total=1826.8299999999999 1625842606118404128
node_cpu,cpu=7,mode=user seconds_total=7717.8699999999999 1625842606118404128
node_cpu,cpu=0,mode=user guest_seconds_total=0 1625842606118404128
node_cpu,cpu=0,mode=nice guest_seconds_total=0 1625842606118404128
node_cpu,cpu=1,mode=user guest_seconds_total=0 1625842606118404128
node_cpu,cpu=1,mode=nice guest_seconds_total=0 1625842606118404128
node_cpu,cpu=2,mode=user guest_seconds_total=0 1625842606118404128
node_cpu,cpu=2,mode=nice guest_seconds_total=0 1625842606118404128
node_cpu,cpu=3,mode=user guest_seconds_total=0 1625842606118404128
node_cpu,cpu=3,mode=nice guest_seconds_total=0 1625842606118404128
node_cpu,cpu=4,mode=user guest_seconds_total=0 1625842606118404128
node_cpu,cpu=4,mode=nice guest_seconds_total=0 1625842606118404128
node_cpu,cpu=5,mode=user guest_seconds_total=0 1625842606118404128
node_cpu,cpu=5,mode=nice guest_seconds_total=0 1625842606118404128
node_cpu,cpu=6,mode=user guest_seconds_total=0 1625842606118404128
node_cpu,cpu=6,mode=nice guest_seconds_total=0 1625842606118404128
node_cpu,cpu=7,mode=user guest_seconds_total=0 1625842606118404128
node_cpu,cpu=7,mode=nice guest_seconds_total=0 1625842606118404128
node_disk,device=nvme0n1 reads_completed_total=1.3411377179449652e-304 1625842606119336434
node_disk,device=dm-0 reads_completed_total=2.1994805592372739e-304 1625842606119375842
node_disk,device=dm-1 reads_completed_total=2.1922059384353575e-304 1625842606119395462
node_disk,device=dm-2 reads_completed_total=6.9694889474988109e-307 1625842606119415905
node_disk,device=nvme0n1 reads_merged_total=8.6959111101821589e-305 1625842606119338423
node_disk,device=dm-0 reads_merged_total=0 1625842606119376783
node_disk,device=dm-1 reads_merged_total=0 1625842606119396342
node_disk,device=dm-2 reads_merged_total=0 1625842606119416865
node_disk,device=nvme0n1 read_bytes_total=7.9506563754880824e-303 1625842606119339523
node_disk,device=dm-0 read_bytes_total=7.9279261327783147e-303 1625842606119377699
node_disk,device=dm-1 read_bytes_total=7.9161757824572278e-303 1625842606119397292
node_disk,device=dm-2 read_bytes_total=1.0474344384425674e-305 1625842606119417800
node_disk,device=nvme0n1 read_time_seconds_total=3.1966722639409534e-305 1625842606119340423
node_disk,device=dm-0 read_time_seconds_total=8.756452044066651e-305 1625842606119378591
node_disk,device=dm-1 read_time_seconds_total=8.747852873623908e-305 1625842606119398381
node_disk,device=dm-2 read_time_seconds_total=5.1317630061530409e-307 1625842606119418668
node_disk,device=nvme0n1 writes_completed_total=1.1674108966428034e-303 1625842606119341584
node_disk,device=dm-0 writes_completed_total=1.8509173462488364e-303 1625842606119379606
node_disk,device=dm-1 writes_completed_total=1.8359312113078591e-303 1625842606119399822
node_disk,device=dm-2 writes_completed_total=5.1303760431906859e-306 1625842606119419727
node_disk,device=nvme0n1 writes_merged_total=6.9575402615493329e-304 1625842606119342542
node_disk,device=dm-0 writes_merged_total=0 1625842606119380441
node_disk,device=dm-1 writes_merged_total=0 1625842606119400695
node_disk,device=dm-2 writes_merged_total=0 1625842606119420658
node_disk,device=nvme0n1 written_bytes_total=6.4553668769213912e-302 1625842606119343475
node_disk,device=dm-0 written_bytes_total=6.4198260893924029e-302 1625842606119381302
node_disk,device=dm-1 written_bytes_total=6.4584224950508319e-302 1625842606119401865
node_disk,device=dm-2 written_bytes_total=4.1043008345703274e-305 1625842606119421570
node_disk,device=nvme0n1 write_time_seconds_total=1.3076709948695808e-303 1625842606119344345
node_disk,device=dm-0 write_time_seconds_total=5.2982706852329569e-303 1625842606119382389
node_disk,device=dm-1 write_time_seconds_total=5.1035632749998652e-303 1625842606119402703
node_disk,device=dm-2 write_time_seconds_total=1.4557563381913418e-305 1625842606119422431
node_disk,device=nvme0n1 io_time_seconds_total=7.7729288595982396e-304 1625842606119346457
node_disk,device=dm-0 io_time_seconds_total=7.9020828517980929e-304 1625842606119383847
node_disk,device=dm-1 io_time_seconds_total=7.8996972754816854e-304 1625842606119404189
node_disk,device=dm-2 io_time_seconds_total=6.9348148732738551e-307 1625842606119424083
node_disk,device=nvme0n1 io_time_weighted_seconds_total=1.4177826789187767e-303 1625842606119347403
node_disk,device=dm-0 io_time_weighted_seconds_total=5.3858352057137237e-303 1625842606119384645
node_disk,device=dm-1 io_time_weighted_seconds_total=5.1910418037747827e-303 1625842606119405059
node_disk,device=dm-2 io_time_weighted_seconds_total=1.5070739682643407e-305 1625842606119425000
node_disk,device=nvme0n1 discards_completed_total=0 1625842606119348254
node_disk,device=dm-0 discards_completed_total=0 1625842606119385405
node_disk,device=dm-1 discards_completed_total=0 1625842606119405820
node_disk,device=dm-2 discards_completed_total=0 1625842606119425800
node_disk,device=nvme0n1 discards_merged_total=0 1625842606119348995
node_disk,device=dm-0 discards_merged_total=0 1625842606119386162
node_disk,device=dm-1 discards_merged_total=0 1625842606119406524
node_disk,device=dm-2 discards_merged_total=0 1625842606119426506
node_disk,device=nvme0n1 discarded_sectors_total=0 1625842606119349745
node_disk,device=dm-0 discarded_sectors_total=0 1625842606119386827
node_disk,device=dm-1 discarded_sectors_total=0 1625842606119407221
node_disk,device=dm-2 discarded_sectors_total=0 1625842606119427251
node_disk,device=nvme0n1 discard_time_seconds_total=0 1625842606119350390
node_disk,device=dm-0 discard_time_seconds_total=0 1625842606119387469
node_disk,device=dm-1 discard_time_seconds_total=0 1625842606119408059
node_disk,device=dm-2 discard_time_seconds_total=0 1625842606119427965
node_disk,device=nvme0n1 flush_requests_total=8.2776030773264636e-305 1625842606119351194
node_disk,device=dm-0 flush_requests_total=0 1625842606119388383
node_disk,device=dm-1 flush_requests_total=0 1625842606119408727
node_disk,device=dm-2 flush_requests_total=0 1625842606119428731
node_disk,device=nvme0n1 flush_requests_time_seconds_total=112.684 1625842606119352392
node_disk,device=dm-0 flush_requests_time_seconds_total=0 1625842606119389100
node_disk,device=dm-1 flush_requests_time_seconds_total=0 1625842606119409394
node_disk,device=dm-2 flush_requests_time_seconds_total=0 1625842606119429446
node intr_total=384262117 1625842606119685058
node context_switches_total=605111048 1625842606119685058
node forks_total=515402 1625842606119685058
node_memory MemTotal_bytes=16445845504 1625842606118926829
node_memory MemFree_bytes=3914833920 1625842606118926829
node_memory MemAvailable_bytes=10026749952 1625842606118926829
node_memory Buffers_bytes=1061900288 1625842606118926829
node_memory Cached_bytes=6257840128 1625842606118926829
node_memory SwapCached_bytes=1974272 1625842606118926829
node_memory Active_bytes=3568754688 1625842606118926829
node_memory Inactive_bytes=7085912064 1625842606118926829
node_memory Active_anon_bytes=58318848 1625842606118926829
node_memory Inactive_anon_bytes=4578213888 1625842606118926829
node_memory Active_file_bytes=3510435840 1625842606118926829
node_memory Inactive_file_bytes=2507698176 1625842606118926829
node_memory Unevictable_bytes=999981056 1625842606118926829
node_memory Mlocked_bytes=1232896 1625842606118926829
node_memory SwapTotal_bytes=1023406080 1625842606118926829
node_memory SwapFree_bytes=1000431616 1625842606118926829
node_memory Dirty_bytes=667648 1625842606118926829
node_memory Writeback_bytes=0 1625842606118926829
node_memory AnonPages_bytes=4332937216 1625842606118926829
node_memory Mapped_bytes=1069518848 1625842606118926829
node_memory Shmem_bytes=1332330496 1625842606118926829
node_memory KReclaimable_bytes=444194816 1625842606118926829
node_memory Slab_bytes=672362496 1625842606118926829
node_memory SReclaimable_bytes=444194816 1625842606118926829
node_memory SUnreclaim_bytes=228167680 1625842606118926829
node_memory KernelStack_bytes=26329088 1625842606118926829
node_memory PageTables_bytes=60489728 1625842606118926829
node_memory NFS_Unstable_bytes=0 1625842606118926829
node_memory Bounce_bytes=0 1625842606118926829
node_memory WritebackTmp_bytes=0 1625842606118926829
node_memory CommitLimit_bytes=9246326784 1625842606118926829
node_memory Committed_AS_bytes=20046229504 1625842606118926829
node_memory VmallocTotal_bytes=35184372087808 1625842606118926829
node_memory VmallocUsed_bytes=60338176 1625842606118926829
node_memory VmallocChunk_bytes=0 1625842606118926829
node_memory Percpu_bytes=13631488 1625842606118926829
node_memory HardwareCorrupted_bytes=0 1625842606118926829
node_memory AnonHugePages_bytes=0 1625842606118926829
node_memory ShmemHugePages_bytes=0 1625842606118926829
node_memory ShmemPmdMapped_bytes=0 1625842606118926829
node_memory FileHugePages_bytes=0 1625842606118926829
node_memory FilePmdMapped_bytes=0 1625842606118926829
node_memory HugePages_Total=0 1625842606118926829
node_memory HugePages_Free=0 1625842606118926829
node_memory HugePages_Rsvd=0 1625842606118926829
node_memory HugePages_Surp=0 1625842606118926829
node_memory Hugepagesize_bytes=2097152 1625842606118926829
node_memory Hugetlb_bytes=0 1625842606118926829
node_memory DirectMap4k_bytes=970108928 1625842606118926829
node_memory DirectMap2M_bytes=14799601664 1625842606118926829
node_memory DirectMap1G_bytes=2147483648 1625842606118926829
node_disk,device=nvme0n1 io_now=0 1625842606119345621
node_disk,device=dm-0 io_now=0 1625842606119383176
node_disk,device=dm-1 io_now=0 1625842606119403515
node_disk,device=dm-2 io_now=0 1625842606119423375
node_uname,sysname=Linux,release=5.11.0-22-generic,version=#23-Ubuntu\ SMP\ Thu\ Jun\ 17\ 00:34:23\ UTC\ 2021,machine=x86_64,nodename=monox,domainname=(none) info=1 1625842606119640661
node boot_time_seconds=1625634352 1625842606119685058
node procs_running=1 1625842606119685058
node procs_blocked=0 1625842606119685058
node time_seconds=1625842606.1201811 1625842606120180970
node load1=1.8 1625842606120195353
node load5=1.2 1625842606120195353
node load15=1.05 1625842606120195353