mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
UnifiedSearch: Introduce a ResourceIndex interface and bleve implementation (#96826)
Co-authored-by: Scott Lepper <scott.lepper@gmail.com>
This commit is contained in:
Ryan McKinley
GitHub
parent
bbae396db4
commit
c6848d4b68
15
go.mod
15
go.mod
@ -38,6 +38,7 @@ require (
|
||||
github.com/beevik/etree v1.4.1 // @grafana/grafana-backend-group
|
||||
github.com/benbjohnson/clock v1.3.5 // @grafana/alerting-backend
|
||||
github.com/blang/semver/v4 v4.0.0 // indirect; @grafana/grafana-developer-enablement-squad
|
||||
github.com/blevesearch/bleve/v2 v2.4.3 // @grafana/grafana-search-and-storage
|
||||
github.com/blugelabs/bluge v0.1.9 // @grafana/grafana-backend-group
|
||||
github.com/blugelabs/bluge_segment_api v0.2.0 // @grafana/grafana-backend-group
|
||||
github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 // @grafana/grafana-backend-group
|
||||
@ -480,11 +481,24 @@ require github.com/openzipkin/zipkin-go v0.4.3 // @grafana/oss-big-tent
|
||||
require (
|
||||
cloud.google.com/go/longrunning v0.6.0 // indirect
|
||||
github.com/at-wat/mqtt-go v0.19.4 // indirect
|
||||
github.com/blevesearch/bleve_index_api v1.1.12 // indirect
|
||||
github.com/blevesearch/geo v0.1.20 // indirect
|
||||
github.com/blevesearch/go-faiss v1.0.23 // indirect
|
||||
github.com/blevesearch/gtreap v0.1.1 // indirect
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.2.16 // indirect
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
|
||||
github.com/blevesearch/zapx/v11 v11.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v12 v12.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v13 v13.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v14 v14.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v15 v15.3.16 // indirect
|
||||
github.com/blevesearch/zapx/v16 v16.1.8 // indirect
|
||||
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 // indirect
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/dolthub/maphash v0.1.0 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/gammazero/deque v0.2.1 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/grafana/jsonparser v0.0.0-20240425183733-ea80629e1a32 // indirect
|
||||
github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 // indirect
|
||||
github.com/grafana/sqlds/v4 v4.1.0 // indirect
|
||||
@ -500,6 +514,7 @@ require (
|
||||
github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
|
||||
github.com/shadowspore/fossil-delta v0.0.0-20240102155221-e3a8590b820b // indirect
|
||||
github.com/sony/gobreaker v0.5.0 // indirect
|
||||
go.etcd.io/bbolt v1.3.10 // indirect
|
||||
go.opentelemetry.io/collector/featuregate v1.9.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 // indirect
|
||||
go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect
|
||||
|
||||
28
go.sum
28
go.sum
@ -1616,19 +1616,45 @@ github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZ
|
||||
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
|
||||
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
|
||||
github.com/blevesearch/bleve/v2 v2.4.3 h1:XDYj+1prgX84L2Cf+V3ojrOPqXxy0qxyd2uLMmeuD+4=
|
||||
github.com/blevesearch/bleve/v2 v2.4.3/go.mod h1:hEPDPrbYw3vyrm5VOa36GyS4bHWuIf4Fflp7460QQXY=
|
||||
github.com/blevesearch/bleve_index_api v1.1.12 h1:P4bw9/G/5rulOF7SJ9l4FsDoo7UFJ+5kexNy1RXfegY=
|
||||
github.com/blevesearch/bleve_index_api v1.1.12/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
|
||||
github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
|
||||
github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
|
||||
github.com/blevesearch/go-faiss v1.0.23 h1:Wmc5AFwDLKGl2L6mjLX1Da3vCL0EKa2uHHSorcIS1Uc=
|
||||
github.com/blevesearch/go-faiss v1.0.23/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
|
||||
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
|
||||
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
|
||||
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
|
||||
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
|
||||
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.2.16 h1:uGvKVvG7zvSxCwcm4/ehBa9cCEuZVE+/zvrSl57QUVY=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.2.16/go.mod h1:VF5oHVbIFTu+znY1v30GjSpT5+9YFs9dV2hjvuh34F0=
|
||||
github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
|
||||
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
|
||||
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
|
||||
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
|
||||
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
|
||||
github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ=
|
||||
github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
|
||||
github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
|
||||
github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
|
||||
github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
|
||||
github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
|
||||
github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
|
||||
github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
|
||||
github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
|
||||
github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
|
||||
github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
|
||||
github.com/blevesearch/zapx/v15 v15.3.16 h1:Ct3rv7FUJPfPk99TI/OofdC+Kpb4IdyfdMH48sb+FmE=
|
||||
github.com/blevesearch/zapx/v15 v15.3.16/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
|
||||
github.com/blevesearch/zapx/v16 v16.1.8 h1:Bxzpw6YQpFs7UjoCV1+RvDw6fmAT2GZxldwX8b3wVBM=
|
||||
github.com/blevesearch/zapx/v16 v16.1.8/go.mod h1:JqQlOqlRVaYDkpLIl3JnKql8u4zKTNlVEa3nLsi0Gn8=
|
||||
github.com/blugelabs/bluge v0.1.9 h1:bPgXlcsWugrXNjzeoLdOnvfJpHsyODKpYaAndayl/SM=
|
||||
github.com/blugelabs/bluge v0.1.9/go.mod h1:5d7LktUkQgvbh5Bmi6tPWtvo4+6uRTm6gAwP+5z6FqQ=
|
||||
github.com/blugelabs/bluge_segment_api v0.2.0 h1:cCX1Y2y8v0LZ7+EEJ6gH7dW6TtVTW4RhG0vp3R+N2Lo=
|
||||
@ -2073,6 +2099,8 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2V
|
||||
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
|
||||
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
|
||||
github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
|
||||
|
||||
12
go.work.sum
12
go.work.sum
@ -571,6 +571,7 @@ github.com/RoaringBitmap/gocroaring v0.4.0 h1:5nufXUgWpBEUNEJXw7926YAA58ZAQRpWPr
|
||||
github.com/RoaringBitmap/real-roaring-datasets v0.0.0-20190726190000-eb7c87156f76 h1:ZYlhPbqQFU+AHfgtCdHGDTtRW1a8geZyiE8c6Q+Sl1s=
|
||||
github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo=
|
||||
github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
|
||||
github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
|
||||
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
|
||||
github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 h1:KkH3I3sJuOLP3TjA/dfr4NAY8bghDwnXiU7cTKxQqo0=
|
||||
github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06/go.mod h1:7erjKLwalezA0k99cWs5L11HWOAPNjdUZ6RxH1BXbbM=
|
||||
@ -651,11 +652,13 @@ github.com/benbjohnson/immutable v0.4.0/go.mod h1:iAr8OjJGLnLmVUr9MZ/rz4PWUy6Ouc
|
||||
github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
|
||||
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=
|
||||
github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
|
||||
github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
|
||||
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA=
|
||||
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
|
||||
github.com/blevesearch/goleveldb v1.0.1 h1:iAtV2Cu5s0GD1lwUiekkFHe2gTMCCNVj2foPclDLIFI=
|
||||
github.com/blevesearch/goleveldb v1.0.1/go.mod h1:WrU8ltZbIp0wAoig/MHbrPCXSOLpe79nz5lv5nqfYrQ=
|
||||
github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc=
|
||||
github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A=
|
||||
github.com/blevesearch/snowball v0.6.1/go.mod h1:ZF0IBg5vgpeoUhnMza2v0A/z8m1cWPlwhke08LpNusg=
|
||||
github.com/blevesearch/stempel v0.2.0 h1:CYzVPaScODMvgE9o+kf6D4RJ/VRomyi9uHF+PtB+Afc=
|
||||
@ -978,8 +981,6 @@ github.com/grafana/go-json v0.0.0-20241106155216-71a03f133f5c/go.mod h1:oq7eo15S
|
||||
github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU=
|
||||
github.com/grafana/grafana-app-sdk v0.19.0/go.mod h1:y0BgzYxc+a7CwOqkwUhN9zXd5cgZJjd2zAbgHEd/xzo=
|
||||
github.com/grafana/pyroscope-go/godeltaprof v0.1.6/go.mod h1:Tk376Nbldo4Cha9RgiU7ik8WKFkNpfds98aUzS8omLE=
|
||||
github.com/grafana/pyroscope/api v1.0.0 h1:RWK3kpv8EAnB7JpOqnf//xwE84DdKF03N/iFxpFAoHY=
|
||||
github.com/grafana/pyroscope/api v1.0.0/go.mod h1:CUrgOgSZDnx4M1mlRoxhrVKkTuKIse9p4FtuPbrGA04=
|
||||
github.com/grafana/tail v0.0.0-20230510142333-77b18831edf0 h1:bjh0PVYSVVFxzINqPFYJmAmJNrWPgnVjuSdYJGHmtFU=
|
||||
github.com/grafana/tail v0.0.0-20230510142333-77b18831edf0/go.mod h1:7t5XR+2IA8P2qggOAHTj/GCZfoLBle3OvNSYh1VkRBU=
|
||||
github.com/grafana/thema v0.0.0-20230511182720-3146087fcc26 h1:HX927q4X1n451pnGb8U0wq74i8PCzuxVjzv7TyD10kc=
|
||||
@ -1061,6 +1062,7 @@ github.com/jon-whit/go-grpc-prometheus v1.4.0 h1:/wmpGDJcLXuEjXryWhVYEGt9YBRhtLw
|
||||
github.com/jon-whit/go-grpc-prometheus v1.4.0/go.mod h1:iTPm+Iuhh3IIqR0iGZ91JJEg5ax6YQEe1I0f6vtBuao=
|
||||
github.com/joncrlsn/dque v0.0.0-20211108142734-c2ef48c5192a h1:sfe532Ipn7GX0V6mHdynBk393rDmqgI0QmjLK7ct7TU=
|
||||
github.com/joncrlsn/dque v0.0.0-20211108142734-c2ef48c5192a/go.mod h1:dNKs71rs2VJGBAmttu7fouEsRQlRjxy0p1Sx+T5wbpY=
|
||||
github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
|
||||
github.com/jsternberg/zap-logfmt v1.2.0 h1:1v+PK4/B48cy8cfQbxL4FmmNZrjnIMr2BsnyEmXqv2o=
|
||||
github.com/jsternberg/zap-logfmt v1.2.0/go.mod h1:kz+1CUmCutPWABnNkOu9hOHKdT2q3TDYCcsFy9hpqb0=
|
||||
@ -1293,8 +1295,6 @@ github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7l
|
||||
github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5 h1:ZCnq+JUrvXcDVhX/xRolRBZifmabN1HcS1wrPSvxhrU=
|
||||
github.com/openzipkin/zipkin-go v0.4.2 h1:zjqfqHjUpPmB3c1GlCvvgsM1G4LkvqQbBDueDOCg/jA=
|
||||
github.com/openzipkin/zipkin-go v0.4.2/go.mod h1:ZeVkFjuuBiSy13y8vpSDCjMi9GoI3hPpCJSBx/EYFhY=
|
||||
github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg=
|
||||
github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w=
|
||||
github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
|
||||
@ -1384,6 +1384,7 @@ github.com/shoenig/test v1.7.1 h1:UJcjSAI3aUKx52kfcfhblgyhZceouhvvs3OYdWgn+PY=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
|
||||
github.com/sony/gobreaker v0.4.1 h1:oMnRNZXX5j85zso6xCPRNPtmAycat+WcoKbklScLDgQ=
|
||||
github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
|
||||
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
|
||||
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
|
||||
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad h1:fiWzISvDn0Csy5H0iwgAuJGQTUpVfEMJJd4nRFXogbc=
|
||||
github.com/stoewer/parquet-cli v0.0.7 h1:rhdZODIbyMS3twr4OM3am8BPPT5pbfMcHLH93whDM5o=
|
||||
@ -1473,8 +1474,6 @@ github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 h1:ESFSdwYZvkeru3RtdrYueztKhOBCSAAzS4Gf+k0tEow=
|
||||
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
|
||||
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
|
||||
github.com/yalue/merged_fs v1.3.0 h1:qCeh9tMPNy/i8cwDsQTJ5bLr6IRxbs6meakNE5O+wyY=
|
||||
github.com/yalue/merged_fs v1.3.0/go.mod h1:WqqchfVYQyclV2tnR7wtRhBddzBvLVR83Cjw9BKQw0M=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf h1:ckwNHVo4bv2tqNkgx3W3HANh3ta1j6TR5qw08J1A7Tw=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7 h1:nL8XwD6fSst7xFUirkaWJmE7kM0CdWRYgu6+YQer1d4=
|
||||
@ -1498,6 +1497,7 @@ github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wK
|
||||
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs=
|
||||
go.einride.tech/aip v0.67.1 h1:d/4TW92OxXBngkSOwWS2CH5rez869KpKMaN44mdxkFI=
|
||||
go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI=
|
||||
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
|
||||
go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg=
|
||||
go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
|
||||
go.opentelemetry.io/collector v0.97.0 h1:qyOju13byHIKEK/JehmTiGMj4pFLa4kDyrOCtTmjHU0=
|
||||
|
||||
@ -159,11 +159,8 @@ func NewIndexableDocument(key *ResourceKey, rv int64, obj utils.GrafanaMetaAcces
|
||||
return doc
|
||||
}
|
||||
|
||||
func StandardDocumentBuilder() DocumentBuilderInfo {
|
||||
return DocumentBuilderInfo{
|
||||
Builder: &standardDocumentBuilder{},
|
||||
Fields: StandardSearchFields(),
|
||||
}
|
||||
func StandardDocumentBuilder() DocumentBuilder {
|
||||
return &standardDocumentBuilder{}
|
||||
}
|
||||
|
||||
type standardDocumentBuilder struct{}
|
||||
@ -295,6 +292,30 @@ func StandardSearchFields() SearchableDocumentFields {
|
||||
FreeText: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: SEARCH_FIELD_TAGS,
|
||||
Type: ResourceTableColumnDefinition_STRING,
|
||||
IsArray: true,
|
||||
Description: "Unique tags",
|
||||
Properties: &ResourceTableColumnDefinition_Properties{
|
||||
Filterable: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: SEARCH_FIELD_FOLDER,
|
||||
Type: ResourceTableColumnDefinition_STRING,
|
||||
Description: "Kubernetes name for the folder",
|
||||
},
|
||||
{
|
||||
Name: SEARCH_FIELD_RV,
|
||||
Type: ResourceTableColumnDefinition_INT64,
|
||||
Description: "resource version",
|
||||
},
|
||||
{
|
||||
Name: SEARCH_FIELD_CREATED,
|
||||
Type: ResourceTableColumnDefinition_INT64,
|
||||
Description: "created timestamp", // date?
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
panic("failed to initialize standard search fields")
|
||||
|
||||
@ -12,7 +12,7 @@ import (
|
||||
|
||||
func TestStandardDocumentBuilder(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
builder := StandardDocumentBuilder().Builder
|
||||
builder := StandardDocumentBuilder()
|
||||
|
||||
body, err := os.ReadFile("testdata/playlist-resource.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -323,6 +323,7 @@ message WatchEvent {
|
||||
Resource previous = 4;
|
||||
}
|
||||
|
||||
// This will soon be deprecated/replaced with ResourceSearchRequest
|
||||
message SearchRequest {
|
||||
// query string for chosen implementation (currently just bleve)
|
||||
string query = 1;
|
||||
@ -342,6 +343,92 @@ message SearchRequest {
|
||||
repeated string filters = 10;
|
||||
}
|
||||
|
||||
// Search within a single resource
|
||||
message ResourceSearchRequest {
|
||||
message Sort {
|
||||
string field = 1;
|
||||
bool desc = 2; // defaults to ascending
|
||||
}
|
||||
|
||||
message Facet {
|
||||
string field = 1;
|
||||
int64 limit = 2;
|
||||
// For now, only term queries, eventually?
|
||||
// numeric queries
|
||||
// date queries
|
||||
}
|
||||
|
||||
// The key must include namespace + group + resource
|
||||
ListOptions options = 1;
|
||||
|
||||
// To search additional resource types, add additional keys to this list
|
||||
// NOTE: queries will only support federation across kinds with common fields
|
||||
repeated ResourceKey federated = 2;
|
||||
|
||||
// When a query exists, it is parsed and used to influence
|
||||
// query string for chosen implementation (currently just bleve)
|
||||
// The score is only relevant when a query exists
|
||||
string query = 3;
|
||||
|
||||
// max results
|
||||
int64 limit = 4;
|
||||
|
||||
// where to start the query (eg, From)
|
||||
int64 offset = 5;
|
||||
|
||||
// sorting
|
||||
repeated Sort sortBy = 6;
|
||||
|
||||
// calculate field statistics
|
||||
map<string,Facet> facet = 7;
|
||||
|
||||
// the return fields (empty will return everything)
|
||||
repeated string fields = 8;
|
||||
|
||||
// explain each result (added to the each row)
|
||||
bool explain = 9;
|
||||
}
|
||||
|
||||
message ResourceSearchResponse {
|
||||
message Facet {
|
||||
string field = 1;
|
||||
// The distinct terms
|
||||
int64 total = 2;
|
||||
// The number of documents that do *not* have this field
|
||||
int64 missing = 3;
|
||||
// Top term stats
|
||||
repeated TermFacet terms = 4;
|
||||
// numeric range
|
||||
// date range facets
|
||||
}
|
||||
|
||||
message TermFacet {
|
||||
string term = 1;
|
||||
int64 count = 2;
|
||||
}
|
||||
|
||||
// Error details
|
||||
ErrorResult error = 1;
|
||||
|
||||
// All results exist within this key
|
||||
ResourceKey key = 2;
|
||||
|
||||
// Query results
|
||||
ResourceTable results = 3;
|
||||
|
||||
// The total hit count
|
||||
uint64 total_hits = 4;
|
||||
|
||||
// indicates how expensive was the query with respect to bytes read
|
||||
uint64 query_cost = 5;
|
||||
|
||||
// maximum score across all fields
|
||||
double max_score = 6;
|
||||
|
||||
// Facet results
|
||||
map<string,Facet> facet = 7;
|
||||
}
|
||||
|
||||
message GroupBy {
|
||||
string name = 1;
|
||||
int64 limit = 2;
|
||||
@ -352,6 +439,7 @@ message Group {
|
||||
int64 count = 2;
|
||||
}
|
||||
|
||||
// This will soon be deprecated/replaced with ResourceSearchResponse
|
||||
message SearchResponse {
|
||||
repeated ResourceWrapper items = 1;
|
||||
repeated Group groups = 2;
|
||||
|
||||
@ -12,6 +12,8 @@ import (
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
|
||||
"github.com/grafana/authlib/authz"
|
||||
)
|
||||
|
||||
type NamespacedResource struct {
|
||||
@ -20,8 +22,52 @@ type NamespacedResource struct {
|
||||
Resource string
|
||||
}
|
||||
|
||||
// All fields are set
|
||||
func (s *NamespacedResource) Valid() bool {
|
||||
return s.Namespace != "" && s.Group != "" && s.Resource != ""
|
||||
}
|
||||
|
||||
type ResourceIndex interface {
|
||||
// Add a document to the index. Note it may not be searchable until after flush is called
|
||||
Write(doc *IndexableDocument) error
|
||||
|
||||
// Mark a resource as deleted. Note it may not be searchable until after flush is called
|
||||
Delete(key *ResourceKey) error
|
||||
|
||||
// Make sure any changes to the index are flushed and available in the next search/origin calls
|
||||
Flush() error
|
||||
|
||||
// Search within a namespaced resource
|
||||
// When working with federated queries, the additional indexes will be passed in explicitly
|
||||
Search(ctx context.Context, access authz.AccessClient, req *ResourceSearchRequest, federate []ResourceIndex) (*ResourceSearchResponse, error)
|
||||
|
||||
// Execute an origin query -- access control is not not checked for each item
|
||||
// NOTE: this will likely be used for provisioning, or it will be removed
|
||||
Origin(ctx context.Context, req *OriginRequest) (*OriginResponse, error)
|
||||
}
|
||||
|
||||
// SearchBackend contains the technology specific logic to support search
|
||||
type SearchBackend interface {
|
||||
// TODO
|
||||
// This will return nil if the key does not exist
|
||||
GetIndex(ctx context.Context, key NamespacedResource) (ResourceIndex, error)
|
||||
|
||||
// Build an index from scratch
|
||||
BuildIndex(ctx context.Context,
|
||||
key NamespacedResource,
|
||||
|
||||
// When the size is known, it will be passed along here
|
||||
// Depending on the size, the backend may choose different options (eg: memory vs disk)
|
||||
size int64,
|
||||
|
||||
// The last known resource version (can be used to know that nothing has changed)
|
||||
resourceVersion int64,
|
||||
|
||||
// The non-standard index fields
|
||||
fields SearchableDocumentFields,
|
||||
|
||||
// The builder will write all documents before returning
|
||||
builder func(index ResourceIndex) (int64, error),
|
||||
) (ResourceIndex, error)
|
||||
}
|
||||
|
||||
const tracingPrexfixSearch = "unified_search."
|
||||
@ -119,7 +165,7 @@ func (s *searchSupport) init(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size int64, rv int64) (any, int64, error) {
|
||||
func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size int64, rv int64) (ResourceIndex, int64, error) {
|
||||
_, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Build")
|
||||
defer span.End()
|
||||
|
||||
@ -127,10 +173,57 @@ func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
fields := s.builders.GetFields(nsr)
|
||||
|
||||
s.log.Debug(fmt.Sprintf("TODO, build %+v (size:%d, rv:%d) // builder:%+v\n", nsr, size, rv, builder))
|
||||
|
||||
return nil, 0, nil
|
||||
key := &ResourceKey{
|
||||
Group: nsr.Group,
|
||||
Resource: nsr.Resource,
|
||||
Namespace: nsr.Namespace,
|
||||
}
|
||||
index, err := s.search.BuildIndex(ctx, nsr, size, rv, fields, func(index ResourceIndex) (int64, error) {
|
||||
rv, err = s.storage.ListIterator(ctx, &ListRequest{
|
||||
Limit: 1000000000000, // big number
|
||||
Options: &ListOptions{
|
||||
Key: key,
|
||||
},
|
||||
}, func(iter ListIterator) error {
|
||||
for iter.Next() {
|
||||
if err = iter.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update the key name
|
||||
// Or should we read it from the body?
|
||||
key.Name = iter.Name()
|
||||
|
||||
// Convert it to an indexable document
|
||||
doc, err := builder.BuildDocument(ctx, key, iter.ResourceVersion(), iter.Value())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// And finally write it to the index
|
||||
if err = index.Write(doc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
})
|
||||
return rv, err
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
err = index.Flush()
|
||||
}
|
||||
|
||||
// rv is the last RV we read. when watching, we must add all events since that time
|
||||
return index, rv, err
|
||||
}
|
||||
|
||||
type builderCache struct {
|
||||
@ -140,6 +233,9 @@ type builderCache struct {
|
||||
// Possible blob support
|
||||
blob BlobSupport
|
||||
|
||||
// searchable fields initialized once on startup
|
||||
fields map[schema.GroupResource]SearchableDocumentFields
|
||||
|
||||
// lookup by group, then resource (namespace)
|
||||
// This is only modified at startup, so we do not need mutex for access
|
||||
lookup map[string]map[string]DocumentBuilderInfo
|
||||
@ -151,6 +247,7 @@ type builderCache struct {
|
||||
|
||||
func newBuilderCache(cfg []DocumentBuilderInfo, nsCacheSize int, ttl time.Duration) (*builderCache, error) {
|
||||
cache := &builderCache{
|
||||
fields: make(map[schema.GroupResource]SearchableDocumentFields),
|
||||
lookup: make(map[string]map[string]DocumentBuilderInfo),
|
||||
ns: expirable.NewLRU[NamespacedResource, DocumentBuilder](nsCacheSize, nil, ttl),
|
||||
}
|
||||
@ -173,10 +270,17 @@ func newBuilderCache(cfg []DocumentBuilderInfo, nsCacheSize int, ttl time.Durati
|
||||
cache.lookup[b.GroupResource.Group] = g
|
||||
}
|
||||
g[b.GroupResource.Resource] = b
|
||||
|
||||
// Any custom fields
|
||||
cache.fields[b.GroupResource] = b.Fields
|
||||
}
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
func (s *builderCache) GetFields(key NamespacedResource) SearchableDocumentFields {
|
||||
return s.fields[schema.GroupResource{Group: key.Group, Resource: key.Resource}]
|
||||
}
|
||||
|
||||
// context is typically background. Holds an LRU cache for a
|
||||
func (s *builderCache) get(ctx context.Context, key NamespacedResource) (DocumentBuilder, error) {
|
||||
g, ok := s.lookup[key.Group]
|
||||
|
||||
@ -7,13 +7,18 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
reflect "reflect"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter"
|
||||
)
|
||||
|
||||
@ -74,6 +79,10 @@ func (x *ResourceTable) ToK8s() (metav1.Table, error) {
|
||||
}
|
||||
} else if r.Key != nil {
|
||||
obj := &metav1.PartialObjectMetadata{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: r.Key.Resource, // :(
|
||||
APIVersion: r.Key.Group, // :(
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: r.Key.Name,
|
||||
Namespace: r.Key.Namespace,
|
||||
@ -102,6 +111,8 @@ type TableBuilder struct {
|
||||
hasDuplicateNames bool
|
||||
}
|
||||
|
||||
type ResourceColumnEncoder = func(v any) ([]byte, error)
|
||||
|
||||
func NewTableBuilder(cols []*ResourceTableColumnDefinition) (*TableBuilder, error) {
|
||||
table := &TableBuilder{
|
||||
ResourceTable: ResourceTable{
|
||||
@ -124,6 +135,15 @@ func NewTableBuilder(cols []*ResourceTableColumnDefinition) (*TableBuilder, erro
|
||||
return table, err
|
||||
}
|
||||
|
||||
func (x *TableBuilder) Encoders() []ResourceColumnEncoder {
|
||||
encoders := make([]ResourceColumnEncoder, len(x.Columns))
|
||||
for i, f := range x.Columns {
|
||||
v := x.lookup[f.Name]
|
||||
encoders[i] = v.Encode
|
||||
}
|
||||
return encoders
|
||||
}
|
||||
|
||||
func (x *TableBuilder) AddRow(key *ResourceKey, rv int64, vals map[string]any) error {
|
||||
row := &ResourceTableRow{
|
||||
Key: key,
|
||||
@ -395,6 +415,8 @@ func (x *resourceTableColumn) Encode(v any) ([]byte, error) {
|
||||
f = int64(typed)
|
||||
case float32:
|
||||
f = int64(typed)
|
||||
case float64:
|
||||
f = int64(typed)
|
||||
case uint64:
|
||||
f = int64(typed)
|
||||
case uint:
|
||||
@ -547,3 +569,29 @@ func (x *resourceTableColumn) Decode(buff []byte) (any, error) {
|
||||
}
|
||||
return v, err
|
||||
}
|
||||
|
||||
// AssertTableSnapshot will match a ResourceTable vs the saved value
|
||||
func AssertTableSnapshot(t *testing.T, path string, table *ResourceTable) {
|
||||
t.Helper()
|
||||
|
||||
k8sTable, err := table.ToK8s()
|
||||
require.NoError(t, err, "unable to create table response", path)
|
||||
actual, err := json.MarshalIndent(k8sTable, "", " ")
|
||||
require.NoError(t, err, "unable to write table json", path)
|
||||
|
||||
// Safe to disable, this is a test.
|
||||
// nolint:gosec
|
||||
expected, err := os.ReadFile(path)
|
||||
if err != nil || len(expected) < 1 {
|
||||
assert.Fail(t, "missing file: %s", path)
|
||||
} else if assert.JSONEq(t, string(expected), string(actual)) {
|
||||
return // everything is OK
|
||||
}
|
||||
|
||||
// Write the snapshot
|
||||
// Safe to disable, this is a test.
|
||||
// nolint:gosec
|
||||
err = os.WriteFile(path, actual, 0600)
|
||||
require.NoError(t, err)
|
||||
fmt.Printf("Updated table snapshot: %s\n", path)
|
||||
}
|
||||
|
||||
@ -1,44 +1,15 @@
|
||||
package resource
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// AssertTableSnapshot will match a ResourceTable vs the saved value
|
||||
func AssertTableSnapshot(t *testing.T, path string, table *ResourceTable) {
|
||||
t.Helper()
|
||||
|
||||
k8sTable, err := table.ToK8s()
|
||||
require.NoError(t, err, "unable to create table response", path)
|
||||
actual, err := json.MarshalIndent(k8sTable, "", " ")
|
||||
require.NoError(t, err, "unable to write table json", path)
|
||||
|
||||
// Safe to disable, this is a test.
|
||||
// nolint:gosec
|
||||
expected, err := os.ReadFile(path)
|
||||
if err != nil || len(expected) < 1 {
|
||||
assert.Fail(t, "missing file")
|
||||
} else if assert.JSONEq(t, string(expected), string(actual)) {
|
||||
return // everything is OK
|
||||
}
|
||||
|
||||
// Write the snapshot
|
||||
// Safe to disable, this is a test.
|
||||
// nolint:gosec
|
||||
err = os.WriteFile(path, actual, 0600)
|
||||
require.NoError(t, err)
|
||||
fmt.Printf("Updated table snapshot: %s\n", path)
|
||||
}
|
||||
|
||||
func TestTableFormat(t *testing.T) {
|
||||
columns := []*ResourceTableColumnDefinition{
|
||||
{
|
||||
|
||||
@ -41,6 +41,8 @@
|
||||
]
|
||||
],
|
||||
"object": {
|
||||
"kind": "xyz",
|
||||
"apiVersion": "ggg",
|
||||
"metadata": {
|
||||
"name": "aaa",
|
||||
"namespace": "default",
|
||||
@ -60,6 +62,8 @@
|
||||
]
|
||||
],
|
||||
"object": {
|
||||
"kind": "xyz",
|
||||
"apiVersion": "ggg",
|
||||
"metadata": {
|
||||
"name": "bbb",
|
||||
"namespace": "default",
|
||||
|
||||
547
pkg/storage/unified/search/bleve.go
Normal file
547
pkg/storage/unified/search/bleve.go
Normal file
@ -0,0 +1,547 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"k8s.io/apimachinery/pkg/selection"
|
||||
|
||||
"github.com/grafana/authlib/authz"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
)
|
||||
|
||||
const tracingPrexfixBleve = "unified_search.bleve."
|
||||
|
||||
var _ resource.SearchBackend = &bleveBackend{}
|
||||
var _ resource.ResourceIndex = &bleveIndex{}
|
||||
|
||||
type bleveOptions struct {
|
||||
// The root folder where file objects are saved
|
||||
Root string
|
||||
|
||||
// The resource count where values switch from memory to file based
|
||||
FileThreshold int64
|
||||
|
||||
// How big should a batch get before flushing
|
||||
// ?? not totally sure the units
|
||||
BatchSize int
|
||||
}
|
||||
|
||||
type bleveBackend struct {
|
||||
tracer trace.Tracer
|
||||
log *slog.Logger
|
||||
opts bleveOptions
|
||||
|
||||
// cache info
|
||||
cache map[resource.NamespacedResource]*bleveIndex
|
||||
cacheMu sync.RWMutex
|
||||
}
|
||||
|
||||
func NewBleveBackend(opts bleveOptions, tracer trace.Tracer, reg prometheus.Registerer) *bleveBackend {
|
||||
b := &bleveBackend{
|
||||
log: slog.Default().With("logger", "bleve-backend"),
|
||||
tracer: tracer,
|
||||
cache: make(map[resource.NamespacedResource]*bleveIndex),
|
||||
opts: opts,
|
||||
}
|
||||
|
||||
if reg != nil {
|
||||
b.log.Info("TODO, register metrics collectors!")
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// This will return nil if the key does not exist
|
||||
func (b *bleveBackend) GetIndex(ctx context.Context, key resource.NamespacedResource) (resource.ResourceIndex, error) {
|
||||
b.cacheMu.RLock()
|
||||
defer b.cacheMu.RUnlock()
|
||||
|
||||
idx, ok := b.cache[key]
|
||||
if ok {
|
||||
return idx, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build an index from scratch
|
||||
func (b *bleveBackend) BuildIndex(ctx context.Context,
|
||||
key resource.NamespacedResource,
|
||||
|
||||
// When the size is known, it will be passed along here
|
||||
// Depending on the size, the backend may choose different options (eg: memory vs disk)
|
||||
size int64,
|
||||
|
||||
// The last known resource version can be used to know that we can skip calling the builder
|
||||
resourceVersion int64,
|
||||
|
||||
// the non-standard searchable fields
|
||||
fields resource.SearchableDocumentFields,
|
||||
|
||||
// The builder will write all documents before returning
|
||||
builder func(index resource.ResourceIndex) (int64, error),
|
||||
) (resource.ResourceIndex, error) {
|
||||
b.cacheMu.Lock()
|
||||
defer b.cacheMu.Unlock()
|
||||
|
||||
_, span := b.tracer.Start(ctx, tracingPrexfixBleve+"BuildIndex")
|
||||
defer span.End()
|
||||
|
||||
var err error
|
||||
var index bleve.Index
|
||||
|
||||
mapper := getBleveMappings(fields)
|
||||
|
||||
if size > b.opts.FileThreshold {
|
||||
dir := filepath.Join(b.opts.Root, key.Namespace, fmt.Sprintf("%s.%s", key.Resource, key.Group))
|
||||
index, err = bleve.New(dir, mapper)
|
||||
if err == nil {
|
||||
b.log.Info("TODO, check last RV so we can see if the numbers have changed", "dir", dir)
|
||||
}
|
||||
} else {
|
||||
index, err = bleve.NewMemOnly(mapper)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Batch all the changes
|
||||
idx := &bleveIndex{
|
||||
key: key,
|
||||
index: index,
|
||||
batch: index.NewBatch(),
|
||||
batchSize: b.opts.BatchSize,
|
||||
fields: fields,
|
||||
standard: resource.StandardSearchFields(),
|
||||
}
|
||||
|
||||
idx.allFields, err = getAllFields(idx.standard, fields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = builder(idx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Flush the batch
|
||||
err = idx.Flush()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b.cache[key] = idx
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
type bleveIndex struct {
|
||||
key resource.NamespacedResource
|
||||
index bleve.Index
|
||||
|
||||
standard resource.SearchableDocumentFields
|
||||
fields resource.SearchableDocumentFields
|
||||
|
||||
// The values returned with all
|
||||
allFields []*resource.ResourceTableColumnDefinition
|
||||
|
||||
// only valid in single thread
|
||||
batch *bleve.Batch
|
||||
batchSize int // ??? not totally sure the units here
|
||||
}
|
||||
|
||||
// Write implements resource.DocumentIndex.
|
||||
func (b *bleveIndex) Write(v *resource.IndexableDocument) error {
|
||||
// remove references (for now!)
|
||||
v.References = nil
|
||||
if b.batch != nil {
|
||||
err := b.batch.Index(v.Key.SearchID(), v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b.batch.Size() > b.batchSize {
|
||||
err = b.index.Batch(b.batch)
|
||||
b.batch.Reset() // clear the batch
|
||||
}
|
||||
return err // nil
|
||||
}
|
||||
return b.index.Index(v.Key.SearchID(), v)
|
||||
}
|
||||
|
||||
// Delete implements resource.DocumentIndex.
|
||||
func (b *bleveIndex) Delete(key *resource.ResourceKey) error {
|
||||
if b.batch != nil {
|
||||
return fmt.Errorf("unexpected delete while building batch")
|
||||
}
|
||||
return b.index.Delete(key.SearchID())
|
||||
}
|
||||
|
||||
// Flush implements resource.DocumentIndex.
|
||||
func (b *bleveIndex) Flush() (err error) {
|
||||
if b.batch != nil {
|
||||
err = b.index.Batch(b.batch)
|
||||
b.batch.Reset()
|
||||
b.batch = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Origin implements resource.DocumentIndex.
|
||||
func (b *bleveIndex) Origin(ctx context.Context, req *resource.OriginRequest) (*resource.OriginResponse, error) {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// Search implements resource.DocumentIndex.
|
||||
func (b *bleveIndex) Search(
|
||||
ctx context.Context,
|
||||
access authz.AccessClient,
|
||||
req *resource.ResourceSearchRequest,
|
||||
federate []resource.ResourceIndex, // For federated queries, these will match the values in req.federate
|
||||
) (*resource.ResourceSearchResponse, error) {
|
||||
if req.Options == nil || req.Options.Key == nil {
|
||||
return &resource.ResourceSearchResponse{
|
||||
Error: resource.NewBadRequestError("missing query key"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
response := &resource.ResourceSearchResponse{
|
||||
Error: b.verifyKey(req.Options.Key),
|
||||
}
|
||||
if response.Error != nil {
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// Verifies the index federation
|
||||
index, err := b.getIndex(req, federate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// convert protobuf request to bleve request
|
||||
searchrequest, e := toBleveSearchRequest(req, access)
|
||||
if e != nil {
|
||||
response.Error = e
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// Show all fields when nothing is selected
|
||||
if len(searchrequest.Fields) < 1 && req.Limit > 0 {
|
||||
f, err := b.index.Fields()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
searchrequest.Fields = f
|
||||
}
|
||||
|
||||
res, err := index.Search(searchrequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
response.TotalHits = res.Total
|
||||
response.QueryCost = res.Cost
|
||||
response.MaxScore = res.MaxScore
|
||||
|
||||
response.Results, err = b.hitsToTable(searchrequest.Fields, res.Hits, req.Explain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Write frame as JSON
|
||||
//response.Frame, err = frame.MarshalJSON()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// parse the facet fields
|
||||
for k, v := range res.Facets {
|
||||
f := &resource.ResourceSearchResponse_Facet{
|
||||
Field: v.Field,
|
||||
Total: int64(v.Total),
|
||||
Missing: int64(v.Missing),
|
||||
}
|
||||
if v.Terms != nil {
|
||||
for _, t := range v.Terms.Terms() {
|
||||
f.Terms = append(f.Terms, &resource.ResourceSearchResponse_TermFacet{
|
||||
Term: t.Term,
|
||||
Count: int64(t.Count),
|
||||
})
|
||||
}
|
||||
}
|
||||
if response.Facet == nil {
|
||||
response.Facet = make(map[string]*resource.ResourceSearchResponse_Facet)
|
||||
}
|
||||
response.Facet[k] = f
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// make sure the request key matches the index
|
||||
func (b *bleveIndex) verifyKey(key *resource.ResourceKey) *resource.ErrorResult {
|
||||
if key.Namespace != b.key.Namespace {
|
||||
return resource.NewBadRequestError("namespace mismatch (expected " + b.key.Namespace + ")")
|
||||
}
|
||||
if key.Group != b.key.Group {
|
||||
return resource.NewBadRequestError("group mismatch (expected " + b.key.Group + ")")
|
||||
}
|
||||
if key.Resource != b.key.Resource {
|
||||
return resource.NewBadRequestError("resource mismatch (expected " + b.key.Resource + ")")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *bleveIndex) getIndex(
|
||||
req *resource.ResourceSearchRequest,
|
||||
federate []resource.ResourceIndex,
|
||||
) (bleve.Index, error) {
|
||||
if len(req.Federated) != len(federate) {
|
||||
return nil, fmt.Errorf("federation is misconfigured")
|
||||
}
|
||||
|
||||
// Search across resources using
|
||||
// https://blevesearch.com/docs/IndexAlias/
|
||||
if len(federate) > 0 {
|
||||
all := []bleve.Index{b.index}
|
||||
for i, extra := range federate {
|
||||
typedindex, ok := extra.(*bleveIndex)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("federated indexes must be the same type")
|
||||
}
|
||||
if typedindex.verifyKey(req.Federated[i]) != nil {
|
||||
return nil, fmt.Errorf("federated index keys do not match")
|
||||
}
|
||||
all = append(all, typedindex.index)
|
||||
}
|
||||
return bleve.NewIndexAlias(all...), nil
|
||||
}
|
||||
return b.index, nil
|
||||
}
|
||||
|
||||
func toBleveSearchRequest(req *resource.ResourceSearchRequest, access authz.AccessClient) (*bleve.SearchRequest, *resource.ErrorResult) {
|
||||
searchrequest := &bleve.SearchRequest{
|
||||
Fields: req.Fields,
|
||||
Size: int(req.Limit),
|
||||
From: int(req.Offset),
|
||||
Explain: req.Explain,
|
||||
}
|
||||
|
||||
// Currently everything is within an AND query
|
||||
queries := []query.Query{}
|
||||
if len(req.Options.Labels) > 0 {
|
||||
for _, v := range req.Options.Labels {
|
||||
q, err := requirementQuery(v, "labels.")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
queries = append(queries, q)
|
||||
}
|
||||
}
|
||||
if len(req.Options.Fields) > 0 {
|
||||
for _, v := range req.Options.Fields {
|
||||
q, err := requirementQuery(v, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
queries = append(queries, q)
|
||||
}
|
||||
}
|
||||
|
||||
if req.Query != "" {
|
||||
// ??? Should expose the full power of query parsing here?
|
||||
// it is great for exploration, but also hard to change in the future
|
||||
q := bleve.NewQueryStringQuery(req.Query)
|
||||
queries = append(queries, q)
|
||||
}
|
||||
|
||||
if access != nil {
|
||||
// TODO AUTHZ!!!!
|
||||
// Need to add an authz filter into the mix
|
||||
// See: https://github.com/grafana/grafana/blob/v11.3.0/pkg/services/searchV2/bluge.go
|
||||
// NOTE, we likely want to pass in the already called checker because the resource server
|
||||
// will first need to check if we can see anything (or everything!) for this resource
|
||||
fmt.Printf("TODO... check authorization")
|
||||
}
|
||||
|
||||
switch len(queries) {
|
||||
case 0:
|
||||
searchrequest.Query = bleve.NewMatchAllQuery()
|
||||
case 1:
|
||||
searchrequest.Query = queries[0]
|
||||
default:
|
||||
searchrequest.Query = bleve.NewConjunctionQuery(queries...) // AND
|
||||
}
|
||||
|
||||
for k, v := range req.Facet {
|
||||
if searchrequest.Facets == nil {
|
||||
searchrequest.Facets = make(bleve.FacetsRequest)
|
||||
}
|
||||
searchrequest.Facets[k] = bleve.NewFacetRequest(v.Field, int(v.Limit))
|
||||
}
|
||||
|
||||
// Add the sort fields
|
||||
for _, sort := range req.SortBy {
|
||||
// hardcoded (for now)
|
||||
if strings.HasPrefix(sort.Field, "stats.") {
|
||||
searchrequest.Sort = append(searchrequest.Sort, &search.SortField{
|
||||
Field: sort.Field,
|
||||
Desc: sort.Desc,
|
||||
Type: search.SortFieldAsNumber, // force for now!
|
||||
Mode: search.SortFieldDefault, // ???
|
||||
Missing: search.SortFieldMissingLast,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// Default support
|
||||
input := sort.Field
|
||||
if sort.Desc {
|
||||
input = "-" + sort.Field
|
||||
}
|
||||
s := search.ParseSearchSortString(input)
|
||||
searchrequest.Sort = append(searchrequest.Sort, s)
|
||||
}
|
||||
|
||||
// Always sort by *something*, otherwise the order is unstable
|
||||
if len(searchrequest.Sort) == 0 {
|
||||
searchrequest.Sort = append(searchrequest.Sort, &search.SortDocID{
|
||||
Desc: false,
|
||||
})
|
||||
}
|
||||
|
||||
return searchrequest, nil
|
||||
}
|
||||
|
||||
// Convert a "requirement" into a bleve query
|
||||
func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *resource.ErrorResult) {
|
||||
switch selection.Operator(req.Operator) {
|
||||
case selection.Equals, selection.DoubleEquals:
|
||||
if len(req.Values) != 1 {
|
||||
return nil, resource.NewBadRequestError("equals query can have one value")
|
||||
}
|
||||
q := query.NewMatchQuery(req.Values[0])
|
||||
q.FieldVal = prefix + req.Key
|
||||
return q, nil
|
||||
|
||||
case selection.NotEquals:
|
||||
case selection.DoesNotExist:
|
||||
case selection.GreaterThan:
|
||||
case selection.LessThan:
|
||||
case selection.Exists:
|
||||
case selection.In:
|
||||
case selection.NotIn:
|
||||
}
|
||||
return nil, resource.NewBadRequestError(
|
||||
fmt.Sprintf("unsupported query operation (%s %s %v)", req.Key, req.Operator, req.Values),
|
||||
)
|
||||
}
|
||||
|
||||
func (b *bleveIndex) hitsToTable(selectFields []string, hits search.DocumentMatchCollection, explain bool) (*resource.ResourceTable, error) {
|
||||
fields := []*resource.ResourceTableColumnDefinition{}
|
||||
for _, name := range selectFields {
|
||||
if name == "_all" {
|
||||
fields = b.allFields
|
||||
break
|
||||
}
|
||||
|
||||
f := b.standard.Field(name)
|
||||
if f == nil && b.fields != nil {
|
||||
f = b.fields.Field(name)
|
||||
}
|
||||
if f == nil {
|
||||
// Labels as a string
|
||||
if strings.HasPrefix(name, "labels.") {
|
||||
f = &resource.ResourceTableColumnDefinition{
|
||||
Name: name,
|
||||
Type: resource.ResourceTableColumnDefinition_STRING,
|
||||
}
|
||||
}
|
||||
|
||||
// return nil, fmt.Errorf("unknown response field: " + name)
|
||||
if f == nil {
|
||||
continue // OK for now
|
||||
}
|
||||
}
|
||||
fields = append(fields, f)
|
||||
}
|
||||
if explain {
|
||||
fields = append(fields, b.standard.Field(resource.SEARCH_FIELD_EXPLAIN))
|
||||
}
|
||||
|
||||
builder, err := resource.NewTableBuilder(fields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encoders := builder.Encoders()
|
||||
|
||||
table := &resource.ResourceTable{
|
||||
Columns: fields,
|
||||
Rows: make([]*resource.ResourceTableRow, hits.Len()),
|
||||
}
|
||||
for rowID, match := range hits {
|
||||
row := &resource.ResourceTableRow{
|
||||
Key: &resource.ResourceKey{},
|
||||
Cells: make([][]byte, len(fields)),
|
||||
}
|
||||
table.Rows[rowID] = row
|
||||
|
||||
err := row.Key.ReadSearchID(match.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i, f := range fields {
|
||||
if f.Name == resource.SEARCH_FIELD_ID {
|
||||
row.Cells[i] = []byte(match.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
// QUICK QUICK... more options yes
|
||||
v := match.Fields[f.Name]
|
||||
if v != nil {
|
||||
// Encode the value to protobuf
|
||||
row.Cells[i], err = encoders[i](v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error encoding (row:%d/col:%d) %v %w", rowID, i, v, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return table, nil
|
||||
}
|
||||
|
||||
func getAllFields(standard resource.SearchableDocumentFields, custom resource.SearchableDocumentFields) ([]*resource.ResourceTableColumnDefinition, error) {
|
||||
fields := []*resource.ResourceTableColumnDefinition{
|
||||
standard.Field(resource.SEARCH_FIELD_ID),
|
||||
standard.Field(resource.SEARCH_FIELD_TITLE),
|
||||
standard.Field(resource.SEARCH_FIELD_TAGS),
|
||||
standard.Field(resource.SEARCH_FIELD_FOLDER),
|
||||
standard.Field(resource.SEARCH_FIELD_RV),
|
||||
standard.Field(resource.SEARCH_FIELD_CREATED),
|
||||
}
|
||||
|
||||
if custom != nil {
|
||||
for _, name := range custom.Fields() {
|
||||
f := custom.Field(name)
|
||||
if f.Priority > 10 {
|
||||
continue
|
||||
}
|
||||
fields = append(fields, f)
|
||||
}
|
||||
}
|
||||
for _, field := range fields {
|
||||
if field == nil {
|
||||
return nil, fmt.Errorf("invalid all field")
|
||||
}
|
||||
}
|
||||
return fields, nil
|
||||
}
|
||||
68
pkg/storage/unified/search/bleve_mappings.go
Normal file
68
pkg/storage/unified/search/bleve_mappings.go
Normal file
@ -0,0 +1,68 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
|
||||
"github.com/blevesearch/bleve/v2/mapping"
|
||||
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
)
|
||||
|
||||
func getBleveMappings(fields resource.SearchableDocumentFields) mapping.IndexMapping {
|
||||
mapper := bleve.NewIndexMapping()
|
||||
mapper.DefaultMapping = getBleveDocMappings(fields)
|
||||
return mapper
|
||||
}
|
||||
|
||||
func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentMapping {
|
||||
mapper := bleve.NewDocumentStaticMapping()
|
||||
mapper.AddFieldMapping(&mapping.FieldMapping{
|
||||
Name: "title",
|
||||
Type: "text",
|
||||
// TODO - if we don't want title to be a keyword, we can use this
|
||||
// set the title field to use keyword analyzer so it sorts by the whole phrase
|
||||
// https://github.com/blevesearch/bleve/issues/417#issuecomment-245273022
|
||||
Analyzer: keyword.Name,
|
||||
Store: true,
|
||||
Index: true,
|
||||
IncludeTermVectors: true,
|
||||
IncludeInAll: true,
|
||||
DocValues: false,
|
||||
})
|
||||
|
||||
mapper.AddFieldMapping(&mapping.FieldMapping{
|
||||
Name: "description",
|
||||
Type: "text",
|
||||
Store: true,
|
||||
Index: true,
|
||||
IncludeTermVectors: false,
|
||||
IncludeInAll: false,
|
||||
DocValues: false,
|
||||
})
|
||||
|
||||
mapper.AddFieldMapping(&mapping.FieldMapping{
|
||||
Name: "tags",
|
||||
Type: "text",
|
||||
Analyzer: keyword.Name,
|
||||
Store: true,
|
||||
Index: true,
|
||||
IncludeTermVectors: false,
|
||||
IncludeInAll: false,
|
||||
DocValues: false,
|
||||
})
|
||||
|
||||
mapper.AddFieldMapping(&mapping.FieldMapping{
|
||||
Name: "folder",
|
||||
Type: "text",
|
||||
Analyzer: keyword.Name,
|
||||
Store: true,
|
||||
Index: true,
|
||||
IncludeTermVectors: false,
|
||||
IncludeInAll: false,
|
||||
DocValues: true, // will be needed for authz client
|
||||
})
|
||||
|
||||
mapper.Dynamic = true
|
||||
|
||||
return mapper
|
||||
}
|
||||
46
pkg/storage/unified/search/bleve_mappings_test.go
Normal file
46
pkg/storage/unified/search/bleve_mappings_test.go
Normal file
@ -0,0 +1,46 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/document"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
)
|
||||
|
||||
func TestDocumentMapping(t *testing.T) {
|
||||
mappings := getBleveMappings(nil)
|
||||
data := resource.IndexableDocument{
|
||||
Title: "title",
|
||||
Description: "descr",
|
||||
Tags: []string{"a", "b"},
|
||||
Created: 12345,
|
||||
Folder: "xyz",
|
||||
CreatedBy: "user:ryan",
|
||||
Labels: map[string]string{
|
||||
"a": "b",
|
||||
"x": "y",
|
||||
},
|
||||
RV: 1234,
|
||||
RepoInfo: &utils.ResourceRepositoryInfo{
|
||||
Name: "nnn",
|
||||
Path: "ppp",
|
||||
Hash: "hhh",
|
||||
},
|
||||
}
|
||||
|
||||
doc := document.NewDocument("id")
|
||||
err := mappings.MapDocument(doc, data)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, f := range doc.Fields {
|
||||
fmt.Printf("%s = %+v\n", f.Name(), f.Value())
|
||||
}
|
||||
|
||||
fmt.Printf("DOC: fields %d\n", len(doc.Fields))
|
||||
fmt.Printf("DOC: size %d\n", doc.Size())
|
||||
require.Equal(t, 15, len(doc.Fields))
|
||||
}
|
||||
289
pkg/storage/unified/search/bleve_test.go
Normal file
289
pkg/storage/unified/search/bleve_test.go
Normal file
@ -0,0 +1,289 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/grafana/grafana/pkg/services/store/kind/dashboard"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
)
|
||||
|
||||
func TestBleveBackend(t *testing.T) {
|
||||
dashboardskey := &resource.ResourceKey{
|
||||
Namespace: "default",
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
}
|
||||
folderKey := &resource.ResourceKey{
|
||||
Namespace: dashboardskey.Namespace,
|
||||
Group: "folder.grafana.app",
|
||||
Resource: "folders",
|
||||
}
|
||||
tmpdir, err := os.CreateTemp("", "bleve-test")
|
||||
require.NoError(t, err)
|
||||
|
||||
backend := NewBleveBackend(
|
||||
bleveOptions{
|
||||
Root: tmpdir.Name(),
|
||||
FileThreshold: 5, // with more than 5 items we create a file on disk
|
||||
},
|
||||
tracing.NewNoopTracerService(),
|
||||
nil,
|
||||
)
|
||||
|
||||
rv := int64(10)
|
||||
ctx := context.Background()
|
||||
var dashboardsIndex resource.ResourceIndex
|
||||
var foldersIndex resource.ResourceIndex
|
||||
|
||||
t.Run("build dashboards", func(t *testing.T) {
|
||||
key := dashboardskey
|
||||
info, err := DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) {
|
||||
return &DashboardDocumentBuilder{
|
||||
Namespace: namespace,
|
||||
Blob: blob,
|
||||
Stats: NewDashboardStatsLookup(nil), // empty stats
|
||||
DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{{}}),
|
||||
}, nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
index, err := backend.BuildIndex(ctx, resource.NamespacedResource{
|
||||
Namespace: key.Namespace,
|
||||
Group: key.Group,
|
||||
Resource: key.Resource,
|
||||
}, 2, rv, info.Fields, func(index resource.ResourceIndex) (int64, error) {
|
||||
_ = index.Write(&resource.IndexableDocument{
|
||||
RV: 1,
|
||||
Key: &resource.ResourceKey{
|
||||
Name: "aaa",
|
||||
Namespace: "ns",
|
||||
Group: "g",
|
||||
Resource: "dash",
|
||||
},
|
||||
Title: "bbb (dash)",
|
||||
Folder: "xxx",
|
||||
Fields: map[string]any{
|
||||
DASHBOARD_LEGACY_ID: 12,
|
||||
DASHBOARD_PANEL_TYPES: []string{"timeseries", "table"},
|
||||
DASHBOARD_ERRORS_TODAY: 25,
|
||||
},
|
||||
Tags: []string{"aa", "bb"},
|
||||
})
|
||||
_ = index.Write(&resource.IndexableDocument{
|
||||
RV: 2,
|
||||
Key: &resource.ResourceKey{
|
||||
Name: "bbb",
|
||||
Namespace: "ns",
|
||||
Group: "g",
|
||||
Resource: "dash",
|
||||
},
|
||||
Title: "aaa (dash)",
|
||||
Folder: "xxx",
|
||||
Fields: map[string]any{
|
||||
DASHBOARD_LEGACY_ID: 12,
|
||||
DASHBOARD_PANEL_TYPES: []string{"timeseries"},
|
||||
DASHBOARD_ERRORS_TODAY: 40,
|
||||
},
|
||||
Tags: []string{"aa"},
|
||||
Labels: map[string]string{
|
||||
"region": "east",
|
||||
},
|
||||
})
|
||||
_ = index.Write(&resource.IndexableDocument{
|
||||
RV: 3,
|
||||
Key: &resource.ResourceKey{
|
||||
Name: "ccc",
|
||||
Namespace: "ns",
|
||||
Group: "g",
|
||||
Resource: "dash",
|
||||
},
|
||||
Title: "ccc (dash)",
|
||||
Folder: "xxx",
|
||||
Fields: map[string]any{
|
||||
DASHBOARD_LEGACY_ID: 12,
|
||||
},
|
||||
Tags: []string{"aa"},
|
||||
Labels: map[string]string{
|
||||
"region": "west",
|
||||
},
|
||||
})
|
||||
return rv, nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, index)
|
||||
dashboardsIndex = index
|
||||
|
||||
rsp, err := index.Search(ctx, nil, &resource.ResourceSearchRequest{
|
||||
Options: &resource.ListOptions{
|
||||
Key: key,
|
||||
},
|
||||
Limit: 100000,
|
||||
SortBy: []*resource.ResourceSearchRequest_Sort{
|
||||
{Field: "title", Desc: true}, // ccc,bbb,aaa
|
||||
},
|
||||
Facet: map[string]*resource.ResourceSearchRequest_Facet{
|
||||
"tags": {
|
||||
Field: "tags",
|
||||
Limit: 100,
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, rsp.Error)
|
||||
require.NotNil(t, rsp.Results)
|
||||
require.NotNil(t, rsp.Facet)
|
||||
|
||||
// Match the results
|
||||
resource.AssertTableSnapshot(t, filepath.Join("testdata", "manual-dashboard.json"), rsp.Results)
|
||||
|
||||
// Get the tags facets
|
||||
facet, ok := rsp.Facet["tags"]
|
||||
require.True(t, ok)
|
||||
disp, err := json.MarshalIndent(facet, "", " ")
|
||||
require.NoError(t, err)
|
||||
//fmt.Printf("%s\n", disp)
|
||||
require.JSONEq(t, `{
|
||||
"field": "tags",
|
||||
"total": 4,
|
||||
"terms": [
|
||||
{
|
||||
"term": "aa",
|
||||
"count": 3
|
||||
},
|
||||
{
|
||||
"term": "bb",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}`, string(disp))
|
||||
})
|
||||
|
||||
t.Run("build folders", func(t *testing.T) {
|
||||
key := folderKey
|
||||
var fields resource.SearchableDocumentFields
|
||||
|
||||
index, err := backend.BuildIndex(ctx, resource.NamespacedResource{
|
||||
Namespace: key.Namespace,
|
||||
Group: key.Group,
|
||||
Resource: key.Resource,
|
||||
}, 2, rv, fields, func(index resource.ResourceIndex) (int64, error) {
|
||||
_ = index.Write(&resource.IndexableDocument{
|
||||
RV: 1,
|
||||
Key: &resource.ResourceKey{
|
||||
Name: "zzz",
|
||||
Namespace: "ns",
|
||||
Group: "g",
|
||||
Resource: "folder",
|
||||
},
|
||||
Title: "zzz (folder)",
|
||||
})
|
||||
_ = index.Write(&resource.IndexableDocument{
|
||||
RV: 2,
|
||||
Key: &resource.ResourceKey{
|
||||
Name: "yyy",
|
||||
Namespace: "ns",
|
||||
Group: "g",
|
||||
Resource: "folder",
|
||||
},
|
||||
Title: "yyy (folder)",
|
||||
Labels: map[string]string{
|
||||
"region": "west",
|
||||
},
|
||||
})
|
||||
return rv, nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, index)
|
||||
foldersIndex = index
|
||||
|
||||
rsp, err := index.Search(ctx, nil, &resource.ResourceSearchRequest{
|
||||
Options: &resource.ListOptions{
|
||||
Key: key,
|
||||
},
|
||||
Limit: 100000,
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, rsp.Error)
|
||||
require.NotNil(t, rsp.Results)
|
||||
require.Nil(t, rsp.Facet)
|
||||
|
||||
resource.AssertTableSnapshot(t, filepath.Join("testdata", "manual-folder.json"), rsp.Results)
|
||||
})
|
||||
|
||||
t.Run("simple federation", func(t *testing.T) {
|
||||
// The other tests must run first to build the indexes
|
||||
require.NotNil(t, dashboardsIndex)
|
||||
require.NotNil(t, foldersIndex)
|
||||
|
||||
// Use a federated query to get both results together, sorted by title
|
||||
rsp, err := dashboardsIndex.Search(ctx, nil, &resource.ResourceSearchRequest{
|
||||
Options: &resource.ListOptions{
|
||||
Key: dashboardskey,
|
||||
},
|
||||
Fields: []string{
|
||||
"title", "_id",
|
||||
},
|
||||
Federated: []*resource.ResourceKey{
|
||||
folderKey, // This will join in the
|
||||
},
|
||||
Limit: 100000,
|
||||
SortBy: []*resource.ResourceSearchRequest_Sort{
|
||||
{Field: "title", Desc: false},
|
||||
},
|
||||
Facet: map[string]*resource.ResourceSearchRequest_Facet{
|
||||
"region": {
|
||||
Field: "labels.region",
|
||||
Limit: 100,
|
||||
},
|
||||
},
|
||||
}, []resource.ResourceIndex{foldersIndex}) // << note the folder index matches the federation request
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, rsp.Error)
|
||||
require.NotNil(t, rsp.Results)
|
||||
require.NotNil(t, rsp.Facet)
|
||||
|
||||
// Sorted across two indexes
|
||||
sorted := []string{}
|
||||
for _, row := range rsp.Results.Rows {
|
||||
sorted = append(sorted, string(row.Cells[0]))
|
||||
}
|
||||
require.Equal(t, []string{
|
||||
"aaa (dash)",
|
||||
"bbb (dash)",
|
||||
"ccc (dash)",
|
||||
"yyy (folder)",
|
||||
"zzz (folder)",
|
||||
}, sorted)
|
||||
|
||||
resource.AssertTableSnapshot(t, filepath.Join("testdata", "manual-federated.json"), rsp.Results)
|
||||
|
||||
facet, ok := rsp.Facet["region"]
|
||||
require.True(t, ok)
|
||||
disp, err := json.MarshalIndent(facet, "", " ")
|
||||
require.NoError(t, err)
|
||||
// fmt.Printf("%s\n", disp)
|
||||
// NOTE, the west values come from *both* dashboards and folders
|
||||
require.JSONEq(t, `{
|
||||
"field": "labels.region",
|
||||
"total": 3,
|
||||
"missing": 2,
|
||||
"terms": [
|
||||
{
|
||||
"term": "west",
|
||||
"count": 2
|
||||
},
|
||||
{
|
||||
"term": "east",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
}`, string(disp))
|
||||
})
|
||||
}
|
||||
@ -26,7 +26,11 @@ func (s *StandardDocumentBuilders) GetDocumentBuilders() ([]resource.DocumentBui
|
||||
})
|
||||
|
||||
return []resource.DocumentBuilderInfo{
|
||||
resource.StandardDocumentBuilder(),
|
||||
// The default builder
|
||||
resource.DocumentBuilderInfo{
|
||||
Builder: resource.StandardDocumentBuilder(),
|
||||
},
|
||||
// Dashboard builder
|
||||
dashboards,
|
||||
}, err
|
||||
}
|
||||
|
||||
@ -80,7 +80,7 @@ func TestDashboardDocumentBuilder(t *testing.T) {
|
||||
})
|
||||
|
||||
// Standard
|
||||
builder = resource.StandardDocumentBuilder().Builder
|
||||
builder = resource.StandardDocumentBuilder()
|
||||
doSnapshotTests(t, builder, "folder", key, []string{
|
||||
"aaa",
|
||||
"bbb",
|
||||
|
||||
143
pkg/storage/unified/search/testdata/manual-dashboard.json
vendored
Normal file
143
pkg/storage/unified/search/testdata/manual-dashboard.json
vendored
Normal file
@ -0,0 +1,143 @@
|
||||
{
|
||||
"metadata": {},
|
||||
"columnDefinitions": [
|
||||
{
|
||||
"name": "_id",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Unique Identifier. {namespace}/{group}/{resource}/{name}",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "title",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Display name for the resource",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "tags",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Unique tags",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "folder",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Kubernetes name for the folder",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "rv",
|
||||
"type": "number",
|
||||
"format": "int64",
|
||||
"description": "resource version",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "created",
|
||||
"type": "number",
|
||||
"format": "int64",
|
||||
"description": "created timestamp",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "schema_version",
|
||||
"type": "number",
|
||||
"format": "int32",
|
||||
"description": "Numeric version saying when the schema was saved",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "link_count",
|
||||
"type": "number",
|
||||
"format": "int32",
|
||||
"description": "How many links appear on the page",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "panel_types",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "How many links appear on the page",
|
||||
"priority": 0
|
||||
}
|
||||
],
|
||||
"rows": [
|
||||
{
|
||||
"cells": [
|
||||
"ns/g/dash/ccc",
|
||||
"ccc (dash)",
|
||||
[
|
||||
"aa"
|
||||
],
|
||||
"xxx",
|
||||
3,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
],
|
||||
"object": {
|
||||
"kind": "dash",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "ccc",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cells": [
|
||||
"ns/g/dash/aaa",
|
||||
"bbb (dash)",
|
||||
[
|
||||
"aa",
|
||||
"bb"
|
||||
],
|
||||
"xxx",
|
||||
1,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
],
|
||||
"object": {
|
||||
"kind": "dash",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "aaa",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cells": [
|
||||
"ns/g/dash/bbb",
|
||||
"aaa (dash)",
|
||||
[
|
||||
"aa"
|
||||
],
|
||||
"xxx",
|
||||
2,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
],
|
||||
"object": {
|
||||
"kind": "dash",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "bbb",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
96
pkg/storage/unified/search/testdata/manual-federated.json
vendored
Normal file
96
pkg/storage/unified/search/testdata/manual-federated.json
vendored
Normal file
@ -0,0 +1,96 @@
|
||||
{
|
||||
"metadata": {},
|
||||
"columnDefinitions": [
|
||||
{
|
||||
"name": "title",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Display name for the resource",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "_id",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Unique Identifier. {namespace}/{group}/{resource}/{name}",
|
||||
"priority": 0
|
||||
}
|
||||
],
|
||||
"rows": [
|
||||
{
|
||||
"cells": [
|
||||
"aaa (dash)",
|
||||
"ns/g/dash/bbb"
|
||||
],
|
||||
"object": {
|
||||
"kind": "dash",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "bbb",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cells": [
|
||||
"bbb (dash)",
|
||||
"ns/g/dash/aaa"
|
||||
],
|
||||
"object": {
|
||||
"kind": "dash",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "aaa",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cells": [
|
||||
"ccc (dash)",
|
||||
"ns/g/dash/ccc"
|
||||
],
|
||||
"object": {
|
||||
"kind": "dash",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "ccc",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cells": [
|
||||
"yyy (folder)",
|
||||
"ns/g/folder/yyy"
|
||||
],
|
||||
"object": {
|
||||
"kind": "folder",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "yyy",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cells": [
|
||||
"zzz (folder)",
|
||||
"ns/g/folder/zzz"
|
||||
],
|
||||
"object": {
|
||||
"kind": "folder",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "zzz",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
87
pkg/storage/unified/search/testdata/manual-folder.json
vendored
Normal file
87
pkg/storage/unified/search/testdata/manual-folder.json
vendored
Normal file
@ -0,0 +1,87 @@
|
||||
{
|
||||
"metadata": {},
|
||||
"columnDefinitions": [
|
||||
{
|
||||
"name": "_id",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Unique Identifier. {namespace}/{group}/{resource}/{name}",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "title",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Display name for the resource",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "tags",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Unique tags",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "folder",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "Kubernetes name for the folder",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "rv",
|
||||
"type": "number",
|
||||
"format": "int64",
|
||||
"description": "resource version",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "created",
|
||||
"type": "number",
|
||||
"format": "int64",
|
||||
"description": "created timestamp",
|
||||
"priority": 0
|
||||
}
|
||||
],
|
||||
"rows": [
|
||||
{
|
||||
"cells": [
|
||||
"ns/g/folder/yyy",
|
||||
"yyy (folder)",
|
||||
null,
|
||||
null,
|
||||
2,
|
||||
0
|
||||
],
|
||||
"object": {
|
||||
"kind": "folder",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "yyy",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"cells": [
|
||||
"ns/g/folder/zzz",
|
||||
"zzz (folder)",
|
||||
null,
|
||||
null,
|
||||
1,
|
||||
0
|
||||
],
|
||||
"object": {
|
||||
"kind": "folder",
|
||||
"apiVersion": "g",
|
||||
"metadata": {
|
||||
"name": "zzz",
|
||||
"namespace": "ns",
|
||||
"creationTimestamp": null
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user