mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Search: create bluge based index (#48606)
This commit is contained in:
Ryan McKinley
GitHub
parent
5c2dee19d5
commit
d6d358ef26
16
go.mod
16
go.mod
@ -141,7 +141,6 @@ require (
|
||||
github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
|
||||
github.com/centrifugal/protocol v0.7.6 // indirect
|
||||
@ -153,7 +152,6 @@ require (
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.0 // indirect
|
||||
github.com/edsrzf/mmap-go v1.0.0 // indirect
|
||||
github.com/emicklei/proto v1.6.15 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.2 // indirect
|
||||
@ -246,6 +244,7 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.4.0
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.17
|
||||
github.com/armon/go-radix v1.0.0
|
||||
github.com/blugelabs/bluge v0.1.9
|
||||
github.com/golang-migrate/migrate/v4 v4.7.0
|
||||
github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f
|
||||
github.com/grafana/thema v0.0.0-20220413232647-fc54c169b508
|
||||
@ -274,9 +273,21 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.2.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||
github.com/RoaringBitmap/roaring v0.9.1 // indirect
|
||||
github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect
|
||||
github.com/bits-and-blooms/bitset v1.2.0 // indirect
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
|
||||
github.com/blevesearch/mmap-go v1.0.2 // indirect
|
||||
github.com/blevesearch/segment v0.9.0 // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
github.com/blevesearch/vellum v1.0.5 // indirect
|
||||
github.com/blugelabs/bluge_segment_api v0.2.0 // indirect
|
||||
github.com/blugelabs/ice v0.2.0 // indirect
|
||||
github.com/caio/go-tdigest v3.1.0+incompatible // indirect
|
||||
github.com/chromedp/cdproto v0.0.0-20220208224320-6efb837e6bc2 // indirect
|
||||
github.com/containerd/containerd v1.6.2 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect
|
||||
github.com/elazarl/goproxy v0.0.0-20220115173737-adb46da277ac // indirect
|
||||
github.com/getkin/kin-openapi v0.94.0 // indirect
|
||||
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
|
||||
@ -287,6 +298,7 @@ require (
|
||||
github.com/imdario/mergo v0.3.12 // indirect
|
||||
github.com/klauspost/compress v1.15.1 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/mschoch/smat v0.2.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.0.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.8 // indirect
|
||||
github.com/segmentio/asm v1.1.1 // indirect
|
||||
|
||||
35
go.sum
35
go.sum
@ -293,6 +293,10 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/RoaringBitmap/gocroaring v0.4.0/go.mod h1:NieMwz7ZqwU2DD73/vvYwv7r4eWBKuPVSXZIpsaMwCI=
|
||||
github.com/RoaringBitmap/real-roaring-datasets v0.0.0-20190726190000-eb7c87156f76/go.mod h1:oM0MHmQ3nDsq609SS36p+oYbRi16+oVvU2Bw4Ipv0SE=
|
||||
github.com/RoaringBitmap/roaring v0.9.1 h1:5PRizBmoN/PfV17nPNQou4dHQ7NcJi8FO/bihdYyCEM=
|
||||
github.com/RoaringBitmap/roaring v0.9.1/go.mod h1:h1B7iIUOmnAeb5ytYMvnHJwxMc6LUrwBnzXWRuqTQUc=
|
||||
github.com/SAP/go-hdb v0.14.1/go.mod h1:7fdQLVC2lER3urZLjZCm0AuMQfApof92n3aylBPEkMo=
|
||||
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
|
||||
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
|
||||
@ -442,6 +446,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4Nx
|
||||
github.com/aws/smithy-go v1.5.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
|
||||
github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE=
|
||||
github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
|
||||
github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f h1:y06x6vGnFYfXUoVMbrcP1Uzpj4JG01eB5vRps9G8agM=
|
||||
github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f/go.mod h1:2stgcRjl6QmW+gU2h5E7BQXg4HU0gzxKWDuT5HviN9s=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
|
||||
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
|
||||
@ -462,6 +468,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
|
||||
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
|
||||
github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
|
||||
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
|
||||
github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
|
||||
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
|
||||
@ -469,6 +476,22 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
|
||||
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
|
||||
github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0=
|
||||
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
|
||||
github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac=
|
||||
github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
|
||||
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
|
||||
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
|
||||
github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A=
|
||||
github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ=
|
||||
github.com/blugelabs/bluge v0.1.9 h1:bPgXlcsWugrXNjzeoLdOnvfJpHsyODKpYaAndayl/SM=
|
||||
github.com/blugelabs/bluge v0.1.9/go.mod h1:5d7LktUkQgvbh5Bmi6tPWtvo4+6uRTm6gAwP+5z6FqQ=
|
||||
github.com/blugelabs/bluge_segment_api v0.2.0 h1:cCX1Y2y8v0LZ7+EEJ6gH7dW6TtVTW4RhG0vp3R+N2Lo=
|
||||
github.com/blugelabs/bluge_segment_api v0.2.0/go.mod h1:95XA+ZXfRj/IXADm7gZ+iTcWOJPg5jQTY1EReIzl3LA=
|
||||
github.com/blugelabs/ice v0.2.0 h1:9N/TRBqAr43emheD1ptk9mohuT6xAVq83gesgE60Qqk=
|
||||
github.com/blugelabs/ice v0.2.0/go.mod h1:7foiDf4V83FIYYnGh2LOoRWsbNoCqAAMNgKn879Iyu0=
|
||||
github.com/bmatcuk/doublestar v1.2.2/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
|
||||
github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
|
||||
@ -481,7 +504,6 @@ github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQ
|
||||
github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
|
||||
github.com/bsm/sarama-cluster v2.1.13+incompatible/go.mod h1:r7ao+4tTNXvWm+VRpRJchr2kQhqxgmAp2iEX5W96gMM=
|
||||
github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
|
||||
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
|
||||
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
|
||||
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
|
||||
@ -491,6 +513,8 @@ github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee h1:BnPxIde0gjtTnc9
|
||||
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
|
||||
github.com/cactus/go-statsd-client/statsd v0.0.0-20191106001114-12b4e2b38748/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI=
|
||||
github.com/caio/go-tdigest v2.3.0+incompatible/go.mod h1:sHQM/ubZStBUmF1WbB8FAm8q9GjDajLC5T7ydxE3JHI=
|
||||
github.com/caio/go-tdigest v3.1.0+incompatible h1:uoVMJ3Q5lXmVLCCqaMGHLBWnbGoN6Lpu7OAUPR60cds=
|
||||
github.com/caio/go-tdigest v3.1.0+incompatible/go.mod h1:sHQM/ubZStBUmF1WbB8FAm8q9GjDajLC5T7ydxE3JHI=
|
||||
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
||||
github.com/casbin/casbin/v2 v2.31.6/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg=
|
||||
github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
@ -762,6 +786,8 @@ github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/
|
||||
github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8=
|
||||
github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
@ -828,7 +854,6 @@ github.com/dop251/goja v0.0.0-20210804101310-32956a348b49/go.mod h1:R9ET47fwRVRP
|
||||
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
|
||||
github.com/drone/envsubst v1.0.2/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0=
|
||||
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
|
||||
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
||||
@ -1627,6 +1652,7 @@ github.com/influxdata/flux v0.120.1/go.mod h1:pGSAvyAA5d3et7SSzajaYShWYXmnRnJJq2
|
||||
github.com/influxdata/go-syslog/v2 v2.0.1/go.mod h1:hjvie1UTaD5E1fTnDmxaCw8RRDrT4Ve+XHr5O2dKSCo=
|
||||
github.com/influxdata/go-syslog/v3 v3.0.1-0.20201128200927-a1889d947b48/go.mod h1:aXdIdfn2OcGnMhOTojXmwZqXKgC3MU5riiNvzwwG9OY=
|
||||
github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69/go.mod h1:pwymjR6SrP3gD3pRj9RJwdl1j5s3doEEV8gS4X9qSzA=
|
||||
github.com/influxdata/influxdb v1.7.6/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
|
||||
github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
|
||||
github.com/influxdata/influxdb v1.8.0/go.mod h1:SIzcnsjaHRFpmlxpJ4S3NT64qtEKYweNTUMb/vh0OMQ=
|
||||
github.com/influxdata/influxdb v1.8.1/go.mod h1:SIzcnsjaHRFpmlxpJ4S3NT64qtEKYweNTUMb/vh0OMQ=
|
||||
@ -1843,6 +1869,7 @@ github.com/lann/builder v0.0.0-20150808151131-f22ce00fd939/go.mod h1:dXGbAdH5GtB
|
||||
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
|
||||
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
|
||||
github.com/leanovate/gopter v0.2.4/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8=
|
||||
github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 h1:X/79QL0b4YJVO5+OsPH9rF2u428CIrGL/jLmPsoOQQ4=
|
||||
github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353/go.mod h1:N0SVk0uhy+E1PZ3C9ctsPRlvOPAFPkCNlcPBDkt0N3U=
|
||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg=
|
||||
@ -2050,6 +2077,8 @@ github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9
|
||||
github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY=
|
||||
github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
|
||||
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
|
||||
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
|
||||
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
|
||||
github.com/multiplay/go-ts3 v1.0.0/go.mod h1:14S6cS3fLNT3xOytrA/DkRyAFNuQLMLEqOYAsf87IbQ=
|
||||
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
@ -3141,6 +3170,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190102155601-82a175fd1598/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -3470,6 +3500,7 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
|
||||
gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
|
||||
gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
|
||||
gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM=
|
||||
gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
|
||||
gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0=
|
||||
gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
|
||||
|
||||
541
pkg/services/searchV2/bluge.go
Normal file
541
pkg/services/searchV2/bluge.go
Normal file
@ -0,0 +1,541 @@
|
||||
package searchV2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/blugelabs/bluge"
|
||||
"github.com/blugelabs/bluge/search"
|
||||
"github.com/blugelabs/bluge/search/aggregations"
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend"
|
||||
"github.com/grafana/grafana-plugin-sdk-go/data"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
)
|
||||
|
||||
const (
|
||||
documentFieldUID = "_id" // actually UID!! but bluge likes "_id"
|
||||
documentFieldKind = "kind"
|
||||
documentFieldTag = "tag"
|
||||
documentFieldURL = "url"
|
||||
documentFieldName = "name"
|
||||
documentFieldDescription = "description"
|
||||
documentFieldLocation = "location" // parent path
|
||||
documentFieldPanelType = "panel_type"
|
||||
documentFieldDSUID = "ds_uid"
|
||||
documentFieldDSType = "ds_type"
|
||||
documentFieldInternalID = "__internal_id" // only for migrations! (indexed as a string)
|
||||
)
|
||||
|
||||
func initBlugeIndex(dashboards []dashboard, logger log.Logger) (*bluge.Reader, error) {
|
||||
config := bluge.InMemoryOnlyConfig()
|
||||
|
||||
// open an index writer using the configuration
|
||||
writer, err := bluge.OpenWriter(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error opening writer: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
logger.Error("Error closing bluge writer", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
start := time.Now()
|
||||
|
||||
logger.Info("Loading dashboards for bluge index", "elapsed", time.Since(start), "numDashboards", len(dashboards))
|
||||
label := time.Now()
|
||||
|
||||
batch := bluge.NewBatch()
|
||||
|
||||
// First index the folders
|
||||
folderIdLookup := make(map[int64]string, 50)
|
||||
for _, dashboard := range dashboards {
|
||||
if !dashboard.isFolder {
|
||||
continue
|
||||
}
|
||||
uid := dashboard.uid
|
||||
url := fmt.Sprintf("/dashboards/f/%s/%s", dashboard.uid, dashboard.slug)
|
||||
if uid == "" {
|
||||
uid = "general"
|
||||
url = "/dashboards"
|
||||
dashboard.info.Title = "General"
|
||||
dashboard.info.Description = ""
|
||||
|
||||
// ARRRG, why is this not in the final index?!!
|
||||
}
|
||||
|
||||
doc := bluge.NewDocument(uid).
|
||||
AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindFolder)).Aggregatable().StoreValue()).
|
||||
AddField(bluge.NewKeywordField(documentFieldURL, url).StoreValue()).
|
||||
AddField(bluge.NewTextField(documentFieldName, dashboard.info.Title).StoreValue().SearchTermPositions()).
|
||||
AddField(bluge.NewTextField(documentFieldDescription, dashboard.info.Description).SearchTermPositions())
|
||||
|
||||
batch.Insert(doc)
|
||||
|
||||
folderIdLookup[dashboard.id] = uid
|
||||
}
|
||||
|
||||
// Then each dashboard
|
||||
for _, dashboard := range dashboards {
|
||||
if dashboard.isFolder {
|
||||
continue
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("/d/%s/%s", dashboard.uid, dashboard.slug)
|
||||
folderUID := folderIdLookup[dashboard.folderID]
|
||||
location := folderUID
|
||||
|
||||
// Dashboard document
|
||||
doc := bluge.NewDocument(dashboard.uid).
|
||||
AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindDashboard)).Aggregatable().StoreValue()).
|
||||
AddField(bluge.NewKeywordField(documentFieldURL, url).StoreValue()).
|
||||
AddField(bluge.NewKeywordField(documentFieldLocation, location).Aggregatable().StoreValue()).
|
||||
AddField(bluge.NewTextField(documentFieldName, dashboard.info.Title).StoreValue().SearchTermPositions()).
|
||||
AddField(bluge.NewTextField(documentFieldDescription, dashboard.info.Description).SearchTermPositions())
|
||||
|
||||
// Add legacy ID (for lookup by internal ID)
|
||||
doc.AddField(bluge.NewKeywordField(documentFieldInternalID, fmt.Sprintf("%d", dashboard.id)))
|
||||
|
||||
for _, tag := range dashboard.info.Tags {
|
||||
doc.AddField(bluge.NewKeywordField(documentFieldTag, tag).
|
||||
StoreValue().
|
||||
Aggregatable().
|
||||
SearchTermPositions())
|
||||
}
|
||||
|
||||
for _, ds := range dashboard.info.Datasource {
|
||||
if ds.UID != "" {
|
||||
doc.AddField(bluge.NewKeywordField(documentFieldDSUID, ds.UID).
|
||||
StoreValue().
|
||||
Aggregatable().
|
||||
SearchTermPositions())
|
||||
}
|
||||
if ds.Type != "" {
|
||||
doc.AddField(bluge.NewKeywordField(documentFieldDSType, ds.Type).
|
||||
StoreValue().
|
||||
Aggregatable().
|
||||
SearchTermPositions())
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: enterprise, add dashboard sorting fields
|
||||
|
||||
batch.Insert(doc)
|
||||
|
||||
location += "/" + dashboard.uid
|
||||
|
||||
// Now add a doc for each panel
|
||||
for _, panel := range dashboard.info.Panels {
|
||||
uid := dashboard.uid + "#" + strconv.FormatInt(panel.ID, 10)
|
||||
purl := url
|
||||
if panel.Type != "row" {
|
||||
purl = fmt.Sprintf("%s?viewPanel=%d", url, panel.ID)
|
||||
}
|
||||
|
||||
doc := bluge.NewDocument(uid).
|
||||
AddField(bluge.NewKeywordField(documentFieldURL, purl).StoreValue()).
|
||||
AddField(bluge.NewTextField(documentFieldName, panel.Title).StoreValue().SearchTermPositions()).
|
||||
AddField(bluge.NewTextField(documentFieldDescription, panel.Description).SearchTermPositions()).
|
||||
AddField(bluge.NewKeywordField(documentFieldPanelType, panel.Type).Aggregatable().StoreValue()).
|
||||
AddField(bluge.NewKeywordField(documentFieldLocation, location).Aggregatable().StoreValue()).
|
||||
AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindPanel)).Aggregatable().StoreValue()) // likely want independent index for this
|
||||
|
||||
batch.Insert(doc)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info("Inserting documents into bluge batch", "elapsed", time.Since(label))
|
||||
label = time.Now()
|
||||
|
||||
err = writer.Batch(batch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader, err := writer.Reader()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logger.Info("Inserting batch into bluge writer", "elapsed", time.Since(label))
|
||||
logger.Info("Finish building bluge index", "totalElapsed", time.Since(start))
|
||||
return reader, err
|
||||
}
|
||||
|
||||
//nolint: gocyclo
|
||||
func doBlugeQuery(ctx context.Context, s *StandardSearchService, reader *bluge.Reader, filter ResourceFilter, q DashboardQuery) *backend.DataResponse {
|
||||
response := &backend.DataResponse{}
|
||||
|
||||
// Folder listing structure
|
||||
idx := strings.Index(q.Query, ":")
|
||||
if idx > 0 {
|
||||
key := q.Query[0:idx]
|
||||
val := q.Query[idx+1:]
|
||||
if key == "list" {
|
||||
q.Limit = 1000
|
||||
q.Query = ""
|
||||
q.Location = ""
|
||||
q.Explain = false
|
||||
q.SkipLocation = true
|
||||
q.Facet = nil
|
||||
if val == "root" || val == "" {
|
||||
q.Kind = []string{string(entityKindFolder)}
|
||||
} else {
|
||||
q.Location = val
|
||||
q.Kind = []string{string(entityKindDashboard)}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hasConstraints := false
|
||||
fullQuery := bluge.NewBooleanQuery()
|
||||
fullQuery.AddMust(newPermissionFilter(filter, s.logger))
|
||||
|
||||
// Only show dashboard / folders
|
||||
if len(q.Kind) > 0 {
|
||||
bq := bluge.NewBooleanQuery()
|
||||
for _, k := range q.Kind {
|
||||
bq.AddShould(bluge.NewTermQuery(k).SetField(documentFieldKind))
|
||||
}
|
||||
fullQuery.AddMust(bq)
|
||||
hasConstraints = true
|
||||
}
|
||||
|
||||
// Explicit UID lookup (stars etc)
|
||||
if len(q.UIDs) > 0 {
|
||||
bq := bluge.NewBooleanQuery()
|
||||
for _, v := range q.UIDs {
|
||||
bq.AddShould(bluge.NewTermQuery(v).SetField(documentFieldUID))
|
||||
}
|
||||
fullQuery.AddMust(bq)
|
||||
hasConstraints = true
|
||||
}
|
||||
|
||||
// Legacy lookup by internal ID
|
||||
if len(q.IDs) > 0 {
|
||||
bq := bluge.NewBooleanQuery()
|
||||
for _, v := range q.IDs {
|
||||
bq.AddShould(bluge.NewTermQuery(fmt.Sprintf("%d", v)).SetField(documentFieldInternalID))
|
||||
}
|
||||
fullQuery.AddMust(bq)
|
||||
hasConstraints = true
|
||||
}
|
||||
|
||||
// Tags
|
||||
if len(q.Tags) > 0 {
|
||||
bq := bluge.NewBooleanQuery()
|
||||
for _, v := range q.Tags {
|
||||
bq.AddMust(bluge.NewTermQuery(v).SetField(documentFieldTag))
|
||||
}
|
||||
fullQuery.AddMust(bq)
|
||||
hasConstraints = true
|
||||
}
|
||||
|
||||
// Datasource
|
||||
if q.Datasource != "" {
|
||||
fullQuery.AddMust(bluge.NewTermQuery(q.Datasource).SetField(documentFieldDSUID))
|
||||
hasConstraints = true
|
||||
}
|
||||
|
||||
// Folder
|
||||
if q.Location != "" {
|
||||
fullQuery.AddMust(bluge.NewTermQuery(q.Location).SetField(documentFieldLocation))
|
||||
hasConstraints = true
|
||||
}
|
||||
|
||||
if q.Query == "*" || q.Query == "" {
|
||||
if !hasConstraints {
|
||||
fullQuery.AddShould(bluge.NewMatchAllQuery())
|
||||
}
|
||||
} else {
|
||||
// The actual se
|
||||
bq := bluge.NewBooleanQuery().
|
||||
AddShould(bluge.NewMatchPhraseQuery(q.Query).SetField("name").SetBoost(6)).
|
||||
AddShould(bluge.NewMatchPhraseQuery(q.Query).SetField("description").SetBoost(3)).
|
||||
AddShould(bluge.NewPrefixQuery(q.Query).SetField("name").SetBoost(1))
|
||||
|
||||
if len(q.Query) > 4 {
|
||||
bq.AddShould(bluge.NewFuzzyQuery(q.Query).SetField("name")).SetBoost(1.5)
|
||||
}
|
||||
fullQuery.AddMust(bq)
|
||||
}
|
||||
|
||||
limit := 50 // default view
|
||||
if q.Limit > 0 {
|
||||
limit = q.Limit
|
||||
}
|
||||
|
||||
req := bluge.NewTopNSearch(limit, fullQuery)
|
||||
if q.From > 0 {
|
||||
req.SetFrom(q.From)
|
||||
}
|
||||
if q.Explain {
|
||||
req.ExplainScores()
|
||||
}
|
||||
req.WithStandardAggregations()
|
||||
|
||||
// SortBy([]string{"-_score", "name"})
|
||||
// req.SortBy([]string{documentFieldName})
|
||||
|
||||
for _, t := range q.Facet {
|
||||
lim := t.Limit
|
||||
if lim < 1 {
|
||||
lim = 50
|
||||
}
|
||||
req.AddAggregation(t.Field, aggregations.NewTermsAggregation(search.Field(t.Field), lim))
|
||||
}
|
||||
|
||||
// execute this search on the reader
|
||||
documentMatchIterator, err := reader.Search(ctx, req)
|
||||
if err != nil {
|
||||
s.logger.Error("error executing search: %v", err)
|
||||
response.Error = err
|
||||
return response
|
||||
}
|
||||
|
||||
dvfieldNames := []string{"type"}
|
||||
sctx := search.NewSearchContext(0, 0)
|
||||
|
||||
// numericFields := map[string]bool{"schemaVersion": true, "panelCount": true}
|
||||
|
||||
fScore := data.NewFieldFromFieldType(data.FieldTypeFloat64, 0)
|
||||
fUID := data.NewFieldFromFieldType(data.FieldTypeString, 0)
|
||||
fKind := data.NewFieldFromFieldType(data.FieldTypeString, 0)
|
||||
fPType := data.NewFieldFromFieldType(data.FieldTypeString, 0)
|
||||
fName := data.NewFieldFromFieldType(data.FieldTypeString, 0)
|
||||
fURL := data.NewFieldFromFieldType(data.FieldTypeString, 0)
|
||||
fLocation := data.NewFieldFromFieldType(data.FieldTypeString, 0)
|
||||
fTags := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, 0)
|
||||
fDSUIDs := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, 0)
|
||||
fExplain := data.NewFieldFromFieldType(data.FieldTypeNullableJSON, 0)
|
||||
|
||||
fScore.Name = "score"
|
||||
fUID.Name = "uid"
|
||||
fKind.Name = "kind"
|
||||
fName.Name = "name"
|
||||
fLocation.Name = "location"
|
||||
fURL.Name = "url"
|
||||
fURL.Config = &data.FieldConfig{
|
||||
Links: []data.DataLink{
|
||||
{Title: "link", URL: "${__value.text}"},
|
||||
},
|
||||
}
|
||||
fPType.Name = "panel_type"
|
||||
fDSUIDs.Name = "ds_uid"
|
||||
fTags.Name = "tags"
|
||||
fExplain.Name = "explain"
|
||||
|
||||
frame := data.NewFrame("Query results", fScore, fKind, fUID, fName, fPType, fURL, fTags, fDSUIDs, fLocation)
|
||||
if q.Explain {
|
||||
frame.Fields = append(frame.Fields, fExplain)
|
||||
}
|
||||
|
||||
locationItems := make(map[string]bool, 50)
|
||||
|
||||
// iterate through the document matches
|
||||
match, err := documentMatchIterator.Next()
|
||||
for err == nil && match != nil {
|
||||
err = match.LoadDocumentValues(sctx, dvfieldNames)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
uid := ""
|
||||
kind := ""
|
||||
ptype := ""
|
||||
name := ""
|
||||
url := ""
|
||||
loc := ""
|
||||
var ds_uids []string
|
||||
var tags []string
|
||||
|
||||
err = match.VisitStoredFields(func(field string, value []byte) bool {
|
||||
// if numericFields[field] {
|
||||
// num, err2 := bluge.DecodeNumericFloat64(value)
|
||||
// if err2 != nil {
|
||||
// vals[field] = num
|
||||
// }
|
||||
// } else {
|
||||
// vals[field] = string(value)
|
||||
// }
|
||||
|
||||
switch field {
|
||||
case documentFieldUID:
|
||||
uid = string(value)
|
||||
case documentFieldKind:
|
||||
kind = string(value)
|
||||
case documentFieldPanelType:
|
||||
ptype = string(value)
|
||||
case documentFieldName:
|
||||
name = string(value)
|
||||
case documentFieldURL:
|
||||
url = string(value)
|
||||
case documentFieldLocation:
|
||||
loc = string(value)
|
||||
case documentFieldDSUID:
|
||||
ds_uids = append(ds_uids, string(value))
|
||||
case documentFieldTag:
|
||||
tags = append(tags, string(value))
|
||||
}
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
s.logger.Error("error loading stored fields: %v", err)
|
||||
response.Error = err
|
||||
return response
|
||||
}
|
||||
|
||||
fScore.Append(match.Score)
|
||||
fKind.Append(kind)
|
||||
fUID.Append(uid)
|
||||
fPType.Append(ptype)
|
||||
fName.Append(name)
|
||||
fURL.Append(url)
|
||||
fLocation.Append(loc)
|
||||
|
||||
// set a key for all path parts we return
|
||||
if !q.SkipLocation {
|
||||
for _, v := range strings.Split(loc, "/") {
|
||||
locationItems[v] = true
|
||||
}
|
||||
}
|
||||
|
||||
if len(tags) > 0 {
|
||||
js, _ := json.Marshal(tags)
|
||||
jsb := json.RawMessage(js)
|
||||
fTags.Append(&jsb)
|
||||
} else {
|
||||
fTags.Append(nil)
|
||||
}
|
||||
|
||||
if len(ds_uids) > 0 {
|
||||
js, _ := json.Marshal(ds_uids)
|
||||
jsb := json.RawMessage(js)
|
||||
fDSUIDs.Append(&jsb)
|
||||
} else {
|
||||
fDSUIDs.Append(nil)
|
||||
}
|
||||
|
||||
if q.Explain {
|
||||
if match.Explanation != nil {
|
||||
js, _ := json.Marshal(&match.Explanation)
|
||||
jsb := json.RawMessage(js)
|
||||
fExplain.Append(&jsb)
|
||||
} else {
|
||||
fExplain.Append(nil)
|
||||
}
|
||||
}
|
||||
|
||||
// load the next document match
|
||||
match, err = documentMatchIterator.Next()
|
||||
}
|
||||
|
||||
// Must call after iterating :)
|
||||
aggs := documentMatchIterator.Aggregations()
|
||||
|
||||
header := &customMeta{
|
||||
Count: aggs.Count(), // Total cound
|
||||
MaxScore: aggs.Metric("max_score"),
|
||||
}
|
||||
if len(locationItems) > 0 && !q.SkipLocation {
|
||||
header.Locations = getLocationLookupInfo(ctx, reader, locationItems)
|
||||
}
|
||||
|
||||
frame.SetMeta(&data.FrameMeta{
|
||||
Type: "search-results",
|
||||
Custom: header,
|
||||
})
|
||||
|
||||
response.Frames = append(response.Frames, frame)
|
||||
|
||||
for _, t := range q.Facet {
|
||||
bbb := aggs.Buckets(t.Field)
|
||||
if bbb != nil {
|
||||
size := len(bbb)
|
||||
|
||||
fName := data.NewFieldFromFieldType(data.FieldTypeString, size)
|
||||
fName.Name = t.Field
|
||||
|
||||
fCount := data.NewFieldFromFieldType(data.FieldTypeUint64, size)
|
||||
fCount.Name = "Count"
|
||||
|
||||
for i, v := range bbb {
|
||||
fName.Set(i, v.Name())
|
||||
fCount.Set(i, v.Count())
|
||||
}
|
||||
|
||||
response.Frames = append(response.Frames, data.NewFrame("Facet: "+t.Field, fName, fCount))
|
||||
}
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
func getLocationLookupInfo(ctx context.Context, reader *bluge.Reader, uids map[string]bool) map[string]locationItem {
|
||||
res := make(map[string]locationItem, len(uids))
|
||||
bq := bluge.NewBooleanQuery()
|
||||
for k := range uids {
|
||||
bq.AddShould(bluge.NewTermQuery(k).SetField(documentFieldUID))
|
||||
}
|
||||
|
||||
req := bluge.NewAllMatches(bq)
|
||||
|
||||
documentMatchIterator, err := reader.Search(ctx, req)
|
||||
if err != nil {
|
||||
return res
|
||||
}
|
||||
|
||||
dvfieldNames := []string{"type"}
|
||||
sctx := search.NewSearchContext(0, 0)
|
||||
|
||||
// execute this search on the reader
|
||||
// iterate through the document matches
|
||||
match, err := documentMatchIterator.Next()
|
||||
for err == nil && match != nil {
|
||||
err = match.LoadDocumentValues(sctx, dvfieldNames)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
uid := ""
|
||||
item := locationItem{}
|
||||
|
||||
_ = match.VisitStoredFields(func(field string, value []byte) bool {
|
||||
switch field {
|
||||
case documentFieldUID:
|
||||
uid = string(value)
|
||||
case documentFieldKind:
|
||||
item.Kind = string(value)
|
||||
case documentFieldName:
|
||||
item.Name = string(value)
|
||||
case documentFieldURL:
|
||||
item.URL = string(value)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
res[uid] = item
|
||||
|
||||
// load the next document match
|
||||
match, err = documentMatchIterator.Next()
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
type locationItem struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type customMeta struct {
|
||||
Count uint64 `json:"count"`
|
||||
MaxScore float64 `json:"max_score,omitempty"`
|
||||
Locations map[string]locationItem `json:"locationInfo,omitempty"`
|
||||
}
|
||||
132
pkg/services/searchV2/filter.go
Normal file
132
pkg/services/searchV2/filter.go
Normal file
@ -0,0 +1,132 @@
|
||||
package searchV2
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
|
||||
"github.com/blugelabs/bluge"
|
||||
"github.com/blugelabs/bluge/search"
|
||||
"github.com/blugelabs/bluge/search/searcher"
|
||||
"github.com/blugelabs/bluge/search/similarity"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
)
|
||||
|
||||
type PermissionFilter struct {
|
||||
log log.Logger
|
||||
filter ResourceFilter
|
||||
}
|
||||
|
||||
type entityKind string
|
||||
|
||||
const (
|
||||
entityKindPanel entityKind = "panel"
|
||||
entityKindDashboard entityKind = "dashboard"
|
||||
entityKindFolder entityKind = "folder"
|
||||
)
|
||||
|
||||
func (r entityKind) IsValid() bool {
|
||||
return r == entityKindPanel || r == entityKindDashboard || r == entityKindFolder
|
||||
}
|
||||
|
||||
func (r entityKind) supportsAuthzCheck() bool {
|
||||
return r == entityKindPanel || r == entityKindDashboard || r == entityKindFolder
|
||||
}
|
||||
|
||||
var (
|
||||
permissionFilterFields = []string{documentFieldUID, documentFieldKind}
|
||||
panelIdFieldRegex = regexp.MustCompile(`^(.*)#([0-9]{1,4})$`)
|
||||
panelIdFieldDashboardUidSubmatchIndex = 1
|
||||
panelIdFieldPanelIdSubmatchIndex = 2
|
||||
panelIdFieldRegexExpectedSubmatchCount = 3 // submatches[0] - whole string
|
||||
|
||||
_ bluge.Query = (*PermissionFilter)(nil)
|
||||
)
|
||||
|
||||
func newPermissionFilter(resourceFilter ResourceFilter, log log.Logger) *PermissionFilter {
|
||||
return &PermissionFilter{
|
||||
filter: resourceFilter,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
func (q *PermissionFilter) logAccessDecision(decision bool, kind interface{}, id string, reason string, ctx ...interface{}) {
|
||||
if true {
|
||||
return // TOO much logging right now
|
||||
}
|
||||
|
||||
ctx = append(ctx, "kind", kind, "id", id, "reason", reason)
|
||||
if decision {
|
||||
q.log.Debug("allowing access", ctx...)
|
||||
} else {
|
||||
q.log.Info("denying access", ctx...)
|
||||
}
|
||||
}
|
||||
|
||||
func (q *PermissionFilter) canAccess(kind entityKind, id string) bool {
|
||||
if !kind.supportsAuthzCheck() {
|
||||
q.logAccessDecision(false, kind, id, "entityDoesNotSupportAuthz")
|
||||
return false
|
||||
}
|
||||
|
||||
// TODO add `kind` to the `ResourceFilter` interface so that we can move the switch out of here
|
||||
//
|
||||
switch kind {
|
||||
case entityKindFolder:
|
||||
if id == "" {
|
||||
q.logAccessDecision(true, kind, id, "generalFolder")
|
||||
return true
|
||||
}
|
||||
fallthrough
|
||||
case entityKindDashboard:
|
||||
decision := q.filter(id)
|
||||
q.logAccessDecision(decision, kind, id, "resourceFilter")
|
||||
return decision
|
||||
case entityKindPanel:
|
||||
matches := panelIdFieldRegex.FindStringSubmatch(id)
|
||||
|
||||
submatchCount := len(matches)
|
||||
if submatchCount != panelIdFieldRegexExpectedSubmatchCount {
|
||||
q.logAccessDecision(false, kind, id, "invalidPanelIdFieldRegexSubmatchCount", "submatchCount", submatchCount, "expectedSubmatchCount", panelIdFieldRegexExpectedSubmatchCount)
|
||||
return false
|
||||
}
|
||||
|
||||
dashboardUid := matches[panelIdFieldDashboardUidSubmatchIndex]
|
||||
decision := q.filter(dashboardUid)
|
||||
|
||||
q.logAccessDecision(decision, kind, id, "resourceFilter", "dashboardUid", dashboardUid, "panelId", matches[panelIdFieldPanelIdSubmatchIndex])
|
||||
return decision
|
||||
default:
|
||||
q.logAccessDecision(false, kind, id, "reason", "unknownKind")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (q *PermissionFilter) Searcher(i search.Reader, options search.SearcherOptions) (search.Searcher, error) {
|
||||
dvReader, err := i.DocumentValueReader(permissionFilterFields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s, err := searcher.NewMatchAllSearcher(i, 1, similarity.ConstantScorer(1), options)
|
||||
return searcher.NewFilteringSearcher(s, func(d *search.DocumentMatch) bool {
|
||||
var kind, id string
|
||||
err := dvReader.VisitDocumentValues(d.Number, func(field string, term []byte) {
|
||||
if field == documentFieldKind {
|
||||
kind = string(term)
|
||||
} else if field == documentFieldUID {
|
||||
id = string(term)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
q.logAccessDecision(false, kind, id, "errorWhenVisitingDocumentValues")
|
||||
return false
|
||||
}
|
||||
|
||||
e := entityKind(kind)
|
||||
if !e.IsValid() {
|
||||
q.logAccessDecision(false, kind, id, "invalidEntityKind")
|
||||
return false
|
||||
}
|
||||
|
||||
return q.canAccess(e, id)
|
||||
}), err
|
||||
}
|
||||
@ -13,6 +13,8 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/searchV2/extract"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
"github.com/grafana/grafana/pkg/services/store"
|
||||
|
||||
"github.com/blugelabs/bluge"
|
||||
)
|
||||
|
||||
type dashboardLoader interface {
|
||||
@ -32,7 +34,8 @@ type eventStore interface {
|
||||
type dashboardIndex struct {
|
||||
mu sync.RWMutex
|
||||
loader dashboardLoader
|
||||
dashboards map[int64][]dashboard // orgId -> []dashboards
|
||||
dashboards map[int64][]dashboard // orgId -> []dashboards
|
||||
reader map[int64]*bluge.Reader // orgId -> bluge index
|
||||
eventStore eventStore
|
||||
logger log.Logger
|
||||
}
|
||||
@ -53,6 +56,7 @@ func newDashboardIndex(dashLoader dashboardLoader, evStore eventStore) *dashboar
|
||||
loader: dashLoader,
|
||||
eventStore: evStore,
|
||||
dashboards: map[int64][]dashboard{},
|
||||
reader: map[int64]*bluge.Reader{},
|
||||
logger: log.New("dashboardIndex"),
|
||||
}
|
||||
}
|
||||
@ -81,6 +85,9 @@ func (i *dashboardIndex) run(ctx context.Context) error {
|
||||
}
|
||||
i.logger.Info("Indexing for main org finished", "mainOrgIndexElapsed", time.Since(started), "numDashboards", len(dashboards))
|
||||
|
||||
// build bluge index on startup (will catch panics)
|
||||
go i.reIndexFromScratchBluge(ctx)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-partialUpdateTicker.C:
|
||||
@ -120,6 +127,60 @@ func (i *dashboardIndex) reIndexFromScratch(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Variation of the above function that builds bluge index from scratch
|
||||
// Once the frontend is wired up, we should switch to this one
|
||||
func (i *dashboardIndex) reIndexFromScratchBluge(ctx context.Context) {
|
||||
// Catch Panic (just in case)
|
||||
defer func() {
|
||||
recv := recover()
|
||||
if recv != nil {
|
||||
i.logger.Error("panic in search runner", "recv", recv) // REMVOE after we are sure it works!
|
||||
}
|
||||
}()
|
||||
|
||||
i.mu.RLock()
|
||||
orgIDs := make([]int64, 0, len(i.dashboards))
|
||||
for orgID := range i.dashboards {
|
||||
orgIDs = append(orgIDs, orgID)
|
||||
}
|
||||
i.mu.RUnlock()
|
||||
if len(orgIDs) < 1 {
|
||||
orgIDs = append(orgIDs, int64(1)) // make sure we index
|
||||
}
|
||||
|
||||
for _, orgID := range orgIDs {
|
||||
started := time.Now()
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
||||
|
||||
dashboards, err := i.loader.LoadDashboards(ctx, orgID, "")
|
||||
if err != nil {
|
||||
cancel()
|
||||
i.logger.Error("Error re-indexing dashboards for organization", "orgId", orgID, "error", err)
|
||||
continue
|
||||
}
|
||||
orgSearchIndexLoadTime := time.Since(started)
|
||||
|
||||
reader, err := initBlugeIndex(dashboards, i.logger)
|
||||
if err != nil {
|
||||
cancel()
|
||||
i.logger.Error("Error re-indexing dashboards for organization", "orgId", orgID, "error", err)
|
||||
continue
|
||||
}
|
||||
orgSearchIndexTotalTime := time.Since(started)
|
||||
orgSearchIndexBuildTime := orgSearchIndexTotalTime - orgSearchIndexLoadTime
|
||||
|
||||
cancel()
|
||||
i.logger.Info("Re-indexed dashboards for organization (bluge)",
|
||||
"orgId", orgID,
|
||||
"orgSearchIndexLoadTime", orgSearchIndexLoadTime,
|
||||
"orgSearchIndexBuildTime", orgSearchIndexBuildTime,
|
||||
"orgSearchIndexTotalTime", orgSearchIndexTotalTime)
|
||||
i.mu.Lock()
|
||||
i.reader[orgID] = reader
|
||||
i.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (i *dashboardIndex) applyIndexUpdates(ctx context.Context, lastEventID int64) int64 {
|
||||
events, err := i.eventStore.GetAllEventsAfter(context.Background(), lastEventID)
|
||||
if err != nil {
|
||||
|
||||
@ -105,8 +105,24 @@ func (s *StandardSearchService) getUser(ctx context.Context, backendUser *backen
|
||||
return user, nil
|
||||
}
|
||||
|
||||
func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, _ DashboardQuery) *backend.DataResponse {
|
||||
func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, q DashboardQuery) *backend.DataResponse {
|
||||
rsp := &backend.DataResponse{}
|
||||
signedInUser, err := s.getUser(ctx, user, orgId)
|
||||
if err != nil {
|
||||
rsp.Error = err
|
||||
return rsp
|
||||
}
|
||||
|
||||
filter, err := s.auth.GetDashboardReadFilter(signedInUser)
|
||||
if err != nil {
|
||||
rsp.Error = err
|
||||
return rsp
|
||||
}
|
||||
|
||||
reader := s.dashboardIndex.reader[orgId]
|
||||
if reader != nil && q.Query != "" { // frontend initializes with empty string
|
||||
return doBlugeQuery(ctx, s, reader, filter, q)
|
||||
}
|
||||
|
||||
dashboards, err := s.dashboardIndex.getDashboards(ctx, orgId)
|
||||
if err != nil {
|
||||
@ -114,29 +130,14 @@ func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *back
|
||||
return rsp
|
||||
}
|
||||
|
||||
signedInUser, err := s.getUser(ctx, user, orgId)
|
||||
if err != nil {
|
||||
rsp.Error = err
|
||||
return rsp
|
||||
}
|
||||
|
||||
dashboards, err = s.applyAuthFilter(signedInUser, dashboards)
|
||||
if err != nil {
|
||||
rsp.Error = err
|
||||
return rsp
|
||||
}
|
||||
dashboards = s.applyAuthFilter(filter, dashboards)
|
||||
|
||||
rsp.Frames = metaToFrame(dashboards)
|
||||
|
||||
return rsp
|
||||
}
|
||||
|
||||
func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dashboards []dashboard) ([]dashboard, error) {
|
||||
filter, err := s.auth.GetDashboardReadFilter(user)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (s *StandardSearchService) applyAuthFilter(filter ResourceFilter, dashboards []dashboard) []dashboard {
|
||||
// create a list of all viewable dashboards for this user.
|
||||
res := make([]dashboard, 0, len(dashboards))
|
||||
for _, dash := range dashboards {
|
||||
@ -144,7 +145,7 @@ func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dashb
|
||||
res = append(res, dash)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
return res
|
||||
}
|
||||
|
||||
type simpleCounter struct {
|
||||
|
||||
@ -8,8 +8,27 @@ import (
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend"
|
||||
)
|
||||
|
||||
type FacetField struct {
|
||||
Field string `json:"field"`
|
||||
Limit int `json:"limit,omitempty"` // explicit page size
|
||||
}
|
||||
|
||||
type DashboardQuery struct {
|
||||
Query string
|
||||
Query string `json:"query"`
|
||||
Location string `json:"location,omitempty"` // parent folder ID
|
||||
Sort string `json:"sort,omitempty"` // field ASC/DESC
|
||||
Datasource string `json:"ds_uid,omitempty"` // "datasource" collides with the JSON value at the same leel :()
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Kind []string `json:"kind,omitempty"`
|
||||
UIDs []string `json:"uid,omitempty"`
|
||||
IDs []int64 `json:"id,omitempty"` // deprecated -- but will convert internal ID to UIDs
|
||||
Explain bool `json:"explain,omitempty"` // adds details on why document matched
|
||||
Facet []FacetField `json:"facet,omitempty"`
|
||||
SkipLocation bool `json:"skipLocation,omitempty"`
|
||||
AccessInfo bool `json:"accessInfo,omitempty"` // adds field for access control
|
||||
HasPreview string `json:"hasPreview,omitempty"` // the light|dark theme
|
||||
Limit int `json:"limit,omitempty"` // explicit page size
|
||||
From int `json:"from,omitempty"` // for paging
|
||||
}
|
||||
|
||||
type SearchService interface {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user