IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Search: Delete old indexes once not in use (#99251)

Browse Source
This commit is contained in:
Ryan McKinley 2025-01-20 20:25:26 +03:00 committed by GitHub
parent 427a601980
commit 52c1a3dccd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 35 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/storage/unified/resource/search.go
View File

@ -490,7 +490,7 @@ func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size
} }
fields := s.builders.GetFields(nsr) fields := s.builders.GetFields(nsr)
s.log.Debug(fmt.Sprintf("TODO, build %+v (size:%d, rv:%d) // builder:%+v\n", nsr, size, rv, builder)) s.log.Debug("Building index", "resource", nsr.Resource, "size", size, "rv", rv)
key := &ResourceKey{ key := &ResourceKey{
Group: nsr.Group, Group: nsr.Group,

38
pkg/storage/unified/search/bleve.go
View File

@ -111,14 +111,14 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
mapper := getBleveMappings(fields) mapper := getBleveMappings(fields)
if size > b.opts.FileThreshold { if size > b.opts.FileThreshold {
resourceDir := filepath.Join(b.opts.Root, key.Namespace,
fmt.Sprintf("%s.%s", key.Resource, key.Group),
)
fname := fmt.Sprintf("rv%d", resourceVersion) fname := fmt.Sprintf("rv%d", resourceVersion)
if resourceVersion == 0 { if resourceVersion == 0 {
fname = b.start.Format("tmp-20060102-150405") fname = b.start.Format("tmp-20060102-150405")
} }
dir := filepath.Join(b.opts.Root, key.Namespace, dir := filepath.Join(resourceDir, fname)
fmt.Sprintf("%s.%s", key.Resource, key.Group),
fname,
)
if resourceVersion > 0 { if resourceVersion > 0 {
info, _ := os.Stat(dir) info, _ := os.Stat(dir)
if info != nil && info.IsDir() { if info != nil && info.IsDir() {
@ -128,6 +128,10 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
if err != nil || int64(found) != size { if err != nil || int64(found) != size {
b.log.Info("this size changed since the last time the index opened") b.log.Info("this size changed since the last time the index opened")
_ = index.Close() _ = index.Close()
// Pick a new file name
fname = b.start.Format("tmp-20060102-150405-changed")
dir = filepath.Join(resourceDir, fname)
index = nil index = nil
} else { } else {
build = false // no need to build the index build = false // no need to build the index
@ -138,8 +142,15 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
if index == nil { if index == nil {
index, err = bleve.New(dir, mapper) index, err = bleve.New(dir, mapper)
if err != nil {
err = fmt.Errorf("error creating new bleve index: %s %w", dir, err)
}
} }
// Start a background task to cleanup the old index directories
if index != nil && err == nil {
go b.cleanOldIndexes(resourceDir, fname)
}
resource.IndexMetrics.IndexTenants.WithLabelValues("file").Inc() resource.IndexMetrics.IndexTenants.WithLabelValues("file").Inc()
} else { } else {
index, err = bleve.NewMemOnly(mapper) index, err = bleve.NewMemOnly(mapper)
@ -183,6 +194,25 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
return idx, nil return idx, nil
} }
func (b *bleveBackend) cleanOldIndexes(dir string, skip string) {
files, err := os.ReadDir(dir)
if err != nil {
b.log.Warn("error cleaning folders from", "directory", dir, "error", err)
return
}
for _, file := range files {
if file.IsDir() && file.Name() != skip {
fpath := filepath.Join(dir, file.Name())
err = os.RemoveAll(fpath)
if err != nil {
b.log.Error("Unable to remove old index folder", "directory", fpath, "error", err)
} else {
b.log.Error("Removed old index folder", "directory", fpath)
}
}
}
}
// TotalDocs returns the total number of documents across all indices // TotalDocs returns the total number of documents across all indices
func (b *bleveBackend) TotalDocs() int64 { func (b *bleveBackend) TotalDocs() int64 {
var totalDocs int64 var totalDocs int64