grafana/pkg/services/provisioning/dashboards/file_reader.go

package dashboards

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/infra/metrics"
	"github.com/grafana/grafana/pkg/services/accesscontrol"
	"github.com/grafana/grafana/pkg/services/dashboards"
	"github.com/grafana/grafana/pkg/services/folder"
	"github.com/grafana/grafana/pkg/services/provisioning/utils"
	"github.com/grafana/grafana/pkg/util"
)

var (
	// ErrFolderNameMissing is returned when folder name is missing.
	ErrFolderNameMissing = errors.New("folder name missing")
)

// FileReader is responsible for reading dashboards from disk and
// insert/update dashboards to the Grafana database using
// `dashboards.DashboardProvisioningService`.
type FileReader struct {
	Cfg                          *config
	Path                         string
	log                          log.Logger
	dashboardProvisioningService dashboards.DashboardProvisioningService
	dashboardStore               utils.DashboardStore
	FoldersFromFilesStructure    bool
	folderService                folder.Service

	mux                     sync.RWMutex
	usageTracker            *usageTracker
	dbWriteAccessRestricted bool
}

// NewDashboardFileReader returns a new filereader based on `config`
func NewDashboardFileReader(cfg *config, log log.Logger, service dashboards.DashboardProvisioningService,
	dashboardStore utils.DashboardStore, folderService folder.Service) (*FileReader, error) {
	var path string
	path, ok := cfg.Options["path"].(string)
	if !ok {
		path, ok = cfg.Options["folder"].(string)
		if !ok {
			return nil, fmt.Errorf("failed to load dashboards, path param is not a string")
		}

		log.Warn("[Deprecated] The folder property is deprecated. Please use path instead.")
	}

	foldersFromFilesStructure, _ := cfg.Options["foldersFromFilesStructure"].(bool)
	if foldersFromFilesStructure && cfg.Folder != "" && cfg.FolderUID != "" {
		return nil, fmt.Errorf("'folder' and 'folderUID' should be empty using 'foldersFromFilesStructure' option")
	}

	return &FileReader{
		Cfg:                          cfg,
		Path:                         path,
		log:                          log,
		dashboardProvisioningService: service,
		dashboardStore:               dashboardStore,
		folderService:                folderService,
		FoldersFromFilesStructure:    foldersFromFilesStructure,
		usageTracker:                 newUsageTracker(),
	}, nil
}

// pollChanges periodically runs walkDisk based on interval specified in the config.
func (fr *FileReader) pollChanges(ctx context.Context) {
	ticker := time.NewTicker(time.Duration(int64(time.Second) * fr.Cfg.UpdateIntervalSeconds))
	for {
		select {
		case <-ticker.C:
			if err := fr.walkDisk(ctx); err != nil {
				fr.log.Error("failed to search for dashboards", "error", err)
			}
		case <-ctx.Done():
			return
		}
	}
}

// walkDisk traverses the file system for the defined path, reading dashboard definition files,
// and applies any change to the database.
func (fr *FileReader) walkDisk(ctx context.Context) error {
	fr.log.Debug("Start walking disk", "path", fr.Path)
	resolvedPath := fr.resolvedPath()
	if _, err := os.Stat(resolvedPath); err != nil {
		return err
	}

	provisionedDashboardRefs, err := getProvisionedDashboardsByPath(ctx, fr.dashboardProvisioningService, fr.Cfg.Name)
	if err != nil {
		return err
	}

	// Find relevant files
	filesFoundOnDisk := map[string]os.FileInfo{}
	if err := filepath.Walk(resolvedPath, createWalkFn(filesFoundOnDisk)); err != nil {
		return err
	}

	fr.handleMissingDashboardFiles(ctx, provisionedDashboardRefs, filesFoundOnDisk)

	usageTracker := newUsageTracker()
	if fr.FoldersFromFilesStructure {
		err = fr.storeDashboardsInFoldersFromFileStructure(ctx, filesFoundOnDisk, provisionedDashboardRefs, resolvedPath, usageTracker)
	} else {
		err = fr.storeDashboardsInFolder(ctx, filesFoundOnDisk, provisionedDashboardRefs, usageTracker)
	}
	if err != nil {
		return err
	}

	fr.mux.Lock()
	defer fr.mux.Unlock()

	fr.usageTracker = usageTracker
	return nil
}

func (fr *FileReader) changeWritePermissions(restrict bool) {
	fr.mux.Lock()
	defer fr.mux.Unlock()

	fr.dbWriteAccessRestricted = restrict
}

func (fr *FileReader) isDatabaseAccessRestricted() bool {
	fr.mux.RLock()
	defer fr.mux.RUnlock()

	return fr.dbWriteAccessRestricted
}

// storeDashboardsInFolder saves dashboards from the filesystem on disk to the folder from config
func (fr *FileReader) storeDashboardsInFolder(ctx context.Context, filesFoundOnDisk map[string]os.FileInfo,
	dashboardRefs map[string]*dashboards.DashboardProvisioning, usageTracker *usageTracker) error {
	folderID, folderUID, err := fr.getOrCreateFolder(ctx, fr.Cfg, fr.dashboardProvisioningService, fr.Cfg.Folder)
	if err != nil && !errors.Is(err, ErrFolderNameMissing) {
		return err
	}

	// save dashboards based on json files
	for path, fileInfo := range filesFoundOnDisk {
		provisioningMetadata, err := fr.saveDashboard(ctx, path, folderID, folderUID, fileInfo, dashboardRefs)
		if err != nil {
			fr.log.Error("failed to save dashboard", "file", path, "error", err)
			continue
		}

		usageTracker.track(provisioningMetadata)
	}
	return nil
}

// storeDashboardsInFoldersFromFilesystemStructure saves dashboards from the filesystem on disk to the same folder
// in Grafana as they are in on the filesystem.
func (fr *FileReader) storeDashboardsInFoldersFromFileStructure(ctx context.Context, filesFoundOnDisk map[string]os.FileInfo,
	dashboardRefs map[string]*dashboards.DashboardProvisioning, resolvedPath string, usageTracker *usageTracker) error {
	for path, fileInfo := range filesFoundOnDisk {
		folderName := ""

		dashboardsFolder := filepath.Dir(path)
		if dashboardsFolder != resolvedPath {
			folderName = filepath.Base(dashboardsFolder)
		}

		folderID, folderUID, err := fr.getOrCreateFolder(ctx, fr.Cfg, fr.dashboardProvisioningService, folderName)
		if err != nil && !errors.Is(err, ErrFolderNameMissing) {
			return fmt.Errorf("can't provision folder %q from file system structure: %w", folderName, err)
		}

		provisioningMetadata, err := fr.saveDashboard(ctx, path, folderID, folderUID, fileInfo, dashboardRefs)
		usageTracker.track(provisioningMetadata)
		if err != nil {
			fr.log.Error("failed to save dashboard", "file", path, "error", err)
		}
	}
	return nil
}

// handleMissingDashboardFiles will unprovision or delete dashboards which are missing on disk.
func (fr *FileReader) handleMissingDashboardFiles(ctx context.Context, provisionedDashboardRefs map[string]*dashboards.DashboardProvisioning,
	filesFoundOnDisk map[string]os.FileInfo) {
	// find dashboards to delete since json file is missing
	var dashboardsToDelete []int64
	for path, provisioningData := range provisionedDashboardRefs {
		_, existsOnDisk := filesFoundOnDisk[path]
		if !existsOnDisk {
			dashboardsToDelete = append(dashboardsToDelete, provisioningData.DashboardID)
		}
	}

	if fr.Cfg.DisableDeletion {
		// If deletion is disabled for the provisioner we just remove provisioning metadata about the dashboard
		// so afterwards the dashboard is considered unprovisioned.
		for _, dashboardID := range dashboardsToDelete {
			fr.log.Debug("unprovisioning provisioned dashboard. missing on disk", "id", dashboardID)
			err := fr.dashboardProvisioningService.UnprovisionDashboard(ctx, dashboardID)
			if err != nil {
				fr.log.Error("failed to unprovision dashboard", "dashboard_id", dashboardID, "error", err)
			}
		}
	} else {
		// delete dashboards missing JSON file
		for _, dashboardID := range dashboardsToDelete {
			fr.log.Debug("deleting provisioned dashboard, missing on disk", "id", dashboardID)
			err := fr.dashboardProvisioningService.DeleteProvisionedDashboard(ctx, dashboardID, fr.Cfg.OrgID)
			if err != nil {
				fr.log.Error("failed to delete dashboard", "id", dashboardID, "error", err)
			}
		}
	}
}

// saveDashboard saves or updates the dashboard provisioning file at path.
func (fr *FileReader) saveDashboard(ctx context.Context, path string, folderID int64, folderUID string, fileInfo os.FileInfo,
	provisionedDashboardRefs map[string]*dashboards.DashboardProvisioning) (provisioningMetadata, error) {
	provisioningMetadata := provisioningMetadata{}
	resolvedFileInfo, err := resolveSymlink(fileInfo, path)
	if err != nil {
		return provisioningMetadata, err
	}

	provisionedData, alreadyProvisioned := provisionedDashboardRefs[path]

	jsonFile, err := fr.readDashboardFromFile(path, resolvedFileInfo.ModTime(), folderID, folderUID)
	if err != nil {
		fr.log.Error("failed to load dashboard from ", "file", path, "error", err)
		return provisioningMetadata, nil
	}

	upToDate := alreadyProvisioned
	if provisionedData != nil {
		upToDate = jsonFile.checkSum == provisionedData.CheckSum
	}

	// keeps track of which UIDs and titles we have already provisioned
	dash := jsonFile.dashboard
	provisioningMetadata.uid = dash.Dashboard.UID
	metrics.MFolderIDsServiceCount.WithLabelValues(metrics.Provisioning).Inc()
	// nolint:staticcheck
	provisioningMetadata.identity = dashboardIdentity{title: dash.Dashboard.Title, folderID: dash.Dashboard.FolderID}

	// fix empty folder_uid from already provisioned dashboards
	if upToDate && folderUID != "" {
		// search for root dashboard with the specified uid or title
		d, err := fr.dashboardStore.GetDashboard(
			ctx,
			&dashboards.GetDashboardQuery{
				OrgID:     jsonFile.dashboard.OrgID,
				UID:       jsonFile.dashboard.Dashboard.UID,
				Title:     &jsonFile.dashboard.Dashboard.Title,
				FolderUID: util.Pointer(""),
			},
		)
		if err != nil {
			// if no problematic entry is found it's safe to ignore
			if !errors.Is(err, dashboards.ErrDashboardNotFound) {
				return provisioningMetadata, err
			}
		} else {
			// inconsistency is detected so force updating the dashboard
			if d.FolderUID != folderUID {
				upToDate = false
			}
		}
	}

	if upToDate {
		metrics.MFolderIDsServiceCount.WithLabelValues(metrics.Provisioning).Inc()
		// nolint:staticcheck
		fr.log.Debug("provisioned dashboard is up to date", "provisioner", fr.Cfg.Name, "file", path, "folderId", dash.Dashboard.FolderID, "folderUid", dash.Dashboard.FolderUID)
		return provisioningMetadata, nil
	}

	if dash.Dashboard.ID != 0 {
		dash.Dashboard.Data.Set("id", nil)
		dash.Dashboard.ID = 0
	}

	if alreadyProvisioned {
		dash.Dashboard.SetID(provisionedData.DashboardID)
	}

	if !fr.isDatabaseAccessRestricted() {
		metrics.MFolderIDsServiceCount.WithLabelValues(metrics.Provisioning).Inc()
		// nolint:staticcheck
		fr.log.Debug("saving new dashboard", "provisioner", fr.Cfg.Name, "file", path, "folderId", dash.Dashboard.FolderID, "folderUid", dash.Dashboard.FolderUID)
		dp := &dashboards.DashboardProvisioning{
			ExternalID: path,
			Name:       fr.Cfg.Name,
			Updated:    resolvedFileInfo.ModTime().Unix(),
			CheckSum:   jsonFile.checkSum,
		}
		_, err := fr.dashboardProvisioningService.SaveProvisionedDashboard(ctx, dash, dp)
		if err != nil {
			return provisioningMetadata, err
		}
	} else {
		metrics.MFolderIDsServiceCount.WithLabelValues(metrics.Provisioning).Inc()
		// nolint:staticcheck
		fr.log.Warn("Not saving new dashboard due to restricted database access", "provisioner", fr.Cfg.Name,
			"file", path, "folderId", dash.Dashboard.FolderID)
	}

	return provisioningMetadata, nil
}

func getProvisionedDashboardsByPath(ctx context.Context, service dashboards.DashboardProvisioningService, name string) (
	map[string]*dashboards.DashboardProvisioning, error) {
	arr, err := service.GetProvisionedDashboardData(ctx, name)
	if err != nil {
		return nil, err
	}

	byPath := map[string]*dashboards.DashboardProvisioning{}
	for _, pd := range arr {
		byPath[pd.ExternalID] = pd
	}

	return byPath, nil
}

func (fr *FileReader) getOrCreateFolder(ctx context.Context, cfg *config, service dashboards.DashboardProvisioningService, folderName string) (int64, string, error) {
	if folderName == "" {
		return 0, "", ErrFolderNameMissing
	}

	// TODO use folder service instead
	metrics.MFolderIDsServiceCount.WithLabelValues(metrics.Provisioning).Inc()
	cmd := &dashboards.GetDashboardQuery{
		FolderID: util.Pointer(int64(0)), // nolint:staticcheck
		OrgID:    cfg.OrgID,
	}

	if cfg.FolderUID != "" {
		cmd.UID = cfg.FolderUID
	} else {
		cmd.Title = &folderName
	}

	result, err := fr.dashboardStore.GetDashboard(ctx, cmd)

	if err != nil && !errors.Is(err, dashboards.ErrDashboardNotFound) {
		return 0, "", err
	}

	// dashboard folder not found. create one.
	if errors.Is(err, dashboards.ErrDashboardNotFound) {
		// set dashboard folderUid if given
		if cfg.FolderUID == accesscontrol.GeneralFolderUID {
			return 0, "", dashboards.ErrFolderInvalidUID
		}

		createCmd := &folder.CreateFolderCommand{
			OrgID: cfg.OrgID,
			UID:   cfg.FolderUID,
			Title: folderName,
		}

		f, err := service.SaveFolderForProvisionedDashboards(ctx, createCmd)
		if err != nil {
			return 0, "", err
		}
		metrics.MFolderIDsServiceCount.WithLabelValues(metrics.Provisioning).Inc()
		// nolint:staticcheck
		return f.ID, f.UID, nil
	}

	if !result.IsFolder {
		return 0, "", fmt.Errorf("got invalid response. expected folder, found dashboard")
	}

	return result.ID, result.UID, nil
}

func resolveSymlink(fileinfo os.FileInfo, path string) (os.FileInfo, error) {
	checkFilepath, err := filepath.EvalSymlinks(path)
	if path != checkFilepath {
		fi, err := os.Lstat(checkFilepath)
		if err != nil {
			return nil, err
		}

		return fi, nil
	}

	return fileinfo, err
}

func createWalkFn(filesOnDisk map[string]os.FileInfo) filepath.WalkFunc {
	return func(path string, fileInfo os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		isValid, err := validateWalkablePath(fileInfo)
		if !isValid {
			return err
		}

		filesOnDisk[path] = fileInfo
		return nil
	}
}

func validateWalkablePath(fileInfo os.FileInfo) (bool, error) {
	if fileInfo.IsDir() {
		if strings.HasPrefix(fileInfo.Name(), ".") {
			return false, filepath.SkipDir
		}
		return false, nil
	}

	if !strings.HasSuffix(fileInfo.Name(), ".json") {
		return false, nil
	}

	return true, nil
}

type dashboardJSONFile struct {
	dashboard    *dashboards.SaveDashboardDTO
	checkSum     string
	lastModified time.Time
}

func (fr *FileReader) readDashboardFromFile(path string, lastModified time.Time, folderID int64, folderUID string) (*dashboardJSONFile, error) {
	// nolint:gosec
	// We can ignore the gosec G304 warning on this one because `path` comes from the provisioning configuration file.
	reader, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := reader.Close(); err != nil {
			fr.log.Warn("Failed to close file", "path", path, "err", err)
		}
	}()

	all, err := io.ReadAll(reader)
	if err != nil {
		return nil, err
	}

	checkSum, err := util.Md5SumString(string(all))
	if err != nil {
		return nil, err
	}

	data, err := simplejson.NewJson(all)
	if err != nil {
		return nil, err
	}

	dash, err := createDashboardJSON(data, lastModified, fr.Cfg, folderID, folderUID)
	if err != nil {
		return nil, err
	}

	return &dashboardJSONFile{
		dashboard:    dash,
		checkSum:     checkSum,
		lastModified: lastModified,
	}, nil
}

func (fr *FileReader) resolvedPath() string {
	if _, err := os.Stat(fr.Path); os.IsNotExist(err) {
		fr.log.Error("Cannot read directory", "error", err)
	}

	path, err := filepath.Abs(fr.Path)
	if err != nil {
		fr.log.Error("Could not create absolute path", "path", fr.Path, "error", err)
	}

	path, err = filepath.EvalSymlinks(path)
	if err != nil {
		fr.log.Error("Failed to read content of symlinked path", "path", fr.Path, "error", err)
	}

	if path == "" {
		path = fr.Path
		fr.log.Info("falling back to original path due to EvalSymlink/Abs failure")
	}
	return path
}

func (fr *FileReader) getUsageTracker() *usageTracker {
	fr.mux.RLock()
	defer fr.mux.RUnlock()

	return fr.usageTracker
}

type provisioningMetadata struct {
	uid      string
	identity dashboardIdentity
}

type dashboardIdentity struct {
	// Deprecated: use folderUID instead
	folderID  int64
	folderUID string
	title     string
}

func (d *dashboardIdentity) Exists() bool {
	return len(d.title) > 0
}

func newUsageTracker() *usageTracker {
	return &usageTracker{
		uidUsage:   map[string]uint8{},
		titleUsage: map[dashboardIdentity]uint8{},
	}
}

type usageTracker struct {
	uidUsage   map[string]uint8
	titleUsage map[dashboardIdentity]uint8
}

func (t *usageTracker) track(pm provisioningMetadata) {
	if len(pm.uid) > 0 {
		t.uidUsage[pm.uid]++
	}
	if pm.identity.Exists() {
		t.titleUsage[pm.identity]++
	}
}