[MM-31247] Add support for compressed export files with attachments (#16614)

* Include filepaths for post attachments * Cleanup * Enable exporting file attachments * Fix file import * Enable zip export * Support creating missing directories when unzipping * Add test * Add translations * Export direct channel posts attachments * Fix returned values order Remove pointer to slice in return * [MM-31597] Implement export process job (#16626) * Implement export process job * Add translations * Remove unused value * [MM-31249] Add /exports API endpoint (#16633) * Implement API endpoints to list, download and delete export files * Add endpoint for single resource * Update i18n/en.json Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> * Update i18n/en.json Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> * Fix var name Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>
2025-02-25 18:55:24 -06:00 · 2021-02-09 11:58:31 +01:00
parent 9a33c3706a
commit 572f861675
27 changed files with 1063 additions and 106 deletions
--- a/api4/api.go
+++ b/api4/api.go
@@ -125,6 +125,8 @@ type Routes struct {
 	Cloud *mux.Router // 'api/v4/cloud'

 	Imports *mux.Router // 'api/v4/imports'
+	Exports *mux.Router // 'api/v4/exports'
+	Export  *mux.Router // 'api/v4/exports/{export_name:.+\\.zip}'
 }

 type API struct {
@@ -238,6 +240,8 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
 	api.BaseRoutes.Cloud = api.BaseRoutes.ApiRoot.PathPrefix("/cloud").Subrouter()

 	api.BaseRoutes.Imports = api.BaseRoutes.ApiRoot.PathPrefix("/imports").Subrouter()
+	api.BaseRoutes.Exports = api.BaseRoutes.ApiRoot.PathPrefix("/exports").Subrouter()
+	api.BaseRoutes.Export = api.BaseRoutes.Exports.PathPrefix("/{export_name:.+\\.zip}").Subrouter()

 	api.InitUser()
 	api.InitBot()
@@ -276,6 +280,7 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
 	api.InitAction()
 	api.InitCloud()
 	api.InitImport()
+	api.InitExport()

 	root.Handle("/api/v4/{anything:.*}", http.HandlerFunc(api.Handle404))

@@ -344,6 +349,8 @@ func InitLocal(configservice configservice.ConfigService, globalOptionsFunc app.
 	api.BaseRoutes.Upload = api.BaseRoutes.Uploads.PathPrefix("/{upload_id:[A-Za-z0-9]+}").Subrouter()

 	api.BaseRoutes.Imports = api.BaseRoutes.ApiRoot.PathPrefix("/imports").Subrouter()
+	api.BaseRoutes.Exports = api.BaseRoutes.ApiRoot.PathPrefix("/exports").Subrouter()
+	api.BaseRoutes.Export = api.BaseRoutes.Exports.PathPrefix("/{export_name:.+\\.zip}").Subrouter()

 	api.BaseRoutes.Jobs = api.BaseRoutes.ApiRoot.PathPrefix("/jobs").Subrouter()

@@ -363,6 +370,7 @@ func InitLocal(configservice configservice.ConfigService, globalOptionsFunc app.
 	api.InitRoleLocal()
 	api.InitUploadLocal()
 	api.InitImportLocal()
+	api.InitExportLocal()
 	api.InitJobLocal()

 	root.Handle("/api/v4/{anything:.*}", http.HandlerFunc(api.Handle404))
--- a/api4/export.go
+++ b/api4/export.go
@@ -0,0 +1,86 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+import (
+	"encoding/json"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/audit"
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+func (api *API) InitExport() {
+	api.BaseRoutes.Exports.Handle("", api.ApiSessionRequired(listExports)).Methods("GET")
+	api.BaseRoutes.Export.Handle("", api.ApiSessionRequired(deleteExport)).Methods("DELETE")
+	api.BaseRoutes.Export.Handle("", api.ApiSessionRequired(downloadExport)).Methods("GET")
+}
+
+func listExports(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.IsSystemAdmin() {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	exports, appErr := c.App.ListExports()
+	if appErr != nil {
+		c.Err = appErr
+		return
+	}
+
+	data, err := json.Marshal(exports)
+	if err != nil {
+		c.Err = model.NewAppError("listImports", "app.export.marshal.app_error", nil, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Write(data)
+}
+
+func deleteExport(c *Context, w http.ResponseWriter, r *http.Request) {
+	auditRec := c.MakeAuditRecord("deleteExport", audit.Fail)
+	defer c.LogAuditRec(auditRec)
+	auditRec.AddMeta("export_name", c.Params.ExportName)
+
+	if !c.IsSystemAdmin() {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	if err := c.App.DeleteExport(c.Params.ExportName); err != nil {
+		c.Err = err
+		return
+	}
+
+	auditRec.Success()
+	ReturnStatusOK(w)
+}
+
+func downloadExport(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.IsSystemAdmin() {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	filePath := filepath.Join(*c.App.Config().ExportSettings.Directory, c.Params.ExportName)
+	if ok, err := c.App.FileExists(filePath); err != nil {
+		c.Err = err
+		return
+	} else if !ok {
+		c.Err = model.NewAppError("downloadExport", "api.export.export_not_found.app_error", nil, "", http.StatusNotFound)
+		return
+	}
+
+	file, err := c.App.FileReader(filePath)
+	if err != nil {
+		c.Err = err
+		return
+	}
+	defer file.Close()
+
+	w.Header().Set("Content-Type", "application/zip")
+	http.ServeContent(w, r, c.Params.ExportName, time.Time{}, file)
+}
--- a/api4/export_local.go
+++ b/api4/export_local.go
@@ -0,0 +1,10 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+func (api *API) InitExportLocal() {
+	api.BaseRoutes.Exports.Handle("", api.ApiLocal(listExports)).Methods("GET")
+	api.BaseRoutes.Export.Handle("", api.ApiLocal(deleteExport)).Methods("DELETE")
+	api.BaseRoutes.Export.Handle("", api.ApiLocal(downloadExport)).Methods("GET")
+}
--- a/api4/export_test.go
+++ b/api4/export_test.go
@@ -0,0 +1,213 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestListExports(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	t.Run("no permissions", func(t *testing.T) {
+		exports, resp := th.Client.ListExports()
+		require.Error(t, resp.Error)
+		require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
+		require.Nil(t, exports)
+	})
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		exports, resp := c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Empty(t, exports)
+	}, "no exports")
+
+	dataDir, found := fileutils.FindDir("data")
+	require.True(t, found)
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory)
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+
+		f, err := os.Create(filepath.Join(exportDir, "export.zip"))
+		require.Nil(t, err)
+		f.Close()
+
+		exports, resp := c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Len(t, exports, 1)
+		require.Equal(t, exports[0], "export.zip")
+	}, "expected exports")
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		value := *th.App.Config().ExportSettings.Directory
+		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ExportSettings.Directory = value + "new" })
+		defer th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ExportSettings.Directory = value })
+
+		exportDir := filepath.Join(dataDir, value+"new")
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+
+		exports, resp := c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Empty(t, exports)
+
+		f, err := os.Create(filepath.Join(exportDir, "export.zip"))
+		require.Nil(t, err)
+		f.Close()
+
+		exports, resp = c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Len(t, exports, 1)
+		require.Equal(t, "export.zip", exports[0])
+	}, "change export directory")
+}
+
+func TestDeleteExport(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	t.Run("no permissions", func(t *testing.T) {
+		ok, resp := th.Client.DeleteExport("export.zip")
+		require.Error(t, resp.Error)
+		require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
+		require.False(t, ok)
+	})
+
+	dataDir, found := fileutils.FindDir("data")
+	require.True(t, found)
+	exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory)
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+		exportName := "export.zip"
+		f, err := os.Create(filepath.Join(exportDir, exportName))
+		require.Nil(t, err)
+		f.Close()
+
+		exports, resp := c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Len(t, exports, 1)
+		require.Equal(t, exports[0], exportName)
+
+		ok, resp := c.DeleteExport(exportName)
+		require.Nil(t, resp.Error)
+		require.True(t, ok)
+
+		exports, resp = c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Empty(t, exports)
+
+		// verify idempotence
+		ok, resp = c.DeleteExport(exportName)
+		require.Nil(t, resp.Error)
+		require.True(t, ok)
+	}, "successfully delete export")
+}
+
+func TestDownloadExport(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	t.Run("no permissions", func(t *testing.T) {
+		var buf bytes.Buffer
+		n, resp := th.Client.DownloadExport("export.zip", &buf, 0)
+		require.Error(t, resp.Error)
+		require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
+		require.Zero(t, n)
+	})
+
+	dataDir, found := fileutils.FindDir("data")
+	require.True(t, found)
+	exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory)
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		var buf bytes.Buffer
+		n, resp := c.DownloadExport("export.zip", &buf, 0)
+		require.Error(t, resp.Error)
+		require.Equal(t, "api.export.export_not_found.app_error", resp.Error.Id)
+		require.Zero(t, n)
+	}, "not found")
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+
+		data := randomBytes(t, 1024*1024)
+		var buf bytes.Buffer
+		exportName := "export.zip"
+		err = ioutil.WriteFile(filepath.Join(exportDir, exportName), data, 0600)
+		require.Nil(t, err)
+
+		n, resp := c.DownloadExport(exportName, &buf, 0)
+		require.Nil(t, resp.Error)
+		require.Equal(t, len(data), int(n))
+		require.Equal(t, data, buf.Bytes())
+	}, "full download")
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+
+		data := randomBytes(t, 1024*1024)
+		var buf bytes.Buffer
+		exportName := "export.zip"
+		err = ioutil.WriteFile(filepath.Join(exportDir, exportName), data, 0600)
+		require.Nil(t, err)
+
+		offset := 1024 * 512
+		n, resp := c.DownloadExport(exportName, &buf, int64(offset))
+		require.Nil(t, resp.Error)
+		require.Equal(t, len(data)-offset, int(n))
+		require.Equal(t, data[offset:], buf.Bytes())
+	}, "download with offset")
+}
+
+func BenchmarkDownloadExport(b *testing.B) {
+	th := Setup(b)
+	defer th.TearDown()
+
+	dataDir, found := fileutils.FindDir("data")
+	require.True(b, found)
+	exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory)
+
+	err := os.Mkdir(exportDir, 0700)
+	require.Nil(b, err)
+	defer os.RemoveAll(exportDir)
+
+	exportName := "export.zip"
+	f, err := os.Create(filepath.Join(exportDir, exportName))
+	require.Nil(b, err)
+	f.Close()
+
+	err = os.Truncate(filepath.Join(exportDir, exportName), 1024*1024*1024)
+	require.Nil(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		outFilePath := filepath.Join(dataDir, fmt.Sprintf("export%d.zip", i))
+		outFile, _ := os.Create(outFilePath)
+		th.SystemAdminClient.DownloadExport(exportName, outFile, 0)
+		outFile.Close()
+		os.Remove(outFilePath)
+	}
+}