IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

Reorganize file util functionality (#7848)

Browse Source 
* reorganize file util functionality

* fix api test compilation

* fix rebase issue
This commit is contained in:
Chris
2017-11-16 15:04:27 -06:00
committed by Jonathan
parent ef69d93abf
commit eb1a00ef5f
22 changed files with 649 additions and 619 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
api/emoji.go
View File

@@ -16,7 +16,6 @@ import (
"github.com/disintegration/imaging"
"github.com/gorilla/mux"
"github.com/mattermost/mattermost-server/app"
"github.com/mattermost/mattermost-server/einterfaces"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/utils"
)
@@ -51,7 +50,7 @@ func createEmoji(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if emojiInterface := einterfaces.GetEmojiInterface(); emojiInterface != nil &&
if emojiInterface := c.App.Emoji; emojiInterface != nil &&
!emojiInterface.CanUserCreateEmoji(c.Session.Roles, c.Session.TeamMembers) {
c.Err = model.NewAppError("createEmoji", "api.emoji.create.permissions.app_error", nil, "user_id="+c.Session.UserId, http.StatusUnauthorized)
return
@@ -106,7 +105,7 @@ func createEmoji(c *Context, w http.ResponseWriter, r *http.Request) {
if imageData := m.File["image"]; len(imageData) == 0 {
c.SetInvalidParam("createEmoji", "image")
return
} else if err := app.UploadEmojiImage(emoji.Id, imageData[0]); err != nil {
} else if err := c.App.UploadEmojiImage(emoji.Id, imageData[0]); err != nil {
c.Err = err
return
}

2
api/emoji_test.go
View File

@@ -272,7 +272,7 @@ func TestDeleteEmoji(t *testing.T) {
func createTestEmoji(t *testing.T, a *app.App, emoji *model.Emoji, imageData []byte) *model.Emoji {
emoji = store.Must(a.Srv.Store.Emoji().Save(emoji)).(*model.Emoji)
if err := utils.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil {
if err := a.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil {
store.Must(a.Srv.Store.Emoji().Delete(emoji.Id, time.Now().Unix()))
t.Fatalf("failed to write image: %v", err.Error())
}

10
api/file.go
View File

@@ -95,7 +95,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if data, err := utils.ReadFile(info.Path); err != nil {
if data, err := c.App.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {
@@ -116,7 +116,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if data, err := utils.ReadFile(info.ThumbnailPath); err != nil {
if data, err := c.App.ReadFile(info.ThumbnailPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, w, r); err != nil {
@@ -137,7 +137,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if data, err := utils.ReadFile(info.PreviewPath); err != nil {
if data, err := c.App.ReadFile(info.PreviewPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, w, r); err != nil {
@@ -186,7 +186,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if data, err := utils.ReadFile(info.Path); err != nil {
if data, err := c.App.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {
@@ -277,7 +277,7 @@ func getPublicFileOld(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if data, err := utils.ReadFile(info.Path); err != nil {
if data, err := c.App.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {

2
api/file_test.go
View File

@@ -849,7 +849,7 @@ func TestGetInfoForFilename(t *testing.T) {
date := time.Now().Format("20060102")
if info := app.GetInfoForFilename(post1, team1.Id, post1.Filenames[0]); info == nil {
if info := th.App.GetInfoForFilename(post1, team1.Id, post1.Filenames[0]); info == nil {
t.Fatal("info shouldn't be nil")
} else if info.Id == "" {
t.Fatal("info.Id shouldn't be empty")

3
api4/emoji.go
View File

@@ -10,7 +10,6 @@ import (
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/mattermost-server/app"
"github.com/mattermost/mattermost-server/einterfaces"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/utils"
)
@@ -31,7 +30,7 @@ func createEmoji(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if emojiInterface := einterfaces.GetEmojiInterface(); emojiInterface != nil &&
if emojiInterface := c.App.Emoji; emojiInterface != nil &&
!emojiInterface.CanUserCreateEmoji(c.Session.Roles, c.Session.TeamMembers) {
c.Err = model.NewAppError("getEmoji", "api.emoji.disabled.app_error", nil, "user_id="+c.Session.UserId, http.StatusUnauthorized)
return

8
api4/file.go
View File

@@ -123,7 +123,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
data, err := utils.ReadFile(info.Path)
data, err := c.App.ReadFile(info.Path)
if err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
@@ -164,7 +164,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if data, err := utils.ReadFile(info.ThumbnailPath); err != nil {
if data, err := c.App.ReadFile(info.ThumbnailPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, forceDownload, w, r); err != nil {
@@ -233,7 +233,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if data, err := utils.ReadFile(info.PreviewPath); err != nil {
if data, err := c.App.ReadFile(info.PreviewPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, forceDownload, w, r); err != nil {
@@ -294,7 +294,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
if data, err := utils.ReadFile(info.Path); err != nil {
if data, err := c.App.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, true, w, r); err != nil {

30
app/app.go
View File

@@ -43,6 +43,7 @@ type App struct {
Compliance einterfaces.ComplianceInterface
DataRetention einterfaces.DataRetentionInterface
Elasticsearch einterfaces.ElasticsearchInterface
Emoji einterfaces.EmojiInterface
Ldap einterfaces.LdapInterface
Metrics einterfaces.MetricsInterface
Mfa einterfaces.MfaInterface
@@ -133,6 +134,12 @@ func RegisterAccountMigrationInterface(f func(*App) einterfaces.AccountMigration
accountMigrationInterface = f
}
var brandInterface func(*App) einterfaces.BrandInterface
func RegisterBrandInterface(f func(*App) einterfaces.BrandInterface) {
brandInterface = f
}
var clusterInterface func(*App) einterfaces.ClusterInterface
func RegisterClusterInterface(f func(*App) einterfaces.ClusterInterface) {
@@ -151,6 +158,18 @@ func RegisterDataRetentionInterface(f func(*App) einterfaces.DataRetentionInterf
dataRetentionInterface = f
}
var elasticsearchInterface func(*App) einterfaces.ElasticsearchInterface
func RegisterElasticsearchInterface(f func(*App) einterfaces.ElasticsearchInterface) {
elasticsearchInterface = f
}
var emojiInterface func(*App) einterfaces.EmojiInterface
func RegisterEmojiInterface(f func(*App) einterfaces.EmojiInterface) {
emojiInterface = f
}
var jobsDataRetentionJobInterface func(*App) ejobs.DataRetentionJobInterface
func RegisterJobsDataRetentionJobInterface(f func(*App) ejobs.DataRetentionJobInterface) {
@@ -203,14 +222,21 @@ func (a *App) initEnterprise() {
if accountMigrationInterface != nil {
a.AccountMigration = accountMigrationInterface(a)
}
a.Brand = einterfaces.GetBrandInterface()
if brandInterface != nil {
a.Brand = brandInterface(a)
}
if clusterInterface != nil {
a.Cluster = clusterInterface(a)
}
if complianceInterface != nil {
a.Compliance = complianceInterface(a)
}
a.Elasticsearch = einterfaces.GetElasticsearchInterface()
if elasticsearchInterface != nil {
a.Elasticsearch = elasticsearchInterface(a)
}
if emojiInterface != nil {
a.Emoji = emojiInterface(a)
}
if ldapInterface != nil {
a.Ldap = ldapInterface(a)
utils.AddConfigListener(func(_, cfg *model.Config) {

22
app/emoji.go
View File

@@ -51,7 +51,7 @@ func (a *App) CreateEmoji(sessionUserId string, emoji *model.Emoji, multiPartIma
if imageData := multiPartImageData.File["image"]; len(imageData) == 0 {
err := model.NewAppError("Context", "api.context.invalid_body_param.app_error", map[string]interface{}{"Name": "createEmoji"}, "", http.StatusBadRequest)
return nil, err
} else if err := UploadEmojiImage(emoji.Id, imageData[0]); err != nil {
} else if err := a.UploadEmojiImage(emoji.Id, imageData[0]); err != nil {
return nil, err
}
@@ -74,7 +74,7 @@ func (a *App) GetEmojiList(page, perPage int) ([]*model.Emoji, *model.AppError)
}
}
func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppError {
func (a *App) UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppError {
file, err := imageData.Open()
if err != nil {
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.open.app_error", nil, "", http.StatusBadRequest)
@@ -100,7 +100,7 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro
if err := gif.EncodeAll(newbuf, resized_gif); err != nil {
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.gif_encode_error", nil, "", http.StatusBadRequest)
}
if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
if err := a.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
@@ -112,18 +112,14 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro
if err := png.Encode(newbuf, resized_image); err != nil {
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.encode_error", nil, "", http.StatusBadRequest)
}
if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
if err := a.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
}
} else {
if err := utils.WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
return nil
return a.WriteFile(buf.Bytes(), getEmojiImagePath(id))
}
func (a *App) DeleteEmoji(emoji *model.Emoji) *model.AppError {
@@ -131,7 +127,7 @@ func (a *App) DeleteEmoji(emoji *model.Emoji) *model.AppError {
return err
}
deleteEmojiImage(emoji.Id)
a.deleteEmojiImage(emoji.Id)
a.deleteReactionsForEmoji(emoji.Name)
return nil
}
@@ -158,7 +154,7 @@ func (a *App) GetEmojiImage(emojiId string) (imageByte []byte, imageType string,
} else {
var img []byte
if data, err := utils.ReadFile(getEmojiImagePath(emojiId)); err != nil {
if data, err := a.ReadFile(getEmojiImagePath(emojiId)); err != nil {
return nil, "", model.NewAppError("getEmojiImage", "api.emoji.get_image.read.app_error", nil, err.Error(), http.StatusNotFound)
} else {
img = data
@@ -217,8 +213,8 @@ func imageToPaletted(img image.Image) *image.Paletted {
return pm
}
func deleteEmojiImage(id string) {
if err := utils.MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil {
func (a *App) deleteEmojiImage(id string) {
if err := a.MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil {
l4g.Error("Failed to rename image when deleting emoji %v", id)
}
}

62
app/file.go
View File

@@ -57,7 +57,43 @@ const (
IMAGE_PREVIEW_PIXEL_WIDTH = 1024
)
func GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo {
func (a *App) FileBackend() (utils.FileBackend, *model.AppError) {
return utils.NewFileBackend(&a.Config().FileSettings)
}
func (a *App) ReadFile(path string) ([]byte, *model.AppError) {
backend, err := a.FileBackend()
if err != nil {
return nil, err
}
return backend.ReadFile(path)
}
func (a *App) MoveFile(oldPath, newPath string) *model.AppError {
backend, err := a.FileBackend()
if err != nil {
return err
}
return backend.MoveFile(oldPath, newPath)
}
func (a *App) WriteFile(f []byte, path string) *model.AppError {
backend, err := a.FileBackend()
if err != nil {
return err
}
return backend.WriteFile(f, path)
}
func (a *App) RemoveFile(path string) *model.AppError {
backend, err := a.FileBackend()
if err != nil {
return err
}
return backend.RemoveFile(path)
}
func (a *App) GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo {
// Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension}
split := strings.SplitN(filename, "/", 5)
if len(split) < 5 {
@@ -79,7 +115,7 @@ func GetInfoForFilename(post *model.Post, teamId string, filename string) *model
// Open the file and populate the fields of the FileInfo
var info *model.FileInfo
if data, err := utils.ReadFile(path); err != nil {
if data, err := a.ReadFile(path); err != nil {
l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.file_not_found.error"), post.Id, filename, path, err)
return nil
} else {
@@ -121,7 +157,7 @@ func (a *App) FindTeamIdForFilename(post *model.Post, filename string) string {
} else {
for _, team := range teams {
path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name)
if _, err := utils.ReadFile(path); err == nil {
if _, err := a.ReadFile(path); err == nil {
// Found the team that this file was posted from
return team.Id
}
@@ -168,7 +204,7 @@ func (a *App) MigrateFilenamesToFileInfos(post *model.Post) []*model.FileInfo {
l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.team_id.error"), post.Id, filenames)
} else {
for _, filename := range filenames {
info := GetInfoForFilename(post, teamId, filename)
info := a.GetInfoForFilename(post, teamId, filename)
if info == nil {
continue
}
@@ -286,7 +322,7 @@ func (a *App) UploadFiles(teamId string, channelId string, userId string, fileHe
}
}
HandleImages(previewPathList, thumbnailPathList, imageDataList)
a.HandleImages(previewPathList, thumbnailPathList, imageDataList)
return resStruct, nil
}
@@ -321,7 +357,7 @@ func (a *App) DoUploadFile(now time.Time, rawTeamId string, rawChannelId string,
info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
}
if err := utils.WriteFile(data, info.Path); err != nil {
if err := a.WriteFile(data, info.Path); err != nil {
return nil, err
}
@@ -332,7 +368,7 @@ func (a *App) DoUploadFile(now time.Time, rawTeamId string, rawChannelId string,
return info, nil
}
func HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) {
func (a *App) HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) {
wg := new(sync.WaitGroup)
for i := range fileData {
@@ -341,12 +377,12 @@ func HandleImages(previewPathList []string, thumbnailPathList []string, fileData
wg.Add(2)
go func(img *image.Image, path string, width int, height int) {
defer wg.Done()
generateThumbnailImage(*img, path, width, height)
a.generateThumbnailImage(*img, path, width, height)
}(img, thumbnailPathList[i], width, height)
go func(img *image.Image, path string, width int) {
defer wg.Done()
generatePreviewImage(*img, path, width)
a.generatePreviewImage(*img, path, width)
}(img, previewPathList[i], width)
}
}
@@ -417,7 +453,7 @@ func getImageOrientation(input io.Reader) (int, error) {
}
}
func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) {
func (a *App) generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) {
thumbWidth := float64(IMAGE_THUMBNAIL_PIXEL_WIDTH)
thumbHeight := float64(IMAGE_THUMBNAIL_PIXEL_HEIGHT)
imgWidth := float64(width)
@@ -438,13 +474,13 @@ func generateThumbnailImage(img image.Image, thumbnailPath string, width int, he
return
}
if err := utils.WriteFile(buf.Bytes(), thumbnailPath); err != nil {
if err := a.WriteFile(buf.Bytes(), thumbnailPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
return
}
}
func generatePreviewImage(img image.Image, previewPath string, width int) {
func (a *App) generatePreviewImage(img image.Image, previewPath string, width int) {
var preview image.Image
if width > IMAGE_PREVIEW_PIXEL_WIDTH {
@@ -460,7 +496,7 @@ func generatePreviewImage(img image.Image, previewPath string, width int) {
return
}
if err := utils.WriteFile(buf.Bytes(), previewPath); err != nil {
if err := a.WriteFile(buf.Bytes(), previewPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
return
}

9
app/file_test.go
View File

@@ -9,7 +9,6 @@ import (
"time"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/utils"
)
func TestGeneratePublicLinkHash(t *testing.T) {
@@ -51,7 +50,7 @@ func TestDoUploadFile(t *testing.T) {
} else {
defer func() {
<-th.App.Srv.Store.FileInfo().PermanentDelete(info1.Id)
utils.RemoveFile(info1.Path)
th.App.RemoveFile(info1.Path)
}()
}
@@ -65,7 +64,7 @@ func TestDoUploadFile(t *testing.T) {
} else {
defer func() {
<-th.App.Srv.Store.FileInfo().PermanentDelete(info2.Id)
utils.RemoveFile(info2.Path)
th.App.RemoveFile(info2.Path)
}()
}
@@ -79,7 +78,7 @@ func TestDoUploadFile(t *testing.T) {
} else {
defer func() {
<-th.App.Srv.Store.FileInfo().PermanentDelete(info3.Id)
utils.RemoveFile(info3.Path)
th.App.RemoveFile(info3.Path)
}()
}
@@ -93,7 +92,7 @@ func TestDoUploadFile(t *testing.T) {
} else {
defer func() {
<-th.App.Srv.Store.FileInfo().PermanentDelete(info3.Id)
utils.RemoveFile(info3.Path)
th.App.RemoveFile(info3.Path)
}()
}

4
app/import.go
View File

@@ -1497,8 +1497,8 @@ func (a *App) OldImportFile(timestamp time.Time, file io.Reader, teamId string,
img, width, height := prepareImage(data)
if img != nil {
generateThumbnailImage(*img, fileInfo.ThumbnailPath, width, height)
generatePreviewImage(*img, fileInfo.PreviewPath, width)
a.generateThumbnailImage(*img, fileInfo.ThumbnailPath, width, height)
a.generatePreviewImage(*img, fileInfo.PreviewPath, width)
}
return fileInfo, nil

6
app/user.go
View File

@@ -751,7 +751,7 @@ func (a *App) GetProfileImage(user *model.User) ([]byte, bool, *model.AppError)
} else {
path := "users/" + user.Id + "/profile.png"
if data, err := utils.ReadFile(path); err != nil {
if data, err := a.ReadFile(path); err != nil {
readFailed = true
if img, err = CreateProfileImage(user.Username, user.Id, a.Config().FileSettings.InitialFont); err != nil {
@@ -759,7 +759,7 @@ func (a *App) GetProfileImage(user *model.User) ([]byte, bool, *model.AppError)
}
if user.LastPictureUpdate == 0 {
if err := utils.WriteFile(img, path); err != nil {
if err := a.WriteFile(img, path); err != nil {
return nil, false, err
}
}
@@ -812,7 +812,7 @@ func (a *App) SetProfileImage(userId string, imageData *multipart.FileHeader) *m
path := "users/" + userId + "/profile.png"
if err := utils.WriteFile(buf.Bytes(), path); err != nil {
if err := a.WriteFile(buf.Bytes(), path); err != nil {
return model.NewAppError("SetProfileImage", "api.user.upload_profile_user.upload_profile.app_error", nil, "", http.StatusInternalServerError)
}

12
cmd/platform/server.go
View File

@@ -60,13 +60,17 @@ func runServer(configFileLocation string) {
l4g.Info(utils.T("mattermost.working_dir"), pwd)
l4g.Info(utils.T("mattermost.config_file"), utils.FindConfigFile(configFileLocation))
if err := utils.TestFileConnection(); err != nil {
l4g.Error("Problem with file storage settings: " + err.Error())
}
a := app.New(app.ConfigFile(configFileLocation))
defer a.Shutdown()
backend, err := a.FileBackend()
if err == nil {
err = backend.TestConnection()
}
if err != nil {
l4g.Error("Problem with file storage settings: " + err.Error())
}
if model.BuildEnterpriseReady == "true" {
a.LoadLicense()
}

10
einterfaces/brand.go
View File

@@ -12,13 +12,3 @@ type BrandInterface interface {
SaveBrandImage(*multipart.FileHeader) *model.AppError
GetBrandImage() ([]byte, *model.AppError)
}
var theBrandInterface BrandInterface
func RegisterBrandInterface(newInterface BrandInterface) {
theBrandInterface = newInterface
}
func GetBrandInterface() BrandInterface {
return theBrandInterface
}

10
einterfaces/elasticsearch.go
View File

@@ -18,13 +18,3 @@ type ElasticsearchInterface interface {
PurgeIndexes() *model.AppError
DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError
}
var theElasticsearchInterface ElasticsearchInterface
func RegisterElasticsearchInterface(newInterface ElasticsearchInterface) {
theElasticsearchInterface = newInterface
}
func GetElasticsearchInterface() ElasticsearchInterface {
return theElasticsearchInterface
}

10
einterfaces/emoji.go
View File

@@ -10,13 +10,3 @@ import (
type EmojiInterface interface {
CanUserCreateEmoji(string, []*model.TeamMember) bool
}
var theEmojiInterface EmojiInterface
func RegisterEmojiInterface(newInterface EmojiInterface) {
theEmojiInterface = newInterface
}
func GetEmojiInterface() EmojiInterface {
return theEmojiInterface
}

359
utils/file.go
View File

@@ -4,372 +4,13 @@
package utils
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
l4g "github.com/alecthomas/log4go"
s3 "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/credentials"
"github.com/mattermost/mattermost-server/model"
)
const (
TEST_FILE_PATH = "/testfile"
)
// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
// If signV2 input is false, function always returns signature v4.
//
// Additionally this function also takes a user defined region, if set
// disables automatic region lookup.
func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) {
var creds *credentials.Credentials
if signV2 {
creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2)
} else {
creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4)
}
s3Clnt, err := s3.NewWithCredentials(endpoint, creds, secure, region)
if err != nil {
return nil, err
}
if *Cfg.FileSettings.AmazonS3Trace {
s3Clnt.TraceOn(os.Stdout)
}
return s3Clnt, nil
}
func TestFileConnection() *model.AppError {
if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := Cfg.FileSettings.AmazonS3Endpoint
accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *Cfg.FileSettings.AmazonS3SSL
signV2 := *Cfg.FileSettings.AmazonS3SignV2
region := Cfg.FileSettings.AmazonS3Region
bucket := Cfg.FileSettings.AmazonS3Bucket
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return model.NewAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error(), http.StatusInternalServerError)
}
exists, err := s3Clnt.BucketExists(bucket)
if err != nil {
return model.NewAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error(), http.StatusInternalServerError)
}
if !exists {
l4g.Warn("Bucket specified does not exist. Attempting to create...")
err := s3Clnt.MakeBucket(bucket, region)
if err != nil {
l4g.Error("Unable to create bucket.")
return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError)
}
}
l4g.Info("Connection to S3 or minio is good. Bucket exists.")
} else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
f := []byte("testingwrite")
if err := writeFileLocally(f, Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil {
return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError)
}
os.Remove(Cfg.FileSettings.Directory + TEST_FILE_PATH)
l4g.Info("Able to write files to local storage.")
} else {
return model.NewAppError("TestFileConnection", "No file driver selected.", nil, "", http.StatusInternalServerError)
}
return nil
}
func ReadFile(path string) ([]byte, *model.AppError) {
if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := Cfg.FileSettings.AmazonS3Endpoint
accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *Cfg.FileSettings.AmazonS3SSL
signV2 := *Cfg.FileSettings.AmazonS3SignV2
region := Cfg.FileSettings.AmazonS3Region
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
bucket := Cfg.FileSettings.AmazonS3Bucket
minioObject, err := s3Clnt.GetObject(bucket, path)
if err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
defer minioObject.Close()
if f, err := ioutil.ReadAll(minioObject); err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
return f, nil
}
} else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if f, err := ioutil.ReadFile(Cfg.FileSettings.Directory + path); err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
return f, nil
}
} else {
return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented)
}
}
func MoveFile(oldPath, newPath string) *model.AppError {
if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := Cfg.FileSettings.AmazonS3Endpoint
accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *Cfg.FileSettings.AmazonS3SSL
signV2 := *Cfg.FileSettings.AmazonS3SignV2
region := Cfg.FileSettings.AmazonS3Region
encrypt := false
if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance {
encrypt = true
}
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
bucket := Cfg.FileSettings.AmazonS3Bucket
source := s3.NewSourceInfo(bucket, oldPath, nil)
destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt))
if err != nil {
return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if err = s3Clnt.CopyObject(destination, source); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
} else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := os.MkdirAll(filepath.Dir(Cfg.FileSettings.Directory+newPath), 0774); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if err := os.Rename(Cfg.FileSettings.Directory+oldPath, Cfg.FileSettings.Directory+newPath); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError)
}
} else {
return model.NewAppError("moveFile", "api.file.move_file.configured.app_error", nil, "", http.StatusNotImplemented)
}
return nil
}
func WriteFile(f []byte, path string) *model.AppError {
if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := Cfg.FileSettings.AmazonS3Endpoint
accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *Cfg.FileSettings.AmazonS3SSL
signV2 := *Cfg.FileSettings.AmazonS3SignV2
region := Cfg.FileSettings.AmazonS3Region
encrypt := false
if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance {
encrypt = true
}
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
bucket := Cfg.FileSettings.AmazonS3Bucket
ext := filepath.Ext(path)
metaData := S3Metadata(encrypt, "binary/octet-stream")
if model.IsFileExtImage(ext) {
metaData = S3Metadata(encrypt, model.GetImageMimeType(ext))
}
_, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil)
if err != nil {
return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
} else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := writeFileLocally(f, Cfg.FileSettings.Directory+path); err != nil {
return err
}
} else {
return model.NewAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "", http.StatusNotImplemented)
}
return nil
}
func writeFileLocally(f []byte, path string) *model.AppError {
if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil {
directory, _ := filepath.Abs(filepath.Dir(path))
return model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError)
}
if err := ioutil.WriteFile(path, f, 0644); err != nil {
return model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func RemoveFile(path string) *model.AppError {
if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := Cfg.FileSettings.AmazonS3Endpoint
accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *Cfg.FileSettings.AmazonS3SSL
signV2 := *Cfg.FileSettings.AmazonS3SignV2
region := Cfg.FileSettings.AmazonS3Region
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
bucket := Cfg.FileSettings.AmazonS3Bucket
if err := s3Clnt.RemoveObject(bucket, path); err != nil {
return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
} else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := os.Remove(Cfg.FileSettings.Directory + path); err != nil {
return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError)
}
} else {
return model.NewAppError("RemoveFile", "utils.file.remove_file.configured.app_error", nil, "", http.StatusNotImplemented)
}
return nil
}
func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string {
out := make(chan string, 1)
go func() {
defer close(out)
for {
info, done := <-in
if !done {
break
}
out <- info.Key
}
}()
return out
}
// Returns a list of all the directories within the path directory provided.
func ListDirectory(path string) (*[]string, *model.AppError) {
var paths []string
if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := Cfg.FileSettings.AmazonS3Endpoint
accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *Cfg.FileSettings.AmazonS3SSL
signV2 := *Cfg.FileSettings.AmazonS3SignV2
region := Cfg.FileSettings.AmazonS3Region
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
doneCh := make(chan struct{})
defer close(doneCh)
bucket := Cfg.FileSettings.AmazonS3Bucket
for object := range s3Clnt.ListObjects(bucket, path, false, doneCh) {
if object.Err != nil {
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError)
}
paths = append(paths, strings.Trim(object.Key, "/"))
}
} else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if fileInfos, err := ioutil.ReadDir(Cfg.FileSettings.Directory + path); err != nil {
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
for _, fileInfo := range fileInfos {
if fileInfo.IsDir() {
paths = append(paths, filepath.Join(path, fileInfo.Name()))
}
}
}
} else {
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.configured.app_error", nil, "", http.StatusInternalServerError)
}
return &paths, nil
}
func RemoveDirectory(path string) *model.AppError {
if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := Cfg.FileSettings.AmazonS3Endpoint
accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *Cfg.FileSettings.AmazonS3SSL
signV2 := *Cfg.FileSettings.AmazonS3SignV2
region := Cfg.FileSettings.AmazonS3Region
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
doneCh := make(chan struct{})
bucket := Cfg.FileSettings.AmazonS3Bucket
for err := range s3Clnt.RemoveObjects(bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(bucket, path, true, doneCh))) {
if err.Err != nil {
doneCh <- struct{}{}
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError)
}
}
close(doneCh)
} else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := os.RemoveAll(Cfg.FileSettings.Directory + path); err != nil {
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError)
}
} else {
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.configured.app_error", nil, "", http.StatusNotImplemented)
}
return nil
}
func S3Metadata(encrypt bool, contentType string) map[string][]string {
metaData := make(map[string][]string)
if contentType != "" {
metaData["Content-Type"] = []string{"contentType"}
}
if encrypt {
metaData["x-amz-server-side-encryption"] = []string{"AES256"}
}
return metaData
}
func CopyMetadata(encrypt bool) map[string]string {
metaData := make(map[string]string)
metaData["x-amz-server-side-encryption"] = "AES256"
return metaData
}
// CopyFile will copy a file from src path to dst path.
// Overwrites any existing files at dst.
// Permissions are copied from file at src to the new file at dst.

44
utils/file_backend.go Normal file
View File

@@ -0,0 +1,44 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package utils
import (
"net/http"
"github.com/mattermost/mattermost-server/model"
)
type FileBackend interface {
TestConnection() *model.AppError
ReadFile(path string) ([]byte, *model.AppError)
MoveFile(oldPath, newPath string) *model.AppError
WriteFile(f []byte, path string) *model.AppError
RemoveFile(path string) *model.AppError
ListDirectory(path string) (*[]string, *model.AppError)
RemoveDirectory(path string) *model.AppError
}
func NewFileBackend(settings *model.FileSettings) (FileBackend, *model.AppError) {
switch *settings.DriverName {
case model.IMAGE_DRIVER_S3:
return &S3FileBackend{
endpoint: settings.AmazonS3Endpoint,
accessKey: settings.AmazonS3AccessKeyId,
secretKey: settings.AmazonS3SecretAccessKey,
secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL,
signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2,
region: settings.AmazonS3Region,
bucket: settings.AmazonS3Bucket,
encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance,
trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace,
}, nil
case model.IMAGE_DRIVER_LOCAL:
return &LocalFileBackend{
directory: settings.Directory,
}, nil
}
return nil, model.NewAppError("NewFileBackend", "No file driver selected.", nil, "", http.StatusInternalServerError)
}

98
utils/file_backend_local.go Normal file
View File

@@ -0,0 +1,98 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package utils
import (
"io/ioutil"
"net/http"
"os"
"path/filepath"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/mattermost-server/model"
)
const (
TEST_FILE_PATH = "/testfile"
)
type LocalFileBackend struct {
directory string
}
func (b *LocalFileBackend) TestConnection() *model.AppError {
f := []byte("testingwrite")
if err := writeFileLocally(f, filepath.Join(b.directory, TEST_FILE_PATH)); err != nil {
return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError)
}
os.Remove(filepath.Join(b.directory, TEST_FILE_PATH))
l4g.Info("Able to write files to local storage.")
return nil
}
func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) {
if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
return f, nil
}
}
func (b *LocalFileBackend) MoveFile(oldPath, newPath string) *model.AppError {
if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0774); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (b *LocalFileBackend) WriteFile(f []byte, path string) *model.AppError {
return writeFileLocally(f, filepath.Join(b.directory, path))
}
func writeFileLocally(f []byte, path string) *model.AppError {
if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil {
directory, _ := filepath.Abs(filepath.Dir(path))
return model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError)
}
if err := ioutil.WriteFile(path, f, 0644); err != nil {
return model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (b *LocalFileBackend) RemoveFile(path string) *model.AppError {
if err := os.Remove(filepath.Join(b.directory, path)); err != nil {
return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (b *LocalFileBackend) ListDirectory(path string) (*[]string, *model.AppError) {
var paths []string
if fileInfos, err := ioutil.ReadDir(filepath.Join(b.directory, path)); err != nil {
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
for _, fileInfo := range fileInfos {
if fileInfo.IsDir() {
paths = append(paths, filepath.Join(path, fileInfo.Name()))
}
}
}
return &paths, nil
}
func (b *LocalFileBackend) RemoveDirectory(path string) *model.AppError {
if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil {
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}

226
utils/file_backend_s3.go Normal file
View File

@@ -0,0 +1,226 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package utils
import (
"bytes"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
l4g "github.com/alecthomas/log4go"
s3 "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/credentials"
"github.com/mattermost/mattermost-server/model"
)
type S3FileBackend struct {
endpoint string
accessKey string
secretKey string
secure bool
signV2 bool
region string
bucket string
encrypt bool
trace bool
}
// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
// If signV2 input is false, function always returns signature v4.
//
// Additionally this function also takes a user defined region, if set
// disables automatic region lookup.
func (b *S3FileBackend) s3New() (*s3.Client, error) {
var creds *credentials.Credentials
if b.signV2 {
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2)
} else {
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4)
}
s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region)
if err != nil {
return nil, err
}
if b.trace {
s3Clnt.TraceOn(os.Stdout)
}
return s3Clnt, nil
}
func (b *S3FileBackend) TestConnection() *model.AppError {
s3Clnt, err := b.s3New()
if err != nil {
return model.NewAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error(), http.StatusInternalServerError)
}
exists, err := s3Clnt.BucketExists(b.bucket)
if err != nil {
return model.NewAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error(), http.StatusInternalServerError)
}
if !exists {
l4g.Warn("Bucket specified does not exist. Attempting to create...")
err := s3Clnt.MakeBucket(b.bucket, b.region)
if err != nil {
l4g.Error("Unable to create bucket.")
return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError)
}
}
l4g.Info("Connection to S3 or minio is good. Bucket exists.")
return nil
}
func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) {
s3Clnt, err := b.s3New()
if err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
minioObject, err := s3Clnt.GetObject(b.bucket, path)
if err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
defer minioObject.Close()
if f, err := ioutil.ReadAll(minioObject); err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
return f, nil
}
}
func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError {
s3Clnt, err := b.s3New()
if err != nil {
return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
source := s3.NewSourceInfo(b.bucket, oldPath, nil)
destination, err := s3.NewDestinationInfo(b.bucket, newPath, nil, s3CopyMetadata(b.encrypt))
if err != nil {
return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if err = s3Clnt.CopyObject(destination, source); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (b *S3FileBackend) WriteFile(f []byte, path string) *model.AppError {
s3Clnt, err := b.s3New()
if err != nil {
return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
ext := filepath.Ext(path)
metaData := s3Metadata(b.encrypt, "binary/octet-stream")
if model.IsFileExtImage(ext) {
metaData = s3Metadata(b.encrypt, model.GetImageMimeType(ext))
}
if _, err = s3Clnt.PutObjectWithMetadata(b.bucket, path, bytes.NewReader(f), metaData, nil); err != nil {
return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (b *S3FileBackend) RemoveFile(path string) *model.AppError {
s3Clnt, err := b.s3New()
if err != nil {
return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if err := s3Clnt.RemoveObject(b.bucket, path); err != nil {
return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string {
out := make(chan string, 1)
go func() {
defer close(out)
for {
info, done := <-in
if !done {
break
}
out <- info.Key
}
}()
return out
}
func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) {
var paths []string
s3Clnt, err := b.s3New()
if err != nil {
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
doneCh := make(chan struct{})
defer close(doneCh)
for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) {
if object.Err != nil {
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError)
}
paths = append(paths, strings.Trim(object.Key, "/"))
}
return &paths, nil
}
func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError {
s3Clnt, err := b.s3New()
if err != nil {
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
doneCh := make(chan struct{})
for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) {
if err.Err != nil {
doneCh <- struct{}{}
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError)
}
}
close(doneCh)
return nil
}
func s3Metadata(encrypt bool, contentType string) map[string][]string {
metaData := make(map[string][]string)
if contentType != "" {
metaData["Content-Type"] = []string{"contentType"}
}
if encrypt {
metaData["x-amz-server-side-encryption"] = []string{"AES256"}
}
return metaData
}
func s3CopyMetadata(encrypt bool) map[string]string {
metaData := make(map[string]string)
metaData["x-amz-server-side-encryption"] = "AES256"
return metaData
}

164
utils/file_backend_test.go Normal file
View File

@@ -0,0 +1,164 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package utils
import (
"fmt"
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/mattermost/mattermost-server/model"
)
type FileBackendTestSuite struct {
suite.Suite
settings model.FileSettings
backend FileBackend
}
func TestLocalFileBackendTestSuite(t *testing.T) {
dir, err := ioutil.TempDir("", "")
require.NoError(t, err)
defer os.RemoveAll(dir)
suite.Run(t, &FileBackendTestSuite{
settings: model.FileSettings{
DriverName: model.NewString(model.IMAGE_DRIVER_LOCAL),
Directory: dir,
},
})
}
func TestS3FileBackendTestSuite(t *testing.T) {
s3Host := os.Getenv("CI_HOST")
if s3Host == "" {
s3Host = "dockerhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9001"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
suite.Run(t, &FileBackendTestSuite{
settings: model.FileSettings{
DriverName: model.NewString(model.IMAGE_DRIVER_S3),
AmazonS3AccessKeyId: "minioaccesskey",
AmazonS3SecretAccessKey: "miniosecretkey",
AmazonS3Bucket: "mattermost-test",
AmazonS3Endpoint: s3Endpoint,
AmazonS3SSL: model.NewBool(false),
},
})
}
func (s *FileBackendTestSuite) SetupTest() {
TranslationsPreInit()
backend, err := NewFileBackend(&s.settings)
require.Nil(s.T(), err)
s.backend = backend
}
func (s *FileBackendTestSuite) TestConnection() {
s.Nil(s.backend.TestConnection())
}
func (s *FileBackendTestSuite) TestReadWriteFile() {
b := []byte("test")
path := "tests/" + model.NewId()
s.Nil(s.backend.WriteFile(b, path))
defer s.backend.RemoveFile(path)
read, err := s.backend.ReadFile(path)
s.Nil(err)
readString := string(read)
s.EqualValues(readString, "test")
}
func (s *FileBackendTestSuite) TestMoveFile() {
b := []byte("test")
path1 := "tests/" + model.NewId()
path2 := "tests/" + model.NewId()
s.Nil(s.backend.WriteFile(b, path1))
defer s.backend.RemoveFile(path1)
s.Nil(s.backend.MoveFile(path1, path2))
defer s.backend.RemoveFile(path2)
_, err := s.backend.ReadFile(path1)
s.Error(err)
_, err = s.backend.ReadFile(path2)
s.Nil(err)
}
func (s *FileBackendTestSuite) TestRemoveFile() {
b := []byte("test")
path := "tests/" + model.NewId()
s.Nil(s.backend.WriteFile(b, path))
s.Nil(s.backend.RemoveFile(path))
_, err := s.backend.ReadFile(path)
s.Error(err)
s.Nil(s.backend.WriteFile(b, "tests2/foo"))
s.Nil(s.backend.WriteFile(b, "tests2/bar"))
s.Nil(s.backend.WriteFile(b, "tests2/asdf"))
s.Nil(s.backend.RemoveDirectory("tests2"))
}
func (s *FileBackendTestSuite) TestListDirectory() {
b := []byte("test")
path1 := "19700101/" + model.NewId()
path2 := "19800101/" + model.NewId()
s.Nil(s.backend.WriteFile(b, path1))
defer s.backend.RemoveFile(path1)
s.Nil(s.backend.WriteFile(b, path2))
defer s.backend.RemoveFile(path2)
paths, err := s.backend.ListDirectory("")
s.Nil(err)
found1 := false
found2 := false
for _, path := range *paths {
if path == "19700101" {
found1 = true
} else if path == "19800101" {
found2 = true
}
}
s.True(found1)
s.True(found2)
}
func (s *FileBackendTestSuite) TestRemoveDirectory() {
b := []byte("test")
s.Nil(s.backend.WriteFile(b, "tests2/foo"))
s.Nil(s.backend.WriteFile(b, "tests2/bar"))
s.Nil(s.backend.WriteFile(b, "tests2/aaa"))
s.Nil(s.backend.RemoveDirectory("tests2"))
_, err := s.backend.ReadFile("tests2/foo")
s.Error(err)
_, err = s.backend.ReadFile("tests2/bar")
s.Error(err)
_, err = s.backend.ReadFile("tests2/asdf")
s.Error(err)
}

172
utils/file_test.go
View File

@@ -4,7 +4,6 @@
package utils
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
@@ -12,179 +11,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/mattermost/mattermost-server/model"
)
type FileTestSuite struct {
suite.Suite
testDriver string
// Config to be reset after tests.
driverName string
amazonS3AccessKeyId string
amazonS3SecretAccessKey string
amazonS3Bucket string
amazonS3Endpoint string
amazonS3SSL bool
}
func TestFileLocalTestSuite(t *testing.T) {
testsuite := FileTestSuite{
testDriver: model.IMAGE_DRIVER_LOCAL,
}
suite.Run(t, &testsuite)
}
func TestFileMinioTestSuite(t *testing.T) {
testsuite := FileTestSuite{
testDriver: model.IMAGE_DRIVER_S3,
}
suite.Run(t, &testsuite)
}
func (s *FileTestSuite) SetupTest() {
TranslationsPreInit()
LoadGlobalConfig("config.json")
InitTranslations(Cfg.LocalizationSettings)
// Save state to restore after the test has run.
s.driverName = *Cfg.FileSettings.DriverName
s.amazonS3AccessKeyId = Cfg.FileSettings.AmazonS3AccessKeyId
s.amazonS3SecretAccessKey = Cfg.FileSettings.AmazonS3SecretAccessKey
s.amazonS3Bucket = Cfg.FileSettings.AmazonS3Bucket
s.amazonS3Endpoint = Cfg.FileSettings.AmazonS3Endpoint
s.amazonS3SSL = *Cfg.FileSettings.AmazonS3SSL
// Set up the state for the tests.
s3Host := os.Getenv("CI_HOST")
if s3Host == "" {
s3Host = "dockerhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9001"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
if s.testDriver == model.IMAGE_DRIVER_LOCAL {
*Cfg.FileSettings.DriverName = model.IMAGE_DRIVER_LOCAL
} else if s.testDriver == model.IMAGE_DRIVER_S3 {
*Cfg.FileSettings.DriverName = model.IMAGE_DRIVER_S3
Cfg.FileSettings.AmazonS3AccessKeyId = "minioaccesskey"
Cfg.FileSettings.AmazonS3SecretAccessKey = "miniosecretkey"
Cfg.FileSettings.AmazonS3Bucket = "mattermost-test"
Cfg.FileSettings.AmazonS3Endpoint = s3Endpoint
*Cfg.FileSettings.AmazonS3SSL = false
} else {
s.T().Fatal("Invalid image driver set for test suite.")
}
}
func (s *FileTestSuite) TearDownTest() {
// Restore the test state.
*Cfg.FileSettings.DriverName = s.driverName
Cfg.FileSettings.AmazonS3AccessKeyId = s.amazonS3AccessKeyId
Cfg.FileSettings.AmazonS3SecretAccessKey = s.amazonS3SecretAccessKey
Cfg.FileSettings.AmazonS3Bucket = s.amazonS3Bucket
Cfg.FileSettings.AmazonS3Endpoint = s.amazonS3Endpoint
*Cfg.FileSettings.AmazonS3SSL = s.amazonS3SSL
}
func (s *FileTestSuite) TestReadWriteFile() {
b := []byte("test")
path := "tests/" + model.NewId()
s.Nil(WriteFile(b, path))
defer RemoveFile(path)
read, err := ReadFile(path)
s.Nil(err)
readString := string(read)
s.EqualValues(readString, "test")
}
func (s *FileTestSuite) TestMoveFile() {
b := []byte("test")
path1 := "tests/" + model.NewId()
path2 := "tests/" + model.NewId()
s.Nil(WriteFile(b, path1))
defer RemoveFile(path1)
s.Nil(MoveFile(path1, path2))
defer RemoveFile(path2)
_, err := ReadFile(path1)
s.Error(err)
_, err = ReadFile(path2)
s.Nil(err)
}
func (s *FileTestSuite) TestRemoveFile() {
b := []byte("test")
path := "tests/" + model.NewId()
s.Nil(WriteFile(b, path))
s.Nil(RemoveFile(path))
_, err := ReadFile(path)
s.Error(err)
s.Nil(WriteFile(b, "tests2/foo"))
s.Nil(WriteFile(b, "tests2/bar"))
s.Nil(WriteFile(b, "tests2/asdf"))
s.Nil(RemoveDirectory("tests2"))
}
func (s *FileTestSuite) TestListDirectory() {
b := []byte("test")
path1 := "19700101/" + model.NewId()
path2 := "19800101/" + model.NewId()
s.Nil(WriteFile(b, path1))
defer RemoveFile(path1)
s.Nil(WriteFile(b, path2))
defer RemoveFile(path2)
paths, err := ListDirectory("")
s.Nil(err)
found1 := false
found2 := false
for _, path := range *paths {
if path == "19700101" {
found1 = true
} else if path == "19800101" {
found2 = true
}
}
s.True(found1)
s.True(found2)
}
func (s *FileTestSuite) TestRemoveDirectory() {
b := []byte("test")
s.Nil(WriteFile(b, "tests2/foo"))
s.Nil(WriteFile(b, "tests2/bar"))
s.Nil(WriteFile(b, "tests2/aaa"))
s.Nil(RemoveDirectory("tests2"))
_, err := ReadFile("tests2/foo")
s.Error(err)
_, err = ReadFile("tests2/bar")
s.Error(err)
_, err = ReadFile("tests2/asdf")
s.Error(err)
}
func TestCopyDir(t *testing.T) {
srcDir, err := ioutil.TempDir("", "src")
require.NoError(t, err)