IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

Move Slack Import to App Layer. (#5135)

Browse Source
This commit is contained in:
George Goldberg
2017-01-20 14:47:14 +00:00
committed by Christopher Speller
parent 6097f93704
commit 2de6c5394e
8 changed files with 215 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
api/file.go
View File

@@ -5,49 +5,20 @@ package api
import (
"bytes"
"image"
"image/color"
"image/draw"
_ "image/gif"
"image/jpeg"
"io"
"net/http"
"net/url"
"path/filepath"
"strconv"
"strings"
l4g "github.com/alecthomas/log4go"
"github.com/disintegration/imaging"
"github.com/gorilla/mux"
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
"github.com/rwcarlsen/goexif/exif"
_ "golang.org/x/image/bmp"
)
const (
/*
EXIF Image Orientations
1 2 3 4 5 6 7 8
888888 888888 88 88 8888888888 88 88 8888888888
88 88 88 88 88 88 88 88 88 88 88 88
8888 8888 8888 8888 88 8888888888 8888888888 88
88 88 88 88
88 88 888888 888888
*/
Upright = 1
UprightMirrored = 2
UpsideDown = 3
UpsideDownMirrored = 4
RotatedCWMirrored = 5
RotatedCCW = 6
RotatedCCWMirrored = 7
RotatedCW = 8
)
func InitFile() {
l4g.Debug(utils.T("api.file.init.debug"))
@@ -119,7 +90,7 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
io.Copy(buf, file)
data := buf.Bytes()
info, err := doUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data)
info, err := app.DoUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data)
if err != nil {
c.Err = err
return
@@ -138,169 +109,11 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
}
}
handleImages(previewPathList, thumbnailPathList, imageDataList)
app.HandleImages(previewPathList, thumbnailPathList, imageDataList)
w.Write([]byte(resStruct.ToJson()))
}
func doUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
filename := filepath.Base(rawFilename)
info, err := model.GetInfoForBytes(filename, data)
if err != nil {
err.StatusCode = http.StatusBadRequest
return nil, err
}
info.Id = model.NewId()
info.CreatorId = userId
pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
info.Path = pathPrefix + filename
if info.IsImage() {
// Check dimensions before loading the whole thing into memory later on
if info.Width*info.Height > model.MaxImageSize {
err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "")
err.StatusCode = http.StatusBadRequest
return nil, err
}
nameWithoutExtension := filename[:strings.LastIndex(filename, ".")]
info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg"
info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
}
if err := app.WriteFile(data, info.Path); err != nil {
return nil, err
}
if result := <-app.Srv.Store.FileInfo().Save(info); result.Err != nil {
return nil, result.Err
}
return info, nil
}
func handleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) {
for i, data := range fileData {
go func(i int, data []byte) {
img, width, height := prepareImage(fileData[i])
if img != nil {
go generateThumbnailImage(*img, thumbnailPathList[i], width, height)
go generatePreviewImage(*img, previewPathList[i], width)
}
}(i, data)
}
}
func prepareImage(fileData []byte) (*image.Image, int, int) {
// Decode image bytes into Image object
img, imgType, err := image.Decode(bytes.NewReader(fileData))
if err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err)
return nil, 0, 0
}
width := img.Bounds().Dx()
height := img.Bounds().Dy()
// Fill in the background of a potentially-transparent png file as white
if imgType == "png" {
dst := image.NewRGBA(img.Bounds())
draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over)
img = dst
}
// Flip the image to be upright
orientation, _ := getImageOrientation(fileData)
switch orientation {
case UprightMirrored:
img = imaging.FlipH(img)
case UpsideDown:
img = imaging.Rotate180(img)
case UpsideDownMirrored:
img = imaging.FlipV(img)
case RotatedCWMirrored:
img = imaging.Transpose(img)
case RotatedCCW:
img = imaging.Rotate270(img)
case RotatedCCWMirrored:
img = imaging.Transverse(img)
case RotatedCW:
img = imaging.Rotate90(img)
}
return &img, width, height
}
func getImageOrientation(imageData []byte) (int, error) {
if exifData, err := exif.Decode(bytes.NewReader(imageData)); err != nil {
return Upright, err
} else {
if tag, err := exifData.Get("Orientation"); err != nil {
return Upright, err
} else {
orientation, err := tag.Int(0)
if err != nil {
return Upright, err
} else {
return orientation, nil
}
}
}
}
func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) {
thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth)
thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight)
imgWidth := float64(width)
imgHeight := float64(height)
var thumbnail image.Image
if imgHeight < thumbHeight && imgWidth < thumbWidth {
thumbnail = img
} else if imgHeight/imgWidth < thumbHeight/thumbWidth {
thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos)
} else {
thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos)
}
buf := new(bytes.Buffer)
if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err)
return
}
if err := app.WriteFile(buf.Bytes(), thumbnailPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
return
}
}
func generatePreviewImage(img image.Image, previewPath string, width int) {
var preview image.Image
if width > int(utils.Cfg.FileSettings.PreviewWidth) {
preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos)
} else {
preview = img
}
buf := new(bytes.Buffer)
if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err)
return
}
if err := app.WriteFile(buf.Bytes(), previewPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
return
}
}
func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
info, err := getFileInfoForRequest(c, r, true)
if err != nil {

2
api/team.go
View File

@@ -516,7 +516,7 @@ func importTeam(c *Context, w http.ResponseWriter, r *http.Request) {
switch importFrom {
case "slack":
var err *model.AppError
if err, log = SlackImport(fileData, fileSize, c.TeamId); err != nil {
if err, log = app.SlackImport(fileData, fileSize, c.TeamId); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusBadRequest
}

191
app/file.go
View File

@@ -5,11 +5,17 @@ package app
import (
"bytes"
_ "image/gif"
"crypto/sha256"
"encoding/base64"
"fmt"
"image"
"image/color"
"image/draw"
"image/jpeg"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
@@ -20,8 +26,33 @@ import (
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
"github.com/disintegration/imaging"
s3 "github.com/minio/minio-go"
"github.com/rwcarlsen/goexif/exif"
_ "golang.org/x/image/bmp"
)
const (
/*
EXIF Image Orientations
1 2 3 4 5 6 7 8
888888 888888 88 88 8888888888 88 88 8888888888
88 88 88 88 88 88 88 88 88 88 88 88
8888 8888 8888 8888 88 8888888888 8888888888 88
88 88 88 88
88 88 888888 888888
*/
Upright = 1
UprightMirrored = 2
UpsideDown = 3
UpsideDownMirrored = 4
RotatedCWMirrored = 5
RotatedCCW = 6
RotatedCCWMirrored = 7
RotatedCW = 8
MaxImageSize = 6048 * 4032 // 24 megapixels, roughly 36MB as a raw image
)
func ReadFile(path string) ([]byte, *model.AppError) {
@@ -338,3 +369,161 @@ func GeneratePublicLinkHash(fileId, salt string) string {
return base64.RawURLEncoding.EncodeToString(hash.Sum(nil))
}
func DoUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
filename := filepath.Base(rawFilename)
info, err := model.GetInfoForBytes(filename, data)
if err != nil {
err.StatusCode = http.StatusBadRequest
return nil, err
}
info.Id = model.NewId()
info.CreatorId = userId
pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
info.Path = pathPrefix + filename
if info.IsImage() {
// Check dimensions before loading the whole thing into memory later on
if info.Width*info.Height > MaxImageSize {
err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "")
err.StatusCode = http.StatusBadRequest
return nil, err
}
nameWithoutExtension := filename[:strings.LastIndex(filename, ".")]
info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg"
info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
}
if err := WriteFile(data, info.Path); err != nil {
return nil, err
}
if result := <-Srv.Store.FileInfo().Save(info); result.Err != nil {
return nil, result.Err
}
return info, nil
}
func HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) {
for i, data := range fileData {
go func(i int, data []byte) {
img, width, height := prepareImage(fileData[i])
if img != nil {
go generateThumbnailImage(*img, thumbnailPathList[i], width, height)
go generatePreviewImage(*img, previewPathList[i], width)
}
}(i, data)
}
}
func prepareImage(fileData []byte) (*image.Image, int, int) {
// Decode image bytes into Image object
img, imgType, err := image.Decode(bytes.NewReader(fileData))
if err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err)
return nil, 0, 0
}
width := img.Bounds().Dx()
height := img.Bounds().Dy()
// Fill in the background of a potentially-transparent png file as white
if imgType == "png" {
dst := image.NewRGBA(img.Bounds())
draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over)
img = dst
}
// Flip the image to be upright
orientation, _ := getImageOrientation(fileData)
switch orientation {
case UprightMirrored:
img = imaging.FlipH(img)
case UpsideDown:
img = imaging.Rotate180(img)
case UpsideDownMirrored:
img = imaging.FlipV(img)
case RotatedCWMirrored:
img = imaging.Transpose(img)
case RotatedCCW:
img = imaging.Rotate270(img)
case RotatedCCWMirrored:
img = imaging.Transverse(img)
case RotatedCW:
img = imaging.Rotate90(img)
}
return &img, width, height
}
func getImageOrientation(imageData []byte) (int, error) {
if exifData, err := exif.Decode(bytes.NewReader(imageData)); err != nil {
return Upright, err
} else {
if tag, err := exifData.Get("Orientation"); err != nil {
return Upright, err
} else {
orientation, err := tag.Int(0)
if err != nil {
return Upright, err
} else {
return orientation, nil
}
}
}
}
func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) {
thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth)
thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight)
imgWidth := float64(width)
imgHeight := float64(height)
var thumbnail image.Image
if imgHeight < thumbHeight && imgWidth < thumbWidth {
thumbnail = img
} else if imgHeight/imgWidth < thumbHeight/thumbWidth {
thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos)
} else {
thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos)
}
buf := new(bytes.Buffer)
if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err)
return
}
if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
return
}
}
func generatePreviewImage(img image.Image, previewPath string, width int) {
var preview image.Image
if width > int(utils.Cfg.FileSettings.PreviewWidth) {
preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos)
} else {
preview = img
}
buf := new(bytes.Buffer)
if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err)
return
}
if err := WriteFile(buf.Bytes(), previewPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
return
}
}

17
api/import.go → app/import.go
View File

@@ -1,7 +1,7 @@
// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package api
package app
import (
"bytes"
@@ -10,7 +10,6 @@ import (
"unicode/utf8"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
)
@@ -35,12 +34,12 @@ func ImportPost(post *model.Post) {
post.Hashtags, _ = model.ParseHashtags(post.Message)
if result := <-app.Srv.Store.Post().Save(post); result.Err != nil {
if result := <-Srv.Store.Post().Save(post); result.Err != nil {
l4g.Debug(utils.T("api.import.import_post.saving.debug"), post.UserId, post.Message)
}
for _, fileId := range post.FileIds {
if result := <-app.Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil {
if result := <-Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil {
l4g.Error(utils.T("api.import.import_post.attach_files.error"), post.Id, post.FileIds, result.Err)
}
}
@@ -56,17 +55,17 @@ func ImportUser(team *model.Team, user *model.User) *model.User {
user.Roles = model.ROLE_SYSTEM_USER.Id
if result := <-app.Srv.Store.User().Save(user); result.Err != nil {
if result := <-Srv.Store.User().Save(user); result.Err != nil {
l4g.Error(utils.T("api.import.import_user.saving.error"), result.Err)
return nil
} else {
ruser := result.Data.(*model.User)
if cresult := <-app.Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil {
if cresult := <-Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil {
l4g.Error(utils.T("api.import.import_user.set_email.error"), cresult.Err)
}
if err := app.JoinUserToTeam(team, user); err != nil {
if err := JoinUserToTeam(team, user); err != nil {
l4g.Error(utils.T("api.import.import_user.join_team.error"), err)
}
@@ -75,7 +74,7 @@ func ImportUser(team *model.Team, user *model.User) *model.User {
}
func ImportChannel(channel *model.Channel) *model.Channel {
if result := <-app.Srv.Store.Channel().Save(channel); result.Err != nil {
if result := <-Srv.Store.Channel().Save(channel); result.Err != nil {
return nil
} else {
sc := result.Data.(*model.Channel)
@@ -89,7 +88,7 @@ func ImportFile(file io.Reader, teamId string, channelId string, userId string,
io.Copy(buf, file)
data := buf.Bytes()
fileInfo, err := doUploadFile(teamId, channelId, userId, fileName, data)
fileInfo, err := DoUploadFile(teamId, channelId, userId, fileName, data)
if err != nil {
return nil, err
}

21
api/slackimport.go → app/slackimport.go
View File

@@ -1,7 +1,7 @@
// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package api
package app
import (
"archive/zip"
@@ -16,7 +16,6 @@ import (
"unicode/utf8"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
)
@@ -138,7 +137,7 @@ func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map
// Need the team
var team *model.Team
if result := <-app.Srv.Store.Team().Get(teamId); result.Err != nil {
if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
return addedUsers
} else {
@@ -160,10 +159,10 @@ func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map
password := model.NewId()
// Check for email conflict and use existing user if found
if result := <-app.Srv.Store.User().GetByEmail(email); result.Err == nil {
if result := <-Srv.Store.User().GetByEmail(email); result.Err == nil {
existingUser := result.Data.(*model.User)
addedUsers[sUser.Id] = existingUser
if err := app.JoinUserToTeam(team, addedUsers[sUser.Id]); err != nil {
if err := JoinUserToTeam(team, addedUsers[sUser.Id]); err != nil {
log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
} else {
log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
@@ -192,7 +191,7 @@ func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map
func SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
var team *model.Team
if result := <-app.Srv.Store.Team().Get(teamId); result.Err != nil {
if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
return nil
} else {
@@ -245,7 +244,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
}
ImportPost(&newPost)
for _, fileId := range newPost.FileIds {
if result := <-app.Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err)
}
}
@@ -413,7 +412,7 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin
}
func deactivateSlackBotUser(user *model.User) {
_, err := app.UpdateActive(user, false)
_, err := UpdateActive(user, false)
if err != nil {
l4g.Warn(utils.T("api.slackimport.slack_deactivate_bot_user.failed_to_deactivate", err))
}
@@ -424,7 +423,7 @@ func addSlackUsersToChannel(members []string, users map[string]*model.User, chan
if user, ok := users[member]; !ok {
log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"}))
} else {
if _, err := app.AddUserToChannel(user, channel); err != nil {
if _, err := AddUserToChannel(user, channel); err != nil {
log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username}))
}
}
@@ -474,7 +473,7 @@ func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[str
mChannel := ImportChannel(&newChannel)
if mChannel == nil {
// Maybe it already exists?
if result := <-app.Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil {
if result := <-Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil {
l4g.Warn(utils.T("api.slackimport.slack_add_channels.import_failed.warn"), newChannel.DisplayName)
log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
continue
@@ -609,7 +608,7 @@ func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model
deactivateSlackBotUser(botUser)
}
app.InvalidateAllCaches()
InvalidateAllCaches()
log.WriteString(utils.T("api.slackimport.slack_import.notes"))
log.WriteString("=======\r\n\r\n")

2
api/slackimport_test.go → app/slackimport_test.go
View File

@@ -1,7 +1,7 @@
// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package api
package app
import (
"github.com/mattermost/platform/model"

4
cmd/platform/import.go
View File

@@ -6,7 +6,7 @@ import (
"errors"
"os"
"github.com/mattermost/platform/api"
"github.com/mattermost/platform/app"
"github.com/spf13/cobra"
)
@@ -54,7 +54,7 @@ func slackImportCmdF(cmd *cobra.Command, args []string) error {
CommandPrettyPrintln("Running Slack Import. This may take a long time for large teams or teams with many messages.")
api.SlackImport(fileReader, fileInfo.Size(), team.Id)
app.SlackImport(fileReader, fileInfo.Size(), team.Id)
CommandPrettyPrintln("Finished Slack Import.")

2
cmd/platform/oldcommands.go
View File

@@ -1075,7 +1075,7 @@ func cmdSlackImport() {
fmt.Fprintln(os.Stdout, "Running Slack Import. This may take a long time for large teams or teams with many messages.")
api.SlackImport(fileReader, fileInfo.Size(), team.Id)
app.SlackImport(fileReader, fileInfo.Size(), team.Id)
flushLogAndExit(0)
}