mirror of
https://github.com/mattermost/mattermost.git
synced 2025-02-25 18:55:24 -06:00
50fc6e1e9e
* Prepare file upload infrastructure for Data Retention. This commit prepares the file upload infrastructure for the data retention feature that is under construction. Changes are: * Move file management code to utils to allow access to it from jobs. * From now on, store all file uploads in a top level folder which is the date of the day on which they were uploaded. This commit is based on Harrison Healey's branch, but updated to work with the latest master. * Use NewAppError
691 lines
23 KiB
Go
691 lines
23 KiB
Go
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See License.txt for license information.
|
|
|
|
package app
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"encoding/json"
|
|
"io"
|
|
"mime/multipart"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
l4g "github.com/alecthomas/log4go"
|
|
"github.com/mattermost/platform/model"
|
|
"github.com/mattermost/platform/utils"
|
|
)
|
|
|
|
type SlackChannel struct {
|
|
Id string `json:"id"`
|
|
Name string `json:"name"`
|
|
Members []string `json:"members"`
|
|
Topic map[string]string `json:"topic"`
|
|
Purpose map[string]string `json:"purpose"`
|
|
}
|
|
|
|
type SlackUser struct {
|
|
Id string `json:"id"`
|
|
Username string `json:"name"`
|
|
Profile map[string]string `json:"profile"`
|
|
}
|
|
|
|
type SlackFile struct {
|
|
Id string `json:"id"`
|
|
Title string `json:"title"`
|
|
}
|
|
|
|
type SlackPost struct {
|
|
User string `json:"user"`
|
|
BotId string `json:"bot_id"`
|
|
BotUsername string `json:"username"`
|
|
Text string `json:"text"`
|
|
TimeStamp string `json:"ts"`
|
|
Type string `json:"type"`
|
|
SubType string `json:"subtype"`
|
|
Comment *SlackComment `json:"comment"`
|
|
Upload bool `json:"upload"`
|
|
File *SlackFile `json:"file"`
|
|
Attachments []*model.SlackAttachment `json:"attachments"`
|
|
}
|
|
|
|
var isValidChannelNameCharacters = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString
|
|
|
|
type SlackComment struct {
|
|
User string `json:"user"`
|
|
Comment string `json:"comment"`
|
|
}
|
|
|
|
func truncateRunes(s string, i int) string {
|
|
runes := []rune(s)
|
|
if len(runes) > i {
|
|
return string(runes[:i])
|
|
}
|
|
return s
|
|
}
|
|
|
|
func SlackConvertTimeStamp(ts string) int64 {
|
|
timeString := strings.SplitN(ts, ".", 2)[0]
|
|
|
|
timeStamp, err := strconv.ParseInt(timeString, 10, 64)
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_convert_timestamp.bad.warn"))
|
|
return 1
|
|
}
|
|
return timeStamp * 1000 // Convert to milliseconds
|
|
}
|
|
|
|
func SlackConvertChannelName(channelName string, channelId string) string {
|
|
newName := strings.Trim(channelName, "_-")
|
|
if len(newName) == 1 {
|
|
return "slack-channel-" + newName
|
|
}
|
|
|
|
if isValidChannelNameCharacters(newName) {
|
|
return newName
|
|
} else {
|
|
return strings.ToLower(channelId)
|
|
}
|
|
}
|
|
|
|
func SlackParseChannels(data io.Reader) ([]SlackChannel, error) {
|
|
decoder := json.NewDecoder(data)
|
|
|
|
var channels []SlackChannel
|
|
if err := decoder.Decode(&channels); err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_parse_channels.error"))
|
|
return channels, err
|
|
}
|
|
return channels, nil
|
|
}
|
|
|
|
func SlackParseUsers(data io.Reader) ([]SlackUser, error) {
|
|
decoder := json.NewDecoder(data)
|
|
|
|
var users []SlackUser
|
|
if err := decoder.Decode(&users); err != nil {
|
|
// This actually returns errors that are ignored.
|
|
// In this case it is erroring because of a null that Slack
|
|
// introduced. So we just return the users here.
|
|
return users, err
|
|
}
|
|
return users, nil
|
|
}
|
|
|
|
func SlackParsePosts(data io.Reader) ([]SlackPost, error) {
|
|
decoder := json.NewDecoder(data)
|
|
|
|
var posts []SlackPost
|
|
if err := decoder.Decode(&posts); err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_parse_posts.error"))
|
|
return posts, err
|
|
}
|
|
return posts, nil
|
|
}
|
|
|
|
func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map[string]*model.User {
|
|
// Log header
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.created"))
|
|
log.WriteString("===============\r\n\r\n")
|
|
|
|
addedUsers := make(map[string]*model.User)
|
|
|
|
// Need the team
|
|
var team *model.Team
|
|
if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
|
|
return addedUsers
|
|
} else {
|
|
team = result.Data.(*model.Team)
|
|
}
|
|
|
|
for _, sUser := range slackusers {
|
|
firstName := ""
|
|
lastName := ""
|
|
if name, ok := sUser.Profile["first_name"]; ok {
|
|
firstName = name
|
|
}
|
|
if name, ok := sUser.Profile["last_name"]; ok {
|
|
lastName = name
|
|
}
|
|
|
|
email := sUser.Profile["email"]
|
|
if email == "" {
|
|
email = sUser.Username + "@example.com"
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.missing_email_address", map[string]interface{}{"Email": email, "Username": sUser.Username}))
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_users.missing_email_address.warn", map[string]interface{}{"Email": email, "Username": sUser.Username}))
|
|
}
|
|
|
|
password := model.NewId()
|
|
|
|
// Check for email conflict and use existing user if found
|
|
if result := <-Srv.Store.User().GetByEmail(email); result.Err == nil {
|
|
existingUser := result.Data.(*model.User)
|
|
addedUsers[sUser.Id] = existingUser
|
|
if err := JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
|
|
} else {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
|
|
}
|
|
continue
|
|
}
|
|
|
|
newUser := model.User{
|
|
Username: sUser.Username,
|
|
FirstName: firstName,
|
|
LastName: lastName,
|
|
Email: email,
|
|
Password: password,
|
|
}
|
|
|
|
if mUser := OldImportUser(team, &newUser); mUser != nil {
|
|
addedUsers[sUser.Id] = mUser
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password}))
|
|
} else {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username}))
|
|
}
|
|
}
|
|
|
|
return addedUsers
|
|
}
|
|
|
|
func SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
|
|
var team *model.Team
|
|
if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
|
|
return nil
|
|
} else {
|
|
team = result.Data.(*model.Team)
|
|
}
|
|
|
|
password := model.NewId()
|
|
username := "slackimportuser_" + model.NewId()
|
|
email := username + "@localhost"
|
|
|
|
botUser := model.User{
|
|
Username: username,
|
|
FirstName: "",
|
|
LastName: "",
|
|
Email: email,
|
|
Password: password,
|
|
}
|
|
|
|
if mUser := OldImportUser(team, &botUser); mUser != nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password}))
|
|
return mUser
|
|
} else {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username}))
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) {
|
|
for _, sPost := range posts {
|
|
switch {
|
|
case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
|
|
if sPost.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug"))
|
|
continue
|
|
} else if users[sPost.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: sPost.Text,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
}
|
|
if sPost.Upload {
|
|
if fileInfo, ok := SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok == true {
|
|
newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
|
|
newPost.Message = sPost.File.Title
|
|
}
|
|
}
|
|
OldImportPost(&newPost)
|
|
for _, fileId := range newPost.FileIds {
|
|
if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
|
|
l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err)
|
|
}
|
|
}
|
|
|
|
case sPost.Type == "message" && sPost.SubType == "file_comment":
|
|
if sPost.Comment == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_comment.debug"))
|
|
continue
|
|
} else if sPost.Comment.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
|
|
continue
|
|
} else if users[sPost.Comment.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.Comment.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: sPost.Comment.Comment,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
}
|
|
OldImportPost(&newPost)
|
|
case sPost.Type == "message" && sPost.SubType == "bot_message":
|
|
if botUser == nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn"))
|
|
continue
|
|
} else if sPost.BotId == "" {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.no_bot_id.warn"))
|
|
continue
|
|
}
|
|
|
|
props := make(model.StringInterface)
|
|
props["override_username"] = sPost.BotUsername
|
|
if len(sPost.Attachments) > 0 {
|
|
props["attachments"] = sPost.Attachments
|
|
}
|
|
|
|
post := &model.Post{
|
|
UserId: botUser.Id,
|
|
ChannelId: channel.Id,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
Message: sPost.Text,
|
|
Type: model.POST_SLACK_ATTACHMENT,
|
|
}
|
|
|
|
OldImportIncomingWebhookPost(post, props)
|
|
case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
|
|
if sPost.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
|
|
continue
|
|
} else if users[sPost.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
|
|
var postType string
|
|
if sPost.SubType == "channel_join" {
|
|
postType = model.POST_JOIN_CHANNEL
|
|
} else {
|
|
postType = model.POST_LEAVE_CHANNEL
|
|
}
|
|
|
|
newPost := model.Post{
|
|
UserId: users[sPost.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: sPost.Text,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
Type: postType,
|
|
Props: model.StringInterface{
|
|
"username": users[sPost.User].Username,
|
|
},
|
|
}
|
|
OldImportPost(&newPost)
|
|
case sPost.Type == "message" && sPost.SubType == "me_message":
|
|
if sPost.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug"))
|
|
continue
|
|
} else if users[sPost.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: "*" + sPost.Text + "*",
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
}
|
|
OldImportPost(&newPost)
|
|
case sPost.Type == "message" && sPost.SubType == "channel_topic":
|
|
if sPost.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
|
|
continue
|
|
} else if users[sPost.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: sPost.Text,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
Type: model.POST_HEADER_CHANGE,
|
|
}
|
|
OldImportPost(&newPost)
|
|
case sPost.Type == "message" && sPost.SubType == "channel_purpose":
|
|
if sPost.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
|
|
continue
|
|
} else if users[sPost.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: sPost.Text,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
Type: model.POST_PURPOSE_CHANGE,
|
|
}
|
|
OldImportPost(&newPost)
|
|
case sPost.Type == "message" && sPost.SubType == "channel_name":
|
|
if sPost.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
|
|
continue
|
|
} else if users[sPost.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: sPost.Text,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
Type: model.POST_DISPLAYNAME_CHANGE,
|
|
}
|
|
OldImportPost(&newPost)
|
|
default:
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.unsupported.warn"), sPost.Type, sPost.SubType)
|
|
}
|
|
}
|
|
}
|
|
|
|
func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId string, channelId string, userId string) (*model.FileInfo, bool) {
|
|
if sPost.File != nil {
|
|
if file, ok := uploads[sPost.File.Id]; ok == true {
|
|
openFile, err := file.Open()
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_open_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
|
|
return nil, false
|
|
}
|
|
defer openFile.Close()
|
|
|
|
timestamp := utils.TimeFromMillis(SlackConvertTimeStamp(sPost.TimeStamp))
|
|
uploadedFile, err := OldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name))
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
|
|
return nil, false
|
|
}
|
|
|
|
return uploadedFile, true
|
|
} else {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_found.warn", map[string]interface{}{"FileId": sPost.File.Id}))
|
|
return nil, false
|
|
}
|
|
} else {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_in_json.warn"))
|
|
return nil, false
|
|
}
|
|
}
|
|
|
|
func deactivateSlackBotUser(user *model.User) {
|
|
_, err := UpdateActive(user, false)
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_deactivate_bot_user.failed_to_deactivate", err))
|
|
}
|
|
}
|
|
|
|
func addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) {
|
|
for _, member := range members {
|
|
if user, ok := users[member]; !ok {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"}))
|
|
} else {
|
|
if _, err := AddUserToChannel(user, channel); err != nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username}))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func SlackSanitiseChannelProperties(channel model.Channel) model.Channel {
|
|
if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES {
|
|
l4g.Warn("api.slackimport.slack_sanitise_channel_properties.display_name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
|
|
channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES)
|
|
}
|
|
|
|
if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH {
|
|
l4g.Warn("api.slackimport.slack_sanitise_channel_properties.name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
|
|
channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH]
|
|
}
|
|
|
|
if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES {
|
|
l4g.Warn("api.slackimport.slack_sanitise_channel_properties.purpose_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
|
|
channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES)
|
|
}
|
|
|
|
if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES {
|
|
l4g.Warn("api.slackimport.slack_sanitise_channel_properties.header_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
|
|
channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES)
|
|
}
|
|
|
|
return channel
|
|
}
|
|
|
|
func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, log *bytes.Buffer) map[string]*model.Channel {
|
|
// Write Header
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.added"))
|
|
log.WriteString("=================\r\n\r\n")
|
|
|
|
addedChannels := make(map[string]*model.Channel)
|
|
for _, sChannel := range slackchannels {
|
|
newChannel := model.Channel{
|
|
TeamId: teamId,
|
|
Type: model.CHANNEL_OPEN,
|
|
DisplayName: sChannel.Name,
|
|
Name: SlackConvertChannelName(sChannel.Name, sChannel.Id),
|
|
Purpose: sChannel.Purpose["value"],
|
|
Header: sChannel.Topic["value"],
|
|
}
|
|
newChannel = SlackSanitiseChannelProperties(newChannel)
|
|
|
|
var mChannel *model.Channel
|
|
if result := <-Srv.Store.Channel().GetByName(teamId, sChannel.Name, true); result.Err == nil {
|
|
// The channel already exists as an active channel. Merge with the existing one.
|
|
mChannel = result.Data.(*model.Channel)
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
|
|
} else if result := <-Srv.Store.Channel().GetDeletedByName(teamId, sChannel.Name); result.Err == nil {
|
|
// The channel already exists but has been deleted. Generate a random string for the handle instead.
|
|
newChannel.Name = model.NewId()
|
|
newChannel = SlackSanitiseChannelProperties(newChannel)
|
|
}
|
|
|
|
if mChannel == nil {
|
|
// Haven't found an existing channel to merge with. Try importing it as a new one.
|
|
mChannel = OldImportChannel(&newChannel)
|
|
if mChannel == nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_channels.import_failed.warn"), newChannel.DisplayName)
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
|
|
continue
|
|
}
|
|
}
|
|
|
|
addSlackUsersToChannel(sChannel.Members, users, mChannel, log)
|
|
log.WriteString(newChannel.DisplayName + "\r\n")
|
|
addedChannels[sChannel.Id] = mChannel
|
|
SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser)
|
|
}
|
|
|
|
return addedChannels
|
|
}
|
|
|
|
func SlackConvertUserMentions(users []SlackUser, posts map[string][]SlackPost) map[string][]SlackPost {
|
|
var regexes = make(map[string]*regexp.Regexp, len(users))
|
|
for _, user := range users {
|
|
r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>")
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_convert_user_mentions.compile_regexp_failed.warn"), user.Id, user.Username)
|
|
continue
|
|
}
|
|
regexes["@"+user.Username] = r
|
|
}
|
|
|
|
// Special cases.
|
|
regexes["@here"], _ = regexp.Compile(`<!here\|@here>`)
|
|
regexes["@channel"], _ = regexp.Compile("<!channel>")
|
|
regexes["@all"], _ = regexp.Compile("<!everyone>")
|
|
|
|
for channelName, channelPosts := range posts {
|
|
for postIdx, post := range channelPosts {
|
|
for mention, r := range regexes {
|
|
post.Text = r.ReplaceAllString(post.Text, mention)
|
|
posts[channelName][postIdx] = post
|
|
}
|
|
}
|
|
}
|
|
|
|
return posts
|
|
}
|
|
|
|
func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]SlackPost) map[string][]SlackPost {
|
|
var regexes = make(map[string]*regexp.Regexp, len(channels))
|
|
for _, channel := range channels {
|
|
r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>")
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_convert_channel_mentions.compile_regexp_failed.warn"), channel.Id, channel.Name)
|
|
continue
|
|
}
|
|
regexes["~"+channel.Name] = r
|
|
}
|
|
|
|
for channelName, channelPosts := range posts {
|
|
for postIdx, post := range channelPosts {
|
|
for channelReplace, r := range regexes {
|
|
post.Text = r.ReplaceAllString(post.Text, channelReplace)
|
|
posts[channelName][postIdx] = post
|
|
}
|
|
}
|
|
}
|
|
|
|
return posts
|
|
}
|
|
|
|
func SlackConvertPostsMarkup(posts map[string][]SlackPost) map[string][]SlackPost {
|
|
regexReplaceAllString := []struct {
|
|
regex *regexp.Regexp
|
|
rpl string
|
|
}{
|
|
// URL
|
|
{
|
|
regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`),
|
|
"[$2]($1)",
|
|
},
|
|
// bold
|
|
{
|
|
regexp.MustCompile(`(^|[\s.;,])\*(\S[^*\n]+)\*`),
|
|
"$1**$2**",
|
|
},
|
|
// strikethrough
|
|
{
|
|
regexp.MustCompile(`(^|[\s.;,])\~(\S[^~\n]+)\~`),
|
|
"$1~~$2~~",
|
|
},
|
|
// single paragraph blockquote
|
|
// Slack converts > character to >
|
|
{
|
|
regexp.MustCompile(`(?sm)^>`),
|
|
">",
|
|
},
|
|
}
|
|
|
|
regexReplaceAllStringFunc := []struct {
|
|
regex *regexp.Regexp
|
|
fn func(string) string
|
|
}{
|
|
// multiple paragraphs blockquotes
|
|
{
|
|
regexp.MustCompile(`(?sm)^>>>(.+)$`),
|
|
func(src string) string {
|
|
// remove >>> prefix, might have leading \n
|
|
prefixRegexp := regexp.MustCompile(`^([\n])?>>>(.*)`)
|
|
src = prefixRegexp.ReplaceAllString(src, "$1$2")
|
|
// append > to start of line
|
|
appendRegexp := regexp.MustCompile(`(?m)^`)
|
|
return appendRegexp.ReplaceAllString(src, ">$0")
|
|
},
|
|
},
|
|
}
|
|
|
|
for channelName, channelPosts := range posts {
|
|
for postIdx, post := range channelPosts {
|
|
result := post.Text
|
|
|
|
for _, rule := range regexReplaceAllString {
|
|
result = rule.regex.ReplaceAllString(result, rule.rpl)
|
|
}
|
|
|
|
for _, rule := range regexReplaceAllStringFunc {
|
|
result = rule.regex.ReplaceAllStringFunc(result, rule.fn)
|
|
}
|
|
posts[channelName][postIdx].Text = result
|
|
}
|
|
}
|
|
|
|
return posts
|
|
}
|
|
|
|
func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) {
|
|
// Create log file
|
|
log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log"))
|
|
|
|
zipreader, err := zip.NewReader(fileData, fileSize)
|
|
if err != nil || zipreader.File == nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error"))
|
|
return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error()), log
|
|
}
|
|
|
|
var channels []SlackChannel
|
|
var users []SlackUser
|
|
posts := make(map[string][]SlackPost)
|
|
uploads := make(map[string]*zip.File)
|
|
for _, file := range zipreader.File {
|
|
reader, err := file.Open()
|
|
if err != nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}))
|
|
return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error()), log
|
|
}
|
|
if file.Name == "channels.json" {
|
|
channels, _ = SlackParseChannels(reader)
|
|
} else if file.Name == "users.json" {
|
|
users, _ = SlackParseUsers(reader)
|
|
} else {
|
|
spl := strings.Split(file.Name, "/")
|
|
if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") {
|
|
newposts, _ := SlackParsePosts(reader)
|
|
channel := spl[0]
|
|
if _, ok := posts[channel]; ok == false {
|
|
posts[channel] = newposts
|
|
} else {
|
|
posts[channel] = append(posts[channel], newposts...)
|
|
}
|
|
} else if len(spl) == 3 && spl[0] == "__uploads" {
|
|
uploads[spl[1]] = file
|
|
}
|
|
}
|
|
}
|
|
|
|
posts = SlackConvertUserMentions(users, posts)
|
|
posts = SlackConvertChannelMentions(channels, posts)
|
|
posts = SlackConvertPostsMarkup(posts)
|
|
|
|
addedUsers := SlackAddUsers(teamID, users, log)
|
|
botUser := SlackAddBotUser(teamID, log)
|
|
|
|
SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log)
|
|
|
|
if botUser != nil {
|
|
deactivateSlackBotUser(botUser)
|
|
}
|
|
|
|
InvalidateAllCaches()
|
|
|
|
log.WriteString(utils.T("api.slackimport.slack_import.notes"))
|
|
log.WriteString("=======\r\n\r\n")
|
|
|
|
log.WriteString(utils.T("api.slackimport.slack_import.note1"))
|
|
log.WriteString(utils.T("api.slackimport.slack_import.note2"))
|
|
log.WriteString(utils.T("api.slackimport.slack_import.note3"))
|
|
|
|
return nil, log
|
|
}
|