Adding ability to export data from mattermost

This commit is contained in:
Christopher Speller
2015-08-26 12:49:07 -04:00
parent 1b92352844
commit f0fd9a9e8b
16 changed files with 589 additions and 0 deletions

292
api/export.go Normal file
View File

@@ -0,0 +1,292 @@
// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.
package api
import (
"archive/zip"
"encoding/json"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
"io"
"os"
)
const (
EXPORT_PATH = "export/"
EXPORT_FILENAME = "MattermostExport.zip"
EXPORT_OPTIONS_FILE = "options.json"
EXPORT_TEAMS_FOLDER = "teams"
EXPORT_CHANNELS_FOLDER = "channels"
EXPORT_CHANNEL_MEMBERS_FOLDER = "members"
EXPORT_POSTS_FOLDER = "posts"
EXPORT_USERS_FOLDER = "users"
EXPORT_LOCAL_STORAGE_FOLDER = "files"
)
type ExportWriter interface {
Create(name string) (io.Writer, error)
}
type ExportOptions struct {
TeamsToExport []string `json:"teams"`
ChannelsToExport []string `json:"channels"`
UsersToExport []string `json:"users"`
ExportLocalStorage bool `json:"export_local_storage"`
}
func (options *ExportOptions) ToJson() string {
b, err := json.Marshal(options)
if err != nil {
return ""
} else {
return string(b)
}
}
func ExportOptionsFromJson(data io.Reader) *ExportOptions {
decoder := json.NewDecoder(data)
var o ExportOptions
decoder.Decode(&o)
return &o
}
func ExportToFile(options *ExportOptions) (link string, err *model.AppError) {
// Open file for export
if file, err := openFileWriteStream(EXPORT_PATH + EXPORT_FILENAME); err != nil {
return "", err
} else {
defer closeFileWriteStream(file)
ExportToWriter(file, options)
}
return "/api/v1/files/get_export", nil
}
func ExportToWriter(w io.Writer, options *ExportOptions) *model.AppError {
// Open a writer to write to zip file
zipWriter := zip.NewWriter(w)
defer zipWriter.Close()
// Write our options to file
if optionsFile, err := zipWriter.Create(EXPORT_OPTIONS_FILE); err != nil {
return model.NewAppError("ExportToWriter", "Unable to create options file", err.Error())
} else {
if _, err := optionsFile.Write([]byte(options.ToJson())); err != nil {
return model.NewAppError("ExportToWriter", "Unable to write to options file", err.Error())
}
}
// Export Teams
ExportTeams(zipWriter, options)
return nil
}
func ExportTeams(writer ExportWriter, options *ExportOptions) *model.AppError {
// Get the teams
var teams []*model.Team
if len(options.TeamsToExport) == 0 {
if result := <-Srv.Store.Team().GetForExport(); result.Err != nil {
return result.Err
} else {
teams = result.Data.([]*model.Team)
}
} else {
for _, teamId := range options.TeamsToExport {
if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
return result.Err
} else {
team := result.Data.(*model.Team)
teams = append(teams, team)
}
}
}
// Export the teams
for i := range teams {
// Sanitize
teams[i].PreExport()
if teamFile, err := writer.Create(EXPORT_TEAMS_FOLDER + "/" + teams[i].Name + ".json"); err != nil {
return model.NewAppError("ExportTeams", "Unable to open file for export", err.Error())
} else {
if _, err := teamFile.Write([]byte(teams[i].ToJson())); err != nil {
return model.NewAppError("ExportTeams", "Unable to write to team export file", err.Error())
}
}
}
// Export the channels, local storage and users
for _, team := range teams {
if err := ExportChannels(writer, options, team.Id); err != nil {
return err
}
if err := ExportUsers(writer, options, team.Id); err != nil {
return err
}
if err := ExportLocalStorage(writer, options, team.Id); err != nil {
return err
}
}
return nil
}
func ExportChannels(writer ExportWriter, options *ExportOptions, teamId string) *model.AppError {
// Get the channels
var channels []*model.Channel
if len(options.ChannelsToExport) == 0 {
if result := <-Srv.Store.Channel().GetForExport(teamId); result.Err != nil {
return result.Err
} else {
channels = result.Data.([]*model.Channel)
}
} else {
for _, channelId := range options.ChannelsToExport {
if result := <-Srv.Store.Channel().Get(channelId); result.Err != nil {
return result.Err
} else {
channel := result.Data.(*model.Channel)
channels = append(channels, channel)
}
}
}
for i := range channels {
// Get members
mchan := Srv.Store.Channel().GetMembers(channels[i].Id)
// Sanitize
channels[i].PreExport()
if channelFile, err := writer.Create(EXPORT_CHANNELS_FOLDER + "/" + channels[i].Id + ".json"); err != nil {
return model.NewAppError("ExportChannels", "Unable to open file for export", err.Error())
} else {
if _, err := channelFile.Write([]byte(channels[i].ToJson())); err != nil {
return model.NewAppError("ExportChannels", "Unable to write to export file", err.Error())
}
}
var members []model.ChannelMember
if result := <-mchan; result.Err != nil {
return result.Err
} else {
members = result.Data.([]model.ChannelMember)
}
if membersFile, err := writer.Create(EXPORT_CHANNELS_FOLDER + "/" + channels[i].Id + "_members.json"); err != nil {
return model.NewAppError("ExportChannels", "Unable to open file for export", err.Error())
} else {
result, err2 := json.Marshal(members)
if err2 != nil {
return model.NewAppError("ExportChannels", "Unable to convert to json", err.Error())
}
if _, err3 := membersFile.Write([]byte(result)); err3 != nil {
return model.NewAppError("ExportChannels", "Unable to write to export file", err.Error())
}
}
}
for _, channel := range channels {
if err := ExportPosts(writer, options, channel.Id); err != nil {
return err
}
}
return nil
}
func ExportPosts(writer ExportWriter, options *ExportOptions, channelId string) *model.AppError {
// Get the posts
var posts []*model.Post
if result := <-Srv.Store.Post().GetForExport(channelId); result.Err != nil {
return result.Err
} else {
posts = result.Data.([]*model.Post)
}
// Export the posts
if postsFile, err := writer.Create(EXPORT_POSTS_FOLDER + "/" + channelId + "_posts.json"); err != nil {
return model.NewAppError("ExportPosts", "Unable to open file for export", err.Error())
} else {
result, err2 := json.Marshal(posts)
if err2 != nil {
return model.NewAppError("ExportPosts", "Unable to convert to json", err.Error())
}
if _, err3 := postsFile.Write([]byte(result)); err3 != nil {
return model.NewAppError("ExportPosts", "Unable to write to export file", err.Error())
}
}
return nil
}
func ExportUsers(writer ExportWriter, options *ExportOptions, teamId string) *model.AppError {
// Get the users
var users []*model.User
if result := <-Srv.Store.User().GetForExport(teamId); result.Err != nil {
return result.Err
} else {
users = result.Data.([]*model.User)
}
// Write the users
if usersFile, err := writer.Create(EXPORT_USERS_FOLDER + "/" + teamId + "_users.json"); err != nil {
return model.NewAppError("ExportUsers", "Unable to open file for export", err.Error())
} else {
result, err2 := json.Marshal(users)
if err2 != nil {
return model.NewAppError("ExportUsers", "Unable to convert to json", err.Error())
}
if _, err3 := usersFile.Write([]byte(result)); err3 != nil {
return model.NewAppError("ExportUsers", "Unable to write to export file", err.Error())
}
}
return nil
}
func copyDirToExportWriter(writer ExportWriter, inPath string, outPath string) *model.AppError {
dir, err := os.Open(inPath)
if err != nil {
return model.NewAppError("copyDirToExportWriter", "Unable to open directory", err.Error())
}
fileInfoList, err := dir.Readdir(0)
if err != nil {
return model.NewAppError("copyDirToExportWriter", "Unable to read directory", err.Error())
}
for _, fileInfo := range fileInfoList {
if fileInfo.IsDir() {
copyDirToExportWriter(writer, inPath+"/"+fileInfo.Name(), outPath+"/"+fileInfo.Name())
} else {
if toFile, err := writer.Create(outPath + "/" + fileInfo.Name()); err != nil {
return model.NewAppError("copyDirToExportWriter", "Unable to open file for export", err.Error())
} else {
fromFile, err := os.Open(inPath + "/" + fileInfo.Name())
if err != nil {
return model.NewAppError("copyDirToExportWriter", "Unable to open file", err.Error())
}
io.Copy(toFile, fromFile)
}
}
}
return nil
}
func ExportLocalStorage(writer ExportWriter, options *ExportOptions, teamId string) *model.AppError {
teamDir := utils.Cfg.ServiceSettings.StorageDirectory + "teams/" + teamId
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
return model.NewAppError("ExportLocalStorage", "S3 is not supported for local storage export.", "")
} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
if err := copyDirToExportWriter(writer, teamDir, EXPORT_LOCAL_STORAGE_FOLDER); err != nil {
return err
}
}
return nil
}

View File

@@ -40,6 +40,7 @@ func InitFile(r *mux.Router) {
sr.Handle("/get/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiAppHandler(getFile)).Methods("GET")
sr.Handle("/get_info/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiAppHandler(getFileInfo)).Methods("GET")
sr.Handle("/get_public_link", ApiUserRequired(getPublicLink)).Methods("POST")
sr.Handle("/get_export", ApiUserRequired(getExport)).Methods("GET")
}
func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -414,6 +415,23 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write([]byte(model.MapToJson(rData)))
}
func getExport(c *Context, w http.ResponseWriter, r *http.Request) {
if !c.HasPermissionsToTeam(c.Session.TeamId, "export") || !c.IsTeamAdmin(c.Session.UserId) {
c.Err = model.NewAppError("getExport", "Only a team admin can retrieve exported data.", "userId="+c.Session.UserId)
c.Err.StatusCode = http.StatusForbidden
return
}
data, err := readFile(EXPORT_PATH + EXPORT_FILENAME)
if err != nil {
c.Err = model.NewAppError("getExport", "Unable to retrieve exported file. Please re-export", err.Error())
return
}
w.Header().Set("Content-Disposition", "attachment; filename="+EXPORT_FILENAME)
w.Header().Set("Content-Type", "application/octet-stream")
w.Write(data)
}
func writeFile(f []byte, path string) *model.AppError {
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
@@ -488,3 +506,27 @@ func readFile(path string) ([]byte, *model.AppError) {
return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
}
}
func openFileWriteStream(path string) (io.Writer, *model.AppError) {
if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage {
return nil, model.NewAppError("openFileWriteStream", "S3 is not supported.", "")
} else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 {
if err := os.MkdirAll(filepath.Dir(utils.Cfg.ServiceSettings.StorageDirectory+path), 0774); err != nil {
return nil, model.NewAppError("openFileWriteStream", "Encountered an error creating the directory for the new file", err.Error())
}
if fileHandle, err := os.Create(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil {
return nil, model.NewAppError("openFileWriteStream", "Encountered an error writing to local server storage", err.Error())
} else {
fileHandle.Chmod(0644)
return fileHandle, nil
}
}
return nil, model.NewAppError("openFileWriteStream", "File storage not configured properly. Please configure for either S3 or local server file storage.", "")
}
func closeFileWriteStream(file io.Writer) {
file.(*os.File).Close()
}

View File

@@ -32,7 +32,9 @@ func InitTeam(r *mux.Router) {
sr.Handle("/update_name", ApiUserRequired(updateTeamDisplayName)).Methods("POST")
sr.Handle("/update_valet_feature", ApiUserRequired(updateValetFeature)).Methods("POST")
sr.Handle("/me", ApiUserRequired(getMyTeam)).Methods("GET")
// These should be moved to the global admain console
sr.Handle("/import_team", ApiUserRequired(importTeam)).Methods("POST")
sr.Handle("/export_team", ApiUserRequired(exportTeam)).Methods("GET")
}
func signupTeam(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -675,3 +677,22 @@ func importTeam(c *Context, w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/octet-stream")
http.ServeContent(w, r, "MattermostImportLog.txt", time.Now(), bytes.NewReader(log.Bytes()))
}
func exportTeam(c *Context, w http.ResponseWriter, r *http.Request) {
if !c.HasPermissionsToTeam(c.Session.TeamId, "export") || !c.IsTeamAdmin(c.Session.UserId) {
c.Err = model.NewAppError("exportTeam", "Only a team admin can export data.", "userId="+c.Session.UserId)
c.Err.StatusCode = http.StatusForbidden
return
}
options := ExportOptionsFromJson(r.Body)
if link, err := ExportToFile(options); err != nil {
c.Err = err
return
} else {
result := map[string]string{}
result["link"] = link
w.Write([]byte(model.MapToJson(result)))
}
}

View File

@@ -117,3 +117,6 @@ func (o *Channel) PreUpdate() {
func (o *Channel) ExtraUpdated() {
o.ExtraUpdateAt = GetMillis()
}
func (o *Channel) PreExport() {
}

View File

@@ -147,3 +147,6 @@ func (o *Post) AddProp(key string, value string) {
o.Props[key] = value
}
func (o *Post) PreExport() {
}

View File

@@ -197,3 +197,6 @@ func CleanTeamName(s string) string {
return s
}
func (o *Team) PreExport() {
}

View File

@@ -272,6 +272,16 @@ func (u *User) GetDisplayName() string {
}
}
func (u *User) PreExport() {
u.Password = ""
u.AuthData = ""
u.LastActivityAt = 0
u.LastPingAt = 0
u.LastPasswordUpdate = 0
u.LastPictureUpdate = 0
u.FailedAttempts = 0
}
// UserFromJson will decode the input and return a User
func UserFromJson(data io.Reader) *User {
decoder := json.NewDecoder(data)

View File

@@ -678,3 +678,25 @@ func (s SqlChannelStore) UpdateNotifyLevel(channelId, userId, notifyLevel string
return storeChannel
}
func (s SqlChannelStore) GetForExport(teamId string) StoreChannel {
storeChannel := make(StoreChannel)
go func() {
result := StoreResult{}
var data []*model.Channel
_, err := s.GetReplica().Select(&data, "SELECT * FROM Channels WHERE TeamId = :TeamId AND DeleteAt = 0 AND Type = 'O'", map[string]interface{}{"TeamId": teamId})
if err != nil {
result.Err = model.NewAppError("SqlChannelStore.GetAllChannels", "We couldn't get all the channels", "teamId="+teamId+", err="+err.Error())
} else {
result.Data = data
}
storeChannel <- result
close(storeChannel)
}()
return storeChannel
}

View File

@@ -506,3 +506,27 @@ func (s SqlPostStore) Search(teamId string, userId string, terms string, isHasht
return storeChannel
}
func (s SqlPostStore) GetForExport(channelId string) StoreChannel {
storeChannel := make(StoreChannel)
go func() {
result := StoreResult{}
var posts []*model.Post
_, err := s.GetReplica().Select(
&posts,
"SELECT * FROM Posts WHERE ChannelId = :ChannelId AND DeleteAt = 0",
map[string]interface{}{"ChannelId": channelId})
if err != nil {
result.Err = model.NewAppError("SqlPostStore.GetForExport", "We couldn't get the posts for the channel", "channelId="+channelId+err.Error())
} else {
result.Data = posts
}
storeChannel <- result
close(storeChannel)
}()
return storeChannel
}

View File

@@ -193,3 +193,23 @@ func (s SqlTeamStore) GetTeamsForEmail(email string) StoreChannel {
return storeChannel
}
func (s SqlTeamStore) GetForExport() StoreChannel {
storeChannel := make(StoreChannel)
go func() {
result := StoreResult{}
var data []*model.Team
if _, err := s.GetReplica().Select(&data, "SELECT * FROM Teams"); err != nil {
result.Err = model.NewAppError("SqlTeamStore.GetForExport", "We could not get all teams", err.Error())
}
result.Data = data
storeChannel <- result
close(storeChannel)
}()
return storeChannel
}

View File

@@ -452,3 +452,30 @@ func (us SqlUserStore) VerifyEmail(userId string) StoreChannel {
return storeChannel
}
func (us SqlUserStore) GetForExport(teamId string) StoreChannel {
storeChannel := make(StoreChannel)
go func() {
result := StoreResult{}
var users []*model.User
if _, err := us.GetReplica().Select(&users, "SELECT * FROM Users WHERE TeamId = :TeamId", map[string]interface{}{"TeamId": teamId}); err != nil {
result.Err = model.NewAppError("SqlUserStore.GetProfiles", "We encounted an error while finding user profiles", err.Error())
} else {
for _, u := range users {
u.Password = ""
u.AuthData = ""
}
result.Data = users
}
storeChannel <- result
close(storeChannel)
}()
return storeChannel
}

View File

@@ -44,6 +44,7 @@ type TeamStore interface {
Get(id string) StoreChannel
GetByName(name string) StoreChannel
GetTeamsForEmail(domain string) StoreChannel
GetForExport() StoreChannel
}
type ChannelStore interface {
@@ -55,6 +56,7 @@ type ChannelStore interface {
GetChannels(teamId string, userId string) StoreChannel
GetMoreChannels(teamId string, userId string) StoreChannel
GetChannelCounts(teamId string, userId string) StoreChannel
GetForExport(teamId string) StoreChannel
SaveMember(member *model.ChannelMember) StoreChannel
GetMembers(channelId string) StoreChannel
@@ -78,6 +80,7 @@ type PostStore interface {
GetPostsSince(channelId string, time int64) StoreChannel
GetEtag(channelId string) StoreChannel
Search(teamId string, userId string, terms string, isHashtagSearch bool) StoreChannel
GetForExport(channelId string) StoreChannel
}
type UserStore interface {
@@ -96,6 +99,7 @@ type UserStore interface {
VerifyEmail(userId string) StoreChannel
GetEtagForProfiles(teamId string) StoreChannel
UpdateFailedPasswordAttempts(userId string, attempts int) StoreChannel
GetForExport(teamId string) StoreChannel
}
type SessionStore interface {

View File

@@ -0,0 +1,96 @@
// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.
var Client = require('../utils/client.jsx');
export default class TeamExportTab extends React.Component {
constructor(props) {
super(props);
this.state = {status: 'request', link: '', err: ''};
this.onExportSuccess = this.onExportSuccess.bind(this);
this.onExportFailure = this.onExportFailure.bind(this);
this.doExport = this.doExport.bind(this);
}
onExportSuccess(data) {
this.setState({status: 'ready', link: data.link, err: ''});
}
onExportFailure(e) {
this.setState({status: 'failure', link: '', err: e.message});
}
doExport() {
if (this.state.status === 'in-progress') {
return;
}
this.setState({status: 'in-progress'});
Client.exportTeam(this.onExportSuccess, this.onExportFailure);
}
render() {
var messageSection = '';
switch (this.state.status) {
case 'request':
messageSection = '';
break;
case 'in-progress':
messageSection = (
<p className='confirm-import alert alert-warning'>
<i className='fa fa-spinner fa-pulse' />
{' Exporting...'}
</p>
);
break;
case 'ready':
messageSection = (
<p className='confirm-import alert alert-success'>
<i className='fa fa-check' />
{' Ready for '}
<a
href={this.state.link}
download={true}
>
{'download'}
</a>
</p>
);
break;
case 'failure':
messageSection = (
<p className='confirm-import alert alert-warning'>
<i className='fa fa-warning' />
{' Unable to export: ' + this.state.err}
</p>
);
break;
}
return (
<div
ref='wrapper'
className='user-settings'
>
<h3 className='tab-header'>{'Export'}</h3>
<div className='divider-dark first'/>
<ul className='section-max'>
<li className='col-xs-12 section-title'>{'Export your team'}</li>
<li className='col-xs-offset-3 col-xs-8'>
<ul className='setting-list'>
<li className='setting-list-item'>
<span className='btn btn-sm btn-primary btn-file sel-btn'>
<a
className='btn btn-sm btn-primary'
href='#'
onClick={this.doExport}
>
{'Export'}
</a>
</span>
</li>
</ul>
</li>
</ul>
<div className='divider-dark'/>
{messageSection}
</div>
);
}
}

View File

@@ -3,6 +3,7 @@
var TeamStore = require('../stores/team_store.jsx');
var ImportTab = require('./team_import_tab.jsx');
var ExportTab = require('./team_export_tab.jsx');
var FeatureTab = require('./team_feature_tab.jsx');
var GeneralTab = require('./team_general_tab.jsx');
var Utils = require('../utils/utils.jsx');
@@ -64,6 +65,13 @@ export default class TeamSettings extends React.Component {
</div>
);
break;
case 'export':
result = (
<div>
<ExportTab />
</div>
);
break;
default:
result = (
<div/>

View File

@@ -36,6 +36,7 @@ export default class TeamSettingsModal extends React.Component {
let tabs = [];
tabs.push({name: 'general', uiName: 'General', icon: 'glyphicon glyphicon-cog'});
tabs.push({name: 'import', uiName: 'Import', icon: 'glyphicon glyphicon-upload'});
tabs.push({name: 'export', uiName: 'Export', icon: 'glyphicon glyphicon-download'});
tabs.push({name: 'feature', uiName: 'Advanced', icon: 'glyphicon glyphicon-wrench'});
return (

View File

@@ -919,6 +919,19 @@ export function importSlack(fileData, success, error) {
});
}
export function exportTeam(success, error) {
$.ajax({
url: '/api/v1/teams/export_team',
type: 'GET',
dataType: 'json',
success: success,
error: function onError(xhr, status, err) {
var e = handleError('exportTeam', xhr, status, err);
error(e);
}
});
}
export function getStatuses(success, error) {
$.ajax({
url: '/api/v1/users/status',