Files
mattermost/api4/apitestlib.go

908 lines
23 KiB
Go
Raw Normal View History

// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package api4
import (
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"reflect"
"strconv"
"strings"
"testing"
2017-01-31 09:31:53 -05:00
"time"
2017-09-06 23:05:10 -07:00
"github.com/mattermost/mattermost-server/app"
"github.com/mattermost/mattermost-server/config"
"github.com/mattermost/mattermost-server/mlog"
2017-09-06 23:05:10 -07:00
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/store"
"github.com/mattermost/mattermost-server/utils"
"github.com/mattermost/mattermost-server/web"
2017-09-06 23:05:10 -07:00
"github.com/mattermost/mattermost-server/wsapi"
s3 "github.com/minio/minio-go/v6"
"github.com/minio/minio-go/v6/pkg/credentials"
)
type TestHelper struct {
App *app.App
Server *app.Server
ConfigStore config.Store
2017-09-06 17:12:54 -05:00
Client *model.Client4
BasicUser *model.User
BasicUser2 *model.User
TeamAdminUser *model.User
BasicTeam *model.Team
BasicChannel *model.Channel
BasicPrivateChannel *model.Channel
BasicDeletedChannel *model.Channel
BasicChannel2 *model.Channel
BasicPost *model.Post
Group *model.Group
2017-01-31 09:31:53 -05:00
SystemAdminClient *model.Client4
SystemAdminUser *model.User
tempWorkspace string
}
Dockerized build updated tests (#9943) * testlib: introduce and leverage This doesn't yet factor out the individual test helpers: many packages still rely on `api4` directly to do this, but now wire up the test store setup through this package. `app` and `store`, in particular, don't use `testlib` because of circular dependencies at the moment. * cmd: command_test.go: use api4 testlib * cmd: plugin_test.go: remove dependence on test-config.json * cmd: config_test.go use configured database settings * ensure test-(te|ee) exit with status code * test-server: run all tests, deprecating test-te/test-ee * cmd/mattermost/commands: fix unit tests Instead of relying on (and modifying) a config.json found in the current path, explicitly create a temporary one from defaults for each test. This was likely the source of various bugs over time, but specifically allows us to override the SqlSettings to point at the configured test database for all tests simultaneously. * wrap run/check into a test helper It was insufficient to set a config for each invocation of CheckCommand or RunCommand: some tests relied on the config having changed in a subsequent assertion. Instead, create a new test helper embedding api4.TestHelper. This has the nice advantage of cleaning up all the teardown. * additional TestConfigGet granularity * customized config path to avoid default location * be explicit if the storetest initialization fails * generate safe coverprofile names in the presence of subtests * additional TestConfigShow granularity * fix permission_test.go typo * fix webhook tests * actually flag.Parse() to skip database setup on os.Execed tests * fix recent regression in #9962, not caught by unit tests
2018-12-06 13:19:32 -05:00
// testStore tracks the active test store.
// This is a bridge between the new testlib ownership of the test store and the existing usage
// of the api4 test helper by many packages. In the future, this test helper would ideally belong
// to the testlib altogether.
var testStore store.Store
Dockerized build updated tests (#9943) * testlib: introduce and leverage This doesn't yet factor out the individual test helpers: many packages still rely on `api4` directly to do this, but now wire up the test store setup through this package. `app` and `store`, in particular, don't use `testlib` because of circular dependencies at the moment. * cmd: command_test.go: use api4 testlib * cmd: plugin_test.go: remove dependence on test-config.json * cmd: config_test.go use configured database settings * ensure test-(te|ee) exit with status code * test-server: run all tests, deprecating test-te/test-ee * cmd/mattermost/commands: fix unit tests Instead of relying on (and modifying) a config.json found in the current path, explicitly create a temporary one from defaults for each test. This was likely the source of various bugs over time, but specifically allows us to override the SqlSettings to point at the configured test database for all tests simultaneously. * wrap run/check into a test helper It was insufficient to set a config for each invocation of CheckCommand or RunCommand: some tests relied on the config having changed in a subsequent assertion. Instead, create a new test helper embedding api4.TestHelper. This has the nice advantage of cleaning up all the teardown. * additional TestConfigGet granularity * customized config path to avoid default location * be explicit if the storetest initialization fails * generate safe coverprofile names in the presence of subtests * additional TestConfigShow granularity * fix permission_test.go typo * fix webhook tests * actually flag.Parse() to skip database setup on os.Execed tests * fix recent regression in #9962, not caught by unit tests
2018-12-06 13:19:32 -05:00
func UseTestStore(store store.Store) {
testStore = store
}
func setupTestHelper(enterprise bool, updateConfig func(*model.Config)) *TestHelper {
Dockerized build updated tests (#9943) * testlib: introduce and leverage This doesn't yet factor out the individual test helpers: many packages still rely on `api4` directly to do this, but now wire up the test store setup through this package. `app` and `store`, in particular, don't use `testlib` because of circular dependencies at the moment. * cmd: command_test.go: use api4 testlib * cmd: plugin_test.go: remove dependence on test-config.json * cmd: config_test.go use configured database settings * ensure test-(te|ee) exit with status code * test-server: run all tests, deprecating test-te/test-ee * cmd/mattermost/commands: fix unit tests Instead of relying on (and modifying) a config.json found in the current path, explicitly create a temporary one from defaults for each test. This was likely the source of various bugs over time, but specifically allows us to override the SqlSettings to point at the configured test database for all tests simultaneously. * wrap run/check into a test helper It was insufficient to set a config for each invocation of CheckCommand or RunCommand: some tests relied on the config having changed in a subsequent assertion. Instead, create a new test helper embedding api4.TestHelper. This has the nice advantage of cleaning up all the teardown. * additional TestConfigGet granularity * customized config path to avoid default location * be explicit if the storetest initialization fails * generate safe coverprofile names in the presence of subtests * additional TestConfigShow granularity * fix permission_test.go typo * fix webhook tests * actually flag.Parse() to skip database setup on os.Execed tests * fix recent regression in #9962, not caught by unit tests
2018-12-06 13:19:32 -05:00
testStore.DropAllTables()
memoryStore, err := config.NewMemoryStoreWithOptions(&config.MemoryStoreOptions{IgnoreEnvironmentOverrides: true})
if err != nil {
panic("failed to initialize memory store: " + err.Error())
}
var options []app.Option
options = append(options, app.ConfigStore(memoryStore))
Dockerized build updated tests (#9943) * testlib: introduce and leverage This doesn't yet factor out the individual test helpers: many packages still rely on `api4` directly to do this, but now wire up the test store setup through this package. `app` and `store`, in particular, don't use `testlib` because of circular dependencies at the moment. * cmd: command_test.go: use api4 testlib * cmd: plugin_test.go: remove dependence on test-config.json * cmd: config_test.go use configured database settings * ensure test-(te|ee) exit with status code * test-server: run all tests, deprecating test-te/test-ee * cmd/mattermost/commands: fix unit tests Instead of relying on (and modifying) a config.json found in the current path, explicitly create a temporary one from defaults for each test. This was likely the source of various bugs over time, but specifically allows us to override the SqlSettings to point at the configured test database for all tests simultaneously. * wrap run/check into a test helper It was insufficient to set a config for each invocation of CheckCommand or RunCommand: some tests relied on the config having changed in a subsequent assertion. Instead, create a new test helper embedding api4.TestHelper. This has the nice advantage of cleaning up all the teardown. * additional TestConfigGet granularity * customized config path to avoid default location * be explicit if the storetest initialization fails * generate safe coverprofile names in the presence of subtests * additional TestConfigShow granularity * fix permission_test.go typo * fix webhook tests * actually flag.Parse() to skip database setup on os.Execed tests * fix recent regression in #9962, not caught by unit tests
2018-12-06 13:19:32 -05:00
options = append(options, app.StoreOverride(testStore))
s, err := app.NewServer(options...)
if err != nil {
panic(err)
}
th := &TestHelper{
App: s.FakeApp(),
Server: s,
ConfigStore: memoryStore,
}
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.TeamSettings.MaxUsersPerTeam = 50
*cfg.RateLimitSettings.Enable = false
*cfg.EmailSettings.SendEmailNotifications = true
// Disable sniffing, otherwise elastic client fails to connect to docker node
// More details: https://github.com/olivere/elastic/wiki/Sniffing
*cfg.ElasticsearchSettings.Sniff = false
})
prevListenAddress := *th.App.Config().ServiceSettings.ListenAddress
Dockerized build updated tests (#9943) * testlib: introduce and leverage This doesn't yet factor out the individual test helpers: many packages still rely on `api4` directly to do this, but now wire up the test store setup through this package. `app` and `store`, in particular, don't use `testlib` because of circular dependencies at the moment. * cmd: command_test.go: use api4 testlib * cmd: plugin_test.go: remove dependence on test-config.json * cmd: config_test.go use configured database settings * ensure test-(te|ee) exit with status code * test-server: run all tests, deprecating test-te/test-ee * cmd/mattermost/commands: fix unit tests Instead of relying on (and modifying) a config.json found in the current path, explicitly create a temporary one from defaults for each test. This was likely the source of various bugs over time, but specifically allows us to override the SqlSettings to point at the configured test database for all tests simultaneously. * wrap run/check into a test helper It was insufficient to set a config for each invocation of CheckCommand or RunCommand: some tests relied on the config having changed in a subsequent assertion. Instead, create a new test helper embedding api4.TestHelper. This has the nice advantage of cleaning up all the teardown. * additional TestConfigGet granularity * customized config path to avoid default location * be explicit if the storetest initialization fails * generate safe coverprofile names in the presence of subtests * additional TestConfigShow granularity * fix permission_test.go typo * fix webhook tests * actually flag.Parse() to skip database setup on os.Execed tests * fix recent regression in #9962, not caught by unit tests
2018-12-06 13:19:32 -05:00
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" })
if updateConfig != nil {
th.App.UpdateConfig(updateConfig)
}
serverErr := th.Server.Start()
if serverErr != nil {
panic(serverErr)
}
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = prevListenAddress })
Init(th.Server, th.Server.AppOptions, th.App.Srv.Router)
web.New(th.Server, th.Server.AppOptions, th.App.Srv.Router)
wsapi.Init(th.App, th.App.Srv.WebSocketRouter)
th.App.Srv.Store.MarkSystemRanUnitTests()
th.App.DoAppMigrations()
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.EnableOpenServer = true })
// Disable strict password requirements for test
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.PasswordSettings.MinimumLength = 5
*cfg.PasswordSettings.Lowercase = false
*cfg.PasswordSettings.Uppercase = false
*cfg.PasswordSettings.Symbol = false
*cfg.PasswordSettings.Number = false
})
if enterprise {
th.App.SetLicense(model.NewTestLicense())
} else {
th.App.SetLicense(nil)
}
th.Client = th.CreateClient()
th.SystemAdminClient = th.CreateClient()
if th.tempWorkspace == "" {
dir, err := ioutil.TempDir("", "apptest")
if err != nil {
panic(err)
}
th.tempWorkspace = dir
}
return th
}
func SetupEnterprise() *TestHelper {
return setupTestHelper(true, nil)
}
func Setup() *TestHelper {
return setupTestHelper(false, nil)
}
func SetupConfig(updateConfig func(cfg *model.Config)) *TestHelper {
return setupTestHelper(false, updateConfig)
}
func (me *TestHelper) ShutdownApp() {
done := make(chan bool)
go func() {
me.Server.Shutdown()
close(done)
}()
select {
case <-done:
case <-time.After(30 * time.Second):
// panic instead of t.Fatal to terminate all tests in this package, otherwise the
// still running App could spuriously fail subsequent tests.
panic("failed to shutdown App within 30 seconds")
}
}
func (me *TestHelper) TearDown() {
utils.DisableDebugLogForTest()
me.ShutdownApp()
utils.EnableDebugLogForTest()
if err := recover(); err != nil {
panic(err)
}
}
func (me *TestHelper) InitBasic() *TestHelper {
me.waitForConnectivity()
me.SystemAdminUser = me.CreateUser()
me.App.UpdateUserRoles(me.SystemAdminUser.Id, model.SYSTEM_USER_ROLE_ID+" "+model.SYSTEM_ADMIN_ROLE_ID, false)
me.LoginSystemAdmin()
2017-01-31 09:31:53 -05:00
me.TeamAdminUser = me.CreateUser()
me.App.UpdateUserRoles(me.TeamAdminUser.Id, model.SYSTEM_USER_ROLE_ID, false)
2017-01-31 09:31:53 -05:00
me.LoginTeamAdmin()
2017-01-31 09:31:53 -05:00
me.BasicTeam = me.CreateTeam()
me.BasicChannel = me.CreatePublicChannel()
me.BasicPrivateChannel = me.CreatePrivateChannel()
me.BasicDeletedChannel = me.CreatePublicChannel()
me.BasicChannel2 = me.CreatePublicChannel()
me.BasicPost = me.CreatePost()
me.BasicUser = me.CreateUser()
me.LinkUserToTeam(me.BasicUser, me.BasicTeam)
2017-01-31 09:31:53 -05:00
me.BasicUser2 = me.CreateUser()
me.LinkUserToTeam(me.BasicUser2, me.BasicTeam)
2017-09-06 17:12:54 -05:00
me.App.AddUserToChannel(me.BasicUser, me.BasicChannel)
me.App.AddUserToChannel(me.BasicUser2, me.BasicChannel)
me.App.AddUserToChannel(me.BasicUser, me.BasicChannel2)
me.App.AddUserToChannel(me.BasicUser2, me.BasicChannel2)
me.App.AddUserToChannel(me.BasicUser, me.BasicPrivateChannel)
me.App.AddUserToChannel(me.BasicUser2, me.BasicPrivateChannel)
me.App.AddUserToChannel(me.BasicUser, me.BasicDeletedChannel)
me.App.AddUserToChannel(me.BasicUser2, me.BasicDeletedChannel)
me.App.UpdateUserRoles(me.BasicUser.Id, model.SYSTEM_USER_ROLE_ID, false)
me.Client.DeleteChannel(me.BasicDeletedChannel.Id)
me.LoginBasic()
me.Group = me.CreateGroup()
return me
}
func (me *TestHelper) waitForConnectivity() {
for i := 0; i < 1000; i++ {
2017-10-16 08:09:43 -07:00
conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%v", me.App.Srv.ListenAddr.Port))
if err == nil {
2017-10-16 08:09:43 -07:00
conn.Close()
return
}
time.Sleep(time.Millisecond * 20)
}
panic("unable to connect")
}
func (me *TestHelper) CreateClient() *model.Client4 {
2017-10-16 08:09:43 -07:00
return model.NewAPIv4Client(fmt.Sprintf("http://localhost:%v", me.App.Srv.ListenAddr.Port))
}
func (me *TestHelper) CreateWebSocketClient() (*model.WebSocketClient, *model.AppError) {
2017-10-16 08:09:43 -07:00
return model.NewWebSocketClient4(fmt.Sprintf("ws://localhost:%v", me.App.Srv.ListenAddr.Port), me.Client.AuthToken)
}
func (me *TestHelper) CreateWebSocketSystemAdminClient() (*model.WebSocketClient, *model.AppError) {
return model.NewWebSocketClient4(fmt.Sprintf("ws://localhost:%v", me.App.Srv.ListenAddr.Port), me.SystemAdminClient.AuthToken)
}
func (me *TestHelper) CreateUser() *model.User {
2017-01-31 09:31:53 -05:00
return me.CreateUserWithClient(me.Client)
}
func (me *TestHelper) CreateTeam() *model.Team {
return me.CreateTeamWithClient(me.Client)
}
func (me *TestHelper) CreateTeamWithClient(client *model.Client4) *model.Team {
id := model.NewId()
team := &model.Team{
DisplayName: "dn_" + id,
Name: GenerateTestTeamName(),
Email: me.GenerateTestEmail(),
2017-01-31 09:31:53 -05:00
Type: model.TEAM_OPEN,
}
utils.DisableDebugLogForTest()
rteam, resp := client.CreateTeam(team)
if resp.Error != nil {
panic(resp.Error)
}
2017-01-31 09:31:53 -05:00
utils.EnableDebugLogForTest()
return rteam
}
func (me *TestHelper) CreateUserWithClient(client *model.Client4) *model.User {
id := model.NewId()
user := &model.User{
Email: me.GenerateTestEmail(),
Username: GenerateTestUsername(),
Nickname: "nn_" + id,
FirstName: "f_" + id,
LastName: "l_" + id,
Password: "Pa$$word11",
}
utils.DisableDebugLogForTest()
ruser, response := client.CreateUser(user)
if response.Error != nil {
panic(response.Error)
}
ruser.Password = "Pa$$word11"
_, err := me.App.Srv.Store.User().VerifyEmail(ruser.Id, ruser.Email)
if err != nil {
return nil
}
utils.EnableDebugLogForTest()
return ruser
}
func (me *TestHelper) CreatePublicChannel() *model.Channel {
return me.CreateChannelWithClient(me.Client, model.CHANNEL_OPEN)
}
func (me *TestHelper) CreatePrivateChannel() *model.Channel {
return me.CreateChannelWithClient(me.Client, model.CHANNEL_PRIVATE)
}
func (me *TestHelper) CreateChannelWithClient(client *model.Client4, channelType string) *model.Channel {
return me.CreateChannelWithClientAndTeam(client, channelType, me.BasicTeam.Id)
}
func (me *TestHelper) CreateChannelWithClientAndTeam(client *model.Client4, channelType string, teamId string) *model.Channel {
id := model.NewId()
channel := &model.Channel{
DisplayName: "dn_" + id,
Name: GenerateTestChannelName(),
Type: channelType,
TeamId: teamId,
}
utils.DisableDebugLogForTest()
rchannel, resp := client.CreateChannel(channel)
if resp.Error != nil {
panic(resp.Error)
}
utils.EnableDebugLogForTest()
return rchannel
}
func (me *TestHelper) CreatePost() *model.Post {
return me.CreatePostWithClient(me.Client, me.BasicChannel)
}
func (me *TestHelper) CreatePinnedPost() *model.Post {
return me.CreatePinnedPostWithClient(me.Client, me.BasicChannel)
}
func (me *TestHelper) CreateMessagePost(message string) *model.Post {
return me.CreateMessagePostWithClient(me.Client, me.BasicChannel, message)
}
func (me *TestHelper) CreatePostWithClient(client *model.Client4, channel *model.Channel) *model.Post {
id := model.NewId()
post := &model.Post{
ChannelId: channel.Id,
Message: "message_" + id,
}
utils.DisableDebugLogForTest()
rpost, resp := client.CreatePost(post)
if resp.Error != nil {
panic(resp.Error)
}
utils.EnableDebugLogForTest()
return rpost
}
func (me *TestHelper) CreatePinnedPostWithClient(client *model.Client4, channel *model.Channel) *model.Post {
id := model.NewId()
post := &model.Post{
ChannelId: channel.Id,
Message: "message_" + id,
IsPinned: true,
}
utils.DisableDebugLogForTest()
rpost, resp := client.CreatePost(post)
if resp.Error != nil {
panic(resp.Error)
}
utils.EnableDebugLogForTest()
return rpost
}
func (me *TestHelper) CreateMessagePostWithClient(client *model.Client4, channel *model.Channel, message string) *model.Post {
post := &model.Post{
ChannelId: channel.Id,
Message: message,
}
utils.DisableDebugLogForTest()
rpost, resp := client.CreatePost(post)
if resp.Error != nil {
panic(resp.Error)
}
utils.EnableDebugLogForTest()
return rpost
}
func (me *TestHelper) CreateMessagePostNoClient(channel *model.Channel, message string, createAtTime int64) *model.Post {
post, err := me.App.Srv.Store.Post().Save(&model.Post{
UserId: me.BasicUser.Id,
ChannelId: channel.Id,
Message: message,
CreateAt: createAtTime,
})
if err != nil {
panic(err)
}
return post
}
func (me *TestHelper) CreateDmChannel(user *model.User) *model.Channel {
utils.DisableDebugLogForTest()
var err *model.AppError
var channel *model.Channel
if channel, err = me.App.GetOrCreateDirectChannel(me.BasicUser.Id, user.Id); err != nil {
mlog.Error(err.Error())
time.Sleep(time.Second)
panic(err)
}
utils.EnableDebugLogForTest()
return channel
}
func (me *TestHelper) LoginBasic() {
2017-01-31 09:31:53 -05:00
me.LoginBasicWithClient(me.Client)
}
func (me *TestHelper) LoginBasic2() {
me.LoginBasic2WithClient(me.Client)
}
func (me *TestHelper) LoginTeamAdmin() {
me.LoginTeamAdminWithClient(me.Client)
}
func (me *TestHelper) LoginSystemAdmin() {
me.LoginSystemAdminWithClient(me.SystemAdminClient)
}
func (me *TestHelper) LoginBasicWithClient(client *model.Client4) {
utils.DisableDebugLogForTest()
_, resp := client.Login(me.BasicUser.Email, me.BasicUser.Password)
if resp.Error != nil {
panic(resp.Error)
}
utils.EnableDebugLogForTest()
}
2017-01-31 09:31:53 -05:00
func (me *TestHelper) LoginBasic2WithClient(client *model.Client4) {
utils.DisableDebugLogForTest()
_, resp := client.Login(me.BasicUser2.Email, me.BasicUser2.Password)
if resp.Error != nil {
panic(resp.Error)
}
2017-01-31 09:31:53 -05:00
utils.EnableDebugLogForTest()
}
func (me *TestHelper) LoginTeamAdminWithClient(client *model.Client4) {
utils.DisableDebugLogForTest()
_, resp := client.Login(me.TeamAdminUser.Email, me.TeamAdminUser.Password)
if resp.Error != nil {
panic(resp.Error)
}
2017-01-31 09:31:53 -05:00
utils.EnableDebugLogForTest()
}
func (me *TestHelper) LoginSystemAdminWithClient(client *model.Client4) {
utils.DisableDebugLogForTest()
_, resp := client.Login(me.SystemAdminUser.Email, me.SystemAdminUser.Password)
if resp.Error != nil {
panic(resp.Error)
}
2017-01-31 09:31:53 -05:00
utils.EnableDebugLogForTest()
}
func (me *TestHelper) UpdateActiveUser(user *model.User, active bool) {
utils.DisableDebugLogForTest()
_, err := me.App.UpdateActive(user, active)
if err != nil {
mlog.Error(err.Error())
time.Sleep(time.Second)
panic(err)
}
utils.EnableDebugLogForTest()
}
func (me *TestHelper) LinkUserToTeam(user *model.User, team *model.Team) {
2017-01-31 09:31:53 -05:00
utils.DisableDebugLogForTest()
err := me.App.JoinUserToTeam(team, user, "")
2017-01-31 09:31:53 -05:00
if err != nil {
mlog.Error(err.Error())
2017-01-31 09:31:53 -05:00
time.Sleep(time.Second)
panic(err)
}
utils.EnableDebugLogForTest()
}
func (me *TestHelper) AddUserToChannel(user *model.User, channel *model.Channel) *model.ChannelMember {
utils.DisableDebugLogForTest()
member, err := me.App.AddUserToChannel(user, channel)
if err != nil {
mlog.Error(err.Error())
time.Sleep(time.Second)
panic(err)
}
utils.EnableDebugLogForTest()
return member
}
func (me *TestHelper) GenerateTestEmail() string {
if *me.App.Config().EmailSettings.SMTPServer != "localhost" && os.Getenv("CI_INBUCKET_PORT") == "" {
2017-10-18 14:13:14 -07:00
return strings.ToLower("success+" + model.NewId() + "@simulator.amazonses.com")
}
return strings.ToLower(model.NewId() + "@localhost")
}
func (me *TestHelper) CreateGroup() *model.Group {
id := model.NewId()
group := &model.Group{
Name: "n-" + id,
DisplayName: "dn_" + id,
Source: model.GroupSourceLdap,
RemoteId: "ri_" + id,
}
utils.DisableDebugLogForTest()
group, err := me.App.CreateGroup(group)
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return group
}
func GenerateTestUsername() string {
return "fakeuser" + model.NewRandomString(10)
2017-01-31 09:31:53 -05:00
}
func GenerateTestTeamName() string {
return "faketeam" + model.NewRandomString(6)
}
func GenerateTestChannelName() string {
return "fakechannel" + model.NewRandomString(10)
}
func GenerateTestAppName() string {
return "fakeoauthapp" + model.NewRandomString(10)
}
func GenerateTestId() string {
return model.NewId()
}
func CheckUserSanitization(t *testing.T, user *model.User) {
t.Helper()
if user.Password != "" {
t.Fatal("password wasn't blank")
}
if user.AuthData != nil && *user.AuthData != "" {
t.Fatal("auth data wasn't blank")
}
if user.MfaSecret != "" {
t.Fatal("mfa secret wasn't blank")
}
}
func CheckEtag(t *testing.T, data interface{}, resp *model.Response) {
t.Helper()
if !reflect.ValueOf(data).IsNil() {
t.Fatal("etag data was not nil")
}
if resp.StatusCode != http.StatusNotModified {
t.Log("actual: " + strconv.Itoa(resp.StatusCode))
t.Log("expected: " + strconv.Itoa(http.StatusNotModified))
t.Fatal("wrong status code for etag")
}
}
func CheckNoError(t *testing.T, resp *model.Response) {
t.Helper()
if resp.Error != nil {
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
t.Fatalf("Expected no error, got %q", resp.Error.Error())
}
}
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
func checkHTTPStatus(t *testing.T, resp *model.Response, expectedStatus int, expectError bool) {
t.Helper()
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
switch {
case resp == nil:
t.Fatalf("Unexpected nil response, expected http:%v, expectError:%v)", expectedStatus, expectError)
case expectError && resp.Error == nil:
t.Fatalf("Expected a non-nil error and http status:%v, got nil, %v", expectedStatus, resp.StatusCode)
case !expectError && resp.Error != nil:
t.Fatalf("Expected no error and http status:%v, got %q, http:%v", expectedStatus, resp.Error, resp.StatusCode)
case resp.StatusCode != expectedStatus:
t.Fatalf("Expected http status:%v, got %v (err: %q)", expectedStatus, resp.StatusCode, resp.Error)
}
}
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
func CheckOKStatus(t *testing.T, resp *model.Response) {
t.Helper()
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
checkHTTPStatus(t, resp, http.StatusOK, false)
}
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
func CheckCreatedStatus(t *testing.T, resp *model.Response) {
t.Helper()
checkHTTPStatus(t, resp, http.StatusCreated, false)
}
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
func CheckForbiddenStatus(t *testing.T, resp *model.Response) {
t.Helper()
checkHTTPStatus(t, resp, http.StatusForbidden, true)
}
func CheckUnauthorizedStatus(t *testing.T, resp *model.Response) {
t.Helper()
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
checkHTTPStatus(t, resp, http.StatusUnauthorized, true)
}
func CheckNotFoundStatus(t *testing.T, resp *model.Response) {
t.Helper()
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
checkHTTPStatus(t, resp, http.StatusNotFound, true)
}
func CheckBadRequestStatus(t *testing.T, resp *model.Response) {
t.Helper()
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
checkHTTPStatus(t, resp, http.StatusBadRequest, true)
}
func CheckNotImplementedStatus(t *testing.T, resp *model.Response) {
t.Helper()
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
checkHTTPStatus(t, resp, http.StatusNotImplemented, true)
}
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
func CheckRequestEntityTooLargeStatus(t *testing.T, resp *model.Response) {
t.Helper()
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
checkHTTPStatus(t, resp, http.StatusRequestEntityTooLarge, true)
}
MM-7633: Optimize memory utilization during file uploads (#9835) * MM-7633: Optimize memory utilization during file uploads Refactored the file upload code to reduce redundant buffering and stream directly to the file store. Added tests. Benchmark results: ``` levs-mbp:mattermost-server levb$ go test -v -run nothing -bench Upload -benchmem ./app ... BenchmarkUploadFile/random-5Mb-gif-raw-ish_DoUploadFile-4 10 122598031 ns/op 21211370 B/op 1008 allocs/op BenchmarkUploadFile/random-5Mb-gif-raw_UploadFileTask-4 100 20211926 ns/op 5678750 B/op 126 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFiles-4 2 1037051184 ns/op 81806360 B/op 3705013 allocs/op BenchmarkUploadFile/random-5Mb-gif-UploadFileTask-4 2 933644431 ns/op 67015868 B/op 3704410 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw-ish_DoUploadFile-4 100 13110509 ns/op 6032614 B/op 8052 allocs/op BenchmarkUploadFile/random-2Mb-jpg-raw_UploadFileTask-4 100 10729867 ns/op 1738303 B/op 125 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFiles-4 2 925274912 ns/op 70326352 B/op 3718856 allocs/op BenchmarkUploadFile/random-2Mb-jpg-UploadFileTask-4 2 995033336 ns/op 58113796 B/op 3710943 allocs/op BenchmarkUploadFile/zero-10Mb-raw-ish_DoUploadFile-4 30 50777211 ns/op 54791929 B/op 2714 allocs/op BenchmarkUploadFile/zero-10Mb-raw_UploadFileTask-4 50 36387339 ns/op 10503920 B/op 126 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFiles-4 30 48657678 ns/op 54791948 B/op 2719 allocs/op BenchmarkUploadFile/zero-10Mb-UploadFileTask-4 50 37506467 ns/op 31492060 B/op 131 allocs/op ... ``` https://mattermost.atlassian.net/browse/MM-7633 https://github.com/mattermost/mattermost-server/issues/7801 [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Added or updated unit tests (required for all new features) - [ ] Added API documentation (required for all new APIs) - [ ] All new/modified APIs include changes to the drivers *N/A*??? - [x] Includes text changes and localization file ([.../i18n/en.json](https://github.com/mattermost/mattermost-server/blob/master/i18n/en.json)) updates Overview of changes: - api4 - Replaced `uploadFile` handler with `uploadFileStream` that reduces unnecessary buffering. - Added/refactored tests for the new API. - Refactored apitestlib/Check...Status functions. - app - Added App.UploadFileTask, a more efficient refactor of UploadFile. - Consistently set `FileInfo.HasPreviewImage` - Added benchmarks for the new and prior implementations - Replaced passing around `*image.Image` with `image.Image` in the existing code. - model - Added a more capable `client4.UploadFiles` API to match the new server API’s capabilities. - I18n - Replaced `api.file.upload_file.bad_parse.app_error` with a more generic `api.file.upload_file.read_request.app_error` - plugin - Removed type `plugin.multiPluginHookRunnerFunc` in favor of using `func(hooks Hooks) bool` explicitly, to help with testing - tests - Added test files for testing images Still remaining, but can be separate PRs - please let me know the preferred course of action - Investigate JS client API - how does it do multipart? - Performance loss from old code on (small) image processing? - Deprecate the old functions, change other API implementations to use UploadFileTask Definitely separate future PRs - should I file tickets foe these? - Only invoke t.readAll() if there are indeed applicable plugins to run - Find a way to leverage goexif buffer rather than re-reading Suggested long-term improvements - should I file separate tickets for these? - Actually allow uploading of large (GB-sized) files. This may require a change in how the file is passed to plugins. - (Many) api4 tests should probably be subtests and share a server setup - will be much faster - Performance improvements in image processing (goexif/thumbnail/preview) (maybe use https://mattermost.atlassian.net/browse/MM-10188 for this) Questions: 1. I am commiting MBs of test images, are there better alternatives? I can probably create much less dense images that would take up considerably less space, even at pretty large sizes 2. I18n: Do I need to do anything special for the string change? Or just wait until it gets picked up and translated/updated? 3. The image dimensions are flipped in resulting FileInfo to match the actual orientation. Is this by design? Should add a test for it, perhaps? 4. What to do in the case of partial success? S3 but not DB, some files but not others? For now, just doing what the old code did, I think. 5. Make maxUploadDrainBytes configurable? Also, should this be the systemic behavior of all APIs with non-empty body? Otherwise dropped altogether? Check all other ioutil.ReadAll() from sockets. Find a way to set a total byte limit on request Body? * WIP - Fixed for GetPluginsEnvironment() changes * WIP - PR feedback 1. Refactored checkHTTPStatus to improve failure messages 2. Use `var name []type` rather than `name := ([]type)(nil)` 3. Replaced single-letter `p` with a more intention-revealing `part` 4. Added tests for full image size, `HasPreviewImage` * WIP - rebased (c.Session->c.App.Session) * WIP - PR feedback: eliminated use of Request.MultipartReader Instead of hacking the request object to use r.MultipartReader now have own functions `parseMultipartRequestHeader` and `multipartReader` eliminating the need to hack the request object to use Request.MultipartReader limitations. * WIP - PR feedback: UploadFileX with functional options * WIP - PR feedback: style * WIP - PR feedback: errors cleanup * WIP - clarified File Upload benchmarks * WIP - PR feedback: display the value of erroneous formname * WIP - PR feedback: fixed handling of multiple channel_ids * WIP - rebased from master - fixed tests * PR Feedback * PR feedback - moved client4.UploadFiles to _test for now
2018-12-13 13:32:07 -08:00
func CheckInternalErrorStatus(t *testing.T, resp *model.Response) {
t.Helper()
checkHTTPStatus(t, resp, http.StatusInternalServerError, true)
}
2017-01-31 09:31:53 -05:00
func CheckErrorMessage(t *testing.T, resp *model.Response, errorId string) {
t.Helper()
if resp.Error == nil {
2017-01-31 09:31:53 -05:00
t.Fatal("should have errored with message:" + errorId)
return
}
2017-01-31 09:31:53 -05:00
if resp.Error.Id != errorId {
t.Log("actual: " + resp.Error.Id)
t.Log("expected: " + errorId)
t.Fatal("incorrect error message")
}
}
// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
// If signV2 input is false, function always returns signature v4.
//
// Additionally this function also takes a user defined region, if set
// disables automatic region lookup.
func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) {
var creds *credentials.Credentials
if signV2 {
creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2)
} else {
creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4)
}
return s3.NewWithCredentials(endpoint, creds, secure, region)
}
2017-11-09 14:46:20 -06:00
func (me *TestHelper) cleanupTestFile(info *model.FileInfo) error {
cfg := me.App.Config()
if *cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := *cfg.FileSettings.AmazonS3Endpoint
accessKey := *cfg.FileSettings.AmazonS3AccessKeyId
secretKey := *cfg.FileSettings.AmazonS3SecretAccessKey
2017-11-09 14:46:20 -06:00
secure := *cfg.FileSettings.AmazonS3SSL
signV2 := *cfg.FileSettings.AmazonS3SignV2
region := *cfg.FileSettings.AmazonS3Region
s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
if err != nil {
return err
}
bucket := *cfg.FileSettings.AmazonS3Bucket
if err := s3Clnt.RemoveObject(bucket, info.Path); err != nil {
return err
}
if info.ThumbnailPath != "" {
if err := s3Clnt.RemoveObject(bucket, info.ThumbnailPath); err != nil {
return err
}
}
if info.PreviewPath != "" {
if err := s3Clnt.RemoveObject(bucket, info.PreviewPath); err != nil {
return err
}
}
2017-11-09 14:46:20 -06:00
} else if *cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := os.Remove(*cfg.FileSettings.Directory + info.Path); err != nil {
return err
}
if info.ThumbnailPath != "" {
if err := os.Remove(*cfg.FileSettings.Directory + info.ThumbnailPath); err != nil {
return err
}
}
if info.PreviewPath != "" {
if err := os.Remove(*cfg.FileSettings.Directory + info.PreviewPath); err != nil {
return err
}
}
}
return nil
}
func (me *TestHelper) MakeUserChannelAdmin(user *model.User, channel *model.Channel) {
utils.DisableDebugLogForTest()
if cm, err := me.App.Srv.Store.Channel().GetMember(channel.Id, user.Id); err == nil {
cm.SchemeAdmin = true
if _, err = me.App.Srv.Store.Channel().UpdateMember(cm); err != nil {
utils.EnableDebugLogForTest()
panic(err)
}
} else {
utils.EnableDebugLogForTest()
panic(err)
}
utils.EnableDebugLogForTest()
}
func (me *TestHelper) UpdateUserToTeamAdmin(user *model.User, team *model.Team) {
utils.DisableDebugLogForTest()
if tm, err := me.App.Srv.Store.Team().GetMember(team.Id, user.Id); err == nil {
tm.SchemeAdmin = true
if _, err = me.App.Srv.Store.Team().UpdateMember(tm); err != nil {
utils.EnableDebugLogForTest()
panic(err)
}
} else {
utils.EnableDebugLogForTest()
mlog.Error(err.Error())
time.Sleep(time.Second)
panic(err)
}
utils.EnableDebugLogForTest()
}
func (me *TestHelper) UpdateUserToNonTeamAdmin(user *model.User, team *model.Team) {
utils.DisableDebugLogForTest()
if tm, err := me.App.Srv.Store.Team().GetMember(team.Id, user.Id); err == nil {
tm.SchemeAdmin = false
if _, err = me.App.Srv.Store.Team().UpdateMember(tm); err != nil {
utils.EnableDebugLogForTest()
panic(err)
}
} else {
utils.EnableDebugLogForTest()
mlog.Error(err.Error())
time.Sleep(time.Second)
panic(err)
}
utils.EnableDebugLogForTest()
}
func (me *TestHelper) SaveDefaultRolePermissions() map[string][]string {
utils.DisableDebugLogForTest()
results := make(map[string][]string)
for _, roleName := range []string{
"system_user",
"system_admin",
"team_user",
"team_admin",
"channel_user",
"channel_admin",
} {
role, err1 := me.App.GetRoleByName(roleName)
if err1 != nil {
utils.EnableDebugLogForTest()
panic(err1)
}
results[roleName] = role.Permissions
}
utils.EnableDebugLogForTest()
return results
}
func (me *TestHelper) RestoreDefaultRolePermissions(data map[string][]string) {
utils.DisableDebugLogForTest()
for roleName, permissions := range data {
role, err1 := me.App.GetRoleByName(roleName)
if err1 != nil {
utils.EnableDebugLogForTest()
panic(err1)
}
if strings.Join(role.Permissions, " ") == strings.Join(permissions, " ") {
continue
}
role.Permissions = permissions
_, err2 := me.App.UpdateRole(role)
if err2 != nil {
utils.EnableDebugLogForTest()
panic(err2)
}
}
utils.EnableDebugLogForTest()
}
func (me *TestHelper) RemovePermissionFromRole(permission string, roleName string) {
utils.DisableDebugLogForTest()
role, err1 := me.App.GetRoleByName(roleName)
if err1 != nil {
utils.EnableDebugLogForTest()
panic(err1)
}
var newPermissions []string
for _, p := range role.Permissions {
if p != permission {
newPermissions = append(newPermissions, p)
}
}
if strings.Join(role.Permissions, " ") == strings.Join(newPermissions, " ") {
utils.EnableDebugLogForTest()
return
}
role.Permissions = newPermissions
_, err2 := me.App.UpdateRole(role)
if err2 != nil {
utils.EnableDebugLogForTest()
panic(err2)
}
utils.EnableDebugLogForTest()
}
func (me *TestHelper) AddPermissionToRole(permission string, roleName string) {
utils.DisableDebugLogForTest()
role, err1 := me.App.GetRoleByName(roleName)
if err1 != nil {
utils.EnableDebugLogForTest()
panic(err1)
}
for _, existingPermission := range role.Permissions {
if existingPermission == permission {
utils.EnableDebugLogForTest()
return
}
}
role.Permissions = append(role.Permissions, permission)
_, err2 := me.App.UpdateRole(role)
if err2 != nil {
utils.EnableDebugLogForTest()
panic(err2)
}
utils.EnableDebugLogForTest()
}