IntenseWebs/gitea
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Storage configuration support [storage] (#13314)

Browse Source 
* Fix minio bug

* Add tests for storage configuration

* Change the Seek flag to keep compitable minio?

* Fix test when first-byte-pos of all ranges is greater than the resource length

Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
Lunny Xiao 2020-11-01 04:51:48 +08:00 committed by GitHub
parent e7750e0f6a
commit e4e85a3e51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 62 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
integrations/integration_test.go
View File

@ -11,7 +11,6 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/http/cookiejar"
"net/http/httptest"
@ -27,8 +26,10 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/routers"
"code.gitea.io/gitea/routers/routes"
@ -59,6 +60,8 @@ func NewNilResponseRecorder() *NilResponseRecorder {
}
func TestMain(m *testing.M) {
defer log.Close()
managerCtx, cancel := context.WithCancel(context.Background())
graceful.InitManager(managerCtx)
defer cancel()
@ -142,6 +145,10 @@ func initIntegrationTest() {
util.RemoveAll(models.LocalCopyPath())
setting.CheckLFSVersion()
setting.InitDBConfig()
if err := storage.Init(); err != nil {
fmt.Printf("Init storage failed: %v", err)
os.Exit(1)
}
switch {
case setting.Database.UseMySQL:
@ -149,27 +156,27 @@ func initIntegrationTest() {
setting.Database.User, setting.Database.Passwd, setting.Database.Host))
defer db.Close()
if err != nil {
log.Fatalf("sql.Open: %v", err)
log.Fatal("sql.Open: %v", err)
}
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
log.Fatalf("db.Exec: %v", err)
log.Fatal("db.Exec: %v", err)
}
case setting.Database.UsePostgreSQL:
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
defer db.Close()
if err != nil {
log.Fatalf("sql.Open: %v", err)
log.Fatal("sql.Open: %v", err)
}
dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
if err != nil {
log.Fatalf("db.Query: %v", err)
log.Fatal("db.Query: %v", err)
}
defer dbrows.Close()
if !dbrows.Next() {
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
log.Fatalf("db.Exec: CREATE DATABASE: %v", err)
log.Fatal("db.Exec: CREATE DATABASE: %v", err)
}
}
// Check if we need to setup a specific schema
@ -183,18 +190,18 @@ func initIntegrationTest() {
// This is a different db object; requires a different Close()
defer db.Close()
if err != nil {
log.Fatalf("sql.Open: %v", err)
log.Fatal("sql.Open: %v", err)
}
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
if err != nil {
log.Fatalf("db.Query: %v", err)
log.Fatal("db.Query: %v", err)
}
defer schrows.Close()
if !schrows.Next() {
// Create and setup a DB schema
if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
log.Fatalf("db.Exec: CREATE SCHEMA: %v", err)
log.Fatal("db.Exec: CREATE SCHEMA: %v", err)
}
}
@ -203,10 +210,10 @@ func initIntegrationTest() {
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
host, port, "master", setting.Database.User, setting.Database.Passwd))
if err != nil {
log.Fatalf("sql.Open: %v", err)
log.Fatal("sql.Open: %v", err)
}
if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil {
log.Fatalf("db.Exec: %v", err)
log.Fatal("db.Exec: %v", err)
}
defer db.Close()
}

3
integrations/lfs_getobject_test.go
View File

@ -78,6 +78,7 @@ func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, exp
}
}
}
resp := session.MakeRequest(t, req, expectedStatus)
return resp
@ -210,7 +211,7 @@ func TestGetLFSRange(t *testing.T) {
{"bytes=0-10", "123456789\n", http.StatusPartialContent},
// end-range bigger than length-1 is ignored
{"bytes=0-11", "123456789\n", http.StatusPartialContent},
{"bytes=11-", "", http.StatusPartialContent},
{"bytes=11-", "Requested Range Not Satisfiable", http.StatusRequestedRangeNotSatisfiable},
// incorrect header value cause whole header to be ignored
{"bytes=-", "123456789\n", http.StatusOK},
{"foobar", "123456789\n", http.StatusOK},

26
integrations/mysql.ini.tmpl
View File

@ -45,19 +45,21 @@ START_SSH_SERVER = true
OFFLINE_MODE = false
LFS_START_SERVER = true
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
LFS_STORE_TYPE = minio
LFS_SERVE_DIRECT = false
LFS_MINIO_ENDPOINT = minio:9000
LFS_MINIO_ACCESS_KEY_ID = 123456
LFS_MINIO_SECRET_ACCESS_KEY = 12345678
LFS_MINIO_BUCKET = gitea
LFS_MINIO_LOCATION = us-east-1
LFS_MINIO_BASE_PATH = lfs/
LFS_MINIO_USE_SSL = false
[lfs]
MINIO_BASE_PATH = lfs/
[attachment]
MINIO_BASE_PATH = attachments/
[avatars]
MINIO_BASE_PATH = avatars/
[repo-avatars]
MINIO_BASE_PATH = repo-avatars/
[storage]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = minio:9000
@ -65,7 +67,6 @@ MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = gitea
MINIO_LOCATION = us-east-1
MINIO_BASE_PATH = attachments/
MINIO_USE_SSL = false
[mailer]
@ -88,9 +89,6 @@ ENABLE_NOTIFY_MAIL = true
DISABLE_GRAVATAR = false
ENABLE_FEDERATED_AVATAR = false
AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/avatars
REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/repo-avatars
[session]
PROVIDER = file
PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions

23
modules/lfs/content_store.go
View File

@ -8,6 +8,7 @@ import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
@ -21,6 +22,21 @@ var (
errSizeMismatch = errors.New("Content size does not match")
)
// ErrRangeNotSatisfiable represents an error which request range is not satisfiable.
type ErrRangeNotSatisfiable struct {
FromByte int64
}
func (err ErrRangeNotSatisfiable) Error() string {
return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
}
// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable
func IsErrRangeNotSatisfiable(err error) bool {
_, ok := err.(ErrRangeNotSatisfiable)
return ok
}
// ContentStore provides a simple file system based storage.
type ContentStore struct {
storage.ObjectStorage
@ -35,7 +51,12 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
return nil, err
}
if fromByte > 0 {
_, err = f.Seek(fromByte, os.SEEK_CUR)
if fromByte >= meta.Size {
return nil, ErrRangeNotSatisfiable{
FromByte: fromByte,
}
}
_, err = f.Seek(fromByte, io.SeekStart)
if err != nil {
log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
}

4
modules/lfs/server.go
View File

@ -191,8 +191,12 @@ func getContentHandler(ctx *context.Context) {
contentStore := &ContentStore{ObjectStorage: storage.LFS}
content, err := contentStore.Get(meta, fromByte)
if err != nil {
if IsErrRangeNotSatisfiable(err) {
writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
} else {
// Errors are logged in contentStore.Get
writeStatus(ctx, 404)
}
return
}
defer content.Close()

6
modules/setting/storage.go
View File

@ -32,14 +32,12 @@ func (s *Storage) MapTo(v interface{}) error {
}
func getStorage(name, typ string, overrides ...*ini.Section) Storage {
sectionName := "storage"
if len(name) > 0 {
sectionName = sectionName + "." + typ
}
const sectionName = "storage"
sec := Cfg.Section(sectionName)
if len(overrides) == 0 {
overrides = []*ini.Section{
Cfg.Section(sectionName + "." + typ),
Cfg.Section(sectionName + "." + name),
}
}