backend.go 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142
  1. package backend
  2. import (
  3. "github.com/seaweedfs/seaweedfs/weed/util"
  4. "io"
  5. "os"
  6. "strings"
  7. "time"
  8. "github.com/seaweedfs/seaweedfs/weed/glog"
  9. "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
  10. "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
  11. )
  12. type BackendStorageFile interface {
  13. io.ReaderAt
  14. io.WriterAt
  15. Truncate(off int64) error
  16. io.Closer
  17. GetStat() (datSize int64, modTime time.Time, err error)
  18. Name() string
  19. Sync() error
  20. }
  21. type BackendStorage interface {
  22. ToProperties() map[string]string
  23. NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) BackendStorageFile
  24. CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error)
  25. DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error) (size int64, err error)
  26. DeleteFile(key string) (err error)
  27. }
  28. type StringProperties interface {
  29. GetString(key string) string
  30. }
  31. type StorageType string
  32. type BackendStorageFactory interface {
  33. StorageType() StorageType
  34. BuildStorage(configuration StringProperties, configPrefix string, id string) (BackendStorage, error)
  35. }
  36. var (
  37. BackendStorageFactories = make(map[StorageType]BackendStorageFactory)
  38. BackendStorages = make(map[string]BackendStorage)
  39. )
  40. // used by master to load remote storage configurations
  41. func LoadConfiguration(config *util.ViperProxy) {
  42. StorageBackendPrefix := "storage.backend"
  43. for backendTypeName := range config.GetStringMap(StorageBackendPrefix) {
  44. backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)]
  45. if !found {
  46. glog.Fatalf("backend storage type %s not found", backendTypeName)
  47. }
  48. for backendStorageId := range config.GetStringMap(StorageBackendPrefix + "." + backendTypeName) {
  49. if !config.GetBool(StorageBackendPrefix + "." + backendTypeName + "." + backendStorageId + ".enabled") {
  50. continue
  51. }
  52. if _, found := BackendStorages[backendTypeName+"."+backendStorageId]; found {
  53. continue
  54. }
  55. backendStorage, buildErr := backendStorageFactory.BuildStorage(config,
  56. StorageBackendPrefix+"."+backendTypeName+"."+backendStorageId+".", backendStorageId)
  57. if buildErr != nil {
  58. glog.Fatalf("fail to create backend storage %s.%s", backendTypeName, backendStorageId)
  59. }
  60. BackendStorages[backendTypeName+"."+backendStorageId] = backendStorage
  61. if backendStorageId == "default" {
  62. BackendStorages[backendTypeName] = backendStorage
  63. }
  64. }
  65. }
  66. }
  67. // used by volume server to receive remote storage configurations from master
  68. func LoadFromPbStorageBackends(storageBackends []*master_pb.StorageBackend) {
  69. for _, storageBackend := range storageBackends {
  70. backendStorageFactory, found := BackendStorageFactories[StorageType(storageBackend.Type)]
  71. if !found {
  72. glog.Warningf("storage type %s not found", storageBackend.Type)
  73. continue
  74. }
  75. if _, found := BackendStorages[storageBackend.Type+"."+storageBackend.Id]; found {
  76. continue
  77. }
  78. backendStorage, buildErr := backendStorageFactory.BuildStorage(newProperties(storageBackend.Properties), "", storageBackend.Id)
  79. if buildErr != nil {
  80. glog.Fatalf("fail to create backend storage %s.%s", storageBackend.Type, storageBackend.Id)
  81. }
  82. BackendStorages[storageBackend.Type+"."+storageBackend.Id] = backendStorage
  83. if storageBackend.Id == "default" {
  84. BackendStorages[storageBackend.Type] = backendStorage
  85. }
  86. }
  87. }
  88. type Properties struct {
  89. m map[string]string
  90. }
  91. func newProperties(m map[string]string) *Properties {
  92. return &Properties{m: m}
  93. }
  94. func (p *Properties) GetString(key string) string {
  95. if v, found := p.m[key]; found {
  96. return v
  97. }
  98. return ""
  99. }
  100. func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
  101. for sName, s := range BackendStorages {
  102. sType, sId := BackendNameToTypeId(sName)
  103. if sType == "" {
  104. continue
  105. }
  106. backends = append(backends, &master_pb.StorageBackend{
  107. Type: sType,
  108. Id: sId,
  109. Properties: s.ToProperties(),
  110. })
  111. }
  112. return
  113. }
  114. func BackendNameToTypeId(backendName string) (backendType, backendId string) {
  115. parts := strings.Split(backendName, ".")
  116. if len(parts) == 1 {
  117. return backendName, "default"
  118. }
  119. if len(parts) != 2 {
  120. return
  121. }
  122. backendType, backendId = parts[0], parts[1]
  123. return
  124. }