command_remote_configure.go 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213
  1. package shell
  2. import (
  3. "context"
  4. "flag"
  5. "fmt"
  6. "github.com/seaweedfs/seaweedfs/weed/filer"
  7. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  8. "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
  9. "github.com/seaweedfs/seaweedfs/weed/remote_storage"
  10. "github.com/seaweedfs/seaweedfs/weed/util"
  11. "google.golang.org/protobuf/proto"
  12. "io"
  13. "regexp"
  14. "strings"
  15. )
  16. func init() {
  17. Commands = append(Commands, &commandRemoteConfigure{})
  18. }
  19. type commandRemoteConfigure struct {
  20. }
  21. func (c *commandRemoteConfigure) Name() string {
  22. return "remote.configure"
  23. }
  24. func (c *commandRemoteConfigure) Help() string {
  25. return `remote storage configuration
  26. # see the current configurations
  27. remote.configure
  28. # set or update a configuration
  29. remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy -s3.region=us-east-2
  30. remote.configure -name=cloud2 -type=gcs -gcs.appCredentialsFile=~/service-account-file.json -gcs.projectId=yyy
  31. remote.configure -name=cloud3 -type=azure -azure.account_name=xxx -azure.account_key=yyy
  32. remote.configure -name=cloud4 -type=aliyun -aliyun.access_key=xxx -aliyun.secret_key=yyy -aliyun.endpoint=oss-cn-shenzhen.aliyuncs.com -aliyun.region=cn-sehnzhen
  33. remote.configure -name=cloud5 -type=tencent -tencent.secret_id=xxx -tencent.secret_key=yyy -tencent.endpoint=cos.ap-guangzhou.myqcloud.com
  34. remote.configure -name=cloud6 -type=wasabi -wasabi.access_key=xxx -wasabi.secret_key=yyy -wasabi.endpoint=s3.us-west-1.wasabisys.com -wasabi.region=us-west-1
  35. remote.configure -name=cloud7 -type=storj -storj.access_key=xxx -storj.secret_key=yyy -storj.endpoint=https://gateway.us1.storjshare.io
  36. remote.configure -name=cloud8 -type=filebase -filebase.access_key=xxx -filebase.secret_key=yyy -filebase.endpoint=https://s3.filebase.com
  37. # delete one configuration
  38. remote.configure -delete -name=cloud1
  39. `
  40. }
  41. func (c *commandRemoteConfigure) HasTag(CommandTag) bool {
  42. return false
  43. }
  44. var (
  45. isAlpha = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9]*$`).MatchString
  46. )
  47. func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
  48. conf := &remote_pb.RemoteConf{}
  49. remoteConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
  50. isDelete := remoteConfigureCommand.Bool("delete", false, "delete one remote storage by its name")
  51. remoteConfigureCommand.StringVar(&conf.Name, "name", "", "a short name to identify the remote storage")
  52. remoteConfigureCommand.StringVar(&conf.Type, "type", "s3", fmt.Sprintf("[%s] storage type", remote_storage.GetAllRemoteStorageNames()))
  53. remoteConfigureCommand.StringVar(&conf.S3AccessKey, "s3.access_key", "", "s3 access key")
  54. remoteConfigureCommand.StringVar(&conf.S3SecretKey, "s3.secret_key", "", "s3 secret key")
  55. remoteConfigureCommand.StringVar(&conf.S3Region, "s3.region", "us-east-2", "s3 region")
  56. remoteConfigureCommand.StringVar(&conf.S3Endpoint, "s3.endpoint", "", "endpoint for s3-compatible local object store")
  57. remoteConfigureCommand.StringVar(&conf.S3StorageClass, "s3.storage_class", "", "s3 storage class")
  58. remoteConfigureCommand.BoolVar(&conf.S3ForcePathStyle, "s3.force_path_style", true, "s3 force path style")
  59. remoteConfigureCommand.BoolVar(&conf.S3V4Signature, "s3.v4_signature", false, "s3 V4 signature")
  60. remoteConfigureCommand.BoolVar(&conf.S3SupportTagging, "s3.support_tagging", true, "s3 supportTagging")
  61. remoteConfigureCommand.StringVar(&conf.GcsGoogleApplicationCredentials, "gcs.appCredentialsFile", "", "google cloud storage credentials file, default to use env GOOGLE_APPLICATION_CREDENTIALS")
  62. remoteConfigureCommand.StringVar(&conf.GcsProjectId, "gcs.projectId", "", "google cloud storage project id, default to use env GOOGLE_CLOUD_PROJECT")
  63. remoteConfigureCommand.StringVar(&conf.AzureAccountName, "azure.account_name", "", "azure account name, default to use env AZURE_STORAGE_ACCOUNT")
  64. remoteConfigureCommand.StringVar(&conf.AzureAccountKey, "azure.account_key", "", "azure account name, default to use env AZURE_STORAGE_ACCESS_KEY")
  65. remoteConfigureCommand.StringVar(&conf.BackblazeKeyId, "b2.key_id", "", "backblaze keyID")
  66. remoteConfigureCommand.StringVar(&conf.BackblazeApplicationKey, "b2.application_key", "", "backblaze applicationKey. Note that your Master Application Key will not work with the S3 Compatible API. You must create a new key that is eligible for use. For more information: https://help.backblaze.com/hc/en-us/articles/360047425453")
  67. remoteConfigureCommand.StringVar(&conf.BackblazeEndpoint, "b2.endpoint", "", "backblaze endpoint")
  68. remoteConfigureCommand.StringVar(&conf.BackblazeRegion, "b2.region", "us-west-002", "backblaze region")
  69. remoteConfigureCommand.StringVar(&conf.AliyunAccessKey, "aliyun.access_key", "", "Aliyun access key, default to use env ALICLOUD_ACCESS_KEY_ID")
  70. remoteConfigureCommand.StringVar(&conf.AliyunSecretKey, "aliyun.secret_key", "", "Aliyun secret key, default to use env ALICLOUD_ACCESS_KEY_SECRET")
  71. remoteConfigureCommand.StringVar(&conf.AliyunEndpoint, "aliyun.endpoint", "", "Aliyun endpoint")
  72. remoteConfigureCommand.StringVar(&conf.AliyunRegion, "aliyun.region", "", "Aliyun region")
  73. remoteConfigureCommand.StringVar(&conf.TencentSecretId, "tencent.secret_id", "", "Tencent Secret Id, default to use env COS_SECRETID")
  74. remoteConfigureCommand.StringVar(&conf.TencentSecretKey, "tencent.secret_key", "", "Tencent secret key, default to use env COS_SECRETKEY")
  75. remoteConfigureCommand.StringVar(&conf.TencentEndpoint, "tencent.endpoint", "", "Tencent endpoint")
  76. remoteConfigureCommand.StringVar(&conf.BaiduAccessKey, "baidu.access_key", "", "Baidu access key, default to use env BDCLOUD_ACCESS_KEY")
  77. remoteConfigureCommand.StringVar(&conf.BaiduSecretKey, "baidu.secret_key", "", "Baidu secret key, default to use env BDCLOUD_SECRET_KEY")
  78. remoteConfigureCommand.StringVar(&conf.BaiduEndpoint, "baidu.endpoint", "", "Baidu endpoint")
  79. remoteConfigureCommand.StringVar(&conf.BaiduRegion, "baidu.region", "", "Baidu region")
  80. remoteConfigureCommand.StringVar(&conf.WasabiAccessKey, "wasabi.access_key", "", "Wasabi access key")
  81. remoteConfigureCommand.StringVar(&conf.WasabiSecretKey, "wasabi.secret_key", "", "Wasabi secret key")
  82. remoteConfigureCommand.StringVar(&conf.WasabiEndpoint, "wasabi.endpoint", "", "Wasabi endpoint, see https://wasabi.com/wp-content/themes/wasabi/docs/API_Guide/index.html#t=topics%2Fapidiff-intro.htm")
  83. remoteConfigureCommand.StringVar(&conf.WasabiRegion, "wasabi.region", "", "Wasabi region")
  84. remoteConfigureCommand.StringVar(&conf.FilebaseAccessKey, "filebase.access_key", "", "Filebase access key")
  85. remoteConfigureCommand.StringVar(&conf.FilebaseSecretKey, "filebase.secret_key", "", "Filebase secret key")
  86. remoteConfigureCommand.StringVar(&conf.FilebaseEndpoint, "filebase.endpoint", "", "Filebase endpoint, https://s3.filebase.com")
  87. remoteConfigureCommand.StringVar(&conf.StorjAccessKey, "storj.access_key", "", "Storj access key")
  88. remoteConfigureCommand.StringVar(&conf.StorjSecretKey, "storj.secret_key", "", "Storj secret key")
  89. remoteConfigureCommand.StringVar(&conf.StorjEndpoint, "storj.endpoint", "", "Storj endpoint")
  90. if err = remoteConfigureCommand.Parse(args); err != nil {
  91. return nil
  92. }
  93. if conf.Type != "s3" {
  94. // clear out the default values
  95. conf.S3Region = ""
  96. conf.S3ForcePathStyle = false
  97. }
  98. if conf.Name == "" {
  99. return c.listExistingRemoteStorages(commandEnv, writer)
  100. }
  101. if !isAlpha(conf.Name) {
  102. return fmt.Errorf("only letters and numbers allowed in name: %v", conf.Name)
  103. }
  104. if *isDelete {
  105. return c.deleteRemoteStorage(commandEnv, writer, conf.Name)
  106. }
  107. return c.saveRemoteStorage(commandEnv, writer, conf)
  108. }
  109. func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandEnv, writer io.Writer) error {
  110. return filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(filer.DirectoryEtcRemote), "", func(entry *filer_pb.Entry, isLast bool) error {
  111. if len(entry.Content) == 0 {
  112. fmt.Fprintf(writer, "skipping %s\n", entry.Name)
  113. return nil
  114. }
  115. if !strings.HasSuffix(entry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
  116. return nil
  117. }
  118. conf := &remote_pb.RemoteConf{}
  119. if err := proto.Unmarshal(entry.Content, conf); err != nil {
  120. return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, entry.Name, err)
  121. }
  122. // change secret key to stars
  123. conf.S3SecretKey = strings.Repeat("*", len(conf.S3SecretKey))
  124. conf.AliyunSecretKey = strings.Repeat("*", len(conf.AliyunSecretKey))
  125. conf.BaiduAccessKey = strings.Repeat("*", len(conf.BaiduAccessKey))
  126. conf.FilebaseSecretKey = strings.Repeat("*", len(conf.FilebaseSecretKey))
  127. conf.StorjSecretKey = strings.Repeat("*", len(conf.StorjSecretKey))
  128. conf.TencentSecretKey = strings.Repeat("*", len(conf.TencentSecretKey))
  129. conf.WasabiSecretKey = strings.Repeat("*", len(conf.WasabiSecretKey))
  130. return filer.ProtoToText(writer, conf)
  131. })
  132. }
  133. func (c *commandRemoteConfigure) deleteRemoteStorage(commandEnv *CommandEnv, writer io.Writer, storageName string) error {
  134. return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
  135. request := &filer_pb.DeleteEntryRequest{
  136. Directory: filer.DirectoryEtcRemote,
  137. Name: storageName + filer.REMOTE_STORAGE_CONF_SUFFIX,
  138. IgnoreRecursiveError: false,
  139. IsDeleteData: true,
  140. IsRecursive: true,
  141. IsFromOtherCluster: false,
  142. Signatures: nil,
  143. }
  144. _, err := client.DeleteEntry(context.Background(), request)
  145. if err == nil {
  146. fmt.Fprintf(writer, "removed: %s\n", storageName)
  147. }
  148. return err
  149. })
  150. }
  151. func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, writer io.Writer, conf *remote_pb.RemoteConf) error {
  152. data, err := proto.Marshal(conf)
  153. if err != nil {
  154. return err
  155. }
  156. if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
  157. return filer.SaveInsideFiler(client, filer.DirectoryEtcRemote, conf.Name+filer.REMOTE_STORAGE_CONF_SUFFIX, data)
  158. }); err != nil && err != filer_pb.ErrNotFound {
  159. return err
  160. }
  161. return nil
  162. }