filer.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340
  1. package command
  2. import (
  3. "fmt"
  4. "net"
  5. "net/http"
  6. "os"
  7. "runtime"
  8. "sort"
  9. "strings"
  10. "time"
  11. "google.golang.org/grpc/reflection"
  12. "github.com/seaweedfs/seaweedfs/weed/filer"
  13. "github.com/seaweedfs/seaweedfs/weed/glog"
  14. "github.com/seaweedfs/seaweedfs/weed/pb"
  15. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  16. "github.com/seaweedfs/seaweedfs/weed/security"
  17. weed_server "github.com/seaweedfs/seaweedfs/weed/server"
  18. stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
  19. "github.com/seaweedfs/seaweedfs/weed/util"
  20. )
  21. var (
  22. f FilerOptions
  23. filerStartS3 *bool
  24. filerS3Options S3Options
  25. filerStartWebDav *bool
  26. filerWebDavOptions WebDavOption
  27. filerStartIam *bool
  28. filerIamOptions IamOptions
  29. )
  30. type FilerOptions struct {
  31. masters *pb.ServerDiscovery
  32. mastersString *string
  33. ip *string
  34. bindIp *string
  35. port *int
  36. portGrpc *int
  37. publicPort *int
  38. filerGroup *string
  39. collection *string
  40. defaultReplicaPlacement *string
  41. disableDirListing *bool
  42. maxMB *int
  43. dirListingLimit *int
  44. dataCenter *string
  45. rack *string
  46. enableNotification *bool
  47. disableHttp *bool
  48. cipher *bool
  49. metricsHttpPort *int
  50. saveToFilerLimit *int
  51. defaultLevelDbDirectory *string
  52. concurrentUploadLimitMB *int
  53. debug *bool
  54. debugPort *int
  55. localSocket *string
  56. showUIDirectoryDelete *bool
  57. downloadMaxMBps *int
  58. diskType *string
  59. allowedOrigins *string
  60. }
  61. func init() {
  62. cmdFiler.Run = runFiler // break init cycle
  63. f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers or a single DNS SRV record of at least 1 master server, prepended with dnssrv+")
  64. f.filerGroup = cmdFiler.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
  65. f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
  66. f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
  67. f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
  68. f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
  69. f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port")
  70. f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
  71. f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
  72. f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
  73. f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
  74. f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
  75. f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
  76. f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
  77. f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
  78. f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
  79. f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
  80. f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
  81. f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
  82. f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
  83. f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
  84. f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
  85. f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
  86. f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
  87. f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
  88. f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  89. f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
  90. // start s3 on filer
  91. filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
  92. filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port")
  93. filerS3Options.portHttps = cmdFiler.Flag.Int("s3.port.https", 0, "s3 server https listen port")
  94. filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
  95. filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
  96. filerS3Options.dataCenter = cmdFiler.Flag.String("s3.dataCenter", "", "prefer to read and write to volumes in this data center")
  97. filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
  98. filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
  99. filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
  100. filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
  101. filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
  102. filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
  103. filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
  104. // start webdav on filer
  105. filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
  106. filerWebDavOptions.port = cmdFiler.Flag.Int("webdav.port", 7333, "webdav server http listen port")
  107. filerWebDavOptions.collection = cmdFiler.Flag.String("webdav.collection", "", "collection to create the files")
  108. filerWebDavOptions.replication = cmdFiler.Flag.String("webdav.replication", "", "replication to create the files")
  109. filerWebDavOptions.disk = cmdFiler.Flag.String("webdav.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  110. filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file")
  111. filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
  112. filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
  113. filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
  114. filerWebDavOptions.filerRootPath = cmdFiler.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
  115. // start iam on filer
  116. filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
  117. filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
  118. filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
  119. }
  120. func filerLongDesc() string {
  121. desc := `start a file server which accepts REST operation for any files.
  122. //create or overwrite the file, the directories /path/to will be automatically created
  123. POST /path/to/file
  124. //get the file content
  125. GET /path/to/file
  126. //create or overwrite the file, the filename in the multipart request will be used
  127. POST /path/to/
  128. //return a json format subdirectory and files listing
  129. GET /path/to/
  130. The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
  131. If the "filer.toml" is not found, an embedded filer store will be created under "-defaultStoreDir".
  132. The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
  133. Supported Filer Stores:
  134. `
  135. storeNames := make([]string, len(filer.Stores))
  136. for i, store := range filer.Stores {
  137. storeNames[i] = "\t" + store.GetName()
  138. }
  139. sort.Strings(storeNames)
  140. storeList := strings.Join(storeNames, "\n")
  141. return desc + storeList
  142. }
  143. var cmdFiler = &Command{
  144. UsageLine: "filer -port=8888 -master=<ip:port>[,<ip:port>]*",
  145. Short: "start a file server that points to a master server, or a list of master servers",
  146. Long: filerLongDesc(),
  147. }
  148. func runFiler(cmd *Command, args []string) bool {
  149. if *f.debug {
  150. go http.ListenAndServe(fmt.Sprintf(":%d", *f.debugPort), nil)
  151. }
  152. util.LoadConfiguration("security", false)
  153. go stats_collect.StartMetricsServer(*f.bindIp, *f.metricsHttpPort)
  154. filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
  155. startDelay := time.Duration(2)
  156. if *filerStartS3 {
  157. filerS3Options.filer = &filerAddress
  158. filerS3Options.bindIp = f.bindIp
  159. filerS3Options.localFilerSocket = f.localSocket
  160. if *f.dataCenter != "" && *filerS3Options.dataCenter == "" {
  161. filerS3Options.dataCenter = f.dataCenter
  162. }
  163. go func(delay time.Duration) {
  164. time.Sleep(delay * time.Second)
  165. filerS3Options.startS3Server()
  166. }(startDelay)
  167. startDelay++
  168. }
  169. if *filerStartWebDav {
  170. filerWebDavOptions.filer = &filerAddress
  171. if *filerWebDavOptions.disk == "" {
  172. filerWebDavOptions.disk = f.diskType
  173. }
  174. go func(delay time.Duration) {
  175. time.Sleep(delay * time.Second)
  176. filerWebDavOptions.startWebDav()
  177. }(startDelay)
  178. startDelay++
  179. }
  180. if *filerStartIam {
  181. filerIamOptions.filer = &filerAddress
  182. filerIamOptions.masters = f.mastersString
  183. go func(delay time.Duration) {
  184. time.Sleep(delay * time.Second)
  185. filerIamOptions.startIamServer()
  186. }(startDelay)
  187. }
  188. f.masters = pb.ServerAddresses(*f.mastersString).ToServiceDiscovery()
  189. f.startFiler()
  190. return true
  191. }
  192. func (fo *FilerOptions) startFiler() {
  193. defaultMux := http.NewServeMux()
  194. publicVolumeMux := defaultMux
  195. if *fo.publicPort != 0 {
  196. publicVolumeMux = http.NewServeMux()
  197. }
  198. if *fo.portGrpc == 0 {
  199. *fo.portGrpc = 10000 + *fo.port
  200. }
  201. if *fo.bindIp == "" {
  202. *fo.bindIp = *fo.ip
  203. }
  204. if *fo.allowedOrigins == "" {
  205. *fo.allowedOrigins = "*"
  206. }
  207. defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
  208. filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc)
  209. fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
  210. Masters: fo.masters,
  211. FilerGroup: *fo.filerGroup,
  212. Collection: *fo.collection,
  213. DefaultReplication: *fo.defaultReplicaPlacement,
  214. DisableDirListing: *fo.disableDirListing,
  215. MaxMB: *fo.maxMB,
  216. DirListingLimit: *fo.dirListingLimit,
  217. DataCenter: *fo.dataCenter,
  218. Rack: *fo.rack,
  219. DefaultLevelDbDir: defaultLevelDbDirectory,
  220. DisableHttp: *fo.disableHttp,
  221. Host: filerAddress,
  222. Cipher: *fo.cipher,
  223. SaveToFilerLimit: int64(*fo.saveToFilerLimit),
  224. ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
  225. ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
  226. DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
  227. DiskType: *fo.diskType,
  228. AllowedOrigins: strings.Split(*fo.allowedOrigins, ","),
  229. })
  230. if nfs_err != nil {
  231. glog.Fatalf("Filer startup error: %v", nfs_err)
  232. }
  233. if *fo.publicPort != 0 {
  234. publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
  235. glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
  236. publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
  237. if e != nil {
  238. glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
  239. }
  240. go func() {
  241. if e := http.Serve(publicListener, publicVolumeMux); e != nil {
  242. glog.Fatalf("Volume server fail to serve public: %v", e)
  243. }
  244. }()
  245. if localPublicListener != nil {
  246. go func() {
  247. if e := http.Serve(localPublicListener, publicVolumeMux); e != nil {
  248. glog.Errorf("Volume server fail to serve public: %v", e)
  249. }
  250. }()
  251. }
  252. }
  253. glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
  254. filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
  255. *fo.bindIp, *fo.port,
  256. time.Duration(10)*time.Second,
  257. )
  258. if e != nil {
  259. glog.Fatalf("Filer listener error: %v", e)
  260. }
  261. // starting grpc server
  262. grpcPort := *fo.portGrpc
  263. grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
  264. if err != nil {
  265. glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
  266. }
  267. grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
  268. filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
  269. reflection.Register(grpcS)
  270. if grpcLocalL != nil {
  271. go grpcS.Serve(grpcLocalL)
  272. }
  273. go grpcS.Serve(grpcL)
  274. httpS := &http.Server{Handler: defaultMux}
  275. if runtime.GOOS != "windows" {
  276. localSocket := *fo.localSocket
  277. if localSocket == "" {
  278. localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port)
  279. }
  280. if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
  281. glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
  282. }
  283. go func() {
  284. // start on local unix socket
  285. filerSocketListener, err := net.Listen("unix", localSocket)
  286. if err != nil {
  287. glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
  288. }
  289. httpS.Serve(filerSocketListener)
  290. }()
  291. }
  292. if filerLocalListener != nil {
  293. go func() {
  294. if err := httpS.Serve(filerLocalListener); err != nil {
  295. glog.Errorf("Filer Fail to serve: %v", e)
  296. }
  297. }()
  298. }
  299. if err := httpS.Serve(filerListener); err != nil {
  300. glog.Fatalf("Filer Fail to serve: %v", e)
  301. }
  302. }