volume.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397
  1. package command
  2. import (
  3. "fmt"
  4. "github.com/chrislusf/seaweedfs/weed/storage/types"
  5. "net/http"
  6. httppprof "net/http/pprof"
  7. "os"
  8. "runtime/pprof"
  9. "strconv"
  10. "strings"
  11. "time"
  12. "github.com/spf13/viper"
  13. "google.golang.org/grpc"
  14. "github.com/chrislusf/seaweedfs/weed/util/grace"
  15. "github.com/chrislusf/seaweedfs/weed/pb"
  16. "github.com/chrislusf/seaweedfs/weed/security"
  17. "github.com/chrislusf/seaweedfs/weed/util/httpdown"
  18. "google.golang.org/grpc/reflection"
  19. "github.com/chrislusf/seaweedfs/weed/glog"
  20. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  21. "github.com/chrislusf/seaweedfs/weed/server"
  22. stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
  23. "github.com/chrislusf/seaweedfs/weed/storage"
  24. "github.com/chrislusf/seaweedfs/weed/util"
  25. )
  26. var (
  27. v VolumeServerOptions
  28. )
  29. type VolumeServerOptions struct {
  30. port *int
  31. portGrpc *int
  32. publicPort *int
  33. folders []string
  34. folderMaxLimits []int
  35. idxFolder *string
  36. ip *string
  37. publicUrl *string
  38. bindIp *string
  39. mastersString *string
  40. masters []pb.ServerAddress
  41. idleConnectionTimeout *int
  42. dataCenter *string
  43. rack *string
  44. whiteList []string
  45. indexType *string
  46. diskType *string
  47. fixJpgOrientation *bool
  48. readMode *string
  49. cpuProfile *string
  50. memProfile *string
  51. compactionMBPerSecond *int
  52. fileSizeLimitMB *int
  53. concurrentUploadLimitMB *int
  54. concurrentDownloadLimitMB *int
  55. pprof *bool
  56. preStopSeconds *int
  57. metricsHttpPort *int
  58. // pulseSeconds *int
  59. enableTcp *bool
  60. }
  61. func init() {
  62. cmdVolume.Run = runVolume // break init cycle
  63. v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
  64. v.portGrpc = cmdVolume.Flag.Int("port.grpc", 0, "grpc listen port")
  65. v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
  66. v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
  67. v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
  68. v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to")
  69. v.mastersString = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
  70. v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
  71. // v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
  72. v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
  73. v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
  74. v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
  75. v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
  76. v.diskType = cmdVolume.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  77. v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
  78. v.readMode = cmdVolume.Flag.String("readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|proxy to remote node|redirect volume location'.")
  79. v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
  80. v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
  81. v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
  82. v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
  83. v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 256, "limit total concurrent upload size")
  84. v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 256, "limit total concurrent download size")
  85. v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
  86. v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
  87. v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
  88. v.enableTcp = cmdVolume.Flag.Bool("tcp", false, "<exprimental> enable tcp port")
  89. }
  90. var cmdVolume = &Command{
  91. UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
  92. Short: "start a volume server",
  93. Long: `start a volume server to provide storage spaces
  94. `,
  95. }
  96. var (
  97. volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
  98. maxVolumeCounts = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured as free disk space divided by volume size.")
  99. volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
  100. minFreeSpacePercent = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
  101. minFreeSpace = cmdVolume.Flag.String("minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
  102. )
  103. func runVolume(cmd *Command, args []string) bool {
  104. util.LoadConfiguration("security", false)
  105. // If --pprof is set we assume the caller wants to be able to collect
  106. // cpu and memory profiles via go tool pprof
  107. if !*v.pprof {
  108. grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
  109. }
  110. go stats_collect.StartMetricsServer(*v.metricsHttpPort)
  111. minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
  112. v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses()
  113. v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)
  114. return true
  115. }
  116. func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption string, minFreeSpaces []util.MinFreeSpace) {
  117. // Set multiple folders and each folder's max volume count limit'
  118. v.folders = strings.Split(volumeFolders, ",")
  119. for _, folder := range v.folders {
  120. if err := util.TestFolderWritable(util.ResolvePath(folder)); err != nil {
  121. glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
  122. }
  123. }
  124. // set max
  125. maxCountStrings := strings.Split(maxVolumeCounts, ",")
  126. for _, maxString := range maxCountStrings {
  127. if max, e := strconv.Atoi(maxString); e == nil {
  128. v.folderMaxLimits = append(v.folderMaxLimits, max)
  129. } else {
  130. glog.Fatalf("The max specified in -max not a valid number %s", maxString)
  131. }
  132. }
  133. if len(v.folderMaxLimits) == 1 && len(v.folders) > 1 {
  134. for i := 0; i < len(v.folders)-1; i++ {
  135. v.folderMaxLimits = append(v.folderMaxLimits, v.folderMaxLimits[0])
  136. }
  137. }
  138. if len(v.folders) != len(v.folderMaxLimits) {
  139. glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
  140. }
  141. if len(minFreeSpaces) == 1 && len(v.folders) > 1 {
  142. for i := 0; i < len(v.folders)-1; i++ {
  143. minFreeSpaces = append(minFreeSpaces, minFreeSpaces[0])
  144. }
  145. }
  146. if len(v.folders) != len(minFreeSpaces) {
  147. glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(minFreeSpaces))
  148. }
  149. // set disk types
  150. var diskTypes []types.DiskType
  151. diskTypeStrings := strings.Split(*v.diskType, ",")
  152. for _, diskTypeString := range diskTypeStrings {
  153. diskTypes = append(diskTypes, types.ToDiskType(diskTypeString))
  154. }
  155. if len(diskTypes) == 1 && len(v.folders) > 1 {
  156. for i := 0; i < len(v.folders)-1; i++ {
  157. diskTypes = append(diskTypes, diskTypes[0])
  158. }
  159. }
  160. if len(v.folders) != len(diskTypes) {
  161. glog.Fatalf("%d directories by -dir, but only %d disk types is set by -disk", len(v.folders), len(diskTypes))
  162. }
  163. // security related white list configuration
  164. if volumeWhiteListOption != "" {
  165. v.whiteList = strings.Split(volumeWhiteListOption, ",")
  166. }
  167. if *v.ip == "" {
  168. *v.ip = util.DetectedHostAddress()
  169. glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
  170. }
  171. if *v.publicPort == 0 {
  172. *v.publicPort = *v.port
  173. }
  174. if *v.portGrpc == 0 {
  175. *v.portGrpc = 10000 + *v.port
  176. }
  177. if *v.publicUrl == "" {
  178. *v.publicUrl = util.JoinHostPort(*v.ip, *v.publicPort)
  179. }
  180. volumeMux := http.NewServeMux()
  181. publicVolumeMux := volumeMux
  182. if v.isSeparatedPublicPort() {
  183. publicVolumeMux = http.NewServeMux()
  184. }
  185. if *v.pprof {
  186. volumeMux.HandleFunc("/debug/pprof/", httppprof.Index)
  187. volumeMux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline)
  188. volumeMux.HandleFunc("/debug/pprof/profile", httppprof.Profile)
  189. volumeMux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol)
  190. volumeMux.HandleFunc("/debug/pprof/trace", httppprof.Trace)
  191. }
  192. volumeNeedleMapKind := storage.NeedleMapInMemory
  193. switch *v.indexType {
  194. case "leveldb":
  195. volumeNeedleMapKind = storage.NeedleMapLevelDb
  196. case "leveldbMedium":
  197. volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
  198. case "leveldbLarge":
  199. volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
  200. }
  201. volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
  202. *v.ip, *v.port, *v.portGrpc, *v.publicUrl,
  203. v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
  204. *v.idxFolder,
  205. volumeNeedleMapKind,
  206. v.masters, 5, *v.dataCenter, *v.rack,
  207. v.whiteList,
  208. *v.fixJpgOrientation, *v.readMode,
  209. *v.compactionMBPerSecond,
  210. *v.fileSizeLimitMB,
  211. int64(*v.concurrentUploadLimitMB)*1024*1024,
  212. int64(*v.concurrentDownloadLimitMB)*1024*1024,
  213. )
  214. // starting grpc server
  215. grpcS := v.startGrpcService(volumeServer)
  216. // starting public http server
  217. var publicHttpDown httpdown.Server
  218. if v.isSeparatedPublicPort() {
  219. publicHttpDown = v.startPublicHttpService(publicVolumeMux)
  220. if nil == publicHttpDown {
  221. glog.Fatalf("start public http service failed")
  222. }
  223. }
  224. // starting tcp server
  225. if *v.enableTcp {
  226. go v.startTcpService(volumeServer)
  227. }
  228. // starting the cluster http server
  229. clusterHttpServer := v.startClusterHttpService(volumeMux)
  230. stopChan := make(chan bool)
  231. grace.OnInterrupt(func() {
  232. fmt.Println("volume server has be killed")
  233. // Stop heartbeats
  234. if !volumeServer.StopHeartbeat() {
  235. volumeServer.SetStopping()
  236. glog.V(0).Infof("stop send heartbeat and wait %d seconds until shutdown ...", *v.preStopSeconds)
  237. time.Sleep(time.Duration(*v.preStopSeconds) * time.Second)
  238. }
  239. shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer)
  240. stopChan <- true
  241. })
  242. select {
  243. case <-stopChan:
  244. }
  245. }
  246. func shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server, grpcS *grpc.Server, volumeServer *weed_server.VolumeServer) {
  247. // firstly, stop the public http service to prevent from receiving new user request
  248. if nil != publicHttpDown {
  249. glog.V(0).Infof("stop public http server ... ")
  250. if err := publicHttpDown.Stop(); err != nil {
  251. glog.Warningf("stop the public http server failed, %v", err)
  252. }
  253. }
  254. glog.V(0).Infof("graceful stop cluster http server ... ")
  255. if err := clusterHttpServer.Stop(); err != nil {
  256. glog.Warningf("stop the cluster http server failed, %v", err)
  257. }
  258. glog.V(0).Infof("graceful stop gRPC ...")
  259. grpcS.GracefulStop()
  260. volumeServer.Shutdown()
  261. pprof.StopCPUProfile()
  262. }
  263. // check whether configure the public port
  264. func (v VolumeServerOptions) isSeparatedPublicPort() bool {
  265. return *v.publicPort != *v.port
  266. }
  267. func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
  268. grpcPort := *v.portGrpc
  269. grpcL, err := util.NewListener(util.JoinHostPort(*v.bindIp, grpcPort), 0)
  270. if err != nil {
  271. glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
  272. }
  273. grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
  274. volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
  275. reflection.Register(grpcS)
  276. go func() {
  277. if err := grpcS.Serve(grpcL); err != nil {
  278. glog.Fatalf("start gRPC service failed, %s", err)
  279. }
  280. }()
  281. return grpcS
  282. }
  283. func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
  284. publicListeningAddress := util.JoinHostPort(*v.bindIp, *v.publicPort)
  285. glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
  286. publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
  287. if e != nil {
  288. glog.Fatalf("Volume server listener error:%v", e)
  289. }
  290. pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
  291. publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener)
  292. go func() {
  293. if err := publicHttpDown.Wait(); err != nil {
  294. glog.Errorf("public http down wait failed, %v", err)
  295. }
  296. }()
  297. return publicHttpDown
  298. }
  299. func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpdown.Server {
  300. var (
  301. certFile, keyFile string
  302. )
  303. if viper.GetString("https.volume.key") != "" {
  304. certFile = viper.GetString("https.volume.cert")
  305. keyFile = viper.GetString("https.volume.key")
  306. }
  307. listeningAddress := util.JoinHostPort(*v.bindIp, *v.port)
  308. glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
  309. listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
  310. if e != nil {
  311. glog.Fatalf("Volume server listener error:%v", e)
  312. }
  313. httpDown := httpdown.HTTP{
  314. KillTimeout: 5 * time.Minute,
  315. StopTimeout: 5 * time.Minute,
  316. CertFile: certFile,
  317. KeyFile: keyFile}
  318. clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
  319. go func() {
  320. if e := clusterHttpServer.Wait(); e != nil {
  321. glog.Fatalf("Volume server fail to serve: %v", e)
  322. }
  323. }()
  324. return clusterHttpServer
  325. }
  326. func (v VolumeServerOptions) startTcpService(volumeServer *weed_server.VolumeServer) {
  327. listeningAddress := util.JoinHostPort(*v.bindIp, *v.port+20000)
  328. glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "tcp at", listeningAddress)
  329. listener, e := util.NewListener(listeningAddress, 0)
  330. if e != nil {
  331. glog.Fatalf("Volume server listener error on %s:%v", listeningAddress, e)
  332. }
  333. defer listener.Close()
  334. for {
  335. c, err := listener.Accept()
  336. if err != nil {
  337. fmt.Println(err)
  338. return
  339. }
  340. go volumeServer.HandleTcpConnection(c)
  341. }
  342. }