server.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322
  1. package command
  2. import (
  3. "fmt"
  4. "net/http"
  5. "os"
  6. "strings"
  7. "time"
  8. stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
  9. "github.com/seaweedfs/seaweedfs/weed/glog"
  10. "github.com/seaweedfs/seaweedfs/weed/pb"
  11. "github.com/seaweedfs/seaweedfs/weed/util"
  12. "github.com/seaweedfs/seaweedfs/weed/util/grace"
  13. )
  14. type ServerOptions struct {
  15. cpuprofile *string
  16. memprofile *string
  17. debug *bool
  18. debugPort *int
  19. v VolumeServerOptions
  20. }
  21. var (
  22. serverOptions ServerOptions
  23. masterOptions MasterOptions
  24. filerOptions FilerOptions
  25. s3Options S3Options
  26. iamOptions IamOptions
  27. webdavOptions WebDavOption
  28. mqBrokerOptions MessageQueueBrokerOptions
  29. )
  30. func init() {
  31. cmdServer.Run = runServer // break init cycle
  32. }
  33. var cmdServer = &Command{
  34. UsageLine: "server -dir=/tmp -volume.max=5 -ip=server_name",
  35. Short: "start a master server, a volume server, and optionally a filer and a S3 gateway",
  36. Long: `start both a volume server to provide storage spaces
  37. and a master server to provide volume=>location mapping service and sequence number of file ids
  38. This is provided as a convenient way to start both volume server and master server.
  39. The servers acts exactly the same as starting them separately.
  40. So other volume servers can connect to this master server also.
  41. Optionally, a filer server can be started.
  42. Also optionally, a S3 gateway can be started.
  43. `,
  44. }
  45. var (
  46. serverIp = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
  47. serverBindIp = cmdServer.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
  48. serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
  49. serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
  50. serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
  51. serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
  52. serverDisableHttp = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
  53. volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
  54. volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured as free disk space divided by volume size.")
  55. volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
  56. volumeMinFreeSpace = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
  57. serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
  58. serverMetricsHttpIp = cmdServer.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
  59. // pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
  60. isStartingMasterServer = cmdServer.Flag.Bool("master", true, "whether to start master server")
  61. isStartingVolumeServer = cmdServer.Flag.Bool("volume", true, "whether to start volume server")
  62. isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer")
  63. isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
  64. isStartingIam = cmdServer.Flag.Bool("iam", false, "whether to start IAM service")
  65. isStartingWebDav = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway")
  66. isStartingMqBroker = cmdServer.Flag.Bool("mq.broker", false, "whether to start message queue broker")
  67. False = false
  68. )
  69. func init() {
  70. serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
  71. serverOptions.memprofile = cmdServer.Flag.String("memprofile", "", "memory profile output file")
  72. serverOptions.debug = cmdServer.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:6060/debug/pprof/goroutine?debug=2")
  73. serverOptions.debugPort = cmdServer.Flag.Int("debug.port", 6060, "http port for debugging")
  74. masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
  75. masterOptions.portGrpc = cmdServer.Flag.Int("master.port.grpc", 0, "master server grpc listen port")
  76. masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
  77. masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
  78. masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
  79. masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
  80. masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server")
  81. masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
  82. masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
  83. masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
  84. masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
  85. masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
  86. masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
  87. masterOptions.raftBootstrap = cmdServer.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
  88. masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
  89. masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
  90. filerOptions.filerGroup = cmdServer.Flag.String("filer.filerGroup", "", "share metadata with other filers in the same filerGroup")
  91. filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
  92. filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
  93. filerOptions.portGrpc = cmdServer.Flag.Int("filer.port.grpc", 0, "filer server grpc listen port")
  94. filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
  95. filerOptions.allowedOrigins = cmdServer.Flag.String("filer.allowedOrigins", "*", "comma separated list of allowed origins")
  96. filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
  97. filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
  98. filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit")
  99. filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
  100. filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
  101. filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
  102. filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
  103. filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
  104. filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
  105. filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
  106. filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  107. filerOptions.exposeDirectoryData = cmdServer.Flag.Bool("filer.exposeDirectoryData", true, "expose directory data via filer. If false, filer UI will be innaccessible.")
  108. serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
  109. serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
  110. serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
  111. serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
  112. serverOptions.v.diskType = cmdServer.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  113. serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
  114. serverOptions.v.readMode = cmdServer.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
  115. serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
  116. serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
  117. serverOptions.v.ldbTimeout = cmdServer.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
  118. serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
  119. serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 64, "limit total concurrent download size")
  120. serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
  121. serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
  122. serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
  123. serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
  124. serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
  125. serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
  126. serverOptions.v.readBufferSizeMB = cmdServer.Flag.Int("volume.readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally")
  127. s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
  128. s3Options.portHttps = cmdServer.Flag.Int("s3.port.https", 0, "s3 server https listen port")
  129. s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
  130. s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
  131. s3Options.allowedOrigins = cmdServer.Flag.String("s3.allowedOrigins", "*", "comma separated list of allowed origins")
  132. s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
  133. s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
  134. s3Options.tlsCACertificate = cmdServer.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
  135. s3Options.tlsVerifyClientCert = cmdServer.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
  136. s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
  137. s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
  138. s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
  139. s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
  140. s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
  141. iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
  142. webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
  143. webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files")
  144. webdavOptions.replication = cmdServer.Flag.String("webdav.replication", "", "replication to create the files")
  145. webdavOptions.disk = cmdServer.Flag.String("webdav.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  146. webdavOptions.tlsPrivateKey = cmdServer.Flag.String("webdav.key.file", "", "path to the TLS private key file")
  147. webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
  148. webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
  149. webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
  150. webdavOptions.maxMB = cmdServer.Flag.Int("webdav.maxMB", 4, "split files larger than the limit")
  151. webdavOptions.filerRootPath = cmdServer.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
  152. mqBrokerOptions.port = cmdServer.Flag.Int("mq.broker.port", 17777, "message queue broker gRPC listen port")
  153. }
  154. func runServer(cmd *Command, args []string) bool {
  155. if *serverOptions.debug {
  156. go http.ListenAndServe(fmt.Sprintf(":%d", *serverOptions.debugPort), nil)
  157. }
  158. util.LoadSecurityConfiguration()
  159. util.LoadConfiguration("master", false)
  160. grace.SetupProfiling(*serverOptions.cpuprofile, *serverOptions.memprofile)
  161. if *isStartingS3 {
  162. *isStartingFiler = true
  163. }
  164. if *isStartingIam {
  165. *isStartingFiler = true
  166. }
  167. if *isStartingWebDav {
  168. *isStartingFiler = true
  169. }
  170. if *isStartingMqBroker {
  171. *isStartingFiler = true
  172. }
  173. if *isStartingMasterServer {
  174. _, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.portGrpc, *masterOptions.peers)
  175. peers := strings.Join(pb.ToAddressStrings(peerList), ",")
  176. masterOptions.peers = &peers
  177. }
  178. if *serverBindIp == "" {
  179. serverBindIp = serverIp
  180. }
  181. if *serverMetricsHttpIp == "" {
  182. *serverMetricsHttpIp = *serverBindIp
  183. }
  184. // ip address
  185. masterOptions.ip = serverIp
  186. masterOptions.ipBind = serverBindIp
  187. filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToServiceDiscovery()
  188. filerOptions.ip = serverIp
  189. filerOptions.bindIp = serverBindIp
  190. s3Options.bindIp = serverBindIp
  191. iamOptions.ip = serverBindIp
  192. iamOptions.masters = masterOptions.peers
  193. serverOptions.v.ip = serverIp
  194. serverOptions.v.bindIp = serverBindIp
  195. serverOptions.v.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
  196. serverOptions.v.idleConnectionTimeout = serverTimeout
  197. serverOptions.v.dataCenter = serverDataCenter
  198. serverOptions.v.rack = serverRack
  199. mqBrokerOptions.ip = serverIp
  200. mqBrokerOptions.masters = filerOptions.masters.GetInstancesAsMap()
  201. mqBrokerOptions.filerGroup = filerOptions.filerGroup
  202. // serverOptions.v.pulseSeconds = pulseSeconds
  203. // masterOptions.pulseSeconds = pulseSeconds
  204. masterOptions.whiteList = serverWhiteListOption
  205. filerOptions.dataCenter = serverDataCenter
  206. filerOptions.rack = serverRack
  207. mqBrokerOptions.dataCenter = serverDataCenter
  208. mqBrokerOptions.rack = serverRack
  209. s3Options.dataCenter = serverDataCenter
  210. filerOptions.disableHttp = serverDisableHttp
  211. masterOptions.disableHttp = serverDisableHttp
  212. filerAddress := string(pb.NewServerAddress(*serverIp, *filerOptions.port, *filerOptions.portGrpc))
  213. s3Options.filer = &filerAddress
  214. iamOptions.filer = &filerAddress
  215. webdavOptions.filer = &filerAddress
  216. mqBrokerOptions.filerGroup = filerOptions.filerGroup
  217. go stats_collect.StartMetricsServer(*serverMetricsHttpIp, *serverMetricsHttpPort)
  218. folders := strings.Split(*volumeDataFolders, ",")
  219. if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
  220. glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
  221. }
  222. if *masterOptions.metaFolder == "" {
  223. *masterOptions.metaFolder = folders[0]
  224. }
  225. if err := util.TestFolderWritable(util.ResolvePath(*masterOptions.metaFolder)); err != nil {
  226. glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
  227. }
  228. filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder
  229. serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
  230. if *isStartingFiler {
  231. if *filerOptions.diskType == "" && *serverOptions.v.diskType != "" {
  232. filerOptions.diskType = serverOptions.v.diskType
  233. }
  234. go func() {
  235. time.Sleep(1 * time.Second)
  236. filerOptions.startFiler()
  237. }()
  238. }
  239. if *isStartingS3 {
  240. go func() {
  241. time.Sleep(2 * time.Second)
  242. s3Options.localFilerSocket = filerOptions.localSocket
  243. s3Options.startS3Server()
  244. }()
  245. }
  246. if *isStartingIam {
  247. go func() {
  248. time.Sleep(2 * time.Second)
  249. iamOptions.startIamServer()
  250. }()
  251. }
  252. if *isStartingWebDav {
  253. go func() {
  254. time.Sleep(2 * time.Second)
  255. webdavOptions.startWebDav()
  256. }()
  257. }
  258. if *isStartingMqBroker {
  259. go func() {
  260. time.Sleep(2 * time.Second)
  261. mqBrokerOptions.startQueueServer()
  262. }()
  263. }
  264. // start volume server
  265. if *isStartingVolumeServer {
  266. minFreeSpaces := util.MustParseMinFreeSpace(*volumeMinFreeSpace, *volumeMinFreeSpacePercent)
  267. go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, minFreeSpaces)
  268. }
  269. if *isStartingMasterServer {
  270. go startMaster(masterOptions, serverWhiteList)
  271. }
  272. select {}
  273. }