backup.go 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154
  1. package command
  2. import (
  3. "fmt"
  4. "github.com/chrislusf/seaweedfs/weed/security"
  5. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  6. "github.com/chrislusf/seaweedfs/weed/storage/super_block"
  7. "github.com/chrislusf/seaweedfs/weed/util"
  8. "github.com/chrislusf/seaweedfs/weed/operation"
  9. "github.com/chrislusf/seaweedfs/weed/storage"
  10. )
  11. var (
  12. s BackupOptions
  13. )
  14. type BackupOptions struct {
  15. master *string
  16. collection *string
  17. dir *string
  18. volumeId *int
  19. ttl *string
  20. replication *string
  21. }
  22. func init() {
  23. cmdBackup.Run = runBackup // break init cycle
  24. s.master = cmdBackup.Flag.String("server", "localhost:9333", "SeaweedFS master location")
  25. s.collection = cmdBackup.Flag.String("collection", "", "collection name")
  26. s.dir = cmdBackup.Flag.String("dir", ".", "directory to store volume data files")
  27. s.volumeId = cmdBackup.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
  28. s.ttl = cmdBackup.Flag.String("ttl", "", `backup volume's time to live, format:
  29. 3m: 3 minutes
  30. 4h: 4 hours
  31. 5d: 5 days
  32. 6w: 6 weeks
  33. 7M: 7 months
  34. 8y: 8 years
  35. default is the same with origin`)
  36. s.replication = cmdBackup.Flag.String("replication", "", "backup volume's replication, default is the same with origin")
  37. }
  38. var cmdBackup = &Command{
  39. UsageLine: "backup -dir=. -volumeId=234 -server=localhost:9333",
  40. Short: "incrementally backup a volume to local folder",
  41. Long: `Incrementally backup volume data.
  42. It is expected that you use this inside a script, to loop through
  43. all possible volume ids that needs to be backup to local folder.
  44. The volume id does not need to exist locally or even remotely.
  45. This will help to backup future new volumes.
  46. Usually backing up is just copying the .dat (and .idx) files.
  47. But it's tricky to incrementally copy the differences.
  48. The complexity comes when there are multiple addition, deletion and compaction.
  49. This tool will handle them correctly and efficiently, avoiding unnecessary data transportation.
  50. `,
  51. }
  52. func runBackup(cmd *Command, args []string) bool {
  53. util.LoadConfiguration("security", false)
  54. grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
  55. if *s.volumeId == -1 {
  56. return false
  57. }
  58. vid := needle.VolumeId(*s.volumeId)
  59. // find volume location, replication, ttl info
  60. lookup, err := operation.Lookup(*s.master, vid.String())
  61. if err != nil {
  62. fmt.Printf("Error looking up volume %d: %v\n", vid, err)
  63. return true
  64. }
  65. volumeServer := lookup.Locations[0].Url
  66. stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
  67. if err != nil {
  68. fmt.Printf("Error get volume %d status: %v\n", vid, err)
  69. return true
  70. }
  71. var ttl *needle.TTL
  72. if *s.ttl != "" {
  73. ttl, err = needle.ReadTTL(*s.ttl)
  74. if err != nil {
  75. fmt.Printf("Error generate volume %d ttl %s: %v\n", vid, *s.ttl, err)
  76. return true
  77. }
  78. } else {
  79. ttl, err = needle.ReadTTL(stats.Ttl)
  80. if err != nil {
  81. fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err)
  82. return true
  83. }
  84. }
  85. var replication *super_block.ReplicaPlacement
  86. if *s.replication != "" {
  87. replication, err = super_block.NewReplicaPlacementFromString(*s.replication)
  88. if err != nil {
  89. fmt.Printf("Error generate volume %d replication %s : %v\n", vid, *s.replication, err)
  90. return true
  91. }
  92. } else {
  93. replication, err = super_block.NewReplicaPlacementFromString(stats.Replication)
  94. if err != nil {
  95. fmt.Printf("Error get volume %d replication %s : %v\n", vid, stats.Replication, err)
  96. return true
  97. }
  98. }
  99. v, err := storage.NewVolume(util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
  100. if err != nil {
  101. fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
  102. return true
  103. }
  104. if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
  105. if err = v.Compact2(30*1024*1024*1024, 0); err != nil {
  106. fmt.Printf("Compact Volume before synchronizing %v\n", err)
  107. return true
  108. }
  109. if err = v.CommitCompact(); err != nil {
  110. fmt.Printf("Commit Compact before synchronizing %v\n", err)
  111. return true
  112. }
  113. v.SuperBlock.CompactionRevision = uint16(stats.CompactRevision)
  114. v.DataBackend.WriteAt(v.SuperBlock.Bytes(), 0)
  115. }
  116. datSize, _, _ := v.FileStat()
  117. if datSize > stats.TailOffset {
  118. // remove the old data
  119. v.Destroy()
  120. // recreate an empty volume
  121. v, err = storage.NewVolume(util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0)
  122. if err != nil {
  123. fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
  124. return true
  125. }
  126. }
  127. defer v.Close()
  128. if err := v.IncrementalBackup(volumeServer, grpcDialOption); err != nil {
  129. fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
  130. return true
  131. }
  132. return true
  133. }