volume_vacuum_test.go 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172
  1. package storage
  2. import (
  3. "io/ioutil"
  4. "math/rand"
  5. "os"
  6. "testing"
  7. "time"
  8. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  9. "github.com/chrislusf/seaweedfs/weed/storage/super_block"
  10. "github.com/chrislusf/seaweedfs/weed/storage/types"
  11. )
  12. /*
  13. makediff test steps
  14. 1. launch weed server at your local/dev environment, (option
  15. "garbageThreshold" for master and option "max" for volume should be set with specific value which would let
  16. preparing test prerequisite easier )
  17. a) ./weed master -garbageThreshold=0.99 -mdir=./m
  18. b) ./weed volume -dir=./data -max=1 -mserver=localhost:9333 -port=8080
  19. 2. upload 4 different files, you could call dir/assign to get 4 different fids
  20. a) upload file A with fid a
  21. b) upload file B with fid b
  22. c) upload file C with fid c
  23. d) upload file D with fid d
  24. 3. update file A and C
  25. a) modify file A and upload file A with fid a
  26. b) modify file C and upload file C with fid c
  27. c) record the current 1.idx's file size(lastCompactIndexOffset value)
  28. 4. Compacting the data file
  29. a) run curl http://localhost:8080/admin/vacuum/compact?volumeId=1
  30. b) verify the 1.cpd and 1.cpx is created under volume directory
  31. 5. update file B and delete file D
  32. a) modify file B and upload file B with fid b
  33. d) delete file B with fid b
  34. 6. Now you could run the following UT case, the case should be run successfully
  35. 7. Compact commit manually
  36. a) mv 1.cpd 1.dat
  37. b) mv 1.cpx 1.idx
  38. 8. Restart Volume Server
  39. 9. Now you should get updated file A,B,C
  40. */
  41. func TestMakeDiff(t *testing.T) {
  42. v := new(Volume)
  43. //lastCompactIndexOffset value is the index file size before step 4
  44. v.lastCompactIndexOffset = 96
  45. v.SuperBlock.Version = 0x2
  46. /*
  47. err := v.makeupDiff(
  48. "/yourpath/1.cpd",
  49. "/yourpath/1.cpx",
  50. "/yourpath/1.dat",
  51. "/yourpath/1.idx")
  52. if err != nil {
  53. t.Errorf("makeupDiff err is %v", err)
  54. } else {
  55. t.Log("makeupDiff Succeeded")
  56. }
  57. */
  58. }
  59. func TestCompaction(t *testing.T) {
  60. dir, err := ioutil.TempDir("", "example")
  61. if err != nil {
  62. t.Fatalf("temp dir creation: %v", err)
  63. }
  64. defer os.RemoveAll(dir) // clean up
  65. v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0)
  66. if err != nil {
  67. t.Fatalf("volume creation: %v", err)
  68. }
  69. beforeCommitFileCount := 10000
  70. afterCommitFileCount := 10000
  71. infos := make([]*needleInfo, beforeCommitFileCount+afterCommitFileCount)
  72. for i := 1; i <= beforeCommitFileCount; i++ {
  73. doSomeWritesDeletes(i, v, t, infos)
  74. }
  75. startTime := time.Now()
  76. v.Compact2(0, 0)
  77. speed := float64(v.ContentSize()) / time.Now().Sub(startTime).Seconds()
  78. t.Logf("compaction speed: %.2f bytes/s", speed)
  79. for i := 1; i <= afterCommitFileCount; i++ {
  80. doSomeWritesDeletes(i+beforeCommitFileCount, v, t, infos)
  81. }
  82. v.CommitCompact()
  83. v.Close()
  84. v, err = NewVolume(dir, dir, "", 1, NeedleMapInMemory, nil, nil, 0, 0)
  85. if err != nil {
  86. t.Fatalf("volume reloading: %v", err)
  87. }
  88. for i := 1; i <= beforeCommitFileCount+afterCommitFileCount; i++ {
  89. if infos[i-1] == nil {
  90. t.Fatal("not found file", i)
  91. continue
  92. }
  93. if infos[i-1].size == 0 {
  94. continue
  95. }
  96. n := newEmptyNeedle(uint64(i))
  97. size, err := v.readNeedle(n, nil)
  98. if err != nil {
  99. t.Fatalf("read file %d: %v", i, err)
  100. }
  101. if infos[i-1].size != types.Size(size) {
  102. t.Fatalf("read file %d size mismatch expected %d found %d", i, infos[i-1].size, size)
  103. }
  104. if infos[i-1].crc != n.Checksum {
  105. t.Fatalf("read file %d checksum mismatch expected %d found %d", i, infos[i-1].crc, n.Checksum)
  106. }
  107. }
  108. }
  109. func doSomeWritesDeletes(i int, v *Volume, t *testing.T, infos []*needleInfo) {
  110. n := newRandomNeedle(uint64(i))
  111. _, size, _, err := v.writeNeedle2(n, false)
  112. if err != nil {
  113. t.Fatalf("write file %d: %v", i, err)
  114. }
  115. infos[i-1] = &needleInfo{
  116. size: size,
  117. crc: n.Checksum,
  118. }
  119. // println("written file", i, "checksum", n.Checksum.Value(), "size", size)
  120. if rand.Float64() < 0.03 {
  121. toBeDeleted := rand.Intn(i) + 1
  122. oldNeedle := newEmptyNeedle(uint64(toBeDeleted))
  123. v.deleteNeedle2(oldNeedle)
  124. // println("deleted file", toBeDeleted)
  125. infos[toBeDeleted-1] = &needleInfo{
  126. size: 0,
  127. crc: n.Checksum,
  128. }
  129. }
  130. }
  131. type needleInfo struct {
  132. size types.Size
  133. crc needle.CRC
  134. }
  135. func newRandomNeedle(id uint64) *needle.Needle {
  136. n := new(needle.Needle)
  137. n.Data = make([]byte, rand.Intn(1024))
  138. rand.Read(n.Data)
  139. n.Checksum = needle.NewCRC(n.Data)
  140. n.Id = types.Uint64ToNeedleId(id)
  141. return n
  142. }
  143. func newEmptyNeedle(id uint64) *needle.Needle {
  144. n := new(needle.Needle)
  145. n.Id = types.Uint64ToNeedleId(id)
  146. return n
  147. }