compact_map_perf_test.go 2.0 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192
  1. package needle_map
  2. import (
  3. "fmt"
  4. "log"
  5. "os"
  6. "runtime"
  7. "testing"
  8. "time"
  9. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  10. )
  11. /*
  12. To see the memory usage:
  13. go test -run TestMemoryUsage
  14. The TotalAlloc section shows the memory increase for each iteration.
  15. go test -run TestMemoryUsage -memprofile=mem.out
  16. go tool pprof --alloc_space needle.test mem.out
  17. */
  18. func TestMemoryUsage(t *testing.T) {
  19. var maps []*CompactMap
  20. totalRowCount := uint64(0)
  21. startTime := time.Now()
  22. for i := 0; i < 10; i++ {
  23. indexFile, ie := os.OpenFile("../../../test/data/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
  24. if ie != nil {
  25. log.Fatalln(ie)
  26. }
  27. m, rowCount := loadNewNeedleMap(indexFile)
  28. maps = append(maps, m)
  29. totalRowCount += rowCount
  30. indexFile.Close()
  31. PrintMemUsage(totalRowCount)
  32. now := time.Now()
  33. fmt.Printf("\tTaken = %v\n", now.Sub(startTime))
  34. startTime = now
  35. }
  36. }
  37. func loadNewNeedleMap(file *os.File) (*CompactMap, uint64) {
  38. m := NewCompactMap()
  39. bytes := make([]byte, NeedleMapEntrySize)
  40. rowCount := uint64(0)
  41. count, e := file.Read(bytes)
  42. for count > 0 && e == nil {
  43. for i := 0; i < count; i += NeedleMapEntrySize {
  44. rowCount++
  45. key := BytesToNeedleId(bytes[i : i+NeedleIdSize])
  46. offset := BytesToOffset(bytes[i+NeedleIdSize : i+NeedleIdSize+OffsetSize])
  47. size := BytesToSize(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize])
  48. if !offset.IsZero() {
  49. m.Set(NeedleId(key), offset, size)
  50. } else {
  51. m.Delete(key)
  52. }
  53. }
  54. count, e = file.Read(bytes)
  55. }
  56. return m, rowCount
  57. }
  58. func PrintMemUsage(totalRowCount uint64) {
  59. runtime.GC()
  60. var m runtime.MemStats
  61. runtime.ReadMemStats(&m)
  62. // For info on each, see: https://golang.org/pkg/runtime/#MemStats
  63. fmt.Printf("Each %.2f Bytes", float64(m.TotalAlloc)/float64(totalRowCount))
  64. fmt.Printf("\tAlloc = %v MiB", bToMb(m.Alloc))
  65. fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
  66. fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
  67. fmt.Printf("\tNumGC = %v", m.NumGC)
  68. }
  69. func bToMb(b uint64) uint64 {
  70. return b / 1024 / 1024
  71. }