walk.go 1.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354
  1. package idx
  2. import (
  3. "io"
  4. "github.com/chrislusf/seaweedfs/weed/glog"
  5. "github.com/chrislusf/seaweedfs/weed/storage/types"
  6. )
  7. // walks through the index file, calls fn function with each key, offset, size
  8. // stops with the error returned by the fn function
  9. func WalkIndexFile(r io.ReaderAt, fn func(key types.NeedleId, offset types.Offset, size types.Size) error) error {
  10. var readerOffset int64
  11. bytes := make([]byte, types.NeedleMapEntrySize*RowsToRead)
  12. count, e := r.ReadAt(bytes, readerOffset)
  13. if count == 0 && e == io.EOF {
  14. return nil
  15. }
  16. glog.V(3).Infof("readerOffset %d count %d err: %v", readerOffset, count, e)
  17. readerOffset += int64(count)
  18. var (
  19. key types.NeedleId
  20. offset types.Offset
  21. size types.Size
  22. i int
  23. )
  24. for count > 0 && e == nil || e == io.EOF {
  25. for i = 0; i+types.NeedleMapEntrySize <= count; i += types.NeedleMapEntrySize {
  26. key, offset, size = IdxFileEntry(bytes[i : i+types.NeedleMapEntrySize])
  27. if e = fn(key, offset, size); e != nil {
  28. return e
  29. }
  30. }
  31. if e == io.EOF {
  32. return nil
  33. }
  34. count, e = r.ReadAt(bytes, readerOffset)
  35. glog.V(3).Infof("readerOffset %d count %d err: %v", readerOffset, count, e)
  36. readerOffset += int64(count)
  37. }
  38. return e
  39. }
  40. func IdxFileEntry(bytes []byte) (key types.NeedleId, offset types.Offset, size types.Size) {
  41. key = types.BytesToNeedleId(bytes[:types.NeedleIdSize])
  42. offset = types.BytesToOffset(bytes[types.NeedleIdSize : types.NeedleIdSize+types.OffsetSize])
  43. size = types.BytesToSize(bytes[types.NeedleIdSize+types.OffsetSize : types.NeedleIdSize+types.OffsetSize+types.SizeSize])
  44. return
  45. }
  46. const (
  47. RowsToRead = 1024
  48. )