lookup.go 3.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139
  1. package operation
  2. import (
  3. "context"
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "google.golang.org/grpc"
  8. "math/rand"
  9. "net/url"
  10. "strings"
  11. "time"
  12. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  13. "github.com/chrislusf/seaweedfs/weed/util"
  14. )
  15. type Location struct {
  16. Url string `json:"url,omitempty"`
  17. PublicUrl string `json:"publicUrl,omitempty"`
  18. }
  19. type LookupResult struct {
  20. VolumeId string `json:"volumeId,omitempty"`
  21. Locations []Location `json:"locations,omitempty"`
  22. Error string `json:"error,omitempty"`
  23. }
  24. func (lr *LookupResult) String() string {
  25. return fmt.Sprintf("VolumeId:%s, Locations:%v, Error:%s", lr.VolumeId, lr.Locations, lr.Error)
  26. }
  27. var (
  28. vc VidCache // caching of volume locations, re-check if after 10 minutes
  29. )
  30. func Lookup(server string, vid string) (ret *LookupResult, err error) {
  31. locations, cache_err := vc.Get(vid)
  32. if cache_err != nil {
  33. if ret, err = do_lookup(server, vid); err == nil {
  34. vc.Set(vid, ret.Locations, 10*time.Minute)
  35. }
  36. } else {
  37. ret = &LookupResult{VolumeId: vid, Locations: locations}
  38. }
  39. return
  40. }
  41. func do_lookup(server string, vid string) (*LookupResult, error) {
  42. values := make(url.Values)
  43. values.Add("volumeId", vid)
  44. jsonBlob, err := util.Post("http://"+server+"/dir/lookup", values)
  45. if err != nil {
  46. return nil, err
  47. }
  48. var ret LookupResult
  49. err = json.Unmarshal(jsonBlob, &ret)
  50. if err != nil {
  51. return nil, err
  52. }
  53. if ret.Error != "" {
  54. return nil, errors.New(ret.Error)
  55. }
  56. return &ret, nil
  57. }
  58. func LookupFileId(server string, fileId string) (fullUrl string, err error) {
  59. parts := strings.Split(fileId, ",")
  60. if len(parts) != 2 {
  61. return "", errors.New("Invalid fileId " + fileId)
  62. }
  63. lookup, lookupError := Lookup(server, parts[0])
  64. if lookupError != nil {
  65. return "", lookupError
  66. }
  67. if len(lookup.Locations) == 0 {
  68. return "", errors.New("File Not Found")
  69. }
  70. return "http://" + lookup.Locations[rand.Intn(len(lookup.Locations))].Url + "/" + fileId, nil
  71. }
  72. // LookupVolumeIds find volume locations by cache and actual lookup
  73. func LookupVolumeIds(server string, grpcDialOption grpc.DialOption, vids []string) (map[string]LookupResult, error) {
  74. ret := make(map[string]LookupResult)
  75. var unknown_vids []string
  76. //check vid cache first
  77. for _, vid := range vids {
  78. locations, cache_err := vc.Get(vid)
  79. if cache_err == nil {
  80. ret[vid] = LookupResult{VolumeId: vid, Locations: locations}
  81. } else {
  82. unknown_vids = append(unknown_vids, vid)
  83. }
  84. }
  85. //return success if all volume ids are known
  86. if len(unknown_vids) == 0 {
  87. return ret, nil
  88. }
  89. //only query unknown_vids
  90. err := WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
  91. req := &master_pb.LookupVolumeRequest{
  92. VolumeIds: unknown_vids,
  93. }
  94. resp, grpcErr := masterClient.LookupVolume(context.Background(), req)
  95. if grpcErr != nil {
  96. return grpcErr
  97. }
  98. //set newly checked vids to cache
  99. for _, vidLocations := range resp.VolumeIdLocations {
  100. var locations []Location
  101. for _, loc := range vidLocations.Locations {
  102. locations = append(locations, Location{
  103. Url: loc.Url,
  104. PublicUrl: loc.PublicUrl,
  105. })
  106. }
  107. if vidLocations.Error != "" {
  108. vc.Set(vidLocations.VolumeId, locations, 10*time.Minute)
  109. }
  110. ret[vidLocations.VolumeId] = LookupResult{
  111. VolumeId: vidLocations.VolumeId,
  112. Locations: locations,
  113. Error: vidLocations.Error,
  114. }
  115. }
  116. return nil
  117. })
  118. if err != nil {
  119. return nil, err
  120. }
  121. return ret, nil
  122. }