filer_grpc_server.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374
  1. package weed_server
  2. import (
  3. "context"
  4. "fmt"
  5. "os"
  6. "path/filepath"
  7. "strconv"
  8. "time"
  9. "github.com/seaweedfs/seaweedfs/weed/cluster"
  10. "github.com/seaweedfs/seaweedfs/weed/filer"
  11. "github.com/seaweedfs/seaweedfs/weed/glog"
  12. "github.com/seaweedfs/seaweedfs/weed/operation"
  13. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  14. "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
  15. "github.com/seaweedfs/seaweedfs/weed/storage/needle"
  16. "github.com/seaweedfs/seaweedfs/weed/util"
  17. )
  18. func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
  19. glog.V(4).Infof("LookupDirectoryEntry %s", filepath.Join(req.Directory, req.Name))
  20. entry, err := fs.filer.FindEntry(ctx, util.JoinPath(req.Directory, req.Name))
  21. if err == filer_pb.ErrNotFound {
  22. return &filer_pb.LookupDirectoryEntryResponse{}, err
  23. }
  24. if err != nil {
  25. glog.V(3).Infof("LookupDirectoryEntry %s: %+v, ", filepath.Join(req.Directory, req.Name), err)
  26. return nil, err
  27. }
  28. return &filer_pb.LookupDirectoryEntryResponse{
  29. Entry: entry.ToProtoEntry(),
  30. }, nil
  31. }
  32. func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) (err error) {
  33. glog.V(4).Infof("ListEntries %v", req)
  34. limit := int(req.Limit)
  35. if limit == 0 {
  36. limit = fs.option.DirListingLimit
  37. }
  38. paginationLimit := filer.PaginationSize
  39. if limit < paginationLimit {
  40. paginationLimit = limit
  41. }
  42. lastFileName := req.StartFromFileName
  43. includeLastFile := req.InclusiveStartFrom
  44. var listErr error
  45. for limit > 0 {
  46. var hasEntries bool
  47. lastFileName, listErr = fs.filer.StreamListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, int64(paginationLimit), req.Prefix, "", "", func(entry *filer.Entry) bool {
  48. hasEntries = true
  49. if err = stream.Send(&filer_pb.ListEntriesResponse{
  50. Entry: entry.ToProtoEntry(),
  51. }); err != nil {
  52. return false
  53. }
  54. limit--
  55. if limit == 0 {
  56. return false
  57. }
  58. return true
  59. })
  60. if listErr != nil {
  61. return listErr
  62. }
  63. if err != nil {
  64. return err
  65. }
  66. if !hasEntries {
  67. return nil
  68. }
  69. includeLastFile = false
  70. }
  71. return nil
  72. }
  73. func (fs *FilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
  74. resp := &filer_pb.LookupVolumeResponse{
  75. LocationsMap: make(map[string]*filer_pb.Locations),
  76. }
  77. for _, vidString := range req.VolumeIds {
  78. vid, err := strconv.Atoi(vidString)
  79. if err != nil {
  80. glog.V(1).Infof("Unknown volume id %d", vid)
  81. return nil, err
  82. }
  83. var locs []*filer_pb.Location
  84. locations, found := fs.filer.MasterClient.GetLocations(uint32(vid))
  85. if !found {
  86. continue
  87. }
  88. for _, loc := range locations {
  89. locs = append(locs, &filer_pb.Location{
  90. Url: loc.Url,
  91. PublicUrl: loc.PublicUrl,
  92. GrpcPort: uint32(loc.GrpcPort),
  93. DataCenter: loc.DataCenter,
  94. })
  95. }
  96. resp.LocationsMap[vidString] = &filer_pb.Locations{
  97. Locations: locs,
  98. }
  99. }
  100. return resp, nil
  101. }
  102. func (fs *FilerServer) lookupFileId(fileId string) (targetUrls []string, err error) {
  103. fid, err := needle.ParseFileIdFromString(fileId)
  104. if err != nil {
  105. return nil, err
  106. }
  107. locations, found := fs.filer.MasterClient.GetLocations(uint32(fid.VolumeId))
  108. if !found || len(locations) == 0 {
  109. return nil, fmt.Errorf("not found volume %d in %s", fid.VolumeId, fileId)
  110. }
  111. for _, loc := range locations {
  112. targetUrls = append(targetUrls, fmt.Sprintf("http://%s/%s", loc.Url, fileId))
  113. }
  114. return
  115. }
  116. func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (resp *filer_pb.CreateEntryResponse, err error) {
  117. glog.V(4).Infof("CreateEntry %v/%v", req.Directory, req.Entry.Name)
  118. resp = &filer_pb.CreateEntryResponse{}
  119. chunks, garbage, err2 := fs.cleanupChunks(util.Join(req.Directory, req.Entry.Name), nil, req.Entry)
  120. if err2 != nil {
  121. return &filer_pb.CreateEntryResponse{}, fmt.Errorf("CreateEntry cleanupChunks %s %s: %v", req.Directory, req.Entry.Name, err2)
  122. }
  123. so, err := fs.detectStorageOption(string(util.NewFullPath(req.Directory, req.Entry.Name)), "", "", 0, "", "", "", "")
  124. if err != nil {
  125. return nil, err
  126. }
  127. newEntry := filer.FromPbEntry(req.Directory, req.Entry)
  128. newEntry.Chunks = chunks
  129. newEntry.TtlSec = so.TtlSeconds
  130. createErr := fs.filer.CreateEntry(ctx, newEntry, req.OExcl, req.IsFromOtherCluster, req.Signatures, req.SkipCheckParentDirectory, so.MaxFileNameLength)
  131. if createErr == nil {
  132. fs.filer.DeleteChunksNotRecursive(garbage)
  133. } else {
  134. glog.V(3).Infof("CreateEntry %s: %v", filepath.Join(req.Directory, req.Entry.Name), createErr)
  135. resp.Error = createErr.Error()
  136. }
  137. return
  138. }
  139. func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
  140. glog.V(4).Infof("UpdateEntry %v", req)
  141. fullpath := util.Join(req.Directory, req.Entry.Name)
  142. entry, err := fs.filer.FindEntry(ctx, util.FullPath(fullpath))
  143. if err != nil {
  144. return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("not found %s: %v", fullpath, err)
  145. }
  146. chunks, garbage, err2 := fs.cleanupChunks(fullpath, entry, req.Entry)
  147. if err2 != nil {
  148. return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("UpdateEntry cleanupChunks %s: %v", fullpath, err2)
  149. }
  150. newEntry := filer.FromPbEntry(req.Directory, req.Entry)
  151. newEntry.Chunks = chunks
  152. if filer.EqualEntry(entry, newEntry) {
  153. return &filer_pb.UpdateEntryResponse{}, err
  154. }
  155. if err = fs.filer.UpdateEntry(ctx, entry, newEntry); err == nil {
  156. fs.filer.DeleteChunksNotRecursive(garbage)
  157. fs.filer.NotifyUpdateEvent(ctx, entry, newEntry, true, req.IsFromOtherCluster, req.Signatures)
  158. } else {
  159. glog.V(3).Infof("UpdateEntry %s: %v", filepath.Join(req.Directory, req.Entry.Name), err)
  160. }
  161. return &filer_pb.UpdateEntryResponse{}, err
  162. }
  163. func (fs *FilerServer) cleanupChunks(fullpath string, existingEntry *filer.Entry, newEntry *filer_pb.Entry) (chunks, garbage []*filer_pb.FileChunk, err error) {
  164. // remove old chunks if not included in the new ones
  165. if existingEntry != nil {
  166. garbage, err = filer.MinusChunks(fs.lookupFileId, existingEntry.GetChunks(), newEntry.GetChunks())
  167. if err != nil {
  168. return newEntry.GetChunks(), nil, fmt.Errorf("MinusChunks: %v", err)
  169. }
  170. }
  171. // files with manifest chunks are usually large and append only, skip calculating covered chunks
  172. manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(newEntry.GetChunks())
  173. chunks, coveredChunks := filer.CompactFileChunks(fs.lookupFileId, nonManifestChunks)
  174. garbage = append(garbage, coveredChunks...)
  175. if newEntry.Attributes != nil {
  176. so, _ := fs.detectStorageOption(fullpath,
  177. "",
  178. "",
  179. newEntry.Attributes.TtlSec,
  180. "",
  181. "",
  182. "",
  183. "",
  184. ) // ignore readonly error for capacity needed to manifestize
  185. chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), chunks)
  186. if err != nil {
  187. // not good, but should be ok
  188. glog.V(0).Infof("MaybeManifestize: %v", err)
  189. }
  190. }
  191. chunks = append(chunks, manifestChunks...)
  192. return
  193. }
  194. func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendToEntryRequest) (*filer_pb.AppendToEntryResponse, error) {
  195. glog.V(4).Infof("AppendToEntry %v", req)
  196. fullpath := util.NewFullPath(req.Directory, req.EntryName)
  197. lockClient := cluster.NewLockClient(fs.grpcDialOption, fs.option.Host)
  198. lock := lockClient.NewLock(string(fullpath), string(fs.option.Host))
  199. defer lock.StopLock()
  200. var offset int64 = 0
  201. entry, err := fs.filer.FindEntry(ctx, fullpath)
  202. if err == filer_pb.ErrNotFound {
  203. entry = &filer.Entry{
  204. FullPath: fullpath,
  205. Attr: filer.Attr{
  206. Crtime: time.Now(),
  207. Mtime: time.Now(),
  208. Mode: os.FileMode(0644),
  209. Uid: OS_UID,
  210. Gid: OS_GID,
  211. },
  212. }
  213. } else {
  214. offset = int64(filer.TotalSize(entry.GetChunks()))
  215. }
  216. for _, chunk := range req.Chunks {
  217. chunk.Offset = offset
  218. offset += int64(chunk.Size)
  219. }
  220. entry.Chunks = append(entry.GetChunks(), req.Chunks...)
  221. so, err := fs.detectStorageOption(string(fullpath), "", "", entry.TtlSec, "", "", "", "")
  222. if err != nil {
  223. glog.Warningf("detectStorageOption: %v", err)
  224. return &filer_pb.AppendToEntryResponse{}, err
  225. }
  226. entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), entry.GetChunks())
  227. if err != nil {
  228. // not good, but should be ok
  229. glog.V(0).Infof("MaybeManifestize: %v", err)
  230. }
  231. err = fs.filer.CreateEntry(context.Background(), entry, false, false, nil, false, fs.filer.MaxFilenameLength)
  232. return &filer_pb.AppendToEntryResponse{}, err
  233. }
  234. func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) {
  235. glog.V(4).Infof("DeleteEntry %v", req)
  236. err = fs.filer.DeleteEntryMetaAndData(ctx, util.JoinPath(req.Directory, req.Name), req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData, req.IsFromOtherCluster, req.Signatures)
  237. resp = &filer_pb.DeleteEntryResponse{}
  238. if err != nil && err != filer_pb.ErrNotFound {
  239. resp.Error = err.Error()
  240. }
  241. return resp, nil
  242. }
  243. func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) {
  244. if req.DiskType == "" {
  245. req.DiskType = fs.option.DiskType
  246. }
  247. so, err := fs.detectStorageOption(req.Path, req.Collection, req.Replication, req.TtlSec, req.DiskType, req.DataCenter, req.Rack, req.DataNode)
  248. if err != nil {
  249. glog.V(3).Infof("AssignVolume: %v", err)
  250. return &filer_pb.AssignVolumeResponse{Error: fmt.Sprintf("assign volume: %v", err)}, nil
  251. }
  252. assignRequest, altRequest := so.ToAssignRequests(int(req.Count))
  253. assignResult, err := operation.Assign(fs.filer.GetMaster, fs.grpcDialOption, assignRequest, altRequest)
  254. if err != nil {
  255. glog.V(3).Infof("AssignVolume: %v", err)
  256. return &filer_pb.AssignVolumeResponse{Error: fmt.Sprintf("assign volume: %v", err)}, nil
  257. }
  258. if assignResult.Error != "" {
  259. glog.V(3).Infof("AssignVolume error: %v", assignResult.Error)
  260. return &filer_pb.AssignVolumeResponse{Error: fmt.Sprintf("assign volume result: %v", assignResult.Error)}, nil
  261. }
  262. return &filer_pb.AssignVolumeResponse{
  263. FileId: assignResult.Fid,
  264. Count: int32(assignResult.Count),
  265. Location: &filer_pb.Location{
  266. Url: assignResult.Url,
  267. PublicUrl: assignResult.PublicUrl,
  268. GrpcPort: uint32(assignResult.GrpcPort),
  269. },
  270. Auth: string(assignResult.Auth),
  271. Collection: so.Collection,
  272. Replication: so.Replication,
  273. }, nil
  274. }
  275. func (fs *FilerServer) CollectionList(ctx context.Context, req *filer_pb.CollectionListRequest) (resp *filer_pb.CollectionListResponse, err error) {
  276. glog.V(4).Infof("CollectionList %v", req)
  277. resp = &filer_pb.CollectionListResponse{}
  278. err = fs.filer.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
  279. masterResp, err := client.CollectionList(context.Background(), &master_pb.CollectionListRequest{
  280. IncludeNormalVolumes: req.IncludeNormalVolumes,
  281. IncludeEcVolumes: req.IncludeEcVolumes,
  282. })
  283. if err != nil {
  284. return err
  285. }
  286. for _, c := range masterResp.Collections {
  287. resp.Collections = append(resp.Collections, &filer_pb.Collection{Name: c.Name})
  288. }
  289. return nil
  290. })
  291. return
  292. }
  293. func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.DeleteCollectionRequest) (resp *filer_pb.DeleteCollectionResponse, err error) {
  294. glog.V(4).Infof("DeleteCollection %v", req)
  295. err = fs.filer.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
  296. _, err := client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{
  297. Name: req.GetCollection(),
  298. })
  299. return err
  300. })
  301. return &filer_pb.DeleteCollectionResponse{}, err
  302. }