replicator.go 3.0 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697
  1. package replication
  2. import (
  3. "context"
  4. "fmt"
  5. "github.com/chrislusf/seaweedfs/weed/pb"
  6. "google.golang.org/grpc"
  7. "strings"
  8. "time"
  9. "github.com/chrislusf/seaweedfs/weed/glog"
  10. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  11. "github.com/chrislusf/seaweedfs/weed/replication/sink"
  12. "github.com/chrislusf/seaweedfs/weed/replication/source"
  13. "github.com/chrislusf/seaweedfs/weed/util"
  14. )
  15. type Replicator struct {
  16. sink sink.ReplicationSink
  17. source *source.FilerSource
  18. }
  19. func NewReplicator(sourceConfig util.Configuration, configPrefix string, dataSink sink.ReplicationSink) *Replicator {
  20. source := &source.FilerSource{}
  21. source.Initialize(sourceConfig, configPrefix)
  22. dataSink.SetSourceFiler(source)
  23. return &Replicator{
  24. sink: dataSink,
  25. source: source,
  26. }
  27. }
  28. func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_pb.EventNotification) error {
  29. if message.IsFromOtherCluster && r.sink.GetName() == "filer" {
  30. return nil
  31. }
  32. if !strings.HasPrefix(key, r.source.Dir) {
  33. glog.V(4).Infof("skipping %v outside of %v", key, r.source.Dir)
  34. return nil
  35. }
  36. var dateKey string
  37. if r.sink.IsIncremental() {
  38. var mTime int64
  39. if message.NewEntry != nil {
  40. mTime = message.NewEntry.Attributes.Mtime
  41. } else if message.OldEntry != nil {
  42. mTime = message.OldEntry.Attributes.Mtime
  43. }
  44. dateKey = time.Unix(mTime, 0).Format("2006-01-02")
  45. }
  46. newKey := util.Join(r.sink.GetSinkToDirectory(), dateKey, key[len(r.source.Dir):])
  47. glog.V(3).Infof("replicate %s => %s", key, newKey)
  48. key = newKey
  49. if message.OldEntry != nil && message.NewEntry == nil {
  50. glog.V(4).Infof("deleting %v", key)
  51. return r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
  52. }
  53. if message.OldEntry == nil && message.NewEntry != nil {
  54. glog.V(4).Infof("creating %v", key)
  55. return r.sink.CreateEntry(key, message.NewEntry, message.Signatures)
  56. }
  57. if message.OldEntry == nil && message.NewEntry == nil {
  58. glog.V(0).Infof("weird message %+v", message)
  59. return nil
  60. }
  61. foundExisting, err := r.sink.UpdateEntry(key, message.OldEntry, message.NewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
  62. if foundExisting {
  63. glog.V(4).Infof("updated %v", key)
  64. return err
  65. }
  66. err = r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, false, message.Signatures)
  67. if err != nil {
  68. return fmt.Errorf("delete old entry %v: %v", key, err)
  69. }
  70. glog.V(4).Infof("creating missing %v", key)
  71. return r.sink.CreateEntry(key, message.NewEntry, message.Signatures)
  72. }
  73. func ReadFilerSignature(grpcDialOption grpc.DialOption, filer string) (filerSignature int32, readErr error) {
  74. if readErr = pb.WithFilerClient(filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  75. if resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}); err != nil {
  76. return fmt.Errorf("GetFilerConfiguration %s: %v", filer, err)
  77. } else {
  78. filerSignature = resp.Signature
  79. }
  80. return nil
  81. }); readErr != nil {
  82. return 0, readErr
  83. }
  84. return filerSignature, nil
  85. }