bucket_metadata_test.go 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231
  1. package s3api
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "github.com/aws/aws-sdk-go/service/s3"
  6. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  7. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
  8. "github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
  9. "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
  10. "reflect"
  11. "sync"
  12. "testing"
  13. "time"
  14. )
  15. type BucketMetadataTestCase struct {
  16. filerEntry *filer_pb.Entry
  17. expectBucketMetadata *BucketMetaData
  18. }
  19. var (
  20. //bad entry
  21. badEntry = &filer_pb.Entry{
  22. Name: "badEntry",
  23. }
  24. //good entry
  25. goodEntryAcl, _ = json.Marshal(s3_constants.PublicRead)
  26. goodEntry = &filer_pb.Entry{
  27. Name: "entryWithValidAcp",
  28. Extended: map[string][]byte{
  29. s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
  30. s3_constants.ExtAmzOwnerKey: []byte(s3account.AccountAdmin.Name),
  31. s3_constants.ExtAmzAclKey: goodEntryAcl,
  32. },
  33. }
  34. //ownership is ""
  35. ownershipEmptyStr = &filer_pb.Entry{
  36. Name: "ownershipEmptyStr",
  37. Extended: map[string][]byte{
  38. s3_constants.ExtOwnershipKey: []byte(""),
  39. },
  40. }
  41. //ownership valid
  42. ownershipValid = &filer_pb.Entry{
  43. Name: "ownershipValid",
  44. Extended: map[string][]byte{
  45. s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
  46. },
  47. }
  48. //owner is ""
  49. acpEmptyStr = &filer_pb.Entry{
  50. Name: "acpEmptyStr",
  51. Extended: map[string][]byte{
  52. s3_constants.ExtAmzOwnerKey: []byte(""),
  53. },
  54. }
  55. //owner not exists
  56. acpEmptyObject = &filer_pb.Entry{
  57. Name: "acpEmptyObject",
  58. Extended: map[string][]byte{
  59. s3_constants.ExtAmzOwnerKey: []byte("xxxxx"),
  60. },
  61. }
  62. //grants is nil
  63. acpOwnerNilAcp, _ = json.Marshal(make([]*s3.Grant, 0))
  64. acpOwnerNil = &filer_pb.Entry{
  65. Name: "acpOwnerNil",
  66. Extended: map[string][]byte{
  67. s3_constants.ExtAmzAclKey: acpOwnerNilAcp,
  68. },
  69. }
  70. //load filer is
  71. loadFilerBucket = make(map[string]int, 1)
  72. //override `loadBucketMetadataFromFiler` to avoid really load from filer
  73. )
  74. var tcs = []*BucketMetadataTestCase{
  75. {
  76. badEntry, &BucketMetaData{
  77. Name: badEntry.Name,
  78. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  79. Owner: &s3.Owner{
  80. DisplayName: &s3account.AccountAdmin.Name,
  81. ID: &s3account.AccountAdmin.Id,
  82. },
  83. Acl: nil,
  84. },
  85. },
  86. {
  87. goodEntry, &BucketMetaData{
  88. Name: goodEntry.Name,
  89. ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
  90. Owner: &s3.Owner{
  91. DisplayName: &s3account.AccountAdmin.Name,
  92. ID: &s3account.AccountAdmin.Id,
  93. },
  94. Acl: s3_constants.PublicRead,
  95. },
  96. },
  97. {
  98. ownershipEmptyStr, &BucketMetaData{
  99. Name: ownershipEmptyStr.Name,
  100. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  101. Owner: &s3.Owner{
  102. DisplayName: &s3account.AccountAdmin.Name,
  103. ID: &s3account.AccountAdmin.Id,
  104. },
  105. Acl: nil,
  106. },
  107. },
  108. {
  109. ownershipValid, &BucketMetaData{
  110. Name: ownershipValid.Name,
  111. ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
  112. Owner: &s3.Owner{
  113. DisplayName: &s3account.AccountAdmin.Name,
  114. ID: &s3account.AccountAdmin.Id,
  115. },
  116. Acl: nil,
  117. },
  118. },
  119. {
  120. acpEmptyStr, &BucketMetaData{
  121. Name: acpEmptyStr.Name,
  122. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  123. Owner: &s3.Owner{
  124. DisplayName: &s3account.AccountAdmin.Name,
  125. ID: &s3account.AccountAdmin.Id,
  126. },
  127. Acl: nil,
  128. },
  129. },
  130. {
  131. acpEmptyObject, &BucketMetaData{
  132. Name: acpEmptyObject.Name,
  133. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  134. Owner: &s3.Owner{
  135. DisplayName: &s3account.AccountAdmin.Name,
  136. ID: &s3account.AccountAdmin.Id,
  137. },
  138. Acl: nil,
  139. },
  140. },
  141. {
  142. acpOwnerNil, &BucketMetaData{
  143. Name: acpOwnerNil.Name,
  144. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  145. Owner: &s3.Owner{
  146. DisplayName: &s3account.AccountAdmin.Name,
  147. ID: &s3account.AccountAdmin.Id,
  148. },
  149. Acl: make([]*s3.Grant, 0),
  150. },
  151. },
  152. }
  153. func TestBuildBucketMetadata(t *testing.T) {
  154. accountManager := &s3account.AccountManager{
  155. IdNameMapping: map[string]string{
  156. s3account.AccountAdmin.Id: s3account.AccountAdmin.Name,
  157. s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name,
  158. },
  159. }
  160. for _, tc := range tcs {
  161. resultBucketMetadata := buildBucketMetadata(accountManager, tc.filerEntry)
  162. if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) {
  163. t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata)
  164. }
  165. }
  166. }
  167. func TestGetBucketMetadata(t *testing.T) {
  168. loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
  169. time.Sleep(time.Second)
  170. loadFilerBucket[bucketName] = loadFilerBucket[bucketName] + 1
  171. return &BucketMetaData{
  172. Name: bucketName,
  173. }, nil
  174. }
  175. br := &BucketRegistry{
  176. metadataCache: make(map[string]*BucketMetaData),
  177. notFound: make(map[string]struct{}),
  178. s3a: nil,
  179. }
  180. //start 40 goroutine for
  181. var wg sync.WaitGroup
  182. closeCh := make(chan struct{})
  183. for i := 0; i < 40; i++ {
  184. wg.Add(1)
  185. go func() {
  186. defer wg.Done()
  187. outLoop:
  188. for {
  189. for j := 0; j < 5; j++ {
  190. select {
  191. case <-closeCh:
  192. break outLoop
  193. default:
  194. reqBucket := fmt.Sprintf("%c", 67+j)
  195. _, errCode := br.GetBucketMetadata(reqBucket)
  196. if errCode != s3err.ErrNone {
  197. close(closeCh)
  198. t.Error("not expect")
  199. }
  200. }
  201. }
  202. time.Sleep(10 * time.Microsecond)
  203. }
  204. }()
  205. }
  206. time.Sleep(time.Second)
  207. close(closeCh)
  208. wg.Wait()
  209. //Each bucket is loaded from the filer only once
  210. for bucketName, loadCount := range loadFilerBucket {
  211. if loadCount != 1 {
  212. t.Fatalf("lock is uneffict: %s, %d", bucketName, loadCount)
  213. }
  214. }
  215. }