post-policy_test.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378
  1. package policy
  2. /*
  3. * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
  4. *
  5. * Licensed under the Apache License, Version 2.0 (the "License");
  6. * you may not use this file except in compliance with the License.
  7. * You may obtain a copy of the License at
  8. *
  9. * http://www.apache.org/licenses/LICENSE-2.0
  10. *
  11. * Unless required by applicable law or agreed to in writing, software
  12. * distributed under the License is distributed on an "AS IS" BASIS,
  13. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. * See the License for the specific language governing permissions and
  15. * limitations under the License.
  16. */
  17. import (
  18. "bytes"
  19. "crypto/hmac"
  20. "crypto/sha1"
  21. "crypto/sha256"
  22. "encoding/base64"
  23. "encoding/hex"
  24. "fmt"
  25. "mime/multipart"
  26. "net/http"
  27. "net/url"
  28. "regexp"
  29. "strings"
  30. "time"
  31. "unicode/utf8"
  32. )
  33. const (
  34. iso8601DateFormat = "20060102T150405Z"
  35. iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
  36. )
  37. func newPostPolicyBytesV4WithContentRange(credential, bucketName, objectKey string, expiration time.Time) []byte {
  38. t := time.Now().UTC()
  39. // Add the expiration date.
  40. expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat))
  41. // Add the bucket condition, only accept buckets equal to the one passed.
  42. bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
  43. // Add the key condition, only accept keys equal to the one passed.
  44. keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s/upload.txt"]`, objectKey)
  45. // Add content length condition, only accept content sizes of a given length.
  46. contentLengthCondStr := `["content-length-range", 1024, 1048576]`
  47. // Add the algorithm condition, only accept AWS SignV4 Sha256.
  48. algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]`
  49. // Add the date condition, only accept the current date.
  50. dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat))
  51. // Add the credential string, only accept the credential passed.
  52. credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential)
  53. // Add the meta-uuid string, set to 1234
  54. uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234")
  55. // Combine all conditions into one string.
  56. conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr,
  57. keyConditionStr, contentLengthCondStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr)
  58. retStr := "{"
  59. retStr = retStr + expirationStr + ","
  60. retStr = retStr + conditionStr
  61. retStr = retStr + "}"
  62. return []byte(retStr)
  63. }
  64. // newPostPolicyBytesV4 - creates a bare bones postpolicy string with key and bucket matches.
  65. func newPostPolicyBytesV4(credential, bucketName, objectKey string, expiration time.Time) []byte {
  66. t := time.Now().UTC()
  67. // Add the expiration date.
  68. expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat))
  69. // Add the bucket condition, only accept buckets equal to the one passed.
  70. bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
  71. // Add the key condition, only accept keys equal to the one passed.
  72. keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s/upload.txt"]`, objectKey)
  73. // Add the algorithm condition, only accept AWS SignV4 Sha256.
  74. algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]`
  75. // Add the date condition, only accept the current date.
  76. dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat))
  77. // Add the credential string, only accept the credential passed.
  78. credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential)
  79. // Add the meta-uuid string, set to 1234
  80. uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234")
  81. // Combine all conditions into one string.
  82. conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr)
  83. retStr := "{"
  84. retStr = retStr + expirationStr + ","
  85. retStr = retStr + conditionStr
  86. retStr = retStr + "}"
  87. return []byte(retStr)
  88. }
  89. // newPostPolicyBytesV2 - creates a bare bones postpolicy string with key and bucket matches.
  90. func newPostPolicyBytesV2(bucketName, objectKey string, expiration time.Time) []byte {
  91. // Add the expiration date.
  92. expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat))
  93. // Add the bucket condition, only accept buckets equal to the one passed.
  94. bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName)
  95. // Add the key condition, only accept keys equal to the one passed.
  96. keyConditionStr := fmt.Sprintf(`["starts-with", "$key", "%s/upload.txt"]`, objectKey)
  97. // Combine all conditions into one string.
  98. conditionStr := fmt.Sprintf(`"conditions":[%s, %s]`, bucketConditionStr, keyConditionStr)
  99. retStr := "{"
  100. retStr = retStr + expirationStr + ","
  101. retStr = retStr + conditionStr
  102. retStr = retStr + "}"
  103. return []byte(retStr)
  104. }
  105. // Wrapper for calling TestPostPolicyBucketHandler tests for both Erasure multiple disks and single node setup.
  106. // testPostPolicyBucketHandler - Tests validate post policy handler uploading objects.
  107. // Wrapper for calling TestPostPolicyBucketHandlerRedirect tests for both Erasure multiple disks and single node setup.
  108. // testPostPolicyBucketHandlerRedirect tests POST Object when success_action_redirect is specified
  109. // postPresignSignatureV4 - presigned signature for PostPolicy requests.
  110. func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
  111. // Get signining key.
  112. signingkey := getSigningKey(secretAccessKey, t, location)
  113. // Calculate signature.
  114. signature := getSignature(signingkey, policyBase64)
  115. return signature
  116. }
  117. // copied from auth_signature_v4.go to break import loop
  118. // sumHMAC calculate hmac between two input byte array.
  119. func sumHMAC(key []byte, data []byte) []byte {
  120. hash := hmac.New(sha256.New, key)
  121. hash.Write(data)
  122. return hash.Sum(nil)
  123. }
  124. // copied from auth_signature_v4.go to break import loop
  125. // getSigningKey hmac seed to calculate final signature.
  126. func getSigningKey(secretKey string, t time.Time, region string) []byte {
  127. date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format("20060102")))
  128. regionBytes := sumHMAC(date, []byte(region))
  129. service := sumHMAC(regionBytes, []byte("s3"))
  130. signingKey := sumHMAC(service, []byte("aws4_request"))
  131. return signingKey
  132. }
  133. // copied from auth_signature_v4.go to break import loop
  134. // getSignature final signature in hexadecimal form.
  135. func getSignature(signingKey []byte, stringToSign string) string {
  136. return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
  137. }
  138. // copied from auth_signature_v4.go to break import loop
  139. func calculateSignatureV2(stringToSign string, secret string) string {
  140. hm := hmac.New(sha1.New, []byte(secret))
  141. hm.Write([]byte(stringToSign))
  142. return base64.StdEncoding.EncodeToString(hm.Sum(nil))
  143. }
  144. func newPostRequestV2(endPoint, bucketName, objectName string, accessKey, secretKey string) (*http.Request, error) {
  145. // Expire the request five minutes from now.
  146. expirationTime := time.Now().UTC().Add(time.Minute * 5)
  147. // Create a new post policy.
  148. policy := newPostPolicyBytesV2(bucketName, objectName, expirationTime)
  149. // Only need the encoding.
  150. encodedPolicy := base64.StdEncoding.EncodeToString(policy)
  151. // Presign with V4 signature based on the policy.
  152. signature := calculateSignatureV2(encodedPolicy, secretKey)
  153. formData := map[string]string{
  154. "AWSAccessKeyId": accessKey,
  155. "bucket": bucketName,
  156. "key": objectName + "/${filename}",
  157. "policy": encodedPolicy,
  158. "signature": signature,
  159. }
  160. // Create the multipart form.
  161. var buf bytes.Buffer
  162. w := multipart.NewWriter(&buf)
  163. // Set the normal formData
  164. for k, v := range formData {
  165. w.WriteField(k, v)
  166. }
  167. // Set the File formData
  168. writer, err := w.CreateFormFile("file", "upload.txt")
  169. if err != nil {
  170. // return nil, err
  171. return nil, err
  172. }
  173. writer.Write([]byte("hello world"))
  174. // Close before creating the new request.
  175. w.Close()
  176. // Set the body equal to the created policy.
  177. reader := bytes.NewReader(buf.Bytes())
  178. req, err := http.NewRequest(http.MethodPost, makeTestTargetURL(endPoint, bucketName, "", nil), reader)
  179. if err != nil {
  180. return nil, err
  181. }
  182. // Set form content-type.
  183. req.Header.Set("Content-Type", w.FormDataContentType())
  184. return req, nil
  185. }
  186. func buildGenericPolicy(t time.Time, accessKey, region, bucketName, objectName string, contentLengthRange bool) []byte {
  187. // Expire the request five minutes from now.
  188. expirationTime := t.Add(time.Minute * 5)
  189. credStr := getCredentialString(accessKey, region, t)
  190. // Create a new post policy.
  191. policy := newPostPolicyBytesV4(credStr, bucketName, objectName, expirationTime)
  192. if contentLengthRange {
  193. policy = newPostPolicyBytesV4WithContentRange(credStr, bucketName, objectName, expirationTime)
  194. }
  195. return policy
  196. }
  197. func newPostRequestV4Generic(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string, region string,
  198. t time.Time, policy []byte, addFormData map[string]string, corruptedB64 bool, corruptedMultipart bool) (*http.Request, error) {
  199. // Get the user credential.
  200. credStr := getCredentialString(accessKey, region, t)
  201. // Only need the encoding.
  202. encodedPolicy := base64.StdEncoding.EncodeToString(policy)
  203. if corruptedB64 {
  204. encodedPolicy = "%!~&" + encodedPolicy
  205. }
  206. // Presign with V4 signature based on the policy.
  207. signature := postPresignSignatureV4(encodedPolicy, t, secretKey, region)
  208. formData := map[string]string{
  209. "bucket": bucketName,
  210. "key": objectName + "/${filename}",
  211. "x-amz-credential": credStr,
  212. "policy": encodedPolicy,
  213. "x-amz-signature": signature,
  214. "x-amz-date": t.Format(iso8601DateFormat),
  215. "x-amz-algorithm": "AWS4-HMAC-SHA256",
  216. "x-amz-meta-uuid": "1234",
  217. "Content-Encoding": "gzip",
  218. }
  219. // Add form data
  220. for k, v := range addFormData {
  221. formData[k] = v
  222. }
  223. // Create the multipart form.
  224. var buf bytes.Buffer
  225. w := multipart.NewWriter(&buf)
  226. // Set the normal formData
  227. for k, v := range formData {
  228. w.WriteField(k, v)
  229. }
  230. // Set the File formData but don't if we want send an incomplete multipart request
  231. if !corruptedMultipart {
  232. writer, err := w.CreateFormFile("file", "upload.txt")
  233. if err != nil {
  234. // return nil, err
  235. return nil, err
  236. }
  237. writer.Write(objData)
  238. // Close before creating the new request.
  239. w.Close()
  240. }
  241. // Set the body equal to the created policy.
  242. reader := bytes.NewReader(buf.Bytes())
  243. req, err := http.NewRequest(http.MethodPost, makeTestTargetURL(endPoint, bucketName, "", nil), reader)
  244. if err != nil {
  245. return nil, err
  246. }
  247. // Set form content-type.
  248. req.Header.Set("Content-Type", w.FormDataContentType())
  249. return req, nil
  250. }
  251. func newPostRequestV4WithContentLength(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) {
  252. t := time.Now().UTC()
  253. region := "us-east-1"
  254. policy := buildGenericPolicy(t, accessKey, region, bucketName, objectName, true)
  255. return newPostRequestV4Generic(endPoint, bucketName, objectName, objData, accessKey, secretKey, region, t, policy, nil, false, false)
  256. }
  257. func newPostRequestV4(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) {
  258. t := time.Now().UTC()
  259. region := "us-east-1"
  260. policy := buildGenericPolicy(t, accessKey, region, bucketName, objectName, false)
  261. return newPostRequestV4Generic(endPoint, bucketName, objectName, objData, accessKey, secretKey, region, t, policy, nil, false, false)
  262. }
  263. // construct URL for http requests for bucket operations.
  264. func makeTestTargetURL(endPoint, bucketName, objectName string, queryValues url.Values) string {
  265. urlStr := endPoint + "/"
  266. if bucketName != "" {
  267. urlStr = urlStr + bucketName + "/"
  268. }
  269. if objectName != "" {
  270. urlStr = urlStr + EncodePath(objectName)
  271. }
  272. if len(queryValues) > 0 {
  273. urlStr = urlStr + "?" + queryValues.Encode()
  274. }
  275. return urlStr
  276. }
  277. // if object matches reserved string, no need to encode them
  278. var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
  279. // EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
  280. //
  281. // This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
  282. // non english characters cannot be parsed due to the nature in which url.Encode() is written
  283. //
  284. // This function on the other hand is a direct replacement for url.Encode() technique to support
  285. // pretty much every UTF-8 character.
  286. func EncodePath(pathName string) string {
  287. if reservedObjectNames.MatchString(pathName) {
  288. return pathName
  289. }
  290. var encodedPathname string
  291. for _, s := range pathName {
  292. if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
  293. encodedPathname = encodedPathname + string(s)
  294. continue
  295. }
  296. switch s {
  297. case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
  298. encodedPathname = encodedPathname + string(s)
  299. continue
  300. default:
  301. len := utf8.RuneLen(s)
  302. if len < 0 {
  303. // if utf8 cannot convert return the same string as is
  304. return pathName
  305. }
  306. u := make([]byte, len)
  307. utf8.EncodeRune(u, s)
  308. for _, r := range u {
  309. hex := hex.EncodeToString([]byte{r})
  310. encodedPathname = encodedPathname + "%" + strings.ToUpper(hex)
  311. }
  312. }
  313. }
  314. return encodedPathname
  315. }
  316. // getCredentialString generate a credential string.
  317. func getCredentialString(accessKeyID, location string, t time.Time) string {
  318. return accessKeyID + "/" + getScope(t, location)
  319. }
  320. // getScope generate a string of a specific date, an AWS region, and a service.
  321. func getScope(t time.Time, region string) string {
  322. scope := strings.Join([]string{
  323. t.Format("20060102"),
  324. region,
  325. string("s3"),
  326. "aws4_request",
  327. }, "/")
  328. return scope
  329. }