auto_signature_v4_test.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477
  1. package s3api
  2. import (
  3. "bytes"
  4. "crypto/md5"
  5. "crypto/sha256"
  6. "encoding/base64"
  7. "encoding/hex"
  8. "errors"
  9. "fmt"
  10. "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
  11. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
  12. "io"
  13. "net/http"
  14. "net/url"
  15. "sort"
  16. "strconv"
  17. "strings"
  18. "sync"
  19. "testing"
  20. "time"
  21. "unicode/utf8"
  22. "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
  23. )
  24. // TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature version v4 detection.
  25. func TestIsRequestPresignedSignatureV4(t *testing.T) {
  26. testCases := []struct {
  27. inputQueryKey string
  28. inputQueryValue string
  29. expectedResult bool
  30. }{
  31. // Test case - 1.
  32. // Test case with query key ""X-Amz-Credential" set.
  33. {"", "", false},
  34. // Test case - 2.
  35. {"X-Amz-Credential", "", true},
  36. // Test case - 3.
  37. {"X-Amz-Content-Sha256", "", false},
  38. }
  39. for i, testCase := range testCases {
  40. // creating an input HTTP request.
  41. // Only the query parameters are relevant for this particular test.
  42. inputReq, err := http.NewRequest("GET", "http://example.com", nil)
  43. if err != nil {
  44. t.Fatalf("Error initializing input HTTP request: %v", err)
  45. }
  46. q := inputReq.URL.Query()
  47. q.Add(testCase.inputQueryKey, testCase.inputQueryValue)
  48. inputReq.URL.RawQuery = q.Encode()
  49. actualResult := isRequestPresignedSignatureV4(inputReq)
  50. if testCase.expectedResult != actualResult {
  51. t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult)
  52. }
  53. }
  54. }
  55. // Tests is requested authenticated function, tests replies for s3 errors.
  56. func TestIsReqAuthenticated(t *testing.T) {
  57. iam := &IdentityAccessManagement{
  58. hashes: make(map[string]*sync.Pool),
  59. hashCounters: make(map[string]*int32),
  60. }
  61. _ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
  62. Identities: []*iam_pb.Identity{
  63. {
  64. Name: "someone",
  65. Credentials: []*iam_pb.Credential{
  66. {
  67. AccessKey: "access_key_1",
  68. SecretKey: "secret_key_1",
  69. },
  70. },
  71. Actions: []string{},
  72. },
  73. },
  74. })
  75. // List of test cases for validating http request authentication.
  76. testCases := []struct {
  77. req *http.Request
  78. s3Error s3err.ErrorCode
  79. }{
  80. // When request is unsigned, access denied is returned.
  81. {mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), s3err.ErrAccessDenied},
  82. // When request is properly signed, error is none.
  83. {mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), s3err.ErrNone},
  84. }
  85. // Validates all testcases.
  86. for i, testCase := range testCases {
  87. if _, s3Error := iam.reqSignatureV4Verify(testCase.req); s3Error != testCase.s3Error {
  88. io.ReadAll(testCase.req.Body)
  89. t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d", i, testCase.s3Error, s3Error)
  90. }
  91. }
  92. }
  93. func TestCheckaAnonymousRequestAuthType(t *testing.T) {
  94. iam := &IdentityAccessManagement{
  95. hashes: make(map[string]*sync.Pool),
  96. hashCounters: make(map[string]*int32),
  97. }
  98. _ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
  99. Identities: []*iam_pb.Identity{
  100. {
  101. Name: "anonymous",
  102. Actions: []string{s3_constants.ACTION_READ},
  103. },
  104. },
  105. })
  106. testCases := []struct {
  107. Request *http.Request
  108. ErrCode s3err.ErrorCode
  109. Action Action
  110. }{
  111. {Request: mustNewRequest("GET", "http://127.0.0.1:9000/bucket", 0, nil, t), ErrCode: s3err.ErrNone, Action: s3_constants.ACTION_READ},
  112. {Request: mustNewRequest("PUT", "http://127.0.0.1:9000/bucket", 0, nil, t), ErrCode: s3err.ErrAccessDenied, Action: s3_constants.ACTION_WRITE},
  113. }
  114. for i, testCase := range testCases {
  115. _, s3Error := iam.authRequest(testCase.Request, testCase.Action)
  116. if s3Error != testCase.ErrCode {
  117. t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
  118. }
  119. if testCase.Request.Header.Get(s3_constants.AmzAuthType) != "Anonymous" {
  120. t.Errorf("Test %d: Unexpected AuthType returned wanted %s, got %s", i, "Anonymous", testCase.Request.Header.Get(s3_constants.AmzAuthType))
  121. }
  122. }
  123. }
  124. func TestCheckAdminRequestAuthType(t *testing.T) {
  125. iam := &IdentityAccessManagement{
  126. hashes: make(map[string]*sync.Pool),
  127. hashCounters: make(map[string]*int32),
  128. }
  129. _ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
  130. Identities: []*iam_pb.Identity{
  131. {
  132. Name: "someone",
  133. Credentials: []*iam_pb.Credential{
  134. {
  135. AccessKey: "access_key_1",
  136. SecretKey: "secret_key_1",
  137. },
  138. },
  139. Actions: []string{},
  140. },
  141. },
  142. })
  143. testCases := []struct {
  144. Request *http.Request
  145. ErrCode s3err.ErrorCode
  146. }{
  147. {Request: mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrAccessDenied},
  148. {Request: mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone},
  149. {Request: mustNewPresignedRequest(iam, "GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone},
  150. }
  151. for i, testCase := range testCases {
  152. if _, s3Error := iam.reqSignatureV4Verify(testCase.Request); s3Error != testCase.ErrCode {
  153. t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
  154. }
  155. }
  156. }
  157. func BenchmarkGetSignature(b *testing.B) {
  158. t := time.Now()
  159. iam := IdentityAccessManagement{
  160. hashes: make(map[string]*sync.Pool),
  161. hashCounters: make(map[string]*int32),
  162. }
  163. b.ReportAllocs()
  164. b.ResetTimer()
  165. for i := 0; i < b.N; i++ {
  166. iam.getSignature("secret-key", t, "us-east-1", "s3", "random data")
  167. }
  168. }
  169. // Provides a fully populated http request instance, fails otherwise.
  170. func mustNewRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
  171. req, err := newTestRequest(method, urlStr, contentLength, body)
  172. if err != nil {
  173. t.Fatalf("Unable to initialize new http request %s", err)
  174. }
  175. return req
  176. }
  177. // This is similar to mustNewRequest but additionally the request
  178. // is signed with AWS Signature V4, fails if not able to do so.
  179. func mustNewSignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
  180. req := mustNewRequest(method, urlStr, contentLength, body, t)
  181. cred := &Credential{"access_key_1", "secret_key_1"}
  182. if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
  183. t.Fatalf("Unable to initialized new signed http request %s", err)
  184. }
  185. return req
  186. }
  187. // This is similar to mustNewRequest but additionally the request
  188. // is presigned with AWS Signature V4, fails if not able to do so.
  189. func mustNewPresignedRequest(iam *IdentityAccessManagement, method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
  190. req := mustNewRequest(method, urlStr, contentLength, body, t)
  191. cred := &Credential{"access_key_1", "secret_key_1"}
  192. if err := preSignV4(iam, req, cred.AccessKey, cred.SecretKey, int64(10*time.Minute.Seconds())); err != nil {
  193. t.Fatalf("Unable to initialized new signed http request %s", err)
  194. }
  195. return req
  196. }
  197. // Returns new HTTP request object.
  198. func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) {
  199. if method == "" {
  200. method = "POST"
  201. }
  202. // Save for subsequent use
  203. var hashedPayload string
  204. var md5Base64 string
  205. switch {
  206. case body == nil:
  207. hashedPayload = getSHA256Hash([]byte{})
  208. default:
  209. payloadBytes, err := io.ReadAll(body)
  210. if err != nil {
  211. return nil, err
  212. }
  213. hashedPayload = getSHA256Hash(payloadBytes)
  214. md5Base64 = getMD5HashBase64(payloadBytes)
  215. }
  216. // Seek back to beginning.
  217. if body != nil {
  218. body.Seek(0, 0)
  219. } else {
  220. body = bytes.NewReader([]byte(""))
  221. }
  222. req, err := http.NewRequest(method, urlStr, body)
  223. if err != nil {
  224. return nil, err
  225. }
  226. if md5Base64 != "" {
  227. req.Header.Set("Content-Md5", md5Base64)
  228. }
  229. req.Header.Set("x-amz-content-sha256", hashedPayload)
  230. // Add Content-Length
  231. req.ContentLength = contentLength
  232. return req, nil
  233. }
  234. // getSHA256Hash returns SHA-256 hash in hex encoding of given data.
  235. func getSHA256Hash(data []byte) string {
  236. return hex.EncodeToString(getSHA256Sum(data))
  237. }
  238. // getMD5HashBase64 returns MD5 hash in base64 encoding of given data.
  239. func getMD5HashBase64(data []byte) string {
  240. return base64.StdEncoding.EncodeToString(getMD5Sum(data))
  241. }
  242. // getSHA256Sum returns SHA-256 sum of given data.
  243. func getSHA256Sum(data []byte) []byte {
  244. hash := sha256.New()
  245. hash.Write(data)
  246. return hash.Sum(nil)
  247. }
  248. // getMD5Sum returns MD5 sum of given data.
  249. func getMD5Sum(data []byte) []byte {
  250. hash := md5.New()
  251. hash.Write(data)
  252. return hash.Sum(nil)
  253. }
  254. // getMD5Hash returns MD5 hash in hex encoding of given data.
  255. func getMD5Hash(data []byte) string {
  256. return hex.EncodeToString(getMD5Sum(data))
  257. }
  258. var ignoredHeaders = map[string]bool{
  259. "Authorization": true,
  260. "Content-Type": true,
  261. "Content-Length": true,
  262. "User-Agent": true,
  263. }
  264. // Sign given request using Signature V4.
  265. func signRequestV4(req *http.Request, accessKey, secretKey string) error {
  266. // Get hashed payload.
  267. hashedPayload := req.Header.Get("x-amz-content-sha256")
  268. if hashedPayload == "" {
  269. return fmt.Errorf("Invalid hashed payload")
  270. }
  271. currTime := time.Now()
  272. // Set x-amz-date.
  273. req.Header.Set("x-amz-date", currTime.Format(iso8601Format))
  274. // Get header map.
  275. headerMap := make(map[string][]string)
  276. for k, vv := range req.Header {
  277. // If request header key is not in ignored headers, then add it.
  278. if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; !ok {
  279. headerMap[strings.ToLower(k)] = vv
  280. }
  281. }
  282. // Get header keys.
  283. headers := []string{"host"}
  284. for k := range headerMap {
  285. headers = append(headers, k)
  286. }
  287. sort.Strings(headers)
  288. region := "us-east-1"
  289. // Get canonical headers.
  290. var buf bytes.Buffer
  291. for _, k := range headers {
  292. buf.WriteString(k)
  293. buf.WriteByte(':')
  294. switch {
  295. case k == "host":
  296. buf.WriteString(req.URL.Host)
  297. fallthrough
  298. default:
  299. for idx, v := range headerMap[k] {
  300. if idx > 0 {
  301. buf.WriteByte(',')
  302. }
  303. buf.WriteString(v)
  304. }
  305. buf.WriteByte('\n')
  306. }
  307. }
  308. canonicalHeaders := buf.String()
  309. // Get signed headers.
  310. signedHeaders := strings.Join(headers, ";")
  311. // Get canonical query string.
  312. req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
  313. // Get canonical URI.
  314. canonicalURI := EncodePath(req.URL.Path)
  315. // Get canonical request.
  316. // canonicalRequest =
  317. // <HTTPMethod>\n
  318. // <CanonicalURI>\n
  319. // <CanonicalQueryString>\n
  320. // <CanonicalHeaders>\n
  321. // <SignedHeaders>\n
  322. // <HashedPayload>
  323. //
  324. canonicalRequest := strings.Join([]string{
  325. req.Method,
  326. canonicalURI,
  327. req.URL.RawQuery,
  328. canonicalHeaders,
  329. signedHeaders,
  330. hashedPayload,
  331. }, "\n")
  332. // Get scope.
  333. scope := strings.Join([]string{
  334. currTime.Format(yyyymmdd),
  335. region,
  336. "s3",
  337. "aws4_request",
  338. }, "/")
  339. stringToSign := "AWS4-HMAC-SHA256" + "\n" + currTime.Format(iso8601Format) + "\n"
  340. stringToSign = stringToSign + scope + "\n"
  341. stringToSign = stringToSign + getSHA256Hash([]byte(canonicalRequest))
  342. date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd)))
  343. regionHMAC := sumHMAC(date, []byte(region))
  344. service := sumHMAC(regionHMAC, []byte("s3"))
  345. signingKey := sumHMAC(service, []byte("aws4_request"))
  346. signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
  347. // final Authorization header
  348. parts := []string{
  349. "AWS4-HMAC-SHA256" + " Credential=" + accessKey + "/" + scope,
  350. "SignedHeaders=" + signedHeaders,
  351. "Signature=" + signature,
  352. }
  353. auth := strings.Join(parts, ", ")
  354. req.Header.Set("Authorization", auth)
  355. return nil
  356. }
  357. // preSignV4 presign the request, in accordance with
  358. // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html.
  359. func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKeyID, secretAccessKey string, expires int64) error {
  360. // Presign is not needed for anonymous credentials.
  361. if accessKeyID == "" || secretAccessKey == "" {
  362. return errors.New("Presign cannot be generated without access and secret keys")
  363. }
  364. region := "us-east-1"
  365. date := time.Now().UTC()
  366. scope := getScope(date, region)
  367. credential := fmt.Sprintf("%s/%s", accessKeyID, scope)
  368. // Set URL query.
  369. query := req.URL.Query()
  370. query.Set("X-Amz-Algorithm", signV4Algorithm)
  371. query.Set("X-Amz-Date", date.Format(iso8601Format))
  372. query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
  373. query.Set("X-Amz-SignedHeaders", "host")
  374. query.Set("X-Amz-Credential", credential)
  375. query.Set("X-Amz-Content-Sha256", unsignedPayload)
  376. // "host" is the only header required to be signed for Presigned URLs.
  377. extractedSignedHeaders := make(http.Header)
  378. extractedSignedHeaders.Set("host", req.Host)
  379. queryStr := strings.Replace(query.Encode(), "+", "%20", -1)
  380. canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method)
  381. stringToSign := getStringToSign(canonicalRequest, date, scope)
  382. signature := iam.getSignature(secretAccessKey, date, region, "s3", stringToSign)
  383. req.URL.RawQuery = query.Encode()
  384. // Add signature header to RawQuery.
  385. req.URL.RawQuery += "&X-Amz-Signature=" + url.QueryEscape(signature)
  386. // Construct the final presigned URL.
  387. return nil
  388. }
  389. // EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
  390. //
  391. // This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
  392. // non english characters cannot be parsed due to the nature in which url.Encode() is written
  393. //
  394. // This function on the other hand is a direct replacement for url.Encode() technique to support
  395. // pretty much every UTF-8 character.
  396. func EncodePath(pathName string) string {
  397. if reservedObjectNames.MatchString(pathName) {
  398. return pathName
  399. }
  400. var encodedPathname string
  401. for _, s := range pathName {
  402. if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
  403. encodedPathname = encodedPathname + string(s)
  404. continue
  405. }
  406. switch s {
  407. case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
  408. encodedPathname = encodedPathname + string(s)
  409. continue
  410. default:
  411. len := utf8.RuneLen(s)
  412. if len < 0 {
  413. // if utf8 cannot convert return the same string as is
  414. return pathName
  415. }
  416. u := make([]byte, len)
  417. utf8.EncodeRune(u, s)
  418. for _, r := range u {
  419. hex := hex.EncodeToString([]byte{r})
  420. encodedPathname = encodedPathname + "%" + strings.ToUpper(hex)
  421. }
  422. }
  423. }
  424. return encodedPathname
  425. }