chunked_reader_v4.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413
  1. package s3api
  2. // the related code is copied and modified from minio source code
  3. /*
  4. * Minio Cloud Storage, (C) 2016 Minio, Inc.
  5. *
  6. * Licensed under the Apache License, Version 2.0 (the "License");
  7. * you may not use this file except in compliance with the License.
  8. * You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. import (
  19. "bufio"
  20. "bytes"
  21. "crypto/sha256"
  22. "encoding/hex"
  23. "errors"
  24. xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
  25. "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
  26. "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
  27. "hash"
  28. "io"
  29. "net/http"
  30. "time"
  31. "github.com/dustin/go-humanize"
  32. )
  33. // getChunkSignature - get chunk signature.
  34. func getChunkSignature(secretKey string, seedSignature string, region string, date time.Time, hashedChunk string) string {
  35. // Calculate string to sign.
  36. stringToSign := signV4ChunkedAlgorithm + "\n" +
  37. date.Format(iso8601Format) + "\n" +
  38. getScope(date, region) + "\n" +
  39. seedSignature + "\n" +
  40. emptySHA256 + "\n" +
  41. hashedChunk
  42. // Get hmac signing key.
  43. signingKey := getSigningKey(secretKey, date, region, "s3")
  44. // Calculate signature.
  45. newSignature := getSignature(signingKey, stringToSign)
  46. return newSignature
  47. }
  48. // calculateSeedSignature - Calculate seed signature in accordance with
  49. // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
  50. // returns signature, error otherwise if the signature mismatches or any other
  51. // error while parsing and validating.
  52. func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cred *Credential, signature string, region string, date time.Time, errCode s3err.ErrorCode) {
  53. // Copy request.
  54. req := *r
  55. // Save authorization header.
  56. v4Auth := req.Header.Get("Authorization")
  57. // Parse signature version '4' header.
  58. signV4Values, errCode := parseSignV4(v4Auth)
  59. if errCode != s3err.ErrNone {
  60. return nil, "", "", time.Time{}, errCode
  61. }
  62. // Payload streaming.
  63. payload := streamingContentSHA256
  64. // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
  65. if payload != req.Header.Get("X-Amz-Content-Sha256") {
  66. return nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch
  67. }
  68. // Extract all the signed headers along with its values.
  69. extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
  70. if errCode != s3err.ErrNone {
  71. return nil, "", "", time.Time{}, errCode
  72. }
  73. // Verify if the access key id matches.
  74. identity, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey)
  75. if !found {
  76. return nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID
  77. }
  78. bucket, object := xhttp.GetBucketAndObject(r)
  79. if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) {
  80. errCode = s3err.ErrAccessDenied
  81. return
  82. }
  83. // Verify if region is valid.
  84. region = signV4Values.Credential.scope.region
  85. // Extract date, if not present throw error.
  86. var dateStr string
  87. if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
  88. if dateStr = r.Header.Get("Date"); dateStr == "" {
  89. return nil, "", "", time.Time{}, s3err.ErrMissingDateHeader
  90. }
  91. }
  92. // Parse date header.
  93. var err error
  94. date, err = time.Parse(iso8601Format, dateStr)
  95. if err != nil {
  96. return nil, "", "", time.Time{}, s3err.ErrMalformedDate
  97. }
  98. // Query string.
  99. queryStr := req.URL.Query().Encode()
  100. // Get canonical request.
  101. canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method)
  102. // Get string to sign from canonical request.
  103. stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())
  104. // Get hmac signing key.
  105. signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, "s3")
  106. // Calculate signature.
  107. newSignature := getSignature(signingKey, stringToSign)
  108. // Verify if signature match.
  109. if !compareSignatureV4(newSignature, signV4Values.Signature) {
  110. return nil, "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch
  111. }
  112. // Return caculated signature.
  113. return cred, newSignature, region, date, s3err.ErrNone
  114. }
  115. const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB
  116. // lineTooLong is generated as chunk header is bigger than 4KiB.
  117. var errLineTooLong = errors.New("header line too long")
  118. // Malformed encoding is generated when chunk header is wrongly formed.
  119. var errMalformedEncoding = errors.New("malformed chunked encoding")
  120. // newSignV4ChunkedReader returns a new s3ChunkedReader that translates the data read from r
  121. // out of HTTP "chunked" format before returning it.
  122. // The s3ChunkedReader returns io.EOF when the final 0-length chunk is read.
  123. func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, s3err.ErrorCode) {
  124. ident, seedSignature, region, seedDate, errCode := iam.calculateSeedSignature(req)
  125. if errCode != s3err.ErrNone {
  126. return nil, errCode
  127. }
  128. return &s3ChunkedReader{
  129. cred: ident,
  130. reader: bufio.NewReader(req.Body),
  131. seedSignature: seedSignature,
  132. seedDate: seedDate,
  133. region: region,
  134. chunkSHA256Writer: sha256.New(),
  135. state: readChunkHeader,
  136. }, s3err.ErrNone
  137. }
  138. // Represents the overall state that is required for decoding a
  139. // AWS Signature V4 chunked reader.
  140. type s3ChunkedReader struct {
  141. cred *Credential
  142. reader *bufio.Reader
  143. seedSignature string
  144. seedDate time.Time
  145. region string
  146. state chunkState
  147. lastChunk bool
  148. chunkSignature string
  149. chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data.
  150. n uint64 // Unread bytes in chunk
  151. err error
  152. }
  153. // Read chunk reads the chunk token signature portion.
  154. func (cr *s3ChunkedReader) readS3ChunkHeader() {
  155. // Read the first chunk line until CRLF.
  156. var hexChunkSize, hexChunkSignature []byte
  157. hexChunkSize, hexChunkSignature, cr.err = readChunkLine(cr.reader)
  158. if cr.err != nil {
  159. return
  160. }
  161. // <hex>;token=value - converts the hex into its uint64 form.
  162. cr.n, cr.err = parseHexUint(hexChunkSize)
  163. if cr.err != nil {
  164. return
  165. }
  166. if cr.n == 0 {
  167. cr.err = io.EOF
  168. }
  169. // Save the incoming chunk signature.
  170. cr.chunkSignature = string(hexChunkSignature)
  171. }
  172. type chunkState int
  173. const (
  174. readChunkHeader chunkState = iota
  175. readChunkTrailer
  176. readChunk
  177. verifyChunk
  178. eofChunk
  179. )
  180. func (cs chunkState) String() string {
  181. stateString := ""
  182. switch cs {
  183. case readChunkHeader:
  184. stateString = "readChunkHeader"
  185. case readChunkTrailer:
  186. stateString = "readChunkTrailer"
  187. case readChunk:
  188. stateString = "readChunk"
  189. case verifyChunk:
  190. stateString = "verifyChunk"
  191. case eofChunk:
  192. stateString = "eofChunk"
  193. }
  194. return stateString
  195. }
  196. func (cr *s3ChunkedReader) Close() (err error) {
  197. return nil
  198. }
  199. // Read - implements `io.Reader`, which transparently decodes
  200. // the incoming AWS Signature V4 streaming signature.
  201. func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
  202. for {
  203. switch cr.state {
  204. case readChunkHeader:
  205. cr.readS3ChunkHeader()
  206. // If we're at the end of a chunk.
  207. if cr.n == 0 && cr.err == io.EOF {
  208. cr.state = readChunkTrailer
  209. cr.lastChunk = true
  210. continue
  211. }
  212. if cr.err != nil {
  213. return 0, cr.err
  214. }
  215. cr.state = readChunk
  216. case readChunkTrailer:
  217. cr.err = readCRLF(cr.reader)
  218. if cr.err != nil {
  219. return 0, errMalformedEncoding
  220. }
  221. cr.state = verifyChunk
  222. case readChunk:
  223. // There is no more space left in the request buffer.
  224. if len(buf) == 0 {
  225. return n, nil
  226. }
  227. rbuf := buf
  228. // The request buffer is larger than the current chunk size.
  229. // Read only the current chunk from the underlying reader.
  230. if uint64(len(rbuf)) > cr.n {
  231. rbuf = rbuf[:cr.n]
  232. }
  233. var n0 int
  234. n0, cr.err = cr.reader.Read(rbuf)
  235. if cr.err != nil {
  236. // We have lesser than chunk size advertised in chunkHeader, this is 'unexpected'.
  237. if cr.err == io.EOF {
  238. cr.err = io.ErrUnexpectedEOF
  239. }
  240. return 0, cr.err
  241. }
  242. // Calculate sha256.
  243. cr.chunkSHA256Writer.Write(rbuf[:n0])
  244. // Update the bytes read into request buffer so far.
  245. n += n0
  246. buf = buf[n0:]
  247. // Update bytes to be read of the current chunk before verifying chunk's signature.
  248. cr.n -= uint64(n0)
  249. // If we're at the end of a chunk.
  250. if cr.n == 0 {
  251. cr.state = readChunkTrailer
  252. continue
  253. }
  254. case verifyChunk:
  255. // Calculate the hashed chunk.
  256. hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil))
  257. // Calculate the chunk signature.
  258. newSignature := getChunkSignature(cr.cred.SecretKey, cr.seedSignature, cr.region, cr.seedDate, hashedChunk)
  259. if !compareSignatureV4(cr.chunkSignature, newSignature) {
  260. // Chunk signature doesn't match we return signature does not match.
  261. cr.err = errors.New("chunk signature does not match")
  262. return 0, cr.err
  263. }
  264. // Newly calculated signature becomes the seed for the next chunk
  265. // this follows the chaining.
  266. cr.seedSignature = newSignature
  267. cr.chunkSHA256Writer.Reset()
  268. if cr.lastChunk {
  269. cr.state = eofChunk
  270. } else {
  271. cr.state = readChunkHeader
  272. }
  273. case eofChunk:
  274. return n, io.EOF
  275. }
  276. }
  277. }
  278. // readCRLF - check if reader only has '\r\n' CRLF character.
  279. // returns malformed encoding if it doesn't.
  280. func readCRLF(reader io.Reader) error {
  281. buf := make([]byte, 2)
  282. _, err := io.ReadFull(reader, buf[:2])
  283. if err != nil {
  284. return err
  285. }
  286. if buf[0] != '\r' || buf[1] != '\n' {
  287. return errMalformedEncoding
  288. }
  289. return nil
  290. }
  291. // Read a line of bytes (up to \n) from b.
  292. // Give up if the line exceeds maxLineLength.
  293. // The returned bytes are owned by the bufio.Reader
  294. // so they are only valid until the next bufio read.
  295. func readChunkLine(b *bufio.Reader) ([]byte, []byte, error) {
  296. buf, err := b.ReadSlice('\n')
  297. if err != nil {
  298. // We always know when EOF is coming.
  299. // If the caller asked for a line, there should be a line.
  300. if err == io.EOF {
  301. err = io.ErrUnexpectedEOF
  302. } else if err == bufio.ErrBufferFull {
  303. err = errLineTooLong
  304. }
  305. return nil, nil, err
  306. }
  307. if len(buf) >= maxLineLength {
  308. return nil, nil, errLineTooLong
  309. }
  310. // Parse s3 specific chunk extension and fetch the values.
  311. hexChunkSize, hexChunkSignature := parseS3ChunkExtension(buf)
  312. return hexChunkSize, hexChunkSignature, nil
  313. }
  314. // trimTrailingWhitespace - trim trailing white space.
  315. func trimTrailingWhitespace(b []byte) []byte {
  316. for len(b) > 0 && isASCIISpace(b[len(b)-1]) {
  317. b = b[:len(b)-1]
  318. }
  319. return b
  320. }
  321. // isASCIISpace - is ascii space?
  322. func isASCIISpace(b byte) bool {
  323. return b == ' ' || b == '\t' || b == '\n' || b == '\r'
  324. }
  325. // Constant s3 chunk encoding signature.
  326. const s3ChunkSignatureStr = ";chunk-signature="
  327. // parses3ChunkExtension removes any s3 specific chunk-extension from buf.
  328. // For example,
  329. // "10000;chunk-signature=..." => "10000", "chunk-signature=..."
  330. func parseS3ChunkExtension(buf []byte) ([]byte, []byte) {
  331. buf = trimTrailingWhitespace(buf)
  332. semi := bytes.Index(buf, []byte(s3ChunkSignatureStr))
  333. // Chunk signature not found, return the whole buffer.
  334. if semi == -1 {
  335. return buf, nil
  336. }
  337. return buf[:semi], parseChunkSignature(buf[semi:])
  338. }
  339. // parseChunkSignature - parse chunk signature.
  340. func parseChunkSignature(chunk []byte) []byte {
  341. chunkSplits := bytes.SplitN(chunk, []byte(s3ChunkSignatureStr), 2)
  342. return chunkSplits[1]
  343. }
  344. // parse hex to uint64.
  345. func parseHexUint(v []byte) (n uint64, err error) {
  346. for i, b := range v {
  347. switch {
  348. case '0' <= b && b <= '9':
  349. b = b - '0'
  350. case 'a' <= b && b <= 'f':
  351. b = b - 'a' + 10
  352. case 'A' <= b && b <= 'F':
  353. b = b - 'A' + 10
  354. default:
  355. return 0, errors.New("invalid byte in chunk length")
  356. }
  357. if i == 16 {
  358. return 0, errors.New("http chunk length too large")
  359. }
  360. n <<= 4
  361. n |= uint64(b)
  362. }
  363. return
  364. }