compression.go 3.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160
  1. package util
  2. import (
  3. "bytes"
  4. "compress/flate"
  5. "compress/gzip"
  6. "fmt"
  7. "io/ioutil"
  8. "strings"
  9. "github.com/chrislusf/seaweedfs/weed/glog"
  10. // "github.com/klauspost/compress/zstd"
  11. )
  12. var (
  13. UnsupportedCompression = fmt.Errorf("unsupported compression")
  14. )
  15. func MaybeGzipData(input []byte) []byte {
  16. if IsGzippedContent(input) {
  17. return input
  18. }
  19. gzipped, err := GzipData(input)
  20. if err != nil {
  21. return input
  22. }
  23. if len(gzipped)*10 > len(input)*9 {
  24. return input
  25. }
  26. return gzipped
  27. }
  28. func MaybeDecompressData(input []byte) []byte {
  29. uncompressed, err := DecompressData(input)
  30. if err != nil {
  31. if err != UnsupportedCompression {
  32. glog.Errorf("decompressed data: %v", err)
  33. }
  34. return input
  35. }
  36. return uncompressed
  37. }
  38. func GzipData(input []byte) ([]byte, error) {
  39. buf := new(bytes.Buffer)
  40. w, _ := gzip.NewWriterLevel(buf, flate.BestSpeed)
  41. if _, err := w.Write(input); err != nil {
  42. glog.V(2).Infof("error gzip data: %v", err)
  43. return nil, err
  44. }
  45. if err := w.Close(); err != nil {
  46. glog.V(2).Infof("error closing gzipped data: %v", err)
  47. return nil, err
  48. }
  49. return buf.Bytes(), nil
  50. }
  51. func DecompressData(input []byte) ([]byte, error) {
  52. if IsGzippedContent(input) {
  53. return ungzipData(input)
  54. }
  55. /*
  56. if IsZstdContent(input) {
  57. return unzstdData(input)
  58. }
  59. */
  60. return input, UnsupportedCompression
  61. }
  62. func ungzipData(input []byte) ([]byte, error) {
  63. buf := bytes.NewBuffer(input)
  64. r, _ := gzip.NewReader(buf)
  65. defer r.Close()
  66. output, err := ioutil.ReadAll(r)
  67. if err != nil {
  68. glog.V(2).Infof("error ungzip data: %v", err)
  69. }
  70. return output, err
  71. }
  72. func IsGzippedContent(data []byte) bool {
  73. if len(data) < 2 {
  74. return false
  75. }
  76. return data[0] == 31 && data[1] == 139
  77. }
  78. /*
  79. var zstdEncoder, _ = zstd.NewWriter(nil)
  80. func ZstdData(input []byte) ([]byte, error) {
  81. return zstdEncoder.EncodeAll(input, nil), nil
  82. }
  83. var decoder, _ = zstd.NewReader(nil)
  84. func unzstdData(input []byte) ([]byte, error) {
  85. return decoder.DecodeAll(input, nil)
  86. }
  87. func IsZstdContent(data []byte) bool {
  88. if len(data) < 4 {
  89. return false
  90. }
  91. return data[3] == 0xFD && data[2] == 0x2F && data[1] == 0xB5 && data[0] == 0x28
  92. }
  93. */
  94. /*
  95. * Default not to compressed since compression can be done on client side.
  96. */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
  97. // text
  98. if strings.HasPrefix(mtype, "text/") {
  99. return true, true
  100. }
  101. // images
  102. switch ext {
  103. case ".svg", ".bmp", ".wav":
  104. return true, true
  105. }
  106. if strings.HasPrefix(mtype, "image/") {
  107. return false, true
  108. }
  109. // by file name extension
  110. switch ext {
  111. case ".zip", ".rar", ".gz", ".bz2", ".xz", ".zst":
  112. return false, true
  113. case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
  114. return true, true
  115. case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
  116. return true, true
  117. case ".png", ".jpg", ".jpeg":
  118. return false, true
  119. }
  120. // by mime type
  121. if strings.HasPrefix(mtype, "application/") {
  122. if strings.HasSuffix(mtype, "zstd") {
  123. return false, true
  124. }
  125. if strings.HasSuffix(mtype, "xml") {
  126. return true, true
  127. }
  128. if strings.HasSuffix(mtype, "script") {
  129. return true, true
  130. }
  131. }
  132. if strings.HasPrefix(mtype, "audio/") {
  133. switch strings.TrimPrefix(mtype, "audio/") {
  134. case "wave", "wav", "x-wav", "x-pn-wav":
  135. return true, true
  136. }
  137. }
  138. return false, false
  139. }