compression.go 3.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153
  1. package util
  2. import (
  3. "bytes"
  4. "fmt"
  5. "strings"
  6. "github.com/chrislusf/seaweedfs/weed/glog"
  7. // "github.com/klauspost/compress/zstd"
  8. )
  9. var (
  10. UnsupportedCompression = fmt.Errorf("unsupported compression")
  11. )
  12. func MaybeGzipData(input []byte) []byte {
  13. if IsGzippedContent(input) {
  14. return input
  15. }
  16. gzipped, err := GzipData(input)
  17. if err != nil {
  18. return input
  19. }
  20. if len(gzipped)*10 > len(input)*9 {
  21. return input
  22. }
  23. return gzipped
  24. }
  25. func MaybeDecompressData(input []byte) []byte {
  26. uncompressed, err := DecompressData(input)
  27. if err != nil {
  28. if err != UnsupportedCompression {
  29. glog.Errorf("decompressed data: %v", err)
  30. }
  31. return input
  32. }
  33. return uncompressed
  34. }
  35. func GzipData(input []byte) ([]byte, error) {
  36. w := new(bytes.Buffer)
  37. _, err := GzipStream(w, bytes.NewReader(input))
  38. if err != nil {
  39. return nil, err
  40. }
  41. return w.Bytes(), nil
  42. }
  43. func ungzipData(input []byte) ([]byte, error) {
  44. w := new(bytes.Buffer)
  45. _, err := GunzipStream(w, bytes.NewReader(input))
  46. if err != nil {
  47. return nil, err
  48. }
  49. return w.Bytes(), nil
  50. }
  51. func DecompressData(input []byte) ([]byte, error) {
  52. if IsGzippedContent(input) {
  53. return ungzipData(input)
  54. }
  55. /*
  56. if IsZstdContent(input) {
  57. return unzstdData(input)
  58. }
  59. */
  60. return input, UnsupportedCompression
  61. }
  62. func IsGzippedContent(data []byte) bool {
  63. if len(data) < 2 {
  64. return false
  65. }
  66. return data[0] == 31 && data[1] == 139
  67. }
  68. /*
  69. var zstdEncoder, _ = zstd.NewWriter(nil)
  70. func ZstdData(input []byte) ([]byte, error) {
  71. return zstdEncoder.EncodeAll(input, nil), nil
  72. }
  73. var decoder, _ = zstd.NewReader(nil)
  74. func unzstdData(input []byte) ([]byte, error) {
  75. return decoder.DecodeAll(input, nil)
  76. }
  77. func IsZstdContent(data []byte) bool {
  78. if len(data) < 4 {
  79. return false
  80. }
  81. return data[3] == 0xFD && data[2] == 0x2F && data[1] == 0xB5 && data[0] == 0x28
  82. }
  83. */
  84. /*
  85. * Default not to compressed since compression can be done on client side.
  86. */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
  87. // text
  88. if strings.HasPrefix(mtype, "text/") {
  89. return true, true
  90. }
  91. // images
  92. switch ext {
  93. case ".svg", ".bmp", ".wav":
  94. return true, true
  95. }
  96. if strings.HasPrefix(mtype, "image/") {
  97. return false, true
  98. }
  99. // by file name extension
  100. switch ext {
  101. case ".zip", ".rar", ".gz", ".bz2", ".xz", ".zst", ".br":
  102. return false, true
  103. case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
  104. return true, true
  105. case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
  106. return true, true
  107. case ".png", ".jpg", ".jpeg":
  108. return false, true
  109. }
  110. // by mime type
  111. if strings.HasPrefix(mtype, "application/") {
  112. if strings.HasSuffix(mtype, "zstd") {
  113. return false, true
  114. }
  115. if strings.HasSuffix(mtype, "xml") {
  116. return true, true
  117. }
  118. if strings.HasSuffix(mtype, "script") {
  119. return true, true
  120. }
  121. if strings.HasSuffix(mtype, "vnd.rar") {
  122. return false, true
  123. }
  124. }
  125. if strings.HasPrefix(mtype, "audio/") {
  126. switch strings.TrimPrefix(mtype, "audio/") {
  127. case "wave", "wav", "x-wav", "x-pn-wav":
  128. return true, true
  129. }
  130. }
  131. return false, false
  132. }