Fixes #660 - S3 compatible storage backend

.gitlab/ci/__includes__/services.yml

@@ -48,6 +48,13 @@
     KEYCLOAK_ADMIN: admin
     KEYCLOAK_ADMIN_PASSWORD: admin
 
+.variables_s3:
+  variables:
+    S3_URL: http://zammadadmin:zammadadmin@ci-service-s3:9000/zammad?region=de-zammad-s1&force_path_style=true
+    MINIO_ROOT_USER: zammadadmin
+    MINIO_ROOT_PASSWORD: zammadadmin
+    MINIO_DEFAULT_BUCKET: zammad
+
 .services:
   mysql:
     name: $CI_REGISTRY/docker/zammad-mysql:stable
@@ -109,3 +116,8 @@
   ldap:
     name: $CI_REGISTRY/docker/zammad-ldap:stable
     alias: ci-service-ldap
+
+  s3:
+    name: $CI_REGISTRY/docker/zammad-s3:stable
+    alias: ci-service-s3
+    command: ['server', '/data']

.gitlab/ci/test/integration.yml

@@ -11,4 +11,5 @@ include:
     - !reference [.services, imap]
     - !reference [.services, redis]
     - !reference [.services, memcached]
-  stage: test
+  stage: test
+

.gitlab/ci/test/rspec.yml

@@ -22,6 +22,7 @@ rspec:integration:
     - .variables_proxy
     - .variables_mattermost
     - .variables_ftp
+    - .variables_s3
   allow_failure: true
   services:
     - !reference [.services, mysql]
@@ -33,6 +34,7 @@ rspec:integration:
     - !reference [.services, ftp]
     - !reference [.services, mattermost]
     - !reference [.services, ldap]
+    - !reference [.services, s3]
   script:
     - !reference [.scripts, zammad_db_init]
     - bundle exec rspec --exclude-pattern "spec/system/**/*_spec.rb" -t integration -t ~integration_standalone --profile 10

Gemfile

@@ -183,6 +183,9 @@ gem 'openssl'
 gem 'byk', require: false
 gem 'PoParser', require: false
 
+# Simple storage
+gem 'aws-sdk-s3', require: false
+
 # Gems used only for develop/test and not required
 # in production environments by default.
 group :development, :test do

Gemfile.lock

@@ -106,6 +106,22 @@ GEM
     autoprefixer-rails (10.4.15.0)
       execjs (~> 2)
     awrence (1.2.1)
+    aws-eventstream (1.2.0)
+    aws-partitions (1.824.0)
+    aws-sdk-core (3.181.1)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.5)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.71.0)
+      aws-sdk-core (~> 3, >= 3.177.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.134.0)
+      aws-sdk-core (~> 3, >= 3.181.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.6)
+    aws-sigv4 (1.6.0)
+      aws-eventstream (~> 1, >= 1.0.2)
     base64 (0.1.1)
     bindata (2.4.15)
     binding_of_caller (1.0.0)
@@ -265,6 +281,7 @@ GEM
     inflection (1.0.0)
     iniparse (1.5.0)
     interception (0.5)
+    jmespath (1.6.2)
     json (2.6.3)
     jwt (2.3.0)
     koala (3.5.0)
@@ -676,6 +693,7 @@ DEPENDENCIES
   argon2
   autodiscover!
   autoprefixer-rails
+  aws-sdk-s3
   biz
   bootsnap
   brakeman

app/assets/javascripts/app/views/settings/storage_provider.jst.eco

@@ -1,17 +1,28 @@
 <form class="settings-entry" id="<%= @setting.name %>">
   <h2><%- @T(@setting.title) %></h2>
-  <p class="help-text"><%- @T('You can switch between the backends for new attachments even on a system that is already in production without any loss of data.') %></p>
+  <p class="help-text"><%- @T('You can switch between the following backends for new attachments even on a system that is already in production without any loss of data.') %></p>
+  <dl class="help-text">
+    <dt><%- @T('Filesystem') %><code>File</code></dt>
+    <dd><%- @T('Attachments are stored in the filesystem.') %></dd>
+    <dt><%- @T('Database') %><code>DB</code></dt>
+    <dd><%- @T('Attachments are stored in the database.') %></dd>
+    <dt><%- @T('Simple Storage (S3)') %><code>S3</code></dt>
+    <dd><%- @T('Attachments are stored in a Simple Storage Service.') %></dd>
+  </dl>
   <p class="help-text"><%- @T('If you want to move already stored attachments from one backend to another, you need to execute the following via console.') %></p>
-  </p>
+  <code>
+  rails&gt; Store::File.move(SOURCE_STORAGE, TARGET_STORAGE)
+  </code>
+  <p class="help-text"><%- @T('Examples:') %></p>
   <p class="help-text"><%- @T('Move all from "%s" to "%s":', 'filesystem', 'database') %></p>
   </p>
   <code>
   rails&gt; Store::File.move('File', 'DB')
   </code>
-  <p class="help-text"><%- @T('Move all from "%s" to "%s":', 'database', 'filesystem') %></p>
+  <p class="help-text"><%- @T('Move all from "%s" to "%s":', 'database', 'simple storage') %></p>
   </p>
   <code>
-  rails&gt; Store::File.move('DB', 'File')
+  rails&gt; Store::File.move('DB', 'S3')
   </code>
   <br>
   <br>

app/models/store.rb

@@ -26,7 +26,7 @@ class Store < ApplicationModel
   end
 
   def set_store_file
-    file = Store::File.add(data)
+    file = Store::File.add(data, content_type: preferences['Content-Type'], filename: filename)
     self.size = data.to_s.bytesize
     self.store_file_id = file.id
   end

app/models/store/file.rb

@@ -17,7 +17,7 @@ do also verify of written data
 
 =end
 
-    def self.add(data, verify = true)
+    def self.add(data, verify = true, *args)
       sha = checksum(data)
 
       file = Store::File.find_by(sha: sha)
@@ -30,7 +30,7 @@ do also verify of written data
         end
 
         adapter = "Store::Provider::#{adapter_name}".constantize
-        adapter.add(data, sha)
+        adapter.add(data, sha, args)
         file = Store::File.create(
           provider: adapter_name,
           sha:      sha,

app/models/store/provider/db.rb

@@ -5,7 +5,7 @@ class Store
     class DB < ApplicationModel
       self.table_name = 'store_provider_dbs'
 
-      def self.add(data, sha)
+      def self.add(data, sha, *)
         Store::Provider::DB.create(
           data: data,
           sha:  sha,

app/models/store/provider/file.rb

@@ -3,7 +3,7 @@
 class Store::Provider::File
 
   # write file to fs
-  def self.add(data, sha)
+  def self.add(data, sha, *)
     location = get_location(sha)
 
     # write file to file system