Maintenance: Ported email_keep_on_server_test.rb to rspec.

.gitlab/ci/test/integration/other.yml

@@ -4,7 +4,5 @@
   allow_failure: true
   script:
     - !reference [.scripts, zammad_db_unseeded]
-    - echo "email_helper_deliver tests..."
-    - bundle exec rails test test/integration/email_keep_on_server_test.rb
     - echo "Clearbit test..."
     - bundle exec rails test test/integration/clearbit_test.rb

spec/models/channel/driver/imap_spec.rb

@@ -66,14 +66,14 @@ RSpec.describe Channel::Driver::Imap, integration: true, required_envs: %w[MAIL_
     end
   end
 
-  describe 'handling of oversized incoming emails' do
-    let(:folder) { "postmaster_to_sender_#{SecureRandom.uuid}" }
+  describe '.fetch', :aggregate_failures do
+    let(:folder) { "imap_spec-#{SecureRandom.uuid}" }
 
     let(:server_address) { ENV['MAIL_SERVER'] }
     let(:server_login)    { ENV['MAIL_ADDRESS'] }
     let(:server_password) { ENV['MAIL_PASS'] }
     let(:email_address)   { create(:email_address, realname: 'Zammad Helpdesk', email: "some-zammad-#{ENV['MAIL_ADDRESS']}") }
-    let(:group)           { create(:group, name: 'PostmasterToSenderTest', email_address: email_address) }
+    let(:group)           { create(:group, email_address: email_address) }
     let(:inbound_options) do
       {
         adapter: 'imap',
@@ -108,24 +108,6 @@ RSpec.describe Channel::Driver::Imap, integration: true, required_envs: %w[MAIL_
     end
 
     let(:imap) { Net::IMAP.new(server_address, 993, true, nil, false).tap { |imap| imap.login(server_login, server_password) } }
-    let(:sender_email_address) { ENV['MAIL_ADDRESS'] }
-    let(:cid)                  { SecureRandom.uuid.tr('-', '.') }
-    let(:oversized_email) do
-      <<~OVERSIZED_EMAIL.gsub(%r{\n}, "\r\n")
-        Subject: Oversized Email Message
-        From: Max Mustermann <#{sender_email_address}>
-        To: shugo@example.com
-        Message-ID: <#{cid}@zammad.test.com>
-
-        Oversized Email Message Body #{'#' * 120_000}
-      OVERSIZED_EMAIL
-    end
-    let(:oversized_email_md5) { Digest::MD5.hexdigest(oversized_email) }
-    let(:oversized_email_size) { format('%<MB>.2f', MB: oversized_email.size.to_f / 1024 / 1024) }
-    let(:oversized_eml_folder) { Rails.root.join('tmp/oversized_mail') }
-    let(:oversized_eml_file) do
-      Dir.entries(oversized_eml_folder).grep(%r{^#{oversized_email_md5}\.eml$}).map { |path| oversized_eml_folder.join(path) }.last
-    end
 
     let(:purge_inbox) do
       imap.select('inbox')
@@ -135,93 +117,253 @@ RSpec.describe Channel::Driver::Imap, integration: true, required_envs: %w[MAIL_
       imap.expunge
     end
 
-    let(:fetch_oversized_email) do
+    before do
+      purge_inbox
       imap.create(folder)
       imap.select(folder)
-      imap.append(folder, oversized_email, [], Time.zone.now)
-      channel.fetch(true)
     end
 
-    context 'with email reply', :aggregate_failures do
-      before do
-        Setting.set('postmaster_max_size', 0.1)
-        purge_inbox
-        fetch_oversized_email
-      end
+    after do
+      imap.delete(folder)
+    end
+
+    context 'when fetching regular emails' do
+      let(:email1) do
+        <<~EMAIL.gsub(%r{\n}, "\r\n")
+          Subject: hello1
+          From: shugo@example.com
+          To: shugo@example.com
+          Message-ID: <some1@example_keep_on_server>
 
-      after do
-        imap.delete(folder)
+          hello world
+        EMAIL
+      end
+      let(:email2) do
+        <<~EMAIL.gsub(%r{\n}, "\r\n")
+          Subject: hello2
+          From: shugo@example.com
+          To: shugo@example.com
+          Message-ID: <some2@example_keep_on_server>
+
+          hello world
+        EMAIL
       end
 
-      let(:oversized_email_reply) do
-        imap.select('inbox')
-        5.times do |i|
-          sleep i
-          msg = imap.sort(['DATE'], ['ALL'], 'US-ASCII').first
-          if msg
-            return imap.fetch(msg, 'RFC822')[0].attr['RFC822']
-          end
+      context 'with keep_on_server flag' do
+        let(:inbound_options) do
+          {
+            adapter: 'imap',
+            options: {
+              host:           ENV['MAIL_SERVER'],
+              user:           ENV['MAIL_ADDRESS'],
+              password:       server_password,
+              ssl:            true,
+              folder:         folder,
+              keep_on_server: true,
+            }
+          }
         end
-        nil
-      end
 
-      let(:parsed_oversized_email_reply) do
-        Channel::EmailParser.new.parse(oversized_email_reply)
+        it 'handles messages correctly' do # rubocop:disable RSpec/ExampleLength
+
+          imap.append(folder, email1, [], Time.zone.now)
+
+          # verify if message is still on server
+          message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
+          expect(message_ids.count).to be(1)
+
+          message_meta = imap.fetch(1, ['FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).not_to include(:Seen)
+
+          # fetch messages - will import
+          expect { channel.fetch(true) }.to change(Ticket::Article, :count)
+
+          # verify if message is still on server
+          message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
+          expect(message_ids.count).to be(1)
+
+          # message now has :seen flag
+          message_meta = imap.fetch(1, ['RFC822.HEADER', 'FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).to include(:Seen)
+
+          # fetch messages - will not import
+          expect { channel.fetch(true) }.not_to change(Ticket::Article, :count)
+
+          # verify if message is still on server
+          message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
+          expect(message_ids.count).to be(1)
+
+          # put unseen message in it
+          imap.append(folder, email2, [], Time.zone.now)
+
+          message_meta = imap.fetch(1, ['FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).to include(:Seen)
+          message_meta = imap.fetch(2, ['FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).not_to include(:Seen)
+
+          # fetch messages - will import new
+          expect { channel.fetch(true) }.to change(Ticket::Article, :count)
+
+          # verify if message is still on server
+          message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
+          expect(message_ids.count).to be(2)
+
+          message_meta = imap.fetch(1, ['FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).to include(:Seen)
+          message_meta = imap.fetch(2, ['FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).to include(:Seen)
+
+          # set messages to not seen
+          imap.store(1, '-FLAGS', [:Seen])
+          imap.store(2, '-FLAGS', [:Seen])
+
+          # fetch messages - will still not import
+          expect { channel.fetch(true) }.not_to change(Ticket::Article, :count)
+        end
       end
 
-      it 'creates email reply correctly' do
-        # 1. verify that the oversized email has been saved locally to:
-        # /tmp/oversized_mail/yyyy-mm-ddThh:mm:ss-:md5.eml
-        expect(oversized_eml_file).to be_present
+      context 'without keep_on_server flag' do
+
+        it 'handles messages correctly' do
+
+          imap.append(folder, email1, [], Time.zone.now)
 
-        # verify that the file is byte for byte identical to the sent message
-        expect(File.read(oversized_eml_file)).to eq(oversized_email)
+          # verify if message is still on server
+          message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
+          expect(message_ids.count).to be(1)
 
-        # 2. verify that a postmaster response email has been sent to the sender
-        expect(oversized_email_reply).to be_present
+          message_meta = imap.fetch(1, ['FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).not_to include(:Seen)
 
-        # parse the reply mail and verify the various headers
-        expect(parsed_oversized_email_reply).to include({
-                                                          from_email: email_address.email,
-          subject: '[undeliverable] Message too large',
-          'references' => "<#{cid}@zammad.test.com>",
-          'in-reply-to' => "<#{cid}@zammad.test.com>",
-                                                        })
+          # fetch messages - will import
+          expect { channel.fetch(true) }.to change(Ticket::Article, :count)
 
-        # verify the reply mail body content
-        expect(parsed_oversized_email_reply[:body]).to match(%r{^Dear Max Mustermann.*Oversized Email Message.*#{oversized_email_size} MB.*0.1 MB.*#{Setting.get('fqdn')}}sm)
+          # verify if message is still on server
+          message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
+          expect(message_ids.count).to be(1)
 
-        # 3. check if original mail got removed
-        imap.select(folder)
-        expect(imap.sort(['DATE'], ['ALL'], 'US-ASCII')).to be_empty
+          message_meta = imap.fetch(1, ['FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).to include(:Seen, :Deleted)
+
+          # put unseen message in it
+          imap.append(folder, email2, [], Time.zone.now)
+
+          # verify if message is still on server
+          message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
+          expect(message_ids.count).to be(1)
+
+          message_meta = imap.fetch(1, ['FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).not_to include(:Seen)
+
+          # fetch messages - will import
+          expect { channel.fetch(true) }.to change(Ticket::Article, :count)
+
+          # verify if message is still on server
+          message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
+          expect(message_ids.count).to be(1)
+
+          message_meta = imap.fetch(1, ['FLAGS'])[0].attr
+          expect(message_meta['FLAGS']).to include(:Seen)
+        end
       end
     end
 
-    context 'without email reply' do
-      before do
-        Setting.set('postmaster_max_size', 0.1)
-        Setting.set('postmaster_send_reject_if_mail_too_large', false)
-        purge_inbox
-        fetch_oversized_email
+    context 'when fetching oversized emails' do
+      let(:sender_email_address) { ENV['MAIL_ADDRESS'] }
+      let(:cid)                  { SecureRandom.uuid.tr('-', '.') }
+      let(:oversized_email) do
+        <<~OVERSIZED_EMAIL.gsub(%r{\n}, "\r\n")
+          Subject: Oversized Email Message
+          From: Max Mustermann <#{sender_email_address}>
+          To: shugo@example.com
+          Message-ID: <#{cid}@zammad.test.com>
+
+          Oversized Email Message Body #{'#' * 120_000}
+        OVERSIZED_EMAIL
       end
+      let(:oversized_email_md5) { Digest::MD5.hexdigest(oversized_email) }
+      let(:oversized_email_size) { format('%<MB>.2f', MB: oversized_email.size.to_f / 1024 / 1024) }
+      let(:oversized_eml_folder) { Rails.root.join('tmp/oversized_mail') }
+      let(:oversized_eml_file) do
+        Dir.entries(oversized_eml_folder).grep(%r{^#{oversized_email_md5}\.eml$}).map { |path| oversized_eml_folder.join(path) }.last
+      end
+
+      let(:fetch_oversized_email) do
+        imap.append(folder, oversized_email, [], Time.zone.now)
+        channel.fetch(true)
+      end
+
+      context 'with email reply' do
+        before do
+          Setting.set('postmaster_max_size', 0.1)
+          fetch_oversized_email
+        end
+
+        let(:oversized_email_reply) do
+          imap.select('inbox')
+          5.times do |i|
+            sleep i
+            msg = imap.sort(['DATE'], ['ALL'], 'US-ASCII').first
+            if msg
+              return imap.fetch(msg, 'RFC822')[0].attr['RFC822']
+            end
+          end
+          nil
+        end
+
+        let(:parsed_oversized_email_reply) do
+          Channel::EmailParser.new.parse(oversized_email_reply)
+        end
+
+        it 'creates email reply correctly' do
+          # 1. verify that the oversized email has been saved locally to:
+          # /tmp/oversized_mail/yyyy-mm-ddThh:mm:ss-:md5.eml
+          expect(oversized_eml_file).to be_present
+
+          # verify that the file is byte for byte identical to the sent message
+          expect(File.read(oversized_eml_file)).to eq(oversized_email)
+
+          # 2. verify that a postmaster response email has been sent to the sender
+          expect(oversized_email_reply).to be_present
 
-      after do
-        imap.delete(folder)
+          # parse the reply mail and verify the various headers
+          expect(parsed_oversized_email_reply).to include({
+                                                            from_email: email_address.email,
+            subject: '[undeliverable] Message too large',
+            'references' => "<#{cid}@zammad.test.com>",
+            'in-reply-to' => "<#{cid}@zammad.test.com>",
+                                                          })
+
+          # verify the reply mail body content
+          expect(parsed_oversized_email_reply[:body]).to match(%r{^Dear Max Mustermann.*Oversized Email Message.*#{oversized_email_size} MB.*0.1 MB.*#{Setting.get('fqdn')}}sm)
+
+          # 3. check if original mail got removed
+          imap.select(folder)
+          expect(imap.sort(['DATE'], ['ALL'], 'US-ASCII')).to be_empty
+        end
       end
 
-      it 'does not create email reply', :aggregate_failures do
+      context 'without email reply' do
+        before do
+          Setting.set('postmaster_max_size', 0.1)
+          Setting.set('postmaster_send_reject_if_mail_too_large', false)
+          fetch_oversized_email
+        end
+
+        it 'does not create email reply' do
 
-        # 1. verify that email was not locally processed
-        expect(oversized_eml_file).to be_nil
+          # 1. verify that email was not locally processed
+          expect(oversized_eml_file).to be_nil
 
-        # 2. verify that no postmaster response email has been sent
-        imap.select('inbox')
-        sleep 1
-        expect(imap.sort(['DATE'], ['ALL'], 'US-ASCII').count).to be_zero
+          # 2. verify that no postmaster response email has been sent
+          imap.select('inbox')
+          sleep 1
+          expect(imap.sort(['DATE'], ['ALL'], 'US-ASCII').count).to be_zero
 
-        # 3. check that original mail is still there
-        imap.select(folder)
-        expect(imap.sort(['DATE'], ['ALL'], 'US-ASCII').count).to be(1)
+          # 3. check that original mail is still there
+          imap.select(folder)
+          expect(imap.sort(['DATE'], ['ALL'], 'US-ASCII').count).to be(1)
+        end
       end
     end
   end

test/integration/email_keep_on_server_test.rb

@@ -1,237 +0,0 @@
-# Copyright (C) 2012-2023 Zammad Foundation, https://zammad-foundation.org/
-
-require 'test_helper'
-require 'net/imap'
-
-class EmailKeepOnServerTest < ActiveSupport::TestCase
-  setup do
-
-    if ENV['MAIL_SERVER'].blank?
-      raise "Need MAIL_SERVER as ENV variable like export MAIL_SERVER='mx.example.com'"
-    end
-    if ENV['MAIL_ADDRESS'].blank?
-      raise 'Need MAIL_ADDRESS as ENV variable'
-    end
-    if ENV['MAIL_PASS'].blank?
-      raise 'Need MAIL_PASS as ENV variable'
-    end
-
-    @server_login = ENV['MAIL_ADDRESS']
-    @server_password = ENV['MAIL_PASS']
-
-    @folder = "keep_on_mail_server_#{SecureRandom.uuid}"
-
-    email_address = EmailAddress.create!(
-      realname:      'me Helpdesk',
-      email:         "me#{SecureRandom.uuid}@example.com",
-      updated_by_id: 1,
-      created_by_id: 1,
-    )
-
-    group = Group.create_or_update(
-      name:             'KeepOnServerTest',
-      email_address_id: email_address.id,
-      updated_by_id:    1,
-      created_by_id:    1,
-    )
-
-    @channel = Channel.create!(
-      area:          'Email::Account',
-      group_id:      group.id,
-      options:       {
-        inbound:  {
-          adapter: 'imap',
-          options: {
-            host:     ENV['MAIL_SERVER'],
-            user:     @server_login,
-            password: @server_password,
-            ssl:      true,
-            folder:   @folder,
-            # keep_on_server: true,
-          }
-        },
-        outbound: {
-          adapter: 'sendmail'
-        }
-      },
-      active:        true,
-      updated_by_id: 1,
-      created_by_id: 1,
-    )
-    email_address.channel_id = @channel.id
-    email_address.save!
-
-  end
-
-  test 'keep on server' do
-    @channel.options[:inbound][:options][:keep_on_server] = true
-    @channel.save!
-
-    # clean mailbox
-    imap = Net::IMAP.new(ENV['MAIL_SERVER'], 993, true, nil, false)
-    imap.login(@server_login, @server_password)
-    imap.create(@folder)
-    imap.select(@folder)
-
-    # put unseen message in it
-    imap.append(@folder, "Subject: hello1
-From: shugo@example.com
-To: shugo@example.com
-Message-ID: <some1@example_keep_on_server>
-
-hello world
-".gsub(%r{\n}, "\r\n"), [], Time.zone.now)
-
-    # verify if message is still on server
-    message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
-    assert_equal(1, message_ids.count)
-
-    message_meta = imap.fetch(1, ['FLAGS'])[0].attr
-    assert_not(message_meta['FLAGS'].include?(:Seen))
-
-    # fetch messages
-    article_count = Ticket::Article.count
-    @channel.fetch(true)
-    assert_equal(article_count + 1, Ticket::Article.count)
-
-    # verify if message is still on server
-    message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
-    assert_equal(1, message_ids.count)
-
-    message_meta = imap.fetch(1, ['RFC822.HEADER', 'FLAGS'])[0].attr
-    assert(message_meta['FLAGS'].include?(:Seen))
-
-    # fetch messages
-    article_count = Ticket::Article.count
-    @channel.fetch(true)
-    assert_equal(article_count, Ticket::Article.count)
-
-    # verify if message is still on server
-    message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
-    assert_equal(1, message_ids.count)
-
-    # put unseen message in it
-    imap.append(@folder, "Subject: hello2
-From: shugo@example.com
-To: shugo@example.com
-Message-ID: <some2@example_keep_on_server>
-
-hello world
-".gsub(%r{\n}, "\r\n"), [], Time.zone.now)
-
-    message_meta = imap.fetch(1, ['FLAGS'])[0].attr
-    assert(message_meta['FLAGS'].include?(:Seen))
-    message_meta = imap.fetch(2, ['FLAGS'])[0].attr
-    assert_not(message_meta['FLAGS'].include?(:Seen))
-
-    # fetch messages
-    article_count = Ticket::Article.count
-    @channel.fetch(true)
-    assert_equal(article_count + 1, Ticket::Article.count)
-
-    # verify if message is still on server
-    message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
-    assert_equal(2, message_ids.count)
-
-    message_meta = imap.fetch(1, ['FLAGS'])[0].attr
-    assert(message_meta['FLAGS'].include?(:Seen))
-    message_meta = imap.fetch(2, ['FLAGS'])[0].attr
-    assert(message_meta['FLAGS'].include?(:Seen))
-
-    # set messages to not seen
-    imap.store(1, '-FLAGS', [:Seen])
-    imap.store(2, '-FLAGS', [:Seen])
-
-    # fetch messages
-    article_count = Ticket::Article.count
-    @channel.fetch(true)
-    assert_equal(article_count, Ticket::Article.count)
-
-    imap.delete(@folder)
-    @channel.destroy!
-  end
-
-  test 'keep not on server' do
-    @channel.options[:inbound][:options][:keep_on_server] = false
-    @channel.save!
-
-    # clean mailbox
-    imap = Net::IMAP.new(ENV['MAIL_SERVER'], 993, true, nil, false)
-    imap.login(@server_login, @server_password)
-    imap.create(@folder)
-    imap.select(@folder)
-
-    # put unseen message in it
-    imap.append(@folder, "Subject: hello1
-From: shugo@example.com
-To: shugo@example.com
-Message-ID: <some1@example_remove_from_server>
-
-hello world
-".gsub(%r{\n}, "\r\n"), [], Time.zone.now)
-
-    # verify if message is still on server
-    message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
-    assert_equal(1, message_ids.count)
-
-    message_meta = imap.fetch(1, ['FLAGS'])[0].attr
-    assert_not(message_meta['FLAGS'].include?(:Seen))
-
-    # fetch messages
-    article_count = Ticket::Article.count
-    @channel.fetch(true)
-    assert_equal(article_count + 1, Ticket::Article.count)
-
-    # verify if message is still on server
-    message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
-    assert_equal(1, message_ids.count)
-
-    # put unseen message in it
-    imap.append(@folder, "Subject: hello2
-From: shugo@example.com
-To: shugo@example.com
-Message-ID: <some2@example_remove_from_server>
-
-hello world
-".gsub(%r{\n}, "\r\n"), [], Time.zone.now)
-
-    # verify if message is still on server
-    message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
-    assert_equal(1, message_ids.count)
-
-    message_meta = imap.fetch(1, ['FLAGS'])[0].attr
-    assert_not(message_meta['FLAGS'].include?(:Seen))
-
-    # fetch messages
-    article_count = Ticket::Article.count
-    @channel.fetch(true)
-    assert_equal(article_count + 1, Ticket::Article.count)
-
-    # verify if message is still on server
-    message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
-    assert_equal(1, message_ids.count)
-
-    # put unseen message in it
-    imap.append(@folder, "Subject: hello2
-From: shugo@example.com
-To: shugo@example.com
-Message-ID: <some2@example_remove_from_server>
-
-hello world
-".gsub(%r{\n}, "\r\n"), [], Time.zone.now)
-
-    # verify if message is still on server
-    message_ids = imap.sort(['DATE'], ['ALL'], 'US-ASCII')
-    assert_equal(1, message_ids.count)
-
-    # fetch messages
-    article_count = Ticket::Article.count
-    @channel.fetch(true)
-    assert_equal(article_count + 1, Ticket::Article.count)
-
-    imap.delete(@folder)
-    @channel.destroy!
-
-  end
-
-end