Maintenance: Port 'email_parser' minitest to rspec.

spec/models/channel/email_parser_spec.rb

@@ -4,6 +4,39 @@ require 'rails_helper'
 
 RSpec.describe Channel::EmailParser, type: :model do
   describe '#parse' do
+
+    shared_examples 'parses email correctly' do |stored_email|
+      context "for #{stored_email}" do
+        let(:yml_file)                 { stored_email.ext('yml') }
+        let(:content)                  { YAML.load(File.read(yml_file)) } # rubocop:disable Security/YAMLLoad
+        let(:parsed)                   { described_class.new.parse(File.read(stored_email)) }
+        let(:expected_msg)             { content.except(:attachments) }
+        let(:parsed_msg)               { parsed.slice(*expected_msg.keys) }
+        let(:content_attachments_md5s) { (content[:attachments]&.map { |a| Digest::MD5.hexdigest(a[:data]) } || []).to_set }
+        let(:parsed_attachments_md5s)  { (parsed[:attachments]&.map { |a| Digest::MD5.hexdigest(a[:data]) } || []).to_set }
+
+        it 'parses correctly' do
+          expect(File).to exist(yml_file)
+          expect(parsed_msg).to include(expected_msg)
+          expect(content_attachments_md5s).to be_subset(parsed_attachments_md5s)
+        end
+      end
+    end
+
+    # To write new .yml files for emails you can use the following code:
+    #
+    # File.write('test/data/mail/mailXXX.yml', Channel::EmailParser.new.parse(File.read('test/data/mail/mailXXX.box')).slice(:from, :from_email, :from_display_name, :to, :cc, :subject, :body, :content_type, :'reply-to', :attachments).to_yaml)
+    #
+    context 'when checking a bunch of stored emails for correct parsing behaviour' do
+      tests = Dir.glob(Rails.root.join('test/data/mail/mail*.box')).each do |stored_email|
+        include_examples('parses email correctly', stored_email)
+      end
+
+      it 'ensures tests were dynamically generated' do
+        expect(tests.count).to eq(106)
+      end
+    end
+
     # regression test for issue 2390 - Add a postmaster filter to not show emails with potential issue
     describe 'handling HTML links in message content' do
       context 'with under 5,000 links' do

test/data/mail/mail033-undelivered-mail-returned-to-sender.yml

@@ -0,0 +1,90 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: MAILER-DAEMON@mx1.zammad.com (Mail Delivery System)
+from_email: MAILER-DAEMON@mx1.zammad.com
+from_display_name: Mail Delivery System
+to: edenhofer@zammad.example
+subject: Undelivered Mail Returned to Sender
+body: |
+  This is the mail system at host mx1.zammad.com.
+
+  I'm sorry to have to inform you that your message could not
+  be delivered to one or more recipients. It's attached below.
+
+  For further assistance, please send mail to postmaster.
+
+  If you do so, please include this problem report. You can
+  delete your own text from the attached returned message.
+
+                     The mail system
+
+  <not_existing@znuny.com>: host arber.znuny.com[88.198.51.81] said: 550 5.1.1
+      <not_existing@znuny.com>: Recipient address rejected: User unknown (in
+      reply to RCPT TO command)
+content_type: text/plain
+attachments:
+- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+  data: |
+    Reporting-MTA: dns; mx1.zammad.com
+    X-Postfix-Queue-ID: EF1A820C3E97
+    X-Postfix-Sender: rfc822; edenhofer@zammad.example
+    Arrival-Date: Sun, 30 Aug 2015 16:56:02 +0200 (CEST)
+
+    Final-Recipient: rfc822; not_existing@znuny.com
+    Original-Recipient: rfc822;not_existing@znuny.com
+    Action: failed
+    Status: 5.1.1
+    Remote-MTA: dns; arber.znuny.com
+    Diagnostic-Code: smtp; 550 5.1.1 <not_existing@znuny.com>: Recipient address
+        rejected: User unknown
+  filename: Delivery report.txt
+  preferences: !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+    Content-Type: message/delivery-status
+    Content-Description: Delivery report
+    Mime-Type: message/delivery-status
+    Charset: UTF-8
+- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+  data: "Return-Path: <edenhofer@zammad.example>\nReceived: from appnode2.dc.zammad.com
+    (appnode2.dc.zammad.com [144.1.1.1])\n\tby mx1.zammad.com (Postfix) with ESMTP
+    id 4B9BB20C3E96\n\tfor <not_existing@znuny.com>; Sun, 30 Aug 2015 16:56:02 +0200
+    (CEST)\nReceived: by appnode2.dc.zammad.com (Postfix, from userid 1001)\n\tid
+    36680141408; Sun, 30 Aug 2015 16:56:07 +0200 (CEST)\nDate: Sun, 30 Aug 2015 16:56:07
+    +0200\nFrom: Martin Edenhofer via Zammad Helpdesk <edenhofer@zammad.example>\nTo:
+    Martin Edenhofer <not_existing@znuny.com>\nMessage-ID: <20150830145601.30.608881@edenhofer.zammad.com>\nIn-Reply-To:
+    \nSubject: test [Ticket#10010]\nMime-Version: 1.0\nContent-Type: multipart/mixed;\n
+    boundary=\"--==_mimepart_55e319872c918_39d5375468c949db\";\n charset=UTF-8\nContent-Transfer-Encoding:
+    7bit\nOrganization: \nX-Mailer: Zammad Mail Service (1.x)\nX-znuny-MailScanner-Information:
+    Please contact the ISP for more information\nX-znuny-MailScanner-ID: 4B9BB20C3E96.A3EE2\nX-znuny-MailScanner:
+    Found to be clean\nX-znuny-MailScanner-From: edenhofer@zammad.example\nX-Spam-Status:
+    No\n\n\n----==_mimepart_55e319872c918_39d5375468c949db\nContent-Type: multipart/alternative;\n
+    boundary=\"--==_mimepart_55e319872c5bd_39d5375468c94778\";\n charset=UTF-8\nContent-Transfer-Encoding:
+    7bit\n\n\n----==_mimepart_55e319872c5bd_39d5375468c94778\nContent-Type: text/plain;\n
+    charset=UTF-8\nContent-Transfer-Encoding: 7bit\n\nHello,\n\nis somebody there?\n\nMartin
+    Edenhofer\n\n--\n Super Support - Waterford Business Park\n 5201 Blue Lagoon Drive
+    - 8th Floor & 9th Floor - Miami, 33126 USA\n Email: [1] hot@example.com - Web:
+    [2] http://www.example.com/\n--\n\n\n[1] mailto:hot@example.com\n[2] http://www.example.com/\n----==_mimepart_55e319872c5bd_39d5375468c94778\nContent-Type:
+    multipart/related;\n boundary=\"--==_mimepart_55e319872c7db_39d5375468c9486e\";\n
+    charset=UTF-8\nContent-Transfer-Encoding: 7bit\n\n\n----==_mimepart_55e319872c7db_39d5375468c9486e\nContent-Type:
+    text/html;\n charset=UTF-8\nContent-Transfer-Encoding: 7bit\n\n<!DOCTYPE html>\n<html>\n
+    \ <head>\n    <meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>\n
+    \   <style type=\"text/css\">\n    body {\n      width:90% !important;\n      -webkit-text-size-adjust:90%;\n
+    \     -ms-text-size-adjust:90%;\n      font-family:'Helvetica Neue', Helvetica,
+    Arial, Geneva, sans-serif; font-size: 12px;;\n    }\n    img {\n      outline:none;
+    text-decoration:none; -ms-interpolation-mode: bicubic;\n    }\n    a img {\n      border:none;\n
+    \   }\n    table td {\n      border-collapse: collapse;\n    }\n    table {\n
+    \     border-collapse: collapse; mso-table-lspace:0pt; mso-table-rspace:0pt;\n
+    \   }\n    p, table, div, td {\n      max-width: 600px;\n    }\n    p {\n      margin:
+    0;\n    }\n    blockquote, pre {\n      margin: 0px;\n      padding: 8px 12px
+    8px 12px;\n    }\n\n    </style>\n  <head>\n  <body style=\"font-family:'Helvetica
+    Neue', Helvetica, Arial, Geneva, sans-serif; font-size: 12px;\"><div>Hello,</div><div><br></div><div>is
+    somebody there?</div><br><div data-signature=\"true\" data-signature-id=\"1\"><div>Martin
+    Edenhofer</div><div><br></div><div>--</div><div> Super Support - Waterford Business
+    Park</div><div> 5201 Blue Lagoon Drive - 8th Floor &amp; 9th Floor - Miami, 33126
+    USA</div><div> Email: <a href=\"mailto:hot@example.com\" title=\"mailto:hot@example.com\"
+    target=\"_blank\">hot@example.com</a> - Web: <a href=\"http://www.example.com/\"
+    title=\"http://www.example.com/\" target=\"_blank\">http://www.example.com/</a></div><div>--</div></div></body>\n</html>\n\n----==_mimepart_55e319872c7db_39d5375468c9486e--\n\n----==_mimepart_55e319872c5bd_39d5375468c94778--\n\n----==_mimepart_55e319872c918_39d5375468c949db--\n"
+  filename: test [Ticket#10010].eml
+  preferences: !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+    Content-Type: message/rfc822
+    Content-Description: Undelivered Message
+    Mime-Type: message/rfc822
+    Charset: UTF-8

test/data/mail/mail035.yml

@@ -0,0 +1,22 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: '"finances8@firstfinanceloanfirm.example.com"'
+from_email: '"finances8@firstfinanceloanfirm.example.com"'
+from_display_name: ''
+to: ''
+subject: Darlehen bieten jetzt bewerben
+body: |+
+  Beantworten :firstfinancelender@example.com
+
+  Ich Mr.Squires Peter ist eine zuverlässige Kreditangebot mit einer
+  Rate von 1.5% für den Zeitraum von 1 bis 40yrs nur, von der minimalen
+  von 5,000.00euro sie an die maximale Menge an 150,000.000.00euro, so
+  dass für mehr Details, wenn interessiert kontaktieren Sie mich unter
+
+  firstfinancelender@example.com
+
+  Beantworten :firstfinancelender@example.com
+
+content_type: text/plain
+reply-to: firstfinanceloanfirm@example.com
+attachments: []
+...

test/data/mail/mail055.yml

@@ -0,0 +1,223 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: MAILER-DAEMON@mx1.zammad.loc (Mail Delivery System)
+from_email: MAILER-DAEMON@mx1.zammad.loc
+from_display_name: Mail Delivery System
+to: example@zammad.com
+subject: Undelivered Mail Returned to Sender
+body: |
+  This is the mail system at host mx1.zammad.loc.
+
+  I'm sorry to have to inform you that your message could not
+  be delivered to one or more recipients. It's attached below.
+
+  For further assistance, please send mail to postmaster.
+
+  If you do so, please include this problem report. You can
+  delete your own text from the attached returned message.
+
+                     The mail system
+
+                     <ticket-bounce-trigger2@example.com>: host aspmx.l.example.com[108.177.96.26] said:
+      550-5.1.1 The email account that you tried to reach does not exist. Please
+      try 550-5.1.1 double-checking the recipient's email address for typos or
+      550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1
+      https://support.example.com/mail/?p=NoSuchUser l59si1635011edl.281 - gsmtp
+      (in reply to RCPT TO command)
+content_type: text/plain
+attachments:
+- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+  data: |
+    Reporting-MTA: dns; mx1.zammad.loc
+    X-Postfix-Queue-ID: 207FF20398ED
+    X-Postfix-Sender: rfc822; example@zammad.com
+    Arrival-Date: Fri, 26 May 2017 17:01:45 +0200 (CEST)
+
+    Final-Recipient: rfc822; ticket-bounce-trigger2@example.com
+    Original-Recipient: rfc822;ticket-bounce-trigger2@example.com
+    Action: failed
+    Status: 5.1.1
+    Remote-MTA: dns; aspmx.l.example.com
+    Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does
+        not exist. Please try 550-5.1.1 double-checking the recipient's email
+        address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.1.1
+        https://support.example.com/mail/?p=NoSuchUser l59si1635011edl.281 - gsmtp
+  filename: Delivery report.txt
+  preferences: !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+    Content-Type: message/delivery-status
+    Content-Description: Delivery report
+    Mime-Type: message/delivery-status
+    Charset: UTF-8
+- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+  data: |
+    Return-Path: <example@zammad.com>
+    Received: from apn0000.dc.zammad.com (apn0000.dc.zammad.com [88.0.0.0])
+      by mx1.zammad.loc (Postfix) with ESMTP id 207FF20398ED
+      for <ticket-bounce-trigger2@example.com>; Fri, 26 May 2017 17:01:45 +0200 (CEST)
+    Received: by apn0000.dc.zammad.com (Postfix, from userid 1050)
+      id 08443420973; Fri, 26 May 2017 17:01:45 +0200 (CEST)
+    Date: Fri, 26 May 2017 17:01:45 +0200
+    From: Twelve SaaS GmbH Helpdesk <example@zammad.com>
+    To: ticket-bounce-trigger2@example.com
+    Message-ID: <20170526150141.232.13312@example.zammad.loc>
+    In-Reply-To:
+    References: <20170526150142.232.819805@example.zammad.loc>
+     <20170526150119.6C5E520A13B2@mx1.zammad.loc>
+     <20170526150141.232.799457@example.zammad.loc>
+     <20170526150117.0560820A13B3@mx1.zammad.loc>
+     <20170526150115.232.175460@example.zammad.loc>
+     <20170526150108.232.482766@example.zammad.loc>
+     <20170526150041.F3D2C20A13B3@mx1.zammad.loc>
+     <20170526150036.232.513248@example.zammad.loc>
+     <20170526150008.6AE8A20A13B8@mx1.zammad.loc>
+     <20170526150004.232.103372@example.zammad.loc>
+     <20170526145940.D799220A13B3@mx1.zammad.loc>
+     <20170526145932.232.91897@example.zammad.loc>
+     <20170526145906.8FCA520A13B2@mx1.zammad.loc>
+     <20170526145901.232.269971@example.zammad.loc>
+     <lyHYjBDWwaU5KGbDrCyOfA@notifications.example.com>
+     Subject: =?UTF-8?Q?[Ticket#1705265400361]_RE:_Thanks_for_your_follow_up?=
+     =?UTF-8?Q?_=28G_Suite:_Benachrichtigung_=C3=BCber_Verl=C3=A4ngerung_in_30?=
+     =?UTF-8?Q?_Tagen=29?=
+     Mime-Version: 1.0
+    Content-Type: multipart/mixed;
+     boundary="--==_mimepart_5928435950b1_22d42086504355bc";
+     charset=UTF-8
+     Content-Transfer-Encoding: 7bit
+    Organization: Twelve SaaS GmbH
+    X-Loop: yes
+    Precedence: bulk
+    Auto-Submitted: auto-generated
+    X-Auto-Response-Suppress: All
+    X-Powered-By: Zammad - Helpdesk/Support (https://zammad.org/)
+    X-Mailer: Zammad Mail Service
+
+
+    ----==_mimepart_5928435950b1_22d42086504355bc
+    Content-Type: multipart/alternative;
+     boundary="--==_mimepart_592843594d35_22d4208650435394";
+     charset=UTF-8
+     Content-Transfer-Encoding: 7bit
+
+
+    ----==_mimepart_592843594d35_22d4208650435394
+    Content-Type: text/plain;
+     charset=UTF-8
+     Content-Transfer-Encoding: 7bit
+
+    Your follow-up for (Ticket#1705265400361) has been received and will be reviewed by our support staff.
+
+    To provide additional information, please reply to this email or click on the following link:[1] https://example.zammad.loc/#ticket/zoom/232
+
+    Your Twelve SaaS Helpdesk Team
+
+    [2] Zammad, your customer support system
+
+    [1] https://example.zammad.loc/#ticket/zoom/232
+    [2] https://zammad.com
+    ----==_mimepart_592843594d35_22d4208650435394
+    Content-Type: multipart/related;
+     boundary="--==_mimepart_592843594e47_22d4208650435484";
+     charset=UTF-8
+     Content-Transfer-Encoding: 7bit
+
+
+    ----==_mimepart_592843594e47_22d4208650435484
+    Content-Type: text/html;
+     charset=UTF-8
+     Content-Transfer-Encoding: 7bit
+
+    <!DOCTYPE html>
+    <html>
+      <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+        <style type="text/css">
+          body {
+            font-family:'Helvetica Neue', Helvetica, Arial, Geneva, sans-serif; font-size: 12px;;
+          }
+          img {
+            outline: none;
+            text-decoration: none;
+            -ms-interpolation-mode: bicubic;
+          }
+          a img {
+            border: none;
+          }
+          table td {
+            border-collapse: collapse;
+          }
+          table {
+            border-collapse: collapse;
+            mso-table-lspace: 0pt;
+            mso-table-rspace: 0pt;
+            border: none;
+            table-layout: auto;
+            display: block;
+            width: 100%;
+            overflow: auto;
+            word-break: keep-all;
+          }
+          table,
+          pre,
+          blockquote {
+            margin: 0 0 16px;
+          }
+          td, th {
+            padding: 7px 12px;
+            border: 1px solid hsl(0,0%,87%);
+          }
+          th {
+            font-weight: bold;
+            text-align: center;
+          }
+          tbody tr:nth-child(even) {
+            background: hsl(0,0%,97%);
+          }
+          col {
+            width: auto;
+          }
+          p {
+            margin: 0;
+          }
+          code {
+            border: none;
+            background: hsl(0,0%,97%);
+            white-space: pre-wrap;
+          }
+          blockquote {
+            padding: 8px 12px;
+            border-left: 5px solid #eee;
+          }
+          pre {
+            padding: 12px 15px;
+            font-size: 13px;
+            line-height: 1.45;
+            background: hsl(0,0%,97%);
+            white-space: pre-wrap;
+            border-radius: 3px;
+            border: none;
+            overflow: auto;
+          }
+        </style>
+      </head>
+      <body style="font-family:'Helvetica Neue', Helvetica, Arial, Geneva, sans-serif; font-size: 12px;"><div>Your follow-up for <b>(Ticket#1705265400361)</b> has been received and will be reviewed by our support staff.</div>
+      <br>
+    <div>To provide additional information, please reply to this email or click on the following link:
+    <a href="https://example.zammad.loc/#ticket/zoom/232" rel="nofollow noreferrer noopener" target="_blank">https://example.zammad.loc/#ticket/zoom/232</a>
+    </div>
+    <br>
+    <div>Your Twelve SaaS Helpdesk Team</div>
+    <br>
+    <div><i><a href="https://zammad.com" rel="nofollow noreferrer noopener" target="_blank">Zammad</a>, your customer support system</i></div></body>
+    </html>
+
+    ----==_mimepart_592843594e47_22d4208650435484--
+
+    ----==_mimepart_592843594d35_22d4208650435394--
+
+    ----==_mimepart_5928435950b1_22d42086504355bc--
+  filename: Undelivered Message.eml
+  preferences: !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+    Content-Type: message/rfc822
+    Content-Description: Undelivered Message
+    Mime-Type: message/rfc822
+    Charset: UTF-8

test/data/mail/mail060.yml

@@ -0,0 +1,9 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: Martin Edenhofer <martin@example.com>
+from_email: martin@example.com
+from_display_name: Martin Edenhofer
+to: metest@znuny.com
+subject: abc
+body: "Here it goes - äÜß - 塎ĺ\x87şäşşHere it goes - äöü - hi ­"
+content_type: text/plain
+attachments: []

test/data/mail/mail065.yml

@@ -0,0 +1,15 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: "=?iso-8859-1?Q?B=FCrling, =20Andreas?= <smith@example.com>"
+from_email: smith@example.com
+from_display_name: "=20Andreas?="
+to: metest@znuny.com
+subject: aaäöüßad asd
+body: |
+  äöüß ad asd
+
+  -Martin
+
+  --
+  Old programmers never die. They just branch to a new address.
+content_type: text/plain
+attachments: []

test/data/mail/mail070.yml

@@ -0,0 +1,9 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: '"http.abc" <http.abc@example.com>'
+from_email: http.abc@example.com
+from_display_name: http.abc
+to: ''
+subject: test
+body: no visible content
+content_type: text/plain
+attachments: []

test/data/mail/mail071.yml

@@ -0,0 +1,9 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: '"your secret past" <power quadrant system@example.com>'
+from_email: power quadrant system@example.com
+from_display_name: your secret past
+to: user@example.com
+subject: how've things been?
+body: no visible content
+content_type: text/plain
+attachments: []