Improved charset handling.

app/models/channel/email_parser.rb

@@ -2,10 +2,14 @@ require 'mail'
 require 'iconv'
 class Channel::EmailParser
   def conv (charset, string)
-    if charset == 'US-ASCII' then
+    if charset == 'US-ASCII' || charset == 'ASCII-8BIT'
       charset = 'LATIN1'
     end
-    Iconv.conv("UTF8", charset, string)
+    return string if charset.downcase == 'utf8' || charset.downcase == 'utf-8'
+#    puts '-------' + charset
+#    puts string
+#    string.encode("UTF-8")
+    Iconv.conv( 'UTF8', charset, string )
   end
   
   def parse (msg)
@@ -16,17 +20,22 @@ class Channel::EmailParser
     data[:from_email]        = Mail::Address.new( mail[:from].value ).address
     data[:from_display_name] = Mail::Address.new( mail[:from].value ).display_name
     ['from', 'to', 'cc', 'subject'].each {|key|
-      data[key.to_sym] = mail[key] ? conv( mail[key].charset || 'LATIN1', mail[key].to_s) : nil
+      data[key.to_sym] = mail[key] ? mail[key].to_s : nil
     }
 
     # message id
     data[:message_id] = mail['message_id'] ? mail['message_id'].to_s : nil
 
     # body
-   #   plain_part = mail.multipart? ? (mail.text_part ? mail.text_part.body.decoded : nil) : mail.body.decoded
-  #    html_part = message.html_part ? message.html_part.body.decoded : nil
-    data[:plain_part] = mail.multipart? ? (mail.text_part ? mail.text_part.body.decoded : nil) : mail.body.decoded
-    data[:plain_part] = conv( mail.body.charset || 'LATIN1', data[:plain_part] )
+#    plain_part = mail.multipart? ? (mail.text_part ? mail.text_part.body.decoded : nil) : mail.body.decoded
+#    html_part = message.html_part ? message.html_part.body.decoded : nil
+    if mail.multipart?
+      data[:plain_part] = mail.text_part.body.decoded
+      data[:plain_part] = conv( mail.text_part.charset || 'LATIN1', data[:plain_part] )
+    else
+      data[:plain_part] = mail.body.decoded
+      data[:plain_part] = conv( mail.body.charset || 'LATIN1', data[:plain_part] )
+    end
 
     # attachments
     if mail.attachments
@@ -100,7 +109,7 @@ class Channel::EmailParser
       # create new ticket
       if !ticket then
         ticket = Ticket.create(
-          :group_id           => channel[:group_id],
+          :group_id           => channel[:group_id] || 1,
           :customer_id        => user.id,
           :title              => mail[:subject],
           :ticket_state_id    => Ticket::State.where(:name => 'new').first.id,

test/fixtures/mail2.box

@@ -0,0 +1,17 @@
+From: Martin Edenhofer <martin@example.com>
+Content-Type: text/plain;
+	charset=iso-8859-1
+Content-Transfer-Encoding: quoted-printable
+Subject: =?iso-8859-1?Q?aa=E4=F6=FC=DFad_asd?=
+X-Universally-Unique-Identifier: d12c15d2-e6d6-4ccd-86c7-abc2c3d0a2a2
+Date: Fri, 4 May 2012 14:01:03 +0200
+Message-Id: <BC182994-03FA-4DC5-8202-98CBFACA0887@example.com>
+To: metest@znuny.com
+Mime-Version: 1.0 (Apple Message framework v1257)
+
+=E4=F6=FC=DF ad asd
+
+-Martin
+
+--
+Old programmers never die. They just branch to a new address.

test/unit/email_parser_test.rb

@@ -14,6 +14,17 @@ class EmailParserTest < ActiveSupport::TestCase
           :subject            => 'CI Daten für PublicView ',
         },
       },
+      {
+        :data     => IO.read('test/fixtures/mail2.box'),
+        :body_md5 => '25a1ff722497271965b55e52659784a6',
+        :params   => {
+          :from               => 'Martin Edenhofer <martin@example.com>',
+          :from_email         => 'martin@example.com',
+          :from_display_name  => 'Martin Edenhofer',
+          :subject            => 'aaäöüßad asd',
+          :plain_part         => "äöüß ad asd\r\n\r\n-Martin\r\n\r\n--\r\nOld programmers never die. They just branch to a new address.",
+        },
+      },
     ]
 
     files.each { |file|
@@ -21,11 +32,17 @@ class EmailParserTest < ActiveSupport::TestCase
       parser = Channel::EmailParser.new
       data = parser.parse( file[:data] )
       
-      # create md5 of body
+      # check body
       md5 = Digest::MD5.hexdigest( data[:plain_part] )
-      assert_equal( file[:body_md5], md5 )      
+      assert_equal( file[:body_md5], md5 )
+      
+      # check params
       file[:params].each { |key, value|
-        assert_equal( file[:params][key.to_sym], data[key.to_sym] )      
+        if key.to_s == 'plain_part'
+          assert_equal( Digest::MD5.hexdigest( file[:params][key.to_sym].to_s ), Digest::MD5.hexdigest( data[key.to_sym].to_s ) )
+        else
+          assert_equal( file[:params][key.to_sym], data[key.to_sym] )
+        end
       }
     }
   end