Fixes #2401, Closes #2402 - Umlaut-Domains are not converted to ACE-Format during sendout

Gemfile

@@ -116,6 +116,9 @@ gem 'mail', git: 'https://github.com/zammad-deps/mail', branch: '2-7-stable'
 gem 'mime-types'
 gem 'rchardet', '>= 1.8.0'
 
+# convert from punycode ACE strings to unicode UTF-8 strings and visa versa
+gem 'simpleidn'
+
 # feature - business hours
 gem 'biz'
 

Gemfile.lock

@@ -690,6 +690,7 @@ DEPENDENCIES
   sassc-rails
   selenium-webdriver
   shoulda-matchers
+  simpleidn
   slack-notifier
   slack-ruby-client
   sprockets (~> 3.7.2)

app/assets/javascripts/app/controllers/_application_controller/form.coffee

@@ -111,6 +111,11 @@ class App.ControllerForm extends App.Controller
       if @isDisabled == true
         attribute.disabled = true
 
+      # We need to use the text attribute and a own validation, because unicode is in the
+      # default email html field not allowed.
+      if attribute.type is 'email'
+        attribute.input_type = 'text'
+
       # add item
       item = @formGenItem(attribute, @idPrefix, fieldset, attributeCount)
       item.appendTo(fieldset)

app/assets/javascripts/app/models/_application_model.coffee

@@ -129,7 +129,7 @@ class App.Model extends Spine.Model
 
         # check email
         if attribute.type is 'email' && data['params'][attributeName]
-          if !data['params'][attributeName].match(/\S+@\S+\.\S+/)
+          if !data['params'][attributeName].match(/^(([^<>()\[\]\.,;:\s@\"]+(\.[^<>()\[\]\.,;:\s@\"]+)*)|(\".+\"))@(([^<>()[\]\.,;:\s@\"]+\.)+[^<>()[\]\.,;:\s@\"]{2,})$/i)
             errors[attributeName] = 'invalid'
           if data['params'][attributeName].match(/ /)
             errors[attributeName] = 'invalid'

app/assets/javascripts/app/views/generic/input.jst.eco

@@ -1 +1 @@
-<input id="<%= @attribute.id %>" type="<%= @attribute.type %>" name="<%= @attribute.name %>" value="<%= @attribute.value %>" class="form-control <%= @attribute.class %>" <% if @attribute.placeholder: %>placeholder="<%- @Ti(@attribute.placeholder) %>"<% end %> <%= @attribute.required %> <%= @attribute.autofocus %> <%- @attribute.autocapitalize %> <%- @attribute.autocomplete %> <% if @attribute.min isnt undefined: %> min="<%= @attribute.min %>"<% end %><% if @attribute.max isnt undefined: %> max="<%= @attribute.max %>"<% end %><% if @attribute.step: %> step="<%= @attribute.step %>"<% end %><% if @attribute.maxlength: %> maxlength="<%= @attribute.maxlength %>"<% end %><% if @attribute.disabled: %> disabled<% end %>/>
+<input id="<%= @attribute.id %>" type="<%= (@attribute.input_type || @attribute.type) %>" name="<%= @attribute.name %>" value="<%= @attribute.value %>" class="form-control <%= @attribute.class %>" <% if @attribute.placeholder: %>placeholder="<%- @Ti(@attribute.placeholder) %>"<% end %> <%= @attribute.required %> <%= @attribute.autofocus %> <%- @attribute.autocapitalize %> <%- @attribute.autocomplete %> <% if @attribute.min isnt undefined: %> min="<%= @attribute.min %>"<% end %><% if @attribute.max isnt undefined: %> max="<%= @attribute.max %>"<% end %><% if @attribute.step: %> step="<%= @attribute.step %>"<% end %><% if @attribute.maxlength: %> maxlength="<%= @attribute.maxlength %>"<% end %><% if @attribute.disabled: %> disabled<% end %><% if @attribute.type is 'email': %> pattern="(([^<>()\[\]\.,;:\s@\u0022]+(\.[^<>()\[\]\.,;:\s@\u0022]+)*)|(\u0022.+\u0022))@(([^<>()[\]\.,;:\s@\u0022]+\.)+[^<>()[\]\.,;:\s@\u0022]{2,})" title="<%- @Ti("Enter a valid email address.")%>"<% end %> />

app/models/channel/driver/sendmail.rb

@@ -1,6 +1,8 @@
 # Copyright (C) 2012-2022 Zammad Foundation, https://zammad-foundation.org/
 
 class Channel::Driver::Sendmail
+  include Channel::EmailHelper
+
   def send(_options, attr, notification = false)
 
     # return if we run import mode
@@ -14,6 +16,7 @@ class Channel::Driver::Sendmail
       attr[:bcc] += ', ' if attr[:bcc].present?
       attr[:bcc] += system_bcc
     end
+    attr = prepare_idn_outbound(attr)
 
     mail = Channel::EmailBuild.build(attr, notification)
     delivery_method(mail)

app/models/channel/driver/smtp.rb

@@ -1,6 +1,7 @@
 # Copyright (C) 2012-2022 Zammad Foundation, https://zammad-foundation.org/
 
 class Channel::Driver::Smtp
+  include Channel::EmailHelper
 
   # we're using the same timeouts like in Net::SMTP gem
   #   but we would like to have the possibility to mock it for tests
@@ -64,6 +65,7 @@ class Channel::Driver::Smtp
       attr[:bcc] += ', ' if attr[:bcc].present?
       attr[:bcc] += system_bcc
     end
+    attr = prepare_idn_outbound(attr)
 
     mail = Channel::EmailBuild.build(attr, notification)
     smtp_params = {

app/models/channel/email_helper.rb

@@ -0,0 +1,30 @@
+# Copyright (C) 2012-2022 Zammad Foundation, https://zammad-foundation.org/
+
+module Channel::EmailHelper
+  PARTICIPANTS = %i[from to cc bcc reply-to return-path sender
+                    resent-from resent-to resent-bcc
+                    delivered-to x-original-to envelope-to].freeze
+
+  def prepare_idn_outbound(mail)
+    prepare_idn(mail, 'to_ascii')
+  end
+
+  def prepare_idn_inbound(mail)
+    prepare_idn(mail, 'to_unicode')
+  end
+
+  private
+
+  def prepare_idn(mail, action)
+    PARTICIPANTS.each do |participant|
+      next if !mail[participant]
+
+      mail[participant] = mail[participant]
+        .split(', ')
+        .map { |address| EmailHelper::Idn.send(action, address) }
+        .join(', ')
+    end
+
+    mail
+  end
+end

app/models/channel/email_parser.rb

@@ -3,6 +3,8 @@
 # encoding: utf-8
 
 class Channel::EmailParser
+  include Channel::EmailHelper
+
   PROZESS_TIME_MAX = 180
   EMAIL_REGEX = %r{.+@.+}
   RECIPIENT_FIELDS = %w[to cc delivered-to x-original-to envelope-to].freeze
@@ -175,6 +177,9 @@ returns
     article      = nil
     session_user = nil
 
+    # https://github.com/zammad/zammad/issues/2401
+    mail = prepare_idn_inbound(mail)
+
     # use transaction
     Transaction.execute(transaction_params) do
 

app/models/channel/filter/identify_sender.rb

@@ -152,6 +152,10 @@ module Channel::Filter::IdentifySender
   def self.user_create(attrs, role_ids = nil)
     populate_attributes!(attrs, role_ids: role_ids)
 
+    if attrs[:login]
+      attrs[:login] = EmailHelper::Idn.to_unicode(attrs[:login])
+    end
+
     if (user = User.find_by('email = :email OR login = :email', attrs))
       user.update!(attrs.slice(:firstname)) if user.no_name? && attrs[:firstname].present?
     elsif (user = User.create!(attrs))
@@ -193,7 +197,7 @@ module Channel::Filter::IdentifySender
   def self.sanitize_email(string)
     string += '@local' if string.exclude?('@')
 
-    string.downcase
+    string = string.downcase
           .strip
           .delete('"')
           .delete("'")
@@ -202,6 +206,8 @@ module Channel::Filter::IdentifySender
           .sub(%r{\A'(.*)'\z}, '\1') # see https://github.com/zammad/zammad/issues/2154
           .gsub(%r{\s}, '')          # see https://github.com/zammad/zammad/issues/2198
           .delete_suffix('.')
+
+    EmailHelper::Idn.to_unicode(string)
   end
 
 end