email_parser.rb 32 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033
  1. # Copyright (C) 2012-2023 Zammad Foundation, https://zammad-foundation.org/
  2. # encoding: utf-8
  3. class Channel::EmailParser
  4. include Channel::EmailHelper
  5. PROZESS_TIME_MAX = 180
  6. EMAIL_REGEX = %r{.+@.+}
  7. RECIPIENT_FIELDS = %w[to cc delivered-to x-original-to envelope-to].freeze
  8. SENDER_FIELDS = %w[from reply-to return-path sender].freeze
  9. EXCESSIVE_LINKS_MSG = __('This message cannot be displayed because it contains over 5,000 links. Download the raw message below and open it via an Email client if you still wish to view it.').freeze
  10. MESSAGE_STRUCT = Struct.new(:from_display_name, :subject, :msg_size).freeze
  11. UNPROCESSABLE_MAIL_DIRECTORY = Rails.root.join('var/spool/unprocessable_mail')
  12. OVERSIZED_MAIL_DIRECTORY = Rails.root.join('var/spool/oversized_mail')
  13. =begin
  14. parser = Channel::EmailParser.new
  15. mail = parser.parse(msg_as_string)
  16. mail = {
  17. from: 'Some Name <some@example.com>',
  18. from_email: 'some@example.com',
  19. from_local: 'some',
  20. from_domain: 'example.com',
  21. from_display_name: 'Some Name',
  22. message_id: 'some_message_id@example.com',
  23. to: 'Some System <system@example.com>',
  24. cc: 'Somebody <somebody@example.com>',
  25. subject: 'some message subject',
  26. body: 'some message body',
  27. content_type: 'text/html', # text/plain
  28. date: Time.zone.now,
  29. attachments: [
  30. {
  31. data: 'binary of attachment',
  32. filename: 'file_name_of_attachment.txt',
  33. preferences: {
  34. 'content-alternative' => true,
  35. 'Mime-Type' => 'text/plain',
  36. 'Charset: => 'iso-8859-1',
  37. },
  38. },
  39. ],
  40. # ignore email header
  41. x-zammad-ignore: 'false',
  42. # customer headers
  43. x-zammad-customer-login: '',
  44. x-zammad-customer-email: '',
  45. x-zammad-customer-firstname: '',
  46. x-zammad-customer-lastname: '',
  47. # ticket headers (for new tickets)
  48. x-zammad-ticket-group: 'some_group',
  49. x-zammad-ticket-state: 'some_state',
  50. x-zammad-ticket-priority: 'some_priority',
  51. x-zammad-ticket-owner: 'some_owner_login',
  52. # ticket headers (for existing tickets)
  53. x-zammad-ticket-followup-group: 'some_group',
  54. x-zammad-ticket-followup-state: 'some_state',
  55. x-zammad-ticket-followup-priority: 'some_priority',
  56. x-zammad-ticket-followup-owner: 'some_owner_login',
  57. # article headers
  58. x-zammad-article-internal: false,
  59. x-zammad-article-type: 'agent',
  60. x-zammad-article-sender: 'customer',
  61. # all other email headers
  62. some-header: 'some_value',
  63. }
  64. =end
  65. def parse(msg)
  66. msg = msg.force_encoding('binary')
  67. # mail 2.6 and earlier accepted non-conforming mails that lacked the correct CRLF seperators,
  68. # mail 2.7 and above require CRLF so we force it on using binary_unsafe_to_crlf
  69. msg = Mail::Utilities.binary_unsafe_to_crlf(msg)
  70. mail = Mail.new(msg)
  71. message_ensure_message_id(msg, mail)
  72. force_parts_encoding_if_needed(mail)
  73. headers = message_header_hash(mail)
  74. body = message_body_hash(mail)
  75. message_attributes = [
  76. { mail_instance: mail },
  77. headers,
  78. body,
  79. self.class.sender_attributes(headers),
  80. { raw: msg },
  81. ]
  82. message_attributes.reduce({}.with_indifferent_access, &:merge)
  83. end
  84. =begin
  85. parser = Channel::EmailParser.new
  86. ticket, article, user, mail = parser.process(channel, email_raw_string)
  87. returns
  88. [ticket, article, user, mail]
  89. do not raise an exception - e. g. if used by scheduler
  90. parser = Channel::EmailParser.new
  91. ticket, article, user, mail = parser.process(channel, email_raw_string, false)
  92. returns
  93. [ticket, article, user, mail] || false
  94. =end
  95. def process(channel, msg, exception = true)
  96. Timeout.timeout(PROZESS_TIME_MAX) do
  97. _process(channel, msg)
  98. end
  99. rescue => e
  100. # store unprocessable email for bug reporting
  101. filename = archive_mail(UNPROCESSABLE_MAIL_DIRECTORY, msg)
  102. message = "Can't process email, you will find it for bug reporting under #{filename}, please create an issue at https://github.com/zammad/zammad/issues"
  103. p "ERROR: #{message}" # rubocop:disable Rails/Output
  104. p "ERROR: #{e.inspect}" # rubocop:disable Rails/Output
  105. Rails.logger.error message
  106. Rails.logger.error e
  107. return false if exception == false
  108. raise %(#{e.inspect}\n#{e.backtrace.join("\n")})
  109. end
  110. def _process(channel, msg)
  111. # parse email
  112. mail = parse(msg)
  113. Rails.logger.info "Process email with msgid '#{mail[:message_id]}'"
  114. # run postmaster pre filter
  115. UserInfo.current_user_id = 1
  116. # set interface handle
  117. original_interface_handle = ApplicationHandleInfo.current
  118. transaction_params = { interface_handle: "#{original_interface_handle}.postmaster", disable: [] }
  119. filters = {}
  120. Setting.where(area: 'Postmaster::PreFilter').reorder(:name).each do |setting|
  121. filters[setting.name] = Setting.get(setting.name).constantize
  122. end
  123. filters.each do |key, backend|
  124. Rails.logger.debug { "run postmaster pre filter #{key}: #{backend}" }
  125. begin
  126. backend.run(channel, mail, transaction_params)
  127. rescue => e
  128. Rails.logger.error "can't run postmaster pre filter #{key}: #{backend}"
  129. Rails.logger.error e.inspect
  130. raise e
  131. end
  132. end
  133. # check ignore header
  134. if mail[:'x-zammad-ignore'] == 'true' || mail[:'x-zammad-ignore'] == true
  135. Rails.logger.info "ignored email with msgid '#{mail[:message_id]}' from '#{mail[:from]}' because of x-zammad-ignore header"
  136. return
  137. end
  138. ticket = nil
  139. article = nil
  140. session_user = nil
  141. # https://github.com/zammad/zammad/issues/2401
  142. mail = prepare_idn_inbound(mail)
  143. # use transaction
  144. Transaction.execute(transaction_params) do
  145. # get sender user
  146. session_user_id = mail[:'x-zammad-session-user-id']
  147. if !session_user_id
  148. raise __('No x-zammad-session-user-id, no sender set!')
  149. end
  150. session_user = User.lookup(id: session_user_id)
  151. if !session_user
  152. raise "No user found for x-zammad-session-user-id: #{session_user_id}!"
  153. end
  154. # set current user
  155. UserInfo.current_user_id = session_user.id
  156. # get ticket# based on email headers
  157. if mail[:'x-zammad-ticket-id']
  158. ticket = Ticket.find_by(id: mail[:'x-zammad-ticket-id'])
  159. end
  160. if mail[:'x-zammad-ticket-number']
  161. ticket = Ticket.find_by(number: mail[:'x-zammad-ticket-number'])
  162. end
  163. # set ticket state to open if not new
  164. if ticket
  165. set_attributes_by_x_headers(ticket, 'ticket', mail, 'followup')
  166. # save changes set by x-zammad-ticket-followup-* headers
  167. ticket.save! if ticket.has_changes_to_save?
  168. # set ticket to open again or keep create state
  169. if !mail[:'x-zammad-ticket-followup-state'] && !mail[:'x-zammad-ticket-followup-state_id']
  170. new_state = Ticket::State.find_by(default_create: true)
  171. if ticket.state_id != new_state.id && !mail[:'x-zammad-out-of-office']
  172. ticket.state = Ticket::State.find_by(default_follow_up: true)
  173. ticket.save!
  174. end
  175. end
  176. end
  177. # create new ticket
  178. if !ticket
  179. preferences = {}
  180. if channel[:id]
  181. preferences = {
  182. channel_id: channel[:id]
  183. }
  184. end
  185. # get default group where ticket is created
  186. group = nil
  187. if channel[:group_id]
  188. group = Group.lookup(id: channel[:group_id])
  189. else
  190. mail_to_group = self.class.mail_to_group(mail[:to])
  191. if mail_to_group.present?
  192. group = mail_to_group
  193. end
  194. end
  195. if group.blank? || group.active == false
  196. group = Group.where(active: true).reorder(id: :asc).first
  197. end
  198. if group.blank?
  199. group = Group.first
  200. end
  201. title = mail[:subject]
  202. if title.blank?
  203. title = '-'
  204. end
  205. ticket = Ticket.new(
  206. group_id: group.id,
  207. title: title,
  208. preferences: preferences,
  209. )
  210. set_attributes_by_x_headers(ticket, 'ticket', mail)
  211. # create ticket
  212. ticket.save!
  213. end
  214. # apply tags to ticket
  215. if mail[:'x-zammad-ticket-tags'].present?
  216. mail[:'x-zammad-ticket-tags'].each do |tag|
  217. ticket.tag_add(tag, sourceable: mail[:'x-zammad-ticket-tags-source'])
  218. end
  219. end
  220. # set attributes
  221. article = Ticket::Article.new(
  222. ticket_id: ticket.id,
  223. type_id: Ticket::Article::Type.find_by(name: 'email').id,
  224. sender_id: Ticket::Article::Sender.find_by(name: 'Customer').id,
  225. content_type: mail[:content_type],
  226. body: mail[:body],
  227. from: mail[:from],
  228. reply_to: mail[:'reply-to'],
  229. to: mail[:to],
  230. cc: mail[:cc],
  231. subject: mail[:subject],
  232. message_id: mail[:message_id],
  233. internal: false,
  234. )
  235. # x-headers lookup
  236. set_attributes_by_x_headers(article, 'article', mail)
  237. # create article
  238. article.save!
  239. # store mail plain
  240. article.save_as_raw(msg)
  241. # store attachments
  242. mail[:attachments]&.each do |attachment|
  243. filename = attachment[:filename].force_encoding('utf-8')
  244. if !filename.force_encoding('UTF-8').valid_encoding?
  245. filename = filename.utf8_encode(fallback: :read_as_sanitized_binary)
  246. end
  247. Store.create!(
  248. object: 'Ticket::Article',
  249. o_id: article.id,
  250. data: attachment[:data],
  251. filename: filename,
  252. preferences: attachment[:preferences]
  253. )
  254. end
  255. end
  256. ticket.reload
  257. article.reload
  258. session_user.reload
  259. # run postmaster post filter
  260. filters = {}
  261. Setting.where(area: 'Postmaster::PostFilter').reorder(:name).each do |setting|
  262. filters[setting.name] = Setting.get(setting.name).constantize
  263. end
  264. filters.each_value do |backend|
  265. Rails.logger.debug { "run postmaster post filter #{backend}" }
  266. begin
  267. backend.run(channel, mail, ticket, article, session_user)
  268. rescue => e
  269. Rails.logger.error "can't run postmaster post filter #{backend}"
  270. Rails.logger.error e.inspect
  271. end
  272. end
  273. # return new objects
  274. [ticket, article, session_user, mail]
  275. end
  276. def self.mail_to_group(to)
  277. begin
  278. to = Mail::AddressList.new(to)&.addresses&.first&.address
  279. rescue
  280. Rails.logger.error 'Can not parse :to field for group destination!'
  281. end
  282. return if to.blank?
  283. email = EmailAddress.find_by(email: to.downcase)
  284. return if email&.channel.blank?
  285. email.channel&.group
  286. end
  287. def self.check_attributes_by_x_headers(header_name, value)
  288. class_name = nil
  289. attribute = nil
  290. # skip check attributes if it is tags
  291. return true if header_name == 'x-zammad-ticket-tags'
  292. if header_name =~ %r{^x-zammad-(.+?)-(followup-|)(.*)$}i
  293. class_name = $1
  294. attribute = $3
  295. end
  296. return true if !class_name
  297. if class_name.casecmp('article').zero?
  298. class_name = 'Ticket::Article'
  299. end
  300. return true if !attribute
  301. key_short = attribute[ attribute.length - 3, attribute.length ]
  302. return true if key_short != '_id'
  303. class_object = class_name.to_classname.constantize
  304. return if !class_object
  305. class_instance = class_object.new
  306. return false if !class_instance.association_id_validation(attribute, value)
  307. true
  308. end
  309. def self.sender_attributes(from)
  310. if from.is_a?(ActiveSupport::HashWithIndifferentAccess)
  311. from = SENDER_FIELDS.filter_map { |f| from[f] }
  312. .map(&:to_utf8).compact_blank
  313. .partition { |address| address.match?(EMAIL_REGEX) }
  314. .flatten.first
  315. end
  316. data = {}.with_indifferent_access
  317. return data if from.blank?
  318. from = from.gsub('<>', '').strip
  319. mail_address = begin
  320. Mail::AddressList.new(from).addresses
  321. .select { |a| a.address.present? }
  322. .partition { |a| a.address.match?(EMAIL_REGEX) }
  323. .flatten.first
  324. rescue Mail::Field::ParseError => e
  325. $stdout.puts e
  326. end
  327. if mail_address&.address.present?
  328. data[:from_email] = mail_address.address
  329. data[:from_local] = mail_address.local
  330. data[:from_domain] = mail_address.domain
  331. data[:from_display_name] = mail_address.display_name || mail_address.comments&.first
  332. elsif from =~ %r{^(.+?)<((.+?)@(.+?))>}
  333. data[:from_email] = $2
  334. data[:from_local] = $3
  335. data[:from_domain] = $4
  336. data[:from_display_name] = $1
  337. else
  338. data[:from_email] = from
  339. data[:from_local] = from
  340. data[:from_domain] = from
  341. data[:from_display_name] = from
  342. end
  343. # do extra decoding because we needed to use field.value
  344. data[:from_display_name] =
  345. Mail::Field.new('X-From', data[:from_display_name].to_utf8)
  346. .to_s
  347. .delete('"')
  348. .strip
  349. .gsub(%r{(^'|'$)}, '')
  350. data
  351. end
  352. def set_attributes_by_x_headers(item_object, header_name, mail, suffix = false)
  353. # loop all x-zammad-header-* headers
  354. item_object.attributes.each_key do |key|
  355. # ignore read only attributes
  356. next if key == 'updated_by_id'
  357. next if key == 'created_by_id'
  358. # check if id exists
  359. key_short = key[ key.length - 3, key.length ]
  360. if key_short == '_id'
  361. key_short = key[ 0, key.length - 3 ]
  362. header = "x-zammad-#{header_name}-#{key_short}"
  363. if suffix
  364. header = "x-zammad-#{header_name}-#{suffix}-#{key_short}"
  365. end
  366. # only set value on _id if value/reference lookup exists
  367. if mail[header.to_sym]
  368. Rails.logger.info "set_attributes_by_x_headers header #{header} found #{mail[header.to_sym]}"
  369. item_object.class.reflect_on_all_associations.map do |assoc|
  370. next if assoc.name.to_s != key_short
  371. Rails.logger.info "set_attributes_by_x_headers found #{assoc.class_name} lookup for '#{mail[header.to_sym]}'"
  372. item = assoc.class_name.constantize
  373. assoc_object = nil
  374. if item.new.respond_to?(:name)
  375. assoc_object = item.lookup(name: mail[header.to_sym])
  376. end
  377. if !assoc_object && item.new.respond_to?(:login)
  378. assoc_object = item.lookup(login: mail[header.to_sym])
  379. end
  380. if !assoc_object && item.new.respond_to?(:email)
  381. assoc_object = item.lookup(email: mail[header.to_sym])
  382. end
  383. if assoc_object.blank?
  384. # no assoc exists, remove header
  385. mail.delete(header.to_sym)
  386. next
  387. end
  388. Rails.logger.info "set_attributes_by_x_headers assign #{item_object.class} #{key}=#{assoc_object.id}"
  389. item_object[key] = assoc_object.id
  390. item_object.history_change_source_attribute(mail[:"#{header}-source"], key)
  391. end
  392. end
  393. end
  394. # check if attribute exists
  395. header = "x-zammad-#{header_name}-#{key}"
  396. if suffix
  397. header = "x-zammad-#{header_name}-#{suffix}-#{key}"
  398. end
  399. next if !mail[header.to_sym]
  400. Rails.logger.info "set_attributes_by_x_headers header #{header} found. Assign #{key}=#{mail[header.to_sym]}"
  401. item_object[key] = mail[header.to_sym]
  402. item_object.history_change_source_attribute(mail[:"#{header}-source"], key)
  403. end
  404. end
  405. =begin
  406. process unprocessable_mails (var/spool/unprocessable_mail/*.eml) again
  407. Channel::EmailParser.process_unprocessable_mails
  408. =end
  409. def self.process_unprocessable_mails(params = {})
  410. files = []
  411. Dir.glob("#{UNPROCESSABLE_MAIL_DIRECTORY}/*.eml") do |entry|
  412. ticket, _article, _user, _mail = Channel::EmailParser.new.process(params, File.binread(entry))
  413. next if ticket.blank?
  414. files.push entry
  415. File.delete(entry)
  416. end
  417. files
  418. end
  419. =begin
  420. process oversized emails by:
  421. 1. Archiving the oversized mail as tmp/oversized_mail/md5.eml
  422. 2. Reply with a postmaster message to inform the sender
  423. =end
  424. def process_oversized_mail(channel, msg)
  425. archive_mail(OVERSIZED_MAIL_DIRECTORY, msg)
  426. postmaster_response(channel, msg)
  427. end
  428. private
  429. # https://github.com/zammad/zammad/issues/2922
  430. def force_parts_encoding_if_needed(mail)
  431. # enforce encoding on both multipart parts and main body
  432. ([mail] + mail.parts).each { |elem| force_single_part_encoding_if_needed(elem) }
  433. end
  434. # https://github.com/zammad/zammad/issues/2922
  435. def force_single_part_encoding_if_needed(part)
  436. return if part.charset&.downcase != 'iso-2022-jp'
  437. part.body = force_japanese_encoding part.body.encoded.unpack1('M')
  438. end
  439. ISO2022JP_REGEXP = %r{=\?ISO-2022-JP\?B\?(.+?)\?=}
  440. # https://github.com/zammad/zammad/issues/3115
  441. def header_field_unpack_japanese(field)
  442. field.value.gsub ISO2022JP_REGEXP do
  443. force_japanese_encoding Base64.decode64($1)
  444. end
  445. end
  446. # generate Message ID on the fly if it was missing
  447. # yes, Mail gem generates one in some cases
  448. # but it is 100% random so duplicate messages would not be detected
  449. def message_ensure_message_id(raw, parsed)
  450. field = parsed.header.fields.find { |elem| elem.name == 'Message-ID' }
  451. return true if field&.unparsed_value.present?
  452. parsed.message_id = generate_message_id(raw, parsed.from)
  453. end
  454. def message_header_hash(mail)
  455. imported_fields = mail.header.fields.to_h do |f|
  456. begin
  457. value = if f.value.match?(ISO2022JP_REGEXP)
  458. value = header_field_unpack_japanese(f)
  459. else
  460. f.decoded.to_utf8
  461. end
  462. # fields that cannot be cleanly parsed fallback to the empty string
  463. rescue Mail::Field::IncompleteParseError
  464. value = ''
  465. rescue Encoding::CompatibilityError => e
  466. try_iso88591 = f.value.force_encoding('iso-8859-1').encode('utf-8')
  467. raise e if !try_iso88591.is_utf8?
  468. f.value = try_iso88591
  469. value = f.decoded.to_utf8
  470. rescue Date::Error => e
  471. raise e if !f.name.eql?('Resent-Date')
  472. f.value = ''
  473. rescue
  474. value = f.decoded.to_utf8(fallback: :read_as_sanitized_binary)
  475. end
  476. [f.name.downcase, value]
  477. end
  478. # imported_fields = mail.header.fields.map { |f| [f.name.downcase, f.to_utf8] }.to_h
  479. raw_fields = mail.header.fields.index_by { |f| "raw-#{f.name.downcase}" }
  480. custom_fields = {}.tap do |h|
  481. h.replace(imported_fields.slice(*RECIPIENT_FIELDS)
  482. .transform_values { |v| v.match?(EMAIL_REGEX) ? v : '' })
  483. h['x-any-recipient'] = h.values.select(&:present?).join(', ')
  484. h['message_id'] = imported_fields['message-id']
  485. h['subject'] = imported_fields['subject']
  486. h['date'] = begin
  487. Time.zone.parse(mail.date.to_s)
  488. rescue
  489. nil
  490. end
  491. end
  492. [imported_fields, raw_fields, custom_fields].reduce({}.with_indifferent_access, &:merge)
  493. end
  494. def message_body_hash(mail)
  495. if mail.html_part&.body.present?
  496. content_type = mail.html_part.mime_type || 'text/plain'
  497. body = body_text(mail.html_part, strict_html: true)
  498. elsif mail.text_part.present? && mail.all_parts.any? { |elem| elem.inline? && elem.content_type&.start_with?('image') }
  499. content_type = 'text/html'
  500. body = mail
  501. .all_parts
  502. .reduce('') do |memo, part|
  503. if part.mime_type == 'text/plain' && !part.attachment?
  504. memo += body_text(part, strict_html: false).text2html
  505. elsif part.inline? && part.content_type&.start_with?('image')
  506. memo += "<img src='cid:#{part.cid}'>"
  507. end
  508. memo
  509. end
  510. elsif mail.text_part.present?
  511. content_type = 'text/plain'
  512. body = mail
  513. .all_parts
  514. .reduce('') do |memo, part|
  515. if part.mime_type == 'text/plain' && !part.attachment?
  516. memo += body_text(part, strict_html: false)
  517. end
  518. memo
  519. end
  520. elsif mail&.body.present? && (mail.mime_type.nil? || mail.mime_type.match?(%r{^text/(plain|html)$}))
  521. content_type = mail.mime_type || 'text/plain'
  522. body = body_text(mail, strict_html: content_type.eql?('text/html'))
  523. end
  524. content_type = 'text/plain' if body.blank?
  525. {
  526. attachments: collect_attachments(mail),
  527. content_type: content_type || 'text/plain',
  528. body: body.presence || 'no visible content'
  529. }.with_indifferent_access
  530. end
  531. def body_text(message, **options)
  532. body_text = begin
  533. message.body.to_s
  534. rescue Mail::UnknownEncodingType # see test/data/mail/mail043.box / issue #348
  535. message.body.raw_source
  536. end
  537. body_text = body_text.utf8_encode(from: message.charset, fallback: :read_as_sanitized_binary)
  538. body_text = Mail::Utilities.to_lf(body_text)
  539. # plaintext body requires no processing
  540. return body_text if !options[:strict_html]
  541. # Issue #2390 - emails with >5k HTML links should be rejected
  542. return EXCESSIVE_LINKS_MSG if body_text.scan(%r{<a[[:space:]]}i).count >= 5_000
  543. body_text.html2html_strict
  544. end
  545. def collect_attachments(mail)
  546. attachments = []
  547. attachments.push(*get_nonplaintext_body_as_attachment(mail))
  548. mail.parts.each do |part|
  549. attachments.push(*gracefully_get_attachments(part, attachments, mail))
  550. end
  551. attachments
  552. end
  553. def get_nonplaintext_body_as_attachment(mail)
  554. if !(mail.html_part&.body.present? || (!mail.multipart? && mail.mime_type.present? && mail.mime_type != 'text/plain'))
  555. return
  556. end
  557. message = mail.html_part || mail
  558. if !mail.mime_type.starts_with?('text/') && mail.html_part.blank?
  559. return gracefully_get_attachments(message, [], mail)
  560. end
  561. filename = message.filename.presence || (message.mime_type.eql?('text/html') ? 'message.html' : '-no name-')
  562. headers_store = {
  563. 'content-alternative' => true,
  564. 'original-format' => message.mime_type.eql?('text/html'),
  565. 'Mime-Type' => message.mime_type,
  566. 'Charset' => message.charset,
  567. }.compact_blank
  568. [{
  569. data: body_text(message),
  570. filename: filename,
  571. preferences: headers_store
  572. }]
  573. end
  574. def gracefully_get_attachments(part, attachments, mail)
  575. get_attachments(part, attachments, mail).flatten.compact
  576. rescue => e # Protect process to work with spam emails (see test/fixtures/mail15.box)
  577. raise e if (fail_count ||= 0).positive?
  578. (fail_count += 1) && retry
  579. end
  580. def get_attachments(file, attachments, mail)
  581. return file.parts.map { |p| get_attachments(p, attachments, mail) } if file.parts.any?
  582. return [] if [mail.text_part&.body&.encoded, mail.html_part&.body&.encoded].include?(file.body.encoded)
  583. return [] if file.content_type&.start_with?('text/plain') && !file.attachment?
  584. # get file preferences
  585. headers_store = {}
  586. file.header.fields.each do |field|
  587. # full line, encode, ready for storage
  588. value = field.to_utf8
  589. if value.blank?
  590. value = field.raw_value
  591. end
  592. headers_store[field.name.to_s] = value
  593. rescue
  594. headers_store[field.name.to_s] = field.raw_value
  595. end
  596. # cleanup content id, <> will be added automatically later
  597. if headers_store['Content-ID'].blank? && headers_store['Content-Id'].present?
  598. headers_store['Content-ID'] = headers_store['Content-Id']
  599. end
  600. if headers_store['Content-ID']
  601. headers_store['Content-ID'].delete_prefix!('<')
  602. headers_store['Content-ID'].delete_suffix!('>')
  603. end
  604. # get filename from content-disposition
  605. # workaround for: NoMethodError: undefined method `filename' for #<Mail::UnstructuredField:0x007ff109e80678>
  606. begin
  607. filename = file.header[:content_disposition].try(:filename)
  608. rescue
  609. begin
  610. case file.header[:content_disposition].to_s
  611. when %r{(filename|name)(\*{0,1})="(.+?)"}i, %r{(filename|name)(\*{0,1})='(.+?)'}i, %r{(filename|name)(\*{0,1})=(.+?);}i
  612. filename = $3
  613. end
  614. rescue
  615. Rails.logger.debug { 'Unable to get filename' }
  616. end
  617. end
  618. begin
  619. case file.header[:content_disposition].to_s
  620. when %r{(filename|name)(\*{0,1})="(.+?)"}i, %r{(filename|name)(\*{0,1})='(.+?)'}i, %r{(filename|name)(\*{0,1})=(.+?);}i
  621. filename = $3
  622. end
  623. rescue
  624. Rails.logger.debug { 'Unable to get filename' }
  625. end
  626. # as fallback, use raw values
  627. if filename.blank?
  628. case headers_store['Content-Disposition'].to_s
  629. when %r{(filename|name)(\*{0,1})="(.+?)"}i, %r{(filename|name)(\*{0,1})='(.+?)'}i, %r{(filename|name)(\*{0,1})=(.+?);}i
  630. filename = $3
  631. end
  632. end
  633. # for some broken sm mail clients (X-MimeOLE: Produced By Microsoft Exchange V6.5)
  634. filename ||= file.header[:content_location].to_s.dup.force_encoding('utf-8')
  635. file_body = String.new(file.body.to_s)
  636. # generate file name based on content type
  637. if filename.blank? && headers_store['Content-Type'].present? && headers_store['Content-Type'].match?(%r{^message/rfc822}i)
  638. begin
  639. parser = Channel::EmailParser.new
  640. mail_local = parser.parse(file_body)
  641. filename = if mail_local[:subject].present?
  642. "#{mail_local[:subject]}.eml"
  643. elsif headers_store['Content-Description'].present?
  644. "#{headers_store['Content-Description']}.eml".to_s.force_encoding('utf-8')
  645. else
  646. 'Mail.eml'
  647. end
  648. rescue
  649. filename = 'Mail.eml'
  650. end
  651. end
  652. # e. g. Content-Type: video/quicktime; name="Video.MOV";
  653. if filename.blank?
  654. ['(filename|name)(\*{0,1})="(.+?)"(;|$)', '(filename|name)(\*{0,1})=\'(.+?)\'(;|$)', '(filename|name)(\*{0,1})=(.+?)(;|$)'].each do |regexp|
  655. if headers_store['Content-Type'] =~ %r{#{regexp}}i
  656. filename = $3
  657. break
  658. end
  659. end
  660. end
  661. # workaround for mail gem - decode filenames
  662. # https://github.com/zammad/zammad/issues/928
  663. if filename.present?
  664. filename = Mail::Encodings.value_decode(filename)
  665. end
  666. if !filename.force_encoding('UTF-8').valid_encoding?
  667. filename = filename.utf8_encode(fallback: :read_as_sanitized_binary)
  668. end
  669. # generate file name based on content-id with file extention
  670. if filename.blank? && headers_store['Content-ID'].present? && headers_store['Content-ID'] =~ %r{(.+?\..{2,6})@.+?}i
  671. filename = $1
  672. end
  673. # e. g. Content-Type: video/quicktime
  674. if filename.blank? && (content_type = headers_store['Content-Type'])
  675. map = {
  676. 'message/delivery-status': %w[txt delivery-status],
  677. 'text/plain': %w[txt document],
  678. 'text/html': %w[html document],
  679. 'video/quicktime': %w[mov video],
  680. 'image/jpeg': %w[jpg image],
  681. 'image/jpg': %w[jpg image],
  682. 'image/png': %w[png image],
  683. 'image/gif': %w[gif image],
  684. }
  685. map.each do |type, ext|
  686. next if !content_type.match?(%r{^#{Regexp.quote(type)}}i)
  687. filename = if headers_store['Content-Description'].present?
  688. "#{headers_store['Content-Description']}.#{ext[0]}".to_s.force_encoding('utf-8')
  689. else
  690. "#{ext[1]}.#{ext[0]}"
  691. end
  692. break
  693. end
  694. end
  695. # generate file name based on content-id without file extention
  696. if filename.blank? && headers_store['Content-ID'].present? && headers_store['Content-ID'] =~ %r{(.+?)@.+?}i
  697. filename = $1
  698. end
  699. # set fallback filename
  700. if filename.blank?
  701. filename = 'file'
  702. end
  703. # create uniq filename
  704. local_filename = ''
  705. local_extention = ''
  706. if filename =~ %r{^(.*?)\.(.+?)$}
  707. local_filename = $1
  708. local_extention = $2
  709. end
  710. 1.upto(1000) do |i|
  711. filename_exists = false
  712. attachments.each do |attachment|
  713. if attachment[:filename] == filename
  714. filename_exists = true
  715. end
  716. end
  717. break if filename_exists == false
  718. filename = if local_extention.present?
  719. "#{local_filename}#{i}.#{local_extention}"
  720. else
  721. "#{local_filename}#{i}"
  722. end
  723. end
  724. # get mime type
  725. if file.header[:content_type]&.string
  726. headers_store['Mime-Type'] = file.header[:content_type].string
  727. end
  728. # get charset
  729. if file.header&.charset
  730. headers_store['Charset'] = file.header.charset
  731. end
  732. # remove not needed header
  733. headers_store.delete('Content-Transfer-Encoding')
  734. headers_store.delete('Content-Disposition')
  735. attach = {
  736. data: file_body,
  737. filename: filename,
  738. preferences: headers_store,
  739. }
  740. [attach]
  741. end
  742. # Archive the given message as tmp/folder/md5.eml
  743. def archive_mail(path, msg)
  744. FileUtils.mkpath path
  745. path.join("#{Digest::MD5.hexdigest(msg)}.eml").tap do |file_path|
  746. File.binwrite(file_path, msg)
  747. end
  748. end
  749. # Auto reply as the postmaster to oversized emails with:
  750. # [undeliverable] Message too large
  751. def postmaster_response(channel, msg)
  752. begin
  753. reply_mail = compose_postmaster_reply(msg)
  754. rescue NotificationFactory::FileNotFoundError => e
  755. Rails.logger.error "No valid postmaster email_oversized template found. Skipping postmaster reply. #{e.inspect}"
  756. return
  757. end
  758. Rails.logger.info "Send mail too large postmaster message to: #{reply_mail[:to]}"
  759. reply_mail[:from] = EmailAddress.find_by(channel: channel).email
  760. channel.deliver(reply_mail)
  761. rescue => e
  762. Rails.logger.error "Error during sending of postmaster oversized email auto-reply: #{e.inspect}\n#{e.backtrace}"
  763. end
  764. # Compose a "Message too large" reply to the given message
  765. def compose_postmaster_reply(raw_incoming_mail, locale = nil)
  766. parsed_incoming_mail = Channel::EmailParser.new.parse(raw_incoming_mail)
  767. # construct a dummy mail object
  768. mail = MESSAGE_STRUCT.new
  769. mail.from_display_name = parsed_incoming_mail[:from_display_name]
  770. mail.subject = parsed_incoming_mail[:subject]
  771. mail.msg_size = format('%<MB>.2f', MB: raw_incoming_mail.size.to_f / 1024 / 1024)
  772. reply = NotificationFactory::Mailer.template(
  773. template: 'email_oversized',
  774. locale: locale,
  775. format: 'txt',
  776. objects: {
  777. mail: mail,
  778. },
  779. raw: true, # will not add application template
  780. standalone: true, # default: false - will send header & footer
  781. )
  782. reply.merge(
  783. to: parsed_incoming_mail[:from_email],
  784. body: reply[:body].gsub(%r{\n}, "\r\n"),
  785. content_type: 'text/plain',
  786. References: parsed_incoming_mail[:message_id],
  787. 'In-Reply-To': parsed_incoming_mail[:message_id],
  788. )
  789. end
  790. def guess_email_fqdn(from)
  791. Mail::Address.new(from).domain.strip
  792. rescue
  793. nil
  794. end
  795. def generate_message_id(raw_message, from)
  796. fqdn = guess_email_fqdn(from) || 'zammad_generated'
  797. "<gen-#{Digest::MD5.hexdigest(raw_message)}@#{fqdn}>"
  798. end
  799. # https://github.com/zammad/zammad/issues/3096
  800. # specific email needs to be forced to ISO-2022-JP
  801. # but that breaks other emails that can be forced to SJIS only
  802. # thus force to ISO-2022-JP but fallback to SJIS
  803. #
  804. # https://github.com/zammad/zammad/issues/3368
  805. # some characters are not included in the official ISO-2022-JP
  806. # ISO-2022-JP-KDDI superset provides support for more characters
  807. def force_japanese_encoding(input)
  808. %w[ISO-2022-JP ISO-2022-JP-KDDI SJIS]
  809. .lazy
  810. .map { |encoding| try_encoding(input, encoding) }
  811. .detect(&:present?)
  812. end
  813. def try_encoding(input, encoding)
  814. input.force_encoding(encoding).encode('UTF-8')
  815. rescue
  816. nil
  817. end
  818. end
  819. module Mail
  820. # workaround to get content of no parseable headers - in most cases with non 7 bit ascii signs
  821. class Field
  822. def raw_value
  823. begin
  824. value = @raw_value.try(:utf8_encode)
  825. rescue
  826. value = @raw_value.utf8_encode(fallback: :read_as_sanitized_binary)
  827. end
  828. return value if value.blank?
  829. value.sub(%r{^.+?:(\s|)}, '')
  830. end
  831. end
  832. # issue#348 - IMAP mail fetching stops because of broken spam email (e. g. broken Content-Transfer-Encoding value see test/fixtures/mail43.box)
  833. # https://github.com/zammad/zammad/issues/348
  834. class Body
  835. def decoded
  836. if Encodings.defined?(encoding)
  837. Encodings.get_encoding(encoding).decode(raw_source)
  838. else
  839. Rails.logger.info "UnknownEncodingType: Don't know how to decode #{encoding}!"
  840. raw_source
  841. end
  842. end
  843. end
  844. end