email_parser.rb 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968
  1. # Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/
  2. # encoding: utf-8
  3. class Channel::EmailParser
  4. PROZESS_TIME_MAX = 180
  5. EMAIL_REGEX = /.+@.+/.freeze
  6. RECIPIENT_FIELDS = %w[to cc delivered-to x-original-to envelope-to].freeze
  7. SENDER_FIELDS = %w[from reply-to return-path sender].freeze
  8. EXCESSIVE_LINKS_MSG = 'This message cannot be displayed because it contains over 5,000 links. Download the raw message below and open it via an Email client if you still wish to view it.'.freeze
  9. =begin
  10. parser = Channel::EmailParser.new
  11. mail = parser.parse(msg_as_string)
  12. mail = {
  13. from: 'Some Name <some@example.com>',
  14. from_email: 'some@example.com',
  15. from_local: 'some',
  16. from_domain: 'example.com',
  17. from_display_name: 'Some Name',
  18. message_id: 'some_message_id@example.com',
  19. to: 'Some System <system@example.com>',
  20. cc: 'Somebody <somebody@example.com>',
  21. subject: 'some message subject',
  22. body: 'some message body',
  23. content_type: 'text/html', # text/plain
  24. date: Time.zone.now,
  25. attachments: [
  26. {
  27. data: 'binary of attachment',
  28. filename: 'file_name_of_attachment.txt',
  29. preferences: {
  30. 'content-alternative' => true,
  31. 'Mime-Type' => 'text/plain',
  32. 'Charset: => 'iso-8859-1',
  33. },
  34. },
  35. ],
  36. # ignore email header
  37. x-zammad-ignore: 'false',
  38. # customer headers
  39. x-zammad-customer-login: '',
  40. x-zammad-customer-email: '',
  41. x-zammad-customer-firstname: '',
  42. x-zammad-customer-lastname: '',
  43. # ticket headers (for new tickets)
  44. x-zammad-ticket-group: 'some_group',
  45. x-zammad-ticket-state: 'some_state',
  46. x-zammad-ticket-priority: 'some_priority',
  47. x-zammad-ticket-owner: 'some_owner_login',
  48. # ticket headers (for existing tickets)
  49. x-zammad-ticket-followup-group: 'some_group',
  50. x-zammad-ticket-followup-state: 'some_state',
  51. x-zammad-ticket-followup-priority: 'some_priority',
  52. x-zammad-ticket-followup-owner: 'some_owner_login',
  53. # article headers
  54. x-zammad-article-internal: false,
  55. x-zammad-article-type: 'agent',
  56. x-zammad-article-sender: 'customer',
  57. # all other email headers
  58. some-header: 'some_value',
  59. }
  60. =end
  61. def parse(msg)
  62. msg = msg.force_encoding('binary')
  63. # mail 2.6 and earlier accepted non-conforming mails that lacked the correct CRLF seperators,
  64. # mail 2.7 and above require CRLF so we force it on using binary_unsafe_to_crlf
  65. msg = Mail::Utilities.binary_unsafe_to_crlf(msg)
  66. mail = Mail.new(msg)
  67. message_ensure_message_id(msg, mail)
  68. force_parts_encoding_if_needed(mail)
  69. headers = message_header_hash(mail)
  70. body = message_body_hash(mail)
  71. message_attributes = [
  72. { mail_instance: mail },
  73. headers,
  74. body,
  75. self.class.sender_attributes(headers),
  76. { raw: msg },
  77. ]
  78. message_attributes.reduce({}.with_indifferent_access, &:merge)
  79. end
  80. =begin
  81. parser = Channel::EmailParser.new
  82. ticket, article, user, mail = parser.process(channel, email_raw_string)
  83. returns
  84. [ticket, article, user, mail]
  85. do not raise an exception - e. g. if used by scheduler
  86. parser = Channel::EmailParser.new
  87. ticket, article, user, mail = parser.process(channel, email_raw_string, false)
  88. returns
  89. [ticket, article, user, mail] || false
  90. =end
  91. def process(channel, msg, exception = true)
  92. Timeout.timeout(PROZESS_TIME_MAX) do
  93. _process(channel, msg)
  94. end
  95. rescue => e
  96. # store unprocessable email for bug reporting
  97. filename = archive_mail('unprocessable_mail', msg)
  98. message = "Can't process email, you will find it for bug reporting under #{filename}, please create an issue at https://github.com/zammad/zammad/issues"
  99. p "ERROR: #{message}" # rubocop:disable Rails/Output
  100. p "ERROR: #{e.inspect}" # rubocop:disable Rails/Output
  101. Rails.logger.error message
  102. Rails.logger.error e
  103. return false if exception == false
  104. raise %(#{e.inspect}\n#{e.backtrace.join("\n")})
  105. end
  106. def _process(channel, msg)
  107. # parse email
  108. mail = parse(msg)
  109. Rails.logger.info "Process email with msgid '#{mail[:message_id]}'"
  110. # run postmaster pre filter
  111. UserInfo.current_user_id = 1
  112. # set interface handle
  113. original_interface_handle = ApplicationHandleInfo.current
  114. transaction_params = { interface_handle: "#{original_interface_handle}.postmaster", disable: [] }
  115. filters = {}
  116. Setting.where(area: 'Postmaster::PreFilter').order(:name).each do |setting|
  117. filters[setting.name] = Setting.get(setting.name).constantize
  118. end
  119. filters.each do |key, backend|
  120. Rails.logger.debug { "run postmaster pre filter #{key}: #{backend}" }
  121. begin
  122. backend.run(channel, mail, transaction_params)
  123. rescue => e
  124. Rails.logger.error "can't run postmaster pre filter #{key}: #{backend}"
  125. Rails.logger.error e.inspect
  126. raise e
  127. end
  128. end
  129. # check ignore header
  130. if mail[:'x-zammad-ignore'] == 'true' || mail[:'x-zammad-ignore'] == true
  131. Rails.logger.info "ignored email with msgid '#{mail[:message_id]}' from '#{mail[:from]}' because of x-zammad-ignore header"
  132. return
  133. end
  134. ticket = nil
  135. article = nil
  136. session_user = nil
  137. # use transaction
  138. Transaction.execute(transaction_params) do
  139. # get sender user
  140. session_user_id = mail[:'x-zammad-session-user-id']
  141. if !session_user_id
  142. raise 'No x-zammad-session-user-id, no sender set!'
  143. end
  144. session_user = User.lookup(id: session_user_id)
  145. if !session_user
  146. raise "No user found for x-zammad-session-user-id: #{session_user_id}!"
  147. end
  148. # set current user
  149. UserInfo.current_user_id = session_user.id
  150. # get ticket# based on email headers
  151. if mail[:'x-zammad-ticket-id']
  152. ticket = Ticket.find_by(id: mail[:'x-zammad-ticket-id'])
  153. end
  154. if mail[:'x-zammad-ticket-number']
  155. ticket = Ticket.find_by(number: mail[:'x-zammad-ticket-number'])
  156. end
  157. # set ticket state to open if not new
  158. if ticket
  159. set_attributes_by_x_headers(ticket, 'ticket', mail, 'followup')
  160. # save changes set by x-zammad-ticket-followup-* headers
  161. ticket.save! if ticket.has_changes_to_save?
  162. # set ticket to open again or keep create state
  163. if !mail[:'x-zammad-ticket-followup-state'] && !mail[:'x-zammad-ticket-followup-state_id']
  164. new_state = Ticket::State.find_by(default_create: true)
  165. if ticket.state_id != new_state.id && !mail[:'x-zammad-out-of-office']
  166. ticket.state = Ticket::State.find_by(default_follow_up: true)
  167. ticket.save!
  168. end
  169. end
  170. end
  171. # create new ticket
  172. if !ticket
  173. preferences = {}
  174. if channel[:id]
  175. preferences = {
  176. channel_id: channel[:id]
  177. }
  178. end
  179. # get default group where ticket is created
  180. group = nil
  181. if channel[:group_id]
  182. group = Group.lookup(id: channel[:group_id])
  183. else
  184. mail_to_group = self.class.mail_to_group(mail[:to])
  185. if mail_to_group.present?
  186. group = mail_to_group
  187. end
  188. end
  189. if group.blank? || group.active == false
  190. group = Group.where(active: true).order(id: :asc).first
  191. end
  192. if group.blank?
  193. group = Group.first
  194. end
  195. title = mail[:subject]
  196. if title.blank?
  197. title = '-'
  198. end
  199. ticket = Ticket.new(
  200. group_id: group.id,
  201. title: title,
  202. preferences: preferences,
  203. )
  204. set_attributes_by_x_headers(ticket, 'ticket', mail)
  205. # create ticket
  206. ticket.save!
  207. end
  208. # apply tags to ticket
  209. if mail[:'x-zammad-ticket-tags'].present?
  210. mail[:'x-zammad-ticket-tags'].each do |tag|
  211. ticket.tag_add(tag)
  212. end
  213. end
  214. # set attributes
  215. ticket.with_lock do
  216. article = Ticket::Article.new(
  217. ticket_id: ticket.id,
  218. type_id: Ticket::Article::Type.find_by(name: 'email').id,
  219. sender_id: Ticket::Article::Sender.find_by(name: 'Customer').id,
  220. content_type: mail[:content_type],
  221. body: mail[:body],
  222. from: mail[:from],
  223. reply_to: mail[:"reply-to"],
  224. to: mail[:to],
  225. cc: mail[:cc],
  226. subject: mail[:subject],
  227. message_id: mail[:message_id],
  228. internal: false,
  229. )
  230. # x-headers lookup
  231. set_attributes_by_x_headers(article, 'article', mail)
  232. # create article
  233. article.save!
  234. # store mail plain
  235. article.save_as_raw(msg)
  236. # store attachments
  237. mail[:attachments]&.each do |attachment|
  238. filename = attachment[:filename].force_encoding('utf-8')
  239. if !filename.force_encoding('UTF-8').valid_encoding?
  240. filename = filename.utf8_encode(fallback: :read_as_sanitized_binary)
  241. end
  242. Store.add(
  243. object: 'Ticket::Article',
  244. o_id: article.id,
  245. data: attachment[:data],
  246. filename: filename,
  247. preferences: attachment[:preferences]
  248. )
  249. end
  250. end
  251. end
  252. ticket.reload
  253. article.reload
  254. session_user.reload
  255. # run postmaster post filter
  256. filters = {}
  257. Setting.where(area: 'Postmaster::PostFilter').order(:name).each do |setting|
  258. filters[setting.name] = Setting.get(setting.name).constantize
  259. end
  260. filters.each_value do |backend|
  261. Rails.logger.debug { "run postmaster post filter #{backend}" }
  262. begin
  263. backend.run(channel, mail, ticket, article, session_user)
  264. rescue => e
  265. Rails.logger.error "can't run postmaster post filter #{backend}"
  266. Rails.logger.error e.inspect
  267. end
  268. end
  269. # return new objects
  270. [ticket, article, session_user, mail]
  271. end
  272. def self.mail_to_group(to)
  273. begin
  274. to = Mail::AddressList.new(to)&.addresses&.first&.address
  275. rescue
  276. Rails.logger.error 'Can not parse :to field for group destination!'
  277. end
  278. return if to.blank?
  279. email = EmailAddress.find_by(email: to.downcase)
  280. return if email&.channel.blank?
  281. email.channel&.group
  282. end
  283. def self.check_attributes_by_x_headers(header_name, value)
  284. class_name = nil
  285. attribute = nil
  286. # skip check attributes if it is tags
  287. return true if header_name == 'x-zammad-ticket-tags'
  288. if header_name =~ /^x-zammad-(.+?)-(followup-|)(.*)$/i
  289. class_name = $1
  290. attribute = $3
  291. end
  292. return true if !class_name
  293. if class_name.casecmp('article').zero?
  294. class_name = 'Ticket::Article'
  295. end
  296. return true if !attribute
  297. key_short = attribute[ attribute.length - 3, attribute.length ]
  298. return true if key_short != '_id'
  299. class_object = class_name.to_classname.constantize
  300. return if !class_object
  301. class_instance = class_object.new
  302. return false if !class_instance.association_id_validation(attribute, value)
  303. true
  304. end
  305. def self.sender_attributes(from)
  306. if from.is_a?(HashWithIndifferentAccess)
  307. from = SENDER_FIELDS.map { |f| from[f] }.compact
  308. .map(&:to_utf8).reject(&:blank?)
  309. .partition { |address| address.match?(EMAIL_REGEX) }
  310. .flatten.first
  311. end
  312. data = {}.with_indifferent_access
  313. return data if from.blank?
  314. from = from.gsub('<>', '').strip
  315. mail_address = begin
  316. Mail::AddressList.new(from).addresses
  317. .select { |a| a.address.present? }
  318. .partition { |a| a.address.match?(EMAIL_REGEX) }
  319. .flatten.first
  320. rescue Mail::Field::ParseError => e
  321. $stdout.puts e
  322. end
  323. if mail_address&.address.present?
  324. data[:from_email] = mail_address.address
  325. data[:from_local] = mail_address.local
  326. data[:from_domain] = mail_address.domain
  327. data[:from_display_name] = mail_address.display_name || mail_address.comments&.first
  328. elsif from =~ /^(.+?)<((.+?)@(.+?))>/
  329. data[:from_email] = $2
  330. data[:from_local] = $3
  331. data[:from_domain] = $4
  332. data[:from_display_name] = $1
  333. else
  334. data[:from_email] = from
  335. data[:from_local] = from
  336. data[:from_domain] = from
  337. data[:from_display_name] = from
  338. end
  339. # do extra decoding because we needed to use field.value
  340. data[:from_display_name] =
  341. Mail::Field.new('X-From', data[:from_display_name].to_utf8)
  342. .to_s
  343. .delete('"')
  344. .strip
  345. .gsub(/(^'|'$)/, '')
  346. data
  347. end
  348. def set_attributes_by_x_headers(item_object, header_name, mail, suffix = false)
  349. # loop all x-zammad-header-* headers
  350. item_object.attributes.each_key do |key|
  351. # ignore read only attributes
  352. next if key == 'updated_by_id'
  353. next if key == 'created_by_id'
  354. # check if id exists
  355. key_short = key[ key.length - 3, key.length ]
  356. if key_short == '_id'
  357. key_short = key[ 0, key.length - 3 ]
  358. header = "x-zammad-#{header_name}-#{key_short}"
  359. if suffix
  360. header = "x-zammad-#{header_name}-#{suffix}-#{key_short}"
  361. end
  362. # only set value on _id if value/reference lookup exists
  363. if mail[header.to_sym]
  364. Rails.logger.info "set_attributes_by_x_headers header #{header} found #{mail[header.to_sym]}"
  365. item_object.class.reflect_on_all_associations.map do |assoc|
  366. next if assoc.name.to_s != key_short
  367. Rails.logger.info "set_attributes_by_x_headers found #{assoc.class_name} lookup for '#{mail[header.to_sym]}'"
  368. item = assoc.class_name.constantize
  369. assoc_object = nil
  370. if item.new.respond_to?(:name)
  371. assoc_object = item.lookup(name: mail[header.to_sym])
  372. end
  373. if !assoc_object && item.new.respond_to?(:login)
  374. assoc_object = item.lookup(login: mail[header.to_sym])
  375. end
  376. if !assoc_object && item.new.respond_to?(:email)
  377. assoc_object = item.lookup(email: mail[header.to_sym])
  378. end
  379. if assoc_object.blank?
  380. # no assoc exists, remove header
  381. mail.delete(header.to_sym)
  382. next
  383. end
  384. Rails.logger.info "set_attributes_by_x_headers assign #{item_object.class} #{key}=#{assoc_object.id}"
  385. item_object[key] = assoc_object.id
  386. end
  387. end
  388. end
  389. # check if attribute exists
  390. header = "x-zammad-#{header_name}-#{key}"
  391. if suffix
  392. header = "x-zammad-#{header_name}-#{suffix}-#{key}"
  393. end
  394. if mail[header.to_sym]
  395. Rails.logger.info "set_attributes_by_x_headers header #{header} found. Assign #{key}=#{mail[header.to_sym]}"
  396. item_object[key] = mail[header.to_sym]
  397. end
  398. end
  399. end
  400. =begin
  401. process unprocessable_mails (tmp/unprocessable_mail/*.eml) again
  402. Channel::EmailParser.process_unprocessable_mails
  403. =end
  404. def self.process_unprocessable_mails(params = {})
  405. path = Rails.root.join('tmp/unprocessable_mail')
  406. files = []
  407. Dir.glob("#{path}/*.eml") do |entry|
  408. ticket, _article, _user, _mail = Channel::EmailParser.new.process(params, IO.binread(entry))
  409. next if ticket.blank?
  410. files.push entry
  411. File.delete(entry)
  412. end
  413. files
  414. end
  415. =begin
  416. process oversized emails by:
  417. 1. Archiving the oversized mail as tmp/oversized_mail/md5.eml
  418. 2. Reply with a postmaster message to inform the sender
  419. =end
  420. def process_oversized_mail(channel, msg)
  421. archive_mail('oversized_mail', msg)
  422. postmaster_response(channel, msg)
  423. end
  424. private
  425. # https://github.com/zammad/zammad/issues/2922
  426. def force_parts_encoding_if_needed(mail)
  427. mail.parts.each { |elem| force_single_part_encoding_if_needed(elem) }
  428. end
  429. # https://github.com/zammad/zammad/issues/2922
  430. def force_single_part_encoding_if_needed(part)
  431. return if part.charset != 'iso-2022-jp'
  432. part.body = part.body.encoded.unpack1('M').force_encoding('ISO-2022-JP').encode('UTF-8')
  433. end
  434. ISO2022JP_REGEXP = /=\?ISO-2022-JP\?B\?(.+?)\?=/.freeze
  435. # https://github.com/zammad/zammad/issues/3115
  436. def header_field_unpack_japanese(field)
  437. field.value.gsub ISO2022JP_REGEXP do
  438. Base64.decode64($1).force_encoding('SJIS').encode('UTF-8')
  439. end
  440. end
  441. # generate Message ID on the fly if it was missing
  442. # yes, Mail gem generates one in some cases
  443. # but it is 100% random so duplicate messages would not be detected
  444. def message_ensure_message_id(raw, parsed)
  445. field = parsed.header.fields.find { |elem| elem.name == 'Message-ID' }
  446. return true if field&.unparsed_value.present?
  447. parsed.message_id = generate_message_id(raw, parsed.from)
  448. end
  449. def message_header_hash(mail)
  450. imported_fields = mail.header.fields.map do |f|
  451. begin
  452. value = if f.value.match?(ISO2022JP_REGEXP)
  453. header_field_unpack_japanese(f)
  454. else
  455. f.to_utf8
  456. end
  457. if value.blank?
  458. value = f.decoded.to_utf8
  459. end
  460. # fields that cannot be cleanly parsed fallback to the empty string
  461. rescue Mail::Field::IncompleteParseError
  462. value = ''
  463. rescue
  464. value = f.decoded.to_utf8(fallback: :read_as_sanitized_binary)
  465. end
  466. [f.name.downcase, value]
  467. end.to_h
  468. # imported_fields = mail.header.fields.map { |f| [f.name.downcase, f.to_utf8] }.to_h
  469. raw_fields = mail.header.fields.index_by { |f| "raw-#{f.name.downcase}" }
  470. custom_fields = {}.tap do |h|
  471. h.replace(imported_fields.slice(*RECIPIENT_FIELDS)
  472. .transform_values { |v| v.match?(EMAIL_REGEX) ? v : '' })
  473. h['x-any-recipient'] = h.values.select(&:present?).join(', ')
  474. h['message_id'] = imported_fields['message-id']
  475. h['subject'] = imported_fields['subject']
  476. begin
  477. h['date'] = Time.zone.parse(mail.date.to_s) || imported_fields['date']
  478. rescue
  479. h['date'] = nil
  480. end
  481. end
  482. [imported_fields, raw_fields, custom_fields].reduce({}.with_indifferent_access, &:merge)
  483. end
  484. def message_body_hash(mail)
  485. message = [mail.html_part, mail.text_part, mail].find { |m| m&.body.present? }
  486. if message.present? && (message.mime_type.nil? || message.mime_type.match?(%r{^text/(plain|html)$}))
  487. content_type = message.mime_type || 'text/plain'
  488. body = body_text(message, strict_html: content_type.eql?('text/html'))
  489. end
  490. content_type = 'text/plain' if body.blank?
  491. {
  492. attachments: collect_attachments(mail),
  493. content_type: content_type || 'text/plain',
  494. body: body.presence || 'no visible content'
  495. }.with_indifferent_access
  496. end
  497. def body_text(message, **options)
  498. body_text = begin
  499. message.body.to_s
  500. rescue Mail::UnknownEncodingType # see test/data/mail/mail043.box / issue #348
  501. message.body.raw_source
  502. end
  503. body_text = body_text.utf8_encode(from: message.charset, fallback: :read_as_sanitized_binary)
  504. body_text = Mail::Utilities.to_lf(body_text)
  505. # plaintext body requires no processing
  506. return body_text if !options[:strict_html]
  507. # Issue #2390 - emails with >5k HTML links should be rejected
  508. return EXCESSIVE_LINKS_MSG if body_text.scan(/<a[[:space:]]/i).count >= 5_000
  509. body_text.html2html_strict
  510. end
  511. def collect_attachments(mail)
  512. attachments = []
  513. attachments.push(*get_nonplaintext_body_as_attachment(mail))
  514. mail.parts.each do |part|
  515. attachments.push(*gracefully_get_attachments(part, attachments, mail))
  516. end
  517. attachments
  518. end
  519. def get_nonplaintext_body_as_attachment(mail)
  520. if !(mail.html_part&.body.present? || (!mail.multipart? && mail.mime_type.present? && mail.mime_type != 'text/plain'))
  521. return
  522. end
  523. message = mail.html_part || mail
  524. if !mail.mime_type.starts_with?('text/') && mail.html_part.blank?
  525. return gracefully_get_attachments(message, [], mail)
  526. end
  527. filename = message.filename.presence || (message.mime_type.eql?('text/html') ? 'message.html' : '-no name-')
  528. headers_store = {
  529. 'content-alternative' => true,
  530. 'original-format' => message.mime_type.eql?('text/html'),
  531. 'Mime-Type' => message.mime_type,
  532. 'Charset' => message.charset,
  533. }.reject { |_, v| v.blank? }
  534. [{
  535. data: body_text(message),
  536. filename: filename,
  537. preferences: headers_store
  538. }]
  539. end
  540. def gracefully_get_attachments(part, attachments, mail)
  541. get_attachments(part, attachments, mail).flatten.compact
  542. rescue => e # Protect process to work with spam emails (see test/fixtures/mail15.box)
  543. raise e if (fail_count ||= 0).positive?
  544. (fail_count += 1) && retry
  545. end
  546. def get_attachments(file, attachments, mail)
  547. return file.parts.map { |p| get_attachments(p, attachments, mail) } if file.parts.any?
  548. return [] if [mail.text_part&.body&.encoded, mail.html_part&.body&.encoded].include?(file.body.encoded)
  549. # get file preferences
  550. headers_store = {}
  551. file.header.fields.each do |field|
  552. # full line, encode, ready for storage
  553. value = field.to_utf8
  554. if value.blank?
  555. value = field.raw_value
  556. end
  557. headers_store[field.name.to_s] = value
  558. rescue
  559. headers_store[field.name.to_s] = field.raw_value
  560. end
  561. # cleanup content id, <> will be added automatically later
  562. if headers_store['Content-ID']
  563. headers_store['Content-ID'].delete_prefix!('<')
  564. headers_store['Content-ID'].delete_suffix!('>')
  565. end
  566. # get filename from content-disposition
  567. # workaround for: NoMethodError: undefined method `filename' for #<Mail::UnstructuredField:0x007ff109e80678>
  568. begin
  569. filename = file.header[:content_disposition].try(:filename)
  570. rescue
  571. begin
  572. case file.header[:content_disposition].to_s
  573. when /(filename|name)(\*{0,1})="(.+?)"/i, /(filename|name)(\*{0,1})='(.+?)'/i, /(filename|name)(\*{0,1})=(.+?);/i
  574. filename = $3
  575. end
  576. rescue
  577. Rails.logger.debug { 'Unable to get filename' }
  578. end
  579. end
  580. begin
  581. case file.header[:content_disposition].to_s
  582. when /(filename|name)(\*{0,1})="(.+?)"/i, /(filename|name)(\*{0,1})='(.+?)'/i, /(filename|name)(\*{0,1})=(.+?);/i
  583. filename = $3
  584. end
  585. rescue
  586. Rails.logger.debug { 'Unable to get filename' }
  587. end
  588. # as fallback, use raw values
  589. if filename.blank?
  590. case headers_store['Content-Disposition'].to_s
  591. when /(filename|name)(\*{0,1})="(.+?)"/i, /(filename|name)(\*{0,1})='(.+?)'/i, /(filename|name)(\*{0,1})=(.+?);/i
  592. filename = $3
  593. end
  594. end
  595. # for some broken sm mail clients (X-MimeOLE: Produced By Microsoft Exchange V6.5)
  596. filename ||= file.header[:content_location].to_s.force_encoding('utf-8')
  597. # generate file name based on content-id
  598. if filename.blank? && headers_store['Content-ID'].present? && headers_store['Content-ID'] =~ /(.+?)@.+?/i
  599. filename = $1
  600. end
  601. file_body = String.new(file.body.to_s)
  602. # generate file name based on content type
  603. if filename.blank? && headers_store['Content-Type'].present? && headers_store['Content-Type'].match?(%r{^message/rfc822}i)
  604. begin
  605. parser = Channel::EmailParser.new
  606. mail_local = parser.parse(file_body)
  607. filename = if mail_local[:subject].present?
  608. "#{mail_local[:subject]}.eml"
  609. elsif headers_store['Content-Description'].present?
  610. "#{headers_store['Content-Description']}.eml".to_s.force_encoding('utf-8')
  611. else
  612. 'Mail.eml'
  613. end
  614. rescue
  615. filename = 'Mail.eml'
  616. end
  617. end
  618. # e. g. Content-Type: video/quicktime; name="Video.MOV";
  619. if filename.blank?
  620. ['(filename|name)(\*{0,1})="(.+?)"(;|$)', '(filename|name)(\*{0,1})=\'(.+?)\'(;|$)', '(filename|name)(\*{0,1})=(.+?)(;|$)'].each do |regexp|
  621. if headers_store['Content-Type'] =~ /#{regexp}/i
  622. filename = $3
  623. break
  624. end
  625. end
  626. end
  627. # workaround for mail gem - decode filenames
  628. # https://github.com/zammad/zammad/issues/928
  629. if filename.present?
  630. filename = Mail::Encodings.value_decode(filename)
  631. end
  632. if !filename.force_encoding('UTF-8').valid_encoding?
  633. filename = filename.utf8_encode(fallback: :read_as_sanitized_binary)
  634. end
  635. # e. g. Content-Type: video/quicktime
  636. if filename.blank? && (content_type = headers_store['Content-Type'])
  637. map = {
  638. 'message/delivery-status': %w[txt delivery-status],
  639. 'text/plain': %w[txt document],
  640. 'text/html': %w[html document],
  641. 'video/quicktime': %w[mov video],
  642. 'image/jpeg': %w[jpg image],
  643. 'image/jpg': %w[jpg image],
  644. 'image/png': %w[png image],
  645. 'image/gif': %w[gif image],
  646. }
  647. map.each do |type, ext|
  648. next if !content_type.match?(/^#{Regexp.quote(type)}/i)
  649. filename = if headers_store['Content-Description'].present?
  650. "#{headers_store['Content-Description']}.#{ext[0]}".to_s.force_encoding('utf-8')
  651. else
  652. "#{ext[1]}.#{ext[0]}"
  653. end
  654. break
  655. end
  656. end
  657. # set fallback filename
  658. if filename.blank?
  659. filename = 'file'
  660. end
  661. # create uniq filename
  662. local_filename = ''
  663. local_extention = ''
  664. if filename =~ /^(.*?)\.(.+?)$/
  665. local_filename = $1
  666. local_extention = $2
  667. end
  668. 1.upto(1000) do |i|
  669. filename_exists = false
  670. attachments.each do |attachment|
  671. if attachment[:filename] == filename
  672. filename_exists = true
  673. end
  674. end
  675. break if filename_exists == false
  676. filename = if local_extention.present?
  677. "#{local_filename}#{i}.#{local_extention}"
  678. else
  679. "#{local_filename}#{i}"
  680. end
  681. end
  682. # get mime type
  683. if file.header[:content_type]&.string
  684. headers_store['Mime-Type'] = file.header[:content_type].string
  685. end
  686. # get charset
  687. if file.header&.charset
  688. headers_store['Charset'] = file.header.charset
  689. end
  690. # remove not needed header
  691. headers_store.delete('Content-Transfer-Encoding')
  692. headers_store.delete('Content-Disposition')
  693. attach = {
  694. data: file_body,
  695. filename: filename,
  696. preferences: headers_store,
  697. }
  698. [attach]
  699. end
  700. # Archive the given message as tmp/folder/md5.eml
  701. def archive_mail(folder, msg)
  702. path = Rails.root.join('tmp', folder)
  703. FileUtils.mkpath path
  704. # MD5 hash the msg and save it as "md5.eml"
  705. md5 = Digest::MD5.hexdigest(msg)
  706. file_path = Rails.root.join('tmp', folder, "#{md5}.eml")
  707. File.open(file_path, 'wb') do |file|
  708. file.write msg
  709. end
  710. file_path
  711. end
  712. # Auto reply as the postmaster to oversized emails with:
  713. # [undeliverable] Message too large
  714. def postmaster_response(channel, msg)
  715. begin
  716. reply_mail = compose_postmaster_reply(msg)
  717. rescue NotificationFactory::FileNotFoundError => e
  718. Rails.logger.error "No valid postmaster email_oversized template found. Skipping postmaster reply. #{e.inspect}"
  719. return
  720. end
  721. Rails.logger.error "Send mail too large postmaster message to: #{reply_mail[:to]}"
  722. reply_mail[:from] = EmailAddress.find_by(channel: channel).email
  723. channel.deliver(reply_mail)
  724. rescue => e
  725. Rails.logger.error "Error during sending of postmaster oversized email auto-reply: #{e.inspect}\n#{e.backtrace}"
  726. end
  727. # Compose a "Message too large" reply to the given message
  728. def compose_postmaster_reply(raw_incoming_mail, locale = nil)
  729. parsed_incoming_mail = Channel::EmailParser.new.parse(raw_incoming_mail)
  730. # construct a dummy mail object
  731. mail = OpenStruct.new
  732. mail.from_display_name = parsed_incoming_mail[:from_display_name]
  733. mail.subject = parsed_incoming_mail[:subject]
  734. mail.msg_size = format('%<MB>.2f', MB: raw_incoming_mail.size.to_f / 1024 / 1024)
  735. reply = NotificationFactory::Mailer.template(
  736. template: 'email_oversized',
  737. locale: locale,
  738. format: 'txt',
  739. objects: {
  740. mail: mail,
  741. },
  742. raw: true, # will not add application template
  743. standalone: true, # default: false - will send header & footer
  744. )
  745. reply.merge(
  746. to: parsed_incoming_mail[:from_email],
  747. body: reply[:body].gsub(/\n/, "\r\n"),
  748. content_type: 'text/plain',
  749. References: parsed_incoming_mail[:message_id],
  750. 'In-Reply-To': parsed_incoming_mail[:message_id],
  751. )
  752. end
  753. def guess_email_fqdn(from)
  754. Mail::Address.new(from).domain.strip
  755. rescue
  756. nil
  757. end
  758. def generate_message_id(raw_message, from)
  759. fqdn = guess_email_fqdn(from) || 'zammad_generated'
  760. "<gen-#{Digest::MD5.hexdigest(raw_message)}@#{fqdn}>"
  761. end
  762. end
  763. module Mail
  764. # workaround to get content of no parseable headers - in most cases with non 7 bit ascii signs
  765. class Field
  766. def raw_value
  767. begin
  768. value = @raw_value.try(:utf8_encode)
  769. rescue
  770. value = @raw_value.utf8_encode(fallback: :read_as_sanitized_binary)
  771. end
  772. return value if value.blank?
  773. value.sub(/^.+?:(\s|)/, '')
  774. end
  775. end
  776. # issue#348 - IMAP mail fetching stops because of broken spam email (e. g. broken Content-Transfer-Encoding value see test/fixtures/mail43.box)
  777. # https://github.com/zammad/zammad/issues/348
  778. class Body
  779. def decoded
  780. if Encodings.defined?(encoding)
  781. Encodings.get_encoding(encoding).decode(raw_source)
  782. else
  783. Rails.logger.info "UnknownEncodingType: Don't know how to decode #{encoding}!"
  784. raw_source
  785. end
  786. end
  787. end
  788. end