email_parser.rb 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691
  1. # Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/
  2. # encoding: utf-8
  3. require 'mail'
  4. require 'encode'
  5. class Channel::EmailParser
  6. =begin
  7. parser = Channel::EmailParser.new
  8. mail = parser.parse(msg_as_string)
  9. mail = {
  10. from: 'Some Name <some@example.com>',
  11. from_email: 'some@example.com',
  12. from_local: 'some',
  13. from_domain: 'example.com',
  14. from_display_name: 'Some Name',
  15. message_id: 'some_message_id@example.com',
  16. to: 'Some System <system@example.com>',
  17. cc: 'Somebody <somebody@example.com>',
  18. subject: 'some message subject',
  19. body: 'some message body',
  20. content_type: 'text/html', # text/plain
  21. date: Time.zone.now,
  22. attachments: [
  23. {
  24. data: 'binary of attachment',
  25. filename: 'file_name_of_attachment.txt',
  26. preferences: {
  27. content-alternative: true,
  28. Mime-Type: 'text/plain',
  29. Charset: 'iso-8859-1',
  30. },
  31. },
  32. ],
  33. # ignore email header
  34. x-zammad-ignore: 'false',
  35. # customer headers
  36. x-zammad-customer-login: '',
  37. x-zammad-customer-email: '',
  38. x-zammad-customer-firstname: '',
  39. x-zammad-customer-lastname: '',
  40. # ticket headers (for new tickets)
  41. x-zammad-ticket-group: 'some_group',
  42. x-zammad-ticket-state: 'some_state',
  43. x-zammad-ticket-priority: 'some_priority',
  44. x-zammad-ticket-owner: 'some_owner_login',
  45. # ticket headers (for existing tickets)
  46. x-zammad-ticket-followup-group: 'some_group',
  47. x-zammad-ticket-followup-state: 'some_state',
  48. x-zammad-ticket-followup-priority: 'some_priority',
  49. x-zammad-ticket-followup-owner: 'some_owner_login',
  50. # article headers
  51. x-zammad-article-internal: false,
  52. x-zammad-article-type: 'agent',
  53. x-zammad-article-sender: 'customer',
  54. # all other email headers
  55. some-header: 'some_value',
  56. }
  57. =end
  58. def parse(msg)
  59. data = {}
  60. mail = Mail.new(msg)
  61. # set all headers
  62. mail.header.fields.each { |field|
  63. next if !field.name
  64. # full line, encode, ready for storage
  65. data[field.name.to_s.downcase.to_sym] = Encode.conv('utf8', field.to_s)
  66. # if we need to access the lines by objects later again
  67. data[ "raw-#{field.name.downcase}".to_sym ] = field
  68. }
  69. # get sender
  70. from = nil
  71. ['from', 'reply-to', 'return-path'].each { |item|
  72. next if !mail[ item.to_sym ]
  73. from = mail[ item.to_sym ].value
  74. break if from
  75. }
  76. # set x-any-recipient
  77. data['x-any-recipient'.to_sym] = ''
  78. ['to', 'cc', 'delivered-to', 'x-original-to', 'envelope-to'].each { |item|
  79. next if !mail[item.to_sym]
  80. if data['x-any-recipient'.to_sym] != ''
  81. data['x-any-recipient'.to_sym] += ', '
  82. end
  83. data['x-any-recipient'.to_sym] += mail[item.to_sym].to_s
  84. }
  85. # set extra headers
  86. begin
  87. data[:from_email] = Mail::Address.new(from).address
  88. data[:from_local] = Mail::Address.new(from).local
  89. data[:from_domain] = Mail::Address.new(from).domain
  90. data[:from_display_name] = Mail::Address.new(from).display_name ||
  91. (Mail::Address.new(from).comments && Mail::Address.new(from).comments[0])
  92. rescue
  93. data[:from_email] = from
  94. data[:from_local] = from
  95. data[:from_domain] = from
  96. end
  97. # do extra decoding because we needed to use field.value
  98. data[:from_display_name] = Mail::Field.new('X-From', data[:from_display_name]).to_s
  99. # compat headers
  100. data[:message_id] = data['message-id'.to_sym]
  101. # body
  102. # plain_part = mail.multipart? ? (mail.text_part ? mail.text_part.body.decoded : nil) : mail.body.decoded
  103. # html_part = message.html_part ? message.html_part.body.decoded : nil
  104. data[:attachments] = []
  105. # multi part email
  106. if mail.multipart?
  107. # html attachment/body may exists and will be converted to strict html
  108. if mail.html_part && mail.html_part.body
  109. data[:body] = mail.html_part.body.to_s
  110. data[:body] = Encode.conv(mail.html_part.charset.to_s, data[:body])
  111. data[:body] = data[:body].html2html_strict.to_s.force_encoding('utf-8')
  112. if !data[:body].force_encoding('UTF-8').valid_encoding?
  113. data[:body] = data[:body].encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '?')
  114. end
  115. data[:content_type] = 'text/html'
  116. end
  117. # text attachment/body exists
  118. if data[:body].empty? && mail.text_part
  119. data[:body] = mail.text_part.body.decoded
  120. data[:body] = Encode.conv(mail.text_part.charset, data[:body])
  121. if !data[:body].valid_encoding?
  122. data[:body] = data[:body].encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '?')
  123. end
  124. data[:content_type] = 'text/plain'
  125. end
  126. # any other attachments
  127. if data[:body].empty?
  128. data[:body] = 'no visible content'
  129. data[:content_type] = 'text/plain'
  130. end
  131. # add html attachment/body as real attachment
  132. if mail.html_part
  133. filename = 'message.html'
  134. headers_store = {
  135. 'content-alternative' => true,
  136. }
  137. if mail.mime_type
  138. headers_store['Mime-Type'] = mail.html_part.mime_type
  139. end
  140. if mail.charset
  141. headers_store['Charset'] = mail.html_part.charset
  142. end
  143. attachment = {
  144. data: mail.html_part.body.to_s,
  145. filename: mail.html_part.filename || filename,
  146. preferences: headers_store
  147. }
  148. data[:attachments].push attachment
  149. end
  150. # get attachments
  151. if mail.parts
  152. mail.parts.each { |part|
  153. # protect process to work fine with spam emails, see test/fixtures/mail15.box
  154. begin
  155. attachs = _get_attachment(part, data[:attachments], mail)
  156. data[:attachments].concat(attachs)
  157. rescue
  158. attachs = _get_attachment(part, data[:attachments], mail)
  159. data[:attachments].concat(attachs)
  160. end
  161. }
  162. end
  163. # not multipart email
  164. # html part only, convert to text and add it as attachment
  165. elsif mail.mime_type && mail.mime_type.to_s.casecmp('text/html').zero?
  166. filename = 'message.html'
  167. data[:body] = mail.body.decoded
  168. data[:body] = Encode.conv(mail.charset, data[:body])
  169. data[:body] = data[:body].html2html_strict.to_s.force_encoding('utf-8')
  170. if !data[:body].valid_encoding?
  171. data[:body] = data[:body].encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '?')
  172. end
  173. data[:content_type] = 'text/html'
  174. # add body as attachment
  175. headers_store = {
  176. 'content-alternative' => true,
  177. }
  178. if mail.mime_type
  179. headers_store['Mime-Type'] = mail.mime_type
  180. end
  181. if mail.charset
  182. headers_store['Charset'] = mail.charset
  183. end
  184. attachment = {
  185. data: mail.body.decoded,
  186. filename: mail.filename || filename,
  187. preferences: headers_store
  188. }
  189. data[:attachments].push attachment
  190. # text part only
  191. elsif !mail.mime_type || mail.mime_type.to_s == '' || mail.mime_type.to_s.casecmp('text/plain').zero?
  192. data[:body] = mail.body.decoded
  193. data[:body] = Encode.conv(mail.charset, data[:body])
  194. if !data[:body].force_encoding('UTF-8').valid_encoding?
  195. data[:body] = data[:body].encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '?')
  196. end
  197. data[:content_type] = 'text/plain'
  198. else
  199. filename = '-no name-'
  200. data[:body] = 'no visible content'
  201. data[:content_type] = 'text/plain'
  202. # add body as attachment
  203. headers_store = {
  204. 'content-alternative' => true,
  205. }
  206. if mail.mime_type
  207. headers_store['Mime-Type'] = mail.mime_type
  208. end
  209. if mail.charset
  210. headers_store['Charset'] = mail.charset
  211. end
  212. attachment = {
  213. data: mail.body.decoded,
  214. filename: mail.filename || filename,
  215. preferences: headers_store
  216. }
  217. data[:attachments].push attachment
  218. end
  219. # strip not wanted chars
  220. data[:body].gsub!(/\n\r/, "\n")
  221. data[:body].gsub!(/\r\n/, "\n")
  222. data[:body].tr!("\r", "\n")
  223. # get mail date
  224. begin
  225. if mail.date
  226. data[:date] = Time.zone.parse(mail.date.to_s)
  227. end
  228. rescue
  229. data[:date] = nil
  230. end
  231. # remember original mail instance
  232. data[:mail_instance] = mail
  233. data
  234. end
  235. def _get_attachment(file, attachments, mail)
  236. # check if sub parts are available
  237. if !file.parts.empty?
  238. a = []
  239. file.parts.each { |p|
  240. attachment = _get_attachment(p, attachments, mail)
  241. a.concat(attachment)
  242. }
  243. return a
  244. end
  245. # ignore text/plain attachments - already shown in view
  246. return [] if mail.text_part && mail.text_part.body.to_s == file.body.to_s
  247. # ignore text/html - html part, already shown in view
  248. return [] if mail.html_part && mail.html_part.body.to_s == file.body.to_s
  249. # get file preferences
  250. headers_store = {}
  251. file.header.fields.each { |field|
  252. headers_store[field.name.to_s] = field.value.to_s
  253. }
  254. # get filename from content-disposition
  255. filename = nil
  256. # workaround for: NoMethodError: undefined method `filename' for #<Mail::UnstructuredField:0x007ff109e80678>
  257. begin
  258. filename = file.header[:content_disposition].filename
  259. rescue
  260. begin
  261. result = file.header[:content_disposition].to_s.scan( /filename=("|)(.+?)("|);/i )
  262. if result && result[0] && result[0][1]
  263. filename = result[0][1]
  264. end
  265. rescue
  266. Rails.logger.debug 'Unable to get filename'
  267. end
  268. end
  269. # for some broken sm mail clients (X-MimeOLE: Produced By Microsoft Exchange V6.5)
  270. if !filename
  271. filename = file.header[:content_location].to_s
  272. end
  273. # generate file name
  274. if !filename || filename.empty?
  275. attachment_count = 0
  276. (1..1000).each { |count|
  277. filename_exists = false
  278. filename = 'file-' + count.to_s
  279. attachments.each { |attachment|
  280. if attachment[:filename] == filename
  281. filename_exists = true
  282. end
  283. }
  284. break if filename_exists == false
  285. }
  286. end
  287. # get mime type
  288. if file.header[:content_type] && file.header[:content_type].string
  289. headers_store['Mime-Type'] = file.header[:content_type].string
  290. end
  291. # get charset
  292. if file.header && file.header.charset
  293. headers_store['Charset'] = file.header.charset
  294. end
  295. # remove not needed header
  296. headers_store.delete('Content-Transfer-Encoding')
  297. headers_store.delete('Content-Disposition')
  298. attach = {
  299. data: file.body.to_s,
  300. filename: filename,
  301. preferences: headers_store,
  302. }
  303. [attach]
  304. end
  305. =begin
  306. parser = Channel::EmailParser.new
  307. ticket, article, user = parser.process(channel, email_raw_string)
  308. returns
  309. [ticket, article, user]
  310. do not raise an exception - e. g. if used by scheduler
  311. parser = Channel::EmailParser.new
  312. ticket, article, user = parser.process(channel, email_raw_string, fakse)
  313. returns
  314. [ticket, article, user] || false
  315. =end
  316. def process(channel, msg, exception = true)
  317. _process(channel, msg)
  318. rescue => e
  319. # store unprocessable email for bug reporting
  320. path = "#{Rails.root}/tmp/unprocessable_mail/"
  321. FileUtils.mkpath path
  322. md5 = Digest::MD5.hexdigest(msg)
  323. filename = "#{path}/#{md5}.eml"
  324. message = "ERROR: Can't process email, you will find it for bug reporting under #{filename}, please create an issue at https://github.com/zammad/zammad/issues"
  325. p message # rubocop:disable Rails/Output
  326. p 'ERROR: ' + e.inspect # rubocop:disable Rails/Output
  327. Rails.logger.error message
  328. Rails.logger.error 'ERROR: ' + e.inspect
  329. File.open(filename, 'wb') { |file|
  330. file.write msg
  331. }
  332. return false if exception == false
  333. raise e.inspect + e.backtrace.inspect
  334. end
  335. def _process(channel, msg)
  336. # parse email
  337. mail = parse(msg)
  338. # run postmaster pre filter
  339. UserInfo.current_user_id = 1
  340. filters = {}
  341. Setting.where(area: 'Postmaster::PreFilter').order(:name).each { |setting|
  342. filters[setting.name] = Kernel.const_get(Setting.get(setting.name))
  343. }
  344. filters.each { |_prio, backend|
  345. Rails.logger.debug "run postmaster pre filter #{backend}"
  346. begin
  347. backend.run(channel, mail)
  348. rescue => e
  349. Rails.logger.error "can't run postmaster pre filter #{backend}"
  350. Rails.logger.error e.inspect
  351. raise e
  352. end
  353. }
  354. # check ignore header
  355. if mail[ 'x-zammad-ignore'.to_sym ] == 'true' || mail[ 'x-zammad-ignore'.to_sym ] == true
  356. Rails.logger.info "ignored email with msgid '#{mail[:message_id]}' from '#{mail[:from]}' because of x-zammad-ignore header"
  357. return true
  358. end
  359. # set interface handle
  360. original_interface_handle = ApplicationHandleInfo.current
  361. ticket = nil
  362. article = nil
  363. session_user = nil
  364. # use transaction
  365. Transaction.execute(interface_handle: "#{original_interface_handle}.postmaster") do
  366. # get sender user
  367. session_user_id = mail[ 'x-zammad-session-user-id'.to_sym ]
  368. if !session_user_id
  369. raise 'No x-zammad-session-user-id, no sender set!'
  370. end
  371. session_user = User.lookup(id: session_user_id)
  372. if !session_user
  373. raise "No user found for x-zammad-session-user-id: #{session_user_id}!"
  374. end
  375. # set current user
  376. UserInfo.current_user_id = session_user.id
  377. # get ticket# based on email headers
  378. if mail[ 'x-zammad-ticket-id'.to_sym ]
  379. ticket = Ticket.find_by(id: mail[ 'x-zammad-ticket-id'.to_sym ])
  380. end
  381. if mail[ 'x-zammad-ticket-number'.to_sym ]
  382. ticket = Ticket.find_by(number: mail[ 'x-zammad-ticket-number'.to_sym ])
  383. end
  384. # set ticket state to open if not new
  385. if ticket
  386. set_attributes_by_x_headers(ticket, 'ticket', mail, 'followup')
  387. # save changes set by x-zammad-ticket-followup-* headers
  388. ticket.save if ticket.changed?
  389. state = Ticket::State.find(ticket.state_id)
  390. state_type = Ticket::StateType.find(state.state_type_id)
  391. # if tickte is merged, find linked ticket
  392. if state_type.name == 'merged'
  393. end
  394. # set ticket to open again
  395. if !mail[ 'x-zammad-ticket-followup-state'.to_sym ]
  396. if state_type.name != 'new' && !mail[ 'x-zammad-out-of-office'.to_sym ]
  397. ticket.state = Ticket::State.find_by(name: 'open')
  398. ticket.save!
  399. end
  400. end
  401. end
  402. # create new ticket
  403. if !ticket
  404. preferences = {}
  405. if channel[:id]
  406. preferences = {
  407. channel_id: channel[:id]
  408. }
  409. end
  410. # get default group where ticket is created
  411. group = nil
  412. if channel[:group_id]
  413. group = Group.lookup(id: channel[:group_id])
  414. end
  415. if !group || group && !group.active
  416. group = Group.where(active: true).order('id ASC').first
  417. end
  418. if !group
  419. group = Group.first
  420. end
  421. ticket = Ticket.new(
  422. group_id: group.id,
  423. title: mail[:subject] || '',
  424. state_id: Ticket::State.find_by(name: 'new').id,
  425. priority_id: Ticket::Priority.find_by(name: '2 normal').id,
  426. preferences: preferences,
  427. )
  428. set_attributes_by_x_headers(ticket, 'ticket', mail)
  429. # create ticket
  430. ticket.save!
  431. end
  432. # set attributes
  433. ticket.with_lock do
  434. article = Ticket::Article.new(
  435. ticket_id: ticket.id,
  436. type_id: Ticket::Article::Type.find_by(name: 'email').id,
  437. sender_id: Ticket::Article::Sender.find_by(name: 'Customer').id,
  438. content_type: mail[:content_type],
  439. body: mail[:body],
  440. from: mail[:from],
  441. to: mail[:to],
  442. cc: mail[:cc],
  443. subject: mail[:subject],
  444. message_id: mail[:message_id],
  445. internal: false,
  446. )
  447. # x-headers lookup
  448. set_attributes_by_x_headers(article, 'article', mail)
  449. # create article
  450. article.save!
  451. # store mail plain
  452. Store.add(
  453. object: 'Ticket::Article::Mail',
  454. o_id: article.id,
  455. data: msg,
  456. filename: "ticket-#{ticket.number}-#{article.id}.eml",
  457. preferences: {}
  458. )
  459. # store attachments
  460. if mail[:attachments]
  461. mail[:attachments].each do |attachment|
  462. Store.add(
  463. object: 'Ticket::Article',
  464. o_id: article.id,
  465. data: attachment[:data],
  466. filename: attachment[:filename],
  467. preferences: attachment[:preferences]
  468. )
  469. end
  470. end
  471. end
  472. end
  473. # run postmaster post filter
  474. filters = {}
  475. Setting.where(area: 'Postmaster::PostFilter').order(:name).each { |setting|
  476. filters[setting.name] = Kernel.const_get(Setting.get(setting.name))
  477. }
  478. filters.each { |_prio, backend|
  479. Rails.logger.debug "run postmaster post filter #{backend}"
  480. begin
  481. backend.run(channel, mail, ticket, article, session_user)
  482. rescue => e
  483. Rails.logger.error "can't run postmaster post filter #{backend}"
  484. Rails.logger.error e.inspect
  485. end
  486. }
  487. # return new objects
  488. [ticket, article, session_user, mail]
  489. end
  490. def set_attributes_by_x_headers(item_object, header_name, mail, suffix = false)
  491. # loop all x-zammad-hedaer-* headers
  492. item_object.attributes.each { |key, _value|
  493. # ignore read only attributes
  494. next if key == 'updated_by_id'
  495. next if key == 'created_by_id'
  496. # check if id exists
  497. key_short = key[ key.length - 3, key.length ]
  498. if key_short == '_id'
  499. key_short = key[ 0, key.length - 3 ]
  500. header = "x-zammad-#{header_name}-#{key_short}"
  501. if suffix
  502. header = "x-zammad-#{header_name}-#{suffix}-#{key_short}"
  503. end
  504. if mail[ header.to_sym ]
  505. Rails.logger.info "header #{header} found #{mail[ header.to_sym ]}"
  506. item_object.class.reflect_on_all_associations.map { |assoc|
  507. next if assoc.name.to_s != key_short
  508. Rails.logger.info "ASSOC found #{assoc.class_name} lookup #{mail[ header.to_sym ]}"
  509. item = assoc.class_name.constantize
  510. if item.respond_to?(:name)
  511. if item.lookup(name: mail[ header.to_sym ])
  512. item_object[key] = item.lookup(name: mail[ header.to_sym ]).id
  513. end
  514. elsif item.respond_to?(:login)
  515. if item.lookup(login: mail[ header.to_sym ])
  516. item_object[key] = item.lookup(login: mail[ header.to_sym ]).id
  517. end
  518. end
  519. }
  520. end
  521. end
  522. # check if attribute exists
  523. header = "x-zammad-#{header_name}-#{key}"
  524. if suffix
  525. header = "x-zammad-#{header_name}-#{suffix}-#{key}"
  526. end
  527. if mail[ header.to_sym ]
  528. Rails.logger.info "header #{header} found #{mail[ header.to_sym ]}"
  529. item_object[key] = mail[ header.to_sym ]
  530. end
  531. }
  532. end
  533. end
  534. module Mail
  535. # workaround to parse subjects with 2 different encodings correctly (e. g. quoted-printable see test/fixtures/mail9.box)
  536. module Encodings
  537. def self.value_decode(str)
  538. # Optimization: If there's no encoded-words in the string, just return it
  539. return str unless str.index('=?')
  540. str = str.gsub(/\?=(\s*)=\?/, '?==?') # Remove whitespaces between 'encoded-word's
  541. # Split on white-space boundaries with capture, so we capture the white-space as well
  542. str.split(/([ \t])/).map do |text|
  543. if text.index('=?') .nil?
  544. text
  545. else
  546. # Join QP encoded-words that are adjacent to avoid decoding partial chars
  547. # text.gsub!(/\?\=\=\?.+?\?[Qq]\?/m, '') if text =~ /\?==\?/
  548. # Search for occurences of quoted strings or plain strings
  549. text.scan(/( # Group around entire regex to include it in matches
  550. \=\?[^?]+\?([QB])\?[^?]+?\?\= # Quoted String with subgroup for encoding method
  551. | # or
  552. .+?(?=\=\?|$) # Plain String
  553. )/xmi).map do |matches|
  554. string, method = *matches
  555. if method == 'b' || method == 'B'
  556. b_value_decode(string)
  557. elsif method == 'q' || method == 'Q'
  558. q_value_decode(string)
  559. else
  560. string
  561. end
  562. end
  563. end
  564. end.join('')
  565. end
  566. end
  567. # issue#348 - IMAP mail fetching stops because of broken spam email (e. g. broken Content-Transfer-Encoding value see test/fixtures/mail43.box)
  568. # https://github.com/zammad/zammad/issues/348
  569. class Body
  570. def decoded
  571. if !Encodings.defined?(encoding)
  572. #raise UnknownEncodingType, "Don't know how to decode #{encoding}, please call #encoded and decode it yourself."
  573. Rails.logger.info "UnknownEncodingType: Don't know how to decode #{encoding}!"
  574. raw_source
  575. else
  576. Encodings.get_encoding(encoding).decode(raw_source)
  577. end
  578. end
  579. end
  580. end