string.rb 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454
  1. class String
  2. alias old_strip strip
  3. alias old_strip! strip!
  4. def strip!
  5. begin
  6. sub!(/\A[[[:space:]]\u{200B}\u{FEFF}]+/, '')
  7. sub!(/[[[:space:]]\u{200B}\u{FEFF}]+\Z/, '')
  8. # if incompatible encoding regexp match (UTF-8 regexp with ASCII-8BIT string) (Encoding::CompatibilityError), use default
  9. rescue
  10. old_strip!
  11. end
  12. self
  13. end
  14. def strip
  15. begin
  16. new_string = sub(/\A[[[:space:]]\u{200B}\u{FEFF}]+/, '')
  17. new_string.sub!(/[[[:space:]]\u{200B}\u{FEFF}]+\Z/, '')
  18. # if incompatible encoding regexp match (UTF-8 regexp with ASCII-8BIT string) (Encoding::CompatibilityError), use default
  19. rescue
  20. new_string = old_strip
  21. end
  22. new_string
  23. end
  24. def message_quote
  25. quote = split("\n")
  26. body_quote = ''
  27. quote.each do |line|
  28. body_quote = body_quote + '> ' + line + "\n"
  29. end
  30. body_quote
  31. end
  32. def word_wrap(*args)
  33. options = args.extract_options!
  34. if args.present?
  35. options[:line_width] = args[0] || 82
  36. end
  37. options.reverse_merge!(line_width: 82)
  38. lines = self
  39. lines.split("\n").collect do |line|
  40. line.length > options[:line_width] ? line.gsub(/(.{1,#{options[:line_width]}})(\s+|$)/, "\\1\n").strip : line
  41. end * "\n"
  42. end
  43. =begin
  44. filename = 'Some::Module'.to_filename
  45. returns
  46. 'some/module'
  47. =end
  48. def to_filename
  49. camel_cased_word = "#{self}" # rubocop:disable Style/UnneededInterpolation
  50. camel_cased_word.gsub(/::/, '/')
  51. .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
  52. .gsub(/([a-z\d])([A-Z])/, '\1_\2')
  53. .tr('-', '_').downcase
  54. end
  55. =begin
  56. filename = 'some/module.rb'.to_classname
  57. returns
  58. 'Some::Module'
  59. =end
  60. def to_classname
  61. camel_cased_word = "#{self}" # rubocop:disable Style/UnneededInterpolation
  62. camel_cased_word.gsub!(/\.rb$/, '')
  63. camel_cased_word.split('/').map(&:camelize).join('::')
  64. end
  65. # because of mysql inno_db limitations, strip 4 bytes utf8 chars (e. g. emojis)
  66. # unfortunaly UTF8mb4 will raise other limitaions of max varchar and lower index sizes
  67. # More details: http://pjambet.github.io/blog/emojis-and-mysql/
  68. def utf8_to_3bytesutf8
  69. return self if Rails.application.config.db_4bytes_utf8
  70. each_char.select do |c|
  71. if c.bytes.count > 3
  72. Rails.logger.warn "strip out 4 bytes utf8 chars '#{c}' of '#{self}'"
  73. next
  74. end
  75. c
  76. end
  77. .join('')
  78. end
  79. =begin
  80. text = html_string.html2text
  81. returns
  82. 'string with text only'
  83. =end
  84. def html2text(string_only = false, strict = false)
  85. string = "#{self}" # rubocop:disable Style/UnneededInterpolation
  86. # in case of invalid encoding, strip invalid chars
  87. # see also test/fixtures/mail21.box
  88. # note: string.encode!('UTF-8', 'UTF-8', :invalid => :replace, :replace => '?') was not detecting invalid chars
  89. if !string.valid_encoding?
  90. string = string.chars.select(&:valid_encoding?).join
  91. end
  92. # remove html comments
  93. string.gsub!(/<!--.+?-->/m, '')
  94. # find <a href=....> and replace it with [x]
  95. link_list = ''
  96. counter = 0
  97. if !string_only
  98. string.gsub!(/<a[[:space:]].*?href=("|')(.+?)("|').*?>/ix) do
  99. link = $2
  100. counter = counter + 1
  101. link_list += "[#{counter}] #{link}\n"
  102. "[#{counter}] "
  103. end
  104. else
  105. string.gsub!(%r{<a[[:space:]]+(|\S+[[:space:]]+)href=("|')(.+?)("|')([[:space:]]*|[[:space:]]+[^>]*)>(.+?)<[[:space:]]*/a[[:space:]]*>}mxi) do |_placeholder|
  106. link = $3
  107. text = $6
  108. text.gsub!(/\<.+?\>/, '')
  109. link_compare = link.dup
  110. if link_compare.present?
  111. link.strip!
  112. link_compare.strip!
  113. link_compare.downcase!
  114. link_compare.sub!(%r{/$}, '')
  115. end
  116. text_compare = text.dup
  117. if text_compare.present?
  118. text.strip!
  119. text_compare.strip!
  120. text_compare.downcase!
  121. text_compare.sub!(%r{/$}, '')
  122. end
  123. placeholder = if link_compare.present? && text_compare.blank?
  124. link
  125. elsif link_compare.blank? && text_compare.present?
  126. text
  127. elsif link_compare && link_compare =~ /^mailto/i
  128. text
  129. elsif link_compare.present? && text_compare.present? && (link_compare == text_compare || link_compare == "mailto:#{text}".downcase || link_compare == "http://#{text}".downcase)
  130. "######LINKEXT:#{link}/TEXT:#{text}######"
  131. elsif text !~ /^http/
  132. "#{text} (######LINKRAW:#{link}######)"
  133. else
  134. "#{link} (######LINKRAW:#{text}######)"
  135. end
  136. end
  137. end
  138. # remove style tags with content
  139. string.gsub!(%r{<style(|[[:space:]].+?)>(.+?)</style>}im, '')
  140. # remove empty lines
  141. string.gsub!(/^[[:space:]]*/m, '')
  142. if strict
  143. string.gsub!(%r{< [[:space:]]* (/*) [[:space:]]* (b|i|ul|ol|li|u|h1|h2|h3|hr) ([[:space:]]*|[[:space:]]+[^>]*) >}mxi, '######\1\2######')
  144. end
  145. # pre/code handling 1/2
  146. string.gsub!(%r{<pre>(.+?)</pre>}m) do |placeholder|
  147. placeholder = placeholder.gsub(/\n/, '###BR###')
  148. end
  149. string.gsub!(%r{<code>(.+?)</code>}m) do |placeholder|
  150. placeholder = placeholder.gsub(/\n/, '###BR###')
  151. end
  152. # insert spaces on [A-z]\n[A-z]
  153. string.gsub!(/([A-z])[[:space:]]([A-z])/m, '\1 \2')
  154. # remove all new lines
  155. string.gsub!(/(\n\r|\r\r\n|\r\n|\n)/, '')
  156. # blockquote handling
  157. string.gsub!(%r{<blockquote(| [^>]*)>(.+?)</blockquote>}m) do
  158. "\n" + $2.html2text(true).gsub(/^(.*)$/, '&gt; \1') + "\n"
  159. end
  160. # pre/code handling 2/2
  161. string.gsub!(/###BR###/, "\n")
  162. # add counting
  163. string.gsub!(/<li(| [^>]*)>/i, "\n* ")
  164. # add hr
  165. string.gsub!(%r{<hr(|/| [^>]*)>}i, "\n___\n")
  166. # add h\d
  167. string.gsub!(%r{</h\d>}i, "\n")
  168. # add new lines
  169. string.gsub!(%r{</div><div(|[[:space:]].+?)>}im, "\n")
  170. string.gsub!(%r{</p><p(|[[:space:]].+?)>}im, "\n")
  171. string.gsub!(%r{<(div|p|pre|br|table|tr|h)(|/| [^>]*)>}i, "\n")
  172. string.gsub!(%r{</(p|br|div)(|[[:space:]].+?)>}i, "\n")
  173. string.gsub!(%r{</td>}i, ' ')
  174. # strip all other tags
  175. string.gsub!(/\<.+?\>/, '')
  176. # replace multiple spaces with one
  177. string.gsub!(/ /, ' ')
  178. # add hyperlinks
  179. if strict
  180. string.gsub!(%r{([[:space:]])((http|https|ftp|tel)://.+?|(www..+?))([[:space:]]|\.[[:space:]]|,[[:space:]])}mxi) do |_placeholder|
  181. pre = $1
  182. content = $2
  183. post = $5
  184. if content.match?(/^www/i)
  185. content = "http://#{content}"
  186. end
  187. placeholder = if content =~ /^(http|https|ftp|tel)/i
  188. "#{pre}######LINKRAW:#{content}#######{post}"
  189. else
  190. "#{pre}#{content}#{post}"
  191. end
  192. end
  193. end
  194. # try HTMLEntities, if it fails on invalid signes, use manual way
  195. begin
  196. coder = HTMLEntities.new
  197. string = coder.decode(string)
  198. rescue
  199. # strip all &amp; &lt; &gt; &quot;
  200. string.gsub!('&amp;', '&')
  201. string.gsub!('&lt;', '<')
  202. string.gsub!('&gt;', '>')
  203. string.gsub!('&quot;', '"')
  204. string.gsub!('&nbsp;', ' ')
  205. # encode html entities like "&#8211;"
  206. string.gsub!(/(&\#(\d+);?)/x) do
  207. $2.chr
  208. end
  209. # encode html entities like "&#3d;"
  210. string.gsub!(/(&\#[xX]([0-9a-fA-F]+);?)/x) do
  211. chr_orig = $1
  212. hex = $2.hex
  213. if hex
  214. chr = hex.chr
  215. if chr
  216. chr_orig = chr
  217. else
  218. chr_orig
  219. end
  220. else
  221. chr_orig
  222. end
  223. # check valid encoding
  224. begin
  225. if !chr_orig.encode('UTF-8').valid_encoding?
  226. chr_orig = '?'
  227. end
  228. rescue
  229. chr_orig = '?'
  230. end
  231. chr_orig
  232. end
  233. end
  234. # remove tailing empty spaces
  235. string.gsub!(/[[:blank:]]+$/, '')
  236. # remove double multiple empty lines
  237. string.gsub!(/\n\n\n+/, "\n\n")
  238. # add extracted links
  239. if link_list != ''
  240. string += "\n\n\n" + link_list
  241. end
  242. # remove double multiple empty lines
  243. string.gsub!(/\n\n\n+/, "\n\n")
  244. string.strip
  245. end
  246. =begin
  247. html = text_string.text2html
  248. =end
  249. def text2html
  250. text = CGI.escapeHTML(self)
  251. text.gsub!(/\n/, '<br>')
  252. text.chomp
  253. end
  254. =begin
  255. html = text_string.text2html
  256. =end
  257. def html2html_strict
  258. string = "#{self}" # rubocop:disable Style/UnneededInterpolation
  259. string = HtmlSanitizer.cleanup_replace_tags(string)
  260. string = HtmlSanitizer.strict(string, true).strip
  261. string = HtmlSanitizer.cleanup(string).strip
  262. # as fallback, use html2text and text2html
  263. if string.blank?
  264. string = html2text.text2html
  265. string.signature_identify('text')
  266. marker_template = '<span class="js-signatureMarker"></span>'
  267. string.sub!(/######SIGNATURE_MARKER######/, marker_template)
  268. string.gsub!(/######SIGNATURE_MARKER######/, '')
  269. return string.chomp
  270. end
  271. string.gsub!(%r{(<p>[[:space:]]*</p>([[:space:]]*)){2,}}im, '<p>&nbsp;</p>\2')
  272. string.gsub!(%r\<div>[[:space:]]*(<br(|/)>([[:space:]]*)){2,}\im, '<div><br>\3')
  273. string.gsub!(%r\[[:space:]]*(<br>[[:space:]]*){3,}[[:space:]]*</div>\im, '<br><br></div>')
  274. string.gsub!(%r\<div>[[:space:]]*(<br>[[:space:]]*){1,}[[:space:]]*</div>\im, '<div>&nbsp;</div>')
  275. string.gsub!(%r\<div>[[:space:]]*(<div>[[:space:]]*</div>[[:space:]]*){2,}</div>\im, '<div>&nbsp;</div>')
  276. string.gsub!(%r\<p>[[:space:]]*</p>(<br(|/)>[[:space:]]*){2,}[[:space:]]*\im, '<p> </p><br>')
  277. string.gsub!(%r{<p>[[:space:]]*</p>(<br(|/)>[[:space:]]*)+<p>[[:space:]]*</p>}im, '<p> </p><p> </p>')
  278. string.gsub!(%r\(<div>[[:space:]]*</div>[[:space:]]*){2,}\im, '<div> </div>')
  279. string.gsub!(%r{<div>&nbsp;</div>[[:space:]]*(<div>&nbsp;</div>){1,}}im, '<div>&nbsp;</div>')
  280. string.gsub!(/(<br>[[:space:]]*){3,}/im, '<br><br>')
  281. string.gsub!(%r\(<br(|/)>[[:space:]]*){3,}\im, '<br/><br/>')
  282. string.gsub!(%r{<p>[[:space:]]+</p>}im, '<p>&nbsp;</p>')
  283. string.gsub!(%r{\A(<br(|\/)>[[:space:]]*)*}i, '')
  284. string.gsub!(%r{[[:space:]]*(<br(|\/)>[[:space:]]*)*\Z}i, '')
  285. string.gsub!(%r{(<p></p>){1,10}\Z}i, '')
  286. string.signature_identify('html')
  287. marker_template = '<span class="js-signatureMarker"></span>'
  288. string.sub!(/######SIGNATURE_MARKER######/, marker_template)
  289. string.gsub!(/######SIGNATURE_MARKER######/, '')
  290. string.chomp
  291. end
  292. def signature_identify(type = 'text', force = false)
  293. string = self
  294. marker = '######SIGNATURE_MARKER######'
  295. if type == 'html'
  296. map = [
  297. '<br(|\/)>[[:space:]]*(--|__)',
  298. '<\/div>[[:space:]]*(--|__)',
  299. '<p>[[:space:]]*(--|__)',
  300. '(<br(|\/)>|<p>|<div>)[[:space:]]*<b>(Von|From|De|от|Z|Od|Ze|Fra|Van|Mistä|Από|Dal|から|Из|од|iz|Från|จาก|з|Từ):[[:space:]]*</b>',
  301. '(<br>|<div>)[[:space:]]*<br>[[:space:]]*(Von|From|De|от|Z|Od|Ze|Fra|Van|Mistä|Από|Dal|から|Из|од|iz|Från|จาก|з|Từ):[[:space:]]+',
  302. '<blockquote(|.+?)>[[:space:]]*<div>[[:space:]]*(On|Am|Le|El|Den|Dňa|W dniu|Il|Op|Dne|Dana)[[:space:]]',
  303. '<div(|.+?)>[[:space:]]*<br>[[:space:]]*(On|Am|Le|El|Den|Dňa|W dniu|Il|Op|Dne|Dana)[[:space:]].{1,500}<blockquote',
  304. ]
  305. map.each do |regexp|
  306. string.sub!(/#{regexp}/m) do |placeholder|
  307. placeholder = "#{marker}#{placeholder}"
  308. end
  309. end
  310. return string
  311. end
  312. # if we do have less then 10 lines and less then 300 chars ignore this
  313. if !force
  314. lines = string.split("\n")
  315. return if lines.count < 10 && string.length < 300
  316. end
  317. # search for signature separator "--\n"
  318. string.sub!(/^\s{0,2}--\s{0,2}$/) do |placeholder|
  319. placeholder = "#{marker}#{placeholder}"
  320. end
  321. map = {}
  322. # Apple Mail
  323. # On 01/04/15 10:55, Bob Smith wrote:
  324. map['apple-en'] = '^(On)[[:space:]].{6,20}[[:space:]].{3,10}[[:space:]].{1,250}[[:space:]](wrote):'
  325. # Am 03.04.2015 um 20:58 schrieb Martin Edenhofer <me@znuny.ink>:
  326. map['apple-de'] = '^(Am)[[:space:]].{6,20}[[:space:]](um)[[:space:]].{3,10}[[:space:]](schrieb)[[:space:]].{1,250}:'
  327. # Thunderbird
  328. # Am 04.03.2015 um 12:47 schrieb Alf Aardvark:
  329. map['thunderbird-de'] = '^(Am)[[:space:]].{6,20}[[:space:]](um)[[:space:]].{3,10}[[:space:]](schrieb)[[:space:]].{1,250}:'
  330. # Thunderbird default - http://kb.mozillazine.org/Reply_header_settings
  331. # On 01-01-2007 11:00 AM, Alf Aardvark wrote:
  332. map['thunderbird-en-default'] = '^(On)[[:space:]].{6,20}[[:space:]].{3,10},[[:space:]].{1,250}(wrote):'
  333. # http://kb.mozillazine.org/Reply_header_settings
  334. # Alf Aardvark wrote, on 01-01-2007 11:00 AM:
  335. map['thunderbird-en'] = '^.{1,250}[[:space:]](wrote),[[:space:]]on[[:space:]].{3,20}:'
  336. # otrs
  337. # 25.02.2015 10:26 - edv hotline wrote:
  338. # 25.02.2015 10:26 - edv hotline schrieb:
  339. map['otrs-en-de'] = '^.{6,10}[[:space:]].{3,10}[[:space:]]-[[:space:]].{1,250}[[:space:]](wrote|schrieb):'
  340. # Ms
  341. # rubocop:disable Style/AsciiComments
  342. # From: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]
  343. # Send: Donnerstag, 2. April 2015 10:00
  344. # To/Cc/Bcc: xxx
  345. # Subject: xxx
  346. # - or -
  347. # From: xxx
  348. # To/Cc/Bcc: xxx
  349. # Date: 01.04.2015 12:41
  350. # Subject: xxx
  351. # - or -
  352. # De : xxx
  353. # À/?/?: xxx
  354. # Envoyé : mercredi 29 avril 2015 17:31
  355. # Objet : xxx
  356. # rubocop:enable Style/AsciiComments
  357. # en/de/fr | sometimes ms adds a space to "xx : value"
  358. map['ms-en-de-fr_from'] = '^(Von|From|De|от|Z|Od|Ze|Fra|Van|Mistä|Από|Dal|から|Из|од|iz|Från|จาก|з|Từ)( ?):[[:space:]].+?'
  359. map['ms-en-de-fr_from_html'] = "\n######b######(From|Von|De)([[:space:]]?):([[:space:]]?)(######\/b######)[[:space:]].+?"
  360. # word 14
  361. # edv hotline wrote:
  362. # edv hotline schrieb:
  363. #map['word-en-de'] = "[^#{marker}].{1,250}\s(wrote|schrieb):"
  364. map.each_value do |regexp|
  365. begin
  366. string.sub!(/#{regexp}/) do |placeholder|
  367. placeholder = "#{marker}#{placeholder}"
  368. end
  369. rescue
  370. # regexp was not possible because of some string encoding issue, use next
  371. Rails.logger.debug { "Invalid string/charset combination with regexp #{regexp} in string" }
  372. end
  373. end
  374. string
  375. end
  376. end