html_sanitizer.rb 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465
  1. class HtmlSanitizer
  2. =begin
  3. satinize html string based on whiltelist
  4. string = HtmlSanitizer.strict(string, external)
  5. =end
  6. def self.strict(string, external = false)
  7. # config
  8. tags_remove_content = Rails.configuration.html_sanitizer_tags_remove_content
  9. tags_quote_content = Rails.configuration.html_sanitizer_tags_quote_content
  10. tags_whitelist = Rails.configuration.html_sanitizer_tags_whitelist
  11. attributes_whitelist = Rails.configuration.html_sanitizer_attributes_whitelist
  12. css_properties_whitelist = Rails.configuration.html_sanitizer_css_properties_whitelist
  13. css_values_blacklist = Rails.application.config.html_sanitizer_css_values_backlist
  14. classes_whitelist = ['js-signatureMarker']
  15. attributes_2_css = %w[width height]
  16. # remove html comments
  17. string.gsub!(/<!--.+?-->/m, '')
  18. scrubber_link = Loofah::Scrubber.new do |node|
  19. # check if text has urls which need to be clickable
  20. if node&.name != 'a' && node.parent && node.parent.name != 'a' && (!node.parent.parent || node.parent.parent.name != 'a')
  21. if node.class == Nokogiri::XML::Text
  22. urls = []
  23. node.content.scan(%r{((http|https|ftp|tel)://.+?)([[:space:]]|\.[[:space:]]|,[[:space:]]|\.$|,$|\)|\(|$)}mxi).each do |match|
  24. if match[0]
  25. urls.push match[0].to_s.strip
  26. end
  27. end
  28. node.content.scan(/(^|:|;|\s)(www\..+?)([[:space:]]|\.[[:space:]]|,[[:space:]]|\.$|,$|\)|\(|$)/mxi).each do |match|
  29. if match[1]
  30. urls.push match[1].to_s.strip
  31. end
  32. end
  33. next if urls.blank?
  34. add_link(node.content, urls, node)
  35. end
  36. end
  37. # prepare links
  38. if node['href']
  39. href = cleanup_target(node['href'])
  40. if external && href.present? && !href.downcase.start_with?('//') && href.downcase !~ %r{^.{1,6}://.+?}
  41. node['href'] = "http://#{node['href']}"
  42. href = node['href']
  43. end
  44. next if !href.downcase.start_with?('http', 'ftp', '//')
  45. node.set_attribute('href', href)
  46. node.set_attribute('rel', 'nofollow noreferrer noopener')
  47. node.set_attribute('target', '_blank')
  48. end
  49. if node.name == 'a' && node['href'].blank?
  50. node.replace node.children.to_s
  51. Loofah::Scrubber::STOP
  52. end
  53. # check if href is different to text
  54. if node.name == 'a' && !url_same?(node['href'], node.text)
  55. if node['title'].blank?
  56. node['title'] = node['href']
  57. end
  58. end
  59. end
  60. scrubber_wipe = Loofah::Scrubber.new do |node|
  61. # remove tags with subtree
  62. if tags_remove_content.include?(node.name)
  63. node.remove
  64. Loofah::Scrubber::STOP
  65. end
  66. # remove tag, insert quoted content
  67. if tags_quote_content.include?(node.name)
  68. string = html_decode(node.content)
  69. text = Nokogiri::XML::Text.new(string, node.document)
  70. node.add_next_sibling(text)
  71. node.remove
  72. Loofah::Scrubber::STOP
  73. end
  74. # replace tags, keep subtree
  75. if !tags_whitelist.include?(node.name)
  76. node.replace node.children.to_s
  77. Loofah::Scrubber::STOP
  78. end
  79. # prepare src attribute
  80. if node['src']
  81. src = cleanup_target(node['src'])
  82. if src =~ /(javascript|livescript|vbscript):/i || src.downcase.start_with?('http', 'ftp', '//')
  83. node.remove
  84. Loofah::Scrubber::STOP
  85. end
  86. end
  87. # clean class / only use allowed classes
  88. if node['class']
  89. classes = node['class'].gsub(/\t|\n|\r/, '').split(' ')
  90. class_new = ''
  91. classes.each do |local_class|
  92. next if !classes_whitelist.include?(local_class.to_s.strip)
  93. if class_new != ''
  94. class_new += ' '
  95. end
  96. class_new += local_class
  97. end
  98. if class_new != ''
  99. node['class'] = class_new
  100. else
  101. node.delete('class')
  102. end
  103. end
  104. # move style attributes to css attributes
  105. attributes_2_css.each do |key|
  106. next if !node[key]
  107. if node['style'].blank?
  108. node['style'] = ''
  109. else
  110. node['style'] += ';'
  111. end
  112. value = node[key]
  113. node.delete(key)
  114. next if value.blank?
  115. if value !~ /%|px|em/i
  116. value += 'px'
  117. end
  118. node['style'] += "#{key}:#{value}"
  119. end
  120. # clean style / only use allowed style properties
  121. if node['style']
  122. pears = node['style'].downcase.gsub(/\t|\n|\r/, '').split(';')
  123. style = ''
  124. pears.each do |local_pear|
  125. prop = local_pear.split(':')
  126. next if !prop[0]
  127. key = prop[0].strip
  128. next if !css_properties_whitelist.include?(node.name)
  129. next if !css_properties_whitelist[node.name].include?(key)
  130. next if css_values_blacklist[node.name]&.include?(local_pear.gsub(/[[:space:]]/, '').strip)
  131. style += "#{local_pear};"
  132. end
  133. node['style'] = style
  134. if style == ''
  135. node.delete('style')
  136. end
  137. end
  138. # scan for invalid link content
  139. %w[href style].each do |attribute_name|
  140. next if !node[attribute_name]
  141. href = cleanup_target(node[attribute_name])
  142. next if href !~ /(javascript|livescript|vbscript):/i
  143. node.delete(attribute_name)
  144. end
  145. # remove attributes if not whitelisted
  146. node.each do |attribute, _value| # rubocop:disable Performance/HashEachMethods
  147. attribute_name = attribute.downcase
  148. next if attributes_whitelist[:all].include?(attribute_name) || (attributes_whitelist[node.name]&.include?(attribute_name))
  149. node.delete(attribute)
  150. end
  151. # remove mailto links
  152. if node['href']
  153. href = cleanup_target(node['href'])
  154. if href =~ /mailto:(.*)$/i
  155. text = Nokogiri::XML::Text.new($1, node.document)
  156. node.add_next_sibling(text)
  157. node.remove
  158. Loofah::Scrubber::STOP
  159. end
  160. end
  161. end
  162. new_string = ''
  163. done = true
  164. while done
  165. new_string = Loofah.fragment(string).scrub!(scrubber_wipe).to_s
  166. if string == new_string
  167. done = false
  168. end
  169. string = new_string
  170. end
  171. Loofah.fragment(string).scrub!(scrubber_link).to_s
  172. end
  173. =begin
  174. cleanup html string:
  175. * remove empty nodes (p, div, span, table)
  176. * remove nodes in general (keep content - span)
  177. string = HtmlSanitizer.cleanup(string)
  178. =end
  179. def self.cleanup(string)
  180. string.gsub!(/<[A-z]:[A-z]>/, '')
  181. string.gsub!(%r{</[A-z]:[A-z]>}, '')
  182. string.delete!("\t")
  183. # remove all new lines
  184. string.gsub!(/(\n\r|\r\r\n|\r\n|\n)/, "\n")
  185. # remove double multiple empty lines
  186. string.gsub!(/\n\n\n+/, "\n\n")
  187. string = cleanup_structure(string, 'pre')
  188. string = cleanup_replace_tags(string)
  189. string = cleanup_structure(string)
  190. string
  191. end
  192. def self.cleanup_replace_tags(string)
  193. #return string
  194. tags_backlist = %w[span center]
  195. scrubber = Loofah::Scrubber.new do |node|
  196. next if !tags_backlist.include?(node.name)
  197. hit = false
  198. local_node = nil
  199. (1..5).each do |_count|
  200. local_node = if local_node
  201. local_node.parent
  202. else
  203. node.parent
  204. end
  205. break if !local_node
  206. next if local_node.name != 'td'
  207. hit = true
  208. end
  209. next if hit && node.keys.count.positive?
  210. node.replace cleanup_replace_tags(node.children.to_s)
  211. Loofah::Scrubber::STOP
  212. end
  213. Loofah.fragment(string).scrub!(scrubber).to_s
  214. end
  215. def self.cleanup_structure(string, type = 'all')
  216. remove_empty_nodes = if type == 'pre'
  217. %w[span]
  218. else
  219. %w[p div span small table]
  220. end
  221. remove_empty_last_nodes = %w[b i u small table]
  222. # remove last empty nodes and empty -not needed- parrent nodes
  223. scrubber_structure = Loofah::Scrubber.new do |node|
  224. if remove_empty_last_nodes.include?(node.name) && node.children.size.zero?
  225. node.remove
  226. Loofah::Scrubber::STOP
  227. end
  228. # remove empty childs
  229. if node.content.blank? && remove_empty_nodes.include?(node.name) && node.children.size == 1 && remove_empty_nodes.include?(node.children.first.name)
  230. node.replace node.children.to_s
  231. Loofah::Scrubber::STOP
  232. end
  233. # remove empty childs
  234. if remove_empty_nodes.include?(node.name) && node.children.size == 1 && remove_empty_nodes.include?(node.children.first.name) && node.children.first.content == node.content
  235. node.replace node.children.to_s
  236. Loofah::Scrubber::STOP
  237. end
  238. # remove node if empty and parent was already a remove node
  239. if node.content.blank? && remove_empty_nodes.include?(node.name) && node.parent && node.children.size.zero? && remove_empty_nodes.include?(node.parent.name)
  240. node.remove
  241. Loofah::Scrubber::STOP
  242. end
  243. end
  244. new_string = ''
  245. done = true
  246. while done
  247. new_string = Loofah.fragment(string).scrub!(scrubber_structure).to_s
  248. if string == new_string
  249. done = false
  250. end
  251. string = new_string
  252. end
  253. scrubber_cleanup = Loofah::Scrubber.new do |node|
  254. # remove mailto links
  255. if node['href']
  256. href = cleanup_target(node['href'])
  257. if href =~ /mailto:(.*)$/i
  258. text = Nokogiri::XML::Text.new($1, node.document)
  259. node.add_next_sibling(text)
  260. node.remove
  261. Loofah::Scrubber::STOP
  262. end
  263. end
  264. # remove not needed new lines
  265. if node.class == Nokogiri::XML::Text
  266. if !node.parent || (node.parent.name != 'pre' && node.parent.name != 'code')
  267. content = node.content
  268. if content
  269. if content != ' ' && content != "\n"
  270. content.gsub!(/[[:space:]]+/, ' ')
  271. end
  272. if node.previous
  273. if node.previous.name == 'div' || node.previous.name == 'p'
  274. content.strip!
  275. end
  276. elsif node.parent && !node.previous && (!node.next || node.next.name == 'div' || node.next.name == 'p' || node.next.name == 'br')
  277. if (node.parent.name == 'div' || node.parent.name == 'p') && content != ' ' && content != "\n"
  278. content.strip!
  279. end
  280. end
  281. node.content = content
  282. end
  283. end
  284. end
  285. end
  286. Loofah.fragment(string).scrub!(scrubber_cleanup).to_s
  287. end
  288. def self.add_link(content, urls, node)
  289. if urls.blank?
  290. text = Nokogiri::XML::Text.new(content, node.document)
  291. node.add_next_sibling(text)
  292. return
  293. end
  294. url = urls.shift
  295. if content =~ /^(.*)#{Regexp.quote(url)}(.*)$/mx
  296. pre = $1
  297. post = $2
  298. if url.match?(/^www/i)
  299. url = "http://#{url}"
  300. end
  301. a = Nokogiri::XML::Node.new 'a', node.document
  302. a['href'] = url
  303. a['rel'] = 'nofollow noreferrer noopener'
  304. a['target'] = '_blank'
  305. a.content = url
  306. if node.class != Nokogiri::XML::Text
  307. text = Nokogiri::XML::Text.new(pre, node.document)
  308. node.add_next_sibling(text).add_next_sibling(a)
  309. return if post.blank?
  310. add_link(post, urls, a)
  311. return
  312. end
  313. node.content = pre
  314. node.add_next_sibling(a)
  315. return if post.blank?
  316. add_link(post, urls, a)
  317. end
  318. true
  319. end
  320. def self.html_decode(string)
  321. string.gsub('&amp;', '&').gsub('&lt;', '<').gsub('&gt;', '>').gsub('&quot;', '"').gsub('&nbsp;', ' ')
  322. end
  323. def self.cleanup_target(string)
  324. string = CGI.unescape(string).encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '?')
  325. string.gsub(/[[:space:]]|\t|\n|\r/, '').gsub(%r{/\*.*?\*/}, '').gsub(/<!--.*?-->/, '').gsub(/\[.+?\]/, '').delete("\u0000")
  326. end
  327. def self.url_same?(url_new, url_old)
  328. url_new = CGI.unescape(url_new.to_s).encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '?').downcase.gsub(%r{/$}, '').gsub(/[[:space:]]|\t|\n|\r/, '').strip
  329. url_old = CGI.unescape(url_old.to_s).encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '?').downcase.gsub(%r{/$}, '').gsub(/[[:space:]]|\t|\n|\r/, '').strip
  330. url_new = html_decode(url_new).sub('/?', '?')
  331. url_old = html_decode(url_old).sub('/?', '?')
  332. return true if url_new == url_old
  333. return true if url_old == "http://#{url_new}"
  334. return true if url_new == "http://#{url_old}"
  335. return true if url_old == "https://#{url_new}"
  336. return true if url_new == "https://#{url_old}"
  337. false
  338. end
  339. =begin
  340. reolace inline images with cid images
  341. string = HtmlSanitizer.replace_inline_images(article.body)
  342. =end
  343. def self.replace_inline_images(string, prefix = rand(999_999_999))
  344. attachments_inline = []
  345. scrubber = Loofah::Scrubber.new do |node|
  346. if node.name == 'img'
  347. if node['src'] && node['src'] =~ %r{^(data:image/(jpeg|png);base64,.+?)$}i
  348. file_attributes = StaticAssets.data_url_attributes($1)
  349. cid = "#{prefix}.#{rand(999_999_999)}@#{Setting.get('fqdn')}"
  350. attachment = {
  351. data: file_attributes[:content],
  352. filename: cid,
  353. preferences: {
  354. 'Content-Type' => file_attributes[:mime_type],
  355. 'Mime-Type' => file_attributes[:mime_type],
  356. 'Content-ID' => cid,
  357. 'Content-Disposition' => 'inline',
  358. },
  359. }
  360. attachments_inline.push attachment
  361. node['src'] = "cid:#{cid}"
  362. end
  363. Loofah::Scrubber::STOP
  364. end
  365. end
  366. [Loofah.fragment(string).scrub!(scrubber).to_s, attachments_inline]
  367. end
  368. =begin
  369. satinize style of img tags
  370. string = HtmlSanitizer.dynamic_image_size(article.body)
  371. =end
  372. def self.dynamic_image_size(string)
  373. scrubber = Loofah::Scrubber.new do |node|
  374. if node.name == 'img'
  375. if node['src']
  376. style = 'max-width:100%;'
  377. if node['style']
  378. pears = node['style'].downcase.gsub(/\t|\n|\r/, '').split(';')
  379. pears.each do |local_pear|
  380. prop = local_pear.split(':')
  381. next if !prop[0]
  382. key = prop[0].strip
  383. if key == 'height'
  384. key = 'max-height'
  385. end
  386. style += "#{key}:#{prop[1]};"
  387. end
  388. end
  389. node['style'] = style
  390. end
  391. Loofah::Scrubber::STOP
  392. end
  393. end
  394. Loofah.fragment(string).scrub!(scrubber).to_s
  395. end
  396. private_class_method :cleanup_target
  397. private_class_method :add_link
  398. private_class_method :url_same?
  399. private_class_method :html_decode
  400. end