Fixes #2810 - Attachments of Knowledge Base Answers don't get added to Elasticsearch

app/controllers/knowledge_base/answer/attachments_controller.rb

@@ -7,27 +7,15 @@ class KnowledgeBase::Answer::AttachmentsController < ApplicationController
   before_action :fetch_answer
 
   def create
-    file = params[:file]
+    @answer.add_attachment params[:file]
 
-    Store.add(
-      object:      @answer.class.name,
-      o_id:        @answer.id,
-      data:        file.read,
-      filename:    file.original_filename,
-      preferences: headers_for_file(file)
-    )
-
-    output
+    render json: @answer.assets({})
   end
 
   def destroy
-    attachment = @answer.attachments.find { |elem| elem.id == params[:id].to_i }
-
-    raise ActiveRecord::RecordNotFound if attachment.nil?
+    @answer.remove_attachment params[:id]
 
-    Store.remove_item(attachment.id)
-
-    output
+    render json: @answer.assets({})
   end
 
   private
@@ -35,21 +23,4 @@ class KnowledgeBase::Answer::AttachmentsController < ApplicationController
   def fetch_answer
     @answer = KnowledgeBase::Answer.find params[:answer_id]
   end
-
-  def output
-    @answer.touch # rubocop:disable Rails/SkipsModelValidations
-    render json: @answer.assets({})
-  end
-
-  def headers_for_file(file)
-    content_type = file.content_type || 'application/octet-stream'
-
-    if content_type == 'application/octet-stream' && (mime = MIME::Types.type_for(file.original_filename).first)
-      content_type = mime
-    end
-
-    {
-      'Content-Type': content_type
-    }
-  end
 end

app/models/application_model/has_attachments.rb

@@ -59,6 +59,46 @@ store attachments for this object
     attachments_buffer_check
   end
 
+=begin
+
+Returns attachments in ElasticSearch-compatible format
+For use in #search_index_attribute_lookup
+
+=end
+
+  def attachments_for_search_index_attribute_lookup
+    # list ignored file extensions
+    attachments_ignore = Setting.get('es_attachment_ignore') || [ '.png', '.jpg', '.jpeg', '.mpeg', '.mpg', '.mov', '.bin', '.exe' ]
+
+    # max attachment size
+    attachment_max_size_in_mb = (Setting.get('es_attachment_max_size_in_mb') || 10).megabytes
+    attachment_total_max_size_in_kb = 314_572.kilobytes
+    attachment_total_max_size_in_kb_current = 0.kilobytes
+
+    attachments.each_with_object([]) do |attachment, memo|
+      # check if attachment exists
+      next if !attachment.content
+
+      size_in_bytes = attachment.content.size.bytes
+
+      # check file size
+      next if size_in_bytes > attachment_max_size_in_mb
+
+      # check ignored files
+      next if !attachment.filename || attachments_ignore.include?(File.extname(attachment.filename).downcase)
+
+      # check if fits into total size limit
+      next if attachment_total_max_size_in_kb_current + size_in_bytes > attachment_total_max_size_in_kb
+
+      attachment_total_max_size_in_kb_current += size_in_bytes
+
+      memo << {
+        '_name'    => attachment.filename,
+        '_content' => Base64.encode64(attachment.content).delete("\n")
+      }
+    end
+  end
+
   private
 
   def attachments_buffer

app/models/knowledge_base/answer.rb

@@ -61,6 +61,37 @@ class KnowledgeBase::Answer < ApplicationModel
     attachments.sort_by { |elem| elem.filename.downcase }
   end
 
+  def add_attachment(file)
+    filename     = file.try(:original_filename) || File.basename(file.path)
+    content_type = file.try(:content_type) || MIME::Types.type_for(filename).first&.content_type || 'application/octet-stream'
+
+    Store.add(
+      object:      self.class.name,
+      o_id:        id,
+      data:        file.read,
+      filename:    filename,
+      preferences: { 'Content-Type': content_type }
+    )
+
+    touch # rubocop:disable Rails/SkipsModelValidations
+    translations.each(&:touch)
+
+    true
+  end
+
+  def remove_attachment(attachment_id)
+    attachment = attachments.find { |elem| elem.id == attachment_id.to_i }
+
+    raise ActiveRecord::RecordNotFound if attachment.nil?
+
+    Store.remove_item(attachment.id)
+
+    touch # rubocop:disable Rails/SkipsModelValidations
+    translations.each(&:touch)
+
+    true
+  end
+
   def api_url
     Rails.application.routes.url_helpers.knowledge_base_answer_path(category.knowledge_base, self)
   end

app/models/knowledge_base/answer/translation.rb

@@ -54,9 +54,10 @@ class KnowledgeBase::Answer::Translation < ApplicationModel
   def search_index_attribute_lookup
     attrs = super
 
-    attrs['title']    = ActionController::Base.helpers.strip_tags attrs['title']
-    attrs['content']  = content.search_index_attribute_lookup if content
-    attrs['scope_id'] = answer.category_id
+    attrs['title']      = ActionController::Base.helpers.strip_tags attrs['title']
+    attrs['content']    = content.search_index_attribute_lookup if content
+    attrs['scope_id']   = answer.category_id
+    attrs['attachment'] = answer.attachments_for_search_index_attribute_lookup
 
     attrs
   end

lib/tasks/search_index_es.rake

@@ -116,6 +116,17 @@ namespace :searchindex do
                   }.merge(pipeline_field_attributes),
                 }
               }.merge(pipeline_field_attributes),
+            },
+            {
+              foreach: {
+                field:     'attachment',
+                processor: {
+                  attachment: {
+                    target_field: '_ingest._value',
+                    field:        '_ingest._value._content',
+                  }.merge(pipeline_field_attributes),
+                }
+              }.merge(pipeline_field_attributes),
             }
           ]
         }
@@ -289,6 +300,23 @@ def get_mapping_properties_object(object)
     end
   end
 
+  if object.name == 'KnowledgeBase::Answer::Translation'
+    # do not server attachments if document is requested
+    result[name][:_source] = {
+      excludes: ['attachment']
+    }
+
+    # for elasticsearch 5.5 and lower
+    if !es_pipeline?
+      result[name][:_source] = {
+        excludes: ['attachment']
+      }
+      result[name][:properties][:attachment] = {
+        type: 'attachment',
+      }
+    end
+  end
+
   return result if es_type_in_mapping?
 
   result[name]

spec/fixtures/upload/test.rtf

@@ -0,0 +1,10 @@
+{\rtf1\ansi\ansicpg1252\cocoartf1671\cocoasubrtf600
+{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;}
+{\colortbl;\red255\green255\blue255;}
+{\*\expandedcolortbl;;}
+\paperw11900\paperh16840\margl1440\margr1440\vieww10800\viewh8400\viewkind0
+\pard\tx566\tx1133\tx1700\tx2267\tx2834\tx3401\tx3968\tx4535\tx5102\tx5669\tx6236\tx6803\pardirnatural\partightenfactor0
+
+\f0\fs24 \cf0 Hello world from Rich Text 
+\f1\b RTF
+\f0\b0  document}

spec/models/knowledge_base/answer/translation/search_with_attachment_spec.rb

@@ -0,0 +1,24 @@
+require 'rails_helper'
+require 'models/concerns/checks_kb_client_notification_examples'
+require 'models/contexts/factory_context'
+
+RSpec.describe KnowledgeBase::Answer::Translation, type: :model, current_user_id: 1, searchindex: 1 do
+  include_context 'basic Knowledge Base'
+
+  let(:user)     { create(:admin_user) }
+  let(:filename) { 'test.rtf' }
+  let(:query)    { 'RTF document' }
+
+  context 'search with attachment' do
+    before do
+      configure_elasticsearch(required: true, rebuild: true) do
+        published_answer.add_attachment File.open "spec/fixtures/upload/#{filename}"
+      end
+    end
+
+    it do
+      expect(described_class.search(query: query, current_user: user))
+        .to include published_answer.translations.first
+    end
+  end
+end