Added support for elasticsearch 6 and 7. Fixes #1688.

.gitlab-ci.yml

@@ -214,16 +214,9 @@ test:integration:clearbit:
 
 ### Elasticsearch
 
-test:integration:es:
+.script_integration_es_template: &script_integration_es_definition
   <<: *base_env
   stage: test
-  services:
-    - name: registry.znuny.com/docker/zammad-mysql:latest
-      alias: mysql
-    - name: registry.znuny.com/docker/zammad-postgresql:latest
-      alias: postgresql
-    - name: registry.znuny.com/docker/zammad-elasticsearch:5.6
-      alias: elasticsearch
   variables:
     RAILS_ENV: "test"
     ES_INDEX_RAND: "true"
@@ -235,6 +228,36 @@ test:integration:es:
     - bundle exec rspec --tag searchindex
     - bundle exec rails test test/integration/report_test.rb
 
+test:integration:es:5.6:
+  <<: *script_integration_es_definition
+  services:
+    - name: registry.znuny.com/docker/zammad-mysql:latest
+      alias: mysql
+    - name: registry.znuny.com/docker/zammad-postgresql:latest
+      alias: postgresql
+    - name: registry.znuny.com/docker/zammad-elasticsearch:5.6
+      alias: elasticsearch
+
+test:integration:es:6:
+  <<: *script_integration_es_definition
+  services:
+    - name: registry.znuny.com/docker/zammad-mysql:latest
+      alias: mysql
+    - name: registry.znuny.com/docker/zammad-postgresql:latest
+      alias: postgresql
+    - name: registry.znuny.com/docker/zammad-elasticsearch:6
+      alias: elasticsearch
+
+test:integration:es:7:
+  <<: *script_integration_es_definition
+  services:
+    - name: registry.znuny.com/docker/zammad-mysql:latest
+      alias: mysql
+    - name: registry.znuny.com/docker/zammad-postgresql:latest
+      alias: postgresql
+    - name: registry.znuny.com/docker/zammad-elasticsearch:7
+      alias: elasticsearch
+
 ### Zendesk
 
 test:integration:zendesk:
@@ -320,7 +343,7 @@ browser:build:
       alias: mysql
     - name: registry.znuny.com/docker/zammad-postgresql:latest
       alias: postgresql
-    - name: registry.znuny.com/docker/zammad-elasticsearch:5.6
+    - name: registry.znuny.com/docker/zammad-elasticsearch:stable
       alias: elasticsearch
     - name: docker.io/elgalu/selenium:3.14.0-p17
       alias: selenium

app/controllers/form_controller.rb

@@ -197,12 +197,16 @@ class FormController < ApplicationController
   def limit_reached?
     return false if !SearchIndexBackend.enabled?
 
+    # quote ipv6 ip'
+    remote_ip = request.remote_ip.gsub(':', '\\:')
+
+    # in elasticsearch7 "created_at:>now-1h" is not working. Needed to catch -2h
     form_limit_by_ip_per_hour = Setting.get('form_ticket_create_by_ip_per_hour') || 20
-    result = SearchIndexBackend.search("preferences.form.remote_ip:'#{request.remote_ip}' AND created_at:>now-1h", 'Ticket', limit: form_limit_by_ip_per_hour)
+    result = SearchIndexBackend.search("preferences.form.remote_ip:'#{remote_ip}' AND created_at:>now-2h", 'Ticket', limit: form_limit_by_ip_per_hour)
     raise Exceptions::NotAuthorized if result.count >= form_limit_by_ip_per_hour.to_i
 
     form_limit_by_ip_per_day = Setting.get('form_ticket_create_by_ip_per_day') || 240
-    result = SearchIndexBackend.search("preferences.form.remote_ip:'#{request.remote_ip}' AND created_at:>now-1d", 'Ticket', limit: form_limit_by_ip_per_day)
+    result = SearchIndexBackend.search("preferences.form.remote_ip:'#{remote_ip}' AND created_at:>now-1d", 'Ticket', limit: form_limit_by_ip_per_day)
     raise Exceptions::NotAuthorized if result.count >= form_limit_by_ip_per_day.to_i
 
     form_limit_per_day = Setting.get('form_ticket_create_per_day') || 5000

app/models/concerns/has_search_sortable.rb

@@ -39,7 +39,11 @@ returns
       end
 
       if sort_by.blank?
-        sort_by.push(default)
+        if default.is_a?(Array)
+          sort_by = default
+        else
+          sort_by.push(default)
+        end
       end
 
       sort_by
@@ -77,7 +81,11 @@ returns
       end
 
       if order_by.blank?
-        order_by.push(default)
+        if default.is_a?(Array)
+          order_by = default
+        else
+          order_by.push(default)
+        end
       end
 
       order_by

app/models/organization/search.rb

@@ -72,11 +72,11 @@ returns
         offset = params[:offset] || 0
         current_user = params[:current_user]
 
-        # check sort
-        sort_by = search_get_sort_by(params, 'updated_at')
+        # check sort - positions related to order by
+        sort_by = search_get_sort_by(params, %w[active updated_at])
 
-        # check order
-        order_by = search_get_order_by(params, 'desc')
+        # check order - positions related to sort by
+        order_by = search_get_order_by(params, %w[desc desc])
 
         # enable search only for agents and admins
         return [] if !search_preferences(current_user)

app/models/user/search.rb

@@ -83,11 +83,11 @@ returns
         offset = params[:offset] || 0
         current_user = params[:current_user]
 
-        # check sort
-        sort_by = search_get_sort_by(params, 'updated_at')
+        # check sort - positions related to order by
+        sort_by = search_get_sort_by(params, %w[active updated_at])
 
-        # check order
-        order_by = search_get_order_by(params, 'desc')
+        # check order - positions related to sort by
+        order_by = search_get_order_by(params, %w[desc desc])
 
         # enable search only for agents and admins
         return [] if !search_preferences(current_user)

db/migrate/20190415000001_setting_es_multi_index.rb

@@ -0,0 +1,18 @@
+class SettingEsMultiIndex < ActiveRecord::Migration[5.1]
+  def up
+
+    # return if it's a new setup
+    return if !Setting.find_by(name: 'system_init_done')
+
+    Setting.create_if_not_exists(
+      title:       'Elasticsearch Multi Index',
+      name:        'es_multi_index',
+      area:        'SearchIndex::Elasticsearch',
+      description: 'Define if Elasticsearch is using multiple indexes.',
+      state:       false,
+      preferences: { online_service_disable: true },
+      frontend:    false
+    )
+
+  end
+end

db/seeds/settings.rb

@@ -2849,6 +2849,15 @@ Setting.create_if_not_exists(
   preferences: { online_service_disable: true },
   frontend:    false
 )
+Setting.create_if_not_exists(
+  title:       'Elasticsearch Multi Index',
+  name:        'es_multi_index',
+  area:        'SearchIndex::Elasticsearch',
+  description: 'Define if Elasticsearch is using multiple indexes.',
+  state:       false,
+  preferences: { online_service_disable: true },
+  frontend:    false
+)
 
 Setting.create_if_not_exists(
   title:       'Import Mode',

lib/report/ticket_generic_time.rb

@@ -45,7 +45,7 @@ returns
     end
     selector.merge!(without_merged_tickets) # do not show merged tickets in reports
 
-    result_es = SearchIndexBackend.selectors(['Ticket'], selector, {}, aggs_interval)
+    result_es = SearchIndexBackend.selectors('Ticket', selector, {}, aggs_interval)
     if params[:interval] == 'month'
       stop_interval = 12
     elsif params[:interval] == 'week'
@@ -166,7 +166,7 @@ returns
     end
     selector.merge!(without_merged_tickets) # do not show merged tickets in reports
 
-    result = SearchIndexBackend.selectors(['Ticket'], selector, { limit: limit }, aggs_interval)
+    result = SearchIndexBackend.selectors('Ticket', selector, { limit: limit }, aggs_interval)
     return result if params[:sheet].present?
 
     assets = {}

lib/search_index_backend.rb

@@ -31,8 +31,11 @@ info about used search index machine
       installed_version = response.data.dig('version', 'number')
       raise "Unable to get elasticsearch version from response: #{response.inspect}" if installed_version.blank?
 
-      version_supported = Gem::Version.new(installed_version) < Gem::Version.new('5.7')
-      raise "Version #{installed_version} of configured elasticsearch is not supported" if !version_supported
+      version_supported = Gem::Version.new(installed_version) < Gem::Version.new('8')
+      raise "Version #{installed_version} of configured elasticsearch is not supported." if !version_supported
+
+      version_supported = Gem::Version.new(installed_version) > Gem::Version.new('2.3')
+      raise "Version #{installed_version} of configured elasticsearch is not supported." if !version_supported
 
       return response.data
     end
@@ -130,6 +133,7 @@ create/update/delete index
 
   SearchIndexBackend.index(
     :action => 'create',  # create/update/delete
+    :name   => 'Ticket',
     :data   => {
       :mappings => {
         :Ticket => {
@@ -148,17 +152,14 @@ create/update/delete index
 
   SearchIndexBackend.index(
     :action => 'delete',  # create/update/delete
-    :name   => 'Ticket',    # optional
+    :name   => 'Ticket',
   )
 
-  SearchIndexBackend.index(
-    :action => 'delete',  # create/update/delete
-  )
 =end
 
   def self.index(data)
 
-    url = build_url(data[:name])
+    url = build_url(data[:name], nil, false, false)
     return if url.blank?
 
     if data[:action] && data[:action] == 'delete'
@@ -245,7 +246,7 @@ remove whole data from index
 =end
 
   def self.remove(type, o_id = nil)
-    url = build_url(type, o_id)
+    url = build_url(type, o_id, false, false)
     return if url.blank?
 
     Rails.logger.info "# curl -X DELETE \"#{url}\""
@@ -275,7 +276,7 @@ remove whole data from index
 =begin
 
 @param query   [String]  search query
-@param index   [String, Array<String>, nil] indexes to search in (see search_by_index)
+@param index   [String, Array<String>] indexes to search in (see search_by_index)
 @param options [Hash] search options (see build_query)
 
 @return search result
@@ -305,7 +306,7 @@ remove whole data from index
 
 =end
 
-  def self.search(query, index = nil, options = {})
+  def self.search(query, index, options = {})
     if !index.is_a? Array
       return search_by_index(query, index, options)
     end
@@ -318,29 +319,21 @@ remove whole data from index
 
 =begin
 
-@param query   [String]  search query
-@param index   [String, Array<String>, nil] index name or list of index names. If index is nil or not present will, search will be performed globally
+@param query   [String] search query
+@param index   [String] index name
 @param options [Hash] search options (see build_query)
 
 @return search result
 
 =end
 
-  def self.search_by_index(query, index = nil, options = {})
+  def self.search_by_index(query, index, options = {})
     return [] if query.blank?
 
     url = build_url
     return if url.blank?
 
-    url += if index
-             if index.is_a?(Array)
-               "/#{index.join(',')}/_search"
-             else
-               "/#{index}/_search"
-             end
-           else
-             '/_search'
-           end
+    url += build_search_url(index)
 
     # real search condition
     condition = {
@@ -396,8 +389,8 @@ remove whole data from index
       Rails.logger.info "... #{item['_type']} #{item['_id']}"
 
       output = {
-        id:   item['_id'].to_i,
-        type: item['_type'],
+        id:   item['_id'],
+        type: index,
       }
 
       if options.dig(:highlight_fields_by_indexes, index.to_sym)
@@ -434,15 +427,6 @@ remove whole data from index
       )
     end
 
-    # add sorting by active if active is not part of the query
-    if result.flat_map(&:keys).exclude?(:active)
-      result.unshift(
-        active: {
-          order: 'desc',
-        },
-      )
-    end
-
     result.push('_score')
 
     result
@@ -456,7 +440,7 @@ get count of tickets and tickets which match on selector
 
 example with a simple search:
 
-  result = SearchIndexBackend.selectors('Ticket', { category: { operator: 'is', value: 'aa::ab' } })
+  result = SearchIndexBackend.selectors('Ticket', { 'category' => { 'operator' => 'is', 'value' => 'aa::ab' } })
 
   result = [
     { id: 1, type: 'Ticket' },
@@ -482,7 +466,7 @@ example for aggregations within one year
     current_user: User.find(123),
   }
 
-  result = SearchIndexBackend.selectors('Ticket', { category: { operator: 'is', value: 'aa::ab' } }, options, aggs_interval)
+  result = SearchIndexBackend.selectors('Ticket', { 'category' => { 'operator' => 'is', 'value' => 'aa::ab' } }, options, aggs_interval)
 
   result = {
     hits:{
@@ -509,21 +493,13 @@ example for aggregations within one year
 
 =end
 
-  def self.selectors(index = nil, selectors = nil, options = {}, aggs_interval = nil)
+  def self.selectors(index, selectors = nil, options = {}, aggs_interval = nil)
     raise 'no selectors given' if !selectors
 
-    url = build_url
+    url = build_url(nil, nil, false, false)
     return if url.blank?
 
-    url += if index
-             if index.is_a?(Array)
-               "/#{index.join(',')}/_search"
-             else
-               "/#{index}/_search"
-             end
-           else
-             '/_search'
-           end
+    url += build_search_url(index)
 
     data = selector2query(selectors, options, aggs_interval)
 
@@ -669,9 +645,9 @@ example for aggregations within one year
           t[:range] = {}
           t[:range][key_tmp] = {}
           if data['operator'] == 'before (absolute)'
-            t[:range][key_tmp][:lt] = (data['value']).to_s
+            t[:range][key_tmp][:lt] = (data['value'])
           else
-            t[:range][key_tmp][:gt] = (data['value']).to_s
+            t[:range][key_tmp][:gt] = (data['value'])
           end
           query_must.push t
         else
@@ -745,25 +721,71 @@ return true if backend is configured
     true
   end
 
-  def self.build_url(type = nil, o_id = nil)
+  def self.build_index_name(index)
+    local_index = "#{Setting.get('es_index')}_#{Rails.env}"
+
+    "#{local_index}_#{index.underscore.tr('/', '_')}"
+  end
+
+  def self.build_url(type = nil, o_id = nil, pipeline = true, with_type = true)
     return if !SearchIndexBackend.enabled?
 
+    # for elasticsearch 5.6 and lower
     index = "#{Setting.get('es_index')}_#{Rails.env}"
-    url   = Setting.get('es_url')
-    url = if type
-            url_pipline = Setting.get('es_pipeline')
-            if url_pipline.present?
-              url_pipline = "?pipeline=#{url_pipline}"
-            end
-            if o_id
-              "#{url}/#{index}/#{type}/#{o_id}#{url_pipline}"
+    if Setting.get('es_multi_index') == false
+      url = Setting.get('es_url')
+      url = if type
+              url_pipline = Setting.get('es_pipeline')
+              if url_pipline.present?
+                url_pipline = "?pipeline=#{url_pipline}"
+              end
+              if o_id
+                "#{url}/#{index}/#{type}/#{o_id}#{url_pipline}"
+              else
+                "#{url}/#{index}/#{type}#{url_pipline}"
+              end
             else
-              "#{url}/#{index}/#{type}#{url_pipline}"
+              "#{url}/#{index}"
             end
-          else
-            "#{url}/#{index}"
-          end
-    url
+      return url
+    end
+
+    # for elasticsearch 6.x and higher
+    url = Setting.get('es_url')
+    if pipeline == true
+      url_pipline = Setting.get('es_pipeline')
+      if url_pipline.present?
+        url_pipline = "?pipeline=#{url_pipline}"
+      end
+    end
+    if type
+      index = build_index_name(type)
+      if with_type == false
+        return "#{url}/#{index}"
+      end
+
+      if o_id
+        return "#{url}/#{index}/_doc/#{o_id}#{url_pipline}"
+      end
+
+      return "#{url}/#{index}/_doc#{url_pipline}"
+    end
+    "#{url}/"
+  end
+
+  def self.build_search_url(index)
+
+    # for elasticsearch 5.6 and lower
+    if Setting.get('es_multi_index') == false
+      if index
+        return "/#{index}/_search"
+      end
+
+      return '/_search'
+    end
+
+    # for elasticsearch 6.x and higher
+    "#{build_index_name(index)}/_doc/_search"
   end
 
   def self.humanized_error(verb:, url:, payload: nil, response:)
@@ -833,4 +855,5 @@ return true if backend is configured
 
     data
   end
+
 end

lib/tasks/search_index_es.rake

@@ -3,12 +3,21 @@ require 'rubygems'
 
 namespace :searchindex do
   task :drop, [:opts] => :environment do |_t, _args|
+    print 'drop indexes...'
 
     # drop indexes
-    print 'drop indexes...'
-    SearchIndexBackend.index(
-      action: 'delete',
-    )
+    if es_multi_index?
+      Models.indexable.each do |local_object|
+        SearchIndexBackend.index(
+          action: 'delete',
+          name:   local_object.name,
+        )
+      end
+    else
+      SearchIndexBackend.index(
+        action: 'delete',
+      )
+    end
     puts 'done'
 
     Rake::Task['searchindex:drop_pipeline'].execute
@@ -17,32 +26,40 @@ namespace :searchindex do
   task :create, [:opts] => :environment do |_t, _args|
     print 'create indexes...'
 
-    # es with mapper-attachments plugin
-    info = SearchIndexBackend.info
-    number = nil
-    if info.present?
-      number = info['version']['number'].to_s
+    if es_multi_index?
+      Setting.set('es_multi_index', true)
+    else
+      Setting.set('es_multi_index', false)
     end
 
     settings = {
       'index.mapping.total_fields.limit': 2000,
     }
-    mapping = {}
-    Models.indexable.each do |local_object|
-      mapping.merge!(get_mapping_properties_object(local_object))
-    end
 
     # create indexes
-    SearchIndexBackend.index(
-      action: 'create',
-      data:   {
-        mappings: mapping,
-        settings: settings,
-      }
-    )
-
-    if number.blank? || number =~ /^[2-4]\./ || number =~ /^5\.[0-5]\./
-      Setting.set('es_pipeline', '')
+    if es_multi_index?
+      Models.indexable.each do |local_object|
+        SearchIndexBackend.index(
+          action: 'create',
+          name:   local_object.name,
+          data:   {
+            mappings: get_mapping_properties_object(local_object),
+            settings: settings,
+          }
+        )
+      end
+    else
+      mapping = {}
+      Models.indexable.each do |local_object|
+        mapping.merge!(get_mapping_properties_object(local_object))
+      end
+      SearchIndexBackend.index(
+        action: 'create',
+        data:   {
+          mappings: mapping,
+          settings: settings,
+        }
+      )
     end
 
     puts 'done'
@@ -51,14 +68,10 @@ namespace :searchindex do
   end
 
   task :create_pipeline, [:opts] => :environment do |_t, _args|
-
-    # es with mapper-attachments plugin
-    info = SearchIndexBackend.info
-    number = nil
-    if info.present?
-      number = info['version']['number'].to_s
+    if !es_pipeline?
+      Setting.set('es_pipeline', '')
+      next
     end
-    next if number.blank? || number =~ /^[2-4]\./ || number =~ /^5\.[0-5]\./
 
     # update processors
     pipeline = Setting.get('es_pipeline')
@@ -103,14 +116,7 @@ namespace :searchindex do
   end
 
   task :drop_pipeline, [:opts] => :environment do |_t, _args|
-
-    # es with mapper-attachments plugin
-    info = SearchIndexBackend.info
-    number = nil
-    if info.present?
-      number = info['version']['number'].to_s
-    end
-    next if number.blank? || number =~ /^[2-4]\./ || number =~ /^5\.[0-5]\./
+    next if !es_pipeline?
 
     # update processors
     pipeline = Setting.get('es_pipeline')
@@ -142,11 +148,9 @@ namespace :searchindex do
   end
 
   task :rebuild, [:opts] => :environment do |_t, _args|
-
     Rake::Task['searchindex:drop'].execute
     Rake::Task['searchindex:create'].execute
     Rake::Task['searchindex:reload'].execute
-
   end
 end
 
@@ -172,74 +176,87 @@ mapping = {
 =end
 
 def get_mapping_properties_object(object)
+
+  name = object.name
+  if es_multi_index?
+    name = '_doc'
+  end
   result = {
-    object.name => {
+    name => {
       properties: {}
     }
   }
 
   store_columns = %w[preferences data]
 
+  # for elasticsearch 6.x and later
+  string_type = 'text'
+  string_raw  = { 'type': 'keyword' }
+  boolean_raw = { 'type': 'boolean' }
+
+  # for elasticsearch 5.6 and lower
+  if !es_multi_index?
+    string_type = 'string'
+    string_raw  = { 'type': 'string', 'index': 'not_analyzed' }
+    boolean_raw = { 'type': 'boolean', 'index': 'not_analyzed' }
+  end
+
   object.columns_hash.each do |key, value|
     if value.type == :string && value.limit && value.limit <= 5000 && store_columns.exclude?(key)
-      result[object.name][:properties][key] = {
-        type:   'string',
+      result[name][:properties][key] = {
+        type:   string_type,
         fields: {
-          raw: { 'type': 'string', 'index': 'not_analyzed' }
+          raw: string_raw,
         }
       }
     elsif value.type == :integer
-      result[object.name][:properties][key] = {
+      result[name][:properties][key] = {
         type: 'integer',
       }
     elsif value.type == :datetime
-      result[object.name][:properties][key] = {
+      result[name][:properties][key] = {
         type: 'date',
       }
     elsif value.type == :boolean
-      result[object.name][:properties][key] = {
+      result[name][:properties][key] = {
         type:   'boolean',
         fields: {
-          raw: { 'type': 'boolean', 'index': 'not_analyzed' }
+          raw: boolean_raw,
         }
       }
     elsif value.type == :binary
-      result[object.name][:properties][key] = {
+      result[name][:properties][key] = {
         type: 'binary',
       }
     elsif value.type == :bigint
-      result[object.name][:properties][key] = {
+      result[name][:properties][key] = {
         type: 'long',
       }
     elsif value.type == :decimal
-      result[object.name][:properties][key] = {
+      result[name][:properties][key] = {
         type: 'float',
       }
     elsif value.type == :date
-      result[object.name][:properties][key] = {
+      result[name][:properties][key] = {
         type: 'date',
       }
     end
   end
 
   # es with mapper-attachments plugin
-  info = SearchIndexBackend.info
-  number = nil
-  if info.present?
-    number = info['version']['number'].to_s
-  end
-
   if object.name == 'Ticket'
 
-    result[object.name][:_source] = {
+    # do not server attachments if document is requested
+    result[name][:_source] = {
       excludes: ['article.attachment']
     }
 
-    if number.blank? || number =~ /^[2-4]\./ || number =~ /^5\.[0-5]\./
-      result[object.name][:_source] = {
+    # for elasticsearch 5.5 and lower
+    if !es_pipeline?
+      result[name][:_source] = {
         excludes: ['article.attachment']
       }
-      result[object.name][:properties][:article] = {
+      result[name][:properties][:article] = {
         type:              'nested',
         include_in_parent: true,
         properties:        {
@@ -251,5 +268,45 @@ def get_mapping_properties_object(object)
     end
   end
 
-  result
+  return result if es_type_in_mapping?
+
+  result[name]
+end
+
+# get es version
+def es_version
+  info = SearchIndexBackend.info
+  number = nil
+  if info.present?
+    number = info['version']['number'].to_s
+  end
+  number
+end
+
+# no es_pipeline for elasticsearch 5.5 and lower
+def es_pipeline?
+  number = es_version
+  return false if number.blank?
+  return false if number =~ /^[2-4]\./
+  return false if number =~ /^5\.[0-5]\./
+
+  true
+end
+
+# no mulit index for elasticsearch 5.6 and lower
+def es_multi_index?
+  number = es_version
+  return false if number.blank?
+  return false if number =~ /^[2-5]\./
+
+  true
+end
+
+# no type in mapping
+def es_type_in_mapping?
+  number = es_version
+  return true if number.blank?
+  return true if number =~ /^[2-6]\./
+
+  false
 end