search_index_backend.rb 28 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102
  1. # Copyright (C) 2012-2025 Zammad Foundation, https://zammad-foundation.org/
  2. class SearchIndexBackend
  3. SUPPORTED_ES_VERSION_MINIMUM = '7.8'.freeze
  4. SUPPORTED_ES_VERSION_LESS_THAN = '9'.freeze
  5. =begin
  6. info about used search index machine
  7. SearchIndexBackend.info
  8. =end
  9. def self.info
  10. url = Setting.get('es_url').to_s
  11. return if url.blank?
  12. response = make_request(url)
  13. if response.success?
  14. installed_version = response.data.dig('version', 'number')
  15. raise "Unable to get elasticsearch version from response: #{response.inspect}" if installed_version.blank?
  16. installed_version_parsed = Gem::Version.new(installed_version)
  17. if (installed_version_parsed >= Gem::Version.new(SUPPORTED_ES_VERSION_LESS_THAN)) ||
  18. (installed_version_parsed < Gem::Version.new(SUPPORTED_ES_VERSION_MINIMUM))
  19. raise "Version #{installed_version} of configured elasticsearch is not supported."
  20. end
  21. return response.data
  22. end
  23. raise humanized_error(
  24. verb: 'GET',
  25. url: url,
  26. response: response,
  27. )
  28. end
  29. =begin
  30. update processors
  31. SearchIndexBackend.processors(
  32. _ingest/pipeline/attachment: {
  33. description: 'Extract attachment information from arrays',
  34. processors: [
  35. {
  36. foreach: {
  37. field: 'ticket.articles.attachments',
  38. processor: {
  39. attachment: {
  40. target_field: '_ingest._value.attachment',
  41. field: '_ingest._value.data'
  42. }
  43. }
  44. }
  45. }
  46. ]
  47. }
  48. )
  49. =end
  50. def self.processors(data)
  51. data.each do |key, items|
  52. url = "#{Setting.get('es_url')}/#{key}"
  53. items.each do |item|
  54. if item[:action] == 'delete'
  55. response = make_request(url, method: :delete)
  56. next if response.success?
  57. next if response.code.to_s == '404'
  58. raise humanized_error(
  59. verb: 'DELETE',
  60. url: url,
  61. response: response,
  62. )
  63. end
  64. item.delete(:action)
  65. make_request_and_validate(url, data: item, method: :put)
  66. end
  67. end
  68. true
  69. end
  70. =begin
  71. create/update/delete index
  72. SearchIndexBackend.index(
  73. :action => 'create', # create/update/delete
  74. :name => 'Ticket',
  75. :data => {
  76. :mappings => {
  77. :Ticket => {
  78. :properties => {
  79. :articles => {
  80. :type => 'nested',
  81. :properties => {
  82. 'attachment' => { :type => 'attachment' }
  83. }
  84. }
  85. }
  86. }
  87. }
  88. }
  89. )
  90. SearchIndexBackend.index(
  91. :action => 'delete', # create/update/delete
  92. :name => 'Ticket',
  93. )
  94. =end
  95. def self.index(data)
  96. url = build_url(type: data[:name], with_pipeline: false, with_document_type: false)
  97. return if url.blank?
  98. if data[:action] && data[:action] == 'delete'
  99. return if !SearchIndexBackend.index_exists?(data[:name])
  100. return SearchIndexBackend.remove(data[:name])
  101. end
  102. make_request_and_validate(url, data: data[:data], method: :put)
  103. end
  104. =begin
  105. add new object to search index
  106. SearchIndexBackend.add('Ticket', some_data_object)
  107. =end
  108. def self.add(type, data)
  109. url = build_url(type: type, object_id: data['id'])
  110. return if url.blank?
  111. make_request_and_validate(url, data: data, method: :post)
  112. end
  113. =begin
  114. get object of search index by id
  115. SearchIndexBackend.get('Ticket', 123)
  116. =end
  117. def self.get(type, data)
  118. url = build_url(type: type, object_id: data, with_pipeline: false)
  119. return if url.blank?
  120. make_request(url, method: :get).try(:data)
  121. end
  122. =begin
  123. Check if an index exists.
  124. SearchIndexBackend.index_exists?('Ticket')
  125. =end
  126. def self.index_exists?(type)
  127. url = build_url(type: type, with_pipeline: false, with_document_type: false)
  128. return if url.blank?
  129. response = make_request(url)
  130. return true if response.success?
  131. return true if response.code.to_s != '404'
  132. false
  133. end
  134. =begin
  135. This function updates specifc attributes of an index based on a query.
  136. It should get used in batches to prevent performance issues on entities which have millions of objects in it.
  137. data = {
  138. organization: {
  139. name: "Zammad Foundation"
  140. }
  141. }
  142. where = {
  143. term: {
  144. organization_id: 1
  145. }
  146. }
  147. SearchIndexBackend.update_by_query('Ticket', data, where)
  148. =end
  149. def self.update_by_query(type, data, where)
  150. return if data.blank?
  151. return if where.blank?
  152. url_params = {
  153. conflicts: 'proceed',
  154. slices: 'auto',
  155. max_docs: 1_000,
  156. }
  157. url = build_url(type: type, action: '_update_by_query', with_pipeline: false, with_document_type: false, url_params: url_params)
  158. return if url.blank?
  159. script_list = []
  160. data.each_key do |key|
  161. script_list.push("ctx._source.#{key}=params.#{key}")
  162. end
  163. data = {
  164. script: {
  165. lang: 'painless',
  166. source: script_list.join(';'),
  167. params: data,
  168. },
  169. query: where,
  170. sort: {
  171. id: 'desc',
  172. },
  173. }
  174. response = make_request(url, data: data, method: :post, read_timeout: 10.minutes)
  175. if !response.success?
  176. Rails.logger.error humanized_error(
  177. verb: 'GET',
  178. url: url,
  179. payload: data,
  180. response: response,
  181. )
  182. return []
  183. end
  184. response.data
  185. end
  186. =begin
  187. remove whole data from index
  188. SearchIndexBackend.remove('Ticket', 123)
  189. SearchIndexBackend.remove('Ticket')
  190. =end
  191. def self.remove(type, o_id = nil)
  192. url = if o_id
  193. build_url(type: type, object_id: o_id, with_pipeline: false, with_document_type: true)
  194. else
  195. build_url(type: type, object_id: o_id, with_pipeline: false, with_document_type: false)
  196. end
  197. return if url.blank?
  198. response = make_request(url, method: :delete)
  199. return true if response.success?
  200. return true if response.code.to_s == '400'
  201. humanized_error = humanized_error(
  202. verb: 'DELETE',
  203. url: url,
  204. response: response,
  205. )
  206. Rails.logger.warn "Can't delete index: #{humanized_error}"
  207. false
  208. end
  209. =begin
  210. @param query [String] search query
  211. @param index [String, Array<String>] indexes to search in (see search_by_index)
  212. @param options [Hash] search options (see build_query)
  213. @return search result
  214. @example Sample queries
  215. result = SearchIndexBackend.search('search query', ['User', 'Organization'], limit: limit)
  216. - result = SearchIndexBackend.search('search query', 'User', limit: limit)
  217. result = SearchIndexBackend.search('search query', 'User', limit: limit, sort_by: ['updated_at'], order_by: ['desc'])
  218. result = SearchIndexBackend.search('search query', 'User', limit: limit, sort_by: ['active', updated_at'], order_by: ['desc', 'desc'])
  219. result = [
  220. {
  221. :id => 123,
  222. :type => 'User',
  223. },
  224. {
  225. :id => 125,
  226. :type => 'User',
  227. },
  228. {
  229. :id => 15,
  230. :type => 'Organization',
  231. }
  232. ]
  233. result = SearchIndexBackend.search('Nicole', 'User', only_total_count: true)
  234. { :total_count => 1 }
  235. =end
  236. def self.search(query, index, options = {})
  237. if options.key? :with_total_count
  238. raise 'Option "with_total_count" is not supported by multi-index search. Please use search_by_index instead.' # rubocop:disable Zammad/DetectTranslatableString
  239. end
  240. if !index.is_a? Array
  241. return search_by_index(query, index, options)
  242. end
  243. index
  244. .filter_map { |local_index| search_by_index(query, local_index, options) }
  245. .flatten(1)
  246. end
  247. =begin
  248. @param query [String] search query
  249. @param index [String] index name
  250. @param options [Hash] search options (see build_query)
  251. @return search result
  252. result = SearchIndexBackend.search_by_index('Nicole', 'User', only_total_count: true)
  253. { :total_count => 1 }
  254. result = SearchIndexBackend.search_by_index('Nicole', 'User', with_total_count: true)
  255. {
  256. :total_count => 1,
  257. :object_metadata => [
  258. {
  259. :id => "2",
  260. :type => "User"
  261. }
  262. ]
  263. }
  264. result = SearchIndexBackend.search_by_index('Nicole', 'User')
  265. [
  266. {
  267. :id => "2",
  268. :type => "User"
  269. }
  270. ]
  271. =end
  272. def self.search_by_index(query, index, options = {})
  273. return if query.blank?
  274. action = '_search'
  275. if options[:only_total_count].present?
  276. action = '_count'
  277. end
  278. url = build_url(type: index, action: action, with_pipeline: false, with_document_type: false)
  279. return if url.blank?
  280. # real search condition
  281. condition = {
  282. 'query_string' => {
  283. 'query' => append_wildcard_to_simple_query(query),
  284. 'time_zone' => Setting.get('timezone_default'),
  285. 'default_operator' => 'AND',
  286. 'analyze_wildcard' => true,
  287. }
  288. }
  289. if (fields = options.dig(:query_fields_by_indexes, index.to_sym))
  290. condition['query_string']['fields'] = fields
  291. end
  292. query_data = build_query(index, condition, options)
  293. if (fields = options.dig(:highlight_fields_by_indexes, index.to_sym)) && options[:only_total_count].blank?
  294. fields_for_highlight = fields.index_with { |_elem| {} }
  295. query_data[:highlight] = { fields: fields_for_highlight }
  296. end
  297. if options[:only_total_count].present?
  298. query_data.slice!(:query)
  299. end
  300. response = make_request(url, data: query_data, method: :post)
  301. if options[:only_total_count].present?
  302. return {
  303. total_count: response.data&.dig('count') || 0,
  304. }
  305. end
  306. data = if response.success?
  307. Array.wrap(response.data&.dig('hits', 'hits'))
  308. else
  309. Rails.logger.error humanized_error(
  310. verb: 'GET',
  311. url: url,
  312. payload: query_data,
  313. response: response,
  314. )
  315. []
  316. end
  317. data.map! do |item|
  318. Rails.logger.debug { "... #{item['_type']} #{item['_id']}" }
  319. output = {
  320. id: item['_id'],
  321. type: index,
  322. }
  323. if options.dig(:highlight_fields_by_indexes, index.to_sym)
  324. output[:highlight] = item['highlight']
  325. end
  326. output
  327. end
  328. if options[:with_total_count].present?
  329. return {
  330. total_count: response.data&.dig('hits', 'total', 'value') || 0,
  331. object_metadata: data,
  332. }
  333. end
  334. data
  335. end
  336. def self.search_by_index_sort(index:, sort_by: nil, order_by: nil, fulltext: false)
  337. result = (sort_by || [])
  338. .map(&:to_s)
  339. .each_with_object([])
  340. .with_index do |(elem, memo), idx|
  341. next if elem.blank?
  342. next if order_by&.at(idx).blank?
  343. # for sorting values use .keyword values (no analyzer is used - plain values)
  344. is_keyword = get_mapping_properties_object(Array.wrap(index).first.constantize).dig(:properties, elem, :fields, :keyword, :type) == 'keyword'
  345. if is_keyword
  346. elem += '.keyword'
  347. end
  348. memo.push(
  349. elem => {
  350. order: order_by[idx],
  351. },
  352. )
  353. end
  354. # if we have no fulltext search then the primary default sort is updated at else score
  355. if result.blank? && !fulltext
  356. result.push(
  357. updated_at: {
  358. order: 'desc',
  359. },
  360. )
  361. end
  362. result.push('_score')
  363. result
  364. end
  365. =begin
  366. get count of tickets and tickets which match on selector
  367. result = SearchIndexBackend.selectors(index, selector)
  368. example with a simple search:
  369. result = SearchIndexBackend.selectors('Ticket', { 'category' => { 'operator' => 'is', 'value' => 'aa::ab' } })
  370. result = [
  371. { id: 1, type: 'Ticket' },
  372. { id: 2, type: 'Ticket' },
  373. { id: 3, type: 'Ticket' },
  374. ]
  375. you also can get aggregations
  376. result = SearchIndexBackend.selectors(index, selector, options, aggs_interval)
  377. example for aggregations within one year
  378. aggs_interval = {
  379. from: '2015-01-01',
  380. to: '2015-12-31',
  381. interval: 'month', # year, quarter, month, week, day, hour, minute, second
  382. field: 'created_at',
  383. }
  384. options = {
  385. limit: 123,
  386. current_user: User.find(123),
  387. }
  388. result = SearchIndexBackend.selectors('Ticket', { 'category' => { 'operator' => 'is', 'value' => 'aa::ab' } }, options, aggs_interval)
  389. result = {
  390. hits:{
  391. total:4819,
  392. },
  393. aggregations:{
  394. time_buckets:{
  395. buckets:[
  396. {
  397. key_as_string:"2014-10-01T00:00:00.000Z",
  398. key:1412121600000,
  399. doc_count:420
  400. },
  401. {
  402. key_as_string:"2014-11-01T00:00:00.000Z",
  403. key:1414800000000,
  404. doc_count:561
  405. },
  406. ...
  407. ]
  408. }
  409. }
  410. }
  411. =end
  412. def self.selectors(index, selectors = nil, options = {}, aggs_interval = nil)
  413. raise 'no selectors given' if !selectors
  414. url = build_url(type: index, action: '_search', with_pipeline: false, with_document_type: false)
  415. return if url.blank?
  416. data = selector2query(index, selectors, options, aggs_interval)
  417. response = make_request(url, data: data, method: :post)
  418. with_interval = aggs_interval.present? && aggs_interval[:interval].present?
  419. if !response.success?
  420. # Work around a bug with ES versions <= 8.5.0, where invalid date range conditions caused an error response from the server.
  421. # https://github.com/zammad/zammad/issues/5105, https://github.com/elastic/elasticsearch/issues/88131
  422. # This can probably be removed when the required minimum ES version is >= 8.5.0.
  423. if with_interval && response.code.to_i == 400 && response.body&.include?('illegal_argument_exception')
  424. return fake_empty_es_aggregation_response
  425. end
  426. raise humanized_error(
  427. verb: 'GET',
  428. url: url,
  429. payload: data,
  430. response: response,
  431. )
  432. end
  433. Rails.logger.debug { response.data.to_json }
  434. if !with_interval
  435. object_ids = response.data['hits']['hits'].pluck('_id')
  436. # in lower ES 6 versions, we get total count directly, in higher
  437. # versions we need to pick it from total has
  438. count = response.data['hits']['total']
  439. if response.data['hits']['total'].class != Integer
  440. count = response.data['hits']['total']['value']
  441. end
  442. return {
  443. count: count,
  444. object_ids: object_ids,
  445. }
  446. end
  447. response.data
  448. end
  449. def self.selector2query(index, selector, options, aggs_interval)
  450. Selector::SearchIndex.new(selector: selector, options: options.merge(aggs_interval: aggs_interval), target_class: index.constantize).get
  451. end
  452. =begin
  453. return true if backend is configured
  454. result = SearchIndexBackend.enabled?
  455. =end
  456. def self.enabled?
  457. return false if Setting.get('es_url').blank?
  458. true
  459. end
  460. def self.build_index_name(index = nil)
  461. local_index = "#{Setting.get('es_index')}_#{Rails.env}"
  462. return local_index if index.blank?
  463. "#{local_index}_#{index.underscore.tr('/', '_')}"
  464. end
  465. =begin
  466. generate url for index or document access (only for internal use)
  467. # url to access single document in index (in case with_pipeline or not)
  468. url = SearchIndexBackend.build_url(type: 'User', object_id: 123, with_pipeline: true)
  469. # url to access whole index
  470. url = SearchIndexBackend.build_url(type: 'User')
  471. # url to access document definition in index (only es6 and higher)
  472. url = SearchIndexBackend.build_url(type: 'User', with_pipeline: false, with_document_type: true)
  473. # base url
  474. url = SearchIndexBackend.build_url
  475. =end
  476. def self.build_url(type: nil, action: nil, object_id: nil, with_pipeline: true, with_document_type: true, url_params: {})
  477. return if !SearchIndexBackend.enabled?
  478. # set index
  479. index = build_index_name(type)
  480. # add pipeline if needed
  481. if index && with_pipeline == true
  482. url_pipline = Setting.get('es_pipeline')
  483. if url_pipline.present?
  484. url_params['pipeline'] = url_pipline
  485. end
  486. end
  487. # prepare url params
  488. params_string = ''
  489. if url_params.present?
  490. params_string = "?#{URI.encode_www_form(url_params)}"
  491. end
  492. url = Setting.get('es_url')
  493. return "#{url}#{params_string}" if index.blank?
  494. # add type information
  495. url = "#{url}/#{index}"
  496. # add document type
  497. if with_document_type
  498. url = "#{url}/_doc"
  499. end
  500. # add action
  501. if action
  502. url = "#{url}/#{action}"
  503. end
  504. # add object id
  505. if object_id.present?
  506. url = "#{url}/#{object_id}"
  507. end
  508. "#{url}#{params_string}"
  509. end
  510. def self.humanized_error(verb:, url:, response:, payload: nil)
  511. prefix = "Unable to process #{verb} request to elasticsearch URL '#{url}'."
  512. suffix = "\n\nResponse:\n#{response.inspect}\n\n"
  513. if payload.respond_to?(:to_json)
  514. suffix += "Payload:\n#{payload.to_json}"
  515. suffix += "\n\nPayload size: #{payload.to_json.bytesize / 1024 / 1024}M"
  516. else
  517. suffix += "Payload:\n#{payload.inspect}"
  518. end
  519. message = if response&.error&.match?('Connection refused') # rubocop:disable Zammad/DetectTranslatableString
  520. __("Elasticsearch is not reachable. It's possible that it's not running. Please check whether it is installed.")
  521. elsif url.end_with?('pipeline/zammad-attachment', 'pipeline=zammad-attachment') && response.code == 400
  522. __('The installed attachment plugin could not handle the request payload. Ensure that the correct attachment plugin is installed (ingest-attachment).')
  523. else
  524. __('Check the response and payload for detailed information:')
  525. end
  526. result = "#{prefix} #{message}#{suffix}"
  527. Rails.logger.error result.first(40_000)
  528. result
  529. end
  530. # add * on simple query like "somephrase23"
  531. def self.append_wildcard_to_simple_query(query)
  532. query = query.strip
  533. query += '*' if query.exclude?(':')
  534. query
  535. end
  536. =begin
  537. @param condition [Hash] search condition
  538. @param options [Hash] search options
  539. @option options [Integer] :from
  540. @option options [Integer] :limit
  541. @option options [Hash] :query_extension applied to ElasticSearch query
  542. @option options [Array<String>] :order_by ordering directions, desc or asc
  543. @option options [Array<String>] :sort_by fields to sort by
  544. @option options [Array<String>] :fulltext If no sorting is defined the current fallback is the sorting by updated_at. But for fulltext searches it makes more sense to search by _score as default. This parameter allows to change to the fallback to _score.
  545. =end
  546. DEFAULT_QUERY_OPTIONS = {
  547. from: 0,
  548. limit: 10
  549. }.freeze
  550. def self.build_query(index, condition, options = {})
  551. options[:from] = options[:from].presence || options[:offset].presence
  552. options = DEFAULT_QUERY_OPTIONS.merge(options.compact_blank.deep_symbolize_keys)
  553. data = {
  554. from: options[:from],
  555. size: options[:limit],
  556. sort: search_by_index_sort(index: index, sort_by: options[:sort_by], order_by: options[:order_by], fulltext: options[:fulltext]),
  557. query: {
  558. bool: {
  559. must: [],
  560. must_not: [],
  561. }
  562. }
  563. }
  564. if (extension = options[:query_extension])
  565. data[:query].deep_merge! extension.deep_dup
  566. end
  567. data[:query][:bool][:must].push condition
  568. if options[:ids].present?
  569. data[:query][:bool][:must].push({ ids: { values: options[:ids] } })
  570. end
  571. if options[:condition].present?
  572. selector_query = SearchIndexBackend.selector2query(index, options[:condition], {}, nil)
  573. data[:query][:bool][:must] += Array.wrap(selector_query[:query][:bool][:must])
  574. data[:query][:bool][:must_not] += Array.wrap(selector_query[:query][:bool][:must_not])
  575. end
  576. data
  577. end
  578. =begin
  579. refreshes all indexes to make previous request data visible in future requests
  580. SearchIndexBackend.refresh
  581. =end
  582. def self.refresh
  583. return if !enabled?
  584. url = "#{Setting.get('es_url')}/_all/_refresh"
  585. make_request_and_validate(url, method: :post)
  586. end
  587. =begin
  588. helper method for making HTTP calls
  589. @param url [String] url
  590. @option params [Hash] :data is a payload hash
  591. @option params [Symbol] :method is a HTTP method
  592. @option params [Integer] :open_timeout is HTTP request open timeout
  593. @option params [Integer] :read_timeout is HTTP request read timeout
  594. @return UserAgent response
  595. =end
  596. def self.make_request(url, data: {}, method: :get, open_timeout: 8, read_timeout: 180)
  597. Rails.logger.debug { "# curl -X #{method} \"#{url}\" " }
  598. Rails.logger.debug { "-d '#{data.to_json}'" } if data.present?
  599. options = {
  600. json: true,
  601. open_timeout: open_timeout,
  602. read_timeout: read_timeout,
  603. total_timeout: (open_timeout + read_timeout + 60),
  604. open_socket_tries: 3,
  605. user: Setting.get('es_user'),
  606. password: Setting.get('es_password'),
  607. verify_ssl: Setting.get('es_ssl_verify'),
  608. }
  609. response = UserAgent.send(method, url, data, options)
  610. Rails.logger.debug { "# #{response.code}" }
  611. response
  612. end
  613. =begin
  614. helper method for making HTTP calls and raising error if response was not success
  615. @param url [String] url
  616. @option args [Hash] see {make_request}
  617. @return [Boolean] always returns true. Raises error if something went wrong.
  618. =end
  619. def self.make_request_and_validate(url, **args)
  620. response = make_request(url, **args)
  621. return true if response.success?
  622. raise humanized_error(
  623. verb: args[:method],
  624. url: url,
  625. payload: args[:data],
  626. response: response
  627. )
  628. end
  629. =begin
  630. This function will return a index mapping based on the
  631. attributes of the database table of the existing object.
  632. mapping = SearchIndexBackend.get_mapping_properties_object(Ticket)
  633. Returns:
  634. mapping = {
  635. User: {
  636. properties: {
  637. firstname: {
  638. type: 'keyword',
  639. },
  640. }
  641. }
  642. }
  643. =end
  644. def self.get_mapping_properties_object(object)
  645. result = {
  646. properties: {}
  647. }
  648. store_columns = %w[preferences data condition condition_selected condition_saved perform options view order match timeplan]
  649. # for elasticsearch 6.x and later
  650. string_type = 'text'
  651. string_raw = { type: 'keyword', ignore_above: 5012 }
  652. boolean_raw = { type: 'boolean' }
  653. object.columns_hash.each do |key, value|
  654. if value.type == :string && value.limit && value.limit <= 5000 && store_columns.exclude?(key)
  655. result[:properties][key] = {
  656. type: string_type,
  657. fields: {
  658. keyword: string_raw,
  659. }
  660. }
  661. elsif value.type == :integer
  662. result[:properties][key] = {
  663. type: 'integer',
  664. }
  665. elsif value.type == :datetime || value.type == :date # rubocop:disable Style/MultipleComparison
  666. result[:properties][key] = {
  667. type: 'date',
  668. }
  669. elsif value.type == :boolean
  670. result[:properties][key] = {
  671. type: 'boolean',
  672. fields: {
  673. keyword: boolean_raw,
  674. }
  675. }
  676. elsif value.type == :binary
  677. result[:properties][key] = {
  678. type: 'binary',
  679. }
  680. elsif value.type == :bigint
  681. result[:properties][key] = {
  682. type: 'long',
  683. }
  684. elsif value.type == :decimal
  685. result[:properties][key] = {
  686. type: 'float',
  687. }
  688. elsif value.type == :jsonb
  689. result[:properties][key] = {
  690. properties: {
  691. label: {
  692. type: string_type,
  693. fields: {
  694. keyword: string_raw
  695. }
  696. },
  697. value: {
  698. type: string_type,
  699. fields: {
  700. keyword: string_raw
  701. }
  702. }
  703. }
  704. }
  705. end
  706. end
  707. case object.name
  708. when 'Ticket'
  709. result[:properties][:article] = {
  710. type: 'nested',
  711. include_in_parent: true,
  712. }
  713. end
  714. result
  715. end
  716. # get es version
  717. def self.version
  718. @version ||= SearchIndexBackend.info&.dig('version', 'number')
  719. end
  720. def self.configured?
  721. Setting.get('es_url').present?
  722. end
  723. def self.model_indexable?(model_name)
  724. Models.indexable.any? { |m| m.name == model_name }
  725. end
  726. def self.default_model_settings
  727. {
  728. 'index.mapping.total_fields.limit' => 2000,
  729. }
  730. end
  731. def self.model_settings(model)
  732. settings = Setting.get('es_model_settings')[model.name] || {}
  733. default_model_settings.merge(settings)
  734. end
  735. def self.all_settings
  736. Models.indexable.each_with_object({}).to_h { |m| [m.name, model_settings(m)] }
  737. end
  738. def self.set_setting(model_name, key, value)
  739. raise "It is not possible to configure settings for the non-indexable model '#{model_name}'." if !model_indexable?(model_name)
  740. raise __("The required parameter 'key' is missing.") if key.blank?
  741. raise __("The required parameter 'value' is missing.") if value.blank?
  742. config = Setting.get('es_model_settings')
  743. config[model_name] ||= {}
  744. config[model_name][key] = value
  745. Setting.set('es_model_settings', config)
  746. end
  747. def self.unset_setting(model_name, key)
  748. raise "It is not possible to configure settings for the non-indexable model '#{model_name}'." if !model_indexable?(model_name)
  749. raise __("The required parameter 'key' is missing.") if key.blank?
  750. config = Setting.get('es_model_settings')
  751. config[model_name] ||= {}
  752. config[model_name].delete(key)
  753. Setting.set('es_model_settings', config)
  754. end
  755. def self.create_index(models = Models.indexable)
  756. models.each do |local_object|
  757. SearchIndexBackend.index(
  758. action: 'create',
  759. name: local_object.name,
  760. data: {
  761. mappings: SearchIndexBackend.get_mapping_properties_object(local_object),
  762. settings: model_settings(local_object),
  763. }
  764. )
  765. end
  766. end
  767. def self.drop_index(models = Models.indexable)
  768. models.each do |local_object|
  769. SearchIndexBackend.index(
  770. action: 'delete',
  771. name: local_object.name,
  772. )
  773. end
  774. end
  775. def self.create_object_index(object)
  776. models = Models.indexable.select { |c| c.to_s == object }
  777. create_index(models)
  778. end
  779. def self.drop_object_index(object)
  780. models = Models.indexable.select { |c| c.to_s == object }
  781. drop_index(models)
  782. end
  783. def self.pipeline(create: false)
  784. pipeline = Setting.get('es_pipeline')
  785. if create && pipeline.blank?
  786. pipeline = "zammad#{SecureRandom.uuid}"
  787. Setting.set('es_pipeline', pipeline)
  788. end
  789. pipeline
  790. end
  791. def self.pipeline_settings
  792. {
  793. ignore_failure: true,
  794. ignore_missing: true,
  795. }
  796. end
  797. def self.create_pipeline
  798. SearchIndexBackend.processors(
  799. "_ingest/pipeline/#{pipeline(create: true)}": [
  800. {
  801. action: 'delete',
  802. },
  803. {
  804. action: 'create',
  805. description: __('Extract zammad-attachment information from arrays'),
  806. processors: [
  807. {
  808. foreach: {
  809. field: 'article',
  810. processor: {
  811. foreach: {
  812. field: '_ingest._value.attachment',
  813. processor: {
  814. attachment: {
  815. target_field: '_ingest._value',
  816. field: '_ingest._value._content',
  817. }.merge(pipeline_settings),
  818. }
  819. }.merge(pipeline_settings),
  820. }
  821. }.merge(pipeline_settings),
  822. },
  823. {
  824. foreach: {
  825. field: 'attachment',
  826. processor: {
  827. attachment: {
  828. target_field: '_ingest._value',
  829. field: '_ingest._value._content',
  830. }.merge(pipeline_settings),
  831. }
  832. }.merge(pipeline_settings),
  833. }
  834. ]
  835. }
  836. ]
  837. )
  838. end
  839. def self.drop_pipeline
  840. return if pipeline.blank?
  841. SearchIndexBackend.processors(
  842. "_ingest/pipeline/#{pipeline}": [
  843. {
  844. action: 'delete',
  845. },
  846. ]
  847. )
  848. end
  849. # Simulate an empty response from ES.
  850. def self.fake_empty_es_aggregation_response
  851. {
  852. 'hits' => { 'total' => { 'value' => 0, 'relation' => 'eq' }, 'max_score' => nil, 'hits' => [] },
  853. 'aggregations' => { 'time_buckets' => { 'buckets' => [] } }
  854. }
  855. end
  856. end