delayed_job.rb 1.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970
  1. # Copyright (C) 2012-2022 Zammad Foundation, https://zammad-foundation.org/
  2. module MonitoringHelper
  3. class HealthChecker
  4. class DelayedJob < Backend
  5. FAILED_JOBS_THRESHOLD = 10
  6. TOTAL_JOBS_THRESHOLD = 8_000
  7. TOTAL_JOBS_TIMEOUT = 15.minutes
  8. def run_health_check
  9. failed_jobs
  10. failed_with_attempts
  11. total_jobs
  12. end
  13. private
  14. def scope
  15. Delayed::Job.where('attempts > 0')
  16. end
  17. def failed_jobs
  18. count_failed_jobs = scope.count
  19. return if count_failed_jobs <= FAILED_JOBS_THRESHOLD
  20. response.issues.push "#{count_failed_jobs} failing background jobs"
  21. end
  22. def failed_with_attempts
  23. scope
  24. .order(:created_at)
  25. .limit(10)
  26. .each_with_object({}) { |elem, memo| map_single_failed_job(elem, memo) }
  27. .sort
  28. .each_with_index do |(job_name, job_data), index|
  29. response.issues.push "Failed to run background job ##{index + 1} '#{job_name}' #{job_data[:count]} time(s) with #{job_data[:attempts]} attempt(s)."
  30. end
  31. end
  32. def map_single_failed_job(job, hash)
  33. job_name = job_name(job)
  34. hash[job_name] ||= {
  35. count: 0,
  36. attempts: 0,
  37. }
  38. hash[job_name][:count] += 1
  39. hash[job_name][:attempts] += job.attempts
  40. end
  41. def job_name(job)
  42. if job.instance_of?(Delayed::Backend::ActiveRecord::Job) && job.payload_object.respond_to?(:job_data)
  43. return job.payload_object.job_data['job_class']
  44. end
  45. job.name
  46. end
  47. def total_jobs
  48. total_jobs = Delayed::Job.where('created_at < ?', TOTAL_JOBS_TIMEOUT.ago).count
  49. return if total_jobs <= TOTAL_JOBS_THRESHOLD
  50. response.issues.push "#{total_jobs} background jobs in queue"
  51. end
  52. end
  53. end
  54. end