Maintenance: Allow dumping of thread status by sending SIGWINCH to Zammad services

For details, see: doc/developer_manual/cookbook/how-to-debug-zammad-processes.md

config/puma.rb

@@ -11,9 +11,19 @@ environment ENV.fetch('RAILS_ENV', 'development')
 
 preload_app!
 
+# Teach pumactl to use 'SIGWINCH' instead of 'SIGINFO', because the latter is not available on Linux.
+if defined?(Puma::ControlCLI)
+  # Suppress const redefinition warning (can't use silcence_warnings from Rails here).
+  old_verbose = $VERBOSE
+  $VERBOSE = nil
+  Puma::ControlCLI::CMD_PATH_SIG_MAP = Puma::ControlCLI::CMD_PATH_SIG_MAP.merge({ 'info' => 'SIGWINCH' }).freeze
+  $VERBOSE = old_verbose
+end
+
 begin
   on_booted do
     AppVersion.start_maintenance_thread(process_name: 'puma')
+    Zammad::ProcessDebug.install_thread_status_handler
   end
 rescue NoMethodError
   # Workaround for https://github.com/puma/puma/issues/3356, can be removed after this is
@@ -23,5 +33,6 @@ end
 if worker_count.positive?
   on_worker_boot do
     ActiveRecord::Base.establish_connection
+    Zammad::ProcessDebug.install_thread_status_handler
   end
 end

doc/developer_manual/cookbook/how-to-debug-zammad-processes.md

@@ -0,0 +1,37 @@
+# How to Debug Zammad Processes
+
+Sometimes it can be helpful to understand the internal state of threads
+running in a Zammad process. This can be achieved by sending a `SIGWINCH`
+signal to any Zammad service (railsserver, websocket or background worker)
+process, which will cause the process to print the state of its threads to
+STDOUT.
+
+- Find out process ID of Zammad webserver
+
+```screen
+$ ps
+  PID TTY           TIME CMD
+…
+77926 ttys001    0:03.02 puma 6.5.0 (tcp://localhost:3000) [zammad]
+…
+```
+
+- Send the SIGWINCH signal to that process or any of its forked workers. This doesn't acutally "kill" the process.
+
+```screen
+kill -SIGWINCH 77926
+```
+
+- Observe the log output in the process' STDOUT, e.g. via journalctl
+
+```screen
+PID: 77926 Thread: TID-jqk AR Pool Reaper
+  /path/to/gems/ruby-3.2.4/gems/activerecord-7.2.2.1/lib/active_record/connection_adapters/abstract/connection_pool/reaper.rb:49:in `sleep'
+  /path/to/gems/ruby-3.2.4/gems/activerecord-7.2.2.1/lib/active_record/connection_adapters/abstract/connection_pool/reaper.rb:49:in `block in spawn_thread'
+PID: 77926 Thread: TID-jr4 listen-wait_thread
+  <internal:thread_sync>:18:in `pop'
+  /path/to/rubies/ruby-3.2.4/lib/ruby/3.2.0/forwardable.rb:240:in `pop'
+  /path/to/gems/ruby-3.2.4/gems/listen-3.9.0/lib/listen/event/processor.rb:89:in `_wait_until_events'
+  /path/to/gems/ruby-3.2.4/gems/listen-3.9.0/lib/listen/event/processor.rb:21:in `block in loop_for'
+…
+```

doc/developer_manual/index.md

@@ -21,6 +21,7 @@ Welcome to the developer docs of Zammad. 👋 This is a work in progress, and yo
 # Cookbook / Recipes
 
 - [How to diagnose email bugs](cookbook/how-to-diagnose-email-bugs.md)
+- [How to Debug Zammad Processes](cookbook/how-to-debug-zammad-processes.md)
 - [How to setup S/MIME integration](cookbook/how-to-setup-smime-integration.md)
 - [How to use debuggers with Zammad](cookbook/how-to-use-debuggers.md)
 - [How to test with RSpec / Capybara](cookbook/how-to-test-with-rspec-and-capybara.md)

lib/background_services.rb

@@ -20,6 +20,7 @@ class BackgroundServices
     @threads    = []
     install_signal_trap
     AppVersion.start_maintenance_thread(process_name: 'background-worker')
+    Zammad::ProcessDebug.install_thread_status_handler
   end
 
   def run

lib/websocket_server.rb

@@ -16,6 +16,7 @@ class WebsocketServer
     Rails.configuration.interface = 'websocket'
 
     AppVersion.start_maintenance_thread(process_name: 'websocket-server')
+    Zammad::ProcessDebug.install_thread_status_handler
 
     EventMachine.run do
       EventMachine::WebSocket.start(host: @options[:b], port: @options[:p], secure: @options[:s], tls_options: @options[:tls_options]) do |ws|

lib/zammad/process_debug.rb

@@ -0,0 +1,26 @@
+# Copyright (C) 2012-2025 Zammad Foundation, https://zammad-foundation.org/
+
+module Zammad
+  module ProcessDebug
+
+    def self.dump_thread_status
+      Thread.list.each do |thread|
+        name = "PID: #{Process.pid} Thread: TID-#{thread.object_id.to_s(36)}"
+        name += " #{thread['label']}" if thread['label']
+        name += " #{thread.name}" if thread.respond_to?(:name) && thread.name
+        backtrace = thread.backtrace || ['<no backtrace available>']
+
+        # rubocop:disable Rails/Output
+        puts name
+        puts(backtrace.map { |bt| "  #{bt}" })
+        # rubocop:enable Rails/Output
+      end
+    end
+
+    def self.install_thread_status_handler
+      Signal.trap 'SIGWINCH' do
+        Zammad::ProcessDebug.dump_thread_status
+      end
+    end
+  end
+end

spec/lib/zammad/process_debug_spec.rb

@@ -0,0 +1,17 @@
+# Copyright (C) 2012-2025 Zammad Foundation, https://zammad-foundation.org/
+
+require 'rails_helper'
+
+RSpec.describe Zammad::ProcessDebug, :aggregate_failures do
+  describe '.dump_thread_status' do
+    it 'prints the thread info and stack trace' do
+      output = ''
+      allow(described_class).to receive(:puts) do |message|
+        output += [message].flatten.join("\n")
+      end
+      described_class.dump_thread_status
+      expect(output).to include("PID: #{Process.pid} Thread:")
+      expect(output).to include('block in dump_thread_status')
+    end
+  end
+end