zammad/db/migrate/20250129140759_add_article_language_detection.rb

# Copyright (C) 2012-2025 Zammad Foundation, https://zammad-foundation.org/

class AddArticleLanguageDetection < ActiveRecord::Migration[7.0]
  def change
    # return if it's a new setup
    return if !Setting.exists?(name: 'system_init_done')

    add_ticket_articles_column
    add_setting
  end

  private

  def add_ticket_articles_column
    change_table :ticket_articles do |t|
      t.string :detected_language, limit: 8, null: true
    end

    # NB: Untranslated list of languages for now, supported by CLD library.
    #   https://github.com/mzsanford/cld/blob/master/languages/internal/languages.cc
    detected_language_options = {
      'ab'     => 'Abkhazian',
      'aa'     => 'Afar',
      'af'     => 'Afrikaans',
      'sq'     => 'Albanian',
      'am'     => 'Amharic',
      'ar'     => 'Arabic',
      'hy'     => 'Armenian',
      'as'     => 'Assamese',
      'ay'     => 'Aymara',
      'az'     => 'Azerbaijani',
      'ba'     => 'Bashkir',
      'eu'     => 'Basque',
      'be'     => 'Belarusian',
      'bn'     => 'Bengali',
      'bh'     => 'Bihari',
      'bi'     => 'Bislama',
      'bs'     => 'Bosnian',
      'br'     => 'Breton',
      'bg'     => 'Bulgarian',
      'my'     => 'Burmese',
      'ca'     => 'Catalan',
      'chr'    => 'Cherokee',
      'zh-TW'  => 'Chinese (Taiwan)',
      'zh'     => 'Chinese',
      'co'     => 'Corsican',
      'cpe'    => 'Creoles and Pidgins (English based)',
      'cpf'    => 'Creoles and Pidgins (French based)',
      'crp'    => 'Creoles and Pidgins (Other)',
      'cpp'    => 'Creoles and Pidgins (Portuguese based)',
      'hr'     => 'Croatian',
      'cs'     => 'Czech',
      'da'     => 'Danish',
      'dv'     => 'Dhivehi',
      'nl'     => 'Dutch',
      'dz'     => 'Dzongkha',
      'en'     => 'English',
      'eo'     => 'Esperanto',
      'et'     => 'Estonian',
      'fo'     => 'Faroese',
      'fj'     => 'Fijian',
      'fi'     => 'Finnish',
      'fr'     => 'French',
      'fy'     => 'Frisian',
      'gl'     => 'Galician',
      'lg'     => 'Ganda',
      'ka'     => 'Georgian',
      'de'     => 'German',
      'el'     => 'Greek',
      'kl'     => 'Greenlandic',
      'gn'     => 'Guarani',
      'gu'     => 'Gujarati',
      'ht'     => 'Haitian Creole',
      'ha'     => 'Hausa',
      'he'     => 'Hebrew',
      'hi'     => 'Hindi',
      'hu'     => 'Hungarian',
      'is'     => 'Icelandic',
      'id'     => 'Indonesian',
      'ia'     => 'Interlingua',
      'ie'     => 'Interlingue',
      'iu'     => 'Inuktitut',
      'ik'     => 'Inupiak',
      'ga'     => 'Irish',
      'it'     => 'Italian',
      'ja'     => 'Japanese',
      'jw'     => 'Javanese',
      'kn'     => 'Kannada',
      'ks'     => 'Kashmiri',
      'kk'     => 'Kazakh',
      'kha'    => 'Khasi',
      'km'     => 'Khmer',
      'rw'     => 'Kinyarwanda',
      'ko'     => 'Korean',
      'ku'     => 'Kurdish',
      'ky'     => 'Kyrgyz',
      'lo'     => 'Laothian',
      'la'     => 'Latin',
      'lv'     => 'Latvian',
      'sit-NP' => 'Limbu',
      'ln'     => 'Lingala',
      'lt'     => 'Lithuanian',
      'lb'     => 'Luxembourgish',
      'mk'     => 'Macedonian',
      'mg'     => 'Malagasy',
      'ms'     => 'Malay',
      'ml'     => 'Malayalam',
      'mt'     => 'Maltese',
      'gv'     => 'Manx',
      'mi'     => 'Maori',
      'mr'     => 'Marathi',
      'mo'     => 'Moldavian',
      'mn'     => 'Mongolian',
      'sr-ME'  => 'Montenegrin',
      'na'     => 'Nauru',
      'ne'     => 'Nepali',
      'nn'     => 'Norwegian (Nynorsk)',
      'nb'     => 'Norwegian',
      'oc'     => 'Occitan',
      'or'     => 'Oriya',
      'om'     => 'Oromo',
      'ps'     => 'Pashto',
      'fa'     => 'Persian',
      'pl'     => 'Polish',
      'pt-BR'  => 'Portuguese (Brazil)',
      'pt-PT'  => 'Portuguese (Portugal)',
      'pt'     => 'Portuguese',
      'pa'     => 'Punjabi',
      'qu'     => 'Quechua',
      'rm'     => 'Rhaeto Romance',
      'ro'     => 'Romanian',
      'rn'     => 'Rundi',
      'ru'     => 'Russian',
      'sm'     => 'Samoan',
      'sg'     => 'Sango',
      'sa'     => 'Sanskrit',
      'gd'     => 'Scots Gaelic',
      'sco'    => 'Scots',
      'sr'     => 'Serbian',
      'sh'     => 'Serbo Croatian',
      'st'     => 'Sesotho',
      'sn'     => 'Shona',
      'sd'     => 'Sindhi',
      'si'     => 'Sinhalese',
      'ss'     => 'Siswant',
      'sk'     => 'Slovak',
      'sl'     => 'Slovenian',
      'so'     => 'Somali',
      'es'     => 'Spanish',
      'su'     => 'Sundanese',
      'sw'     => 'Swahili',
      'sv'     => 'Swedish',
      'syr'    => 'Syriac',
      'fil'    => 'Tagalog',
      'tg'     => 'Tajik',
      'ta'     => 'Tamil',
      'tt'     => 'Tatar',
      'te'     => 'Telugu',
      'th'     => 'Thai',
      'bo'     => 'Tibetan',
      'ti'     => 'Tigrinya',
      'to'     => 'Tonga',
      'ts'     => 'Tsonga',
      'tn'     => 'Tswana',
      'tr'     => 'Turkish',
      'tk'     => 'Turkmen',
      'tw'     => 'Twi',
      'ug'     => 'Uighur',
      'uk'     => 'Ukrainian',
      'ur'     => 'Urdu',
      'uz'     => 'Uzbek',
      'vi'     => 'Vietnamese',
      'vo'     => 'Volapuk',
      'cy'     => 'Welsh',
      'wo'     => 'Wolof',
      'xh'     => 'Xhosa',
      'yi'     => 'Yiddish',
      'yo'     => 'Yoruba',
      'za'     => 'Zhuang',
      'zu'     => 'Zulu',
    }.freeze

    ObjectManager::Attribute.add(
      force:         true,
      object:        'TicketArticle',
      name:          'detected_language',
      display:       'Detected Language',
      data_type:     'select',
      data_option:   {
        nulloption:         true,
        multiple:           false,
        null:               true,
        default:            '',
        translate:          false,
        options:            detected_language_options,
        # FIXME: Remove this after the automatic population of historical options for seeded attributes is in place.
        historical_options: detected_language_options,
      },
      editable:      false,
      active:        true,
      screens:       {
        create_middle: {},
        edit:          {
          'ticket.agent' => {
            null: false,
          },
        },
      },
      to_create:     false,
      to_migrate:    false,
      to_delete:     false,
      position:      100,
      updated_by_id: 1,
      created_by_id: 1,
    )

    Ticket::Article.reset_column_information
  end

  def add_setting
    Setting.create_if_not_exists(
      title:       'Language Detection Service',
      name:        'language_detection_article',
      area:        'Ticket::LanguageDetection',
      description: 'Defines the backend service for ticket article language detection.',
      options:     {
        form: [
          {
            display: '',
            null:    true,
            name:    'language_detection_article',
            tag:     'select',
            options: {
              ''    => '-',
              'cld' => 'Compact Language Detector (CLD)',
            },
          },
        ],
      },
      state:       '',
      preferences: {},
      frontend:    false
    )
  end
end