zammad/app/models/concerns/can_csv_import.rb

# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/

require 'csv'

module CanCsvImport
  extend ActiveSupport::Concern

  # methods defined here are going to extend the class, not the instance of it
  class_methods do

=begin

  result = Model.csv_import(
    string: csv_string,
    parse_params: {
      col_sep: ',',
    },
    try: true,
    delete: false,
  )

  result = Model.csv_import(
    file: '/file/location/of/file.csv',
    parse_params: {
      col_sep: ',',
    },
    try: true,
    delete: false,
  )

  result = TextModule.csv_import(
    file: '/Users/me/Downloads/Textbausteine_final.csv',
    parse_params: {
      col_sep: ',',
    },
    try: false,
    delete: false,
  )

returns

  {
    records: [record1, ...]
    try: true, # true|false
    success: true, # true|false
  }

=end

    def csv_import(data)
      try = true
      if data[:try] != 'true' && data[:try] != true
        try = false
      end
      delete = false
      if data[:delete] == true || data[:delete] == 'true'
        delete = true
      end

      errors = []
      if delete == true && @csv_delete_possible != true
        errors.push "Delete is not possible for #{new.class}."
        result = {
          errors: errors,
          try: try,
          result: 'failed',
        }
        return result
      end

      if data[:file].present?
        raise Exceptions::UnprocessableEntity, "No such file '#{data[:file]}'" if !File.exist?(data[:file])

        begin
          file = File.open(data[:file], 'r:UTF-8')
          data[:string] = file.read
        rescue => e
          raise Exceptions::UnprocessableEntity, "Unable to read file '#{data[:file]}': #{e.inspect}"
        end
      end
      if data[:string].blank?
        errors.push "Unable to parse empty file/string for #{new.class}."
        result = {
          errors: errors,
          try: try,
          result: 'failed',
        }
        return result
      end

      rows = ::CSV.parse(data[:string], data[:parse_params])
      header = rows.shift
      if header.blank?
        errors.push "Unable to parse file/string without header for #{new.class}."
        result = {
          errors: errors,
          try: try,
          result: 'failed',
        }
        return result
      end
      header.each do |item|
        if item.respond_to?(:strip!)
          item.strip!
        end
        next if !item.respond_to?(:downcase!)

        item.downcase!
      end

      if rows[0].blank?
        errors.push "No records found in file/string for #{new.class}."
        result = {
          errors: errors,
          try: try,
          result: 'failed',
        }
        return result
      end

      # check if min one lookup key exists
      if header.count == (header - lookup_keys.map(&:to_s)).count
        errors.push "No lookup column like #{lookup_keys.map(&:to_s).join(',')} for #{new.class} found."
        result = {
          errors: errors,
          try: try,
          result: 'failed',
        }
        return result
      end

      # get payload based on csv
      payload = []
      rows.each do |row|
        if row[0].blank? && row[1].blank?
          payload_last = payload.last
          row.each_with_index do |item, count|
            next if item.blank?
            next if header[count].nil?

            if payload_last[header[count].to_sym].class != Array
              payload_last[header[count].to_sym] = [payload_last[header[count].to_sym]]
            end
            payload_last[header[count].to_sym].push item.strip
          end
          next
        end
        attributes = {}
        row.each_with_index do |item, count|
          next if !item
          next if header[count].blank?
          next if @csv_attributes_ignored&.include?(header[count].to_sym)

          attributes[header[count].to_sym] = if item.respond_to?(:strip)
                                               item.strip
                                             else
                                               item
                                             end
        end
        data[:fixed_params]&.each do |key, value|
          attributes[key] = value
        end
        payload.push attributes
      end

      stats = {
        created: 0,
        updated: 0,
      }

      # delete
      if delete == true
        stats[:deleted] = self.count
        if try == false
          destroy_all
        end
      end

      # create or update records
      csv_object_ids_ignored = @csv_object_ids_ignored || []
      records = []
      line_count = 0
      payload.each do |attributes|
        line_count += 1
        record = nil
        lookup_keys.each do |lookup_by|
          next if attributes[lookup_by].blank?

          params = {}
          params[lookup_by] = if %i[email login].include?(lookup_by)
                                attributes[lookup_by].downcase
                              else
                                attributes[lookup_by]
                              end
          record = lookup(params)
          break if record
        end

        if attributes[:id].present? && !record
          errors.push "Line #{line_count}: unknown record with id '#{attributes[:id]}' for #{new.class}."
          next
        end

        if record && csv_object_ids_ignored.include?(record.id)
          errors.push "Line #{line_count}: unable to update record with id '#{attributes[:id]}' for #{new.class}."
          next
        end

        begin
          clean_params = association_name_to_id_convert(attributes)
        rescue => e
          errors.push "Line #{line_count}: #{e.message}"
          next
        end

        # create object
        BulkImportInfo.enable
        Transaction.execute(disable_notification: true, reset_user_id: true) do
          UserInfo.current_user_id = clean_params[:updated_by_id] || clean_params[:created_by_id]
          if !record || delete == true
            stats[:created] += 1
            begin
              csv_verify_attributes(clean_params)
              clean_params = param_cleanup(clean_params)

              if !UserInfo.current_user_id
                clean_params[:created_by_id] = 1
                clean_params[:updated_by_id] = 1
              end
              record = new(clean_params)
              next if try == true

              record.associations_from_param(attributes)
              record.save!
            rescue => e
              errors.push "Line #{line_count}: Unable to create record - #{e.message}"
              next
            end
          else
            stats[:updated] += 1
            next if try == true

            begin
              csv_verify_attributes(clean_params)
              clean_params = param_cleanup(clean_params)

              if !UserInfo.current_user_id
                clean_params[:updated_by_id] = 1
              end

              record.with_lock do
                record.associations_from_param(attributes)
                clean_params.each do |key, value|
                  record[key] = value
                end
                next if !record.changed?

                record.save!
              end
            rescue => e
              errors.push "Line #{line_count}: Unable to update record - #{e.message}"
              next
            end
          end
        end
        BulkImportInfo.disable

        records.push record
      end

      result = 'success'
      if errors.present?
        result = 'failed'
      end

      {
        stats: stats,
        records: records,
        errors: errors,
        try: try,
        result: result,
      }
    end

=begin

verify if attributes are valid, will raise an ArgumentError with "unknown attribute '#{key}' for #{new.class}."

  Model.csv_verify_attributes({'attribute': 'some value'})

=end

    def csv_verify_attributes(clean_params)
      all_clean_attributes = {}
      new.attributes.each_key do |attribute|
        all_clean_attributes[attribute.to_sym] = true
      end
      reflect_on_all_associations.map do |assoc|
        all_clean_attributes[assoc.name.to_sym] = true
        ref = if assoc.name.to_s.end_with?('_id')
                "#{assoc.name}_id"
              else
                "#{assoc.name.to_s.chop}_ids"
              end
        all_clean_attributes[ref.to_sym] = true
      end
      clean_params.each_key do |key|
        next if all_clean_attributes.key?(key.to_sym)

        raise ArgumentError, "unknown attribute '#{key}' for #{new.class}."
      end
      true
    end

=begin

  csv_string = Model.csv_example(
    col_sep: ',',
  )

returns

  csv_string

=end

    def csv_example(params = {})
      header = []
      csv_object_ids_ignored = @csv_object_ids_ignored || []
      records = where.not(id: csv_object_ids_ignored).offset(1).limit(23).to_a
      if records.count < 20
        record_ids = records.pluck(:id).concat(csv_object_ids_ignored)
        local_records = where.not(id: record_ids).limit(20 - records.count)
        records = records.concat(local_records)
      end
      records_attributes_with_association_names = []
      records.each do |record|
        record_attributes_with_association_names = record.attributes_with_association_names
        records_attributes_with_association_names.push record_attributes_with_association_names
        record_attributes_with_association_names.each do |key, value|
          next if value.class == ActiveSupport::HashWithIndifferentAccess
          next if value.class == Hash
          next if @csv_attributes_ignored&.include?(key.to_sym)
          next if key.match?(/_id$/)
          next if key.match?(/_ids$/)
          next if key == 'created_by'
          next if key == 'updated_by'
          next if key == 'created_at'
          next if key == 'updated_at'
          next if header.include?(key)

          header.push key
        end
      end

      rows = []
      records_attributes_with_association_names.each do |record|
        row = []
        rows_to_add = []
        position = -1
        header.each do |key|
          position += 1
          if record[key].class == ActiveSupport::TimeWithZone
            row.push record[key].iso8601
            next
          end
          if record[key].class == Array
            entry_count = -2
            record[key].each do |entry|
              entry_count += 1
              next if entry_count == -1

              if !rows_to_add[entry_count]
                rows_to_add[entry_count] = Array.new(header.count + 1) { '' }
              end
              rows_to_add[entry_count][position] = entry
            end
            record[key] = record[key][0]
          end
          row.push record[key]
        end
        rows.push row
        next if rows_to_add.count.zero?

        rows_to_add.each do |item|
          rows.push item
        end
        rows_to_add = []
      end
      ::CSV.generate(params) do |csv|
        csv << header
        rows.each do |row|
          csv << row
        end
      end
    end

=begin

serve methode to ignore model based on id

class Model < ApplicationModel
  include CanCsvImport
  csv_object_ids_ignored(1, 2, 3)
end

=end

    def csv_object_ids_ignored(*object_ids)
      @csv_object_ids_ignored = object_ids
    end

=begin

serve methode to ignore model attributes

class Model < ApplicationModel
  include CanCsvImport
  csv_attributes_ignored :password,
    :image_source,
    :login_failed,
    :source,
    :image_source,
    :image,
    :authorizations,
    :organizations

end

=end

    def csv_attributes_ignored(*attributes)
      @csv_attributes_ignored = attributes
    end

=begin

serve methode to define if delete option is possible or not

class Model < ApplicationModel
  include CanCsvImport
  csv_delete_possible true

end

=end

    def csv_delete_possible(value)
      @csv_delete_possible = value
    end
  end
end