<NME> product_importer.rb
<BEF> # frozen_string_literal: true

# This is the main class for product import. It handles the initial processing of the CSV file,
# and begins the processing of the spreadsheet entries by the other product import classes.
# As spreadsheets can contain any number of entries (1000+), the import is split into smaller chunks
# of 100 items, and processed sequentially over a number of requests to avoid server timeouts.
# The various bits of collated info such as file upload status, per-item errors or user feedback
# on the saving process are made available to the controller through this object.

require 'roo'

module ProductImport
  class ProductImporter
    extend ActiveModel::Naming
    include ActiveModel::Conversion
    include ActiveModel::Validations

    attr_reader :entries, :updated_ids, :import_settings

    def initialize(file, current_user, import_settings = {})
      unless file.is_a?(File)
        errors.add(:importer, I18n.t(:product_importer_file_error))
        return
      end

      @file = file
      @sheet = open_spreadsheet
      @entries = []

      @import_time = Time.zone.now
      @import_settings = import_settings || {}

      @current_user = current_user
      @editable_enterprises = {}
      @inventory_permissions = {}

      @reset_counts = {}
      @updated_ids = []

      init_product_importer if @sheet
    end

    def persisted?
      false # ActiveModel
    end

    def entries?
      @entries.count > 0
    end

    def valid_entries?
      @entries.each do |entry|
        return true if entry.validates_as.present?
      end
      false
    end

    def item_count
      @sheet ? @sheet.last_row - 1 : 0
    end

    def product_field_errors?
      @entries.each do |entry|
        return true if entry.errors.messages.
          value?([I18n.t('admin.product_import.model.not_updatable')])
      end
      false
    end

    def reset_counts
      # Return indexed data about existing product count, reset count, and
      # updates count per enterprise
      @reset_counts.each do |enterprise_id, values|
        values[:updates_count] = 0 if values[:updates_count].blank?

        if values[:updates_count] && values[:existing_products]
          @reset_counts[enterprise_id][:reset_count] =
            values[:existing_products] - values[:updates_count]
        end
      end
      @reset_counts
    end

    def enterprises_index
      @spreadsheet_data.enterprises_index
    end

    def enterprise_products
      @processor&.enterprise_products
    end

    def total_enterprise_products
      @processor.total_enterprise_products
    end

    def entries_json
      entries = {}
      @entries.each do |entry|
        entries[entry.line_number] = {
          attributes: entry.displayable_attributes,
          validates_as: entry.validates_as,
          errors: entry.invalid_attributes,
          product_validations: entry.product_validations
        }
      end
      entries.to_json
    end

    def table_headings
      return unless @entries.first

      @entries.first.displayable_attributes.keys.map do |key|
        I18n.t "admin.product_import.product_headings.#{key}"
      end
    end

    def products_created_count
      @processor.products_created + @processor.variants_created
    end

    def products_updated_count
      @processor.variants_updated
    end

    def inventory_created_count
      @processor.inventory_created
    end

    def inventory_updated_count
      @processor.inventory_updated
    end

    def products_reset_count
      @processor.products_reset_count
    end

    def total_saved_count
      @processor.total_saved_count
    end

    def import_results
      { entries: entries_json, reset_counts: reset_counts }
    end

    def save_results
      {
        results: {
          products_created: products_created_count,
          products_updated: products_updated_count,
          inventory_created: inventory_created_count,
          inventory_updated: inventory_updated_count,
          products_reset: products_reset_count,
        },
        updated_ids: updated_ids,
        errors: errors.full_messages
      }
    end

    def validate_entries
      @validator.validate_all(@entries)
    end

    def save_entries
      validate_entries
      save_all_valid
    end

    def reset_absent(updated_ids)
      @products_created = updated_ids.count
      @updated_ids = updated_ids
      @processor.reset_absent_items
    end

    def permission_by_id?(enterprise_id)
      @editable_enterprises.value?(Integer(enterprise_id))
    end

    private

    def init_product_importer
      init_permissions

      if staged_import?
        build_entries_in_range
      else
        build_entries
      end

      @spreadsheet_data = SpreadsheetData.new(@entries, @import_settings)
      @validator = EntryValidator.new(@current_user, @import_time, @spreadsheet_data,
    end

    def staged_import?
      @import_settings&.key?(:start) && @import_settings.key?(:end)
    end

    def init_permissions

    def staged_import?
      @import_settings&.key?(:start) && @import_settings&.key?(:end)
    end

    def init_permissions
      permissions = OpenFoodNetwork::Permissions.new(@current_user)

      permissions.editable_enterprises.
        order('is_primary_producer ASC, name').
        map { |e| @editable_enterprises[e.name] = e.id }

      @inventory_permissions = permissions.variant_override_enterprises_per_hub
    end

    def open_spreadsheet
      if accepted_mimetype
        Roo::Spreadsheet.open(@file, extension: accepted_mimetype, encoding: Encoding::UTF_8)
      else
        errors.add(:importer, I18n.t(:product_importer_spreadsheet_error))
        delete_uploaded_file
        nil
      end
    end

    def accepted_mimetype
      return false unless ['.csv'].include? File.extname(@file.path)

      @file.path.split('.').last.to_sym
    end

    def headers
      @sheet.row(1)
    end

    def rows
      return [] unless @sheet&.last_row

      (2..@sheet.last_row).map do |i|
        @sheet.row(i)
      end
    rescue ArgumentError => e
      if e.message.include? 'invalid byte sequence'
        errors.add(:importer, I18n.t('admin.product_import.model.encoding_error'))
      else
        errors.add(:importer, I18n.t('admin.product_import.model.unexpected_error',
                                     error_message: e.message))
      end
      []
    rescue CSV::MalformedCSVError => e
      add_malformed_csv_error e.message
      []
    end

    # This error is raised twice because init_product_importer calls both
    # build_entries and buils_all_entries
    def add_malformed_csv_error(error_message)
      unless errors.added?(:importer, I18n.t('admin.product_import.model.malformed_csv',
                                             error_message: error_message))
        errors.add(:importer, I18n.t('admin.product_import.model.malformed_csv',
                                     error_message: error_message))
      end
    end

    def build_entries_in_range
      # In the JS, start and end are calculated like this:
      # start = (batchIndex * $scope.batchSize) + 1
      # end = (batchIndex + 1) * $scope.batchSize
      start_data_index = @import_settings[:start] - 1
      end_data_index = @import_settings[:end] - 1

      data_rows = rows[start_data_index..end_data_index]
      @entries = build_entries_from_rows(data_rows, start_data_index)
    end

    def build_entries
      @entries = build_entries_from_rows(rows)
    end

    def build_all_entries
      build_entries_from_rows(rows)
    end

    def save_all_valid
      @processor.save_all(@entries)
      @processor.reset_absent_items unless staged_import?
      @processor.total_saved_count
    end

    def delete_uploaded_file
      return unless @file.path == Rails.root.join('tmp', 'product_import').to_s

      File.delete(@file)
    end

    def build_entries_from_rows(rows, offset = 0)
      rows.each_with_index.inject([]) do |entries, (row, i)|
        row_data = Hash[[headers, row].transpose]
        entry = SpreadsheetEntry.new(row_data)
        entry.line_number = offset + i + 2
        entries.push entry
      end
    end
  end
end

<MSG> Fix rubocop issue

<DFF> @@ -191,7 +191,7 @@ module ProductImport
     end
 
     def staged_import?
-      @import_settings&.key?(:start) && @import_settings.key?(:end)
+      @import_settings&.key?(:start) && @import_settings&.key?(:end)
     end
 
     def init_permissions

{'commit': "diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml\nindex 4c24cf1d7..49e20f078 100644\n--- a/.rubocop_todo.yml\n+++ b/.rubocop_todo.yml\n@@ -27,14 +27,6 @@ Lint/IneffectiveAccessModifier:\n     - 'lib/open_food_network/feature_toggle.rb'\n     - 'spec/lib/open_food_network/reports/report_spec.rb'\n \n-# Offense count: 1\n-# Cop supports --auto-correct.\n-# Configuration parameters: AllowedMethods.\n-# AllowedMethods: present?, blank?, presence, try, try!\n-Lint/SafeNavigationConsistency:\n-  Exclude:\n-    - 'app/models/product_import/product_importer.rb'\n-\n # Offense count: 3\n Lint/ShadowingOuterLocalVariable:\n   Exclude:\ndiff --git a/app/models/product_import/product_importer.rb b/app/models/product_import/product_importer.rb\nindex 901db7087..75e9d7884 100644\n--- a/app/models/product_import/product_importer.rb\n+++ b/app/models/product_import/product_importer.rb\n@@ -191,7 +191,7 @@ module ProductImport\n     end\n \n     def staged_import?\n-      @import_settings&.key?(:start) && @import_settings.key?(:end)\n+      @import_settings&.key?(:start) && @import_settings&.key?(:end)\n     end\n \n     def init_permissions\n", 'commit_subject': 'Fix rubocop issue', 'repo_name': 'openfoodfoundation/openfoodnetwork', 'committer': {'name': 'Luis Ramos', 'email': '76e2c1b37d8af1e69d2a845c129e65d6effab7d3@gmail.com', 'time_sec': '1592843960', 'tz_offset': '60', 'date': {'seconds': '1592843960'}}, 'commit_tree': 'e545690aeadcd0ea5797f211038c6f816fe577a6', 'commit_message': 'Fix rubocop issue\n', 'language_name': 'HTML', 'license': 'agpl-3.0', 'hunks': '@@ -191,7 +191,7 @@ module ProductImport\n     end\n \n     def staged_import?\n-      @import_settings&.key?(:start) && @import_settings.key?(:end)\n+      @import_settings&.key?(:start) && @import_settings&.key?(:end)\n     end\n \n     def init_permissions\n', 'addition_count': 1, 'deletion_count': 1, 'src_file': 'a/app/models/product_import/product_importer.rb', 'tgt_file': 'b/app/models/product_import/product_importer.rb', 'file_extension': '.rb', 'before_file': '# frozen_string_literal: true\n\n# This is the main class for product import. It handles the initial processing of the CSV file,\n# and begins the processing of the spreadsheet entries by the other product import classes.\n# As spreadsheets can contain any number of entries (1000+), the import is split into smaller chunks\n# of 100 items, and processed sequentially over a number of requests to avoid server timeouts.\n# The various bits of collated info such as file upload status, per-item errors or user feedback\n# on the saving process are made available to the controller through this object.\n\nrequire \'roo\'\n\nmodule ProductImport\n  class ProductImporter\n    extend ActiveModel::Naming\n    include ActiveModel::Conversion\n    include ActiveModel::Validations\n\n    attr_reader :entries, :updated_ids, :import_settings\n\n    def initialize(file, current_user, import_settings = {})\n      unless file.is_a?(File)\n        errors.add(:importer, I18n.t(:product_importer_file_error))\n        return\n      end\n\n      @file = file\n      @sheet = open_spreadsheet\n      @entries = []\n\n      @import_time = Time.zone.now\n      @import_settings = import_settings || {}\n\n      @current_user = current_user\n      @editable_enterprises = {}\n      @inventory_permissions = {}\n\n      @reset_counts = {}\n      @updated_ids = []\n\n      init_product_importer if @sheet\n    end\n\n    def persisted?\n      false # ActiveModel\n    end\n\n    def entries?\n      @entries.count > 0\n    end\n\n    def valid_entries?\n      @entries.each do |entry|\n        return true if entry.validates_as.present?\n      end\n      false\n    end\n\n    def item_count\n      @sheet ? @sheet.last_row - 1 : 0\n    end\n\n    def product_field_errors?\n      @entries.each do |entry|\n        return true if entry.errors.messages.\n          value?([I18n.t(\'admin.product_import.model.not_updatable\')])\n      end\n      false\n    end\n\n    def reset_counts\n      # Return indexed data about existing product count, reset count, and\n      # updates count per enterprise\n      @reset_counts.each do |enterprise_id, values|\n        values[:updates_count] = 0 if values[:updates_count].blank?\n\n        if values[:updates_count] && values[:existing_products]\n          @reset_counts[enterprise_id][:reset_count] =\n            values[:existing_products] - values[:updates_count]\n        end\n      end\n      @reset_counts\n    end\n\n    def enterprises_index\n      @spreadsheet_data.enterprises_index\n    end\n\n    def enterprise_products\n      @processor&.enterprise_products\n    end\n\n    def total_enterprise_products\n      @processor.total_enterprise_products\n    end\n\n    def entries_json\n      entries = {}\n      @entries.each do |entry|\n        entries[entry.line_number] = {\n          attributes: entry.displayable_attributes,\n          validates_as: entry.validates_as,\n          errors: entry.invalid_attributes,\n          product_validations: entry.product_validations\n        }\n      end\n      entries.to_json\n    end\n\n    def table_headings\n      return unless @entries.first\n\n      @entries.first.displayable_attributes.keys.map do |key|\n        I18n.t "admin.product_import.product_headings.#{key}"\n      end\n    end\n\n    def products_created_count\n      @processor.products_created + @processor.variants_created\n    end\n\n    def products_updated_count\n      @processor.variants_updated\n    end\n\n    def inventory_created_count\n      @processor.inventory_created\n    end\n\n    def inventory_updated_count\n      @processor.inventory_updated\n    end\n\n    def products_reset_count\n      @processor.products_reset_count\n    end\n\n    def total_saved_count\n      @processor.total_saved_count\n    end\n\n    def import_results\n      { entries: entries_json, reset_counts: reset_counts }\n    end\n\n    def save_results\n      {\n        results: {\n          products_created: products_created_count,\n          products_updated: products_updated_count,\n          inventory_created: inventory_created_count,\n          inventory_updated: inventory_updated_count,\n          products_reset: products_reset_count,\n        },\n        updated_ids: updated_ids,\n        errors: errors.full_messages\n      }\n    end\n\n    def validate_entries\n      @validator.validate_all(@entries)\n    end\n\n    def save_entries\n      validate_entries\n      save_all_valid\n    end\n\n    def reset_absent(updated_ids)\n      @products_created = updated_ids.count\n      @updated_ids = updated_ids\n      @processor.reset_absent_items\n    end\n\n    def permission_by_id?(enterprise_id)\n      @editable_enterprises.value?(Integer(enterprise_id))\n    end\n\n    private\n\n    def init_product_importer\n      init_permissions\n\n      if staged_import?\n        build_entries_in_range\n      else\n        build_entries\n      end\n\n      @spreadsheet_data = SpreadsheetData.new(@entries, @import_settings)\n      @validator = EntryValidator.new(@current_user, @import_time, @spreadsheet_data,\n    end\n\n    def staged_import?\n      @import_settings&.key?(:start) && @import_settings.key?(:end)\n    end\n\n    def init_permissions\n\n    def staged_import?\n      @import_settings&.key?(:start) && @import_settings&.key?(:end)\n    end\n\n    def init_permissions\n      permissions = OpenFoodNetwork::Permissions.new(@current_user)\n\n      permissions.editable_enterprises.\n        order(\'is_primary_producer ASC, name\').\n        map { |e| @editable_enterprises[e.name] = e.id }\n\n      @inventory_permissions = permissions.variant_override_enterprises_per_hub\n    end\n\n    def open_spreadsheet\n      if accepted_mimetype\n        Roo::Spreadsheet.open(@file, extension: accepted_mimetype, encoding: Encoding::UTF_8)\n      else\n        errors.add(:importer, I18n.t(:product_importer_spreadsheet_error))\n        delete_uploaded_file\n        nil\n      end\n    end\n\n    def accepted_mimetype\n      return false unless [\'.csv\'].include? File.extname(@file.path)\n\n      @file.path.split(\'.\').last.to_sym\n    end\n\n    def headers\n      @sheet.row(1)\n    end\n\n    def rows\n      return [] unless @sheet&.last_row\n\n      (2..@sheet.last_row).map do |i|\n        @sheet.row(i)\n      end\n    rescue ArgumentError => e\n      if e.message.include? \'invalid byte sequence\'\n        errors.add(:importer, I18n.t(\'admin.product_import.model.encoding_error\'))\n      else\n        errors.add(:importer, I18n.t(\'admin.product_import.model.unexpected_error\',\n                                     error_message: e.message))\n      end\n      []\n    rescue CSV::MalformedCSVError => e\n      add_malformed_csv_error e.message\n      []\n    end\n\n    # This error is raised twice because init_product_importer calls both\n    # build_entries and buils_all_entries\n    def add_malformed_csv_error(error_message)\n      unless errors.added?(:importer, I18n.t(\'admin.product_import.model.malformed_csv\',\n                                             error_message: error_message))\n        errors.add(:importer, I18n.t(\'admin.product_import.model.malformed_csv\',\n                                     error_message: error_message))\n      end\n    end\n\n    def build_entries_in_range\n      # In the JS, start and end are calculated like this:\n      # start = (batchIndex * $scope.batchSize) + 1\n      # end = (batchIndex + 1) * $scope.batchSize\n      start_data_index = @import_settings[:start] - 1\n      end_data_index = @import_settings[:end] - 1\n\n      data_rows = rows[start_data_index..end_data_index]\n      @entries = build_entries_from_rows(data_rows, start_data_index)\n    end\n\n    def build_entries\n      @entries = build_entries_from_rows(rows)\n    end\n\n    def build_all_entries\n      build_entries_from_rows(rows)\n    end\n\n    def save_all_valid\n      @processor.save_all(@entries)\n      @processor.reset_absent_items unless staged_import?\n      @processor.total_saved_count\n    end\n\n    def delete_uploaded_file\n      return unless @file.path == Rails.root.join(\'tmp\', \'product_import\').to_s\n\n      File.delete(@file)\n    end\n\n    def build_entries_from_rows(rows, offset = 0)\n      rows.each_with_index.inject([]) do |entries, (row, i)|\n        row_data = Hash[[headers, row].transpose]\n        entry = SpreadsheetEntry.new(row_data)\n        entry.line_number = offset + i + 2\n        entries.push entry\n      end\n    end\n  end\nend\n', 'message': 'Fix rubocop issue\n', 'author_name': 'Luis Ramos', 'lang': 'rb'}