Feature/nrmi 59 urn nightly job

app/controllers/admin/urn_lists_controller.rb

@@ -2,15 +2,15 @@ class Admin::UrnListsController < AdminController
   before_action :find_latest_list, only: %i[index download]
 
   def index
-    @urn_lists = UrnList.order(created_at: :desc).all
+    @urn_lists = UrnList.order(created_at: :desc).page(params[:page])
   end
 
   def new
     @urn_list = UrnList.new
   end
 
   def create
-    @urn_list = UrnList.new(urn_list_params)
+    @urn_list = UrnList.new(urn_list_params.merge(source: 'manual_upload'))
 
     if @urn_list.save
       UrnListImporterJob.perform_later(@urn_list)
@@ -33,7 +33,7 @@ def urn_list_params
   end
 
   def find_latest_list
-    @latest_urn_list = UrnList.processed.order(created_at: :desc).first
+    @latest_urn_list = UrnList.where(source: 'manual_upload', aasm_state: 'processed').order(created_at: :desc).first
   end
 
   def s3_client

app/jobs/urn_list_api_sync_job.rb

@@ -0,0 +1,21 @@
+class UrnListApiSyncJob < ApplicationJob
+  def perform
+    urn_list = UrnList.create!(aasm_state: :pending, source: 'api_import')
+
+    rows = UrnLists::ApiClient.new.fetch_rows
+    count = UrnLists::ImportCustomers.new(rows: rows).call
+
+    urn_list.update!(
+      aasm_state: :processed,
+      completed_at: Time.current,
+      processed_count: count
+    )
+  rescue StandardError => e
+    urn_list.update!(
+      aasm_state: :failed,
+      completed_at: Time.current,
+      processed_count: count || 0
+    )
+    raise e
+  end
+end

app/jobs/urn_list_importer_job.rb

@@ -1,5 +1,4 @@
 require 'tempfile'
-require 'csv'
 require 'aws-sdk-s3'
 require 'rubyXL'
 require 'rubyXL/convenience_methods/workbook'
@@ -8,97 +7,59 @@
 class UrnListImporterJob < ApplicationJob
   class AlreadyImported < StandardError; end
 
-  class InvalidFormat < StandardError; end
-
-  REQUIRED_COLUMNS = ['URN', 'CustomerName', 'PostCode', 'Sector'].freeze
-
   discard_on ActiveJob::DeserializationError
   discard_on AlreadyImported
 
-  discard_on InvalidFormat do |job, _error|
-    job.arguments.first.update!(aasm_state: :failed)
-  end
-
   retry_on Aws::S3::Errors::ServiceError
 
+  # rubocop:disable Metrics/AbcSize
   def perform(urn_list)
     raise AlreadyImported unless urn_list.pending?
 
     downloader = AttachedFileDownloader.new(urn_list.excel_file)
     downloader.download!
 
-    convert_to_csv(downloader.temp_file.path)
-
-    customers = customers_from_csv
-
-    soft_delete!(customers)
-    upsert!(customers)
-
-    remove_published_column(urn_list, downloader.temp_file.path)
-
-    urn_list.update!(aasm_state: :processed)
-
-    downloader.temp_file.close
-    downloader.temp_file.unlink
+    rows = UrnLists::ReadExcel.new(file_path: downloader.temp_file.path).call
+    count = UrnLists::ImportCustomers.new(rows: rows).call
+
+    workbook_temp_file = build_workbook_temp_file(urn_list)
+    remove_published_column(urn_list, workbook_temp_file.path)
+
+    urn_list.update!(
+      aasm_state: :processed,
+      completed_at: Time.current,
+      processed_count: count
+    )
+  rescue Aws::S3::Errors::ServiceError
+    raise
+  rescue UrnLists::ReadExcel::InvalidFormat
+    mark_failed!(urn_list)
+    raise
+  rescue StandardError => e
+    mark_failed!(urn_list) if urn_list.persisted? && urn_list.pending?
+    raise e
+  ensure
+    cleanup_downloader_temp_file(downloader&.temp_file)
+    cleanup_downloader_temp_file(workbook_temp_file)
   end
+  # rubocop:enable Metrics/AbcSize
 
   private
 
-  def convert_to_csv(path)
-    command = "in2csv --sheet=\"Customers\" --locale=en_GB --blanks --skipinitialspace #{path}"
-    command += " | csvcut -c 'URN,CustomerName,PostCode,Sector,Published'"
-    command += " > \"#{csv_temp_file.path}\""
-
-    result = Ingest::CommandRunner.new(command).run!
-    raise InvalidFormat if result.stderr.any? { |s| s.include?('Error') }
+  def build_workbook_temp_file(urn_list)
+    file = Tempfile.new(['urn_list_workbook', '.xlsx'])
+    file.binmode
+    file.write(urn_list.excel_file.download)
+    file.flush
+    file.rewind
+    file
   end
 
-  def csv_temp_file
-    @csv_temp_file ||= Tempfile.new('customer')
-  end
-
-  def customers_from_csv
-    customers = []
-
-    CSV.foreach(csv_temp_file, headers: true) do |row|
-      raise InvalidFormat unless (row.headers & REQUIRED_COLUMNS) == REQUIRED_COLUMNS
+  def cleanup_downloader_temp_file(file)
+    return unless file
 
-      customers << Customer.new(
-        name: row['CustomerName'],
-        urn: row['URN'].to_i,
-        postcode: row['PostCode'],
-        sector: (row['Sector'] == 'Central Government' ? :central_government : :wider_public_sector),
-        deleted: false,
-        published: (row['Published'] == 'False' ? false : true)
-      )
-    end
-
-    csv_temp_file.close
-    csv_temp_file.unlink
-
-    customers
-  end
-
-  def upsert!(customers)
-    Customer.transaction do
-      Customer.import(
-        customers,
-        batch_size: 100,
-        on_duplicate_key_update: {
-          conflict_target: [:urn],
-          columns: %i[name postcode sector deleted published]
-        }
-      )
-    end
-  end
-
-  def soft_delete!(customers)
-    existing_urns = Customer.pluck(:urn)
-    importing_urns = customers.map(&:urn)
-
-    urns_to_be_deleted = existing_urns - importing_urns
-
-    Customer.where(urn: urns_to_be_deleted).update(deleted: true)
+    file.close unless file.closed?
+    file.unlink
   end
 
   def remove_published_column(urn_list, path)
@@ -134,4 +95,12 @@ def delete_non_publish_row(worksheet, row_num, row)
     worksheet.delete_row(row_num)
     true
   end
+
+  def mark_failed!(urn_list, processed_count: 0)
+    urn_list.update!(
+      aasm_state: :failed,
+      completed_at: Time.current,
+      processed_count: processed_count
+    )
+  end
 end

app/models/urn_list.rb

@@ -1,6 +1,11 @@
 class UrnList < ApplicationRecord
   include AASM
 
+  enum :source, {
+    manual_upload: 'manual_upload',
+    api_import: 'api_import'
+  }
+
   aasm do
     state :pending, initial: true
     state :processed

app/services/urn_lists/api_client.rb

@@ -0,0 +1,65 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+module UrnLists
+  class ApiClient
+    class ApiError < StandardError; end
+
+    def fetch_rows
+      token = fetch_access_token
+      fetch_urn_list(token)
+    end
+
+    private
+
+    def fetch_access_token
+      uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL'))
+
+      response = Net::HTTP.post_form(uri, {
+                                       grant_type: 'client_credentials',
+        client_id: ENV.fetch('MDM_API_CLIENT_ID'),
+        client_secret: ENV.fetch('MDM_API_CLIENT_SECRET'),
+        scope: ENV.fetch('MDM_API_SCOPE')
+                                     })
+
+      raise ApiError, "Failed to fetch access token: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+      body = JSON.parse(response.body)
+      body.fetch('access_token')
+    end
+
+    def fetch_urn_list(token)
+      base_url = 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/'
+      params = {
+        'api-version' => '2016-10-01',
+        'sp' => '/triggers/manual/run',
+        'sv' => '1.0',
+        'filter' => "Published eq 'True'"
+      }
+
+      uri = URI(base_url)
+      uri.query = URI.encode_www_form(params)
+
+      request = Net::HTTP::Get.new(uri.to_s)
+      request['Authorization'] = "Bearer #{token}"
+      request['Accept'] = 'application/json'
+
+      response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+        http.request(request)
+      end
+
+      raise ApiError, "Failed to fetch URN list: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+      rows = JSON.parse(response.body)
+      validate_rows!(rows)
+      rows
+    end
+
+    def validate_rows!(rows)
+      return if rows.is_a?(Array) && rows.all? { |row| row.is_a?(Hash) }
+
+      raise ApiError, 'Invalid URN list format: expected an array of objects'
+    end
+  end
+end

app/services/urn_lists/import_customers.rb

@@ -0,0 +1,66 @@
+module UrnLists
+  class ImportCustomers
+    def initialize(rows:)
+      @rows = rows
+    end
+
+    def call
+      customers = build_customers
+      soft_delete!(customers)
+      upsert!(customers)
+
+      customers.count
+    end
+
+    private
+
+    attr_reader :rows
+
+    def build_customers
+      rows.map do |row|
+        next row if row.is_a?(Customer)
+
+        Customer.new(
+          name: row['CustomerName'],
+          urn: row['URN'].to_i,
+          postcode: row['PostCode'],
+          sector: normalize_sector(row['Sector']),
+          deleted: false,
+          published: normalize_published(row['Published'])
+        )
+      end
+    end
+
+    def normalize_sector(value)
+      value == 'Central Government' ? :central_government : :wider_public_sector
+    end
+
+    def normalize_published(value)
+      return true if value.nil?
+
+      value != 'False'
+    end
+
+    def upsert!(customers)
+      Customer.transaction do
+        Customer.import(
+          customers,
+          batch_size: 100,
+          on_duplicate_key_update: {
+            conflict_target: [:urn],
+            columns: %i[name postcode sector deleted published]
+          }
+        )
+      end
+    end
+
+    def soft_delete!(customers)
+      existing_urns = Customer.pluck(:urn)
+      importing_urns = customers.map(&:urn)
+
+      urns_to_be_deleted = existing_urns - importing_urns
+
+      Customer.where(urn: urns_to_be_deleted).update(deleted: true)
+    end
+  end
+end