From 211e63d4e3a329f3321e03f86342359e7ce29bab Mon Sep 17 00:00:00 2001 From: Reem Ibrahim Date: Thu, 16 Apr 2026 18:28:27 +0100 Subject: [PATCH 1/6] updated urn import code to allow for nightly api sync as well as manual upload and updated test suite --- app/controllers/admin/urn_lists_controller.rb | 4 +- app/jobs/urn_list_api_sync_job.rb | 21 ++ app/jobs/urn_list_importer_job.rb | 115 +++---- app/models/urn_list.rb | 5 + app/services/urn_lists/api_client.rb | 65 ++++ app/services/urn_lists/import_customers.rb | 66 ++++ app/services/urn_lists/read_excel.rb | 66 ++++ app/views/admin/urn_lists/index.html.haml | 4 +- config/sidekiq_schedule.yml | 4 + ...13130608_add_import_fields_to_urn_lists.rb | 10 + db/schema.rb | 147 ++++----- spec/jobs/urn_list_api_sync_job_spec.rb | 77 +++++ spec/jobs/urn_list_importer_job_spec.rb | 301 ++++++++++++++---- spec/services/urn_lists/api_client_spec.rb | 50 +++ .../urn_lists/import_customers_spec.rb | 98 ++++++ spec/services/urn_lists/read_excel_spec.rb | 29 ++ 16 files changed, 845 insertions(+), 217 deletions(-) create mode 100644 app/jobs/urn_list_api_sync_job.rb create mode 100644 app/services/urn_lists/api_client.rb create mode 100644 app/services/urn_lists/import_customers.rb create mode 100644 app/services/urn_lists/read_excel.rb create mode 100644 db/migrate/20260413130608_add_import_fields_to_urn_lists.rb create mode 100644 spec/jobs/urn_list_api_sync_job_spec.rb create mode 100644 spec/services/urn_lists/api_client_spec.rb create mode 100644 spec/services/urn_lists/import_customers_spec.rb create mode 100644 spec/services/urn_lists/read_excel_spec.rb diff --git a/app/controllers/admin/urn_lists_controller.rb b/app/controllers/admin/urn_lists_controller.rb index c67b456e6..4453295f1 100644 --- a/app/controllers/admin/urn_lists_controller.rb +++ b/app/controllers/admin/urn_lists_controller.rb @@ -10,7 +10,7 @@ def new end def create - @urn_list = UrnList.new(urn_list_params) + @urn_list = UrnList.new(urn_list_params.merge(source: 'manual_upload')) if @urn_list.save UrnListImporterJob.perform_later(@urn_list) @@ -33,7 +33,7 @@ def urn_list_params end def find_latest_list - @latest_urn_list = UrnList.processed.order(created_at: :desc).first + @latest_urn_list = UrnList.where(source: 'manual_upload', aasm_state: 'processed').order(created_at: :desc).first end def s3_client diff --git a/app/jobs/urn_list_api_sync_job.rb b/app/jobs/urn_list_api_sync_job.rb new file mode 100644 index 000000000..0ceea29f0 --- /dev/null +++ b/app/jobs/urn_list_api_sync_job.rb @@ -0,0 +1,21 @@ +class UrnListApiSyncJob < ApplicationJob + def perform + urn_list = UrnList.create!(aasm_state: :pending, source: 'api_import') + + rows = UrnLists::ApiClient.new.fetch_rows + count = UrnLists::ImportCustomers.new(rows: rows).call + + urn_list.update!( + aasm_state: :processed, + completed_at: Time.current, + processed_count: count + ) + rescue => e + urn_list.update!( + aasm_state: :failed, + completed_at: Time.current, + processed_count: count || 0 + ) + raise e + end +end \ No newline at end of file diff --git a/app/jobs/urn_list_importer_job.rb b/app/jobs/urn_list_importer_job.rb index 6284a6218..72a14514d 100644 --- a/app/jobs/urn_list_importer_job.rb +++ b/app/jobs/urn_list_importer_job.rb @@ -1,5 +1,4 @@ require 'tempfile' -require 'csv' require 'aws-sdk-s3' require 'rubyXL' require 'rubyXL/convenience_methods/workbook' @@ -8,17 +7,9 @@ class UrnListImporterJob < ApplicationJob class AlreadyImported < StandardError; end - class InvalidFormat < StandardError; end - - REQUIRED_COLUMNS = ['URN', 'CustomerName', 'PostCode', 'Sector'].freeze - discard_on ActiveJob::DeserializationError discard_on AlreadyImported - discard_on InvalidFormat do |job, _error| - job.arguments.first.update!(aasm_state: :failed) - end - retry_on Aws::S3::Errors::ServiceError def perform(urn_list) @@ -27,78 +18,46 @@ def perform(urn_list) downloader = AttachedFileDownloader.new(urn_list.excel_file) downloader.download! - convert_to_csv(downloader.temp_file.path) - - customers = customers_from_csv - - soft_delete!(customers) - upsert!(customers) - - remove_published_column(urn_list, downloader.temp_file.path) - - urn_list.update!(aasm_state: :processed) - - downloader.temp_file.close - downloader.temp_file.unlink + rows = UrnLists::ReadExcel.new(file_path: downloader.temp_file.path).call + count = UrnLists::ImportCustomers.new(rows: rows).call + + workbook_temp_file = build_workbook_temp_file(urn_list) + remove_published_column(urn_list, workbook_temp_file.path) + + urn_list.update!( + aasm_state: :processed, + completed_at: Time.current, + processed_count: count + ) + rescue Aws::S3::Errors::ServiceError => e + raise + rescue UrnLists::ReadExcel::InvalidFormat => e + mark_failed!(urn_list) + raise + rescue => e + mark_failed!(urn_list) if urn_list.persisted? && urn_list.pending? + raise e + ensure + cleanup_downloader_temp_file(downloader&.temp_file) + cleanup_downloader_temp_file(workbook_temp_file) end private - def convert_to_csv(path) - command = "in2csv --sheet=\"Customers\" --locale=en_GB --blanks --skipinitialspace #{path}" - command += " | csvcut -c 'URN,CustomerName,PostCode,Sector,Published'" - command += " > \"#{csv_temp_file.path}\"" - - result = Ingest::CommandRunner.new(command).run! - raise InvalidFormat if result.stderr.any? { |s| s.include?('Error') } + def build_workbook_temp_file(urn_list) + file = Tempfile.new(['urn_list_workbook', '.xlsx']) + file.binmode + file.write(urn_list.excel_file.download) + file.flush + file.rewind + file end - def csv_temp_file - @csv_temp_file ||= Tempfile.new('customer') - end - - def customers_from_csv - customers = [] - - CSV.foreach(csv_temp_file, headers: true) do |row| - raise InvalidFormat unless (row.headers & REQUIRED_COLUMNS) == REQUIRED_COLUMNS + def cleanup_downloader_temp_file(file) + return unless file - customers << Customer.new( - name: row['CustomerName'], - urn: row['URN'].to_i, - postcode: row['PostCode'], - sector: (row['Sector'] == 'Central Government' ? :central_government : :wider_public_sector), - deleted: false, - published: (row['Published'] == 'False' ? false : true) - ) - end - - csv_temp_file.close - csv_temp_file.unlink - - customers - end - - def upsert!(customers) - Customer.transaction do - Customer.import( - customers, - batch_size: 100, - on_duplicate_key_update: { - conflict_target: [:urn], - columns: %i[name postcode sector deleted published] - } - ) - end - end - - def soft_delete!(customers) - existing_urns = Customer.pluck(:urn) - importing_urns = customers.map(&:urn) - - urns_to_be_deleted = existing_urns - importing_urns - - Customer.where(urn: urns_to_be_deleted).update(deleted: true) + file.close unless file.closed? + file.unlink end def remove_published_column(urn_list, path) @@ -134,4 +93,12 @@ def delete_non_publish_row(worksheet, row_num, row) worksheet.delete_row(row_num) true end + + def mark_failed!(urn_list, processed_count: 0) + urn_list.update!( + aasm_state: :failed, + completed_at: Time.current, + processed_count: processed_count + ) + end end diff --git a/app/models/urn_list.rb b/app/models/urn_list.rb index f2505ecde..98273e9d6 100644 --- a/app/models/urn_list.rb +++ b/app/models/urn_list.rb @@ -1,6 +1,11 @@ class UrnList < ApplicationRecord include AASM + enum :source, { + manual_upload: 'manual_upload', + api_import: 'api_import' + } + aasm do state :pending, initial: true state :processed diff --git a/app/services/urn_lists/api_client.rb b/app/services/urn_lists/api_client.rb new file mode 100644 index 000000000..38e37431a --- /dev/null +++ b/app/services/urn_lists/api_client.rb @@ -0,0 +1,65 @@ +require 'net/http' +require 'uri' +require 'json' + +module UrnLists + class ApiClient + class ApiError < StandardError; end + + def fetch_rows + token = fetch_access_token + fetch_urn_list(token) + end + + private + + def fetch_access_token + uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL')) + + response = Net::HTTP.post_form(uri, { + grant_type: 'client_credentials', + client_id: ENV.fetch('MDM_API_CLIENT_ID'), + client_secret: ENV.fetch('MDM_API_CLIENT_SECRET'), + scope: ENV.fetch('MDM_API_SCOPE') + }) + + raise ApiError, "Failed to fetch access token: #{response.code}" unless response.is_a?(Net::HTTPSuccess) + + body = JSON.parse(response.body) + body.fetch('access_token') + end + + def fetch_urn_list(token) + base_url = 'https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/' + params = { + 'api-version' => '2016-10-01', + 'sp' => '/triggers/manual/run', + 'sv' => '1.0', + 'filter' => "Published eq 'True'" + } + + uri = URI(base_url) + uri.query = URI.encode_www_form(params) + + request = Net::HTTP::Get.new(uri.to_s) + request['Authorization'] = "Bearer #{token}" + request['Accept'] = 'application/json' + + response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http| + http.request(request) + end + + raise ApiError, "Failed to fetch URN list: #{response.code}" unless response.is_a?(Net::HTTPSuccess) + + rows = JSON.parse(response.body) + validate_rows!(rows) + rows + end + + def validate_rows!(rows) + unless rows.is_a?(Array) && rows.all? { |row| row.is_a?(Hash) } + raise ApiError, "Invalid URN list format: expected an array of objects" + end + end + end +end \ No newline at end of file diff --git a/app/services/urn_lists/import_customers.rb b/app/services/urn_lists/import_customers.rb new file mode 100644 index 000000000..862a12c35 --- /dev/null +++ b/app/services/urn_lists/import_customers.rb @@ -0,0 +1,66 @@ +module UrnLists + class ImportCustomers + def initialize(rows:) + @rows = rows + end + + def call + customers = build_customers + soft_delete!(customers) + upsert!(customers) + + customers.count + end + + private + + attr_reader :rows + + def build_customers + rows.map do |row| + next row if row.is_a?(Customer) + + Customer.new( + name: row['CustomerName'], + urn: row['URN'].to_i, + postcode: row['PostCode'], + sector: normalize_sector(row['Sector']), + deleted: false, + published: normalize_published(row['Published']) + ) + end + end + + def normalize_sector(value) + value == 'Central Government' ? :central_government : :wider_public_sector + end + + def normalize_published(value) + return true if value.nil? + + value == 'False' ? false : true + end + + def upsert!(customers) + Customer.transaction do + Customer.import( + customers, + batch_size: 100, + on_duplicate_key_update: { + conflict_target: [:urn], + columns: %i[name postcode sector deleted published] + } + ) + end + end + + def soft_delete!(customers) + existing_urns = Customer.pluck(:urn) + importing_urns = customers.map(&:urn) + + urns_to_be_deleted = existing_urns - importing_urns + + Customer.where(urn: urns_to_be_deleted).update_all(deleted: true) + end + end +end \ No newline at end of file diff --git a/app/services/urn_lists/read_excel.rb b/app/services/urn_lists/read_excel.rb new file mode 100644 index 000000000..7f4548916 --- /dev/null +++ b/app/services/urn_lists/read_excel.rb @@ -0,0 +1,66 @@ +require 'csv' +require 'tempfile' +require 'shellwords' + +module UrnLists + class ReadExcel + class InvalidFormat < StandardError; end + + REQUIRED_COLUMNS = ['URN', 'CustomerName', 'PostCode', 'Sector', 'Published'].freeze + + def initialize(file_path:) + @file_path = file_path + end + + def call + convert_to_csv + rows_from_csv + ensure + cleanup_csv_temp_file + end + + private + + attr_reader :file_path + + def convert_to_csv + command = "in2csv --sheet=\"Customers\" --locale=en_GB --blanks --skipinitialspace #{file_path}" + command += " | csvcut -c 'URN,CustomerName,PostCode,Sector,Published'" + command += " > \"#{csv_temp_file.path}\"" + + result = Ingest::CommandRunner.new(command).run! + + raise InvalidFormat if result.stderr.any? { |s| s.include?('Error') } + end + + def rows_from_csv + rows = [] + + CSV.foreach(csv_temp_file, headers: true) do |row| + raise InvalidFormat unless (row.headers & REQUIRED_COLUMNS) == REQUIRED_COLUMNS + + rows << Customer.new( + name: row['CustomerName'], + urn: row['URN'].to_i, + postcode: row['PostCode'], + sector: (row['Sector'] == 'Central Government' ? :central_government : :wider_public_sector), + deleted: false, + published: (row['Published'] == 'False' ? false : true) + ) + end + + rows + end + + def csv_temp_file + @csv_temp_file ||= Tempfile.new('customer') + end + + def cleanup_csv_temp_file + return unless @csv_temp_file + + @csv_temp_file.close unless @csv_temp_file.closed? + @csv_temp_file.unlink + end + end +end \ No newline at end of file diff --git a/app/views/admin/urn_lists/index.html.haml b/app/views/admin/urn_lists/index.html.haml index e0a606f23..a057bcf90 100644 --- a/app/views/admin/urn_lists/index.html.haml +++ b/app/views/admin/urn_lists/index.html.haml @@ -19,13 +19,15 @@ %table.govuk-table{:class => 'govuk-!-margin-top-7'} %thead.govuk-table__head %tr.govuk-table__row + %th.govuk-table__header Source %th.govuk-table__header Filename %th.govuk-table__header Upload Date %th.govuk-table__header Status %tbody.govuk-table__body - @urn_lists.each do |list| %tr.govuk-table__row - %td.govuk-table__cell= list.excel_file.filename + %td.govuk-table__cell= list.source.humanize + %td.govuk-table__cell= list.excel_file.filename || '-' %td.govuk-table__cell= list.created_at %td.govuk-table__cell= list.aasm_state diff --git a/config/sidekiq_schedule.yml b/config/sidekiq_schedule.yml index 366788011..12f5ccc78 100644 --- a/config/sidekiq_schedule.yml +++ b/config/sidekiq_schedule.yml @@ -10,3 +10,7 @@ kill_stuck_submissions: cron: '*/5 * * * *' class: KillStuckSubmissionsJob queue: default +urn_list_importer: + cron: '0 19 * * * Europe/London' + class: UrnListApiSyncJob + queue: default diff --git a/db/migrate/20260413130608_add_import_fields_to_urn_lists.rb b/db/migrate/20260413130608_add_import_fields_to_urn_lists.rb new file mode 100644 index 000000000..c96d1d3d5 --- /dev/null +++ b/db/migrate/20260413130608_add_import_fields_to_urn_lists.rb @@ -0,0 +1,10 @@ +class AddImportFieldsToUrnLists < ActiveRecord::Migration[8.1] + def change + add_column :urn_lists, :source, :string, null: false, default: 'manual_upload' + add_column :urn_lists, :completed_at, :datetime + add_column :urn_lists, :processed_count, :integer + + add_index :urn_lists, :source + add_index :urn_lists, :completed_at + end +end diff --git a/db/schema.rb b/db/schema.rb index 9ceb4bca5..d4bca3270 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2025_10_20_131706) do +ActiveRecord::Schema[8.1].define(version: 2026_04_13_130608) do # These are extensions that must be enabled in order to support this database enable_extension "citext" enable_extension "pg_catalog.plpgsql" @@ -18,23 +18,23 @@ enable_extension "uuid-ossp" create_table "active_storage_attachments", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "name", null: false - t.string "record_type", null: false - t.uuid "record_id", null: false t.uuid "blob_id", null: false t.datetime "created_at", precision: nil, null: false + t.string "name", null: false + t.uuid "record_id", null: false + t.string "record_type", null: false t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true end create_table "active_storage_blobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "key", null: false - t.string "filename", null: false - t.string "content_type" - t.text "metadata" t.bigint "byte_size", null: false t.string "checksum" + t.string "content_type" t.datetime "created_at", precision: nil, null: false + t.string "filename", null: false + t.string "key", null: false + t.text "metadata" t.string "service_name", null: false t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true end @@ -47,54 +47,54 @@ create_table "agreement_framework_lots", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "agreement_id", null: false - t.uuid "framework_lot_id", null: false t.datetime "created_at", precision: nil, null: false + t.uuid "framework_lot_id", null: false t.datetime "updated_at", precision: nil, null: false t.index ["agreement_id"], name: "index_agreement_framework_lots_on_agreement_id" t.index ["framework_lot_id"], name: "index_agreement_framework_lots_on_framework_lot_id" end create_table "agreements", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.boolean "active", default: true t.uuid "framework_id", null: false t.uuid "supplier_id", null: false - t.boolean "active", default: true t.index ["active"], name: "index_agreements_on_active" t.index ["framework_id"], name: "index_agreements_on_framework_id" t.index ["supplier_id"], name: "index_agreements_on_supplier_id" end create_table "api_keys", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "key", null: false - t.string "description", null: false t.datetime "created_at", null: false + t.string "description", null: false + t.string "key", null: false t.datetime "updated_at", null: false t.index ["key"], name: "index_api_keys_on_key", unique: true end create_table "bulk_user_uploads", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "aasm_state" t.datetime "created_at", precision: nil, null: false t.datetime "updated_at", precision: nil, null: false - t.string "aasm_state" t.index ["aasm_state"], name: "index_bulk_user_uploads_on_aasm_state" end create_table "customer_effort_scores", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.integer "rating", null: false t.string "comments" t.datetime "created_at", precision: nil + t.integer "rating", null: false t.uuid "user_id" t.index ["user_id"], name: "index_customer_effort_scores_on_user_id" end create_table "customers", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", precision: nil, null: false + t.boolean "deleted", default: false t.string "name", null: false t.string "postcode" - t.integer "urn", null: false + t.boolean "published", default: true t.integer "sector", null: false - t.datetime "created_at", precision: nil, null: false t.datetime "updated_at", precision: nil, null: false - t.boolean "deleted", default: false - t.boolean "published", default: true + t.integer "urn", null: false t.index ["name"], name: "index_customers_on_name" t.index ["postcode"], name: "index_customers_on_postcode" t.index ["sector"], name: "index_customers_on_sector" @@ -108,11 +108,11 @@ end create_table "event_store_events", id: :serial, force: :cascade do |t| + t.datetime "created_at", precision: nil, null: false + t.text "data", null: false t.uuid "event_id", null: false t.string "event_type", null: false t.text "metadata" - t.text "data", null: false - t.datetime "created_at", precision: nil, null: false t.datetime "valid_at" t.index ["created_at"], name: "index_event_store_events_on_created_at" t.index ["event_id"], name: "index_event_store_events_on_event_id", unique: true @@ -120,39 +120,39 @@ end create_table "event_store_events_in_streams", id: :serial, force: :cascade do |t| - t.string "stream", null: false - t.integer "position" - t.uuid "event_id", null: false t.datetime "created_at", precision: nil, null: false + t.uuid "event_id", null: false + t.integer "position" + t.string "stream", null: false t.index ["created_at"], name: "index_event_store_events_in_streams_on_created_at" t.index ["stream", "event_id"], name: "index_event_store_events_in_streams_on_stream_and_event_id", unique: true t.index ["stream", "position"], name: "index_event_store_events_in_streams_on_stream_and_position", unique: true end create_table "framework_lots", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", precision: nil, null: false + t.string "description" t.uuid "framework_id", null: false t.string "number", null: false - t.string "description" - t.datetime "created_at", precision: nil, null: false t.datetime "updated_at", precision: nil, null: false t.index ["framework_id", "number"], name: "index_framework_lots_on_framework_id_and_number", unique: true t.index ["framework_id"], name: "index_framework_lots_on_framework_id" end create_table "frameworks", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "aasm_state", default: "new", null: false + t.text "definition_source", null: false t.string "name" t.string "short_name", null: false - t.text "definition_source", null: false - t.string "aasm_state", default: "new", null: false t.index ["aasm_state"], name: "index_frameworks_on_aasm_state" t.index ["short_name"], name: "index_frameworks_on_short_name", unique: true end create_table "memberships", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "user_id", null: false - t.uuid "supplier_id", null: false t.datetime "created_at", precision: nil, null: false + t.uuid "supplier_id", null: false t.datetime "updated_at", precision: nil, null: false + t.uuid "user_id", null: false t.index ["supplier_id"], name: "index_memberships_on_supplier_id" t.index ["user_id"], name: "index_memberships_on_user_id" end @@ -161,34 +161,34 @@ t.text "notification_message" t.boolean "published", default: false t.datetime "published_at" + t.text "summary", null: false t.datetime "unpublished_at" t.string "user" - t.text "summary", null: false t.index ["published"], name: "index_notifications_on_published" end create_table "release_notes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.text "header", null: false t.text "body", null: false - t.boolean "published", default: false t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.text "header", null: false + t.boolean "published", default: false t.datetime "published_at" + t.datetime "updated_at", null: false end create_table "submission_entries", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "submission_id", null: false - t.uuid "submission_file_id" - t.jsonb "source" - t.jsonb "data" - t.datetime "created_at", precision: nil, null: false - t.datetime "updated_at", precision: nil, null: false t.string "aasm_state" - t.jsonb "validation_errors" + t.datetime "created_at", precision: nil, null: false + t.integer "customer_urn" + t.jsonb "data" t.string "entry_type" - t.decimal "total_value" t.decimal "management_charge", precision: 18, scale: 4 - t.integer "customer_urn" + t.jsonb "source" + t.uuid "submission_file_id" + t.uuid "submission_id", null: false + t.decimal "total_value" + t.datetime "updated_at", precision: nil, null: false + t.jsonb "validation_errors" t.index ["aasm_state"], name: "index_submission_entries_on_aasm_state" t.index ["entry_type"], name: "index_submission_entries_on_entry_type" t.index ["entry_type"], name: "index_submission_entries_on_invoice_entry_type", where: "((entry_type)::text = 'invoice'::text)" @@ -199,18 +199,18 @@ end create_table "submission_entries_stages", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "submission_id", null: false - t.uuid "submission_file_id" - t.jsonb "source" - t.jsonb "data" - t.datetime "created_at", precision: nil, null: false - t.datetime "updated_at", precision: nil, null: false t.string "aasm_state" - t.jsonb "validation_errors" + t.datetime "created_at", precision: nil, null: false + t.integer "customer_urn" + t.jsonb "data" t.string "entry_type" - t.decimal "total_value" t.decimal "management_charge", precision: 18, scale: 4 - t.integer "customer_urn" + t.jsonb "source" + t.uuid "submission_file_id" + t.uuid "submission_id", null: false + t.decimal "total_value" + t.datetime "updated_at", precision: nil, null: false + t.jsonb "validation_errors" t.index ["aasm_state"], name: "index_submission_entries_stages_on_aasm_state" t.index ["entry_type"], name: "index_submission_entries_stage_on_invoice_entry_type", where: "((entry_type)::text = 'invoice'::text)" t.index ["entry_type"], name: "index_submission_entries_stages_on_entry_type" @@ -221,36 +221,36 @@ end create_table "submission_files", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "submission_id", null: false - t.integer "rows" t.datetime "created_at", precision: nil, null: false + t.integer "rows" + t.uuid "submission_id", null: false t.datetime "updated_at", precision: nil, null: false t.index ["submission_id"], name: "index_submission_files_on_submission_id" end create_table "submission_invoices", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "submission_id", null: false - t.string "workday_reference" t.datetime "created_at", precision: nil, null: false - t.datetime "updated_at", precision: nil, null: false t.boolean "reversal", default: false, null: false + t.uuid "submission_id", null: false + t.datetime "updated_at", precision: nil, null: false + t.string "workday_reference" t.index ["submission_id"], name: "index_submission_invoices_on_submission_id" end create_table "submissions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "framework_id", null: false - t.uuid "supplier_id", null: false t.string "aasm_state" - t.uuid "task_id", null: false + t.boolean "cleanup_processed", default: false, null: false t.datetime "created_at", precision: nil, null: false - t.datetime "updated_at", precision: nil, null: false - t.string "purchase_order_number" t.uuid "created_by_id" - t.uuid "submitted_by_id" - t.datetime "submitted_at", precision: nil - t.decimal "management_charge_total", precision: 18, scale: 4 + t.uuid "framework_id", null: false t.decimal "invoice_total", precision: 18, scale: 4 - t.boolean "cleanup_processed", default: false, null: false + t.decimal "management_charge_total", precision: 18, scale: 4 + t.string "purchase_order_number" + t.datetime "submitted_at", precision: nil + t.uuid "submitted_by_id" + t.uuid "supplier_id", null: false + t.uuid "task_id", null: false + t.datetime "updated_at", precision: nil, null: false t.index ["aasm_state"], name: "index_submissions_on_aasm_state" t.index ["cleanup_processed"], name: "index_submissions_on_cleanup_processed" t.index ["created_at"], name: "index_submissions_on_created_at", order: :desc @@ -270,15 +270,15 @@ end create_table "tasks", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "status", null: false t.datetime "created_at", precision: nil, null: false - t.datetime "updated_at", precision: nil, null: false t.string "description" t.date "due_on" + t.uuid "framework_id" t.integer "period_month" t.integer "period_year" + t.string "status", null: false t.uuid "supplier_id" - t.uuid "framework_id" + t.datetime "updated_at", precision: nil, null: false t.index ["framework_id"], name: "index_tasks_on_framework_id" t.index ["status"], name: "index_tasks_on_status" t.index ["supplier_id"], name: "index_tasks_on_supplier_id" @@ -286,17 +286,22 @@ end create_table "urn_lists", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "aasm_state" + t.datetime "completed_at" t.datetime "created_at", precision: nil, null: false + t.integer "processed_count" + t.string "source", default: "manual_upload", null: false t.datetime "updated_at", precision: nil, null: false - t.string "aasm_state" t.index ["aasm_state"], name: "index_urn_lists_on_aasm_state" + t.index ["completed_at"], name: "index_urn_lists_on_completed_at" + t.index ["source"], name: "index_urn_lists_on_source" end create_table "users", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.string "auth_id" - t.string "name" - t.string "email" t.datetime "created_at", precision: nil, null: false + t.string "email" + t.string "name" t.datetime "updated_at", precision: nil, null: false t.index ["auth_id"], name: "index_users_on_auth_id", unique: true end diff --git a/spec/jobs/urn_list_api_sync_job_spec.rb b/spec/jobs/urn_list_api_sync_job_spec.rb new file mode 100644 index 000000000..e50206d40 --- /dev/null +++ b/spec/jobs/urn_list_api_sync_job_spec.rb @@ -0,0 +1,77 @@ +require 'rails_helper' + +RSpec.describe UrnListApiSyncJob do + describe '#perform' do + let(:rows) do + [ + Customer.new( + urn: 10009655, + name: 'Government Commercial Agency', + postcode: 'L3 9PP', + sector: 'central_government', + deleted: false, + published: true + ), + Customer.new( + urn: 10009656, + name: 'Another Organisation', + postcode: 'AB1 2CD', + sector: 'wider_public_sector', + deleted: false, + published: true + ) + ] + end + + let(:api_client_service) { double('UrnLists::ApiClient', fetch_rows: rows) } + let(:import_customers_service) { double('UrnLists::ImportCustomers', call: rows.count) } + + before do + allow(UrnLists::ApiClient).to receive(:new).and_return(api_client_service) + allow(UrnLists::ImportCustomers).to receive(:new).with(rows: rows).and_return(import_customers_service) + end + + it 'creates a pending urn list, imports the rows, and marks it as processed' do + expect { + described_class.perform_now + }.to change(UrnList, :count).by(1) + + expect(api_client_service).to have_received(:fetch_rows) + expect(import_customers_service).to have_received(:call) + + urn_list = UrnList.last + expect(urn_list.source).to eq('api_import') + expect(urn_list).to be_processed + expect(urn_list.processed_count).to eq(rows.count) + expect(urn_list.completed_at).to be_present + end + + it 'marks the urn list as failed when the api call fails' do + allow(api_client_service).to receive(:fetch_rows).and_raise(StandardError.new('token failed')) + + expect { + described_class.perform_now + }.to raise_error(StandardError, 'token failed') + + urn_list = UrnList.last + expect(urn_list.source).to eq('api_import') + expect(urn_list).to be_failed + expect(urn_list.processed_count).to eq(0) + expect(urn_list.completed_at).to be_present + end + + it 'marks the urn list as failed when the import fails after rows are fetched' do + allow(import_customers_service).to receive(:call).and_raise(StandardError.new('import failed')) + + expect { + described_class.perform_now + }.to raise_error(StandardError, 'import failed') + + urn_list = UrnList.last + expect(urn_list.source).to eq('api_import') + expect(urn_list).to be_failed + expect(urn_list.processed_count).to eq(0) + expect(urn_list.completed_at).to be_present + end + end +end \ No newline at end of file diff --git a/spec/jobs/urn_list_importer_job_spec.rb b/spec/jobs/urn_list_importer_job_spec.rb index 9d0af24a5..501588efd 100644 --- a/spec/jobs/urn_list_importer_job_spec.rb +++ b/spec/jobs/urn_list_importer_job_spec.rb @@ -1,114 +1,277 @@ require 'rails_helper' RSpec.describe UrnListImporterJob do - around do |example| - ClimateControl.modify AWS_S3_BUCKET: 'fake', AWS_S3_REGION: 'zz-north-1' do - example.run + describe '#perform' do + let(:urn_list) do + create( + :urn_list, + source: 'manual_upload', + aasm_state: :pending + ) end - end - describe '#perform' do - before { stub_s3_get_object('customers_test.xlsx') } + let(:download_tempfile) do + file = Tempfile.new(['test', '.xlsx']) + file.binmode + file.write('dummy-content') + file.flush + file.rewind + file + end - let(:urn_list) { create(:urn_list, filename: 'customers_test.xlsx') } + let(:workbook_tempfile) do + file = Tempfile.new(['workbook', '.xlsx']) + file.binmode + file.write('dummy-workbook-content') + file.flush + file.rewind + file + end - context 'given a well-formed URN list in Excel format' do - it 'inserts all customers' do - expect { UrnListImporterJob.perform_now(urn_list) } - .to change { Customer.count } - .by(3) + let(:downloader) do + double( + 'AttachedFileDownloader', + download!: true, + temp_file: download_tempfile + ) + end - expect(urn_list).to be_processed + let(:rows) do + [ + Customer.new( + urn: 10009655, + name: 'Crown Commercial Service', + postcode: 'L3 9PP', + sector: 'central_government', + deleted: false, + published: true + ), + Customer.new( + urn: 10009656, + name: 'Another Organisation', + postcode: 'AB1 2CD', + sector: 'wider_public_sector', + deleted: false, + published: true + ) + ] + end + + let(:read_excel_service) { double('UrnLists::ReadExcel', call: rows) } + let(:import_customers_service) { double('UrnLists::ImportCustomers', call: rows.count) } + + before do + allow(AttachedFileDownloader).to receive(:new).with(urn_list.excel_file).and_return(downloader) + allow(UrnLists::ReadExcel).to receive(:new) + .with(file_path: download_tempfile.path) + .and_return(read_excel_service) + allow(UrnLists::ImportCustomers).to receive(:new) + .with(rows: rows) + .and_return(import_customers_service) + allow_any_instance_of(described_class).to receive(:build_workbook_temp_file).with(urn_list).and_return(workbook_tempfile) + allow_any_instance_of(described_class).to receive(:remove_published_column).with(urn_list, workbook_tempfile.path) + allow_any_instance_of(described_class).to receive(:cleanup_downloader_temp_file) + end + + after do + [download_tempfile, workbook_tempfile].each do |file| + next if file.closed? + file.close + file.unlink + rescue StandardError + nil end + end - it 'replaces customers that already exist' do - customer = create(:customer, urn: 10009655, name: 'CCS') + it 'downloads the file, processes it, and updates the URN list record' do + described_class.perform_now(urn_list) - UrnListImporterJob.perform_now(urn_list) + expect(AttachedFileDownloader).to have_received(:new).with(urn_list.excel_file) + expect(downloader).to have_received(:download!) + expect(UrnLists::ReadExcel).to have_received(:new).with(file_path: download_tempfile.path) + expect(read_excel_service).to have_received(:call) + expect(UrnLists::ImportCustomers).to have_received(:new).with(rows: rows) + expect(import_customers_service).to have_received(:call) - customer.reload + urn_list.reload + expect(urn_list).to be_processed + expect(urn_list.completed_at).to be_present + expect(urn_list.processed_count).to eq(rows.count) + end - expect(customer.urn).to eql 10009655 - expect(customer.name).to eql 'Crown Commercial Service' - expect(customer.postcode).to eql 'L3 9PP' - expect(customer.sector).to eql 'central_government' - expect(urn_list).to be_processed - end + it 'raises AlreadyImported if the URN list is not pending' do + urn_list.update!(aasm_state: :processed) - it 'soft deletes obsolete customer records' do - customer = create(:customer, urn: 10009656, name: 'Deleted organisation') + expect(AttachedFileDownloader).not_to receive(:new) - UrnListImporterJob.perform_now(urn_list) + described_class.perform_now(urn_list) - customer.reload + expect(urn_list.reload).to be_processed + end - expect(customer.urn).to eql 10009656 - expect(customer.name).to eql 'Deleted organisation' - expect(customer).to be_deleted - expect(urn_list).to be_processed - end + it 'marks the URN list as failed when the spreadsheet is invalid' do + bad_read_excel = double('UrnLists::ReadExcel') + allow(bad_read_excel).to receive(:call).and_raise(UrnLists::ReadExcel::InvalidFormat) + allow(UrnLists::ReadExcel).to receive(:new).and_return(bad_read_excel) - it 'restores a previously deleted customer now back in the list' do - customer = create(:customer, - urn: 10009655, - deleted: true, - name: 'Crown Commercial Service') + expect { + described_class.perform_now(urn_list) + }.to raise_error(UrnLists::ReadExcel::InvalidFormat) - UrnListImporterJob.perform_now(urn_list) + urn_list.reload + expect(urn_list).to be_failed + expect(urn_list.completed_at).to be_present + expect(urn_list.processed_count).to eq(0) + end - customer.reload + it 'retries the job when a transient S3 error occurs' do + bad_downloader = double('AttachedFileDownloader') + allow(bad_downloader).to receive(:download!).and_raise(Aws::S3::Errors::ServiceError.new(nil, 'S3 error')) + allow(AttachedFileDownloader).to receive(:new).and_return(bad_downloader) - expect(customer.urn).to eql 10009655 - expect(customer.name).to eql 'Crown Commercial Service' - expect(customer).not_to be_deleted - expect(urn_list).to be_processed - end + urn_list.reload + expect(urn_list).to be_pending + expect(urn_list.completed_at).to be_nil end - context 'given a URN list which fails to download' do - before { stub_s3_get_object_with_exception(Timeout::Error) } + it 'marks the URN list as failed for unexpected errors' do + bad_importer = double('UrnLists::ImportCustomers') + allow(bad_importer).to receive(:call).and_raise(StandardError.new('Unexpected error')) + allow(UrnLists::ImportCustomers).to receive(:new).and_return(bad_importer) - let(:urn_list) { create(:urn_list, filename: 'customers_test.xlsx') } + expect { + described_class.perform_now(urn_list) + }.to raise_error(StandardError) - it 'throws an error, and retries the job' do - allow_any_instance_of(AttachedFileDownloader) - .to receive(:download!) - .and_raise(Aws::S3::Errors::NoSuchKey.new('fake', 'fake')) + urn_list.reload + expect(urn_list).to be_failed + expect(urn_list.completed_at).to be_present + expect(urn_list.processed_count).to eq(0) + end + end - expect_any_instance_of(UrnListImporterJob).to receive(:retry_job) + describe 'private helper methods' do + let(:job) { described_class.new } + + describe '#build_workbook_temp_file' do + let(:urn_list) do + create(:urn_list, source: 'manual_upload', aasm_state: :pending) + end - UrnListImporterJob.perform_now(urn_list) + it 'writes the attached workbook to a tempfile' do + tempfile = job.send(:build_workbook_temp_file, urn_list) - expect(urn_list).to be_pending + expect(File.exist?(tempfile.path)).to be true + expect(File.binread(tempfile.path)).to eq(urn_list.excel_file.download) + ensure + tempfile&.close + tempfile&.unlink end end - context 'given a URN list without the required sheet present' do - before { stub_s3_get_object('customers_with_wrong_sheet_name.xlsx') } + describe '#cleanup_downloader_temp_file' do + it 'closes and deletes the tempfile' do + file = Tempfile.new(['cleanup_test', '.xlsx']) + path = file.path - let(:urn_list) { create(:urn_list, filename: 'customers_with_wrong_sheet_name.xlsx') } + job.send(:cleanup_downloader_temp_file, file) - it 'throws an error, and is not retried' do - expect_any_instance_of(UrnListImporterJob).not_to receive(:retry_job) + expect(file.closed?).to be true + expect(File.exist?(path)).to be false + end + + it 'handles nil temp file' do + expect { job.send(:cleanup_downloader_temp_file, nil) }.not_to raise_error + end + end + + describe '#delete_non_publish_row' do + it 'deletes rows where published is false' do + workbook = RubyXL::Workbook.new + worksheet = workbook[0] + + worksheet.add_cell(0, 0, 'URN') + worksheet.add_cell(0, 4, 'Published') + + worksheet.add_cell(1, 0, '10009655') + worksheet.add_cell(1, 4, 'False') + + result = job.send(:delete_non_publish_row, worksheet, 1, worksheet[1]) + + expect(result).to be true + expect(worksheet[1]).to be_nil + end - UrnListImporterJob.perform_now(urn_list) + it 'does not delete rows where published is true' do + workbook = RubyXL::Workbook.new + worksheet = workbook[0] - expect(urn_list).to be_failed + worksheet.add_cell(0, 0, 'URN') + worksheet.add_cell(0, 4, 'Published') + + worksheet.add_cell(1, 0, '10009655') + worksheet.add_cell(1, 4, 'True') + + result = job.send(:delete_non_publish_row, worksheet, 1, worksheet[1]) + + expect(result).to be_nil + expect(worksheet[1]).not_to be_nil end end - context 'given a URN list without the required columns' do - before { stub_s3_get_object('customers_with_missing_columns.xlsx') } + describe '#id_and_remove_non_publish_rows' do + it 'iterates through rows and deletes non-published rows' do + workbook = RubyXL::Workbook.new + worksheet = workbook[0] + + worksheet.add_cell(0, 0, 'URN') + worksheet.add_cell(0, 4, 'Published') + + # Published row + worksheet.add_cell(1, 0, '10009655') + worksheet.add_cell(1, 4, 'True') - let(:urn_list) { create(:urn_list, filename: 'customers_with_missing_columns.xlsx') } + # Non-published row + worksheet.add_cell(2, 0, '10009656') + worksheet.add_cell(2, 4, 'False') - it 'throws an error, and is not retried' do - expect_any_instance_of(UrnListImporterJob).not_to receive(:retry_job) + row_count = worksheet.sheet_data.rows.size - UrnListImporterJob.perform_now(urn_list) + job.send(:id_and_remove_non_publish_rows, worksheet, row_count, 1) - expect(urn_list).to be_failed + remaining_urns = worksheet.sheet_data.rows.compact.map { |row| row[0].value } + + expect(remaining_urns).to include('10009655') + expect(remaining_urns).not_to include('10009656') + end + end + + describe '#remove_published_column' do + let(:urn_list) do + create(:urn_list, filename: 'customers_test.xlsx') + end + it 'removes the Published column from the workbook' do + tempfile = job.send(:build_workbook_temp_file, urn_list) + + job.send(:remove_published_column, urn_list, tempfile.path) + + rewritten = Tempfile.new(['rewritten', '.xlsx']) + rewritten.binmode + rewritten.write(urn_list.excel_file.download) + rewritten.flush + rewritten.rewind + + workbook = RubyXL::Parser.parse(rewritten.path) + worksheet = workbook[0] + + header_values = worksheet[0].cells.map(&:value) + expect(header_values).not_to include('Published') + ensure + tempfile&.close + tempfile&.unlink + rewritten&.close + rewritten&.unlink end end end -end +end \ No newline at end of file diff --git a/spec/services/urn_lists/api_client_spec.rb b/spec/services/urn_lists/api_client_spec.rb new file mode 100644 index 000000000..a4d50ff2b --- /dev/null +++ b/spec/services/urn_lists/api_client_spec.rb @@ -0,0 +1,50 @@ +require 'rails_helper' + +RSpec.describe UrnLists::ApiClient do + describe '#fetch_rows' do + before do + stub_request(:post, ENV.fetch('MDM_API_TOKEN_URL')) + .to_return( + status: 200, + body: { access_token: 'abc123' }.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0"). + with( + headers: { + 'Accept'=>'application/json', + 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization'=>'Bearer abc123', + 'User-Agent'=>'Ruby' + }). + to_return( + status: 200, + body: [ + { + urn: 10009655, + name: 'Government Commercial Agency', + postcode: 'L3 9PP', + sector: 'central_government', + published: true + } + ].to_json, + headers: { 'Content-Type' => 'application/json' } + ) + end + + it 'fetches and returns customer data' do + client = described_class.new + customers = client.fetch_rows + + expect(customers.size).to eq(1) + expect(customers.first['urn']).to eq(10009655) + expect(customers.first['name']).to eq('Government Commercial Agency') + expect(customers.first['postcode']).to eq('L3 9PP') + expect(customers.first['sector']).to eq('central_government') + expect(customers.first['published']).to eq(true) + end + + + end +end \ No newline at end of file diff --git a/spec/services/urn_lists/import_customers_spec.rb b/spec/services/urn_lists/import_customers_spec.rb new file mode 100644 index 000000000..52ea0f154 --- /dev/null +++ b/spec/services/urn_lists/import_customers_spec.rb @@ -0,0 +1,98 @@ +require 'rails_helper' + +RSpec.describe UrnLists::ImportCustomers do + describe '#call' do + context 'when given customer objects' do + let(:rows) do + [ + Customer.new( + urn: 10009655, + name: 'Government Commercial Agency', + postcode: 'L3 9PP', + sector: 'central_government', + deleted: false, + published: true + ) + ] + end + + it 'upserts the customers' do + expect { + described_class.new(rows: rows).call + }.to change(Customer, :count).by(1) + end + end + + context 'when given raw data rows' do + let(:rows) do + [ + { + 'URN' => '10009655', + 'CustomerName' => 'Government Commercial Agency', + 'PostCode' => 'L3 9PP', + 'Sector' => 'Central Government', + 'Published' => 'True' + } + ] + end + + it 'builds and upserts the customers' do + expect { + described_class.new(rows: rows).call + }.to change(Customer, :count).by(1) + + customer = Customer.last + + expect(customer.urn).to eq(10009655) + expect(customer.name).to eq('Government Commercial Agency') + expect(customer.postcode).to eq('L3 9PP') + expect(customer.sector).to eq('central_government') + expect(customer.deleted).to eq(false) + expect(customer.published).to eq(true) + end + + it 'updates an existing customer' do + existing_customer = create(:customer, urn: 10009655, name: 'Old Name', deleted: true) + + expect { + described_class.new(rows: rows).call + }.not_to change(Customer, :count) + + existing_customer.reload + + expect(existing_customer.name).to eq('Government Commercial Agency') + expect(existing_customer.deleted).to eq(false) + end + + it 'soft deletes customers not in the new list' do + obsolete = create(:customer, urn: 10009656, name: 'Obsolete Customer', deleted: false) + + rows = [ + { + 'URN' => '10009655', + 'CustomerName' => 'Government Commercial Agency', + 'PostCode' => 'L3 9PP', + 'Sector' => 'Central Government', + 'Published' => 'True' + } + ] + + described_class.new(rows: rows).call + + expect(obsolete.reload.deleted).to eq(true) + end + + it 'restores a previously deleted customer if it reappears in the list' do + deleted_customer = create(:customer, urn: 10009655, name: 'Government Commercial Agency', deleted: true) + + expect { + described_class.new(rows: rows).call + }.not_to change(Customer, :count) + + deleted_customer.reload + + expect(deleted_customer.deleted).to eq(false) + end + end + end +end \ No newline at end of file diff --git a/spec/services/urn_lists/read_excel_spec.rb b/spec/services/urn_lists/read_excel_spec.rb new file mode 100644 index 000000000..a3f773c01 --- /dev/null +++ b/spec/services/urn_lists/read_excel_spec.rb @@ -0,0 +1,29 @@ +require 'rails_helper' + +RSpec.describe UrnLists::ReadExcel do + describe '#call' do + subject(:rows) { described_class.new(file_path: path).call } + + context 'with a valid workbook' do + let(:path) { Rails.root.join('spec', 'fixtures', 'customers_test.xlsx') } + it 'returns customer data' do + expect(rows.size).to eq(3) + expect(rows.first).to be_a(Customer) + end + + it 'maps fields correctly' do + expect(rows.first.urn).to eq(10009655) + expect(rows.first.name).to eq('Crown Commercial Service') + expect(rows.first.postcode).to eq('L3 9PP') + end + end + + context 'with missing columns' do + let(:path) { Rails.root.join('spec', 'fixtures', 'customers_missing_columns.xlsx') } + + it 'raises invalid format' do + expect { rows }.to raise_error(UrnLists::ReadExcel::InvalidFormat) + end + end + end +end \ No newline at end of file From 922ea29617bfe5eb332e64e2006caf86c88be596 Mon Sep 17 00:00:00 2001 From: Reem Ibrahim Date: Thu, 16 Apr 2026 18:48:09 +0100 Subject: [PATCH 2/6] rubocop --- app/jobs/urn_list_api_sync_job.rb | 4 +- app/jobs/urn_list_importer_job.rb | 10 ++-- app/models/urn_list.rb | 6 +-- app/services/urn_lists/api_client.rb | 12 ++--- app/services/urn_lists/import_customers.rb | 8 +-- app/services/urn_lists/read_excel.rb | 22 ++++---- spec/jobs/urn_list_api_sync_job_spec.rb | 28 +++++----- spec/jobs/urn_list_importer_job_spec.rb | 34 +++++++------ spec/services/urn_lists/api_client_spec.rb | 51 +++++++++---------- .../urn_lists/import_customers_spec.rb | 34 ++++++------- spec/services/urn_lists/read_excel_spec.rb | 2 +- 11 files changed, 107 insertions(+), 104 deletions(-) diff --git a/app/jobs/urn_list_api_sync_job.rb b/app/jobs/urn_list_api_sync_job.rb index 0ceea29f0..8a012c7c4 100644 --- a/app/jobs/urn_list_api_sync_job.rb +++ b/app/jobs/urn_list_api_sync_job.rb @@ -10,7 +10,7 @@ def perform completed_at: Time.current, processed_count: count ) - rescue => e + rescue StandardError => e urn_list.update!( aasm_state: :failed, completed_at: Time.current, @@ -18,4 +18,4 @@ def perform ) raise e end -end \ No newline at end of file +end diff --git a/app/jobs/urn_list_importer_job.rb b/app/jobs/urn_list_importer_job.rb index 72a14514d..71fe2269f 100644 --- a/app/jobs/urn_list_importer_job.rb +++ b/app/jobs/urn_list_importer_job.rb @@ -12,6 +12,7 @@ class AlreadyImported < StandardError; end retry_on Aws::S3::Errors::ServiceError + # rubocop:disable Metrics/AbcSize def perform(urn_list) raise AlreadyImported unless urn_list.pending? @@ -29,18 +30,19 @@ def perform(urn_list) completed_at: Time.current, processed_count: count ) - rescue Aws::S3::Errors::ServiceError => e + rescue Aws::S3::Errors::ServiceError raise - rescue UrnLists::ReadExcel::InvalidFormat => e + rescue UrnLists::ReadExcel::InvalidFormat mark_failed!(urn_list) raise - rescue => e + rescue StandardError => e mark_failed!(urn_list) if urn_list.persisted? && urn_list.pending? raise e ensure cleanup_downloader_temp_file(downloader&.temp_file) cleanup_downloader_temp_file(workbook_temp_file) end + # rubocop:enable Metrics/AbcSize private @@ -96,7 +98,7 @@ def delete_non_publish_row(worksheet, row_num, row) def mark_failed!(urn_list, processed_count: 0) urn_list.update!( - aasm_state: :failed, + aasm_state: :failed, completed_at: Time.current, processed_count: processed_count ) diff --git a/app/models/urn_list.rb b/app/models/urn_list.rb index 98273e9d6..ba0a8b26e 100644 --- a/app/models/urn_list.rb +++ b/app/models/urn_list.rb @@ -1,9 +1,9 @@ class UrnList < ApplicationRecord include AASM - enum :source, { - manual_upload: 'manual_upload', - api_import: 'api_import' + enum :source, { + manual_upload: 'manual_upload', + api_import: 'api_import' } aasm do diff --git a/app/services/urn_lists/api_client.rb b/app/services/urn_lists/api_client.rb index 38e37431a..837fe56c5 100644 --- a/app/services/urn_lists/api_client.rb +++ b/app/services/urn_lists/api_client.rb @@ -17,11 +17,11 @@ def fetch_access_token uri = URI.parse(ENV.fetch('MDM_API_TOKEN_URL')) response = Net::HTTP.post_form(uri, { - grant_type: 'client_credentials', + grant_type: 'client_credentials', client_id: ENV.fetch('MDM_API_CLIENT_ID'), client_secret: ENV.fetch('MDM_API_CLIENT_SECRET'), scope: ENV.fetch('MDM_API_SCOPE') - }) + }) raise ApiError, "Failed to fetch access token: #{response.code}" unless response.is_a?(Net::HTTPSuccess) @@ -57,9 +57,9 @@ def fetch_urn_list(token) end def validate_rows!(rows) - unless rows.is_a?(Array) && rows.all? { |row| row.is_a?(Hash) } - raise ApiError, "Invalid URN list format: expected an array of objects" - end + return if rows.is_a?(Array) && rows.all? { |row| row.is_a?(Hash) } + + raise ApiError, 'Invalid URN list format: expected an array of objects' end end -end \ No newline at end of file +end diff --git a/app/services/urn_lists/import_customers.rb b/app/services/urn_lists/import_customers.rb index 862a12c35..10983841d 100644 --- a/app/services/urn_lists/import_customers.rb +++ b/app/services/urn_lists/import_customers.rb @@ -37,8 +37,8 @@ def normalize_sector(value) def normalize_published(value) return true if value.nil? - - value == 'False' ? false : true + + value != 'False' end def upsert!(customers) @@ -60,7 +60,7 @@ def soft_delete!(customers) urns_to_be_deleted = existing_urns - importing_urns - Customer.where(urn: urns_to_be_deleted).update_all(deleted: true) + Customer.where(urn: urns_to_be_deleted).update(deleted: true) end end -end \ No newline at end of file +end diff --git a/app/services/urn_lists/read_excel.rb b/app/services/urn_lists/read_excel.rb index 7f4548916..2dd95e3d5 100644 --- a/app/services/urn_lists/read_excel.rb +++ b/app/services/urn_lists/read_excel.rb @@ -37,16 +37,16 @@ def rows_from_csv rows = [] CSV.foreach(csv_temp_file, headers: true) do |row| - raise InvalidFormat unless (row.headers & REQUIRED_COLUMNS) == REQUIRED_COLUMNS - - rows << Customer.new( - name: row['CustomerName'], - urn: row['URN'].to_i, - postcode: row['PostCode'], - sector: (row['Sector'] == 'Central Government' ? :central_government : :wider_public_sector), - deleted: false, - published: (row['Published'] == 'False' ? false : true) - ) + raise InvalidFormat unless (row.headers & REQUIRED_COLUMNS) == REQUIRED_COLUMNS + + rows << Customer.new( + name: row['CustomerName'], + urn: row['URN'].to_i, + postcode: row['PostCode'], + sector: (row['Sector'] == 'Central Government' ? :central_government : :wider_public_sector), + deleted: false, + published: (row['Published'] != 'False') + ) end rows @@ -63,4 +63,4 @@ def cleanup_csv_temp_file @csv_temp_file.unlink end end -end \ No newline at end of file +end diff --git a/spec/jobs/urn_list_api_sync_job_spec.rb b/spec/jobs/urn_list_api_sync_job_spec.rb index e50206d40..48484bc4b 100644 --- a/spec/jobs/urn_list_api_sync_job_spec.rb +++ b/spec/jobs/urn_list_api_sync_job_spec.rb @@ -5,19 +5,19 @@ let(:rows) do [ Customer.new( - urn: 10009655, - name: 'Government Commercial Agency', - postcode: 'L3 9PP', + urn: 10009655, + name: 'Government Commercial Agency', + postcode: 'L3 9PP', sector: 'central_government', - deleted: false, + deleted: false, published: true ), Customer.new( - urn: 10009656, + urn: 10009656, name: 'Another Organisation', - postcode: 'AB1 2CD', + postcode: 'AB1 2CD', sector: 'wider_public_sector', - deleted: false, + deleted: false, published: true ) ] @@ -32,9 +32,9 @@ end it 'creates a pending urn list, imports the rows, and marks it as processed' do - expect { + expect do described_class.perform_now - }.to change(UrnList, :count).by(1) + end.to change(UrnList, :count).by(1) expect(api_client_service).to have_received(:fetch_rows) expect(import_customers_service).to have_received(:call) @@ -49,9 +49,9 @@ it 'marks the urn list as failed when the api call fails' do allow(api_client_service).to receive(:fetch_rows).and_raise(StandardError.new('token failed')) - expect { + expect do described_class.perform_now - }.to raise_error(StandardError, 'token failed') + end.to raise_error(StandardError, 'token failed') urn_list = UrnList.last expect(urn_list.source).to eq('api_import') @@ -63,9 +63,9 @@ it 'marks the urn list as failed when the import fails after rows are fetched' do allow(import_customers_service).to receive(:call).and_raise(StandardError.new('import failed')) - expect { + expect do described_class.perform_now - }.to raise_error(StandardError, 'import failed') + end.to raise_error(StandardError, 'import failed') urn_list = UrnList.last expect(urn_list.source).to eq('api_import') @@ -74,4 +74,4 @@ expect(urn_list.completed_at).to be_present end end -end \ No newline at end of file +end diff --git a/spec/jobs/urn_list_importer_job_spec.rb b/spec/jobs/urn_list_importer_job_spec.rb index 501588efd..3cbc5e41c 100644 --- a/spec/jobs/urn_list_importer_job_spec.rb +++ b/spec/jobs/urn_list_importer_job_spec.rb @@ -30,28 +30,28 @@ let(:downloader) do double( - 'AttachedFileDownloader', - download!: true, + 'AttachedFileDownloader', + download!: true, temp_file: download_tempfile - ) + ) end let(:rows) do [ Customer.new( - urn: 10009655, - name: 'Crown Commercial Service', - postcode: 'L3 9PP', + urn: 10009655, + name: 'Crown Commercial Service', + postcode: 'L3 9PP', sector: 'central_government', - deleted: false, + deleted: false, published: true ), Customer.new( - urn: 10009656, + urn: 10009656, name: 'Another Organisation', - postcode: 'AB1 2CD', + postcode: 'AB1 2CD', sector: 'wider_public_sector', - deleted: false, + deleted: false, published: true ) ] @@ -68,7 +68,8 @@ allow(UrnLists::ImportCustomers).to receive(:new) .with(rows: rows) .and_return(import_customers_service) - allow_any_instance_of(described_class).to receive(:build_workbook_temp_file).with(urn_list).and_return(workbook_tempfile) + allow_any_instance_of(described_class).to receive(:build_workbook_temp_file) + .with(urn_list).and_return(workbook_tempfile) allow_any_instance_of(described_class).to receive(:remove_published_column).with(urn_list, workbook_tempfile.path) allow_any_instance_of(described_class).to receive(:cleanup_downloader_temp_file) end @@ -76,6 +77,7 @@ after do [download_tempfile, workbook_tempfile].each do |file| next if file.closed? + file.close file.unlink rescue StandardError @@ -114,9 +116,9 @@ allow(bad_read_excel).to receive(:call).and_raise(UrnLists::ReadExcel::InvalidFormat) allow(UrnLists::ReadExcel).to receive(:new).and_return(bad_read_excel) - expect { + expect do described_class.perform_now(urn_list) - }.to raise_error(UrnLists::ReadExcel::InvalidFormat) + end.to raise_error(UrnLists::ReadExcel::InvalidFormat) urn_list.reload expect(urn_list).to be_failed @@ -139,9 +141,9 @@ allow(bad_importer).to receive(:call).and_raise(StandardError.new('Unexpected error')) allow(UrnLists::ImportCustomers).to receive(:new).and_return(bad_importer) - expect { + expect do described_class.perform_now(urn_list) - }.to raise_error(StandardError) + end.to raise_error(StandardError) urn_list.reload expect(urn_list).to be_failed @@ -274,4 +276,4 @@ end end end -end \ No newline at end of file +end diff --git a/spec/services/urn_lists/api_client_spec.rb b/spec/services/urn_lists/api_client_spec.rb index a4d50ff2b..72ab0db00 100644 --- a/spec/services/urn_lists/api_client_spec.rb +++ b/spec/services/urn_lists/api_client_spec.rb @@ -5,31 +5,32 @@ before do stub_request(:post, ENV.fetch('MDM_API_TOKEN_URL')) .to_return( - status: 200, - body: { access_token: 'abc123' }.to_json, - headers: { 'Content-Type' => 'application/json' } + status: 200, + body: { access_token: 'abc123' }.to_json, + headers: { 'Content-Type' => 'application/json' } ) - stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0"). - with( - headers: { - 'Accept'=>'application/json', - 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Authorization'=>'Bearer abc123', - 'User-Agent'=>'Ruby' - }). - to_return( - status: 200, - body: [ - { - urn: 10009655, - name: 'Government Commercial Agency', - postcode: 'L3 9PP', - sector: 'central_government', - published: true - } - ].to_json, - headers: { 'Content-Type' => 'application/json' } + stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") + .with( + headers: { + 'Accept' => 'application/json', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Bearer abc123', + 'User-Agent' => 'Ruby' + } + ) + .to_return( + status: 200, + body: [ + { + urn: 10009655, + name: 'Government Commercial Agency', + postcode: 'L3 9PP', + sector: 'central_government', + published: true + } + ].to_json, + headers: { 'Content-Type' => 'application/json' } ) end @@ -44,7 +45,5 @@ expect(customers.first['sector']).to eq('central_government') expect(customers.first['published']).to eq(true) end - - end -end \ No newline at end of file +end diff --git a/spec/services/urn_lists/import_customers_spec.rb b/spec/services/urn_lists/import_customers_spec.rb index 52ea0f154..67519df00 100644 --- a/spec/services/urn_lists/import_customers_spec.rb +++ b/spec/services/urn_lists/import_customers_spec.rb @@ -17,9 +17,9 @@ end it 'upserts the customers' do - expect { + expect do described_class.new(rows: rows).call - }.to change(Customer, :count).by(1) + end.to change(Customer, :count).by(1) end end @@ -37,9 +37,9 @@ end it 'builds and upserts the customers' do - expect { + expect do described_class.new(rows: rows).call - }.to change(Customer, :count).by(1) + end.to change(Customer, :count).by(1) customer = Customer.last @@ -54,9 +54,9 @@ it 'updates an existing customer' do existing_customer = create(:customer, urn: 10009655, name: 'Old Name', deleted: true) - expect { + expect do described_class.new(rows: rows).call - }.not_to change(Customer, :count) + end.not_to change(Customer, :count) existing_customer.reload @@ -68,13 +68,13 @@ obsolete = create(:customer, urn: 10009656, name: 'Obsolete Customer', deleted: false) rows = [ - { - 'URN' => '10009655', - 'CustomerName' => 'Government Commercial Agency', - 'PostCode' => 'L3 9PP', - 'Sector' => 'Central Government', - 'Published' => 'True' - } + { + 'URN' => '10009655', + 'CustomerName' => 'Government Commercial Agency', + 'PostCode' => 'L3 9PP', + 'Sector' => 'Central Government', + 'Published' => 'True' + } ] described_class.new(rows: rows).call @@ -82,12 +82,12 @@ expect(obsolete.reload.deleted).to eq(true) end - it 'restores a previously deleted customer if it reappears in the list' do + it 'restores a previously deleted customer if it reappears in the list' do deleted_customer = create(:customer, urn: 10009655, name: 'Government Commercial Agency', deleted: true) - expect { + expect do described_class.new(rows: rows).call - }.not_to change(Customer, :count) + end.not_to change(Customer, :count) deleted_customer.reload @@ -95,4 +95,4 @@ end end end -end \ No newline at end of file +end diff --git a/spec/services/urn_lists/read_excel_spec.rb b/spec/services/urn_lists/read_excel_spec.rb index a3f773c01..d407371d2 100644 --- a/spec/services/urn_lists/read_excel_spec.rb +++ b/spec/services/urn_lists/read_excel_spec.rb @@ -26,4 +26,4 @@ end end end -end \ No newline at end of file +end From e84a5f5f4c518f9b0fe756b8ed38513ac4f5551d Mon Sep 17 00:00:00 2001 From: Reem Ibrahim Date: Fri, 17 Apr 2026 13:40:50 +0100 Subject: [PATCH 3/6] added pagination to audit log --- app/controllers/admin/urn_lists_controller.rb | 2 +- app/views/admin/urn_lists/index.html.haml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/admin/urn_lists_controller.rb b/app/controllers/admin/urn_lists_controller.rb index 4453295f1..20dd7612b 100644 --- a/app/controllers/admin/urn_lists_controller.rb +++ b/app/controllers/admin/urn_lists_controller.rb @@ -2,7 +2,7 @@ class Admin::UrnListsController < AdminController before_action :find_latest_list, only: %i[index download] def index - @urn_lists = UrnList.order(created_at: :desc).all + @urn_lists = UrnList.order(created_at: :desc).page(params[:page]) end def new diff --git a/app/views/admin/urn_lists/index.html.haml b/app/views/admin/urn_lists/index.html.haml index a057bcf90..c5ff34a6c 100644 --- a/app/views/admin/urn_lists/index.html.haml +++ b/app/views/admin/urn_lists/index.html.haml @@ -30,5 +30,6 @@ %td.govuk-table__cell= list.excel_file.filename || '-' %td.govuk-table__cell= list.created_at %td.govuk-table__cell= list.aasm_state + = paginate @urn_lists From 1e73dd86dc5dc9c5ed7429719de1358ad99175de Mon Sep 17 00:00:00 2001 From: Reem Ibrahim Date: Fri, 17 Apr 2026 13:53:15 +0100 Subject: [PATCH 4/6] fixed test issue --- spec/services/urn_lists/api_client_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/services/urn_lists/api_client_spec.rb b/spec/services/urn_lists/api_client_spec.rb index 72ab0db00..a8fe688ab 100644 --- a/spec/services/urn_lists/api_client_spec.rb +++ b/spec/services/urn_lists/api_client_spec.rb @@ -3,7 +3,7 @@ RSpec.describe UrnLists::ApiClient do describe '#fetch_rows' do before do - stub_request(:post, ENV.fetch('MDM_API_TOKEN_URL')) + stub_request(:post, 'https://login.microsoftonline.com/9f8c0d79-3e87-4cd3-9799-c3443146ea5e/oauth2/v2.0/token') .to_return( status: 200, body: { access_token: 'abc123' }.to_json, @@ -13,7 +13,7 @@ stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") .with( headers: { - 'Accept' => 'application/json', + 'Accept' => 'application/json', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization' => 'Bearer abc123', 'User-Agent' => 'Ruby' From 52aa189699c8e8e647976ec67c88ee8f2b19565e Mon Sep 17 00:00:00 2001 From: Reem Ibrahim Date: Fri, 17 Apr 2026 15:53:04 +0100 Subject: [PATCH 5/6] fixed test issue --- spec/services/urn_lists/api_client_spec.rb | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/spec/services/urn_lists/api_client_spec.rb b/spec/services/urn_lists/api_client_spec.rb index a8fe688ab..9fb957302 100644 --- a/spec/services/urn_lists/api_client_spec.rb +++ b/spec/services/urn_lists/api_client_spec.rb @@ -3,9 +3,18 @@ RSpec.describe UrnLists::ApiClient do describe '#fetch_rows' do before do - stub_request(:post, 'https://login.microsoftonline.com/9f8c0d79-3e87-4cd3-9799-c3443146ea5e/oauth2/v2.0/token') + stub_request(:post, "https://example.com/oauth/token") + .with( + body: {"client_id" => "test_client_id", "client_secret" => "test_client_secret", "grant_type" => "client_credentials", "scope" => "test_scope"}, + headers: { + 'Accept'=>'*/*', + 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Content-Type'=>'application/x-www-form-urlencoded', + 'Host'=>'example.com', + 'User-Agent'=>'Ruby' + }) .to_return( - status: 200, + status: 200, body: { access_token: 'abc123' }.to_json, headers: { 'Content-Type' => 'application/json' } ) @@ -32,6 +41,12 @@ ].to_json, headers: { 'Content-Type' => 'application/json' } ) + + allow(ENV).to receive(:fetch).and_call_original + allow(ENV).to receive(:fetch).with('MDM_API_TOKEN_URL').and_return('https://example.com/oauth/token') + allow(ENV).to receive(:fetch).with('MDM_API_CLIENT_ID').and_return('test_client_id') + allow(ENV).to receive(:fetch).with('MDM_API_CLIENT_SECRET').and_return('test_client_secret') + allow(ENV).to receive(:fetch).with('MDM_API_SCOPE').and_return('test_scope') end it 'fetches and returns customer data' do From e8fc0af625acda34603a812787493ecd3d0f3520 Mon Sep 17 00:00:00 2001 From: Reem Ibrahim Date: Fri, 17 Apr 2026 16:11:51 +0100 Subject: [PATCH 6/6] rubocop --- spec/services/urn_lists/api_client_spec.rb | 34 ++++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/spec/services/urn_lists/api_client_spec.rb b/spec/services/urn_lists/api_client_spec.rb index 9fb957302..e6e195298 100644 --- a/spec/services/urn_lists/api_client_spec.rb +++ b/spec/services/urn_lists/api_client_spec.rb @@ -3,18 +3,20 @@ RSpec.describe UrnLists::ApiClient do describe '#fetch_rows' do before do - stub_request(:post, "https://example.com/oauth/token") + stub_request(:post, 'https://example.com/oauth/token') .with( - body: {"client_id" => "test_client_id", "client_secret" => "test_client_secret", "grant_type" => "client_credentials", "scope" => "test_scope"}, - headers: { - 'Accept'=>'*/*', - 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Content-Type'=>'application/x-www-form-urlencoded', - 'Host'=>'example.com', - 'User-Agent'=>'Ruby' - }) + body: { 'client_id' => 'test_client_id', 'client_secret' => 'test_client_secret', +'grant_type' => 'client_credentials', 'scope' => 'test_scope' }, + headers: { + 'Accept' => '*/*', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Content-Type' => 'application/x-www-form-urlencoded', + 'Host' => 'example.com', + 'User-Agent' => 'Ruby' + } + ) .to_return( - status: 200, + status: 200, body: { access_token: 'abc123' }.to_json, headers: { 'Content-Type' => 'application/json' } ) @@ -22,7 +24,7 @@ stub_request(:get, "https://apim.crowncommercial.gov.uk/website-data/manual/paths/invoke/%5Batt%5D.%5Bvw_RMIActiveURNList%5D/?api-version=2016-10-01&filter=Published%20eq%20'True'&sp=/triggers/manual/run&sv=1.0") .with( headers: { - 'Accept' => 'application/json', + 'Accept' => 'application/json', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization' => 'Bearer abc123', 'User-Agent' => 'Ruby' @@ -42,11 +44,11 @@ headers: { 'Content-Type' => 'application/json' } ) - allow(ENV).to receive(:fetch).and_call_original - allow(ENV).to receive(:fetch).with('MDM_API_TOKEN_URL').and_return('https://example.com/oauth/token') - allow(ENV).to receive(:fetch).with('MDM_API_CLIENT_ID').and_return('test_client_id') - allow(ENV).to receive(:fetch).with('MDM_API_CLIENT_SECRET').and_return('test_client_secret') - allow(ENV).to receive(:fetch).with('MDM_API_SCOPE').and_return('test_scope') + allow(ENV).to receive(:fetch).and_call_original + allow(ENV).to receive(:fetch).with('MDM_API_TOKEN_URL').and_return('https://example.com/oauth/token') + allow(ENV).to receive(:fetch).with('MDM_API_CLIENT_ID').and_return('test_client_id') + allow(ENV).to receive(:fetch).with('MDM_API_CLIENT_SECRET').and_return('test_client_secret') + allow(ENV).to receive(:fetch).with('MDM_API_SCOPE').and_return('test_scope') end it 'fetches and returns customer data' do