5db7f693-1ba8-464b-88d6-caaa168a35c8

Committed 30 Dec 2024 07:22PM UTC coverage: 91.223% (-0.6%) from 91.859%

Build # 5db7f693-1ba8-464b-88d6-caaa168a35c8

Build Type

Pull #2563

circleci

Committed by

maxkadel

Commit Message

Rubocop

Pull Request Pull Request #2563: I2321 Shift SCSB full index tasks into separate background jobs

Run Details

134 of 163 new or added lines in 10 files covered. (82.21%)

5 existing lines in 1 file now uncovered.

3461 of 3794 relevant lines covered (91.22%)

366.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.11

/app/models/scsb/partner_updates/full.rb

require 'json/add/regexp'

module Scsb
  class PartnerUpdates
    class Full < Update
      def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)
        @dump = dump
        @institutions = [
          { inst: 'NYPL', prefix: 'scsbfull_nypl_' },
          { inst: 'CUL', prefix: 'scsbfull_cul_' },
          { inst: 'HL', prefix: 'scsbfull_hl_' }
        ]
        super
      end

      def process_full_files
        prepare_directory
        dump_id = @dump.id
        @institutions.each do |institution|
          params = { inst: institution[:inst], prefix: institution[:prefix], dump_id: }.stringify_keys
          DownloadAndProcessFullJob.perform_async(params)
        end
        set_generated_date
        log_record_fixes
      end

      def self.download_partner_files(file_filter:, dump_id:, file_prefix:)
        file = Scsb::PartnerUpdates::Full.download_full_file(file_filter)
        if file
          params = { dump_id:, file_prefix:, file: file.to_s }.stringify_keys
          ProcessPartnerUpdatesJob.perform_async(params)
        else
          Scsb::PartnerUpdates::Full.add_error(message: "No full dump files found matching #{file_filter}", dump_id:)
        end
      end

      # Ensures that CSV is present and that it does not include any private records
      def self.validate_csv(inst:, dump_id:)
        @update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
        @scsb_file_dir = ENV['SCSB_FILE_DIR']
        matcher = /#{inst}.*\.csv/.as_json
        file = Scsb::PartnerUpdates::Full.download_full_file(matcher)
        matches_expected_collections = false
        if file
          csv = CSV.read(file, headers: true)
          group_ids = csv['Collection Group Id(s)'].first
          matches_expected_collections = group_ids == '1*2*5*6'
          unless matches_expected_collections
            add_error(message: "Metadata file indicates that dump for #{inst} does not include the correct Group IDs, not processing. Group ids: #{group_ids}", dump_id:)
          end
          filename = File.basename(file)
          destination_filepath = "#{@scsb_file_dir}/#{filename}"
          FileUtils.move(file, destination_filepath)
          Dump.attach_dump_file(dump_id:, filepath: destination_filepath, dump_file_type: :recap_records_full_metadata)
          File.unlink(destination_filepath) if File.exist?(destination_filepath)
        else
          add_error(message: "No metadata files found matching #{inst}", dump_id:)
        end
        matches_expected_collections
      end

      def self.add_error(message:, dump_id:)
        dump_event = Dump.find(dump_id).event
        error = Array.wrap(dump_event.error)
        error << message
        dump_event.error = error.join('; ')
        dump_event.save
      end

      def self.download_full_file(file_filter)
        update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
        prefix = ENV['SCSB_S3_PARTNER_FULLS'] || 'data-exports/PUL/MARCXml/Full'
        s3_bucket = Scsb::S3Bucket.partner_transfer_client
        file_filter = Regexp.json_create(file_filter)
        s3_bucket.download_recent(prefix:, output_directory: update_directory, file_filter:)
      end
    end
  end
end

1	require 'json/add/regexp'	1✔
2
3	module Scsb	1✔
4	class PartnerUpdates	1✔
5	class Full < Update	1✔
6	def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)	1✔
7	@dump = dump	6✔
8	@institutions = [
9	{ inst: 'NYPL', prefix: 'scsbfull_nypl_' },	6✔
10	{ inst: 'CUL', prefix: 'scsbfull_cul_' },
11	{ inst: 'HL', prefix: 'scsbfull_hl_' }
12	]
13	super	6✔
14	end
15
16	def process_full_files	1✔
17	prepare_directory	5✔
18	dump_id = @dump.id	5✔
19	@institutions.each do \|institution\|	5✔
20	params = { inst: institution[:inst], prefix: institution[:prefix], dump_id: }.stringify_keys	7✔
21	DownloadAndProcessFullJob.perform_async(params)	7✔
22	end
23	set_generated_date	1✔
24	log_record_fixes	1✔
25	end
26
27	def self.download_partner_files(file_filter:, dump_id:, file_prefix:)	1✔
28	file = Scsb::PartnerUpdates::Full.download_full_file(file_filter)	9✔
29	if file	9✔
30	params = { dump_id:, file_prefix:, file: file.to_s }.stringify_keys	9✔
31	ProcessPartnerUpdatesJob.perform_async(params)	9✔
32	else
NEW 33	Scsb::PartnerUpdates::Full.add_error(message: "No full dump files found matching #{file_filter}", dump_id:)	×
34	end
35	end
36
37	# Ensures that CSV is present and that it does not include any private records
38	def self.validate_csv(inst:, dump_id:)	1✔
39	@update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	15✔
40	@scsb_file_dir = ENV['SCSB_FILE_DIR']	15✔
41	matcher = /#{inst}.*\.csv/.as_json	15✔
42	file = Scsb::PartnerUpdates::Full.download_full_file(matcher)	15✔
43	matches_expected_collections = false	13✔
44	if file	13✔
45	csv = CSV.read(file, headers: true)	10✔
46	group_ids = csv['Collection Group Id(s)'].first	10✔
47	matches_expected_collections = group_ids == '125*6'	10✔
48	unless matches_expected_collections	10✔
49	add_error(message: "Metadata file indicates that dump for #{inst} does not include the correct Group IDs, not processing. Group ids: #{group_ids}", dump_id:)	1✔
50	end
51	filename = File.basename(file)	10✔
52	destination_filepath = "#{@scsb_file_dir}/#{filename}"	10✔
53	FileUtils.move(file, destination_filepath)	10✔
54	Dump.attach_dump_file(dump_id:, filepath: destination_filepath, dump_file_type: :recap_records_full_metadata)	10✔
55	File.unlink(destination_filepath) if File.exist?(destination_filepath)	10✔
56	else
57	add_error(message: "No metadata files found matching #{inst}", dump_id:)	3✔
58	end
59	matches_expected_collections	13✔
60	end
61
62	def self.add_error(message:, dump_id:)	1✔
63	dump_event = Dump.find(dump_id).event	4✔
64	error = Array.wrap(dump_event.error)	4✔
65	error << message	4✔
66	dump_event.error = error.join('; ')	4✔
67	dump_event.save	4✔
68	end
69
70	def self.download_full_file(file_filter)	1✔
71	update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	24✔
72	prefix = ENV['SCSB_S3_PARTNER_FULLS'] \|\| 'data-exports/PUL/MARCXml/Full'	24✔
73	s3_bucket = Scsb::S3Bucket.partner_transfer_client	24✔
74	file_filter = Regexp.json_create(file_filter)	24✔
75	s3_bucket.download_recent(prefix:, output_directory: update_directory, file_filter:)	24✔
76	end
77	end
78	end
79	end

pulibrary / bibdata / 5db7f693-1ba8-464b-88d6-caaa168a35c8

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous