83785282-b493-4102-b828-c01dc1ed4e60

Committed 31 Dec 2024 07:48PM UTC coverage: 91.93% (+0.07%) from 91.859%

Build # 83785282-b493-4102-b828-c01dc1ed4e60

Build Type

Pull #2563

circleci

Committed by

maxkadel

Commit Message

Create more nested batches

Pull Request Pull Request #2563: I2321 Shift SCSB full index tasks into separate background jobs

Run Details

184 of 188 new or added lines in 11 files covered. (97.87%)

4 existing lines in 1 file now uncovered.

3497 of 3804 relevant lines covered (91.93%)

366.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.18

/app/models/scsb/partner_updates/full.rb

require 'json/add/regexp'

module Scsb
  class PartnerUpdates
    class Full < Update
      def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)
        @dump = dump
        @institutions = [
          { inst: 'NYPL', prefix: 'scsbfull_nypl_' },
          { inst: 'CUL', prefix: 'scsbfull_cul_' },
          { inst: 'HL', prefix: 'scsbfull_hl_' }
        ]
        super
      end

      def process_full_files
        prepare_directory
        dump_id = @dump.id
        batch = Sidekiq::Batch.new
        batch.jobs do
          @institutions.each do |institution|
            params = { inst: institution[:inst], prefix: institution[:prefix], dump_id: }.stringify_keys
            DownloadAndProcessFullJob.perform_async(params)
          end
        end
        set_generated_date
        log_record_fixes
      end

      def self.download_partner_files(file_filter:, dump_id:, file_prefix:)
        file = Scsb::PartnerUpdates::Full.download_full_file(file_filter)
        if file
          params = { dump_id:, file_prefix:, file: file.to_s }.stringify_keys
          ProcessPartnerUpdatesJob.perform_async(params)
        else
          Scsb::PartnerUpdates::Full.add_error(message: "No full dump files found matching #{file_filter}", dump_id:)
        end
      end

      # Ensures that CSV is present and that it does not include any private records
      def self.validate_csv(inst:, dump_id:)
        @update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
        @scsb_file_dir = ENV['SCSB_FILE_DIR']
        matcher = /#{inst}.*\.csv/.as_json
        file = Scsb::PartnerUpdates::Full.download_full_file(matcher)
        matches_expected_collections = false
        if file
          csv = CSV.read(file, headers: true)
          group_ids = csv['Collection Group Id(s)'].first
          matches_expected_collections = group_ids == '1*2*5*6'
          unless matches_expected_collections
            add_error(message: "Metadata file indicates that dump for #{inst} does not include the correct Group IDs, not processing. Group ids: #{group_ids}", dump_id:)
          end
          filename = File.basename(file)
          destination_filepath = "#{@scsb_file_dir}/#{filename}"
          FileUtils.move(file, destination_filepath)
          Dump.attach_dump_file(dump_id:, filepath: destination_filepath, dump_file_type: :recap_records_full_metadata)
          File.unlink(destination_filepath) if File.exist?(destination_filepath)
        else
          add_error(message: "No metadata files found matching #{inst}", dump_id:)
        end
        matches_expected_collections
      end

      def self.add_error(message:, dump_id:)
        dump_event = Dump.find(dump_id).event
        error = Array.wrap(dump_event.error)
        error << message
        dump_event.error = error.join('; ')
        dump_event.save
      end

      def self.download_full_file(file_filter)
        update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
        prefix = ENV['SCSB_S3_PARTNER_FULLS'] || 'data-exports/PUL/MARCXml/Full'
        s3_bucket = Scsb::S3Bucket.partner_transfer_client
        file_filter = Regexp.json_create(file_filter)
        s3_bucket.download_recent(prefix:, output_directory: update_directory, file_filter:)
      end
    end
  end
end

1	require 'json/add/regexp'	1✔
2
3	module Scsb	1✔
4	class PartnerUpdates	1✔
5	class Full < Update	1✔
6	def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)	1✔
7	@dump = dump	6✔
8	@institutions = [
9	{ inst: 'NYPL', prefix: 'scsbfull_nypl_' },	6✔
10	{ inst: 'CUL', prefix: 'scsbfull_cul_' },
11	{ inst: 'HL', prefix: 'scsbfull_hl_' }
12	]
13	super	6✔
14	end
15
16	def process_full_files	1✔
17	prepare_directory	5✔
18	dump_id = @dump.id	5✔
19	batch = Sidekiq::Batch.new	5✔
20	batch.jobs do	5✔
21	@institutions.each do \|institution\|	5✔
22	params = { inst: institution[:inst], prefix: institution[:prefix], dump_id: }.stringify_keys	15✔
23	DownloadAndProcessFullJob.perform_async(params)	15✔
24	end
25	end
26	set_generated_date	5✔
27	log_record_fixes	5✔
28	end
29
30	def self.download_partner_files(file_filter:, dump_id:, file_prefix:)	1✔
31	file = Scsb::PartnerUpdates::Full.download_full_file(file_filter)	13✔
32	if file	13✔
33	params = { dump_id:, file_prefix:, file: file.to_s }.stringify_keys	13✔
34	ProcessPartnerUpdatesJob.perform_async(params)	13✔
35	else
NEW 36	Scsb::PartnerUpdates::Full.add_error(message: "No full dump files found matching #{file_filter}", dump_id:)	×
37	end
38	end
39
40	# Ensures that CSV is present and that it does not include any private records
41	def self.validate_csv(inst:, dump_id:)	1✔
42	@update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	23✔
43	@scsb_file_dir = ENV['SCSB_FILE_DIR']	23✔
44	matcher = /#{inst}.*\.csv/.as_json	23✔
45	file = Scsb::PartnerUpdates::Full.download_full_file(matcher)	23✔
46	matches_expected_collections = false	21✔
47	if file	21✔
48	csv = CSV.read(file, headers: true)	16✔
49	group_ids = csv['Collection Group Id(s)'].first	16✔
50	matches_expected_collections = group_ids == '125*6'	16✔
51	unless matches_expected_collections	16✔
52	add_error(message: "Metadata file indicates that dump for #{inst} does not include the correct Group IDs, not processing. Group ids: #{group_ids}", dump_id:)	3✔
53	end
54	filename = File.basename(file)	16✔
55	destination_filepath = "#{@scsb_file_dir}/#{filename}"	16✔
56	FileUtils.move(file, destination_filepath)	16✔
57	Dump.attach_dump_file(dump_id:, filepath: destination_filepath, dump_file_type: :recap_records_full_metadata)	16✔
58	File.unlink(destination_filepath) if File.exist?(destination_filepath)	16✔
59	else
60	add_error(message: "No metadata files found matching #{inst}", dump_id:)	5✔
61	end
62	matches_expected_collections	21✔
63	end
64
65	def self.add_error(message:, dump_id:)	1✔
66	dump_event = Dump.find(dump_id).event	8✔
67	error = Array.wrap(dump_event.error)	8✔
68	error << message	8✔
69	dump_event.error = error.join('; ')	8✔
70	dump_event.save	8✔
71	end
72
73	def self.download_full_file(file_filter)	1✔
74	update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	36✔
75	prefix = ENV['SCSB_S3_PARTNER_FULLS'] \|\| 'data-exports/PUL/MARCXml/Full'	36✔
76	s3_bucket = Scsb::S3Bucket.partner_transfer_client	36✔
77	file_filter = Regexp.json_create(file_filter)	36✔
78	s3_bucket.download_recent(prefix:, output_directory: update_directory, file_filter:)	36✔
79	end
80	end
81	end
82	end

pulibrary / bibdata / 83785282-b493-4102-b828-c01dc1ed4e60

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous