373ad6ff-fad2-405a-ab26-3d30fb5ceecf

Committed 24 Dec 2024 08:24PM UTC coverage: 91.938% (+0.08%) from 91.859%

Build # 373ad6ff-fad2-405a-ab26-3d30fb5ceecf

Build Type

Pull #2563

circleci

Committed by

maxkadel

Commit Message

Put attaching xml files in their own batch

Pull Request Pull Request #2563: I2321 Shift SCSB full index tasks into separate background jobs

Run Details

152 of 156 new or added lines in 10 files covered. (97.44%)

65 existing lines in 17 files now uncovered.

3478 of 3783 relevant lines covered (91.94%)

366.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.11

/app/models/scsb/partner_updates/full.rb

require 'json/add/regexp'

module Scsb
  class PartnerUpdates
    class Full < Update
      def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)
        @dump = dump
        @institutions = [
          { inst: "NYPL", prefix: 'scsbfull_nypl_' },
          { inst: "CUL", prefix: 'scsbfull_cul_' },
          { inst: "HL", prefix: 'scsbfull_hl_' }
        ]
        super
      end

      def process_full_files
        prepare_directory
        dump_id = @dump.id
        @institutions.each do |institution|
          params = { inst: institution[:inst], prefix: institution[:prefix], dump_id: }.stringify_keys
          DownloadAndProcessFullJob.perform_async(params)
        end
        set_generated_date
        log_record_fixes
      end

      def self.download_partner_files(file_filter:, dump_id:, file_prefix:)
        file = Scsb::PartnerUpdates::Full.download_full_file(file_filter)
        if file
          params = { dump_id:, file_prefix:, file: file.to_s }.stringify_keys
          ProcessPartnerUpdatesJob.perform_async(params)
        else
          Scsb::PartnerUpdates::Full.add_error(message: "No full dump files found matching #{file_filter}", dump_id:)
        end
      end

      # Ensures that CSV is present and that it does not include any private records
      def self.validate_csv(inst:, dump_id:)
        @update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
        @scsb_file_dir = ENV['SCSB_FILE_DIR']
        matcher = /#{inst}.*\.csv/.as_json
        file = Scsb::PartnerUpdates::Full.download_full_file(matcher)
        matches_expected_collections = false
        if file
          csv = CSV.read(file, headers: true)
          group_ids = csv["Collection Group Id(s)"].first
          matches_expected_collections = group_ids == '1*2*5*6'
          unless matches_expected_collections
            add_error(message: "Metadata file indicates that dump for #{inst} does not include the correct Group IDs, not processing. Group ids: #{group_ids}", dump_id:)
          end
          filename = File.basename(file)
          destination_filepath = "#{@scsb_file_dir}/#{filename}"
          FileUtils.move(file, destination_filepath)
          Dump.attach_dump_file(dump_id:, filepath: destination_filepath, dump_file_type: :recap_records_full_metadata)
          File.unlink(destination_filepath) if File.exist?(destination_filepath)
        else
          add_error(message: "No metadata files found matching #{inst}", dump_id:)
        end
        matches_expected_collections
      end

      def self.add_error(message:, dump_id:)
        dump_event = Dump.find(dump_id).event
        error = Array.wrap(dump_event.error)
        error << message
        dump_event.error = error.join("; ")
        dump_event.save
      end

      def self.download_full_file(file_filter)
        update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
        prefix = ENV['SCSB_S3_PARTNER_FULLS'] || 'data-exports/PUL/MARCXml/Full'
        s3_bucket = Scsb::S3Bucket.partner_transfer_client
        file_filter = Regexp.json_create(file_filter)
        s3_bucket.download_recent(prefix:, output_directory: update_directory, file_filter:)
      end
    end
  end
end

1	require 'json/add/regexp'	1✔
2
3	module Scsb	1✔
4	class PartnerUpdates	1✔
5	class Full < Update	1✔
6	def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)	1✔
7	@dump = dump	6✔
8	@institutions = [
9	{ inst: "NYPL", prefix: 'scsbfull_nypl_' },	6✔
10	{ inst: "CUL", prefix: 'scsbfull_cul_' },
11	{ inst: "HL", prefix: 'scsbfull_hl_' }
12	]
13	super	6✔
14	end
15
16	def process_full_files	1✔
17	prepare_directory	5✔
18	dump_id = @dump.id	5✔
19	@institutions.each do \|institution\|	5✔
20	params = { inst: institution[:inst], prefix: institution[:prefix], dump_id: }.stringify_keys	15✔
21	DownloadAndProcessFullJob.perform_async(params)	15✔
22	end
23	set_generated_date	5✔
24	log_record_fixes	5✔
25	end
26
27	def self.download_partner_files(file_filter:, dump_id:, file_prefix:)	1✔
28	file = Scsb::PartnerUpdates::Full.download_full_file(file_filter)	13✔
29	if file	13✔
30	params = { dump_id:, file_prefix:, file: file.to_s }.stringify_keys	13✔
31	ProcessPartnerUpdatesJob.perform_async(params)	13✔
32	else
NEW 33	Scsb::PartnerUpdates::Full.add_error(message: "No full dump files found matching #{file_filter}", dump_id:)	×
34	end
35	end
36
37	# Ensures that CSV is present and that it does not include any private records
38	def self.validate_csv(inst:, dump_id:)	1✔
39	@update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	23✔
40	@scsb_file_dir = ENV['SCSB_FILE_DIR']	23✔
41	matcher = /#{inst}.*\.csv/.as_json	23✔
42	file = Scsb::PartnerUpdates::Full.download_full_file(matcher)	23✔
43	matches_expected_collections = false	21✔
44	if file	21✔
45	csv = CSV.read(file, headers: true)	16✔
46	group_ids = csv["Collection Group Id(s)"].first	16✔
47	matches_expected_collections = group_ids == '125*6'	16✔
48	unless matches_expected_collections	16✔
49	add_error(message: "Metadata file indicates that dump for #{inst} does not include the correct Group IDs, not processing. Group ids: #{group_ids}", dump_id:)	3✔
50	end
51	filename = File.basename(file)	16✔
52	destination_filepath = "#{@scsb_file_dir}/#{filename}"	16✔
53	FileUtils.move(file, destination_filepath)	16✔
54	Dump.attach_dump_file(dump_id:, filepath: destination_filepath, dump_file_type: :recap_records_full_metadata)	16✔
55	File.unlink(destination_filepath) if File.exist?(destination_filepath)	16✔
56	else
57	add_error(message: "No metadata files found matching #{inst}", dump_id:)	5✔
58	end
59	matches_expected_collections	21✔
60	end
61
62	def self.add_error(message:, dump_id:)	1✔
63	dump_event = Dump.find(dump_id).event	8✔
64	error = Array.wrap(dump_event.error)	8✔
65	error << message	8✔
66	dump_event.error = error.join("; ")	8✔
67	dump_event.save	8✔
68	end
69
70	def self.download_full_file(file_filter)	1✔
71	update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	36✔
72	prefix = ENV['SCSB_S3_PARTNER_FULLS'] \|\| 'data-exports/PUL/MARCXml/Full'	36✔
73	s3_bucket = Scsb::S3Bucket.partner_transfer_client	36✔
74	file_filter = Regexp.json_create(file_filter)	36✔
75	s3_bucket.download_recent(prefix:, output_directory: update_directory, file_filter:)	36✔
76	end
77	end
78	end
79	end

pulibrary / bibdata / 373ad6ff-fad2-405a-ab26-3d30fb5ceecf

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous