Build has been set to done!

373ad6ff-fad2-405a-ab26-3d30fb5ceecf

Committed 24 Dec 2024 08:24PM UTC coverage: 91.938% (+0.08%) from 91.859%

Build # 373ad6ff-fad2-405a-ab26-3d30fb5ceecf

Build Type

Pull #2563

circleci

Committed by

maxkadel

Commit Message

Put attaching xml files in their own batch

Pull Request Pull Request #2563: I2321 Shift SCSB full index tasks into separate background jobs

Run Details

152 of 156 new or added lines in 10 files covered. (97.44%)

65 existing lines in 17 files now uncovered.

3478 of 3783 relevant lines covered (91.94%)

366.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.39

/app/jobs/alma_dump_transfer_job.rb

require 'net/sftp'

# Downloads the files from the sftp server and attaches them to Event, Dump,
# DumpFile objects. Kicks off further processing if neccessary
class AlmaDumpTransferJob < ApplicationJob
  queue_as :default
  attr_reader :dump_file_type

  def perform(dump:, job_id:)
    @dump_file_type = find_dump_file_type(dump)
    AlmaDownloader.files_for(job_id:, dump_file_type:).each do |file|
      dump.dump_files << file
    end

    dump.save

    IndexManager.for(Rails.application.config.solr["url"]).index_remaining! if incremental_dump?
  end

  def incremental_dump?
    dump_file_type == :updated_records
  end

  def find_dump_file_type(dump)
    job_config = find_job_configuration(dump:)
    job_config["dump_file_type"].downcase.to_sym
  end

  class AlmaDownloader
    def self.files_for(job_id:, dump_file_type:)
      new(job_id:, dump_file_type:).files_for
    end

    attr_reader :job_id, :dump_file_type
    def initialize(job_id:, dump_file_type:)
      @job_id = job_id
      @dump_file_type = dump_file_type
    end

    def files_for
      dump_files = []
      Net::SFTP.start(sftp_host, sftp_username, password: sftp_password) do |sftp|
        downloads = []
        remote_paths(sftp_session: sftp).each do |remote_path|
          df = DumpFile.create(dump_file_type:, path: dump_file_path(remote_path))
          dump_files << df
          download = transfer_file(sftp_session: sftp, remote_path:, local_path: df.path)
          downloads << download
        end

        # wait for all asynchronous downloads to complete before closing sftp
        # session
        downloads.each(&:wait)
      end

      dump_files
    end

    # look to sftp server and identify the desired files using job_id
    def remote_paths(sftp_session:)
      sftp_session.dir.entries(remote_base_path).select { |entry| parse_job_id(entry.name) == job_id }.map { |entry| File.join(remote_base_path, entry.name) }
    end

    def remote_base_path
      Rails.configuration.alma["sftp_alma_base_path"]
    end

    def dump_file_path(remote_path)
      File.join(MARC_LIBERATION_CONFIG['data_dir'], File.basename(remote_path))
    end

    # By default alma puts the timestamp before the job_id in filenames, and the
    #   default timestamp used differed from the documented timestamp
    #
    # So we configured the job_id to come before the timestamp to
    # protect against future variation in the timestamp format.
    # configured form is:
    # fulldump_<job ID>_<time stamp>_<new or update or delete>_<counter>.xml.tar.gz
    #
    # documentation is at
    # https://knowledge.exlibrisgroup.com/Alma/Product_Documentation/010Alma_Online_Help_(English)/090Integrations_with_External_Systems/030Resource_Management/080Publishing_and_Inventory_Enrichment#File_name
    def parse_job_id(name)
      name.split("_")[1]
    end

    # Do the actual download from the sftp server
    def transfer_file(sftp_session:, remote_path:, local_path:)
      File.truncate(local_path, 0) if File.exist?(local_path)
      sftp_session.download(remote_path, local_path)
    end

    def sftp_options
      {
        username: sftp_username,
        password: sftp_password,
        passive: true,
        ssl: true
      }
    end

    def sftp_username
      Rails.configuration.alma["sftp_username"]
    end

    def sftp_password
      Rails.configuration.alma["sftp_password"]
    end

    def sftp_host
      Rails.configuration.alma["sftp_host"]
    end
  end

  private

    def event_message(dump:)
      event = dump.event
      return {} if event.nil?

      JSON.parse(event.message_body)
    end

    def jobs_configuration
      Rails.configuration.alma[:jobs] || {}
    end

    def find_job_configuration(dump:)
      job_name = event_message(dump:).dig("job_instance", "name")

      jobs_configuration[job_name] || {}
    end
end

1	require 'net/sftp'	1✔
2
3	# Downloads the files from the sftp server and attaches them to Event, Dump,
4	# DumpFile objects. Kicks off further processing if neccessary
5	class AlmaDumpTransferJob < ApplicationJob	1✔
6	queue_as :default	1✔
7	attr_reader :dump_file_type	1✔
8
9	def perform(dump:, job_id:)	1✔
10	@dump_file_type = find_dump_file_type(dump)	2✔
11	AlmaDownloader.files_for(job_id:, dump_file_type:).each do \|file\|	2✔
12	dump.dump_files << file	3✔
13	end
14
15	dump.save	2✔
16
17	IndexManager.for(Rails.application.config.solr["url"]).index_remaining! if incremental_dump?	2✔
18	end
19
20	def incremental_dump?	1✔
21	dump_file_type == :updated_records	2✔
22	end
23
24	def find_dump_file_type(dump)	1✔
25	job_config = find_job_configuration(dump:)	2✔
26	job_config["dump_file_type"].downcase.to_sym	2✔
27	end
28
29	class AlmaDownloader	1✔
30	def self.files_for(job_id:, dump_file_type:)	1✔
31	new(job_id:, dump_file_type:).files_for	2✔
32	end
33
34	attr_reader :job_id, :dump_file_type	1✔
35	def initialize(job_id:, dump_file_type:)	1✔
36	@job_id = job_id	2✔
37	@dump_file_type = dump_file_type	2✔
38	end
39
40	def files_for	1✔
41	dump_files = []	2✔
42	Net::SFTP.start(sftp_host, sftp_username, password: sftp_password) do \|sftp\|	2✔
43	downloads = []	2✔
44	remote_paths(sftp_session: sftp).each do \|remote_path\|	2✔
45	df = DumpFile.create(dump_file_type:, path: dump_file_path(remote_path))	3✔
46	dump_files << df	3✔
47	download = transfer_file(sftp_session: sftp, remote_path:, local_path: df.path)	3✔
48	downloads << download	3✔
49	end
50
51	# wait for all asynchronous downloads to complete before closing sftp
52	# session
53	downloads.each(&:wait)	2✔
54	end
55
56	dump_files	2✔
57	end
58
59	# look to sftp server and identify the desired files using job_id
60	def remote_paths(sftp_session:)	1✔
61	sftp_session.dir.entries(remote_base_path).select { \|entry\| parse_job_id(entry.name) == job_id }.map { \|entry\| File.join(remote_base_path, entry.name) }	9✔
62	end
63
64	def remote_base_path	1✔
65	Rails.configuration.alma["sftp_alma_base_path"]	5✔
66	end
67
68	def dump_file_path(remote_path)	1✔
69	File.join(MARC_LIBERATION_CONFIG['data_dir'], File.basename(remote_path))	3✔
70	end
71
72	# By default alma puts the timestamp before the job_id in filenames, and the
73	# default timestamp used differed from the documented timestamp
74	#
75	# So we configured the job_id to come before the timestamp to
76	# protect against future variation in the timestamp format.
77	# configured form is:
78	# fulldump_<job ID>_<time stamp>_<new or update or delete>_<counter>.xml.tar.gz
79	#
80	# documentation is at
81	# https://knowledge.exlibrisgroup.com/Alma/Product_Documentation/010Alma_Online_Help_(English)/090Integrations_with_External_Systems/030Resource_Management/080Publishing_and_Inventory_Enrichment#File_name
82	def parse_job_id(name)	1✔
83	name.split("_")[1]	4✔
84	end
85
86	# Do the actual download from the sftp server
87	def transfer_file(sftp_session:, remote_path:, local_path:)	1✔
88	File.truncate(local_path, 0) if File.exist?(local_path)	3✔
89	sftp_session.download(remote_path, local_path)	3✔
90	end
91
92	def sftp_options	1✔
93	{
UNCOV 94	username: sftp_username,	×
95	password: sftp_password,
96	passive: true,
97	ssl: true
98	}
99	end
100
101	def sftp_username	1✔
102	Rails.configuration.alma["sftp_username"]	2✔
103	end
104
105	def sftp_password	1✔
106	Rails.configuration.alma["sftp_password"]	2✔
107	end
108
109	def sftp_host	1✔
110	Rails.configuration.alma["sftp_host"]	2✔
111	end
112	end
113
114	private	1✔
115
116	def event_message(dump:)	1✔
117	event = dump.event	2✔
118	return {} if event.nil?	2✔
119
120	JSON.parse(event.message_body)	2✔
121	end
122
123	def jobs_configuration	1✔
124	Rails.configuration.alma[:jobs] \|\| {}	2✔
125	end
126
127	def find_job_configuration(dump:)	1✔
128	job_name = event_message(dump:).dig("job_instance", "name")	2✔
129
130	jobs_configuration[job_name] \|\| {}	2✔
131	end
132	end

pulibrary / bibdata / 373ad6ff-fad2-405a-ab26-3d30fb5ceecf

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous