2c0ea772-0154-4a7d-8c5e-4a73107ca30a

Committed 01 Oct 2025 03:07PM UTC coverage: 96.355% (+0.001%) from 96.354%

Build # 2c0ea772-0154-4a7d-8c5e-4a73107ca30a

Build Type

circleci

Committed by Ryan Laddusaw

Commit Message

Fix failing tests

Pull Request Pull Request #971: Checkout files before pull

Run Details

5 of 5 new or added lines in 2 files covered. (100.0%)

3 existing lines in 1 file now uncovered.

3225 of 3347 relevant lines covered (96.35%)

1268.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.59

/app/models/aspace_version_control/get_eads_job.rb

# frozen_string_literal: true
require 'archivesspace/client'
require 'nokogiri'
require 'fileutils'

# rubocop:disable Metrics/ClassLength
module AspaceVersionControl
  class GetEadsJob < LibJob
    attr_reader :repos
    def initialize(local_git_lab_dir: Rails.application.config.aspace.local_git_lab_dir)
      super(category: "EAD_export")
      @errors = []
      @local_git_lab_dir = local_git_lab_dir
      @repos = Rails.application.config.aspace.repos
    end

    def aspace_login
      aspace_client(aspace_config)
    end

    def aspace_config
      @config ||= ArchivesSpace::Configuration.new({
                                                     base_uri: ENV['ASPACE_URL'],
                                                     base_repo: "",
                                                     username: ENV['ASPACE_USER'],
                                                     password: ENV['ASPACE_PASSWORD'],
                                                     throttle: 0,
                                                     verify_ssl: false
                                                   })
    end

    def aspace_client(config)
      @client ||= ArchivesSpace::Client.new(config).login
    end

    def handle(data_set:)
      aspace_login
      GitLab.new.update
      repos.each do |repo, path|
        get_resource_ids_for_repo(repo)
        next unless @resource_ids

        prepare_and_commit_to_git_lab(repo, path)
      end
      data_set.data = report
      data_set.report_time = Time.zone.now
      data_set
    end

    def repo_path(dir, path)
      "#{dir}/#{path}"
    end

    def make_directories(dir)
      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
    end

    def get_resource_ids_for_repo(repo)
      retries ||= 0
      @resource_ids = @client.get("/repositories/#{repo}/resources", {
                                    query: { all_ids: true }
                                  }).parsed
      @resource_ids
    rescue Net::ReadTimeout => error
      while (retries += 1) <= 3
        Rails.logger.warn("Encountered #{error.class}: '#{error.message}' when connecting at #{Time.now.utc}, retrying in #{retries} second(s)...")
        sleep(retries)
        retry
      end
      Rails.logger.error("Encountered #{error.class}: '#{error.message}' at #{Time.now.utc}, unsuccessful in connecting after #{retries} retries")
    end

    def get_eads_from_ids(dir, repo, resource_ids)
      resource_ids.map { |id| write_eads_to_file(dir, repo, id) }
    end

    def report
      if @errors.empty?
        "EADs successfully exported."
      else
        @errors.join(', ')
      end
    end

    private

    def prepare_and_commit_to_git_lab(repo, path)
      git_lab_repo_path = repo_path(@local_git_lab_dir, path)
      Rails.logger.info("Preparing commit to GitLab for #{git_lab_repo_path}")

      make_directories(git_lab_repo_path)
      get_eads_from_ids(git_lab_repo_path, repo, @resource_ids)
      GitLab.new.commit_eads_to_git(path:)
    rescue Git::Error => error
      Rails.logger.error("Error updating EADS using GitLab for repo #{repo} at path #{path}.\nError: #{error}")
    end

    # Replace the namespace with the correct loc.gov one,
    # then write the results to the file
    def write_eads_to_file(dir, repo, id)
      Rails.logger.info("Now processing #{repo}/#{id}")
      record = aspace_record(repo, id)
      ead = Nokogiri::XML(record.body)
      rewrite_namespace(dir:, ead:)
    rescue Nokogiri::XML::SyntaxError
      err = "Unable to process XML for record #{repo}/#{id}, please check the source XML for errors"
      log_stdout(err)
      log_stderr(err)
    end

    def aspace_record(repo, id)
      retries ||= 0
      @client.get("/repositories/#{repo}/resource_descriptions/#{id}.xml", {
                    query: { include_daos: true }
                  })
    rescue Net::ReadTimeout => error
      while (retries += 1) <= 3
        Rails.logger.warn("Encountered #{error.class}: '#{error.message}' when connecting at #{Time.now.utc}, retrying in #{retries} second(s)...")
        sleep(retries)
        retry
      end
      Rails.logger.error("Encountered #{error.class}: '#{error.message}' at #{Time.now.utc}, unsuccessful in connecting after #{retries} retries")
    end

    def rewrite_namespace(dir:, ead:)
      ead.remove_namespaces!
      ead_id = ead.at_xpath('//eadid/text()')
      file = File.open("#{dir}/#{ead_id}.EAD.xml", "w")
      ead.child.add_namespace('ead', 'http://www.loc.gov/ead/ead')
      file << ead
      file.close
    end

    def log_stderr(stderr_str)
      @errors << stderr_str unless stderr_str.empty?
    end

    def log_stdout(stdout_str)
      Rails.logger.info(stdout_str) unless stdout_str.empty?
    end
  end
end
# rubocop:enable Metrics/ClassLength

1	# frozen_string_literal: true
2	require 'archivesspace/client'	1✔
3	require 'nokogiri'	1✔
4	require 'fileutils'	1✔
5
6	# rubocop:disable Metrics/ClassLength
7	module AspaceVersionControl	1✔
8	class GetEadsJob < LibJob	1✔
9	attr_reader :repos	1✔
10	def initialize(local_git_lab_dir: Rails.application.config.aspace.local_git_lab_dir)	1✔
11	super(category: "EAD_export")	6✔
12	@errors = []	6✔
13	@local_git_lab_dir = local_git_lab_dir	6✔
14	@repos = Rails.application.config.aspace.repos	6✔
15	end
16
17	def aspace_login	1✔
18	aspace_client(aspace_config)	6✔
19	end
20
21	def aspace_config	1✔
22	@config \|\|= ArchivesSpace::Configuration.new({	6✔
23	base_uri: ENV['ASPACE_URL'],
24	base_repo: "",
25	username: ENV['ASPACE_USER'],
26	password: ENV['ASPACE_PASSWORD'],
27	throttle: 0,
28	verify_ssl: false
29	})
30	end
31
32	def aspace_client(config)	1✔
33	@client \|\|= ArchivesSpace::Client.new(config).login	6✔
34	end
35
36	def handle(data_set:)	1✔
37	aspace_login	5✔
38	GitLab.new.update	5✔
39	repos.each do \|repo, path\|	5✔
40	get_resource_ids_for_repo(repo)	20✔
41	next unless @resource_ids	20✔
42
43	prepare_and_commit_to_git_lab(repo, path)	20✔
44	end
45	data_set.data = report	5✔
46	data_set.report_time = Time.zone.now	5✔
47	data_set	5✔
48	end
49
50	def repo_path(dir, path)	1✔
51	"#{dir}/#{path}"	20✔
52	end
53
54	def make_directories(dir)	1✔
55	FileUtils.mkdir_p(dir) unless Dir.exist?(dir)	20✔
56	end
57
58	def get_resource_ids_for_repo(repo)	1✔
59	retries \|\|= 0	24✔
60	@resource_ids = @client.get("/repositories/#{repo}/resources", {	24✔
61	query: { all_ids: true }
62	}).parsed
63	@resource_ids	20✔
64	rescue Net::ReadTimeout => error
65	while (retries += 1) <= 3	8✔
66	Rails.logger.warn("Encountered #{error.class}: '#{error.message}' when connecting at #{Time.now.utc}, retrying in #{retries} second(s)...")	3✔
67	sleep(retries)	3✔
68	retry	3✔
69	end
70	Rails.logger.error("Encountered #{error.class}: '#{error.message}' at #{Time.now.utc}, unsuccessful in connecting after #{retries} retries")	1✔
71	end
72
73	def get_eads_from_ids(dir, repo, resource_ids)	1✔
74	resource_ids.map { \|id\| write_eads_to_file(dir, repo, id) }	60✔
75	end
76
77	def report	1✔
78	if @errors.empty?	7✔
79	"EADs successfully exported."	5✔
80	else
81	@errors.join(', ')	2✔
82	end
83	end
84
85	private	1✔
86
87	def prepare_and_commit_to_git_lab(repo, path)	1✔
88	git_lab_repo_path = repo_path(@local_git_lab_dir, path)	20✔
89	Rails.logger.info("Preparing commit to GitLab for #{git_lab_repo_path}")	20✔
90
91	make_directories(git_lab_repo_path)	20✔
92	get_eads_from_ids(git_lab_repo_path, repo, @resource_ids)	20✔
93	GitLab.new.commit_eads_to_git(path:)	20✔
94	rescue Git::Error => error
UNCOV 95	Rails.logger.error("Error updating EADS using GitLab for repo #{repo} at path #{path}.\nError: #{error}")	×
96	end
97
98	# Replace the namespace with the correct loc.gov one,
99	# then write the results to the file
100	def write_eads_to_file(dir, repo, id)	1✔
101	Rails.logger.info("Now processing #{repo}/#{id}")	40✔
102	record = aspace_record(repo, id)	39✔
103	ead = Nokogiri::XML(record.body)	39✔
104	rewrite_namespace(dir:, ead:)	39✔
105	rescue Nokogiri::XML::SyntaxError
106	err = "Unable to process XML for record #{repo}/#{id}, please check the source XML for errors"	1✔
107	log_stdout(err)	1✔
108	log_stderr(err)	1✔
109	end
110
111	def aspace_record(repo, id)	1✔
112	retries \|\|= 0	39✔
113	@client.get("/repositories/#{repo}/resource_descriptions/#{id}.xml", {	39✔
114	query: { include_daos: true }
115	})
116	rescue Net::ReadTimeout => error
UNCOV 117	while (retries += 1) <= 3	×
118	Rails.logger.warn("Encountered #{error.class}: '#{error.message}' when connecting at #{Time.now.utc}, retrying in #{retries} second(s)...")	×
119	sleep(retries)	×
120	retry	×
121	end
UNCOV 122	Rails.logger.error("Encountered #{error.class}: '#{error.message}' at #{Time.now.utc}, unsuccessful in connecting after #{retries} retries")	×
123	end
124
125	def rewrite_namespace(dir:, ead:)	1✔
126	ead.remove_namespaces!	39✔
127	ead_id = ead.at_xpath('//eadid/text()')	39✔
128	file = File.open("#{dir}/#{ead_id}.EAD.xml", "w")	39✔
129	ead.child.add_namespace('ead', 'http://www.loc.gov/ead/ead')	39✔
130	file << ead	39✔
131	file.close	39✔
132	end
133
134	def log_stderr(stderr_str)	1✔
135	@errors << stderr_str unless stderr_str.empty?	1✔
136	end
137
138	def log_stdout(stdout_str)	1✔
139	Rails.logger.info(stdout_str) unless stdout_str.empty?	1✔
140	end
141	end
142	end
143	# rubocop:enable Metrics/ClassLength

pulibrary / lib_jobs / 2c0ea772-0154-4a7d-8c5e-4a73107ca30a

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous