ab0f75ee-b22c-4a73-9960-46f76c4800c9

Committed 13 Oct 2025 02:58PM UTC coverage: 96.319%. Remained the same

Build # ab0f75ee-b22c-4a73-9960-46f76c4800c9

Build Type

circleci

Committed by Ryan Laddusaw

Commit Message

Add symlink for staging repo

Pull Request Pull Request #975: Use a environment variable to chose the gitlab repo path

Run Details

10 of 10 new or added lines in 2 files covered. (100.0%)

6 existing lines in 1 file now uncovered.

3349 of 3477 relevant lines covered (96.32%)

1221.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.68

/app/models/aspace_version_control/get_eads_job.rb

# frozen_string_literal: true
require 'archivesspace/client'
require 'nokogiri'
require 'fileutils'

# rubocop:disable Metrics/ClassLength
module AspaceVersionControl
  class GetEadsJob < LibJob
    attr_reader :repos
    def initialize(local_git_lab_dir: Rails.application.config.aspace.local_git_lab_dir)
      super(category: "EAD_export")
      @errors = []
      @local_git_lab_dir = ENV['GIT_LAB_DIR'] || local_git_lab_dir
      @repos = Rails.application.config.aspace.repos
    end

    def aspace_login
      aspace_client(aspace_config)
    end

    def aspace_config
      @config ||= ArchivesSpace::Configuration.new({
                                                     base_uri: ENV['ASPACE_URL'],
                                                     base_repo: "",
                                                     username: ENV['ASPACE_USER'],
                                                     password: ENV['ASPACE_PASSWORD'],
                                                     throttle: 0,
                                                     verify_ssl: false
                                                   })
    end

    def aspace_client(config)
      @client ||= ArchivesSpace::Client.new(config).login
    end

    def handle(data_set:)
      aspace_login
      Rails.logger.info("Opening Repo at #{@local_git_lab_dir}")
      GitLab.new.update(path: @local_git_lab_dir)
      repos.each do |repo, path|
        get_resource_ids_for_repo(repo)
        next unless @resource_ids

        prepare_and_commit_to_git_lab(repo, path)
      end
      data_set.data = report
      data_set.report_time = Time.zone.now
      data_set
    end

    def repo_path(dir, path)
      "#{dir}/#{path}"
    end

    def make_directories(dir)
      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
    end

    def get_resource_ids_for_repo(repo)
      retries ||= 0
      @resource_ids = @client.get("/repositories/#{repo}/resources", {
                                    query: { all_ids: true }
                                  }).parsed
      @resource_ids
    rescue Net::ReadTimeout => error
      while (retries += 1) <= 3
        Rails.logger.warn("Encountered #{error.class}: '#{error.message}' when connecting at #{Time.now.utc}, retrying in #{retries} second(s)...")
        sleep(retries)
        retry
      end
      Rails.logger.error("Encountered #{error.class}: '#{error.message}' at #{Time.now.utc}, unsuccessful in connecting after #{retries} retries")
    end

    def get_eads_from_ids(dir, repo, resource_ids)
      resource_ids.map { |id| write_eads_to_file(dir, repo, id) }
    end

    def report
      if @errors.empty?
        "EADs successfully exported."
      else
        @errors.join(', ')
      end
    end

    private

    def prepare_and_commit_to_git_lab(repo, path)
      git_lab_repo_path = repo_path(@local_git_lab_dir, path)
      Rails.logger.info("Preparing commit to GitLab for #{git_lab_repo_path}")

      make_directories(git_lab_repo_path)
      get_eads_from_ids(git_lab_repo_path, repo, @resource_ids)
      GitLab.new.commit_eads_to_git(path:)
    rescue Git::Error => error
      Rails.logger.error("Error updating EADS using GitLab for repo #{repo} at path #{path}.\nError: #{error}")
    end

    # Replace the namespace with the correct loc.gov one,
    # then write the results to the file
    def write_eads_to_file(dir, repo, id)
      Rails.logger.info("Now processing #{repo}/#{id}")
      record = aspace_record(repo, id)
      ead = Nokogiri::XML(record.body)
      rewrite_namespace(dir:, ead:)
    rescue Nokogiri::XML::SyntaxError
      err = "Unable to process XML for record #{repo}/#{id}, please check the source XML for errors"
      log_stdout(err)
      log_stderr(err)
    end

    def aspace_record(repo, id)
      retries ||= 0
      @client.get("/repositories/#{repo}/resource_descriptions/#{id}.xml", {
                    query: { include_daos: true }
                  })
    rescue Net::ReadTimeout => error
      while (retries += 1) <= 3
        Rails.logger.warn("Encountered #{error.class}: '#{error.message}' when connecting at #{Time.now.utc}, retrying in #{retries} second(s)...")
        sleep(retries)
        retry
      end
      Rails.logger.error("Encountered #{error.class}: '#{error.message}' at #{Time.now.utc}, unsuccessful in connecting after #{retries} retries")
    end

    def rewrite_namespace(dir:, ead:)
      ead.remove_namespaces!
      ead_id = ead.at_xpath('//eadid/text()')
      file = File.open("#{dir}/#{ead_id}.EAD.xml", "w")
      ead.child.add_namespace('ead', 'http://www.loc.gov/ead/ead')
      file << ead
      file.close
    end

    def log_stderr(stderr_str)
      @errors << stderr_str unless stderr_str.empty?
    end

    def log_stdout(stdout_str)
      Rails.logger.info(stdout_str) unless stdout_str.empty?
    end
  end
end
# rubocop:enable Metrics/ClassLength

1	# frozen_string_literal: true
2	require 'archivesspace/client'	1✔
3	require 'nokogiri'	1✔
4	require 'fileutils'	1✔
5
6	# rubocop:disable Metrics/ClassLength
7	module AspaceVersionControl	1✔
8	class GetEadsJob < LibJob	1✔
9	attr_reader :repos	1✔
10	def initialize(local_git_lab_dir: Rails.application.config.aspace.local_git_lab_dir)	1✔
11	super(category: "EAD_export")	6✔
12	@errors = []	6✔
13	@local_git_lab_dir = ENV['GIT_LAB_DIR'] \|\| local_git_lab_dir	6✔
14	@repos = Rails.application.config.aspace.repos	6✔
15	end
16
17	def aspace_login	1✔
18	aspace_client(aspace_config)	6✔
19	end
20
21	def aspace_config	1✔
22	@config \|\|= ArchivesSpace::Configuration.new({	6✔
23	base_uri: ENV['ASPACE_URL'],
24	base_repo: "",
25	username: ENV['ASPACE_USER'],
26	password: ENV['ASPACE_PASSWORD'],
27	throttle: 0,
28	verify_ssl: false
29	})
30	end
31
32	def aspace_client(config)	1✔
33	@client \|\|= ArchivesSpace::Client.new(config).login	6✔
34	end
35
36	def handle(data_set:)	1✔
37	aspace_login	5✔
38	Rails.logger.info("Opening Repo at #{@local_git_lab_dir}")	5✔
39	GitLab.new.update(path: @local_git_lab_dir)	5✔
40	repos.each do \|repo, path\|	5✔
41	get_resource_ids_for_repo(repo)	20✔
42	next unless @resource_ids	20✔
43
44	prepare_and_commit_to_git_lab(repo, path)	20✔
45	end
46	data_set.data = report	5✔
47	data_set.report_time = Time.zone.now	5✔
48	data_set	5✔
49	end
50
51	def repo_path(dir, path)	1✔
52	"#{dir}/#{path}"	20✔
53	end
54
55	def make_directories(dir)	1✔
56	FileUtils.mkdir_p(dir) unless Dir.exist?(dir)	20✔
57	end
58
59	def get_resource_ids_for_repo(repo)	1✔
60	retries \|\|= 0	24✔
61	@resource_ids = @client.get("/repositories/#{repo}/resources", {	24✔
62	query: { all_ids: true }
63	}).parsed
64	@resource_ids	20✔
65	rescue Net::ReadTimeout => error
66	while (retries += 1) <= 3	8✔
67	Rails.logger.warn("Encountered #{error.class}: '#{error.message}' when connecting at #{Time.now.utc}, retrying in #{retries} second(s)...")	3✔
68	sleep(retries)	3✔
69	retry	3✔
70	end
71	Rails.logger.error("Encountered #{error.class}: '#{error.message}' at #{Time.now.utc}, unsuccessful in connecting after #{retries} retries")	1✔
72	end
73
74	def get_eads_from_ids(dir, repo, resource_ids)	1✔
75	resource_ids.map { \|id\| write_eads_to_file(dir, repo, id) }	60✔
76	end
77
78	def report	1✔
79	if @errors.empty?	7✔
80	"EADs successfully exported."	5✔
81	else
82	@errors.join(', ')	2✔
83	end
84	end
85
86	private	1✔
87
88	def prepare_and_commit_to_git_lab(repo, path)	1✔
89	git_lab_repo_path = repo_path(@local_git_lab_dir, path)	20✔
90	Rails.logger.info("Preparing commit to GitLab for #{git_lab_repo_path}")	20✔
91
92	make_directories(git_lab_repo_path)	20✔
93	get_eads_from_ids(git_lab_repo_path, repo, @resource_ids)	20✔
94	GitLab.new.commit_eads_to_git(path:)	20✔
95	rescue Git::Error => error
UNCOV 96	Rails.logger.error("Error updating EADS using GitLab for repo #{repo} at path #{path}.\nError: #{error}")	×
97	end
98
99	# Replace the namespace with the correct loc.gov one,
100	# then write the results to the file
101	def write_eads_to_file(dir, repo, id)	1✔
102	Rails.logger.info("Now processing #{repo}/#{id}")	40✔
103	record = aspace_record(repo, id)	39✔
104	ead = Nokogiri::XML(record.body)	39✔
105	rewrite_namespace(dir:, ead:)	39✔
106	rescue Nokogiri::XML::SyntaxError
107	err = "Unable to process XML for record #{repo}/#{id}, please check the source XML for errors"	1✔
108	log_stdout(err)	1✔
109	log_stderr(err)	1✔
110	end
111
112	def aspace_record(repo, id)	1✔
113	retries \|\|= 0	39✔
114	@client.get("/repositories/#{repo}/resource_descriptions/#{id}.xml", {	39✔
115	query: { include_daos: true }
116	})
117	rescue Net::ReadTimeout => error
UNCOV 118	while (retries += 1) <= 3	×
UNCOV 119	Rails.logger.warn("Encountered #{error.class}: '#{error.message}' when connecting at #{Time.now.utc}, retrying in #{retries} second(s)...")	×
UNCOV 120	sleep(retries)	×
UNCOV 121	retry	×
122	end
UNCOV 123	Rails.logger.error("Encountered #{error.class}: '#{error.message}' at #{Time.now.utc}, unsuccessful in connecting after #{retries} retries")	×
124	end
125
126	def rewrite_namespace(dir:, ead:)	1✔
127	ead.remove_namespaces!	39✔
128	ead_id = ead.at_xpath('//eadid/text()')	39✔
129	file = File.open("#{dir}/#{ead_id}.EAD.xml", "w")	39✔
130	ead.child.add_namespace('ead', 'http://www.loc.gov/ead/ead')	39✔
131	file << ead	39✔
132	file.close	39✔
133	end
134
135	def log_stderr(stderr_str)	1✔
136	@errors << stderr_str unless stderr_str.empty?	1✔
137	end
138
139	def log_stdout(stdout_str)	1✔
140	Rails.logger.info(stdout_str) unless stdout_str.empty?	1✔
141	end
142	end
143	end
144	# rubocop:enable Metrics/ClassLength

pulibrary / lib_jobs / ab0f75ee-b22c-4a73-9960-46f76c4800c9

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous