2b274cd3-5d04-438b-ba47-a8d5eeac0c16

Committed 11 Aug 2025 04:31PM UTC coverage: 87.857% (-4.3%) from 92.158%

Build # 2b274cd3-5d04-438b-ba47-a8d5eeac0c16

Build Type

Pull #2830

circleci

Committed by

sandbergja

Commit Message

[#2746] Begin indexing access restriction notes

Advances #2746

Pull Request Pull Request #2830: [#2746] Begin indexing access restriction notes

Run Details

3350 of 3813 relevant lines covered (87.86%)

668.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.09

/app/models/index_manager.rb

class IndexManager < ActiveRecord::Base
  # @param solr_url Solr URL to get the index manager for (or initialize a new
  #   one)
  # @returns [IndexManager]
  def self.for(solr_url)
    IndexManager.find_or_initialize_by(solr_collection: solr_url)
  end

  def self.reindex!(solr_url: nil)
    solr_url ||= rebuild_solr_url
    manager = self.for(solr_url)
    return false if manager.in_progress?

    manager.last_dump_completed = nil
    manager.save
    manager.wipe!
    manager.index_remaining!
    true
  end

  def self.rebuild_solr_url
    "#{Rails.application.config.solr['url']}-rebuild"
  end

  belongs_to :dump_in_progress, class_name: 'Dump'
  belongs_to :last_dump_completed, class_name: 'Dump'

  def wipe!
    RSolr.connect(url: solr_collection).delete_by_query('*:*')
  end

  def index_next_dump!
    return unless next_dump

    self.dump_in_progress = next_dump
    self.in_progress = true
    save
    generate_batch do
      next_dump.dump_files.each do |dump_file|
        Index::DumpFileJob.perform_async(dump_file.id, solr_collection)
      end
    end
  end

  def index_remaining!
    logger.warn 'IndexManager is in progress' if in_progress?

    # Don't do anything unless there's a job to index and we're not already
    # indexing.
    return unless next_dump && !in_progress?

    save!
    # Create an overall catchup batch.
    batch = Sidekiq::Batch.new
    batch.on(:success, 'IndexManager::Workflow#indexed_remaining', 'index_manager_id' => id)
    batch.description = "Performing catch-up index into #{solr_collection}"
    batch.jobs do
      Index::RemainingDumpsJob.perform_async(id)
    end
  end

  def generate_batch(&)
    batch = Sidekiq::Batch.new
    batch.on(:success, IndexManager::Workflow, 'dump_id' => next_dump.id, 'index_manager_id' => id)
    batch.description = "Indexing Dump #{next_dump.id} into #{solr_collection}"
    batch.jobs(&)
  end

  def next_dump
    @next_dump ||=
      if last_dump_completed&.full_dump?
        previous_to_full_incremental || next_incremental
      elsif last_dump_completed
        next_incremental
      else
        recent_full_dump || first_incremental
      end
  end

  def recent_full_dump
    Dump.full_dump.joins(:event).order('events.start' => 'DESC').first
  end

  def first_incremental
    Dump.changed_records.joins(:event).order('events.start' => 'ASC').first
  end

  def previous_to_full_incremental
    Dump.changed_records.joins(:event).where('events.start < ?', last_dump_completed.event.start.to_s).where.not(id: last_dump_completed.id).order('events.start' => 'DESC').first
  end

  def next_incremental
    incremental_dump = Dump.changed_records.joins(:event).where('events.start': last_dump_completed.event.start..Float::INFINITY).where.not(id: last_dump_completed.id).order('events.start' => 'ASC').first
    if incremental_dump&.dump_files&.empty?
      self.last_dump_completed = incremental_dump
      incremental_dump = next_incremental
    end
    incremental_dump
  end

  class Workflow
    # Callback for when the batch of DumpFiles is done indexing.
    def on_success(status, options)
      index_manager = IndexManager.find(options['index_manager_id'])
      dump = Dump.find(options['dump_id'])
      index_manager.last_dump_completed = dump
      index_manager.dump_in_progress = nil
      index_manager.save
      # If there's a parent batch it's meant to keep going until it runs out of
      # dumps.
      if status.parent_bid
        return unless index_manager.next_dump

        overall = Sidekiq::Batch.new(status.parent_bid)
        overall.jobs do
          index_manager.index_next_dump!
        end
      else
        index_manager.in_progress = false
        index_manager.save
      end
    end

    def indexed_remaining(_status, options)
      index_manager = IndexManager.find(options['index_manager_id'])
      RSolr.connect(url: index_manager.solr_collection).commit
      index_manager.dump_in_progress = nil
      index_manager.in_progress = false
      index_manager.save
    end
  end
end

1	class IndexManager < ActiveRecord::Base	1✔
2	# @param solr_url Solr URL to get the index manager for (or initialize a new
3	# one)
4	# @returns [IndexManager]
5	def self.for(solr_url)	1✔
6	IndexManager.find_or_initialize_by(solr_collection: solr_url)	7✔
7	end
8
9	def self.reindex!(solr_url: nil)	1✔
10	solr_url \|\|= rebuild_solr_url	1✔
11	manager = self.for(solr_url)	1✔
12	return false if manager.in_progress?	1✔
13
14	manager.last_dump_completed = nil	1✔
15	manager.save	1✔
16	manager.wipe!	1✔
17	manager.index_remaining!	1✔
18	true	×
19	end
20
21	def self.rebuild_solr_url	1✔
22	"#{Rails.application.config.solr['url']}-rebuild"	1✔
23	end
24
25	belongs_to :dump_in_progress, class_name: 'Dump'	1✔
26	belongs_to :last_dump_completed, class_name: 'Dump'	1✔
27
28	def wipe!	1✔
29	RSolr.connect(url: solr_collection).delete_by_query(':')	1✔
30	end
31
32	def index_next_dump!	1✔
33	return unless next_dump	5✔
34
35	self.dump_in_progress = next_dump	5✔
36	self.in_progress = true	5✔
37	save	5✔
38	generate_batch do	5✔
39	next_dump.dump_files.each do \|dump_file\|	5✔
40	Index::DumpFileJob.perform_async(dump_file.id, solr_collection)	10✔
41	end
42	end
43	end
44
45	def index_remaining!	1✔
46	logger.warn 'IndexManager is in progress' if in_progress?	3✔
47
48	# Don't do anything unless there's a job to index and we're not already
49	# indexing.
50	return unless next_dump && !in_progress?	3✔
51
52	save!	3✔
53	# Create an overall catchup batch.
54	batch = Sidekiq::Batch.new	3✔
55	batch.on(:success, 'IndexManager::Workflow#indexed_remaining', 'index_manager_id' => id)	3✔
56	batch.description = "Performing catch-up index into #{solr_collection}"	3✔
57	batch.jobs do	3✔
58	Index::RemainingDumpsJob.perform_async(id)	3✔
59	end
60	end
61
62	def generate_batch(&)	1✔
63	batch = Sidekiq::Batch.new	5✔
64	batch.on(:success, IndexManager::Workflow, 'dump_id' => next_dump.id, 'index_manager_id' => id)	5✔
65	batch.description = "Indexing Dump #{next_dump.id} into #{solr_collection}"	5✔
66	batch.jobs(&)	5✔
67	end
68
69	def next_dump	1✔
70	@next_dump \|\|=	30✔
71	if last_dump_completed&.full_dump?	8✔
72	previous_to_full_incremental \|\| next_incremental	×
73	elsif last_dump_completed	8✔
74	next_incremental	×
75	else
76	recent_full_dump \|\| first_incremental	8✔
77	end
78	end
79
80	def recent_full_dump	1✔
81	Dump.full_dump.joins(:event).order('events.start' => 'DESC').first	8✔
82	end
83
84	def first_incremental	1✔
85	Dump.changed_records.joins(:event).order('events.start' => 'ASC').first	1✔
86	end
87
88	def previous_to_full_incremental	1✔
89	Dump.changed_records.joins(:event).where('events.start < ?', last_dump_completed.event.start.to_s).where.not(id: last_dump_completed.id).order('events.start' => 'DESC').first	×
90	end
91
92	def next_incremental	1✔
93	incremental_dump = Dump.changed_records.joins(:event).where('events.start': last_dump_completed.event.start..Float::INFINITY).where.not(id: last_dump_completed.id).order('events.start' => 'ASC').first	×
94	if incremental_dump&.dump_files&.empty?	×
95	self.last_dump_completed = incremental_dump	×
96	incremental_dump = next_incremental	×
97	end
98	incremental_dump	×
99	end
100
101	class Workflow	1✔
102	# Callback for when the batch of DumpFiles is done indexing.
103	def on_success(status, options)	1✔
104	index_manager = IndexManager.find(options['index_manager_id'])	×
105	dump = Dump.find(options['dump_id'])	×
106	index_manager.last_dump_completed = dump	×
107	index_manager.dump_in_progress = nil	×
108	index_manager.save	×
109	# If there's a parent batch it's meant to keep going until it runs out of
110	# dumps.
111	if status.parent_bid	×
112	return unless index_manager.next_dump	×
113
114	overall = Sidekiq::Batch.new(status.parent_bid)	×
115	overall.jobs do	×
116	index_manager.index_next_dump!	×
117	end
118	else
119	index_manager.in_progress = false	×
120	index_manager.save	×
121	end
122	end
123
124	def indexed_remaining(_status, options)	1✔
125	index_manager = IndexManager.find(options['index_manager_id'])	×
126	RSolr.connect(url: index_manager.solr_collection).commit	×
127	index_manager.dump_in_progress = nil	×
128	index_manager.in_progress = false	×
129	index_manager.save	×
130	end
131	end
132	end

pulibrary / bibdata / 2b274cd3-5d04-438b-ba47-a8d5eeac0c16

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous