4f01b5ec-96e4-40e5-bb74-6da450726a2e

Build # 4f01b5ec-96e4-40e5-bb74-6da450726a2e

Build Type

circleci

Committed by Jane Sandberg

Commit Message

explicitly require redis gem

Pull Request Pull Request #2143: Bump sidekiq and sidekiq-pro

Run Details

3555 of 3942 relevant lines covered (90.18%)

307.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

40.0

/app/models/index_manager.rb

class IndexManager < ActiveRecord::Base
  # @param solr_url Solr URL to get the index manager for (or initialize a new
  #   one)
  # @returns [IndexManager]
  def self.for(solr_url)
    IndexManager.find_or_initialize_by(solr_collection: solr_url)
  end

  def self.reindex!(solr_url: nil)
    solr_url ||= rebuild_solr_url
    manager = self.for(solr_url)
    return false if manager.in_progress?
    manager.last_dump_completed = nil
    manager.save
    manager.wipe!
    manager.index_remaining!
    true
  end

  def self.rebuild_solr_url
    "#{Rails.application.config.solr['url']}-rebuild"
  end

  belongs_to :dump_in_progress, class_name: "Dump"
  belongs_to :last_dump_completed, class_name: "Dump"

  def wipe!
    RSolr.connect(url: solr_collection).delete_by_query("*:*")
  end

  def index_next_dump!
    return unless next_dump
    self.dump_in_progress = next_dump
    self.in_progress = true
    save
    generate_batch do
      next_dump.dump_files.each do |dump_file|
        DumpFileIndexJob.perform_async(dump_file.id, solr_collection)
      end
    end
  end

  def index_remaining!
    logger.warn "IndexManager is in progress" if in_progress?

    # Don't do anything unless there's a job to index and we're not already
    # indexing.
    return unless next_dump && !in_progress?
    save!
    # Create an overall catchup batch.
    batch = Sidekiq::Batch.new
    batch.on(:success, "IndexManager::Workflow#indexed_remaining", 'index_manager_id' => id)
    batch.description = "Performing catch-up index into #{solr_collection}"
    batch.jobs do
      IndexRemainingDumpsJob.perform_async(id)
    end
  end

  def generate_batch
    batch = Sidekiq::Batch.new
    batch.on(:success, IndexManager::Workflow, 'dump_id' => next_dump.id, 'index_manager_id' => id)
    batch.description = "Indexing Dump #{next_dump.id} into #{solr_collection}"
    batch.jobs do
      yield
    end
  end

  def next_dump
    @next_dump ||=
      if last_dump_completed&.full_dump?
        previous_to_full_incremental || next_incremental
      elsif last_dump_completed
        next_incremental
      else
        recent_full_dump || first_incremental
      end
  end

  def recent_full_dump
    Dump.full_dumps.joins(:event).order("events.start" => "DESC").first
  end

  def first_incremental
    Dump.changed_records.joins(:event).order("events.start" => "ASC").first
  end

  def previous_to_full_incremental
    Dump.changed_records.joins(:event).where("events.start < ?", last_dump_completed.event.start.to_s).where.not(id: last_dump_completed.id).order("events.start" => "DESC").first
  end

  def next_incremental
    incremental_dump = Dump.changed_records.joins(:event).where("events.start": last_dump_completed.event.start..Float::INFINITY).where.not(id: last_dump_completed.id).order("events.start" => "ASC").first
    if incremental_dump&.dump_files&.empty?
      self.last_dump_completed = incremental_dump
      incremental_dump = next_incremental
    end
    incremental_dump
  end

  class Workflow
    # Callback for when the batch of DumpFiles is done indexing.
    def on_success(status, options)
      index_manager = IndexManager.find(options['index_manager_id'])
      dump = Dump.find(options['dump_id'])
      index_manager.last_dump_completed = dump
      index_manager.dump_in_progress = nil
      index_manager.save
      # If there's a parent batch it's meant to keep going until it runs out of
      # dumps.
      if status.parent_bid
        return unless index_manager.next_dump
        overall = Sidekiq::Batch.new(status.parent_bid)
        overall.jobs do
          index_manager.index_next_dump!
        end
      else
        index_manager.in_progress = false
        index_manager.save
      end
    end

    def indexed_remaining(_status, options)
      index_manager = IndexManager.find(options['index_manager_id'])
      RSolr.connect(url: index_manager.solr_collection).commit
      index_manager.dump_in_progress = nil
      index_manager.in_progress = false
      index_manager.save
    end
  end
end

1	class IndexManager < ActiveRecord::Base	1✔
2	# @param solr_url Solr URL to get the index manager for (or initialize a new
3	# one)
4	# @returns [IndexManager]
5	def self.for(solr_url)	1✔
6	IndexManager.find_or_initialize_by(solr_collection: solr_url)	1✔
7	end
8
9	def self.reindex!(solr_url: nil)	1✔
10	solr_url \|\|= rebuild_solr_url	×
11	manager = self.for(solr_url)	×
12	return false if manager.in_progress?	×
13	manager.last_dump_completed = nil	×
14	manager.save	×
15	manager.wipe!	×
16	manager.index_remaining!	×
17	true	×
18	end
19
20	def self.rebuild_solr_url	1✔
21	"#{Rails.application.config.solr['url']}-rebuild"	×
22	end
23
24	belongs_to :dump_in_progress, class_name: "Dump"	1✔
25	belongs_to :last_dump_completed, class_name: "Dump"	1✔
26
27	def wipe!	1✔
28	RSolr.connect(url: solr_collection).delete_by_query(":")	×
29	end
30
31	def index_next_dump!	1✔
32	return unless next_dump	×
33	self.dump_in_progress = next_dump	×
34	self.in_progress = true	×
35	save	×
36	generate_batch do	×
37	next_dump.dump_files.each do \|dump_file\|	×
38	DumpFileIndexJob.perform_async(dump_file.id, solr_collection)	×
39	end
40	end
41	end
42
43	def index_remaining!	1✔
44	logger.warn "IndexManager is in progress" if in_progress?	1✔
45
46	# Don't do anything unless there's a job to index and we're not already
47	# indexing.
48	return unless next_dump && !in_progress?	1✔
49	save!	1✔
50	# Create an overall catchup batch.
51	batch = Sidekiq::Batch.new	1✔
52	batch.on(:success, "IndexManager::Workflow#indexed_remaining", 'index_manager_id' => id)	1✔
53	batch.description = "Performing catch-up index into #{solr_collection}"	1✔
54	batch.jobs do	1✔
55	IndexRemainingDumpsJob.perform_async(id)	×
56	end
57	end
58
59	def generate_batch	1✔
60	batch = Sidekiq::Batch.new	×
61	batch.on(:success, IndexManager::Workflow, 'dump_id' => next_dump.id, 'index_manager_id' => id)	×
62	batch.description = "Indexing Dump #{next_dump.id} into #{solr_collection}"	×
63	batch.jobs do	×
64	yield	×
65	end
66	end
67
68	def next_dump	1✔
69	@next_dump \|\|=	1✔
70	if last_dump_completed&.full_dump?	1✔
71	previous_to_full_incremental \|\| next_incremental	×
72	elsif last_dump_completed	1✔
73	next_incremental	×
74	else
75	recent_full_dump \|\| first_incremental	1✔
76	end
77	end
78
79	def recent_full_dump	1✔
80	Dump.full_dumps.joins(:event).order("events.start" => "DESC").first	1✔
81	end
82
83	def first_incremental	1✔
84	Dump.changed_records.joins(:event).order("events.start" => "ASC").first	1✔
85	end
86
87	def previous_to_full_incremental	1✔
88	Dump.changed_records.joins(:event).where("events.start < ?", last_dump_completed.event.start.to_s).where.not(id: last_dump_completed.id).order("events.start" => "DESC").first	×
89	end
90
91	def next_incremental	1✔
92	incremental_dump = Dump.changed_records.joins(:event).where("events.start": last_dump_completed.event.start..Float::INFINITY).where.not(id: last_dump_completed.id).order("events.start" => "ASC").first	×
93	if incremental_dump&.dump_files&.empty?	×
94	self.last_dump_completed = incremental_dump	×
95	incremental_dump = next_incremental	×
96	end
97	incremental_dump	×
98	end
99
100	class Workflow	1✔
101	# Callback for when the batch of DumpFiles is done indexing.
102	def on_success(status, options)	1✔
103	index_manager = IndexManager.find(options['index_manager_id'])	×
104	dump = Dump.find(options['dump_id'])	×
105	index_manager.last_dump_completed = dump	×
106	index_manager.dump_in_progress = nil	×
107	index_manager.save	×
108	# If there's a parent batch it's meant to keep going until it runs out of
109	# dumps.
110	if status.parent_bid	×
111	return unless index_manager.next_dump	×
112	overall = Sidekiq::Batch.new(status.parent_bid)	×
113	overall.jobs do	×
114	index_manager.index_next_dump!	×
115	end
116	else
117	index_manager.in_progress = false	×
118	index_manager.save	×
119	end
120	end
121
122	def indexed_remaining(_status, options)	1✔
123	index_manager = IndexManager.find(options['index_manager_id'])	×
124	RSolr.connect(url: index_manager.solr_collection).commit	×
125	index_manager.dump_in_progress = nil	×
126	index_manager.in_progress = false	×
127	index_manager.save	×
128	end
129	end
130	end

pulibrary / bibdata / 4f01b5ec-96e4-40e5-bb74-6da450726a2e

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous