373ad6ff-fad2-405a-ab26-3d30fb5ceecf

Committed 24 Dec 2024 08:24PM UTC coverage: 91.938% (+0.08%) from 91.859%

Build # 373ad6ff-fad2-405a-ab26-3d30fb5ceecf

Build Type

Pull #2563

circleci

Committed by

maxkadel

Commit Message

Put attaching xml files in their own batch

Pull Request Pull Request #2563: I2321 Shift SCSB full index tasks into separate background jobs

Run Details

152 of 156 new or added lines in 10 files covered. (97.44%)

65 existing lines in 17 files now uncovered.

3478 of 3783 relevant lines covered (91.94%)

366.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.88

/app/models/dump.rb

# encoding: utf-8
require 'zip'
require 'net/sftp'
require 'date'

class Dump < ActiveRecord::Base
  ENQUEUED = 'enqueued'.freeze
  STARTED = 'started'.freeze
  DONE = 'done'.freeze

  belongs_to :event
  has_many :dump_files
  # These only apply to change dumps (stored in db rather than text files)
  serialize :delete_ids
  serialize :update_ids
  serialize :recap_barcodes
  validates :event_id, presence: true

  before_destroy do
    self.dump_files.each do |df|
      df.destroy
    end
  end

  enum dump_type: {
    full_dump: 1,
    changed_records: 2,
    princeton_recap: 3,
    partner_recap: 4,
    partner_recap_full: 5,
    bib_ids: 6,
    merged_ids: 7
  }

  class << self
    ##
    # Create a dump of partner recap incremental records.
    def partner_update
      dump = nil
      timestamp = incremental_update_timestamp
      Event.record do |event|
        event.save
        dump = Dump.create(dump_type: :partner_recap, event_id: event.id)
        ScsbImportJob.perform_later(dump.id, timestamp)
        dump.save
      end
      dump
    end

    def latest_generated
      order('generated_date desc').first
    end

    def attach_dump_file(dump_id:, filepath:, dump_file_type: :recap_records_full)
      df = DumpFile.create(dump_file_type:, path: filepath)
      df.zip
      df.save!
      dump = Dump.find(dump_id)
      dump.dump_files << df
      dump.save!
    end

    private

      ##
      # Figure out when was the last time we got partner update records, and find
      # anything that has appeared since then. The ways we can know the last time
      # we got partner update records are:
      # * via an environment variable
      # * look for the last partner recap dump and get its creation date
      # * or, if neither of those, use yesterday
      def incremental_update_timestamp
        (ENV['TIMESTAMP'] || last_incremental_update || DateTime.now - 1).to_time.strftime('%Y-%m-%d %H:%M:%S.%6N %z')
      end

      def last_incremental_update
        Dump.partner_recap.last&.created_at
      end
  end # class << self

  def subsequent_partner_incrementals
    Dump.partner_recap.where(generated_date: generated_date..Float::INFINITY)
  end

  def enqueued?
    index_status == ENQUEUED
  end

  def started?
    index_status == STARTED
  end

  def done?
    index_status == DONE
  end
end

1	# encoding: utf-8
2	require 'zip'	1✔
3	require 'net/sftp'	1✔
4	require 'date'	1✔
5
6	class Dump < ActiveRecord::Base	1✔
7	ENQUEUED = 'enqueued'.freeze	1✔
8	STARTED = 'started'.freeze	1✔
9	DONE = 'done'.freeze	1✔
10
11	belongs_to :event	1✔
12	has_many :dump_files	1✔
13	# These only apply to change dumps (stored in db rather than text files)
14	serialize :delete_ids	1✔
15	serialize :update_ids	1✔
16	serialize :recap_barcodes	1✔
17	validates :event_id, presence: true	1✔
18
19	before_destroy do	1✔
20	self.dump_files.each do \|df\|	32✔
21	df.destroy	22✔
22	end
23	end
24
25	enum dump_type: {	1✔
26	full_dump: 1,
27	changed_records: 2,
28	princeton_recap: 3,
29	partner_recap: 4,
30	partner_recap_full: 5,
31	bib_ids: 6,
32	merged_ids: 7
33	}
34
35	class << self	1✔
36	##
37	# Create a dump of partner recap incremental records.
38	def partner_update	1✔
39	dump = nil	2✔
40	timestamp = incremental_update_timestamp	2✔
41	Event.record do \|event\|	2✔
42	event.save	2✔
43	dump = Dump.create(dump_type: :partner_recap, event_id: event.id)	2✔
44	ScsbImportJob.perform_later(dump.id, timestamp)	2✔
45	dump.save	2✔
46	end
47	dump	2✔
48	end
49
50	def latest_generated	1✔
51	order('generated_date desc').first	1✔
52	end
53
54	def attach_dump_file(dump_id:, filepath:, dump_file_type: :recap_records_full)	1✔
55	df = DumpFile.create(dump_file_type:, path: filepath)	57✔
56	df.zip	57✔
57	df.save!	57✔
58	dump = Dump.find(dump_id)	57✔
59	dump.dump_files << df	57✔
60	dump.save!	57✔
61	end
62
63	private	1✔
64
65	##
66	# Figure out when was the last time we got partner update records, and find
67	# anything that has appeared since then. The ways we can know the last time
68	# we got partner update records are:
69	# * via an environment variable
70	# * look for the last partner recap dump and get its creation date
71	# * or, if neither of those, use yesterday
72	def incremental_update_timestamp	1✔
73	(ENV['TIMESTAMP'] \|\| last_incremental_update \|\| DateTime.now - 1).to_time.strftime('%Y-%m-%d %H:%M:%S.%6N %z')	10✔
74	end
75
76	def last_incremental_update	1✔
77	Dump.partner_recap.last&.created_at	11✔
78	end
79	end # class << self
80
81	def subsequent_partner_incrementals	1✔
82	Dump.partner_recap.where(generated_date: generated_date..Float::INFINITY)	1✔
83	end
84
85	def enqueued?	1✔
UNCOV 86	index_status == ENQUEUED	×
87	end
88
89	def started?	1✔
UNCOV 90	index_status == STARTED	×
91	end
92
93	def done?	1✔
UNCOV 94	index_status == DONE	×
95	end
96	end

pulibrary / bibdata / 373ad6ff-fad2-405a-ab26-3d30fb5ceecf

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous