cace366a-ffad-45f1-9b60-678e607fa527

Committed 14 May 2024 02:21PM UTC coverage: 60.862% (-35.0%) from 95.908%

Build # cace366a-ffad-45f1-9b60-678e607fa527

Build Type

push

circleci

Committed by

jrgriffiniii

Commit Message

wip

Run Details

1 of 3 new or added lines in 2 files covered. (33.33%)

1194 existing lines in 57 files now uncovered.

2076 of 3411 relevant lines covered (60.86%)

22.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

16.98

/app/jobs/dspace_bitstream_copy_job.rb

# frozen_string_literal: true
class DspaceBitstreamCopyJob < ApplicationJob
  queue_as :default

  # For each file from DSpace, queue up a migration.
  # If the file contains characters that are not S3 safe, re-name the file.
  # Note that the dspace_file.filename_display will be the file's key in S3.
  # Files that are re-named must be re-named sequentially, and we must provide
  # a list of all of the re-naming that occurred, and include that file in
  # what is uploaded to S3.
  def perform(dspace_files_json:, work_id:, migration_snapshot_id:)
    dspace_files = JSON.parse(dspace_files_json).map { |json_file| S3File.from_json(json_file) }
    @work = Work.find(work_id)
    frms = FileRenameMappingService.new(upload_snapshot: MigrationUploadSnapshot.find(migration_snapshot_id))
    dspace_files.each do |dspace_file|
      if FileRenameService.new(filename: dspace_file.filename_display).needs_rename?
        # Rename files so they are S3 safe
        dspace_file.filename_display = frms.renamed_files[dspace_file.filename_display]
      end
      migrate_file(dspace_file, migration_snapshot_id)
    end
    upload_rename_mapping(frms)
  end

  private

    def migrate_file(dspace_file, migration_snapshot_id)
      # Allow a restart if there is an error with one file
      snapshot = MigrationUploadSnapshot.find(migration_snapshot_id)
      return if file_complete?(snapshot, dspace_file)

      downloaded_file = download_dspace_file(dspace_file, migration_snapshot_id)
      return if downloaded_file.nil?
      aws_connector = PULDspaceAwsConnector.new(@work, @work.doi)
      result = aws_connector.upload_to_s3([dspace_file]).first
      update_migration_status(migration_snapshot_id) do |migration_snapshot|
        if result[:error].present?
          migration_snapshot.mark_error(result[:file], result[:error])
        else
          # update the checksum here
          migration_snapshot.mark_complete(result[:file])
        end
      end
      File.delete(downloaded_file.filename) if File.exist?(downloaded_file.filename)
    end

    def download_dspace_file(dspace_file, migration_snapshot_id)
      dspace_connector = PULDspaceConnector.new(@work)
      downloaded_file = dspace_connector.download_bitstreams([dspace_file]).first
      if downloaded_file.is_a?(Hash)
        update_migration_status(migration_snapshot_id) do |migration_snapshot|
          migration_snapshot.mark_error(downloaded_file[:file], downloaded_file[:error])
        end
        nil
      else
        downloaded_file
      end
    end

    def update_migration_status(migration_snapshot_id)
      migration_snapshot = MigrationUploadSnapshot.find(migration_snapshot_id)
      migration_snapshot.with_lock do
        migration_snapshot.reload
        yield migration_snapshot
        migration_snapshot.save!
      end
    end

    def file_complete?(migration_snapshot, dspace_file)
      s3_file = dspace_file.clone
      s3_file.filename = s3_file.filename_display
      migration_snapshot.complete?(s3_file)
    end

    # If any files were renamed, upload a text file containing
    # the original file names and what they were renamed to.
    def upload_rename_mapping(frms)
      return unless frms.rename_needed?

      filename = "renamed_files.txt"
      io = StringIO.new
      io.write frms.renaming_document
      io.rewind
      size = io.size
      checksum = Digest::MD5.new
      checksum.update(io.read)
      base64 = checksum.base64digest
      io.rewind
      @work.s3_query_service.upload_file(io:, filename:, size:, md5_digest: base64)
    end
end

1	# frozen_string_literal: true
2	class DspaceBitstreamCopyJob < ApplicationJob	1✔
3	queue_as :default	1✔
4
5	# For each file from DSpace, queue up a migration.
6	# If the file contains characters that are not S3 safe, re-name the file.
7	# Note that the dspace_file.filename_display will be the file's key in S3.
8	# Files that are re-named must be re-named sequentially, and we must provide
9	# a list of all of the re-naming that occurred, and include that file in
10	# what is uploaded to S3.
11	def perform(dspace_files_json:, work_id:, migration_snapshot_id:)	1✔
UNCOV 12	dspace_files = JSON.parse(dspace_files_json).map { \|json_file\| S3File.from_json(json_file) }	×
UNCOV 13	@work = Work.find(work_id)	×
UNCOV 14	frms = FileRenameMappingService.new(upload_snapshot: MigrationUploadSnapshot.find(migration_snapshot_id))	×
UNCOV 15	dspace_files.each do \|dspace_file\|	×
UNCOV 16	if FileRenameService.new(filename: dspace_file.filename_display).needs_rename?	×
17	# Rename files so they are S3 safe
18	dspace_file.filename_display = frms.renamed_files[dspace_file.filename_display]	×
19	end
UNCOV 20	migrate_file(dspace_file, migration_snapshot_id)	×
21	end
UNCOV 22	upload_rename_mapping(frms)	×
23	end
24
25	private	1✔
26
27	def migrate_file(dspace_file, migration_snapshot_id)	1✔
28	# Allow a restart if there is an error with one file
UNCOV 29	snapshot = MigrationUploadSnapshot.find(migration_snapshot_id)	×
UNCOV 30	return if file_complete?(snapshot, dspace_file)	×
31
UNCOV 32	downloaded_file = download_dspace_file(dspace_file, migration_snapshot_id)	×
UNCOV 33	return if downloaded_file.nil?	×
UNCOV 34	aws_connector = PULDspaceAwsConnector.new(@work, @work.doi)	×
UNCOV 35	result = aws_connector.upload_to_s3([dspace_file]).first	×
UNCOV 36	update_migration_status(migration_snapshot_id) do \|migration_snapshot\|	×
UNCOV 37	if result[:error].present?	×
UNCOV 38	migration_snapshot.mark_error(result[:file], result[:error])	×
39	else
40	# update the checksum here
UNCOV 41	migration_snapshot.mark_complete(result[:file])	×
42	end
43	end
UNCOV 44	File.delete(downloaded_file.filename) if File.exist?(downloaded_file.filename)	×
45	end
46
47	def download_dspace_file(dspace_file, migration_snapshot_id)	1✔
UNCOV 48	dspace_connector = PULDspaceConnector.new(@work)	×
UNCOV 49	downloaded_file = dspace_connector.download_bitstreams([dspace_file]).first	×
UNCOV 50	if downloaded_file.is_a?(Hash)	×
UNCOV 51	update_migration_status(migration_snapshot_id) do \|migration_snapshot\|	×
UNCOV 52	migration_snapshot.mark_error(downloaded_file[:file], downloaded_file[:error])	×
53	end
54	nil
55	else
UNCOV 56	downloaded_file	×
57	end
58	end
59
60	def update_migration_status(migration_snapshot_id)	1✔
UNCOV 61	migration_snapshot = MigrationUploadSnapshot.find(migration_snapshot_id)	×
UNCOV 62	migration_snapshot.with_lock do	×
UNCOV 63	migration_snapshot.reload	×
UNCOV 64	yield migration_snapshot	×
UNCOV 65	migration_snapshot.save!	×
66	end
67	end
68
69	def file_complete?(migration_snapshot, dspace_file)	1✔
UNCOV 70	s3_file = dspace_file.clone	×
UNCOV 71	s3_file.filename = s3_file.filename_display	×
UNCOV 72	migration_snapshot.complete?(s3_file)	×
73	end
74
75	# If any files were renamed, upload a text file containing
76	# the original file names and what they were renamed to.
77	def upload_rename_mapping(frms)	1✔
UNCOV 78	return unless frms.rename_needed?	×
79
80	filename = "renamed_files.txt"	×
81	io = StringIO.new	×
82	io.write frms.renaming_document	×
83	io.rewind	×
84	size = io.size	×
85	checksum = Digest::MD5.new	×
86	checksum.update(io.read)	×
87	base64 = checksum.base64digest	×
88	io.rewind	×
89	@work.s3_query_service.upload_file(io:, filename:, size:, md5_digest: base64)	×
90	end
91	end

pulibrary / pdc_describe / cace366a-ffad-45f1-9b60-678e607fa527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous