9091a1ae-29be-458c-984a-339d213919c4

Committed 12 Dec 2024 07:41PM UTC coverage: 26.434% (-69.7%) from 96.113%

Build # 9091a1ae-29be-458c-984a-339d213919c4

Build Type

Pull #2000

circleci

Committed by

jrgriffiniii

Commit Message

Removing integration with ActiveStorage

Pull Request Pull Request #2000: Bump actionpack from 7.2.1.1 to 7.2.2.1

Run Details

945 of 3575 relevant lines covered (26.43%)

0.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

16.98

/app/jobs/dspace_bitstream_copy_job.rb

# frozen_string_literal: true
class DspaceBitstreamCopyJob < ApplicationJob
  queue_as :default

  # For each file from DSpace, queue up a migration.
  # If the file contains characters that are not S3 safe, re-name the file.
  # Note that the dspace_file.filename_display will be the file's key in S3.
  # Files that are re-named must be re-named sequentially, and we must provide
  # a list of all of the re-naming that occurred, and include that file in
  # what is uploaded to S3.
  def perform(dspace_files_json:, work_id:, migration_snapshot_id:)
    dspace_files = JSON.parse(dspace_files_json).map { |json_file| S3File.from_json(json_file) }
    @work = Work.find(work_id)
    frms = FileRenameMappingService.new(upload_snapshot: MigrationUploadSnapshot.find(migration_snapshot_id))
    dspace_files.each do |dspace_file|
      if FileRenameService.new(filename: dspace_file.filename_display).needs_rename?
        # Rename files so they are S3 safe
        dspace_file.filename_display = frms.renamed_files[dspace_file.filename_display]
      end
      migrate_file(dspace_file, migration_snapshot_id)
    end
    upload_rename_mapping(frms)
  end

  private

    def migrate_file(dspace_file, migration_snapshot_id)
      # Allow a restart if there is an error with one file
      snapshot = MigrationUploadSnapshot.find(migration_snapshot_id)
      return if file_complete?(snapshot, dspace_file)

      downloaded_file = download_dspace_file(dspace_file, migration_snapshot_id)
      return if downloaded_file.nil?
      aws_connector = PULDspaceAwsConnector.new(@work, @work.doi)
      result = aws_connector.upload_to_s3([dspace_file]).first
      update_migration_status(migration_snapshot_id) do |migration_snapshot|
        if result[:error].present?
          migration_snapshot.mark_error(result[:file], result[:error])
        else
          # update the checksum here
          migration_snapshot.mark_complete(result[:file])
        end
      end
      File.delete(downloaded_file.filename) if File.exist?(downloaded_file.filename)
    end

    def download_dspace_file(dspace_file, migration_snapshot_id)
      dspace_connector = PULDspaceConnector.new(@work)
      downloaded_file = dspace_connector.download_bitstreams([dspace_file]).first
      if downloaded_file.is_a?(Hash)
        update_migration_status(migration_snapshot_id) do |migration_snapshot|
          migration_snapshot.mark_error(downloaded_file[:file], downloaded_file[:error])
        end
        nil
      else
        downloaded_file
      end
    end

    def update_migration_status(migration_snapshot_id)
      migration_snapshot = MigrationUploadSnapshot.find(migration_snapshot_id)
      migration_snapshot.with_lock do
        migration_snapshot.reload
        yield migration_snapshot
        migration_snapshot.save!
      end
    end

    def file_complete?(migration_snapshot, dspace_file)
      s3_file = dspace_file.clone
      s3_file.filename = s3_file.filename_display
      migration_snapshot.complete?(s3_file)
    end

    # If any files were renamed, upload a text file containing
    # the original file names and what they were renamed to.
    def upload_rename_mapping(frms)
      return unless frms.rename_needed?

      filename = "renamed_files.txt"
      io = StringIO.new
      io.write frms.renaming_document
      io.rewind
      size = io.size
      checksum = Digest::MD5.new
      checksum.update(io.read)
      base64 = checksum.base64digest
      io.rewind
      @work.s3_query_service.upload_file(io:, filename:, size:, md5_digest: base64)
    end
end

1	# frozen_string_literal: true
2	class DspaceBitstreamCopyJob < ApplicationJob	1✔
3	queue_as :default	1✔
4
5	# For each file from DSpace, queue up a migration.
6	# If the file contains characters that are not S3 safe, re-name the file.
7	# Note that the dspace_file.filename_display will be the file's key in S3.
8	# Files that are re-named must be re-named sequentially, and we must provide
9	# a list of all of the re-naming that occurred, and include that file in
10	# what is uploaded to S3.
11	def perform(dspace_files_json:, work_id:, migration_snapshot_id:)	1✔
12	dspace_files = JSON.parse(dspace_files_json).map { \|json_file\| S3File.from_json(json_file) }	×
13	@work = Work.find(work_id)	×
14	frms = FileRenameMappingService.new(upload_snapshot: MigrationUploadSnapshot.find(migration_snapshot_id))	×
15	dspace_files.each do \|dspace_file\|	×
16	if FileRenameService.new(filename: dspace_file.filename_display).needs_rename?	×
17	# Rename files so they are S3 safe
18	dspace_file.filename_display = frms.renamed_files[dspace_file.filename_display]	×
19	end
20	migrate_file(dspace_file, migration_snapshot_id)	×
21	end
22	upload_rename_mapping(frms)	×
23	end
24
25	private	1✔
26
27	def migrate_file(dspace_file, migration_snapshot_id)	1✔
28	# Allow a restart if there is an error with one file
29	snapshot = MigrationUploadSnapshot.find(migration_snapshot_id)	×
30	return if file_complete?(snapshot, dspace_file)	×
31
32	downloaded_file = download_dspace_file(dspace_file, migration_snapshot_id)	×
33	return if downloaded_file.nil?	×
34	aws_connector = PULDspaceAwsConnector.new(@work, @work.doi)	×
35	result = aws_connector.upload_to_s3([dspace_file]).first	×
36	update_migration_status(migration_snapshot_id) do \|migration_snapshot\|	×
37	if result[:error].present?	×
38	migration_snapshot.mark_error(result[:file], result[:error])	×
39	else
40	# update the checksum here
41	migration_snapshot.mark_complete(result[:file])	×
42	end
43	end
44	File.delete(downloaded_file.filename) if File.exist?(downloaded_file.filename)	×
45	end
46
47	def download_dspace_file(dspace_file, migration_snapshot_id)	1✔
48	dspace_connector = PULDspaceConnector.new(@work)	×
49	downloaded_file = dspace_connector.download_bitstreams([dspace_file]).first	×
50	if downloaded_file.is_a?(Hash)	×
51	update_migration_status(migration_snapshot_id) do \|migration_snapshot\|	×
52	migration_snapshot.mark_error(downloaded_file[:file], downloaded_file[:error])	×
53	end
54	nil
55	else
56	downloaded_file	×
57	end
58	end
59
60	def update_migration_status(migration_snapshot_id)	1✔
61	migration_snapshot = MigrationUploadSnapshot.find(migration_snapshot_id)	×
62	migration_snapshot.with_lock do	×
63	migration_snapshot.reload	×
64	yield migration_snapshot	×
65	migration_snapshot.save!	×
66	end
67	end
68
69	def file_complete?(migration_snapshot, dspace_file)	1✔
70	s3_file = dspace_file.clone	×
71	s3_file.filename = s3_file.filename_display	×
72	migration_snapshot.complete?(s3_file)	×
73	end
74
75	# If any files were renamed, upload a text file containing
76	# the original file names and what they were renamed to.
77	def upload_rename_mapping(frms)	1✔
78	return unless frms.rename_needed?	×
79
80	filename = "renamed_files.txt"	×
81	io = StringIO.new	×
82	io.write frms.renaming_document	×
83	io.rewind	×
84	size = io.size	×
85	checksum = Digest::MD5.new	×
86	checksum.update(io.read)	×
87	base64 = checksum.base64digest	×
88	io.rewind	×
89	@work.s3_query_service.upload_file(io:, filename:, size:, md5_digest: base64)	×
90	end
91	end

pulibrary / pdc_describe / 9091a1ae-29be-458c-984a-339d213919c4

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous