9091a1ae-29be-458c-984a-339d213919c4

Committed 12 Dec 2024 07:41PM UTC coverage: 26.434% (-69.7%) from 96.113%

Build # 9091a1ae-29be-458c-984a-339d213919c4

Build Type

Pull #2000

circleci

Committed by

jrgriffiniii

Commit Message

Removing integration with ActiveStorage

Pull Request Pull Request #2000: Bump actionpack from 7.2.1.1 to 7.2.2.1

Run Details

945 of 3575 relevant lines covered (26.43%)

0.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

20.51

/app/services/pul_dspace_migrate.rb

# frozen_string_literal: true
class PULDspaceMigrate
  attr_reader :work, :ark, :file_keys, :directory_keys, :current_user,
              :aws_connector, :migration_snapshot, :dspace_files, :aws_files_and_directories

  delegate :doi, to: :dspace_connector

  def initialize(work, current_user)
    @work = work
    @ark = work.ark&.gsub("ark:/", "")
    @file_keys = []
    @directory_keys = []
    @aws_connector = PULDspaceAwsConnector.new(work, work.doi)
    @migration_snapshot = nil
    @aws_files_and_directories = nil
    @dspace_files = []
    @current_user = current_user
  end

  def migrate
    return if ark.nil?
    work.resource.migrated = true
    work.save
    @aws_files_and_directories = aws_connector.aws_files
    migrate_dspace

    aws_copy(aws_files_and_directories)
  end

  def dspace_connector
    @dspace_connector ||= PULDspaceConnector.new(work)
  end

  def migration_message(input_file_keys = file_keys, input_directory_keys = directory_keys)
    message = []
    # rubocop:disable Layout/LineLength
    message << "Migration for #{input_file_keys.count} #{'file'.pluralize(input_file_keys.count)} and #{input_directory_keys.count} #{'directory'.pluralize(input_directory_keys.count)} is running in the background. Depending on the file sizes this may take some time."
    # rubocop:enable Layout/LineLength
    message.join(" ")
  end

  private

    def generate_migration_snapshot
      files = remove_overlap_and_combine
      snapshot = MigrationUploadSnapshot.new(work:, url: work.s3_query_service.prefix)
      last_snapshot = work.upload_snapshots.first
      snapshot.store_files(files, pre_existing_files: last_snapshot&.files)
      snapshot.save!
      directories = aws_files_and_directories.select(&:directory?)
      WorkActivity.add_work_activity(work.id, { migration_id: snapshot.id,
                                                message: migration_message(files, directories), file_count: files.count,
                                                directory_count: directories.count }.to_json,
                                      current_user.id, activity_type: WorkActivity::MIGRATION_START)
      @migration_snapshot = snapshot
    end

    def remove_overlap_and_combine
      dpsace_update_display_to_final_key
      aws_update_display_to_final_key(aws_files_and_directories)
      aws_files_only = aws_files_and_directories.reject(&:directory?)
      aws_file_names = aws_files_only.map(&:filename_display)
      files_to_remove = []
      dspace_files.each do |s3_file|
        idx = aws_file_names.index(s3_file.filename_display)
        if idx.present?
          check_matching_files(aws_files_only[idx], s3_file, files_to_remove)
        end
      end
      @dspace_files = dspace_files - files_to_remove
      @dspace_files.each { |file| file_keys << file.filename_display }
      dspace_files + aws_files_only
    end

    def dpsace_update_display_to_final_key
      dspace_files.each do |s3_file|
        s3_file.filename_display = work.s3_query_service.prefix + s3_file.filename_display
      end
    end

    def aws_update_display_to_final_key(aws_files)
      aws_files.each do |s3_file|
        s3_file.filename_display = s3_file.filename.gsub("#{doi}/".tr(".", "-"), work.s3_query_service.prefix)
      end
    end

    def check_matching_files(aws_file, dpace_file, files_to_remove)
      if dpace_file.checksum == aws_file.checksum
        files_to_remove << dpace_file
      else
        basename = File.basename(dpace_file.filename_display)
        dpace_file.filename_display.gsub!(basename, "data_space_#{basename}")
        aws_file.filename_display.gsub!(basename, "globus_#{basename}")
      end
    end

    def migrate_dspace
      @dspace_files = dspace_connector.list_bitsteams
      generate_migration_snapshot
      dspace_files_json = "[#{dspace_files.map(&:to_json).join(',')}]"
      DspaceBitstreamCopyJob.perform_later(dspace_files_json:, work_id: work.id, migration_snapshot_id: migration_snapshot.id)
    end

    def aws_copy(files)
      Honeybadger.notify("DspaceFileCopyJob started for ark #{work.ark} doi #{work.doi} without a migration snapshot") unless @migration_snapshot
      files.each do |s3_file|
        DspaceFileCopyJob.perform_later(s3_file_json: s3_file.to_json, work_id: work.id, migration_snapshot_id: @migration_snapshot&.id)
        if s3_file.directory?
          directory_keys << s3_file.key
        else
          file_keys << s3_file.filename_display
        end
      end
    end

    def request_http(url)
      uri = URI(url)
      http = Net::HTTP.new(uri.host, uri.port)
      http.use_ssl = true
      http
    end
end

1	# frozen_string_literal: true
2	class PULDspaceMigrate	1✔
3	attr_reader :work, :ark, :file_keys, :directory_keys, :current_user,	1✔
4	:aws_connector, :migration_snapshot, :dspace_files, :aws_files_and_directories
5
6	delegate :doi, to: :dspace_connector	1✔
7
8	def initialize(work, current_user)	1✔
9	@work = work	×
10	@ark = work.ark&.gsub("ark:/", "")	×
11	@file_keys = []	×
12	@directory_keys = []	×
13	@aws_connector = PULDspaceAwsConnector.new(work, work.doi)	×
14	@migration_snapshot = nil	×
15	@aws_files_and_directories = nil	×
16	@dspace_files = []	×
17	@current_user = current_user	×
18	end
19
20	def migrate	1✔
21	return if ark.nil?	×
22	work.resource.migrated = true	×
23	work.save	×
24	@aws_files_and_directories = aws_connector.aws_files	×
25	migrate_dspace	×
26
27	aws_copy(aws_files_and_directories)	×
28	end
29
30	def dspace_connector	1✔
31	@dspace_connector \|\|= PULDspaceConnector.new(work)	×
32	end
33
34	def migration_message(input_file_keys = file_keys, input_directory_keys = directory_keys)	1✔
35	message = []	×
36	# rubocop:disable Layout/LineLength
37	message << "Migration for #{input_file_keys.count} #{'file'.pluralize(input_file_keys.count)} and #{input_directory_keys.count} #{'directory'.pluralize(input_directory_keys.count)} is running in the background. Depending on the file sizes this may take some time."	×
38	# rubocop:enable Layout/LineLength
39	message.join(" ")	×
40	end
41
42	private	1✔
43
44	def generate_migration_snapshot	1✔
45	files = remove_overlap_and_combine	×
46	snapshot = MigrationUploadSnapshot.new(work:, url: work.s3_query_service.prefix)	×
47	last_snapshot = work.upload_snapshots.first	×
48	snapshot.store_files(files, pre_existing_files: last_snapshot&.files)	×
49	snapshot.save!	×
50	directories = aws_files_and_directories.select(&:directory?)	×
51	WorkActivity.add_work_activity(work.id, { migration_id: snapshot.id,	×
52	message: migration_message(files, directories), file_count: files.count,
53	directory_count: directories.count }.to_json,
54	current_user.id, activity_type: WorkActivity::MIGRATION_START)
55	@migration_snapshot = snapshot	×
56	end
57
58	def remove_overlap_and_combine	1✔
59	dpsace_update_display_to_final_key	×
60	aws_update_display_to_final_key(aws_files_and_directories)	×
61	aws_files_only = aws_files_and_directories.reject(&:directory?)	×
62	aws_file_names = aws_files_only.map(&:filename_display)	×
63	files_to_remove = []	×
64	dspace_files.each do \|s3_file\|	×
65	idx = aws_file_names.index(s3_file.filename_display)	×
66	if idx.present?	×
67	check_matching_files(aws_files_only[idx], s3_file, files_to_remove)	×
68	end
69	end
70	@dspace_files = dspace_files - files_to_remove	×
71	@dspace_files.each { \|file\| file_keys << file.filename_display }	×
72	dspace_files + aws_files_only	×
73	end
74
75	def dpsace_update_display_to_final_key	1✔
76	dspace_files.each do \|s3_file\|	×
77	s3_file.filename_display = work.s3_query_service.prefix + s3_file.filename_display	×
78	end
79	end
80
81	def aws_update_display_to_final_key(aws_files)	1✔
82	aws_files.each do \|s3_file\|	×
83	s3_file.filename_display = s3_file.filename.gsub("#{doi}/".tr(".", "-"), work.s3_query_service.prefix)	×
84	end
85	end
86
87	def check_matching_files(aws_file, dpace_file, files_to_remove)	1✔
88	if dpace_file.checksum == aws_file.checksum	×
89	files_to_remove << dpace_file	×
90	else
91	basename = File.basename(dpace_file.filename_display)	×
92	dpace_file.filename_display.gsub!(basename, "data_space_#{basename}")	×
93	aws_file.filename_display.gsub!(basename, "globus_#{basename}")	×
94	end
95	end
96
97	def migrate_dspace	1✔
98	@dspace_files = dspace_connector.list_bitsteams	×
99	generate_migration_snapshot	×
100	dspace_files_json = "[#{dspace_files.map(&:to_json).join(',')}]"	×
101	DspaceBitstreamCopyJob.perform_later(dspace_files_json:, work_id: work.id, migration_snapshot_id: migration_snapshot.id)	×
102	end
103
104	def aws_copy(files)	1✔
105	Honeybadger.notify("DspaceFileCopyJob started for ark #{work.ark} doi #{work.doi} without a migration snapshot") unless @migration_snapshot	×
106	files.each do \|s3_file\|	×
107	DspaceFileCopyJob.perform_later(s3_file_json: s3_file.to_json, work_id: work.id, migration_snapshot_id: @migration_snapshot&.id)	×
108	if s3_file.directory?	×
109	directory_keys << s3_file.key	×
110	else
111	file_keys << s3_file.filename_display	×
112	end
113	end
114	end
115
116	def request_http(url)	1✔
117	uri = URI(url)	×
118	http = Net::HTTP.new(uri.host, uri.port)	×
119	http.use_ssl = true	×
120	http	×
121	end
122	end

pulibrary / pdc_describe / 9091a1ae-29be-458c-984a-339d213919c4

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous