c6144e85-a248-439a-9220-278fd67b307a

Committed 13 Feb 2025 08:19PM UTC coverage: 84.455%. First build

Build # c6144e85-a248-439a-9220-278fd67b307a

Build Type

push

circleci

Committed by

carolyncole

Commit Message

Allow for liberal parsing of the CSV data

Some of the decriptions have commas and quotes and new lines which cause issues if they are all present
Also, check for a mediaflux error response before parsing CSV response for project import

Run Details

4 of 4 branches covered (100.0%)

6 of 7 new or added lines in 1 file covered. (85.71%)

2385 of 2824 relevant lines covered (84.45%)

462.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.12

/app/services/project_import.rb

require "csv"

class ProjectImport

    attr_accessor :csv_data, :test_run

    def initialize(csv_data, test_run: false)
        @csv_data = csv_data
        @test_run = test_run
    end

    def self.run_with_report(mediaflux_session:)
      report = Mediaflux::ProjectReport.new(session_token: mediaflux_session)
      return [report.response_error] if report.error?
        
      importer = self.new(report.csv_data)
      importer.run
    end

    def run
        output = []
        mediaflux_projects = CSV.new(csv_data, headers: true, liberal_parsing: true)
        mediaflux_projects.each do |project_metadata|
          # skip projects not part of the current namespace in dev & test mode since we have both mediaflux instances in one server
          if Rails.env.development? || Rails.env.test?
            next unless project_metadata["path"].starts_with?(Rails.configuration.mediaflux["api_root_collection_namespace"])
          end
          project_id = project_metadata["projectID"]
          existing_project = Project.where("metadata_json @> ?", JSON.dump(project_id:))
          if existing_project.count > 0
            output << "Skipping project #{project_id}.  There are already #{existing_project.count} version of that project in the system"
          else
            metadata = convert_csv(project_metadata:, project_id:)
            if test_run
              output << metadata.to_json
            else
              project = Project.create(metadata:, mediaflux_id: project_metadata["asset"])
              if (project.valid?)
                output << "Created project for #{project_id}"
              else
                output << "Error creating project for #{project_metadata["asset"]}: #{project.errors.to_a.join(";")}"
              end
            end
          end
        end
        output
    rescue CSV::MalformedCSVError => error
      ["Error parsing response #{ csv_data.to_s.slice(0,200) } error: #{error}"]
    end

    private
        def convert_csv(project_metadata:, project_id:)
            data_user = parse_multiple(project_metadata, "dataUser")
            department_names = parse_multiple(project_metadata,"department")
            departments = department_names.map {|name| Affiliation.find_fuzzy_by_name(name)&.code || name }
            
            storage_size_gb = project_metadata["quota"].downcase.to_f/1000000000.0
            ProjectMetadata.new_from_hash({
              project_id:,
              title: project_metadata["title"],
              description: project_metadata["description"],
              status: Project::ACTIVE_STATUS,
              data_sponsor: project_metadata["dataSponsor"],
              data_manager: project_metadata["dataManager"],
              departments: departments,
              data_user_read_only: data_user,
              project_directory: project_metadata["path"],
              storage_capacity: {size: { approved: storage_size_gb, requested: storage_size_gb}, unit: {approved: "GB", requested: "GB"}},
              storage_performance_expectations: { requested: "Standard", approved: "Standard" },
              created_by: project_metadata["creatorUser"],
              created_on: project_metadata["createdOn"]
            })
        end

        def parse_multiple(project_metadata, key)
            if project_metadata[key].blank?
            []
            else
            project_metadata[key].split(",").map(&:strip)
            end
        end
end

1	require "csv"	1✔
2
3	class ProjectImport	1✔
4
5	attr_accessor :csv_data, :test_run	1✔
6
7	def initialize(csv_data, test_run: false)	1✔
8	@csv_data = csv_data	8✔
9	@test_run = test_run	8✔
10	end
11
12	def self.run_with_report(mediaflux_session:)	1✔
13	report = Mediaflux::ProjectReport.new(session_token: mediaflux_session)	4✔
14	return [report.response_error] if report.error?	4✔
15
16	importer = self.new(report.csv_data)	4✔
17	importer.run	4✔
18	end
19
20	def run	1✔
21	output = []	9✔
22	mediaflux_projects = CSV.new(csv_data, headers: true, liberal_parsing: true)	9✔
23	mediaflux_projects.each do \|project_metadata\|	9✔
24	# skip projects not part of the current namespace in dev & test mode since we have both mediaflux instances in one server
25	if Rails.env.development? \|\| Rails.env.test?	18✔
26	next unless project_metadata["path"].starts_with?(Rails.configuration.mediaflux["api_root_collection_namespace"])	18✔
27	end
28	project_id = project_metadata["projectID"]	18✔
29	existing_project = Project.where("metadata_json @> ?", JSON.dump(project_id:))	18✔
30	if existing_project.count > 0	18✔
31	output << "Skipping project #{project_id}. There are already #{existing_project.count} version of that project in the system"	3✔
32	else
33	metadata = convert_csv(project_metadata:, project_id:)	15✔
34	if test_run	15✔
35	output << metadata.to_json	×
36	else
37	project = Project.create(metadata:, mediaflux_id: project_metadata["asset"])	15✔
38	if (project.valid?)	15✔
39	output << "Created project for #{project_id}"	8✔
40	else
41	output << "Error creating project for #{project_metadata["asset"]}: #{project.errors.to_a.join(";")}"	7✔
42	end
43	end
44	end
45	end
46	output	9✔
47	rescue CSV::MalformedCSVError => error
NEW 48	["Error parsing response #{ csv_data.to_s.slice(0,200) } error: #{error}"]	×
49	end
50
51	private	1✔
52	def convert_csv(project_metadata:, project_id:)	1✔
53	data_user = parse_multiple(project_metadata, "dataUser")	15✔
54	department_names = parse_multiple(project_metadata,"department")	15✔
55	departments = department_names.map {\|name\| Affiliation.find_fuzzy_by_name(name)&.code \|\| name }	35✔
56
57	storage_size_gb = project_metadata["quota"].downcase.to_f/1000000000.0	15✔
58	ProjectMetadata.new_from_hash({	15✔
59	project_id:,
60	title: project_metadata["title"],
61	description: project_metadata["description"],
62	status: Project::ACTIVE_STATUS,
63	data_sponsor: project_metadata["dataSponsor"],
64	data_manager: project_metadata["dataManager"],
65	departments: departments,
66	data_user_read_only: data_user,
67	project_directory: project_metadata["path"],
68	storage_capacity: {size: { approved: storage_size_gb, requested: storage_size_gb}, unit: {approved: "GB", requested: "GB"}},
69	storage_performance_expectations: { requested: "Standard", approved: "Standard" },
70	created_by: project_metadata["creatorUser"],
71	created_on: project_metadata["createdOn"]
72	})
73	end
74
75	def parse_multiple(project_metadata, key)	1✔
76	if project_metadata[key].blank?	30✔
77	[]	6✔
78	else
79	project_metadata[key].split(",").map(&:strip)	24✔
80	end
81	end
82	end

pulibrary / tigerdata-app / c6144e85-a248-439a-9220-278fd67b307a

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous