af0427fc-ec69-4b12-9b18-e8ea02051032

Committed 30 Jan 2025 06:16PM UTC coverage: 83.508%. Remained the same

Build # af0427fc-ec69-4b12-9b18-e8ea02051032

Build Type

push

circleci

Committed by

web-flow

Commit Message

Change to float parsing as the numbers were too large for integer parsing (#1264)

Co-authored-by: Robert-Anthony Lee-Faison <108823963+leefaisonr@users.noreply.github.com>

Run Details

4 of 4 branches covered (100.0%)

0 of 1 new or added line in 1 file covered. (0.0%)

2309 of 2765 relevant lines covered (83.51%)

383.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

20.0

/lib/tasks/import.rake

require 'csv'
# frozen_string_literal: true
#
# The input file can be created by running by running the following script in aterm:
#
# set ids [xvalues id [asset.query :where xpath(tigerdata:project/ProjectID) has value]]
# puts "asset,path,creatorDomain,creatorUser,createdOn,quota,store,projectDirectory,title,description,dataSponsor,dataManager,dataUser,department,projectID"
# foreach id $ids {
#     set asset [asset.get :id $id]
#     set path [xvalue asset/path $asset]
#     set creatorDomain [xvalue asset/creator/domain $asset]
#     set creatorUser [xvalue asset/creator/user $asset]
#     set createdOn [xvalue  asset/ctime $asset]
#     set store [xvalue asset/collection/store $asset]
#     set projectDirectory [xvalue asset/meta/tigerdata:project/ProjectDirectory $asset]
#     set title [xvalue asset/meta/tigerdata:project/Title $asset]
#     set description [xvalue asset/meta/tigerdata:project/Description $asset]
#     set dataSponsor [xvalue asset/meta/tigerdata:project/DataSponsor $asset]
#     set dataManager [xvalue asset/meta/tigerdata:project/DataManager $asset]
#     set dataUser [xvalue asset/meta/tigerdata:project/DataUser $asset]
#     set department [xvalue asset/meta/tigerdata:project/Department $asset]
#     set projectID [xvalue asset/meta/tigerdata:project/ProjectID $asset]
#     set quota [xvalue asset/collection/quota/allocation $asset]
#     puts $id "," $path "," $creatorDomain "," $creatorUser "," $createdOn "," \
#         $quota "," $store "," $projectDirectory "," \"$title\" "," \"$description\" "," \
#         $dataSponsor "," $dataManager ",\"" $dataUser "\",\"" $department "\""," $projectID 
# }
#
#

def parse_multiple(project_metadata, key)
  if project_metadata[key].blank?
    []
  else
    project_metadata[key].split(",").map(&:strip)
  end
end

namespace :import do
    # command line syntax: bundle exec rake metadata:update_pppl_subcommunities\["netid"\]
    desc "import projects from mediaflux csv file"
    task :mediaflux_projects, [:project_file, :test_run] => [:environment] do |_, args|
      project_file = args[:project_file]
      test_run = args[:test_run] || false
      mediaflux_projects = CSV.read(project_file, headers: true)
      mediaflux_projects.each do |project_metadata|
        project_id = project_metadata["projectID"]
        existing_project = Project.where("metadata_json @> ?", JSON.dump(project_id:))
        if existing_project.count > 0
          puts "Skipping project #{project_id}.  There are already #{existing_project.count} version of that project in the system"
        else
          data_user = parse_multiple(project_metadata, "dataUser")
          department_names = parse_multiple(project_metadata,"department")
          departments = department_names.map {|name| Affiliation.find_fuzzy_by_name(name)&.code || name }
          
          storage_size_gb = project_metadata["quota"].downcase.to_f/1000000000.0
          metadata = ProjectMetadata.new_from_hash({
            project_id:,
            title: project_metadata["title"],
            description: project_metadata["description"],
            status: Project::ACTIVE_STATUS,
            data_sponsor: project_metadata["dataSponsor"],
            data_manager: project_metadata["dataManager"],
            departments: departments,
            data_user_read_only: data_user,
            project_directory: project_metadata["path"],
            storage_capacity: {size: { approved: storage_size_gb, requested: storage_size_gb}, unit: {approved: "GB", requested: "GB"}},
            storage_performance_expectations: { requested: "Standard", approved: "Standard" },
            created_by: project_metadata["creatorUser"],
            created_on: project_metadata["createdOn"]
          })
          if test_run
            puts metadata.to_json
          else
            project = Project.create!(metadata:, mediaflux_id: project_metadata["asset"])
            puts "Created project for #{project_id}"
          end
        end
      end
    end
  end

1	require 'csv'	1✔
2	# frozen_string_literal: true
3	#
4	# The input file can be created by running by running the following script in aterm:
5	#
6	# set ids [xvalues id [asset.query :where xpath(tigerdata:project/ProjectID) has value]]
7	# puts "asset,path,creatorDomain,creatorUser,createdOn,quota,store,projectDirectory,title,description,dataSponsor,dataManager,dataUser,department,projectID"
8	# foreach id $ids {
9	# set asset [asset.get :id $id]
10	# set path [xvalue asset/path $asset]
11	# set creatorDomain [xvalue asset/creator/domain $asset]
12	# set creatorUser [xvalue asset/creator/user $asset]
13	# set createdOn [xvalue asset/ctime $asset]
14	# set store [xvalue asset/collection/store $asset]
15	# set projectDirectory [xvalue asset/meta/tigerdata:project/ProjectDirectory $asset]
16	# set title [xvalue asset/meta/tigerdata:project/Title $asset]
17	# set description [xvalue asset/meta/tigerdata:project/Description $asset]
18	# set dataSponsor [xvalue asset/meta/tigerdata:project/DataSponsor $asset]
19	# set dataManager [xvalue asset/meta/tigerdata:project/DataManager $asset]
20	# set dataUser [xvalue asset/meta/tigerdata:project/DataUser $asset]
21	# set department [xvalue asset/meta/tigerdata:project/Department $asset]
22	# set projectID [xvalue asset/meta/tigerdata:project/ProjectID $asset]
23	# set quota [xvalue asset/collection/quota/allocation $asset]
24	# puts $id "," $path "," $creatorDomain "," $creatorUser "," $createdOn "," \
25	# $quota "," $store "," $projectDirectory "," \"$title\" "," \"$description\" "," \
26	# $dataSponsor "," $dataManager ",\"" $dataUser "\",\"" $department "\""," $projectID
27	# }
28	#
29	#
30
31	def parse_multiple(project_metadata, key)	1✔
32	if project_metadata[key].blank?	×
33	[]	×
34	else
35	project_metadata[key].split(",").map(&:strip)	×
36	end
37	end
38
39	namespace :import do	1✔
40	# command line syntax: bundle exec rake metadata:update_pppl_subcommunities\["netid"\]
41	desc "import projects from mediaflux csv file"	1✔
42	task :mediaflux_projects, [:project_file, :test_run] => [:environment] do \|_, args\|	1✔
43	project_file = args[:project_file]	×
44	test_run = args[:test_run] \|\| false	×
45	mediaflux_projects = CSV.read(project_file, headers: true)	×
46	mediaflux_projects.each do \|project_metadata\|	×
47	project_id = project_metadata["projectID"]	×
48	existing_project = Project.where("metadata_json @> ?", JSON.dump(project_id:))	×
49	if existing_project.count > 0	×
50	puts "Skipping project #{project_id}. There are already #{existing_project.count} version of that project in the system"	×
51	else
52	data_user = parse_multiple(project_metadata, "dataUser")	×
53	department_names = parse_multiple(project_metadata,"department")	×
54	departments = department_names.map {\|name\| Affiliation.find_fuzzy_by_name(name)&.code \|\| name }	×
55
NEW 56	storage_size_gb = project_metadata["quota"].downcase.to_f/1000000000.0	×
57	metadata = ProjectMetadata.new_from_hash({	×
58	project_id:,
59	title: project_metadata["title"],
60	description: project_metadata["description"],
61	status: Project::ACTIVE_STATUS,
62	data_sponsor: project_metadata["dataSponsor"],
63	data_manager: project_metadata["dataManager"],
64	departments: departments,
65	data_user_read_only: data_user,
66	project_directory: project_metadata["path"],
67	storage_capacity: {size: { approved: storage_size_gb, requested: storage_size_gb}, unit: {approved: "GB", requested: "GB"}},
68	storage_performance_expectations: { requested: "Standard", approved: "Standard" },
69	created_by: project_metadata["creatorUser"],
70	created_on: project_metadata["createdOn"]
71	})
72	if test_run	×
73	puts metadata.to_json	×
74	else
75	project = Project.create!(metadata:, mediaflux_id: project_metadata["asset"])	×
76	puts "Created project for #{project_id}"	×
77	end
78	end
79	end
80	end
81	end

pulibrary / tigerdata-app / af0427fc-ec69-4b12-9b18-e8ea02051032

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous