5e867592-a902-4c05-97f1-a03662fb2c14

pending completion

Build # 5e867592-a902-4c05-97f1-a03662fb2c14

Build Type

Pull #441

circleci

Committed by

hectorcorrea

Commit Message

Rubocop nit picking

Pull Request Pull Request #441: Indexing to a new collection

Run Details

98 of 98 new or added lines in 3 files covered. (100.0%)

1791 of 2219 relevant lines covered (80.71%)

8.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.77

/lib/traject/solr_cloud_helper.rb

# frozen_string_literal: true

class SolrCloudHelper
  def self.alias_uri
    Blacklight.default_index.connection.uri
  end

  def self.alias_url
    Blacklight.default_index.connection.uri.to_s
  end

  def self.config_set
    Blacklight.default_index.connection.options[:solr_config_set]
  end

  def self.collection_writer_commit!
    commit_url = "#{collection_writer_url}/update?commit=true"
    HTTParty.get(commit_url)
  end

  # Returns the Solr collection that we should use for quering data
  def self.collection_reader_url
    collection = current_collection_for_alias(alias_uri)
    build_solr_url_for_collection(alias_uri, collection)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exist)
  def self.collection_writer_url
    collection_writer_for_alias(alias_uri, false)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exists, deletes & recreates it if already exists)
  def self.create_collection_writer
    collection_writer_for_alias(alias_uri, true)
  end

  # For a given Solr alias, returns a URL that is suitable for writing new data without affecting
  # the existing Solr index. It does this by creating a new alternate Solr collection to write data,
  # instead of overwriting the data on the collection used by the provided solr_alias_uri.
  #
  # The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
  #
  # For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
  # and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
  # the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
  #
  # If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
  # then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
  def self.collection_writer_for_alias(solr_alias_uri, recreate)
    if config_set.nil?
      # We are not running in a Solr cloud environment - nothing to do.
      return solr_alias_uri.to_s
    end

    alternate_collection = alternate_collection_for_alias(solr_alias_uri)
    if collection_exist?(solr_alias_uri, alternate_collection)
      if recreate
        # Re-create it
        delete_collection!(solr_alias_uri, alternate_collection)
        create_collection(solr_alias_uri, alternate_collection)
      end
    else
      # Create it
      create_collection(solr_alias_uri, alternate_collection)
    end

    build_solr_url_for_collection(solr_alias_uri, alternate_collection)
  end

  # Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
  # the indicated `collection` instead of pointing to the alias.
  def self.build_solr_url_for_collection(solr_alias_uri, collection)
    URI::HTTP.build(schema: solr_alias_uri.scheme, host: solr_alias_uri.host, port: solr_alias_uri.port, path: "/solr/#{collection}").to_s
  end

  def self.current_collection_for_alias(solr_alias_uri)
    alias_name = solr_alias_uri.path.split("/").last
    alias_list_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=LISTALIASES"
    )
    response = HTTParty.get(alias_list_query.to_s)
    # The response will indicate the actual collection the alias points to.
    collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200
    collection_name || alias_name
  end

  def self.alternate_collection_for_alias(solr_alias_uri)
    solr_collection = current_collection_for_alias(solr_alias_uri)
    if solr_collection.end_with?("-1")
      solr_collection.gsub("-1", "-2")
    elsif solr_collection.end_with?("-2")
      solr_collection.gsub("-2", "-1")
    else
      solr_collection
    end
  end

  def self.collection_exist?(solr_alias_uri, collection_name)
    collection_list_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=LIST"
    )
    response = HTTParty.get(collection_list_query.to_s)
    collections = if response.code == 200
                    response.parsed_response.dig("collections") || []
                  else
                    []
                  end
    collections.any?(collection_name)
  end

  def self.create_collection(solr_alias_uri, collection_name)
    create_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
    )
    response = HTTParty.get(create_query.to_s)
    response.parsed_response.key?("success") ? collection_name : nil
  end

  def self.delete_collection!(solr_alias_uri, collection_name)
    create_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=DELETE&name=#{collection_name}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end

  # Set the solr_alias to point to the current writer collection
  def self.update_solr_alias!
    writer_collection = collection_writer_url.split("/").last

    alias_name = alias_uri.path.split("/").last
    if alias_name == writer_collection
      # Nothing to do
      # (we are probably using standalone Solr in development)
      return true
    end

    create_query = URI::HTTP.build(
      schema: alias_uri.scheme,
      host: alias_uri.host,
      port: alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end
end

1	# frozen_string_literal: true
2
3	class SolrCloudHelper	1✔
4	def self.alias_uri	1✔
5	Blacklight.default_index.connection.uri	207✔
6	end
7
8	def self.alias_url	1✔
9	Blacklight.default_index.connection.uri.to_s	×
10	end
11
12	def self.config_set	1✔
13	Blacklight.default_index.connection.options[:solr_config_set]	415✔
14	end
15
16	def self.collection_writer_commit!	1✔
17	commit_url = "#{collection_writer_url}/update?commit=true"	×
18	HTTParty.get(commit_url)	×
19	end
20
21	# Returns the Solr collection that we should use for quering data
22	def self.collection_reader_url	1✔
23	collection = current_collection_for_alias(alias_uri)	×
24	build_solr_url_for_collection(alias_uri, collection)	×
25	end
26
27	# Returns the Solr collection that we should write to
28	# (creates it if does not exist)
29	def self.collection_writer_url	1✔
30	collection_writer_for_alias(alias_uri, false)	207✔
31	end
32
33	# Returns the Solr collection that we should write to
34	# (creates it if does not exists, deletes & recreates it if already exists)
35	def self.create_collection_writer	1✔
36	collection_writer_for_alias(alias_uri, true)	×
37	end
38
39	# For a given Solr alias, returns a URL that is suitable for writing new data without affecting
40	# the existing Solr index. It does this by creating a new alternate Solr collection to write data,
41	# instead of overwriting the data on the collection used by the provided solr_alias_uri.
42	#
43	# The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
44	#
45	# For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
46	# and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
47	# the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
48	#
49	# If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
50	# then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
51	def self.collection_writer_for_alias(solr_alias_uri, recreate)	1✔
52	if config_set.nil?	208✔
53	# We are not running in a Solr cloud environment - nothing to do.
54	return solr_alias_uri.to_s	×
55	end
56
57	alternate_collection = alternate_collection_for_alias(solr_alias_uri)	208✔
58	if collection_exist?(solr_alias_uri, alternate_collection)	208✔
59	if recreate	1✔
60	# Re-create it
61	delete_collection!(solr_alias_uri, alternate_collection)	×
62	create_collection(solr_alias_uri, alternate_collection)	×
63	end
64	else
65	# Create it
66	create_collection(solr_alias_uri, alternate_collection)	207✔
67	end
68
69	build_solr_url_for_collection(solr_alias_uri, alternate_collection)	1✔
70	end
71
72	# Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
73	# the indicated `collection` instead of pointing to the alias.
74	def self.build_solr_url_for_collection(solr_alias_uri, collection)	1✔
75	URI::HTTP.build(schema: solr_alias_uri.scheme, host: solr_alias_uri.host, port: solr_alias_uri.port, path: "/solr/#{collection}").to_s	1✔
76	end
77
78	def self.current_collection_for_alias(solr_alias_uri)	1✔
79	alias_name = solr_alias_uri.path.split("/").last	212✔
80	alias_list_query = URI::HTTP.build(	212✔
81	schema: solr_alias_uri.scheme,
82	host: solr_alias_uri.host,
83	port: solr_alias_uri.port,
84	path: "/solr/admin/collections",
85	query: "action=LISTALIASES"
86	)
87	response = HTTParty.get(alias_list_query.to_s)	212✔
88	# The response will indicate the actual collection the alias points to.
89	collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200	212✔
90	collection_name \|\| alias_name	212✔
91	end
92
93	def self.alternate_collection_for_alias(solr_alias_uri)	1✔
94	solr_collection = current_collection_for_alias(solr_alias_uri)	211✔
95	if solr_collection.end_with?("-1")	211✔
96	solr_collection.gsub("-1", "-2")	2✔
97	elsif solr_collection.end_with?("-2")	209✔
98	solr_collection.gsub("-2", "-1")	1✔
99	else
100	solr_collection	208✔
101	end
102	end
103
104	def self.collection_exist?(solr_alias_uri, collection_name)	1✔
105	collection_list_query = URI::HTTP.build(	210✔
106	schema: solr_alias_uri.scheme,
107	host: solr_alias_uri.host,
108	port: solr_alias_uri.port,
109	path: "/solr/admin/collections",
110	query: "action=LIST"
111	)
112	response = HTTParty.get(collection_list_query.to_s)	210✔
113	collections = if response.code == 200	210✔
114	response.parsed_response.dig("collections") \|\| []	3✔
115	else
116	[]	207✔
117	end
118	collections.any?(collection_name)	210✔
119	end
120
121	def self.create_collection(solr_alias_uri, collection_name)	1✔
122	create_query = URI::HTTP.build(	207✔
123	schema: solr_alias_uri.scheme,
124	host: solr_alias_uri.host,
125	port: solr_alias_uri.port,
126	path: "/solr/admin/collections",
127	query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
128	)
129	response = HTTParty.get(create_query.to_s)	207✔
130	response.parsed_response.key?("success") ? collection_name : nil	207✔
131	end
132
133	def self.delete_collection!(solr_alias_uri, collection_name)	1✔
134	create_query = URI::HTTP.build(	×
135	schema: solr_alias_uri.scheme,
136	host: solr_alias_uri.host,
137	port: solr_alias_uri.port,
138	path: "/solr/admin/collections",
139	query: "action=DELETE&name=#{collection_name}"
140	)
141	response = HTTParty.get(create_query.to_s)	×
142	response.code == 200	×
143	end
144
145	# Set the solr_alias to point to the current writer collection
146	def self.update_solr_alias!	1✔
147	writer_collection = collection_writer_url.split("/").last	×
148
149	alias_name = alias_uri.path.split("/").last	×
150	if alias_name == writer_collection	×
151	# Nothing to do
152	# (we are probably using standalone Solr in development)
153	return true	×
154	end
155
156	create_query = URI::HTTP.build(	×
157	schema: alias_uri.scheme,
158	host: alias_uri.host,
159	port: alias_uri.port,
160	path: "/solr/admin/collections",
161	query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
162	)
163	response = HTTParty.get(create_query.to_s)	×
164	response.code == 200	×
165	end
166	end

pulibrary / pdc_discovery / 5e867592-a902-4c05-97f1-a03662fb2c14

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous