fa8e4699-854f-4a43-adc1-7e92603d9eea

Build # fa8e4699-854f-4a43-adc1-7e92603d9eea

Build Type

circleci

Committed by Hector Correa

Commit Message

Rubocop nitpicking

Pull Request Pull Request #441: Indexing to a new collection

Run Details

70 of 70 new or added lines in 2 files covered. (100.0%)

2073 of 2216 relevant lines covered (93.55%)

170.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

32.31

/lib/traject/solr_cloud_helper.rb

# frozen_string_literal: true

# rubocop:disable Style/ClassVars (OK here since we are not inheriting this class)
# https://rubydoc.info/gems/rubocop/RuboCop/Cop/Style/ClassVars
class SolrCloudHelper
  def self.alias_uri
    Blacklight.default_index.connection.uri
  end

  def self.alias_url
    Blacklight.default_index.connection.uri.to_s
  end

  def self.config_set
    Blacklight.default_index.connection.options[:solr_config_set]
  end

  def self.collection_writer_commit!
    commit_url = "#{collection_writer_url}/update?commit=true"
    HTTParty.get(commit_url)
  end

  # Returns the Solr collection that we should use for quering data
  def self.collection_reader_url
    collection = current_collection_for_alias(alias_uri)
    build_solr_url_for_collection(alias_uri, collection)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exist)
  def self.collection_writer_url
    collection_writer_for_alias(alias_uri, false)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exists, deletes & recreates it if already exists)
  def self.create_collection_writer
    collection_writer_for_alias(alias_uri, true)
  end

  # For a given Solr alias, returns a URL that is suitable for writing new data without affecting
  # the existing Solr index. It does this by creating a new alternate Solr collection to write data,
  # instead of overwriting the data on the collection used by the provided solr_alias_uri.
  #
  # The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
  #
  # For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
  # and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
  # the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
  #
  # If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
  # then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
  def self.collection_writer_for_alias(solr_alias_uri, recreate)
    if config_set.nil?
      # We are not running in a Solr cloud environment - nothing to do.
      return solr_alias_uri.to_s
    end

    alternate_collection = alternate_collection_for_alias(solr_alias_uri)
    if collection_exist?(solr_alias_uri, alternate_collection)
      if recreate
        # Re-create it
        delete_collection!(solr_alias_uri, alternate_collection)
        create_collection(solr_alias_uri, alternate_collection)
      end
    else
      # Create it
      create_collection(solr_alias_uri, alternate_collection)
    end

    build_solr_url_for_collection(solr_alias_uri, alternate_collection)
  end

  # Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
  # the indicated `collection` instead of pointing to the alias.
  def self.build_solr_url_for_collection(solr_alias_uri, collection)
    URI::HTTP.build(schema: solr_alias_uri.scheme, host: solr_alias_uri.host, port: solr_alias_uri.port, path: "/solr/#{collection}").to_s
  end

  def self.current_collection_for_alias(solr_alias_uri)
    alias_name = solr_alias_uri.path.split("/").last
    alias_list_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=LISTALIASES"
    )
    response = HTTParty.get(alias_list_query.to_s)
    # The response will indicate the actual collection the alias points to.
    collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200
    collection_name || alias_name
  end

  def self.alternate_collection_for_alias(solr_alias_uri)
    solr_collection = current_collection_for_alias(solr_alias_uri)
    if solr_collection.end_with?("-1")
      solr_collection.gsub("-1", "-2")
    elsif solr_collection.end_with?("-2")
      solr_collection.gsub("-2", "-1")
    else
      solr_collection
    end
  end

  def self.collection_exist?(solr_alias_uri, collection_name)
    collection_list_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=LIST"
    )
    response = HTTParty.get(collection_list_query.to_s)
    collections = if response.code == 200
                    response.parsed_response.dig("collections") || []
                  else
                    []
                  end
    collections.any?(collection_name)
  end

  def self.create_collection(solr_alias_uri, collection_name)
    create_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
    )
    response = HTTParty.get(create_query.to_s)
    response.parsed_response.key?("success") ? collection_name : nil
  end

  def self.delete_collection!(solr_alias_uri, collection_name)
    create_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=DELETE&name=#{collection_name}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end

  # Set the solr_alias to point to the current writer collection
  def self.update_solr_alias!
    writer_collection = collection_writer_url.split("/").last

    alias_name = alias_uri.path.split("/").last
    if alias_name == writer_collection
      # Nothing to do
      # (we are probably using standalone Solr in development)
      return true
    end

    create_query = URI::HTTP.build(
      schema: alias_uri.scheme,
      host: alias_uri.host,
      port: alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end
end
# rubocop:enable Style/ClassVars

1	# frozen_string_literal: true
2
3	# rubocop:disable Style/ClassVars (OK here since we are not inheriting this class)
4	# https://rubydoc.info/gems/rubocop/RuboCop/Cop/Style/ClassVars
5	class SolrCloudHelper	1✔
6	def self.alias_uri	1✔
7	Blacklight.default_index.connection.uri	206✔
8	end
9
10	def self.alias_url	1✔
11	Blacklight.default_index.connection.uri.to_s	×
12	end
13
14	def self.config_set	1✔
15	Blacklight.default_index.connection.options[:solr_config_set]	206✔
16	end
17
18	def self.collection_writer_commit!	1✔
19	commit_url = "#{collection_writer_url}/update?commit=true"	×
20	HTTParty.get(commit_url)	×
21	end
22
23	# Returns the Solr collection that we should use for quering data
24	def self.collection_reader_url	1✔
25	collection = current_collection_for_alias(alias_uri)	×
26	build_solr_url_for_collection(alias_uri, collection)	×
27	end
28
29	# Returns the Solr collection that we should write to
30	# (creates it if does not exist)
31	def self.collection_writer_url	1✔
32	collection_writer_for_alias(alias_uri, false)	206✔
33	end
34
35	# Returns the Solr collection that we should write to
36	# (creates it if does not exists, deletes & recreates it if already exists)
37	def self.create_collection_writer	1✔
38	collection_writer_for_alias(alias_uri, true)	×
39	end
40
41	# For a given Solr alias, returns a URL that is suitable for writing new data without affecting
42	# the existing Solr index. It does this by creating a new alternate Solr collection to write data,
43	# instead of overwriting the data on the collection used by the provided solr_alias_uri.
44	#
45	# The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
46	#
47	# For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
48	# and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
49	# the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
50	#
51	# If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
52	# then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
53	def self.collection_writer_for_alias(solr_alias_uri, recreate)	1✔
54	if config_set.nil?	206✔
55	# We are not running in a Solr cloud environment - nothing to do.
56	return solr_alias_uri.to_s	206✔
57	end
58
59	alternate_collection = alternate_collection_for_alias(solr_alias_uri)	×
60	if collection_exist?(solr_alias_uri, alternate_collection)	×
61	if recreate	×
62	# Re-create it
63	delete_collection!(solr_alias_uri, alternate_collection)	×
64	create_collection(solr_alias_uri, alternate_collection)	×
65	end
66	else
67	# Create it
68	create_collection(solr_alias_uri, alternate_collection)	×
69	end
70
71	build_solr_url_for_collection(solr_alias_uri, alternate_collection)	×
72	end
73
74	# Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
75	# the indicated `collection` instead of pointing to the alias.
76	def self.build_solr_url_for_collection(solr_alias_uri, collection)	1✔
77	URI::HTTP.build(schema: solr_alias_uri.scheme, host: solr_alias_uri.host, port: solr_alias_uri.port, path: "/solr/#{collection}").to_s	×
78	end
79
80	def self.current_collection_for_alias(solr_alias_uri)	1✔
81	alias_name = solr_alias_uri.path.split("/").last	×
82	alias_list_query = URI::HTTP.build(	×
83	schema: solr_alias_uri.scheme,
84	host: solr_alias_uri.host,
85	port: solr_alias_uri.port,
86	path: "/solr/admin/collections",
87	query: "action=LISTALIASES"
88	)
89	response = HTTParty.get(alias_list_query.to_s)	×
90	# The response will indicate the actual collection the alias points to.
91	collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200	×
92	collection_name \|\| alias_name	×
93	end
94
95	def self.alternate_collection_for_alias(solr_alias_uri)	1✔
96	solr_collection = current_collection_for_alias(solr_alias_uri)	×
97	if solr_collection.end_with?("-1")	×
98	solr_collection.gsub("-1", "-2")	×
99	elsif solr_collection.end_with?("-2")	×
100	solr_collection.gsub("-2", "-1")	×
101	else
102	solr_collection	×
103	end
104	end
105
106	def self.collection_exist?(solr_alias_uri, collection_name)	1✔
107	collection_list_query = URI::HTTP.build(	×
108	schema: solr_alias_uri.scheme,
109	host: solr_alias_uri.host,
110	port: solr_alias_uri.port,
111	path: "/solr/admin/collections",
112	query: "action=LIST"
113	)
114	response = HTTParty.get(collection_list_query.to_s)	×
115	collections = if response.code == 200	×
116	response.parsed_response.dig("collections") \|\| []	×
117	else
118	[]	×
119	end
120	collections.any?(collection_name)	×
121	end
122
123	def self.create_collection(solr_alias_uri, collection_name)	1✔
124	create_query = URI::HTTP.build(	×
125	schema: solr_alias_uri.scheme,
126	host: solr_alias_uri.host,
127	port: solr_alias_uri.port,
128	path: "/solr/admin/collections",
129	query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
130	)
131	response = HTTParty.get(create_query.to_s)	×
132	response.parsed_response.key?("success") ? collection_name : nil	×
133	end
134
135	def self.delete_collection!(solr_alias_uri, collection_name)	1✔
136	create_query = URI::HTTP.build(	×
137	schema: solr_alias_uri.scheme,
138	host: solr_alias_uri.host,
139	port: solr_alias_uri.port,
140	path: "/solr/admin/collections",
141	query: "action=DELETE&name=#{collection_name}"
142	)
143	response = HTTParty.get(create_query.to_s)	×
144	response.code == 200	×
145	end
146
147	# Set the solr_alias to point to the current writer collection
148	def self.update_solr_alias!	1✔
149	writer_collection = collection_writer_url.split("/").last	×
150
151	alias_name = alias_uri.path.split("/").last	×
152	if alias_name == writer_collection	×
153	# Nothing to do
154	# (we are probably using standalone Solr in development)
155	return true	×
156	end
157
158	create_query = URI::HTTP.build(	×
159	schema: alias_uri.scheme,
160	host: alias_uri.host,
161	port: alias_uri.port,
162	path: "/solr/admin/collections",
163	query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
164	)
165	response = HTTParty.get(create_query.to_s)	×
166	response.code == 200	×
167	end
168	end
169	# rubocop:enable Style/ClassVars

pulibrary / pdc_discovery / fa8e4699-854f-4a43-adc1-7e92603d9eea

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous