6d837dbf-1d25-4c27-9192-9f932836fb97

pending completion

Build # 6d837dbf-1d25-4c27-9192-9f932836fb97

Build Type

Pull #441

circleci

Committed by

hectorcorrea

Commit Message

Added a few more tests

Pull Request Pull Request #441: Indexing to a new collection

Run Details

111 of 111 new or added lines in 3 files covered. (100.0%)

2146 of 2232 relevant lines covered (96.15%)

171.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.41

/lib/traject/solr_cloud_helper.rb

# frozen_string_literal: true

class SolrCloudHelper
  def self.alias_uri
    Blacklight.default_index.connection.uri
  end

  def self.alias_url
    Blacklight.default_index.connection.uri.to_s
  end

  def self.config_set
    Blacklight.default_index.connection.options[:solr_config_set]
  end

  def self.collection_writer_commit!
    commit_url = "#{collection_writer_url}/update?commit=true"
    HTTParty.get(commit_url)
  end

  # Returns the Solr collection that we should use for quering data
  def self.collection_reader_url
    collection = current_collection_for_alias(alias_uri)
    build_solr_url_for_collection(alias_uri, collection)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exist)
  def self.collection_writer_url
    collection_writer_for_alias(alias_uri, false)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exists, deletes & recreates it if already exists)
  def self.create_collection_writer
    collection_writer_for_alias(alias_uri, true)
  end

  # For a given Solr alias, returns a URL that is suitable for writing new data without affecting
  # the existing Solr index. It does this by creating a new alternate Solr collection to write data,
  # instead of overwriting the data on the collection used by the provided solr_alias_uri.
  #
  # The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
  #
  # For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
  # and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
  # the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
  #
  # If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
  # then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
  def self.collection_writer_for_alias(solr_alias_uri, recreate)
    if config_set.nil?
      # We are not running in a Solr cloud environment - nothing to do.
      return solr_alias_uri.to_s
    end

    alternate_collection = alternate_collection_for_alias(solr_alias_uri)
    if collection_exist?(solr_alias_uri, alternate_collection)
      if recreate
        # Re-create it
        delete_collection!(solr_alias_uri, alternate_collection)
        create_collection(solr_alias_uri, alternate_collection)
      end
    else
      # Create it
      create_collection(solr_alias_uri, alternate_collection)
    end

    build_solr_url_for_collection(solr_alias_uri, alternate_collection)
  end

  # Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
  # the indicated `collection` instead of pointing to the alias.
  def self.build_solr_url_for_collection(solr_alias_uri, collection)
    build_uri(base_uri: solr_alias_uri, path: "/solr/#{collection}").to_s
  end

  def self.current_collection_for_alias(solr_alias_uri)
    alias_name = solr_alias_uri.path.split("/").last
    alias_list_query = build_uri(base_uri: solr_alias_uri, path: "/solr/admin/collections", query: "action=LISTALIASES")
    response = HTTParty.get(alias_list_query.to_s)
    # The response will indicate the actual collection the alias points to.
    collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200
    collection_name || alias_name
  end

  def self.alternate_collection_for_alias(solr_alias_uri)
    solr_collection = current_collection_for_alias(solr_alias_uri)
    if solr_collection.end_with?("-1")
      solr_collection.gsub("-1", "-2")
    elsif solr_collection.end_with?("-2")
      solr_collection.gsub("-2", "-1")
    else
      solr_collection
    end
  end

  def self.collection_exist?(solr_alias_uri, collection_name)
    collection_list_query = build_uri(base_uri: solr_alias_uri, path: "/solr/admin/collections", query: "action=LIST")
    response = HTTParty.get(collection_list_query.to_s)
    collections = if response.code == 200
                    response.parsed_response.dig("collections") || []
                  else
                    []
                  end
    collections.any?(collection_name)
  end

  def self.create_collection(solr_alias_uri, collection_name)
    create_query = build_uri(
      base_uri: solr_alias_uri,
      path: "/solr/admin/collections",
      query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
    )
    response = HTTParty.get(create_query.to_s)
    return unless response.code == 200 && response.parsed_response.key?("success")
    collection_name
  end

  def self.delete_collection!(solr_alias_uri, collection_name)
    create_query = build_uri(
      base_uri: solr_alias_uri,
      path: "/solr/admin/collections",
      query: "action=DELETE&name=#{collection_name}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end

  # Set the solr_alias to point to the current writer collection
  def self.update_solr_alias!
    writer_collection = collection_writer_url.split("/").last

    alias_name = alias_uri.path.split("/").last
    if alias_name == writer_collection
      # Nothing to do
      # (we are probably using standalone Solr in development)
      return true
    end

    create_query = build_uri(
      base_uri: solr_alias_uri,
      path: "/solr/admin/collections",
      query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end

  # Build a URI using another URI as the base.
  def self.build_uri(base_uri:, path:, query: nil)
    URI::HTTP.build(
      schema: base_uri.scheme,
      userinfo: base_uri.userinfo,
      host: base_uri.host,
      port: base_uri.port,
      path: path,
      query: query
    )
  end
end

1	# frozen_string_literal: true
2
3	class SolrCloudHelper	1✔
4	def self.alias_uri	1✔
5	Blacklight.default_index.connection.uri	243✔
6	end
7
8	def self.alias_url	1✔
9	Blacklight.default_index.connection.uri.to_s	×
10	end
11
12	def self.config_set	1✔
13	Blacklight.default_index.connection.options[:solr_config_set]	244✔
14	end
15
16	def self.collection_writer_commit!	1✔
17	commit_url = "#{collection_writer_url}/update?commit=true"	×
18	HTTParty.get(commit_url)	×
19	end
20
21	# Returns the Solr collection that we should use for quering data
22	def self.collection_reader_url	1✔
23	collection = current_collection_for_alias(alias_uri)	1✔
24	build_solr_url_for_collection(alias_uri, collection)	1✔
25	end
26
27	# Returns the Solr collection that we should write to
28	# (creates it if does not exist)
29	def self.collection_writer_url	1✔
30	collection_writer_for_alias(alias_uri, false)	241✔
31	end
32
33	# Returns the Solr collection that we should write to
34	# (creates it if does not exists, deletes & recreates it if already exists)
35	def self.create_collection_writer	1✔
36	collection_writer_for_alias(alias_uri, true)	×
37	end
38
39	# For a given Solr alias, returns a URL that is suitable for writing new data without affecting
40	# the existing Solr index. It does this by creating a new alternate Solr collection to write data,
41	# instead of overwriting the data on the collection used by the provided solr_alias_uri.
42	#
43	# The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
44	#
45	# For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
46	# and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
47	# the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
48	#
49	# If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
50	# then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
51	def self.collection_writer_for_alias(solr_alias_uri, recreate)	1✔
52	if config_set.nil?	243✔
53	# We are not running in a Solr cloud environment - nothing to do.
54	return solr_alias_uri.to_s	×
55	end
56
57	alternate_collection = alternate_collection_for_alias(solr_alias_uri)	243✔
58	if collection_exist?(solr_alias_uri, alternate_collection)	243✔
59	if recreate	243✔
60	# Re-create it
61	delete_collection!(solr_alias_uri, alternate_collection)	1✔
62	create_collection(solr_alias_uri, alternate_collection)	1✔
63	end
64	else
65	# Create it
66	create_collection(solr_alias_uri, alternate_collection)	×
67	end
68
69	build_solr_url_for_collection(solr_alias_uri, alternate_collection)	243✔
70	end
71
72	# Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
73	# the indicated `collection` instead of pointing to the alias.
74	def self.build_solr_url_for_collection(solr_alias_uri, collection)	1✔
75	build_uri(base_uri: solr_alias_uri, path: "/solr/#{collection}").to_s	244✔
76	end
77
78	def self.current_collection_for_alias(solr_alias_uri)	1✔
79	alias_name = solr_alias_uri.path.split("/").last	248✔
80	alias_list_query = build_uri(base_uri: solr_alias_uri, path: "/solr/admin/collections", query: "action=LISTALIASES")	248✔
81	response = HTTParty.get(alias_list_query.to_s)	248✔
82	# The response will indicate the actual collection the alias points to.
83	collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200	248✔
84	collection_name \|\| alias_name	248✔
85	end
86
87	def self.alternate_collection_for_alias(solr_alias_uri)	1✔
88	solr_collection = current_collection_for_alias(solr_alias_uri)	246✔
89	if solr_collection.end_with?("-1")	246✔
90	solr_collection.gsub("-1", "-2")	3✔
91	elsif solr_collection.end_with?("-2")	243✔
92	solr_collection.gsub("-2", "-1")	1✔
93	else
94	solr_collection	242✔
95	end
96	end
97
98	def self.collection_exist?(solr_alias_uri, collection_name)	1✔
99	collection_list_query = build_uri(base_uri: solr_alias_uri, path: "/solr/admin/collections", query: "action=LIST")	245✔
100	response = HTTParty.get(collection_list_query.to_s)	245✔
101	collections = if response.code == 200	245✔
102	response.parsed_response.dig("collections") \|\| []	245✔
103	else
104	[]	×
105	end
106	collections.any?(collection_name)	245✔
107	end
108
109	def self.create_collection(solr_alias_uri, collection_name)	1✔
110	create_query = build_uri(	1✔
111	base_uri: solr_alias_uri,
112	path: "/solr/admin/collections",
113	query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
114	)
115	response = HTTParty.get(create_query.to_s)	1✔
116	return unless response.code == 200 && response.parsed_response.key?("success")	1✔
117	collection_name	1✔
118	end
119
120	def self.delete_collection!(solr_alias_uri, collection_name)	1✔
121	create_query = build_uri(	1✔
122	base_uri: solr_alias_uri,
123	path: "/solr/admin/collections",
124	query: "action=DELETE&name=#{collection_name}"
125	)
126	response = HTTParty.get(create_query.to_s)	1✔
127	response.code == 200	1✔
128	end
129
130	# Set the solr_alias to point to the current writer collection
131	def self.update_solr_alias!	1✔
132	writer_collection = collection_writer_url.split("/").last	×
133
134	alias_name = alias_uri.path.split("/").last	×
135	if alias_name == writer_collection	×
136	# Nothing to do
137	# (we are probably using standalone Solr in development)
138	return true	×
139	end
140
141	create_query = build_uri(	×
142	base_uri: solr_alias_uri,
143	path: "/solr/admin/collections",
144	query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
145	)
146	response = HTTParty.get(create_query.to_s)	×
147	response.code == 200	×
148	end
149
150	# Build a URI using another URI as the base.
151	def self.build_uri(base_uri:, path:, query: nil)	1✔
152	URI::HTTP.build(	739✔
153	schema: base_uri.scheme,
154	userinfo: base_uri.userinfo,
155	host: base_uri.host,
156	port: base_uri.port,
157	path: path,
158	query: query
159	)
160	end
161	end

pulibrary / pdc_discovery / 6d837dbf-1d25-4c27-9192-9f932836fb97

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous