8702269235

Committed 16 Apr 2024 08:17AM UTC coverage: 67.82% (-0.01%) from 67.831%

Build # 8702269235

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #12881 from keynslug/fix/ds-repl-flaky

fix(dsrepl): make replication-related tests more stable

Run Details

11 of 17 new or added lines in 1 file covered. (64.71%)

32 existing lines in 11 files now uncovered.

37936 of 55936 relevant lines covered (67.82%)

7895.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.77

/apps/emqx_retainer/src/emqx_retainer_index.erl

%%--------------------------------------------------------------------
%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------

-module(emqx_retainer_index).

-export([
    foreach_index_key/3,
    to_index_key/2,
    index_score/2,
    select_index/2,
    condition/1,
    condition/2,
    restore_topic/1
]).

-export_type([index/0]).

-type index() :: list(pos_integer()).

%% @doc Index key is a term that can be effectively searched in the index table.
-type index_key() :: {index(), {emqx_types:words(), emqx_types:words()}}.

-type match_pattern_part() :: term().

%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------

%% @doc Given words of a concrete topic (`Tokens') and a list of `Indices',
%% constructs index keys for the topic and each of the indices.
%% `Fun' is called with each of these keys.
-spec foreach_index_key(fun((index_key()) -> any()), list(index()), emqx_types:words()) -> ok.
foreach_index_key(_Fun, [], _Tokens) ->
    ok;
foreach_index_key(Fun, [Index | Indices], Tokens) ->
    Key = to_index_key(Index, Tokens),
    _ = Fun(Key),
    foreach_index_key(Fun, Indices, Tokens).

%% @doc Given a concrete topic and an index
%% returns the corresponding index key.
%%
%% In an index key words from indexed and unindexed positions are split.
%%
%% E.g given `[2, 3]' index and `[<<"a">>, <<"b">>, <<"c">>, <<"d">>]' topic,
%% returns `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' term.
%%
%% @see foreach_index_key/3
-spec to_index_key(index(), emqx_types:words()) -> index_key().
to_index_key(Index, Tokens) ->
    {Index, split_index_tokens(Index, Tokens, 1, [], [])}.

%% @doc Given an index and a wildcard topic
%% returns the length of the constant prefix of the
%% according index key.
%%
%% E.g. for `[2,3]' index and <code>['+', <<"b">>, '+', <<"d">>]</code> wildcard topic
%% the score is `1', because the according index key pattern is
%% <code>{[<<"b">>, '_'], ['_', <<"d">>]}</code>.
%%
%% @see foreach_index_key/3
%% @see to_index_key/2
-spec index_score(index(), emqx_types:words()) -> non_neg_integer().
index_score(Index, Tokens) ->
    index_score(Index, Tokens, 1, 0).

%% @doc Given a list of indices and a wildcard topic
%% returns index with the best score.
%%
%% Returns `undefined' if there are no indices with score `> 0'.
%%
%% @see index_score/2
-spec select_index(emqx_types:words(), list(index())) -> index() | undefined.
select_index(Tokens, Indices) ->
    select_index(Tokens, Indices, 0, undefined).

%% @doc For an index and a wildcard topic
%% returns a tuple of:
%% * matchspec pattern for the corresponding index key
%% * boolean flag indicating whether the pattern is exact
%%
%% E.g. for `[2, 3]' index and <code>['+', <<"b">>, '+', <<"d">>]</code> wildcard topic
%% returns <code>{[2, 3], {[<<"b">>, '_'], ['_', <<"d">>]}}</code> pattern.
-spec condition(index(), emqx_types:words()) -> {match_pattern_part(), boolean()}.
condition(Index, Tokens) ->
    {Condition, IsExact} = condition(Index, Tokens, 1, [], []),
    {{Index, Condition}, IsExact}.

%% @doc Returns a matchspec pattern for a wildcard topic.
%%
%% E.g. for <code>['+', <<"b">>, '+', <<"d">>, '#']</code> wildcard topic
%% returns <code>['_', <<"b">>, '_', <<"d">> | '_']</code> pattern.
-spec condition(emqx_types:words()) -> match_pattern_part().
condition(Tokens) ->
    Tokens1 = [
        case W of
            '+' -> '_';
            _ -> W
        end
     || W <- Tokens
    ],
    case length(Tokens1) > 0 andalso lists:last(Tokens1) =:= '#' of
        false ->
            Tokens1;
        _ ->
            (Tokens1 -- ['#']) ++ '_'
    end.

%% @doc Restores concrete topic from its index key representation.
%%
%% E.g given `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' index key
%% returns `[<<"a">>, <<"b">>, <<"c">>, <<"d">>]' topic.
-spec restore_topic(index_key()) -> emqx_types:words().
restore_topic({Index, {IndexTokens, OtherTokens}}) ->
    restore_topic(Index, IndexTokens, OtherTokens, 1, []).

%%--------------------------------------------------------------------
%% Private
%%--------------------------------------------------------------------

split_index_tokens([NIndex | OtherIndex], [Token | Tokens], N, IndexTokens, OtherTokens) when
    NIndex == N
->
    split_index_tokens(OtherIndex, Tokens, N + 1, [Token | IndexTokens], OtherTokens);
split_index_tokens([_NIndex | _] = Index, [Token | Tokens], N, IndexTokens, OtherTokens) ->
    split_index_tokens(Index, Tokens, N + 1, IndexTokens, [Token | OtherTokens]);
split_index_tokens([], Tokens, _N, IndexTokens, OtherTokens) ->
    {lists:reverse(IndexTokens), lists:reverse(OtherTokens) ++ Tokens};
split_index_tokens(_Index, [], _N, IndexTokens, OtherTokens) ->
    {lists:reverse(IndexTokens), lists:reverse(OtherTokens)}.

index_score([N | _Index], [Ph | _Tokens], N, Score) when
    Ph =:= '+'; Ph =:= '#'
->
    Score;
index_score([N | Index], [_Word | Tokens], N, Score) ->
    index_score(Index, Tokens, N + 1, Score + 1);
index_score(Index, [_Word | Tokens], N, Score) ->
    index_score(Index, Tokens, N + 1, Score);
index_score([], _Tokens, _N, Score) ->
    Score;
index_score(_Index, [], _N, Score) ->
    Score.

select_index(_Tokens, [], _MaxScore, SelectedIndex) ->
    SelectedIndex;
select_index(Tokens, [Index | Indices], MaxScore, SelectedIndex) ->
    Score = index_score(Index, Tokens),
    case Score > MaxScore of
        true ->
            select_index(Tokens, Indices, Score, Index);
        false ->
            select_index(Tokens, Indices, MaxScore, SelectedIndex)
    end.

condition([_NIndex | _OtherIndex], ['#' | _OtherTokens], _N, IndexMatch, OtherMatch) ->
    {{lists:reverse(IndexMatch) ++ '_', lists:reverse(OtherMatch) ++ '_'}, false};
condition([], ['#' | _OtherTokens], _N, IndexMatch, OtherMatch) ->
    {{lists:reverse(IndexMatch), lists:reverse(OtherMatch) ++ '_'}, true};
condition([], Tokens, _N, IndexMatch, OtherMatch) ->
    {{lists:reverse(IndexMatch), lists:reverse(OtherMatch) ++ condition(Tokens)}, true};
condition([_NIndex | _OtherIndex], [], _N, IndexMatch, OtherMatch) ->
    {{lists:reverse(IndexMatch), lists:reverse(OtherMatch)}, true};
condition([NIndex | OtherIndex], ['+' | OtherTokens], N, IndexMatch, OtherMatch) when
    NIndex =:= N
->
    condition(OtherIndex, OtherTokens, N + 1, ['_' | IndexMatch], OtherMatch);
condition(Index, ['+' | OtherTokens], N, IndexMatch, OtherMatch) ->
    condition(Index, OtherTokens, N + 1, IndexMatch, ['_' | OtherMatch]);
condition([NIndex | OtherIndex], [Token | OtherTokens], N, IndexMatch, OtherMatch) when
    NIndex =:= N, is_binary(Token) orelse Token =:= ''
->
    condition(OtherIndex, OtherTokens, N + 1, [Token | IndexMatch], OtherMatch);
condition(Index, [Token | OtherTokens], N, IndexMatch, OtherMatch) when
    is_binary(Token) orelse Token =:= ''
->
    condition(Index, OtherTokens, N + 1, IndexMatch, [Token | OtherMatch]).

restore_topic(_Index, [], OtherTokens, _N, Tokens) ->
    lists:reverse(Tokens) ++ OtherTokens;
restore_topic([NIndex | OtherIndex], [IndexToken | OtherIndexTokens], OtherTokens, N, Tokens) when
    NIndex =:= N
->
    restore_topic(OtherIndex, OtherIndexTokens, OtherTokens, N + 1, [IndexToken | Tokens]);
restore_topic(OtherIndex, IndexTokens, [Token | OtherTokens], N, Tokens) ->
    restore_topic(OtherIndex, IndexTokens, OtherTokens, N + 1, [Token | Tokens]).

1	%%--------------------------------------------------------------------
2	%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
3	%%
4	%% Licensed under the Apache License, Version 2.0 (the "License");
5	%% you may not use this file except in compliance with the License.
6	%% You may obtain a copy of the License at
7	%%
8	%% http://www.apache.org/licenses/LICENSE-2.0
9	%%
10	%% Unless required by applicable law or agreed to in writing, software
11	%% distributed under the License is distributed on an "AS IS" BASIS,
12	%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	%% See the License for the specific language governing permissions and
14	%% limitations under the License.
15	%%--------------------------------------------------------------------
16
17	-module(emqx_retainer_index).
18
19	-export([
20	foreach_index_key/3,
21	to_index_key/2,
22	index_score/2,
23	select_index/2,
24	condition/1,
25	condition/2,
26	restore_topic/1
27	]).
28
29	-export_type([index/0]).
30
31	-type index() :: list(pos_integer()).
32
33	%% @doc Index key is a term that can be effectively searched in the index table.
34	-type index_key() :: {index(), {emqx_types:words(), emqx_types:words()}}.
35
36	-type match_pattern_part() :: term().
37
38	%%--------------------------------------------------------------------
39	%% API
40	%%--------------------------------------------------------------------
41
42	%% @doc Given words of a concrete topic (`Tokens') and a list of `Indices',
43	%% constructs index keys for the topic and each of the indices.
44	%% `Fun' is called with each of these keys.
45	-spec foreach_index_key(fun((index_key()) -> any()), list(index()), emqx_types:words()) -> ok.
46	foreach_index_key(_Fun, [], _Tokens) ->
47	ok;	49✔
48	foreach_index_key(Fun, [Index \| Indices], Tokens) ->
49	Key = to_index_key(Index, Tokens),	196✔
50	_ = Fun(Key),	196✔
51	foreach_index_key(Fun, Indices, Tokens).	196✔
52
53	%% @doc Given a concrete topic and an index
54	%% returns the corresponding index key.
55	%%
56	%% In an index key words from indexed and unindexed positions are split.
57	%%
58	%% E.g given `[2, 3]' index and `[<<"a">>, <<"b">>, <<"c">>, <<"d">>]' topic,
59	%% returns `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' term.
60	%%
61	%% @see foreach_index_key/3
62	-spec to_index_key(index(), emqx_types:words()) -> index_key().
63	to_index_key(Index, Tokens) ->
64	{Index, split_index_tokens(Index, Tokens, 1, [], [])}.	1,265✔
65
66	%% @doc Given an index and a wildcard topic
67	%% returns the length of the constant prefix of the
68	%% according index key.
69	%%
70	%% E.g. for `[2,3]' index and <code>['+', <<"b">>, '+', <<"d">>]</code> wildcard topic
71	%% the score is `1', because the according index key pattern is
72	%% <code>{[<<"b">>, '_'], ['_', <<"d">>]}</code>.
73	%%
74	%% @see foreach_index_key/3
75	%% @see to_index_key/2
76	-spec index_score(index(), emqx_types:words()) -> non_neg_integer().
77	index_score(Index, Tokens) ->
78	index_score(Index, Tokens, 1, 0).	×
79
80	%% @doc Given a list of indices and a wildcard topic
81	%% returns index with the best score.
82	%%
83	%% Returns `undefined' if there are no indices with score `> 0'.
84	%%
85	%% @see index_score/2
86	-spec select_index(emqx_types:words(), list(index())) -> index() \| undefined.
87	select_index(Tokens, Indices) ->
88	select_index(Tokens, Indices, 0, undefined).	×
89
90	%% @doc For an index and a wildcard topic
91	%% returns a tuple of:
92	%% * matchspec pattern for the corresponding index key
93	%% * boolean flag indicating whether the pattern is exact
94	%%
95	%% E.g. for `[2, 3]' index and <code>['+', <<"b">>, '+', <<"d">>]</code> wildcard topic
96	%% returns <code>{[2, 3], {[<<"b">>, '_'], ['_', <<"d">>]}}</code> pattern.
97	-spec condition(index(), emqx_types:words()) -> {match_pattern_part(), boolean()}.
98	condition(Index, Tokens) ->
99	{Condition, IsExact} = condition(Index, Tokens, 1, [], []),	100✔
100	{{Index, Condition}, IsExact}.	100✔
101
102	%% @doc Returns a matchspec pattern for a wildcard topic.
103	%%
104	%% E.g. for <code>['+', <<"b">>, '+', <<"d">>, '#']</code> wildcard topic
105	%% returns <code>['_', <<"b">>, '_', <<"d">> \| '_']</code> pattern.
106	-spec condition(emqx_types:words()) -> match_pattern_part().
107	condition(Tokens) ->
108	Tokens1 = [	12✔
109	case W of	26✔
110	'+' -> '_';	11✔
111	_ -> W	15✔
112	end
113	\|\| W <- Tokens	12✔
114	],
115	case length(Tokens1) > 0 andalso lists:last(Tokens1) =:= '#' of	12✔
116	false ->
117	Tokens1;	6✔
118	_ ->
119	(Tokens1 -- ['#']) ++ '_'	6✔
120	end.
121
122	%% @doc Restores concrete topic from its index key representation.
123	%%
124	%% E.g given `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' index key
125	%% returns `[<<"a">>, <<"b">>, <<"c">>, <<"d">>]' topic.
126	-spec restore_topic(index_key()) -> emqx_types:words().
127	restore_topic({Index, {IndexTokens, OtherTokens}}) ->
128	restore_topic(Index, IndexTokens, OtherTokens, 1, []).	155✔
129
130	%%--------------------------------------------------------------------
131	%% Private
132	%%--------------------------------------------------------------------
133
134	split_index_tokens([NIndex \| OtherIndex], [Token \| Tokens], N, IndexTokens, OtherTokens) when
135	NIndex == N
136	->
137	split_index_tokens(OtherIndex, Tokens, N + 1, [Token \| IndexTokens], OtherTokens);	1,195✔
138	split_index_tokens([_NIndex \| _] = Index, [Token \| Tokens], N, IndexTokens, OtherTokens) ->
139	split_index_tokens(Index, Tokens, N + 1, IndexTokens, [Token \| OtherTokens]);	4,388✔
140	split_index_tokens([], Tokens, _N, IndexTokens, OtherTokens) ->
141	{lists:reverse(IndexTokens), lists:reverse(OtherTokens) ++ Tokens};	354✔
142	split_index_tokens(_Index, [], _N, IndexTokens, OtherTokens) ->
143	{lists:reverse(IndexTokens), lists:reverse(OtherTokens)}.	911✔
144
145	index_score([N \| _Index], [Ph \| _Tokens], N, Score) when
146	Ph =:= '+'; Ph =:= '#'
147	->
148	Score;	×
149	index_score([N \| Index], [_Word \| Tokens], N, Score) ->
150	index_score(Index, Tokens, N + 1, Score + 1);	×
151	index_score(Index, [_Word \| Tokens], N, Score) ->
152	index_score(Index, Tokens, N + 1, Score);	×
153	index_score([], _Tokens, _N, Score) ->
154	Score;	×
155	index_score(_Index, [], _N, Score) ->
156	Score.	×
157
158	select_index(_Tokens, [], _MaxScore, SelectedIndex) ->
159	SelectedIndex;	×
160	select_index(Tokens, [Index \| Indices], MaxScore, SelectedIndex) ->
161	Score = index_score(Index, Tokens),	×
162	case Score > MaxScore of	×
163	true ->
164	select_index(Tokens, Indices, Score, Index);	×
165	false ->
166	select_index(Tokens, Indices, MaxScore, SelectedIndex)	×
167	end.
168
169	condition([_NIndex \| _OtherIndex], ['#' \| _OtherTokens], _N, IndexMatch, OtherMatch) ->
170	{{lists:reverse(IndexMatch) ++ '_', lists:reverse(OtherMatch) ++ '_'}, false};	43✔
171	condition([], ['#' \| _OtherTokens], _N, IndexMatch, OtherMatch) ->
UNCOV 172	{{lists:reverse(IndexMatch), lists:reverse(OtherMatch) ++ '_'}, true};	×
173	condition([], Tokens, _N, IndexMatch, OtherMatch) ->
174	{{lists:reverse(IndexMatch), lists:reverse(OtherMatch) ++ condition(Tokens)}, true};	12✔
175	condition([_NIndex \| _OtherIndex], [], _N, IndexMatch, OtherMatch) ->
176	{{lists:reverse(IndexMatch), lists:reverse(OtherMatch)}, true};	45✔
177	condition([NIndex \| OtherIndex], ['+' \| OtherTokens], N, IndexMatch, OtherMatch) when
178	NIndex =:= N
179	->
180	condition(OtherIndex, OtherTokens, N + 1, ['_' \| IndexMatch], OtherMatch);	21✔
181	condition(Index, ['+' \| OtherTokens], N, IndexMatch, OtherMatch) ->
182	condition(Index, OtherTokens, N + 1, IndexMatch, ['_' \| OtherMatch]);	116✔
183	condition([NIndex \| OtherIndex], [Token \| OtherTokens], N, IndexMatch, OtherMatch) when
184	NIndex =:= N, is_binary(Token) orelse Token =:= ''
185	->
186	condition(OtherIndex, OtherTokens, N + 1, [Token \| IndexMatch], OtherMatch);	20✔
187	condition(Index, [Token \| OtherTokens], N, IndexMatch, OtherMatch) when
188	is_binary(Token) orelse Token =:= ''
189	->
190	condition(Index, OtherTokens, N + 1, IndexMatch, [Token \| OtherMatch]).	92✔
191
192	restore_topic(_Index, [], OtherTokens, _N, Tokens) ->
193	lists:reverse(Tokens) ++ OtherTokens;	155✔
194	restore_topic([NIndex \| OtherIndex], [IndexToken \| OtherIndexTokens], OtherTokens, N, Tokens) when
195	NIndex =:= N
196	->
197	restore_topic(OtherIndex, OtherIndexTokens, OtherTokens, N + 1, [IndexToken \| Tokens]);	155✔
198	restore_topic(OtherIndex, IndexTokens, [Token \| OtherTokens], N, Tokens) ->
199	restore_topic(OtherIndex, IndexTokens, OtherTokens, N + 1, [Token \| Tokens]).	321✔

emqx / emqx / 8702269235

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous