8644497754

Committed 11 Apr 2024 09:24AM UTC coverage: 62.388% (-0.05%) from 62.44%

Build # 8644497754

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #12858 from zmstone/0410-fix-variform-number-handling

fix(variform): allow numbers to be numbers

Run Details

2 of 3 new or added lines in 1 file covered. (66.67%)

67 existing lines in 12 files now uncovered.

34873 of 55897 relevant lines covered (62.39%)

6476.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.65

/apps/emqx_utils/src/emqx_variform.erl

%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------

%% @doc This module provides a single-line expression string rendering engine.
%% A predefined set of functions are allowed to be called in the expressions.
%% Only simple string expressions are supported, and no control flow is allowed.
%% However, with the help from the functions, some control flow can be achieved.
%% For example, the `coalesce` function can be used to provide a default value,
%% or used to choose the first non-empty value from a list of variables.
-module(emqx_variform).

-export([
    inject_allowed_module/1,
    inject_allowed_modules/1,
    erase_allowed_module/1,
    erase_allowed_modules/1
]).
-export([render/2, render/3]).

%% @doc Render a variform expression with bindings.
%% A variform expression is a template string which supports variable substitution
%% and function calls.
%%
%% The function calls are in the form of `module.function(arg1, arg2, ...)` where `module`
%% is optional, and if not provided, the function is assumed to be in the `emqx_variform_str` module.
%% Both module and function must be existing atoms, and only whitelisted functions are allowed.
%%
%% A function arg can be a constant string or a number.
%% Strings can be quoted with single quotes or double quotes, without support of escape characters.
%% If some special characters are needed, the function `unescape' can be used convert a escaped string
%% to raw bytes.
%% For example, to get the first line of a multi-line string, the expression can be
%% `coalesce(tokens(variable_name, unescape("\n")))'.
%%
%% The bindings is a map of variables to their values.
%%
%% For unresolved variables, empty string (but not "undefined") is used.
%% In case of runtime exeption, an error is returned.
-spec render(string(), map()) -> {ok, binary()} | {error, term()}.
render(Expression, Bindings) ->
    render(Expression, Bindings, #{}).

render(Expression, Bindings, Opts) when is_binary(Expression) ->
    render(unicode:characters_to_list(Expression), Bindings, Opts);
render(Expression, Bindings, Opts) ->
    case emqx_variform_scan:string(Expression) of
        {ok, Tokens, _Line} ->
            case emqx_variform_parser:parse(Tokens) of
                {ok, Expr} ->
                    eval_as_string(Expr, Bindings, Opts);
                {error, {_, emqx_variform_parser, Msg}} ->
                    %% syntax error
                    {error, lists:flatten(Msg)};
                {error, Reason} ->
                    {error, Reason}
            end;
        {error, Reason, _Line} ->
            {error, Reason}
    end.

eval_as_string(Expr, Bindings, _Opts) ->
    try
        {ok, str(eval(Expr, Bindings))}
    catch
        throw:Reason ->
            {error, Reason};
        C:E:S ->
            {error, #{exception => C, reason => E, stack_trace => S}}
    end.

eval({str, Str}, _Bindings) ->
    str(Str);
eval({integer, Num}, _Bindings) ->
    Num;
eval({float, Num}, _Bindings) ->
    Num;
eval({array, Args}, Bindings) ->
    eval(Args, Bindings);
eval({call, FuncNameStr, Args}, Bindings) ->
    {Mod, Fun} = resolve_func_name(FuncNameStr),
    ok = assert_func_exported(Mod, Fun, length(Args)),
    call(Mod, Fun, eval(Args, Bindings));
eval({var, VarName}, Bindings) ->
    resolve_var_value(VarName, Bindings);
eval([Arg | Args], Bindings) ->
    [eval(Arg, Bindings) | eval(Args, Bindings)];
eval([], _Bindings) ->
    [].

%% Some functions accept arbitrary number of arguments but implemented as /1.
call(emqx_variform_str, concat, Args) ->
    str(emqx_variform_str:concat(Args));
call(emqx_variform_str, coalesce, Args) ->
    str(emqx_variform_str:coalesce(Args));
call(Mod, Fun, Args) ->
    erlang:apply(Mod, Fun, Args).

resolve_func_name(FuncNameStr) ->
    case string:tokens(FuncNameStr, ".") of
        [Mod0, Fun0] ->
            Mod =
                try
                    list_to_existing_atom(Mod0)
                catch
                    error:badarg ->
                        throw(#{
                            reason => unknown_variform_module,
                            module => Mod0
                        })
                end,
            ok = assert_module_allowed(Mod),
            Fun =
                try
                    list_to_existing_atom(Fun0)
                catch
                    error:badarg ->
                        throw(#{
                            reason => unknown_variform_function,
                            function => Fun0
                        })
                end,
            {Mod, Fun};
        [Fun] ->
            FuncName =
                try
                    list_to_existing_atom(Fun)
                catch
                    error:badarg ->
                        throw(#{
                            reason => unknown_variform_function,
                            function => Fun
                        })
                end,
            {emqx_variform_str, FuncName};
        _ ->
            throw(#{reason => invalid_function_reference, function => FuncNameStr})
    end.

resolve_var_value(VarName, Bindings) ->
    case emqx_template:lookup_var(split(VarName), Bindings) of
        {ok, Value} ->
            Value;
        {error, _Reason} ->
            <<>>
    end.

assert_func_exported(emqx_variform_str, concat, _Arity) ->
    ok;
assert_func_exported(emqx_variform_str, coalesce, _Arity) ->
    ok;
assert_func_exported(Mod, Fun, Arity) ->
    ok = try_load(Mod),
    case erlang:function_exported(Mod, Fun, Arity) of
        true ->
            ok;
        false ->
            throw(#{
                reason => unknown_variform_function,
                module => Mod,
                function => Fun,
                arity => Arity
            })
    end.

%% best effort to load the module because it might not be loaded as a part of the release modules
%% e.g. from a plugin.
%% do not call code server, just try to call a function in the module.
try_load(Mod) ->
    try
        _ = erlang:apply(Mod, module_info, [md5]),
        ok
    catch
        _:_ ->
            ok
    end.

assert_module_allowed(emqx_variform_str) ->
    ok;
assert_module_allowed(Mod) ->
    Allowed = get_allowed_modules(),
    case lists:member(Mod, Allowed) of
        true ->
            ok;
        false ->
            throw(#{
                reason => unallowed_veriform_module,
                module => Mod
            })
    end.

inject_allowed_module(Module) when is_atom(Module) ->
    inject_allowed_modules([Module]).

inject_allowed_modules(Modules) when is_list(Modules) ->
    Allowed0 = get_allowed_modules(),
    Allowed = lists:usort(Allowed0 ++ Modules),
    persistent_term:put({emqx_variform, allowed_modules}, Allowed).

erase_allowed_module(Module) when is_atom(Module) ->
    erase_allowed_modules([Module]).

erase_allowed_modules(Modules) when is_list(Modules) ->
    Allowed0 = get_allowed_modules(),
    Allowed = Allowed0 -- Modules,
    persistent_term:put({emqx_variform, allowed_modules}, Allowed).

get_allowed_modules() ->
    persistent_term:get({emqx_variform, allowed_modules}, []).

str(Value) ->
    emqx_utils_conv:bin(Value).

split(VarName) ->
    lists:map(fun erlang:iolist_to_binary/1, string:tokens(VarName, ".")).

1	%%--------------------------------------------------------------------
2	%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
3	%%
4	%% Licensed under the Apache License, Version 2.0 (the "License");
5	%% you may not use this file except in compliance with the License.
6	%% You may obtain a copy of the License at
7	%%
8	%% http://www.apache.org/licenses/LICENSE-2.0
9	%%
10	%% Unless required by applicable law or agreed to in writing, software
11	%% distributed under the License is distributed on an "AS IS" BASIS,
12	%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	%% See the License for the specific language governing permissions and
14	%% limitations under the License.
15	%%--------------------------------------------------------------------
16
17	%% @doc This module provides a single-line expression string rendering engine.
18	%% A predefined set of functions are allowed to be called in the expressions.
19	%% Only simple string expressions are supported, and no control flow is allowed.
20	%% However, with the help from the functions, some control flow can be achieved.
21	%% For example, the `coalesce` function can be used to provide a default value,
22	%% or used to choose the first non-empty value from a list of variables.
23	-module(emqx_variform).
24
25	-export([
26	inject_allowed_module/1,
27	inject_allowed_modules/1,
28	erase_allowed_module/1,
29	erase_allowed_modules/1
30	]).
31	-export([render/2, render/3]).
32
33	%% @doc Render a variform expression with bindings.
34	%% A variform expression is a template string which supports variable substitution
35	%% and function calls.
36	%%
37	%% The function calls are in the form of `module.function(arg1, arg2, ...)` where `module`
38	%% is optional, and if not provided, the function is assumed to be in the `emqx_variform_str` module.
39	%% Both module and function must be existing atoms, and only whitelisted functions are allowed.
40	%%
41	%% A function arg can be a constant string or a number.
42	%% Strings can be quoted with single quotes or double quotes, without support of escape characters.
43	%% If some special characters are needed, the function `unescape' can be used convert a escaped string
44	%% to raw bytes.
45	%% For example, to get the first line of a multi-line string, the expression can be
46	%% `coalesce(tokens(variable_name, unescape("\n")))'.
47	%%
48	%% The bindings is a map of variables to their values.
49	%%
50	%% For unresolved variables, empty string (but not "undefined") is used.
51	%% In case of runtime exeption, an error is returned.
52	-spec render(string(), map()) -> {ok, binary()} \| {error, term()}.
53	render(Expression, Bindings) ->
54	render(Expression, Bindings, #{}).	29✔
55
56	render(Expression, Bindings, Opts) when is_binary(Expression) ->
57	render(unicode:characters_to_list(Expression), Bindings, Opts);	1✔
58	render(Expression, Bindings, Opts) ->
59	case emqx_variform_scan:string(Expression) of	29✔
60	{ok, Tokens, _Line} ->
61	case emqx_variform_parser:parse(Tokens) of	29✔
62	{ok, Expr} ->
63	eval_as_string(Expr, Bindings, Opts);	25✔
64	{error, {_, emqx_variform_parser, Msg}} ->
65	%% syntax error
66	{error, lists:flatten(Msg)};	4✔
67	{error, Reason} ->
68	{error, Reason}	×
69	end;
70	{error, Reason, _Line} ->
71	{error, Reason}	×
72	end.
73
74	eval_as_string(Expr, Bindings, _Opts) ->
75	try	25✔
76	{ok, str(eval(Expr, Bindings))}	25✔
77	catch
78	throw:Reason ->
79	{error, Reason};	7✔
80	C:E:S ->
81	{error, #{exception => C, reason => E, stack_trace => S}}	×
82	end.
83
84	eval({str, Str}, _Bindings) ->
85	str(Str);	17✔
86	eval({integer, Num}, _Bindings) ->
87	Num;	4✔
88	eval({float, Num}, _Bindings) ->
NEW 89	Num;	×
90	eval({array, Args}, Bindings) ->
91	eval(Args, Bindings);	1✔
92	eval({call, FuncNameStr, Args}, Bindings) ->
93	{Mod, Fun} = resolve_func_name(FuncNameStr),	27✔
94	ok = assert_func_exported(Mod, Fun, length(Args)),	22✔
95	call(Mod, Fun, eval(Args, Bindings));	21✔
96	eval({var, VarName}, Bindings) ->
97	resolve_var_value(VarName, Bindings);	17✔
98	eval([Arg \| Args], Bindings) ->
99	[eval(Arg, Bindings) \| eval(Args, Bindings)];	41✔
100	eval([], _Bindings) ->
101	[].	22✔
102
103	%% Some functions accept arbitrary number of arguments but implemented as /1.
104	call(emqx_variform_str, concat, Args) ->
105	str(emqx_variform_str:concat(Args));	3✔
106	call(emqx_variform_str, coalesce, Args) ->
107	str(emqx_variform_str:coalesce(Args));	5✔
108	call(Mod, Fun, Args) ->
109	erlang:apply(Mod, Fun, Args).	13✔
110
111	resolve_func_name(FuncNameStr) ->
112	case string:tokens(FuncNameStr, ".") of	27✔
113	[Mod0, Fun0] ->
114	Mod =	5✔
115	try
116	list_to_existing_atom(Mod0)	5✔
117	catch
118	error:badarg ->
119	throw(#{	1✔
120	reason => unknown_variform_module,
121	module => Mod0
122	})
123	end,
124	ok = assert_module_allowed(Mod),	4✔
125	Fun =	3✔
126	try
127	list_to_existing_atom(Fun0)	3✔
128	catch
129	error:badarg ->
130	throw(#{	1✔
131	reason => unknown_variform_function,
132	function => Fun0
133	})
134	end,
135	{Mod, Fun};	2✔
136	[Fun] ->
137	FuncName =	21✔
138	try
139	list_to_existing_atom(Fun)	21✔
140	catch
141	error:badarg ->
142	throw(#{	1✔
143	reason => unknown_variform_function,
144	function => Fun
145	})
146	end,
147	{emqx_variform_str, FuncName};	20✔
148	_ ->
149	throw(#{reason => invalid_function_reference, function => FuncNameStr})	1✔
150	end.
151
152	resolve_var_value(VarName, Bindings) ->
153	case emqx_template:lookup_var(split(VarName), Bindings) of	17✔
154	{ok, Value} ->
155	Value;	13✔
156	{error, _Reason} ->
157	<<>>	4✔
158	end.
159
160	assert_func_exported(emqx_variform_str, concat, _Arity) ->
161	ok;	3✔
162	assert_func_exported(emqx_variform_str, coalesce, _Arity) ->
163	ok;	5✔
164	assert_func_exported(Mod, Fun, Arity) ->
165	ok = try_load(Mod),	14✔
166	case erlang:function_exported(Mod, Fun, Arity) of	14✔
167	true ->
168	ok;	13✔
169	false ->
170	throw(#{	1✔
171	reason => unknown_variform_function,
172	module => Mod,
173	function => Fun,
174	arity => Arity
175	})
176	end.
177
178	%% best effort to load the module because it might not be loaded as a part of the release modules
179	%% e.g. from a plugin.
180	%% do not call code server, just try to call a function in the module.
181	try_load(Mod) ->
182	try	14✔
183	_ = erlang:apply(Mod, module_info, [md5]),	14✔
184	ok	14✔
185	catch
186	_:_ ->
187	ok	×
188	end.
189
190	assert_module_allowed(emqx_variform_str) ->
191	ok;	1✔
192	assert_module_allowed(Mod) ->
193	Allowed = get_allowed_modules(),	3✔
194	case lists:member(Mod, Allowed) of	3✔
195	true ->
196	ok;	2✔
197	false ->
198	throw(#{	1✔
199	reason => unallowed_veriform_module,
200	module => Mod
201	})
202	end.
203
204	inject_allowed_module(Module) when is_atom(Module) ->
205	inject_allowed_modules([Module]).	1✔
206
207	inject_allowed_modules(Modules) when is_list(Modules) ->
208	Allowed0 = get_allowed_modules(),	1✔
209	Allowed = lists:usort(Allowed0 ++ Modules),	1✔
210	persistent_term:put({emqx_variform, allowed_modules}, Allowed).	1✔
211
212	erase_allowed_module(Module) when is_atom(Module) ->
213	erase_allowed_modules([Module]).	1✔
214
215	erase_allowed_modules(Modules) when is_list(Modules) ->
216	Allowed0 = get_allowed_modules(),	1✔
217	Allowed = Allowed0 -- Modules,	1✔
218	persistent_term:put({emqx_variform, allowed_modules}, Allowed).	1✔
219
220	get_allowed_modules() ->
221	persistent_term:get({emqx_variform, allowed_modules}, []).	5✔
222
223	str(Value) ->
224	emqx_utils_conv:bin(Value).	43✔
225
226	split(VarName) ->
227	lists:map(fun erlang:iolist_to_binary/1, string:tokens(VarName, ".")).	17✔

emqx / emqx / 8644497754

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous