8628139215

Committed 10 Apr 2024 08:18AM UTC coverage: 62.44% (-0.05%) from 62.489%

Build # 8628139215

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #12851 from zmstone/0327-feat-add-emqx_variform

emqx_variform for string substitution and transform

Run Details

206 of 238 new or added lines in 3 files covered. (86.55%)

28 existing lines in 16 files now uncovered.

34895 of 55886 relevant lines covered (62.44%)

6585.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.03

/apps/emqx_utils/src/emqx_variform.erl

%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------

%% @doc This module provides a single-line expression string rendering engine.
%% A predefined set of functions are allowed to be called in the expressions.
%% Only simple string expressions are supported, and no control flow is allowed.
%% However, with the help from the functions, some control flow can be achieved.
%% For example, the `coalesce` function can be used to provide a default value,
%% or used to choose the first non-empty value from a list of variables.
-module(emqx_variform).

-export([
    inject_allowed_module/1,
    inject_allowed_modules/1,
    erase_allowed_module/1,
    erase_allowed_modules/1
]).
-export([render/2, render/3]).

%% @doc Render a variform expression with bindings.
%% A variform expression is a template string which supports variable substitution
%% and function calls.
%%
%% The function calls are in the form of `module.function(arg1, arg2, ...)` where `module`
%% is optional, and if not provided, the function is assumed to be in the `emqx_variform_str` module.
%% Both module and function must be existing atoms, and only whitelisted functions are allowed.
%%
%% A function arg can be a constant string or a number.
%% Strings can be quoted with single quotes or double quotes, without support of escape characters.
%% If some special characters are needed, the function `unescape' can be used convert a escaped string
%% to raw bytes.
%% For example, to get the first line of a multi-line string, the expression can be
%% `coalesce(tokens(variable_name, unescape("\n")))'.
%%
%% The bindings is a map of variables to their values.
%%
%% For unresolved variables, empty string (but not "undefined") is used.
%% In case of runtime exeption, an error is returned.
-spec render(string(), map()) -> {ok, binary()} | {error, term()}.
render(Expression, Bindings) ->
    render(Expression, Bindings, #{}).

render(Expression, Bindings, Opts) when is_binary(Expression) ->
    render(unicode:characters_to_list(Expression), Bindings, Opts);
render(Expression, Bindings, Opts) ->
    case emqx_variform_scan:string(Expression) of
        {ok, Tokens, _Line} ->
            case emqx_variform_parser:parse(Tokens) of
                {ok, Expr} ->
                    eval_as_string(Expr, Bindings, Opts);
                {error, {_, emqx_variform_parser, Msg}} ->
                    %% syntax error
                    {error, lists:flatten(Msg)};
                {error, Reason} ->
                    {error, Reason}
            end;
        {error, Reason, _Line} ->
            {error, Reason}
    end.

eval_as_string(Expr, Bindings, _Opts) ->
    try
        {ok, str(eval(Expr, Bindings))}
    catch
        throw:Reason ->
            {error, Reason};
        C:E:S ->
            {error, #{exception => C, reason => E, stack_trace => S}}
    end.

eval({str, Str}, _Bindings) ->
    str(Str);
eval({num, Num}, _Bindings) ->
    str(Num);
eval({array, Args}, Bindings) ->
    eval(Args, Bindings);
eval({call, FuncNameStr, Args}, Bindings) ->
    {Mod, Fun} = resolve_func_name(FuncNameStr),
    ok = assert_func_exported(Mod, Fun, length(Args)),
    call(Mod, Fun, eval(Args, Bindings));
eval({var, VarName}, Bindings) ->
    resolve_var_value(VarName, Bindings);
eval([Arg | Args], Bindings) ->
    [eval(Arg, Bindings) | eval(Args, Bindings)];
eval([], _Bindings) ->
    [].

%% Some functions accept arbitrary number of arguments but implemented as /1.
call(emqx_variform_str, concat, Args) ->
    str(emqx_variform_str:concat(Args));
call(emqx_variform_str, coalesce, Args) ->
    str(emqx_variform_str:coalesce(Args));
call(Mod, Fun, Args) ->
    erlang:apply(Mod, Fun, Args).

resolve_func_name(FuncNameStr) ->
    case string:tokens(FuncNameStr, ".") of
        [Mod0, Fun0] ->
            Mod =
                try
                    list_to_existing_atom(Mod0)
                catch
                    error:badarg ->
                        throw(#{
                            reason => unknown_variform_module,
                            module => Mod0
                        })
                end,
            ok = assert_module_allowed(Mod),
            Fun =
                try
                    list_to_existing_atom(Fun0)
                catch
                    error:badarg ->
                        throw(#{
                            reason => unknown_variform_function,
                            function => Fun0
                        })
                end,
            {Mod, Fun};
        [Fun] ->
            FuncName =
                try
                    list_to_existing_atom(Fun)
                catch
                    error:badarg ->
                        throw(#{
                            reason => unknown_variform_function,
                            function => Fun
                        })
                end,
            {emqx_variform_str, FuncName};
        _ ->
            throw(#{reason => invalid_function_reference, function => FuncNameStr})
    end.

resolve_var_value(VarName, Bindings) ->
    case emqx_template:lookup_var(split(VarName), Bindings) of
        {ok, Value} ->
            str(Value);
        {error, _Reason} ->
            <<>>
    end.

assert_func_exported(emqx_variform_str, concat, _Arity) ->
    ok;
assert_func_exported(emqx_variform_str, coalesce, _Arity) ->
    ok;
assert_func_exported(Mod, Fun, Arity) ->
    ok = try_load(Mod),
    case erlang:function_exported(Mod, Fun, Arity) of
        true ->
            ok;
        false ->
            throw(#{
                reason => unknown_variform_function,
                module => Mod,
                function => Fun,
                arity => Arity
            })
    end.

%% best effort to load the module because it might not be loaded as a part of the release modules
%% e.g. from a plugin.
%% do not call code server, just try to call a function in the module.
try_load(Mod) ->
    try
        _ = erlang:apply(Mod, module_info, [md5]),
        ok
    catch
        _:_ ->
            ok
    end.

assert_module_allowed(emqx_variform_str) ->
    ok;
assert_module_allowed(Mod) ->
    Allowed = get_allowed_modules(),
    case lists:member(Mod, Allowed) of
        true ->
            ok;
        false ->
            throw(#{
                reason => unallowed_veriform_module,
                module => Mod
            })
    end.

inject_allowed_module(Module) when is_atom(Module) ->
    inject_allowed_modules([Module]).

inject_allowed_modules(Modules) when is_list(Modules) ->
    Allowed0 = get_allowed_modules(),
    Allowed = lists:usort(Allowed0 ++ Modules),
    persistent_term:put({emqx_variform, allowed_modules}, Allowed).

erase_allowed_module(Module) when is_atom(Module) ->
    erase_allowed_modules([Module]).

erase_allowed_modules(Modules) when is_list(Modules) ->
    Allowed0 = get_allowed_modules(),
    Allowed = Allowed0 -- Modules,
    persistent_term:put({emqx_variform, allowed_modules}, Allowed).

get_allowed_modules() ->
    persistent_term:get({emqx_variform, allowed_modules}, []).

str(Value) ->
    emqx_utils_conv:bin(Value).

split(VarName) ->
    lists:map(fun erlang:iolist_to_binary/1, string:tokens(VarName, ".")).

1	%%--------------------------------------------------------------------
2	%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
3	%%
4	%% Licensed under the Apache License, Version 2.0 (the "License");
5	%% you may not use this file except in compliance with the License.
6	%% You may obtain a copy of the License at
7	%%
8	%% http://www.apache.org/licenses/LICENSE-2.0
9	%%
10	%% Unless required by applicable law or agreed to in writing, software
11	%% distributed under the License is distributed on an "AS IS" BASIS,
12	%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	%% See the License for the specific language governing permissions and
14	%% limitations under the License.
15	%%--------------------------------------------------------------------
16
17	%% @doc This module provides a single-line expression string rendering engine.
18	%% A predefined set of functions are allowed to be called in the expressions.
19	%% Only simple string expressions are supported, and no control flow is allowed.
20	%% However, with the help from the functions, some control flow can be achieved.
21	%% For example, the `coalesce` function can be used to provide a default value,
22	%% or used to choose the first non-empty value from a list of variables.
23	-module(emqx_variform).
24
25	-export([
26	inject_allowed_module/1,
27	inject_allowed_modules/1,
28	erase_allowed_module/1,
29	erase_allowed_modules/1
30	]).
31	-export([render/2, render/3]).
32
33	%% @doc Render a variform expression with bindings.
34	%% A variform expression is a template string which supports variable substitution
35	%% and function calls.
36	%%
37	%% The function calls are in the form of `module.function(arg1, arg2, ...)` where `module`
38	%% is optional, and if not provided, the function is assumed to be in the `emqx_variform_str` module.
39	%% Both module and function must be existing atoms, and only whitelisted functions are allowed.
40	%%
41	%% A function arg can be a constant string or a number.
42	%% Strings can be quoted with single quotes or double quotes, without support of escape characters.
43	%% If some special characters are needed, the function `unescape' can be used convert a escaped string
44	%% to raw bytes.
45	%% For example, to get the first line of a multi-line string, the expression can be
46	%% `coalesce(tokens(variable_name, unescape("\n")))'.
47	%%
48	%% The bindings is a map of variables to their values.
49	%%
50	%% For unresolved variables, empty string (but not "undefined") is used.
51	%% In case of runtime exeption, an error is returned.
52	-spec render(string(), map()) -> {ok, binary()} \| {error, term()}.
53	render(Expression, Bindings) ->
54	render(Expression, Bindings, #{}).	23✔
55
56	render(Expression, Bindings, Opts) when is_binary(Expression) ->
57	render(unicode:characters_to_list(Expression), Bindings, Opts);	1✔
58	render(Expression, Bindings, Opts) ->
59	case emqx_variform_scan:string(Expression) of	23✔
60	{ok, Tokens, _Line} ->
61	case emqx_variform_parser:parse(Tokens) of	23✔
62	{ok, Expr} ->
63	eval_as_string(Expr, Bindings, Opts);	19✔
64	{error, {_, emqx_variform_parser, Msg}} ->
65	%% syntax error
66	{error, lists:flatten(Msg)};	4✔
67	{error, Reason} ->
NEW 68	{error, Reason}	×
69	end;
70	{error, Reason, _Line} ->
NEW 71	{error, Reason}	×
72	end.
73
74	eval_as_string(Expr, Bindings, _Opts) ->
75	try	19✔
76	{ok, str(eval(Expr, Bindings))}	19✔
77	catch
78	throw:Reason ->
79	{error, Reason};	7✔
80	C:E:S ->
NEW 81	{error, #{exception => C, reason => E, stack_trace => S}}	×
82	end.
83
84	eval({str, Str}, _Bindings) ->
85	str(Str);	14✔
86	eval({num, Num}, _Bindings) ->
87	str(Num);	2✔
88	eval({array, Args}, Bindings) ->
89	eval(Args, Bindings);	1✔
90	eval({call, FuncNameStr, Args}, Bindings) ->
91	{Mod, Fun} = resolve_func_name(FuncNameStr),	20✔
92	ok = assert_func_exported(Mod, Fun, length(Args)),	15✔
93	call(Mod, Fun, eval(Args, Bindings));	14✔
94	eval({var, VarName}, Bindings) ->
95	resolve_var_value(VarName, Bindings);	11✔
96	eval([Arg \| Args], Bindings) ->
97	[eval(Arg, Bindings) \| eval(Args, Bindings)];	29✔
98	eval([], _Bindings) ->
99	[].	15✔
100
101	%% Some functions accept arbitrary number of arguments but implemented as /1.
102	call(emqx_variform_str, concat, Args) ->
103	str(emqx_variform_str:concat(Args));	2✔
104	call(emqx_variform_str, coalesce, Args) ->
105	str(emqx_variform_str:coalesce(Args));	5✔
106	call(Mod, Fun, Args) ->
107	erlang:apply(Mod, Fun, Args).	7✔
108
109	resolve_func_name(FuncNameStr) ->
110	case string:tokens(FuncNameStr, ".") of	20✔
111	[Mod0, Fun0] ->
112	Mod =	5✔
113	try
114	list_to_existing_atom(Mod0)	5✔
115	catch
116	error:badarg ->
117	throw(#{	1✔
118	reason => unknown_variform_module,
119	module => Mod0
120	})
121	end,
122	ok = assert_module_allowed(Mod),	4✔
123	Fun =	3✔
124	try
125	list_to_existing_atom(Fun0)	3✔
126	catch
127	error:badarg ->
128	throw(#{	1✔
129	reason => unknown_variform_function,
130	function => Fun0
131	})
132	end,
133	{Mod, Fun};	2✔
134	[Fun] ->
135	FuncName =	14✔
136	try
137	list_to_existing_atom(Fun)	14✔
138	catch
139	error:badarg ->
140	throw(#{	1✔
141	reason => unknown_variform_function,
142	function => Fun
143	})
144	end,
145	{emqx_variform_str, FuncName};	13✔
146	_ ->
147	throw(#{reason => invalid_function_reference, function => FuncNameStr})	1✔
148	end.
149
150	resolve_var_value(VarName, Bindings) ->
151	case emqx_template:lookup_var(split(VarName), Bindings) of	11✔
152	{ok, Value} ->
153	str(Value);	7✔
154	{error, _Reason} ->
155	<<>>	4✔
156	end.
157
158	assert_func_exported(emqx_variform_str, concat, _Arity) ->
159	ok;	2✔
160	assert_func_exported(emqx_variform_str, coalesce, _Arity) ->
161	ok;	5✔
162	assert_func_exported(Mod, Fun, Arity) ->
163	ok = try_load(Mod),	8✔
164	case erlang:function_exported(Mod, Fun, Arity) of	8✔
165	true ->
166	ok;	7✔
167	false ->
168	throw(#{	1✔
169	reason => unknown_variform_function,
170	module => Mod,
171	function => Fun,
172	arity => Arity
173	})
174	end.
175
176	%% best effort to load the module because it might not be loaded as a part of the release modules
177	%% e.g. from a plugin.
178	%% do not call code server, just try to call a function in the module.
179	try_load(Mod) ->
180	try	8✔
181	_ = erlang:apply(Mod, module_info, [md5]),	8✔
182	ok	8✔
183	catch
184	_:_ ->
NEW 185	ok	×
186	end.
187
188	assert_module_allowed(emqx_variform_str) ->
189	ok;	1✔
190	assert_module_allowed(Mod) ->
191	Allowed = get_allowed_modules(),	3✔
192	case lists:member(Mod, Allowed) of	3✔
193	true ->
194	ok;	2✔
195	false ->
196	throw(#{	1✔
197	reason => unallowed_veriform_module,
198	module => Mod
199	})
200	end.
201
202	inject_allowed_module(Module) when is_atom(Module) ->
203	inject_allowed_modules([Module]).	1✔
204
205	inject_allowed_modules(Modules) when is_list(Modules) ->
206	Allowed0 = get_allowed_modules(),	1✔
207	Allowed = lists:usort(Allowed0 ++ Modules),	1✔
208	persistent_term:put({emqx_variform, allowed_modules}, Allowed).	1✔
209
210	erase_allowed_module(Module) when is_atom(Module) ->
211	erase_allowed_modules([Module]).	1✔
212
213	erase_allowed_modules(Modules) when is_list(Modules) ->
214	Allowed0 = get_allowed_modules(),	1✔
215	Allowed = Allowed0 -- Modules,	1✔
216	persistent_term:put({emqx_variform, allowed_modules}, Allowed).	1✔
217
218	get_allowed_modules() ->
219	persistent_term:get({emqx_variform, allowed_modules}, []).	5✔
220
221	str(Value) ->
222	emqx_utils_conv:bin(Value).	42✔
223
224	split(VarName) ->
225	lists:map(fun erlang:iolist_to_binary/1, string:tokens(VarName, ".")).	11✔

emqx / emqx / 8628139215

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous