714256730

Build Type

gitlab-ci

Committed by Fabio Tranchitella

Commit Message

refac: optimize the mapping cache taking into account the max size

Pull Request Pull Request #79: MEN-5598: map inventory attributes to sequential fields

Run Details

343 of 410 new or added lines in 12 files covered. (83.66%)

4 existing lines in 2 files now uncovered.

1689 of 2125 relevant lines covered (79.48%)

12.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

45.85

/app/reporting/reporting.go

// Copyright 2022 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package reporting

import (
        "context"
        "errors"
        "sort"
        "time"

        "github.com/mendersoftware/go-lib-micro/log"

        "github.com/mendersoftware/reporting/client/inventory"
        "github.com/mendersoftware/reporting/mapping"
        "github.com/mendersoftware/reporting/model"
        "github.com/mendersoftware/reporting/store"
)

//go:generate ../../x/mockgen.sh
type App interface {
        HealthCheck(ctx context.Context) error
        GetSearchableInvAttrs(ctx context.Context, tid string) ([]model.FilterAttribute, error)
        InventorySearchDevices(ctx context.Context, searchParams *model.SearchParams) (
                []inventory.Device, int, error)
}

type app struct {
        store  store.Store
        mapper mapping.Mapper
        ds     store.DataStore
}

func NewApp(store store.Store, ds store.DataStore) App {
        mapper := mapping.NewMapper(ds)
        return &app{
                store:  store,
                mapper: mapper,
                ds:     ds,
        }
}

// HealthCheck performs a health check and returns an error if it fails
func (a *app) HealthCheck(ctx context.Context) error {
        err := a.ds.Ping(ctx)
        if err == nil {
                err = a.store.Ping(ctx)
        }
        return err
}

func (app *app) InventorySearchDevices(
        ctx context.Context,
        searchParams *model.SearchParams,
) ([]inventory.Device, int, error) {
        if err := app.mapSearchParamsAttributes(ctx, searchParams); err != nil {
                return nil, 0, err
        }
        query, err := model.BuildQuery(*searchParams)
        if err != nil {
                return nil, 0, err
        }

        if searchParams.TenantID != "" {
                query = query.Must(model.M{
                        "term": model.M{
                                model.FieldNameTenantID: searchParams.TenantID,
                        },
                })
        }

        if len(searchParams.DeviceIDs) > 0 {
                query = query.Must(model.M{
                        "terms": model.M{
                                model.FieldNameID: searchParams.DeviceIDs,
                        },
                })
        }

        esRes, err := app.store.Search(ctx, query)
        if err != nil {
                return nil, 0, err
        }

        res, total, err := app.storeToInventoryDevs(ctx, searchParams.TenantID, esRes)
        if err != nil {
                return nil, 0, err
        }

        return res, total, err
}

func (app *app) mapSearchParamsAttributes(ctx context.Context,
        searchParams *model.SearchParams) error {
        if len(searchParams.Attributes) > 0 {
                attributes := make(inventory.DeviceAttributes, 0, len(searchParams.Attributes))
                for i := 0; i < len(searchParams.Attributes); i++ {
                        attributes = append(attributes, inventory.DeviceAttribute{
                                Name:  searchParams.Attributes[i].Attribute,
                                Scope: searchParams.Attributes[i].Scope,
                        })
                }
                attributes, err := app.mapper.MapInventoryAttributes(ctx, searchParams.TenantID,
                        attributes, false)
                if err != nil {
                        return err
                }
                searchParams.Attributes = make([]model.SelectAttribute, 0, len(searchParams.Attributes))
                for _, attribute := range attributes {
                        searchParams.Attributes = append(searchParams.Attributes, model.SelectAttribute{
                                Attribute: attribute.Name,
                                Scope:     attribute.Scope,
                        })
                }
        }
        return nil
}

// storeToInventoryDevs translates ES results directly to iventory devices
func (a *app) storeToInventoryDevs(
        ctx context.Context, tenantID string, storeRes map[string]interface{},
) ([]inventory.Device, int, error) {
        devs := []inventory.Device{}

        hitsM, ok := storeRes["hits"].(map[string]interface{})
        if !ok {
                return nil, 0, errors.New("can't process store hits map")
        }

        hitsTotalM, ok := hitsM["total"].(map[string]interface{})
        if !ok {
                return nil, 0, errors.New("can't process total hits struct")
        }

        total, ok := hitsTotalM["value"].(float64)
        if !ok {
                return nil, 0, errors.New("can't process total hits value")
        }

        hitsS, ok := hitsM["hits"].([]interface{})
        if !ok {
                return nil, 0, errors.New("can't process store hits slice")
        }

        for _, v := range hitsS {
                res, err := a.storeToInventoryDev(ctx, tenantID, v)
                if err != nil {
                        return nil, 0, err
                }

                devs = append(devs, *res)
        }

        return devs, int(total), nil
}

func (a *app) storeToInventoryDev(ctx context.Context, tenantID string,
        storeRes interface{}) (*inventory.Device, error) {
        resM, ok := storeRes.(map[string]interface{})
        if !ok {
                return nil, errors.New("can't process individual hit")
        }

        // if query has a 'fields' clause, use 'fields' instead of '_source'
        sourceM, ok := resM["_source"].(map[string]interface{})
        if !ok {
                sourceM, ok = resM["fields"].(map[string]interface{})
                if !ok {
                        return nil, errors.New("can't process hit's '_source' nor 'fields'")
                }
        }

        // if query has a 'fields' clause, all results will be arrays incl. device id, so extract it
        id, ok := sourceM["id"].(string)
        if !ok {
                idarr, ok := sourceM["id"].([]interface{})
                if !ok {
                        return nil, errors.New(
                                "can't parse device id as neither single value nor array",
                        )
                }

                id, ok = idarr[0].(string)
                if !ok {
                        return nil, errors.New(
                                "can't parse device id as neither single value nor array",
                        )
                }
        }

        ret := &inventory.Device{
                ID: inventory.DeviceID(id),
        }

        attrs := []inventory.DeviceAttribute{}

        for k, v := range sourceM {
                s, n, err := model.MaybeParseAttr(k)
                if err != nil {
                        return nil, err
                }

                if vArray, ok := v.([]interface{}); ok && len(vArray) == 1 {
                        v = vArray[0]
                }

                if n != "" {
                        a := inventory.DeviceAttribute{
                                Name:  model.Redot(n),
                                Scope: s,
                                Value: v,
                        }

                        if a.Scope == model.ScopeSystem &&
                                a.Name == model.AttrNameUpdatedAt {
                                ret.UpdatedTs = parseTime(v)
                        } else if a.Scope == model.ScopeSystem &&
                                a.Name == model.AttrNameCreatedAt {
                                ret.CreatedTs = parseTime(v)
                        }

                        attrs = append(attrs, a)
                }
        }

        attributes, err := a.mapper.ReverseInventoryAttributes(ctx, tenantID, attrs)
        if err != nil {
                return nil, err
        }
        ret.Attributes = attributes

        return ret, nil
}

func parseTime(v interface{}) time.Time {
        val, _ := v.(string)
        if t, err := time.Parse(time.RFC3339, val); err == nil {
                return t
        }
        return time.Time{}
}

func (app *app) GetSearchableInvAttrs(
        ctx context.Context,
        tid string,
) ([]model.FilterAttribute, error) {
        l := log.FromContext(ctx)

        index, err := app.store.GetDevicesIndexMapping(ctx, tid)
        if err != nil {
                return nil, err
        }

        // inventory attributes are under 'mappings.properties'
        mappings, ok := index["mappings"]
        if !ok {
                return nil, errors.New("can't parse index mappings")
        }

        mappingsM, ok := mappings.(map[string]interface{})
        if !ok {
                return nil, errors.New("can't parse index mappings")
        }

        props, ok := mappingsM["properties"]
        if !ok {
                return nil, errors.New("can't parse index properties")
        }

        propsM, ok := props.(map[string]interface{})
        if !ok {
                return nil, errors.New("can't parse index properties")
        }

        ret := []model.FilterAttribute{}

        for k := range propsM {
                s, n, err := model.MaybeParseAttr(k)

                if err != nil {
                        return nil, err
                }

                if n != "" {
                        ret = append(ret, model.FilterAttribute{Name: n, Scope: s, Count: 1})
                }
        }

        sort.Slice(ret, func(i, j int) bool {
                if ret[j].Scope > ret[i].Scope {
                        return true
                }

                if ret[j].Scope < ret[i].Scope {
                        return false
                }

                return ret[j].Name > ret[i].Name
        })

        l.Debugf("parsed searchable attributes %v\n", ret)

        return ret, nil
}

1	// Copyright 2022 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package reporting
16
17	import (
18	"context"
19	"errors"
20	"sort"
21	"time"
22
23	"github.com/mendersoftware/go-lib-micro/log"
24
25	"github.com/mendersoftware/reporting/client/inventory"
26	"github.com/mendersoftware/reporting/mapping"
27	"github.com/mendersoftware/reporting/model"
28	"github.com/mendersoftware/reporting/store"
29	)
30
31	//go:generate ../../x/mockgen.sh
32	type App interface {
33	HealthCheck(ctx context.Context) error
34	GetSearchableInvAttrs(ctx context.Context, tid string) ([]model.FilterAttribute, error)
35	InventorySearchDevices(ctx context.Context, searchParams *model.SearchParams) (
36	[]inventory.Device, int, error)
37	}
38
39	type app struct {
40	store store.Store
41	mapper mapping.Mapper
42	ds store.DataStore
43	}
44
45	func NewApp(store store.Store, ds store.DataStore) App {	11✔
46	mapper := mapping.NewMapper(ds)	11✔
47	return &app{	11✔
48	store: store,	11✔
49	mapper: mapper,	11✔
50	ds: ds,	11✔
51	}	11✔
52	}	11✔
53
54	// HealthCheck performs a health check and returns an error if it fails
NEW 55	func (a *app) HealthCheck(ctx context.Context) error {	×
NEW 56	err := a.ds.Ping(ctx)	×
NEW 57	if err == nil {	×
NEW 58	err = a.store.Ping(ctx)	×
NEW 59	}	×
NEW 60	return err	×
61	}
62
63	func (app *app) InventorySearchDevices(
64	ctx context.Context,
65	searchParams *model.SearchParams,
66	) ([]inventory.Device, int, error) {	25✔
67	if err := app.mapSearchParamsAttributes(ctx, searchParams); err != nil {	25✔
NEW 68	return nil, 0, err	×
NEW 69	}	×
70	query, err := model.BuildQuery(*searchParams)	25✔
71	if err != nil {	27✔
72	return nil, 0, err	2✔
73	}	2✔
74
75	if searchParams.TenantID != "" {	38✔
76	query = query.Must(model.M{	15✔
77	"term": model.M{	15✔
78	model.FieldNameTenantID: searchParams.TenantID,	15✔
79	},	15✔
80	})	15✔
81	}	15✔
82
83	if len(searchParams.DeviceIDs) > 0 {	25✔
84	query = query.Must(model.M{	2✔
85	"terms": model.M{	2✔
86	model.FieldNameID: searchParams.DeviceIDs,	2✔
87	},	2✔
88	})	2✔
89	}	2✔
90
91	esRes, err := app.store.Search(ctx, query)	23✔
92	if err != nil {	25✔
93	return nil, 0, err	2✔
94	}	2✔
95
96	res, total, err := app.storeToInventoryDevs(ctx, searchParams.TenantID, esRes)	21✔
97	if err != nil {	23✔
98	return nil, 0, err	2✔
99	}	2✔
100
101	return res, total, err	19✔
102	}
103
104	func (app *app) mapSearchParamsAttributes(ctx context.Context,
105	searchParams *model.SearchParams) error {	25✔
106	if len(searchParams.Attributes) > 0 {	25✔
NEW 107	attributes := make(inventory.DeviceAttributes, 0, len(searchParams.Attributes))	×
NEW 108	for i := 0; i < len(searchParams.Attributes); i++ {	×
NEW 109	attributes = append(attributes, inventory.DeviceAttribute{	×
NEW 110	Name: searchParams.Attributes[i].Attribute,	×
NEW 111	Scope: searchParams.Attributes[i].Scope,	×
NEW 112	})	×
NEW 113	}	×
NEW 114	attributes, err := app.mapper.MapInventoryAttributes(ctx, searchParams.TenantID,	×
NEW 115	attributes, false)	×
NEW 116	if err != nil {	×
NEW 117	return err	×
NEW 118	}	×
NEW 119	searchParams.Attributes = make([]model.SelectAttribute, 0, len(searchParams.Attributes))	×
NEW 120	for _, attribute := range attributes {	×
NEW 121	searchParams.Attributes = append(searchParams.Attributes, model.SelectAttribute{	×
NEW 122	Attribute: attribute.Name,	×
NEW 123	Scope: attribute.Scope,	×
NEW 124	})	×
NEW 125	}	×
126	}
127	return nil	25✔
128	}
129
130	// storeToInventoryDevs translates ES results directly to iventory devices
131	func (a *app) storeToInventoryDevs(
132	ctx context.Context, tenantID string, storeRes map[string]interface{},
133	) ([]inventory.Device, int, error) {	21✔
134	devs := []inventory.Device{}	21✔
135		21✔
136	hitsM, ok := storeRes["hits"].(map[string]interface{})	21✔
137	if !ok {	21✔
138	return nil, 0, errors.New("can't process store hits map")	×
139	}	×
140
141	hitsTotalM, ok := hitsM["total"].(map[string]interface{})	21✔
142	if !ok {	21✔
143	return nil, 0, errors.New("can't process total hits struct")	×
144	}	×
145
146	total, ok := hitsTotalM["value"].(float64)	21✔
147	if !ok {	23✔
148	return nil, 0, errors.New("can't process total hits value")	2✔
149	}	2✔
150
151	hitsS, ok := hitsM["hits"].([]interface{})	19✔
152	if !ok {	19✔
153	return nil, 0, errors.New("can't process store hits slice")	×
154	}	×
155
156	for _, v := range hitsS {	48✔
157	res, err := a.storeToInventoryDev(ctx, tenantID, v)	29✔
158	if err != nil {	29✔
159	return nil, 0, err	×
160	}	×
161
162	devs = append(devs, *res)	29✔
163	}
164
165	return devs, int(total), nil	19✔
166	}
167
168	func (a *app) storeToInventoryDev(ctx context.Context, tenantID string,
169	storeRes interface{}) (*inventory.Device, error) {	29✔
170	resM, ok := storeRes.(map[string]interface{})	29✔
171	if !ok {	29✔
172	return nil, errors.New("can't process individual hit")	×
173	}	×
174
175	// if query has a 'fields' clause, use 'fields' instead of '_source'
176	sourceM, ok := resM["_source"].(map[string]interface{})	29✔
177	if !ok {	29✔
178	sourceM, ok = resM["fields"].(map[string]interface{})	×
179	if !ok {	×
180	return nil, errors.New("can't process hit's '_source' nor 'fields'")	×
181	}	×
182	}
183
184	// if query has a 'fields' clause, all results will be arrays incl. device id, so extract it
185	id, ok := sourceM["id"].(string)	29✔
186	if !ok {	29✔
187	idarr, ok := sourceM["id"].([]interface{})	×
188	if !ok {	×
189	return nil, errors.New(	×
190	"can't parse device id as neither single value nor array",	×
191	)	×
192	}	×
193
194	id, ok = idarr[0].(string)	×
195	if !ok {	×
196	return nil, errors.New(	×
197	"can't parse device id as neither single value nor array",	×
198	)	×
199	}	×
200	}
201
202	ret := &inventory.Device{	29✔
203	ID: inventory.DeviceID(id),	29✔
204	}	29✔
205		29✔
206	attrs := []inventory.DeviceAttribute{}	29✔
207		29✔
208	for k, v := range sourceM {	245✔
209	s, n, err := model.MaybeParseAttr(k)	216✔
210	if err != nil {	216✔
211	return nil, err	×
212	}	×
213
214	if vArray, ok := v.([]interface{}); ok && len(vArray) == 1 {	264✔
215	v = vArray[0]	48✔
216	}	48✔
217
218	if n != "" {	266✔
219	a := inventory.DeviceAttribute{	50✔
220	Name: model.Redot(n),	50✔
221	Scope: s,	50✔
222	Value: v,	50✔
223	}	50✔
224		50✔
225	if a.Scope == model.ScopeSystem &&	50✔
226	a.Name == model.AttrNameUpdatedAt {	50✔
227	ret.UpdatedTs = parseTime(v)	×
228	} else if a.Scope == model.ScopeSystem &&	50✔
229	a.Name == model.AttrNameCreatedAt {	50✔
230	ret.CreatedTs = parseTime(v)	×
231	}	×
232
233	attrs = append(attrs, a)	50✔
234	}
235	}
236
237	attributes, err := a.mapper.ReverseInventoryAttributes(ctx, tenantID, attrs)	29✔
238	if err != nil {	29✔
NEW 239	return nil, err	×
NEW 240	}	×
241	ret.Attributes = attributes	29✔
242		29✔
243	return ret, nil	29✔
244	}
245
246	func parseTime(v interface{}) time.Time {	×
247	val, _ := v.(string)	×
248	if t, err := time.Parse(time.RFC3339, val); err == nil {	×
249	return t	×
250	}	×
251	return time.Time{}	×
252	}
253
254	func (app *app) GetSearchableInvAttrs(
255	ctx context.Context,
256	tid string,
257	) ([]model.FilterAttribute, error) {	×
258	l := log.FromContext(ctx)	×
259		×
260	index, err := app.store.GetDevicesIndexMapping(ctx, tid)	×
261	if err != nil {	×
262	return nil, err	×
263	}	×
264
265	// inventory attributes are under 'mappings.properties'
266	mappings, ok := index["mappings"]	×
267	if !ok {	×
268	return nil, errors.New("can't parse index mappings")	×
269	}	×
270
271	mappingsM, ok := mappings.(map[string]interface{})	×
272	if !ok {	×
273	return nil, errors.New("can't parse index mappings")	×
274	}	×
275
276	props, ok := mappingsM["properties"]	×
277	if !ok {	×
278	return nil, errors.New("can't parse index properties")	×
279	}	×
280
281	propsM, ok := props.(map[string]interface{})	×
282	if !ok {	×
283	return nil, errors.New("can't parse index properties")	×
284	}	×
285
286	ret := []model.FilterAttribute{}	×
287		×
288	for k := range propsM {	×
289	s, n, err := model.MaybeParseAttr(k)	×
290		×
291	if err != nil {	×
292	return nil, err	×
293	}	×
294
295	if n != "" {	×
296	ret = append(ret, model.FilterAttribute{Name: n, Scope: s, Count: 1})	×
297	}	×
298	}
299
300	sort.Slice(ret, func(i, j int) bool {	×
301	if ret[j].Scope > ret[i].Scope {	×
302	return true	×
303	}	×
304
305	if ret[j].Scope < ret[i].Scope {	×
306	return false	×
307	}	×
308
309	return ret[j].Name > ret[i].Name	×
310	})
311
312	l.Debugf("parsed searchable attributes %v\n", ret)	×
313		×
314	return ret, nil	×
315	}

mendersoftware / reporting / 714256730

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous