1648363360

Committed 30 Jan 2025 11:32AM UTC coverage: 76.628% (+0.03%) from 76.597%

Build # 1648363360

Build Type

Pull #394

gitlab-ci

Committed by

bahaa-ghazal

Commit Message

fix: Limiting the size of metadata when uploading and generate artifacts

Changelog: Title
Ticket: MED-7166
Signed-off-by: Bahaa Aldeen Ghazal <bahaa.ghazal@northern.tech>

Pull Request Pull Request #394: fix: Limiting the size of metadata when uploading and generate artifacts

Run Details

4328 of 6288 branches covered (68.83%)

Branch coverage included in aggregate %.

24 of 30 new or added lines in 1 file covered. (80.0%)

4 existing lines in 2 files now uncovered.

45444 of 58665 relevant lines covered (77.46%)

20.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.39

/backend/services/reporting/app/reporting/reporting.go

// Copyright 2023 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package reporting

import (
        "context"
        "errors"
        "sort"
        "time"

        "github.com/mendersoftware/mender-server/pkg/log"

        "github.com/mendersoftware/mender-server/services/reporting/client/inventory"
        "github.com/mendersoftware/mender-server/services/reporting/mapping"
        "github.com/mendersoftware/mender-server/services/reporting/model"
        "github.com/mendersoftware/mender-server/services/reporting/store"
)

//go:generate ../../../../utils/mockgen.sh
type App interface {
        HealthCheck(ctx context.Context) error
        GetMapping(ctx context.Context, tid string) (*model.Mapping, error)
        GetSearchableInvAttrs(ctx context.Context, tid string) ([]model.FilterAttribute, error)
        AggregateDevices(ctx context.Context, aggregateParams *model.AggregateParams) (
                []model.DeviceAggregation, error)
        SearchDevices(ctx context.Context, searchParams *model.SearchParams) (
                []inventory.Device, int, error)
        AggregateDeployments(ctx context.Context, aggregateParams *model.AggregateDeploymentsParams) (
                []model.DeviceAggregation, error)
        SearchDeployments(ctx context.Context, searchParams *model.DeploymentsSearchParams) (
                []model.Deployment, int, error)
}

type app struct {
        store  store.Store
        mapper mapping.Mapper
        ds     store.DataStore
}

func NewApp(store store.Store, ds store.DataStore) App {
        mapper := mapping.NewMapper(ds)
        return &app{
                store:  store,
                mapper: mapper,
                ds:     ds,
        }
}

// HealthCheck performs a health check and returns an error if it fails
func (a *app) HealthCheck(ctx context.Context) error {
        err := a.ds.Ping(ctx)
        if err == nil {
                err = a.store.Ping(ctx)
        }
        return err
}

// GetMapping returns the mapping for the specified tenant
func (app *app) GetMapping(ctx context.Context, tid string) (*model.Mapping, error) {
        return app.ds.GetMapping(ctx, tid)
}

// AggregateDevices aggregates device data
func (app *app) AggregateDevices(
        ctx context.Context,
        aggregateParams *model.AggregateParams,
) ([]model.DeviceAggregation, error) {
        searchParams := &model.SearchParams{
                Filters:  aggregateParams.Filters,
                Groups:   aggregateParams.Groups,
                TenantID: aggregateParams.TenantID,
        }
        if err := app.mapSearchParams(ctx, searchParams); err != nil {
                return nil, err
        }
        query, err := model.BuildQuery(*searchParams)
        if err != nil {
                return nil, err
        }
        if searchParams.TenantID != "" {
                query = query.Must(model.M{
                        "term": model.M{
                                model.FieldNameTenantID: searchParams.TenantID,
                        },
                })
        }

        if err := app.mapAggregations(ctx, searchParams.TenantID,
                aggregateParams.Aggregations); err != nil {
                return nil, err
        }
        aggregations, err := model.BuildAggregations(aggregateParams.Aggregations)
        if err != nil {
                return nil, err
        }

        query = query.WithSize(0).With(map[string]interface{}{
                "aggs": aggregations,
        })
        esRes, err := app.store.AggregateDevices(ctx, query)
        if err != nil {
                return nil, err
        }

        aggregationsS, ok := esRes["aggregations"].(map[string]interface{})
        if !ok {
                return nil, errors.New("can't process store aggregations slice")
        }
        res, err := app.storeToDeviceAggregations(ctx, searchParams.TenantID, aggregationsS)
        if err != nil {
                return nil, err
        }

        return res, nil
}

// storeToDeviceAggregations translates ES results directly to device aggregations
func (a *app) storeToDeviceAggregations(
        ctx context.Context, tenantID string, aggregationsS map[string]interface{},
) ([]model.DeviceAggregation, error) {
        aggs := []model.DeviceAggregation{}
        for name, aggregationS := range aggregationsS {
                if _, ok := aggregationS.(map[string]interface{}); !ok {
                        continue
                }
                bucketsS, ok := aggregationS.(map[string]interface{})["buckets"].([]interface{})
                if !ok {
                        continue
                }
                items := make([]model.DeviceAggregationItem, 0, len(bucketsS))
                for _, bucket := range bucketsS {
                        bucketMap, ok := bucket.(map[string]interface{})
                        if !ok {
                                return nil, errors.New("can't process store bucket item")
                        }
                        key, ok := bucketMap["key"].(string)
                        if !ok {
                                return nil, errors.New("can't process store key attribute")
                        }
                        count, ok := bucketMap["doc_count"].(float64)
                        if !ok {
                                return nil, errors.New("can't process store doc_count attribute")
                        }
                        item := model.DeviceAggregationItem{
                                Key:   key,
                                Count: int(count),
                        }
                        subaggs, err := a.storeToDeviceAggregations(ctx, tenantID, bucketMap)
                        if err == nil && len(subaggs) > 0 {
                                item.Aggregations = subaggs
                        }
                        items = append(items, item)
                }

                otherCount := 0
                if count, ok := aggregationS.(map[string]interface{})["sum_other_doc_count"].(float64); ok {
                        otherCount = int(count)
                }

                aggs = append(aggs, model.DeviceAggregation{
                        Name:       name,
                        Items:      items,
                        OtherCount: otherCount,
                })
        }
        return aggs, nil
}

// SearchDevices searches device data
func (app *app) SearchDevices(
        ctx context.Context,
        searchParams *model.SearchParams,
) ([]inventory.Device, int, error) {
        if err := app.mapSearchParams(ctx, searchParams); err != nil {
                return nil, 0, err
        }
        query, err := model.BuildQuery(*searchParams)
        if err != nil {
                return nil, 0, err
        }

        if searchParams.TenantID != "" {
                query = query.Must(model.M{
                        "term": model.M{
                                model.FieldNameTenantID: searchParams.TenantID,
                        },
                })
        }

        if len(searchParams.DeviceIDs) > 0 {
                query = query.Must(model.M{
                        "terms": model.M{
                                model.FieldNameID: searchParams.DeviceIDs,
                        },
                })
        }

        esRes, err := app.store.SearchDevices(ctx, query)
        if err != nil {
                return nil, 0, err
        }

        res, total, err := app.storeToInventoryDevs(ctx, searchParams.TenantID, esRes)
        if err != nil {
                return nil, 0, err
        }

        return res, total, err
}

func (app *app) mapAggregations(ctx context.Context, tenantID string,
        aggregations []model.AggregationTerm) error {
        attributes := make(inventory.DeviceAttributes, 0, len(aggregations))
        for i := range aggregations {
                attributes = append(attributes, inventory.DeviceAttribute{
                        Name:  aggregations[i].Attribute,
                        Scope: aggregations[i].Scope,
                })
        }
        attributes, err := app.mapper.MapInventoryAttributes(ctx, tenantID,
                attributes, false, true)
        if err == nil {
                for i, attr := range attributes {
                        aggregations[i].Attribute = attr.Name
                        aggregations[i].Scope = attr.Scope
                        if len(aggregations[i].Aggregations) > 0 {
                                err = app.mapAggregations(ctx, tenantID, aggregations[i].Aggregations)
                                if err != nil {
                                        break
                                }
                        }
                }
        }
        return err
}

func (app *app) mapSearchParams(ctx context.Context, searchParams *model.SearchParams) error {
        if len(searchParams.Filters) > 0 {
                attributes := make(inventory.DeviceAttributes, 0, len(searchParams.Attributes))
                for i := 0; i < len(searchParams.Filters); i++ {
                        attributes = append(attributes, inventory.DeviceAttribute{
                                Name:        searchParams.Filters[i].Attribute,
                                Scope:       searchParams.Filters[i].Scope,
                                Value:       searchParams.Filters[i].Value,
                                Description: &searchParams.Filters[i].Type,
                        })
                }
                attributes, err := app.mapper.MapInventoryAttributes(ctx, searchParams.TenantID,
                        attributes, false, true)
                if err != nil {
                        return err
                }
                searchParams.Filters = make([]model.FilterPredicate, 0, len(searchParams.Filters))
                for _, attribute := range attributes {
                        searchParams.Filters = append(searchParams.Filters, model.FilterPredicate{
                                Attribute: attribute.Name,
                                Scope:     attribute.Scope,
                                Value:     attribute.Value,
                                Type:      *attribute.Description,
                        })
                }
        }
        if len(searchParams.Attributes) > 0 {
                attributes := make(inventory.DeviceAttributes, 0, len(searchParams.Attributes))
                for i := 0; i < len(searchParams.Attributes); i++ {
                        attributes = append(attributes, inventory.DeviceAttribute{
                                Name:  searchParams.Attributes[i].Attribute,
                                Scope: searchParams.Attributes[i].Scope,
                        })
                }
                attributes, err := app.mapper.MapInventoryAttributes(ctx, searchParams.TenantID,
                        attributes, false, false)
                if err != nil {
                        return err
                }
                searchParams.Attributes = make([]model.SelectAttribute, 0, len(searchParams.Attributes))
                for _, attribute := range attributes {
                        searchParams.Attributes = append(searchParams.Attributes, model.SelectAttribute{
                                Attribute: attribute.Name,
                                Scope:     attribute.Scope,
                        })
                }
        }
        if len(searchParams.Sort) > 0 {
                attributes := make(inventory.DeviceAttributes, 0, len(searchParams.Sort))
                for i := 0; i < len(searchParams.Sort); i++ {
                        attributes = append(attributes, inventory.DeviceAttribute{
                                Name:        searchParams.Sort[i].Attribute,
                                Scope:       searchParams.Sort[i].Scope,
                                Description: &searchParams.Sort[i].Order,
                        })
                }
                attributes, err := app.mapper.MapInventoryAttributes(ctx, searchParams.TenantID,
                        attributes, false, false)
                if err != nil {
                        return err
                }
                searchParams.Sort = make([]model.SortCriteria, 0, len(searchParams.Attributes))
                for _, attribute := range attributes {
                        searchParams.Sort = append(searchParams.Sort, model.SortCriteria{
                                Attribute: attribute.Name,
                                Scope:     attribute.Scope,
                                Order:     *attribute.Description,
                        })
                }
        }

        return nil
}

// storeToInventoryDevs translates ES results directly to inventory devices
func (a *app) storeToInventoryDevs(
        ctx context.Context, tenantID string, storeRes map[string]interface{},
) ([]inventory.Device, int, error) {
        devs := []inventory.Device{}

        hitsM, ok := storeRes["hits"].(map[string]interface{})
        if !ok {
                return nil, 0, errors.New("can't process store hits map")
        }

        hitsTotalM, ok := hitsM["total"].(map[string]interface{})
        if !ok {
                return nil, 0, errors.New("can't process total hits struct")
        }

        total, ok := hitsTotalM["value"].(float64)
        if !ok {
                return nil, 0, errors.New("can't process total hits value")
        }

        hitsS, ok := hitsM["hits"].([]interface{})
        if !ok {
                return nil, 0, errors.New("can't process store hits slice")
        }

        for _, v := range hitsS {
                res, err := a.storeToInventoryDev(ctx, tenantID, v)
                if err != nil {
                        return nil, 0, err
                }

                devs = append(devs, *res)
        }

        return devs, int(total), nil
}

func (a *app) storeToInventoryDev(ctx context.Context, tenantID string,
        storeRes interface{}) (*inventory.Device, error) {
        resM, ok := storeRes.(map[string]interface{})
        if !ok {
                return nil, errors.New("can't process individual hit")
        }

        // if query has a 'fields' clause, use 'fields' instead of '_source'
        sourceM, ok := resM["_source"].(map[string]interface{})
        if !ok {
                sourceM, ok = resM["fields"].(map[string]interface{})
                if !ok {
                        return nil, errors.New("can't process hit's '_source' nor 'fields'")
                }
        }

        // if query has a 'fields' clause, all results will be arrays incl. device id, so extract it
        id, ok := sourceM["id"].(string)
        if !ok {
                idarr, ok := sourceM["id"].([]interface{})
                if !ok {
                        return nil, errors.New(
                                "can't parse device id as neither single value nor array",
                        )
                }

                id, ok = idarr[0].(string)
                if !ok {
                        return nil, errors.New(
                                "can't parse device id as neither single value nor array",
                        )
                }
        }

        ret := &inventory.Device{
                ID: inventory.DeviceID(id),
        }
        t := getTime(sourceM, model.FieldNameCheckIn)
        if t != nil && !t.IsZero() {
                ret.LastCheckinDate = t
        }
        attrs := []inventory.DeviceAttribute{}

        for k, v := range sourceM {
                s, n, err := model.MaybeParseAttr(k)
                if err != nil {
                        return nil, err
                }

                if vArray, ok := v.([]interface{}); ok && len(vArray) == 1 {
                        v = vArray[0]
                }

                if n != "" {
                        a := inventory.DeviceAttribute{
                                Name:  model.Redot(n),
                                Scope: s,
                                Value: v,
                        }

                        if a.Scope == model.ScopeSystem &&
                                a.Name == model.AttrNameUpdatedAt {
                                ret.UpdatedTs = parseTime(v)
                        } else if a.Scope == model.ScopeSystem &&
                                a.Name == model.AttrNameCreatedAt {
                                ret.CreatedTs = parseTime(v)
                        }

                        attrs = append(attrs, a)
                }
        }

        attributes, err := a.mapper.ReverseInventoryAttributes(ctx, tenantID, attrs)
        if err != nil {
                return nil, err
        }
        ret.Attributes = attributes

        return ret, nil
}

func getTime(m map[string]interface{}, s string) *time.Time {
        if v, ok := m[s]; ok && v != nil {
                timeString := ""
                if vString, ok := v.(string); ok && len(vString) > 0 {
                        timeString = v.(string)
                }
                if vArray, ok := v.([]interface{}); ok && len(vArray) > 0 {
                        timeString = v.([]interface{})[0].(string)
                }
                if len(timeString) > 0 {
                        t, e := time.Parse(time.RFC3339, timeString)
                        if e != nil {
                                return nil
                        }
                        if ok {
                                return &t
                        }
                }
        }
        return nil
}

func parseTime(v interface{}) time.Time {
        val, _ := v.(string)
        if t, err := time.Parse(time.RFC3339, val); err == nil {
                return t
        }
        return time.Time{}
}

func (app *app) GetSearchableInvAttrs(
        ctx context.Context,
        tid string,
) ([]model.FilterAttribute, error) {
        l := log.FromContext(ctx)

        index, err := app.store.GetDevicesIndexMapping(ctx, tid)
        if err != nil {
                return nil, err
        }

        // inventory attributes are under 'mappings.properties'
        mappings, ok := index["mappings"]
        if !ok {
                return nil, errors.New("can't parse index mappings")
        }

        mappingsM, ok := mappings.(map[string]interface{})
        if !ok {
                return nil, errors.New("can't parse index mappings")
        }

        props, ok := mappingsM["properties"]
        if !ok {
                return nil, errors.New("can't parse index properties")
        }

        propsM, ok := props.(map[string]interface{})
        if !ok {
                return nil, errors.New("can't parse index properties")
        }

        attrs := []inventory.DeviceAttribute{}
        for k := range propsM {
                s, n, err := model.MaybeParseAttr(k)

                if err != nil {
                        return nil, err
                }

                if n != "" {
                        attrs = append(attrs, inventory.DeviceAttribute{Name: n, Scope: s})
                }
        }
        attributes, err := app.mapper.ReverseInventoryAttributes(ctx, tid, attrs)
        if err != nil {
                return nil, err
        }

        ret := []model.FilterAttribute{}
        for _, attr := range attributes {
                ret = append(ret, model.FilterAttribute{Name: attr.Name, Scope: attr.Scope, Count: 1})
        }

        sort.Slice(ret, func(i, j int) bool {
                if ret[j].Scope > ret[i].Scope {
                        return true
                }

                if ret[j].Scope < ret[i].Scope {
                        return false
                }

                return ret[j].Name > ret[i].Name
        })

        l.Debugf("parsed searchable attributes %v\n", ret)

        return ret, nil
}

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package reporting
16
17	import (
18	"context"
19	"errors"
20	"sort"
21	"time"
22
23	"github.com/mendersoftware/mender-server/pkg/log"
24
25	"github.com/mendersoftware/mender-server/services/reporting/client/inventory"
26	"github.com/mendersoftware/mender-server/services/reporting/mapping"
27	"github.com/mendersoftware/mender-server/services/reporting/model"
28	"github.com/mendersoftware/mender-server/services/reporting/store"
29	)
30
31	//go:generate ../../../../utils/mockgen.sh
32	type App interface {
33	HealthCheck(ctx context.Context) error
34	GetMapping(ctx context.Context, tid string) (*model.Mapping, error)
35	GetSearchableInvAttrs(ctx context.Context, tid string) ([]model.FilterAttribute, error)
36	AggregateDevices(ctx context.Context, aggregateParams *model.AggregateParams) (
37	[]model.DeviceAggregation, error)
38	SearchDevices(ctx context.Context, searchParams *model.SearchParams) (
39	[]inventory.Device, int, error)
40	AggregateDeployments(ctx context.Context, aggregateParams *model.AggregateDeploymentsParams) (
41	[]model.DeviceAggregation, error)
42	SearchDeployments(ctx context.Context, searchParams *model.DeploymentsSearchParams) (
43	[]model.Deployment, int, error)
44	}
45
46	type app struct {
47	store store.Store
48	mapper mapping.Mapper
49	ds store.DataStore
50	}
51
52	func NewApp(store store.Store, ds store.DataStore) App {	1✔
53	mapper := mapping.NewMapper(ds)	1✔
54	return &app{	1✔
55	store: store,	1✔
56	mapper: mapper,	1✔
57	ds: ds,	1✔
58	}	1✔
59	}	1✔
60
61	// HealthCheck performs a health check and returns an error if it fails
62	func (a *app) HealthCheck(ctx context.Context) error {	1✔
63	err := a.ds.Ping(ctx)	1✔
64	if err == nil {	2✔
65	err = a.store.Ping(ctx)	1✔
66	}	1✔
67	return err	1✔
68	}
69
70	// GetMapping returns the mapping for the specified tenant
71	func (app app) GetMapping(ctx context.Context, tid string) (model.Mapping, error) {	1✔
72	return app.ds.GetMapping(ctx, tid)	1✔
73	}	1✔
74
75	// AggregateDevices aggregates device data
76	func (app *app) AggregateDevices(
77	ctx context.Context,
78	aggregateParams *model.AggregateParams,
79	) ([]model.DeviceAggregation, error) {	1✔
80	searchParams := &model.SearchParams{	1✔
81	Filters: aggregateParams.Filters,	1✔
82	Groups: aggregateParams.Groups,	1✔
83	TenantID: aggregateParams.TenantID,	1✔
84	}	1✔
85	if err := app.mapSearchParams(ctx, searchParams); err != nil {	1✔
86	return nil, err	×
87	}	×
88	query, err := model.BuildQuery(*searchParams)	1✔
89	if err != nil {	1✔
90	return nil, err	×
91	}	×
92	if searchParams.TenantID != "" {	2✔
93	query = query.Must(model.M{	1✔
94	"term": model.M{	1✔
95	model.FieldNameTenantID: searchParams.TenantID,	1✔
96	},	1✔
97	})	1✔
98	}	1✔
99
100	if err := app.mapAggregations(ctx, searchParams.TenantID,	1✔
101	aggregateParams.Aggregations); err != nil {	1✔
102	return nil, err	×
103	}	×
104	aggregations, err := model.BuildAggregations(aggregateParams.Aggregations)	1✔
105	if err != nil {	1✔
106	return nil, err	×
107	}	×
108
109	query = query.WithSize(0).With(map[string]interface{}{	1✔
110	"aggs": aggregations,	1✔
111	})	1✔
112	esRes, err := app.store.AggregateDevices(ctx, query)	1✔
113	if err != nil {	1✔
114	return nil, err	×
115	}	×
116
117	aggregationsS, ok := esRes["aggregations"].(map[string]interface{})	1✔
118	if !ok {	1✔
119	return nil, errors.New("can't process store aggregations slice")	×
120	}	×
121	res, err := app.storeToDeviceAggregations(ctx, searchParams.TenantID, aggregationsS)	1✔
122	if err != nil {	1✔
123	return nil, err	×
124	}	×
125
126	return res, nil	1✔
127	}
128
129	// storeToDeviceAggregations translates ES results directly to device aggregations
130	func (a *app) storeToDeviceAggregations(
131	ctx context.Context, tenantID string, aggregationsS map[string]interface{},
132	) ([]model.DeviceAggregation, error) {	1✔
133	aggs := []model.DeviceAggregation{}	1✔
134	for name, aggregationS := range aggregationsS {	2✔
135	if _, ok := aggregationS.(map[string]interface{}); !ok {	2✔
136	continue	1✔
137	}
138	bucketsS, ok := aggregationS.(map[string]interface{})["buckets"].([]interface{})	1✔
139	if !ok {	1✔
140	continue	×
141	}
142	items := make([]model.DeviceAggregationItem, 0, len(bucketsS))	1✔
143	for _, bucket := range bucketsS {	2✔
144	bucketMap, ok := bucket.(map[string]interface{})	1✔
145	if !ok {	1✔
146	return nil, errors.New("can't process store bucket item")	×
147	}	×
148	key, ok := bucketMap["key"].(string)	1✔
149	if !ok {	1✔
150	return nil, errors.New("can't process store key attribute")	×
151	}	×
152	count, ok := bucketMap["doc_count"].(float64)	1✔
153	if !ok {	1✔
154	return nil, errors.New("can't process store doc_count attribute")	×
155	}	×
156	item := model.DeviceAggregationItem{	1✔
157	Key: key,	1✔
158	Count: int(count),	1✔
159	}	1✔
160	subaggs, err := a.storeToDeviceAggregations(ctx, tenantID, bucketMap)	1✔
161	if err == nil && len(subaggs) > 0 {	2✔
162	item.Aggregations = subaggs	1✔
163	}	1✔
164	items = append(items, item)	1✔
165	}
166
167	otherCount := 0	1✔
168	if count, ok := aggregationS.(map[string]interface{})["sum_other_doc_count"].(float64); ok {	2✔
169	otherCount = int(count)	1✔
170	}	1✔
171
172	aggs = append(aggs, model.DeviceAggregation{	1✔
173	Name: name,	1✔
174	Items: items,	1✔
175	OtherCount: otherCount,	1✔
176	})	1✔
177	}
178	return aggs, nil	1✔
179	}
180
181	// SearchDevices searches device data
182	func (app *app) SearchDevices(
183	ctx context.Context,
184	searchParams *model.SearchParams,
185	) ([]inventory.Device, int, error) {	1✔
186	if err := app.mapSearchParams(ctx, searchParams); err != nil {	1✔
187	return nil, 0, err	×
188	}	×
189	query, err := model.BuildQuery(*searchParams)	1✔
190	if err != nil {	2✔
191	return nil, 0, err	1✔
192	}	1✔
193
194	if searchParams.TenantID != "" {	1✔
195	query = query.Must(model.M{	×
196	"term": model.M{	×
197	model.FieldNameTenantID: searchParams.TenantID,	×
198	},	×
199	})	×
200	}	×
201
202	if len(searchParams.DeviceIDs) > 0 {	2✔
203	query = query.Must(model.M{	1✔
204	"terms": model.M{	1✔
205	model.FieldNameID: searchParams.DeviceIDs,	1✔
206	},	1✔
207	})	1✔
208	}	1✔
209
210	esRes, err := app.store.SearchDevices(ctx, query)	1✔
211	if err != nil {	2✔
212	return nil, 0, err	1✔
213	}	1✔
214
215	res, total, err := app.storeToInventoryDevs(ctx, searchParams.TenantID, esRes)	1✔
216	if err != nil {	2✔
217	return nil, 0, err	1✔
218	}	1✔
219
220	return res, total, err	1✔
221	}
222
223	func (app *app) mapAggregations(ctx context.Context, tenantID string,
224	aggregations []model.AggregationTerm) error {	1✔
225	attributes := make(inventory.DeviceAttributes, 0, len(aggregations))	1✔
226	for i := range aggregations {	2✔
227	attributes = append(attributes, inventory.DeviceAttribute{	1✔
228	Name: aggregations[i].Attribute,	1✔
229	Scope: aggregations[i].Scope,	1✔
230	})	1✔
231	}	1✔
232	attributes, err := app.mapper.MapInventoryAttributes(ctx, tenantID,	1✔
233	attributes, false, true)	1✔
234	if err == nil {	2✔
235	for i, attr := range attributes {	2✔
236	aggregations[i].Attribute = attr.Name	1✔
237	aggregations[i].Scope = attr.Scope	1✔
238	if len(aggregations[i].Aggregations) > 0 {	2✔
239	err = app.mapAggregations(ctx, tenantID, aggregations[i].Aggregations)	1✔
240	if err != nil {	1✔
241	break	×
242	}
243	}
244	}
245	}
246	return err	1✔
247	}
248
249	func (app app) mapSearchParams(ctx context.Context, searchParams model.SearchParams) error {	1✔
250	if len(searchParams.Filters) > 0 {	2✔
251	attributes := make(inventory.DeviceAttributes, 0, len(searchParams.Attributes))	1✔
252	for i := 0; i < len(searchParams.Filters); i++ {	2✔
253	attributes = append(attributes, inventory.DeviceAttribute{	1✔
254	Name: searchParams.Filters[i].Attribute,	1✔
255	Scope: searchParams.Filters[i].Scope,	1✔
256	Value: searchParams.Filters[i].Value,	1✔
257	Description: &searchParams.Filters[i].Type,	1✔
258	})	1✔
259	}	1✔
260	attributes, err := app.mapper.MapInventoryAttributes(ctx, searchParams.TenantID,	1✔
261	attributes, false, true)	1✔
262	if err != nil {	1✔
263	return err	×
264	}	×
265	searchParams.Filters = make([]model.FilterPredicate, 0, len(searchParams.Filters))	1✔
266	for _, attribute := range attributes {	2✔
267	searchParams.Filters = append(searchParams.Filters, model.FilterPredicate{	1✔
268	Attribute: attribute.Name,	1✔
269	Scope: attribute.Scope,	1✔
270	Value: attribute.Value,	1✔
271	Type: *attribute.Description,	1✔
272	})	1✔
273	}	1✔
274	}
275	if len(searchParams.Attributes) > 0 {	2✔
276	attributes := make(inventory.DeviceAttributes, 0, len(searchParams.Attributes))	1✔
277	for i := 0; i < len(searchParams.Attributes); i++ {	2✔
278	attributes = append(attributes, inventory.DeviceAttribute{	1✔
279	Name: searchParams.Attributes[i].Attribute,	1✔
280	Scope: searchParams.Attributes[i].Scope,	1✔
281	})	1✔
282	}	1✔
283	attributes, err := app.mapper.MapInventoryAttributes(ctx, searchParams.TenantID,	1✔
284	attributes, false, false)	1✔
285	if err != nil {	1✔
286	return err	×
287	}	×
288	searchParams.Attributes = make([]model.SelectAttribute, 0, len(searchParams.Attributes))	1✔
289	for _, attribute := range attributes {	2✔
290	searchParams.Attributes = append(searchParams.Attributes, model.SelectAttribute{	1✔
291	Attribute: attribute.Name,	1✔
292	Scope: attribute.Scope,	1✔
293	})	1✔
294	}	1✔
295	}
296	if len(searchParams.Sort) > 0 {	2✔
297	attributes := make(inventory.DeviceAttributes, 0, len(searchParams.Sort))	1✔
298	for i := 0; i < len(searchParams.Sort); i++ {	2✔
299	attributes = append(attributes, inventory.DeviceAttribute{	1✔
300	Name: searchParams.Sort[i].Attribute,	1✔
301	Scope: searchParams.Sort[i].Scope,	1✔
302	Description: &searchParams.Sort[i].Order,	1✔
303	})	1✔
304	}	1✔
305	attributes, err := app.mapper.MapInventoryAttributes(ctx, searchParams.TenantID,	1✔
306	attributes, false, false)	1✔
307	if err != nil {	1✔
308	return err	×
309	}	×
310	searchParams.Sort = make([]model.SortCriteria, 0, len(searchParams.Attributes))	1✔
311	for _, attribute := range attributes {	2✔
312	searchParams.Sort = append(searchParams.Sort, model.SortCriteria{	1✔
313	Attribute: attribute.Name,	1✔
314	Scope: attribute.Scope,	1✔
315	Order: *attribute.Description,	1✔
316	})	1✔
317	}	1✔
318	}
319
320	return nil	1✔
321	}
322
323	// storeToInventoryDevs translates ES results directly to inventory devices
324	func (a *app) storeToInventoryDevs(
325	ctx context.Context, tenantID string, storeRes map[string]interface{},
326	) ([]inventory.Device, int, error) {	1✔
327	devs := []inventory.Device{}	1✔
328		1✔
329	hitsM, ok := storeRes["hits"].(map[string]interface{})	1✔
330	if !ok {	1✔
331	return nil, 0, errors.New("can't process store hits map")	×
332	}	×
333
334	hitsTotalM, ok := hitsM["total"].(map[string]interface{})	1✔
335	if !ok {	1✔
336	return nil, 0, errors.New("can't process total hits struct")	×
337	}	×
338
339	total, ok := hitsTotalM["value"].(float64)	1✔
340	if !ok {	2✔
341	return nil, 0, errors.New("can't process total hits value")	1✔
342	}	1✔
343
344	hitsS, ok := hitsM["hits"].([]interface{})	1✔
345	if !ok {	1✔
346	return nil, 0, errors.New("can't process store hits slice")	×
347	}	×
348
349	for _, v := range hitsS {	2✔
350	res, err := a.storeToInventoryDev(ctx, tenantID, v)	1✔
351	if err != nil {	1✔
352	return nil, 0, err	×
353	}	×
354
355	devs = append(devs, *res)	1✔
356	}
357
358	return devs, int(total), nil	1✔
359	}
360
361	func (a *app) storeToInventoryDev(ctx context.Context, tenantID string,
362	storeRes interface{}) (*inventory.Device, error) {	1✔
363	resM, ok := storeRes.(map[string]interface{})	1✔
364	if !ok {	1✔
365	return nil, errors.New("can't process individual hit")	×
366	}	×
367
368	// if query has a 'fields' clause, use 'fields' instead of '_source'
369	sourceM, ok := resM["_source"].(map[string]interface{})	1✔
370	if !ok {	2✔
371	sourceM, ok = resM["fields"].(map[string]interface{})	1✔
372	if !ok {	1✔
373	return nil, errors.New("can't process hit's '_source' nor 'fields'")	×
374	}	×
375	}
376
377	// if query has a 'fields' clause, all results will be arrays incl. device id, so extract it
378	id, ok := sourceM["id"].(string)	1✔
379	if !ok {	1✔
380	idarr, ok := sourceM["id"].([]interface{})	×
381	if !ok {	×
382	return nil, errors.New(	×
383	"can't parse device id as neither single value nor array",	×
384	)	×
385	}	×
386
387	id, ok = idarr[0].(string)	×
388	if !ok {	×
389	return nil, errors.New(	×
390	"can't parse device id as neither single value nor array",	×
391	)	×
392	}	×
393	}
394
395	ret := &inventory.Device{	1✔
396	ID: inventory.DeviceID(id),	1✔
397	}	1✔
398	t := getTime(sourceM, model.FieldNameCheckIn)	1✔
399	if t != nil && !t.IsZero() {	1✔
400	ret.LastCheckinDate = t	×
401	}	×
402	attrs := []inventory.DeviceAttribute{}	1✔
403		1✔
404	for k, v := range sourceM {	2✔
405	s, n, err := model.MaybeParseAttr(k)	1✔
406	if err != nil {	1✔
407	return nil, err	×
408	}	×
409
410	if vArray, ok := v.([]interface{}); ok && len(vArray) == 1 {	1✔
411	v = vArray[0]	×
412	}	×
413
414	if n != "" {	2✔
415	a := inventory.DeviceAttribute{	1✔
416	Name: model.Redot(n),	1✔
417	Scope: s,	1✔
418	Value: v,	1✔
419	}	1✔
420		1✔
421	if a.Scope == model.ScopeSystem &&	1✔
422	a.Name == model.AttrNameUpdatedAt {	1✔
423	ret.UpdatedTs = parseTime(v)	×
424	} else if a.Scope == model.ScopeSystem &&	1✔
425	a.Name == model.AttrNameCreatedAt {	1✔
426	ret.CreatedTs = parseTime(v)	×
427	}	×
428
429	attrs = append(attrs, a)	1✔
430	}
431	}
432
433	attributes, err := a.mapper.ReverseInventoryAttributes(ctx, tenantID, attrs)	1✔
434	if err != nil {	1✔
435	return nil, err	×
436	}	×
437	ret.Attributes = attributes	1✔
438		1✔
439	return ret, nil	1✔
440	}
441
442	func getTime(m map[string]interface{}, s string) *time.Time {	1✔
443	if v, ok := m[s]; ok && v != nil {	2✔
444	timeString := ""	1✔
445	if vString, ok := v.(string); ok && len(vString) > 0 {	1✔
446	timeString = v.(string)	×
447	}	×
448	if vArray, ok := v.([]interface{}); ok && len(vArray) > 0 {	2✔
449	timeString = v.([]interface{})[0].(string)	1✔
450	}	1✔
451	if len(timeString) > 0 {	2✔
452	t, e := time.Parse(time.RFC3339, timeString)	1✔
453	if e != nil {	1✔
454	return nil	×
455	}	×
456	if ok {	2✔
457	return &t	1✔
458	}	1✔
459	}
460	}
461	return nil	1✔
462	}
463
464	func parseTime(v interface{}) time.Time {	×
465	val, _ := v.(string)	×
466	if t, err := time.Parse(time.RFC3339, val); err == nil {	×
467	return t	×
468	}	×
469	return time.Time{}	×
470	}
471
472	func (app *app) GetSearchableInvAttrs(
473	ctx context.Context,
474	tid string,
475	) ([]model.FilterAttribute, error) {	1✔
476	l := log.FromContext(ctx)	1✔
477		1✔
478	index, err := app.store.GetDevicesIndexMapping(ctx, tid)	1✔
479	if err != nil {	2✔
480	return nil, err	1✔
481	}	1✔
482
483	// inventory attributes are under 'mappings.properties'
484	mappings, ok := index["mappings"]	1✔
485	if !ok {	1✔
486	return nil, errors.New("can't parse index mappings")	×
487	}	×
488
489	mappingsM, ok := mappings.(map[string]interface{})	1✔
490	if !ok {	1✔
491	return nil, errors.New("can't parse index mappings")	×
492	}	×
493
494	props, ok := mappingsM["properties"]	1✔
495	if !ok {	1✔
496	return nil, errors.New("can't parse index properties")	×
497	}	×
498
499	propsM, ok := props.(map[string]interface{})	1✔
500	if !ok {	1✔
501	return nil, errors.New("can't parse index properties")	×
502	}	×
503
504	attrs := []inventory.DeviceAttribute{}	1✔
505	for k := range propsM {	2✔
506	s, n, err := model.MaybeParseAttr(k)	1✔
507		1✔
508	if err != nil {	1✔
509	return nil, err	×
510	}	×
511
512	if n != "" {	2✔
513	attrs = append(attrs, inventory.DeviceAttribute{Name: n, Scope: s})	1✔
514	}	1✔
515	}
516	attributes, err := app.mapper.ReverseInventoryAttributes(ctx, tid, attrs)	1✔
517	if err != nil {	1✔
518	return nil, err	×
519	}	×
520
521	ret := []model.FilterAttribute{}	1✔
522	for _, attr := range attributes {	2✔
523	ret = append(ret, model.FilterAttribute{Name: attr.Name, Scope: attr.Scope, Count: 1})	1✔
524	}	1✔
525
526	sort.Slice(ret, func(i, j int) bool {	2✔
527	if ret[j].Scope > ret[i].Scope {	1✔
UNCOV 528	return true	×
UNCOV 529	}	×
530
531	if ret[j].Scope < ret[i].Scope {	2✔
532	return false	1✔
533	}	1✔
534
535	return ret[j].Name > ret[i].Name	1✔
536	})
537
538	l.Debugf("parsed searchable attributes %v\n", ret)	1✔
539		1✔
540	return ret, nil	1✔
541	}

mendersoftware / mender-server / 1648363360

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous