715476761

Build Type

gitlab-ci

Committed by Fabio Tranchitella

Commit Message

feat: end-point to list the filterable attributes usage and limits

Pull Request Pull Request #79: MEN-5598: map inventory attributes to sequential fields

Run Details

441 of 510 new or added lines in 14 files covered. (86.47%)

13 existing lines in 2 files now uncovered.

1832 of 2225 relevant lines covered (82.34%)

15.09 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.54

/mapping/mapping.go

// Copyright 2022 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package mapping

import (
        "context"
        "fmt"
        "math"
        "os"
        "path"
        "strings"
        "sync"

        "github.com/mendersoftware/reporting/client/inventory"
        "github.com/mendersoftware/reporting/model"
        "github.com/mendersoftware/reporting/store"
)

const (
        inventoryAttributeTemplate = "attribute%d"
)

// Mapping is an interface to map and reverse attributes
type Mapper interface {
        MapInventoryAttributes(ctx context.Context, tenantID string,
                attrs inventory.DeviceAttributes, update bool) (inventory.DeviceAttributes, error)
        ReverseInventoryAttributes(ctx context.Context, tenantID string,
                attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error)
}

type tenantMapCache struct {
        inventory        map[string]string
        inventoryReverse map[string]string
}

type mapper struct {
        ds    store.DataStore
        cache map[string]*tenantMapCache
        lock  sync.RWMutex
}

func NewMapper(ds store.DataStore) Mapper {
        return newMapper(ds)
}

func newMapper(ds store.DataStore) *mapper {
        return &mapper{
                ds:    ds,
                cache: make(map[string]*tenantMapCache),
                lock:  sync.RWMutex{},
        }
}

// MapInventoryAttributes maps inventory attributes to ES fields
func (m *mapper) MapInventoryAttributes(ctx context.Context, tenantID string,
        attrs inventory.DeviceAttributes, update bool) (inventory.DeviceAttributes, error) {
        attributesToFieldsMap := m.lookupMapping(tenantID, attrs, false)
        if attributesToFieldsMap == nil {
                var mapping *model.Mapping
                var err error
                if update {
                        mapping, err = m.updateAndGetMapping(ctx, tenantID, attrs)
                } else {
                        mapping, err = m.getMapping(ctx, tenantID)
                }
                if err != nil {
                        return nil, err
                }
                n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))
                attributesToFieldsMap = attributesToFields(mapping.Inventory[:n])
        }
        return mapAttributes(attrs, attributesToFieldsMap, false), nil
}

// ReverseInventoryAttributes looks up the inventory attribute names from the ES fields
func (m *mapper) ReverseInventoryAttributes(ctx context.Context, tenantID string,
        attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error) {
        attributesToFieldsMap := m.lookupMapping(tenantID, attrs, true)
        if attributesToFieldsMap == nil {
                mapping, err := m.getMapping(ctx, tenantID)
                if err != nil {
                        return nil, err
                }
                n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))
                attributesToFieldsMap = fieldsToAttributes(mapping.Inventory[:n])
        }
        return mapAttributes(attrs, attributesToFieldsMap, true), nil
}

func (m *mapper) getMapping(ctx context.Context, tenantID string) (*model.Mapping, error) {
        mapping, err := m.ds.GetMapping(ctx, tenantID)
        if err == nil {
                m.cacheMapping(tenantID, mapping)
        }
        return mapping, err
}

func (m *mapper) cacheMapping(tenantID string, mapping *model.Mapping) {
        cache := &tenantMapCache{
                inventory:        make(map[string]string),
                inventoryReverse: make(map[string]string),
        }
        n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))
        for i, attr := range mapping.Inventory[:n] {
                attrName := fmt.Sprintf(inventoryAttributeTemplate, i+1)
                cache.inventory[attr] = attrName
                cache.inventoryReverse[attrName] = attr
        }
        m.lock.Lock()
        m.cache[tenantID] = cache
        m.lock.Unlock()
}

func (m *mapper) lookupMapping(tenantID string, attrs inventory.DeviceAttributes,
        reverse bool) map[string]string {
        m.lock.RLock()
        cache, ok := m.cache[tenantID]
        m.lock.RUnlock()
        if ok {
                var cacheAttributes map[string]string
                if reverse {
                        cacheAttributes = cache.inventoryReverse
                } else {
                        cacheAttributes = cache.inventory
                }
                if len(cacheAttributes) < model.MaxMappingInventoryAttributes {
                        for i := 0; i < len(attrs); i++ {
                                if shouldMapScope(attrs[i].Scope, attrs[i].Name) {
                                        var key string
                                        if reverse {
                                                key = attrs[i].Name
                                        } else {
                                                key = path.Join(attrs[i].Scope, attrs[i].Name)
                                        }
                                        if _, ok := cacheAttributes[key]; !ok {
                                                return nil
                                        }
                                }
                        }
                }
                return cacheAttributes
        }
        return nil
}

func (m *mapper) updateAndGetMapping(ctx context.Context, tenantID string,
        attrs inventory.DeviceAttributes) (*model.Mapping, error) {
        inventoryMapping := make([]string, 0, len(attrs))
        for i := 0; i < len(attrs); i++ {
                if shouldMapScope(attrs[i].Scope, attrs[i].Name) {
                        key := path.Join(attrs[i].Scope, attrs[i].Name)
                        inventoryMapping = append(inventoryMapping, key)
                }
        }
        if len(inventoryMapping) > model.MaxMappingInventoryAttributes {
                inventoryMapping = inventoryMapping[:model.MaxMappingInventoryAttributes]
        }
        mapping, err := m.ds.UpdateAndGetMapping(ctx, tenantID, inventoryMapping)
        if err != nil {
                return nil, err
        }
        m.cacheMapping(tenantID, mapping)
        return mapping, nil
}

func mapAttributes(attrs inventory.DeviceAttributes,
        mapping map[string]string, reverse bool) inventory.DeviceAttributes {
        mappedAttrs := make(inventory.DeviceAttributes, 0, len(attrs))
        for i := 0; i < len(attrs); i++ {
                var attrName string
                if !shouldMapScope(attrs[i].Scope, attrs[i].Name) {
                        attrName = attrs[i].Name
                } else if reverse {
                        if name, ok := mapping[attrs[i].Name]; ok {
                                parts := strings.SplitN(name, string(os.PathSeparator), 2)
                                attrName = parts[len(parts)-1]
                        }
                } else if name, ok := mapping[path.Join(attrs[i].Scope, attrs[i].Name)]; ok {
                        attrName = name
                }
                if attrName != "" {
                        mappedAttr := inventory.DeviceAttribute{
                                Name:        attrName,
                                Scope:       attrs[i].Scope,
                                Value:       attrs[i].Value,
                                Description: attrs[i].Description,
                        }
                        mappedAttrs = append(mappedAttrs, mappedAttr)
                }
        }
        return mappedAttrs
}

func attributesToFields(attrs []string) map[string]string {
        var attributesToFields = make(map[string]string, len(attrs))
        for i := 0; i < len(attrs); i++ {
                attributesToFields[attrs[i]] = fmt.Sprintf(inventoryAttributeTemplate, i+1)
        }
        return attributesToFields
}

func fieldsToAttributes(attrs []string) map[string]string {
        var fieldsToAttributes = make(map[string]string, len(attrs))
        for i := 0; i < len(attrs); i++ {
                fieldsToAttributes[fmt.Sprintf(inventoryAttributeTemplate, i+1)] = attrs[i]
        }
        return fieldsToAttributes
}

func shouldMapScope(scope, attribute string) bool {
        return scope != model.ScopeSystem &&
                !(scope == model.ScopeIdentity && attribute == model.AttrNameStatus)
}

1	// Copyright 2022 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package mapping
16
17	import (
18	"context"
19	"fmt"
20	"math"
21	"os"
22	"path"
23	"strings"
24	"sync"
25
26	"github.com/mendersoftware/reporting/client/inventory"
27	"github.com/mendersoftware/reporting/model"
28	"github.com/mendersoftware/reporting/store"
29	)
30
31	const (
32	inventoryAttributeTemplate = "attribute%d"
33	)
34
35	// Mapping is an interface to map and reverse attributes
36	type Mapper interface {
37	MapInventoryAttributes(ctx context.Context, tenantID string,
38	attrs inventory.DeviceAttributes, update bool) (inventory.DeviceAttributes, error)
39	ReverseInventoryAttributes(ctx context.Context, tenantID string,
40	attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error)
41	}
42
43	type tenantMapCache struct {
44	inventory map[string]string
45	inventoryReverse map[string]string
46	}
47
48	type mapper struct {
49	ds store.DataStore
50	cache map[string]*tenantMapCache
51	lock sync.RWMutex
52	}
53
54	func NewMapper(ds store.DataStore) Mapper {	9✔
55	return newMapper(ds)	9✔
56	}	9✔
57
58	func newMapper(ds store.DataStore) *mapper {	11✔
59	return &mapper{	11✔
60	ds: ds,	11✔
61	cache: make(map[string]*tenantMapCache),	11✔
62	lock: sync.RWMutex{},	11✔
63	}	11✔
64	}	11✔
65
66	// MapInventoryAttributes maps inventory attributes to ES fields
67	func (m *mapper) MapInventoryAttributes(ctx context.Context, tenantID string,
68	attrs inventory.DeviceAttributes, update bool) (inventory.DeviceAttributes, error) {	39✔
69	attributesToFieldsMap := m.lookupMapping(tenantID, attrs, false)	39✔
70	if attributesToFieldsMap == nil {	47✔
71	var mapping *model.Mapping	8✔
72	var err error	8✔
73	if update {	12✔
74	mapping, err = m.updateAndGetMapping(ctx, tenantID, attrs)	4✔
75	} else {	8✔
76	mapping, err = m.getMapping(ctx, tenantID)	4✔
77	}	4✔
78	if err != nil {	9✔
79	return nil, err	1✔
80	}	1✔
81	n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))	7✔
82	attributesToFieldsMap = attributesToFields(mapping.Inventory[:n])	7✔
83	}
84	return mapAttributes(attrs, attributesToFieldsMap, false), nil	38✔
85	}
86
87	// ReverseInventoryAttributes looks up the inventory attribute names from the ES fields
88	func (m *mapper) ReverseInventoryAttributes(ctx context.Context, tenantID string,
89	attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error) {	31✔
90	attributesToFieldsMap := m.lookupMapping(tenantID, attrs, true)	31✔
91	if attributesToFieldsMap == nil {	34✔
92	mapping, err := m.getMapping(ctx, tenantID)	3✔
93	if err != nil {	4✔
94	return nil, err	1✔
95	}	1✔
96	n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))	2✔
97	attributesToFieldsMap = fieldsToAttributes(mapping.Inventory[:n])	2✔
98	}
99	return mapAttributes(attrs, attributesToFieldsMap, true), nil	30✔
100	}
101
102	func (m mapper) getMapping(ctx context.Context, tenantID string) (model.Mapping, error) {	7✔
103	mapping, err := m.ds.GetMapping(ctx, tenantID)	7✔
104	if err == nil {	13✔
105	m.cacheMapping(tenantID, mapping)	6✔
106	}	6✔
107	return mapping, err	7✔
108	}
109
110	func (m mapper) cacheMapping(tenantID string, mapping model.Mapping) {	10✔
111	cache := &tenantMapCache{	10✔
112	inventory: make(map[string]string),	10✔
113	inventoryReverse: make(map[string]string),	10✔
114	}	10✔
115	n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))	10✔
116	for i, attr := range mapping.Inventory[:n] {	29✔
117	attrName := fmt.Sprintf(inventoryAttributeTemplate, i+1)	19✔
118	cache.inventory[attr] = attrName	19✔
119	cache.inventoryReverse[attrName] = attr	19✔
120	}	19✔
121	m.lock.Lock()	10✔
122	m.cache[tenantID] = cache	10✔
123	m.lock.Unlock()	10✔
124	}
125
126	func (m *mapper) lookupMapping(tenantID string, attrs inventory.DeviceAttributes,
127	reverse bool) map[string]string {	70✔
128	m.lock.RLock()	70✔
129	cache, ok := m.cache[tenantID]	70✔
130	m.lock.RUnlock()	70✔
131	if ok {	130✔
132	var cacheAttributes map[string]string	60✔
133	if reverse {	89✔
134	cacheAttributes = cache.inventoryReverse	29✔
135	} else {	60✔
136	cacheAttributes = cache.inventory	31✔
137	}	31✔
138	if len(cacheAttributes) < model.MaxMappingInventoryAttributes {	120✔
139	for i := 0; i < len(attrs); i++ {	180✔
140	if shouldMapScope(attrs[i].Scope, attrs[i].Name) {	211✔
141	var key string	91✔
142	if reverse {	144✔
143	key = attrs[i].Name	53✔
144	} else {	91✔
145	key = path.Join(attrs[i].Scope, attrs[i].Name)	38✔
146	}	38✔
147	if _, ok := cacheAttributes[key]; !ok {	92✔
148	return nil	1✔
149	}	1✔
150	}
151	}
152	}
153	return cacheAttributes	59✔
154	}
155	return nil	10✔
156	}
157
158	func (m *mapper) updateAndGetMapping(ctx context.Context, tenantID string,
159	attrs inventory.DeviceAttributes) (*model.Mapping, error) {	6✔
160	inventoryMapping := make([]string, 0, len(attrs))	6✔
161	for i := 0; i < len(attrs); i++ {	21✔
162	if shouldMapScope(attrs[i].Scope, attrs[i].Name) {	29✔
163	key := path.Join(attrs[i].Scope, attrs[i].Name)	14✔
164	inventoryMapping = append(inventoryMapping, key)	14✔
165	}	14✔
166	}
167	if len(inventoryMapping) > model.MaxMappingInventoryAttributes {	6✔
NEW 168	inventoryMapping = inventoryMapping[:model.MaxMappingInventoryAttributes]	×
NEW 169	}	×
170	mapping, err := m.ds.UpdateAndGetMapping(ctx, tenantID, inventoryMapping)	6✔
171	if err != nil {	8✔
172	return nil, err	2✔
173	}	2✔
174	m.cacheMapping(tenantID, mapping)	4✔
175	return mapping, nil	4✔
176	}
177
178	func mapAttributes(attrs inventory.DeviceAttributes,
179	mapping map[string]string, reverse bool) inventory.DeviceAttributes {	69✔
180	mappedAttrs := make(inventory.DeviceAttributes, 0, len(attrs))	69✔
181	for i := 0; i < len(attrs); i++ {	210✔
182	var attrName string	141✔
183	if !shouldMapScope(attrs[i].Scope, attrs[i].Name) {	174✔
184	attrName = attrs[i].Name	33✔
185	} else if reverse {	198✔
186	if name, ok := mapping[attrs[i].Name]; ok {	114✔
187	parts := strings.SplitN(name, string(os.PathSeparator), 2)	57✔
188	attrName = parts[len(parts)-1]	57✔
189	}	57✔
190	} else if name, ok := mapping[path.Join(attrs[i].Scope, attrs[i].Name)]; ok {	100✔
191	attrName = name	49✔
192	}	49✔
193	if attrName != "" {	280✔
194	mappedAttr := inventory.DeviceAttribute{	139✔
195	Name: attrName,	139✔
196	Scope: attrs[i].Scope,	139✔
197	Value: attrs[i].Value,	139✔
198	Description: attrs[i].Description,	139✔
199	}	139✔
200	mappedAttrs = append(mappedAttrs, mappedAttr)	139✔
201	}	139✔
202	}
203	return mappedAttrs	69✔
204	}
205
206	func attributesToFields(attrs []string) map[string]string {	8✔
207	var attributesToFields = make(map[string]string, len(attrs))	8✔
208	for i := 0; i < len(attrs); i++ {	22✔
209	attributesToFields[attrs[i]] = fmt.Sprintf(inventoryAttributeTemplate, i+1)	14✔
210	}	14✔
211	return attributesToFields	8✔
212	}
213
214	func fieldsToAttributes(attrs []string) map[string]string {	3✔
215	var fieldsToAttributes = make(map[string]string, len(attrs))	3✔
216	for i := 0; i < len(attrs); i++ {	10✔
217	fieldsToAttributes[fmt.Sprintf(inventoryAttributeTemplate, i+1)] = attrs[i]	7✔
218	}	7✔
219	return fieldsToAttributes	3✔
220	}
221
222	func shouldMapScope(scope, attribute string) bool {	276✔
223	return scope != model.ScopeSystem &&	276✔
224	!(scope == model.ScopeIdentity && attribute == model.AttrNameStatus)	276✔
225	}	276✔

mendersoftware / reporting / 715476761

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous