714559549

Build Type

gitlab-ci

Committed by Fabio Tranchitella

Commit Message

chore: minor improvements to the mapping utility

Pull Request Pull Request #79: MEN-5598: map inventory attributes to sequential fields

Run Details

334 of 403 new or added lines in 12 files covered. (82.88%)

4 existing lines in 2 files now uncovered.

1680 of 2118 relevant lines covered (79.32%)

12.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.33

/mapping/mapping.go

// Copyright 2022 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package mapping

import (
        "context"
        "fmt"
        "math"
        "sync"

        "github.com/mendersoftware/reporting/client/inventory"
        "github.com/mendersoftware/reporting/model"
        "github.com/mendersoftware/reporting/store"
)

const (
        inventoryAttributeTemplate = "attribute%d"
)

// Mapping is an interface to map and reverse attributes
type Mapper interface {
        MapInventoryAttributes(ctx context.Context, tenantID string,
                attrs inventory.DeviceAttributes, update bool) (inventory.DeviceAttributes, error)
        ReverseInventoryAttributes(ctx context.Context, tenantID string,
                attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error)
}

type tenantMapCache struct {
        inventory        map[string]string
        inventoryReverse map[string]string
}

type mapper struct {
        ds    store.DataStore
        cache map[string]*tenantMapCache
        lock  sync.RWMutex
}

func NewMapper(ds store.DataStore) Mapper {
        return newMapper(ds)
}

func newMapper(ds store.DataStore) *mapper {
        return &mapper{
                ds:    ds,
                cache: make(map[string]*tenantMapCache),
                lock:  sync.RWMutex{},
        }
}

// MapInventoryAttributes maps inventory attributes to ES fields
func (m *mapper) MapInventoryAttributes(ctx context.Context, tenantID string,
        attrs inventory.DeviceAttributes, update bool) (inventory.DeviceAttributes, error) {
        attributesToFieldsMap := m.lookupMapping(tenantID, attrs, false)
        if attributesToFieldsMap == nil {
                var mapping *model.Mapping
                var err error
                if update {
                        mapping, err = m.updateAndGetMapping(ctx, tenantID, attrs)
                } else {
                        mapping, err = m.getMapping(ctx, tenantID)
                }
                if err != nil {
                        return nil, err
                }
                n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))
                attributesToFieldsMap = attributesToFields(mapping.Inventory[:n])
        }
        return mapAttributes(attrs, attributesToFieldsMap), nil
}

// ReverseInventoryAttributes looks up the inventory attribute names from the ES fields
func (m *mapper) ReverseInventoryAttributes(ctx context.Context, tenantID string,
        attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error) {
        attributesToFieldsMap := m.lookupMapping(tenantID, attrs, true)
        if attributesToFieldsMap == nil {
                mapping, err := m.getMapping(ctx, tenantID)
                if err != nil {
                        return nil, err
                }
                n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))
                attributesToFieldsMap = fieldsToAttributes(mapping.Inventory[:n])
        }
        return mapAttributes(attrs, attributesToFieldsMap), nil
}

func (m *mapper) getMapping(ctx context.Context, tenantID string) (*model.Mapping, error) {
        mapping, err := m.ds.GetMapping(ctx, tenantID)
        if err == nil {
                m.cacheMapping(tenantID, mapping)
        }
        return mapping, err
}

func (m *mapper) cacheMapping(tenantID string, mapping *model.Mapping) {
        cache := &tenantMapCache{
                inventory:        make(map[string]string),
                inventoryReverse: make(map[string]string),
        }
        n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))
        for i, attr := range mapping.Inventory[:n] {
                attrName := fmt.Sprintf(inventoryAttributeTemplate, i+1)
                cache.inventory[attr] = attrName
                cache.inventoryReverse[attrName] = attr
        }
        m.lock.Lock()
        m.cache[tenantID] = cache
        m.lock.Unlock()
}

func (m *mapper) lookupMapping(tenantID string, attrs inventory.DeviceAttributes,
        reverse bool) map[string]string {
        m.lock.RLock()
        cache, ok := m.cache[tenantID]
        m.lock.RUnlock()
        if ok {
                var cacheAttributes map[string]string
                if reverse {
                        cacheAttributes = cache.inventoryReverse
                } else {
                        cacheAttributes = cache.inventory
                }
                if len(cacheAttributes) < model.MaxMappingInventoryAttributes {
                        for i := 0; i < len(attrs); i++ {
                                if attrs[i].Scope == inventory.AttrScopeInventory {
                                        if _, ok := cacheAttributes[attrs[i].Name]; !ok {
                                                return nil
                                        }
                                }
                        }
                }
                return cacheAttributes
        }
        return nil
}

func (m *mapper) updateAndGetMapping(ctx context.Context, tenantID string,
        attrs inventory.DeviceAttributes) (*model.Mapping, error) {
        inventoryMapping := make([]string, 0, len(attrs))
        for i := 0; i < len(attrs); i++ {
                if attrs[i].Scope == inventory.AttrScopeInventory {
                        inventoryMapping = append(inventoryMapping, attrs[i].Name)
                }
        }
        if len(inventoryMapping) > model.MaxMappingInventoryAttributes {
                inventoryMapping = inventoryMapping[:model.MaxMappingInventoryAttributes]
        }
        mapping, err := m.ds.UpdateAndGetMapping(ctx, tenantID, inventoryMapping)
        if err != nil {
                return nil, err
        }
        m.cacheMapping(tenantID, mapping)
        return mapping, nil
}

func mapAttributes(attrs inventory.DeviceAttributes,
        mapping map[string]string) inventory.DeviceAttributes {
        mappedAttrs := make(inventory.DeviceAttributes, 0, len(attrs))
        for i := 0; i < len(attrs); i++ {
                var attrName string
                if attrs[i].Scope != inventory.AttrScopeInventory {
                        attrName = attrs[i].Name
                } else if name, ok := mapping[attrs[i].Name]; ok {
                        attrName = name
                } else {
                        attrName = attrs[i].Name
                }
                mappedAttr := inventory.DeviceAttribute{
                        Name:  attrName,
                        Value: attrs[i].Value,
                        Scope: attrs[i].Scope,
                }
                mappedAttrs = append(mappedAttrs, mappedAttr)
        }
        return mappedAttrs
}

func attributesToFields(attrs []string) map[string]string {
        var attributesToFields = make(map[string]string, len(attrs))
        for i := 0; i < len(attrs); i++ {
                attributesToFields[attrs[i]] = fmt.Sprintf(inventoryAttributeTemplate, i+1)
        }
        return attributesToFields
}

func fieldsToAttributes(attrs []string) map[string]string {
        var fieldsToAttributes = make(map[string]string, len(attrs))
        for i := 0; i < len(attrs); i++ {
                fieldsToAttributes[fmt.Sprintf(inventoryAttributeTemplate, i+1)] = attrs[i]
        }
        return fieldsToAttributes
}

1	// Copyright 2022 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package mapping
16
17	import (
18	"context"
19	"fmt"
20	"math"
21	"sync"
22
23	"github.com/mendersoftware/reporting/client/inventory"
24	"github.com/mendersoftware/reporting/model"
25	"github.com/mendersoftware/reporting/store"
26	)
27
28	const (
29	inventoryAttributeTemplate = "attribute%d"
30	)
31
32	// Mapping is an interface to map and reverse attributes
33	type Mapper interface {
34	MapInventoryAttributes(ctx context.Context, tenantID string,
35	attrs inventory.DeviceAttributes, update bool) (inventory.DeviceAttributes, error)
36	ReverseInventoryAttributes(ctx context.Context, tenantID string,
37	attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error)
38	}
39
40	type tenantMapCache struct {
41	inventory map[string]string
42	inventoryReverse map[string]string
43	}
44
45	type mapper struct {
46	ds store.DataStore
47	cache map[string]*tenantMapCache
48	lock sync.RWMutex
49	}
50
51	func NewMapper(ds store.DataStore) Mapper {	9✔
52	return newMapper(ds)	9✔
53	}	9✔
54
55	func newMapper(ds store.DataStore) *mapper {	11✔
56	return &mapper{	11✔
57	ds: ds,	11✔
58	cache: make(map[string]*tenantMapCache),	11✔
59	lock: sync.RWMutex{},	11✔
60	}	11✔
61	}	11✔
62
63	// MapInventoryAttributes maps inventory attributes to ES fields
64	func (m *mapper) MapInventoryAttributes(ctx context.Context, tenantID string,
65	attrs inventory.DeviceAttributes, update bool) (inventory.DeviceAttributes, error) {	5✔
66	attributesToFieldsMap := m.lookupMapping(tenantID, attrs, false)	5✔
67	if attributesToFieldsMap == nil {	9✔
68	var mapping *model.Mapping	4✔
69	var err error	4✔
70	if update {	6✔
71	mapping, err = m.updateAndGetMapping(ctx, tenantID, attrs)	2✔
72	} else {	4✔
73	mapping, err = m.getMapping(ctx, tenantID)	2✔
74	}	2✔
75	if err != nil {	5✔
76	return nil, err	1✔
77	}	1✔
78	n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))	3✔
79	attributesToFieldsMap = attributesToFields(mapping.Inventory[:n])	3✔
80	}
81	return mapAttributes(attrs, attributesToFieldsMap), nil	4✔
82	}
83
84	// ReverseInventoryAttributes looks up the inventory attribute names from the ES fields
85	func (m *mapper) ReverseInventoryAttributes(ctx context.Context, tenantID string,
86	attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error) {	31✔
87	attributesToFieldsMap := m.lookupMapping(tenantID, attrs, true)	31✔
88	if attributesToFieldsMap == nil {	61✔
89	mapping, err := m.getMapping(ctx, tenantID)	30✔
90	if err != nil {	31✔
91	return nil, err	1✔
92	}	1✔
93	n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))	29✔
94	attributesToFieldsMap = fieldsToAttributes(mapping.Inventory[:n])	29✔
95	}
96	return mapAttributes(attrs, attributesToFieldsMap), nil	30✔
97	}
98
99	func (m mapper) getMapping(ctx context.Context, tenantID string) (model.Mapping, error) {	32✔
100	mapping, err := m.ds.GetMapping(ctx, tenantID)	32✔
101	if err == nil {	63✔
102	m.cacheMapping(tenantID, mapping)	31✔
103	}	31✔
104	return mapping, err	32✔
105	}
106
107	func (m mapper) cacheMapping(tenantID string, mapping model.Mapping) {	33✔
108	cache := &tenantMapCache{	33✔
109	inventory: make(map[string]string),	33✔
110	inventoryReverse: make(map[string]string),	33✔
111	}	33✔
112	n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))	33✔
113	for i, attr := range mapping.Inventory[:n] {	46✔
114	attrName := fmt.Sprintf(inventoryAttributeTemplate, i+1)	13✔
115	cache.inventory[attr] = attrName	13✔
116	cache.inventoryReverse[attrName] = attr	13✔
117	}	13✔
118	m.lock.Lock()	33✔
119	m.cache[tenantID] = cache	33✔
120	m.lock.Unlock()	33✔
121	}
122
123	func (m *mapper) lookupMapping(tenantID string, attrs inventory.DeviceAttributes,
124	reverse bool) map[string]string {	36✔
125	m.lock.RLock()	36✔
126	cache, ok := m.cache[tenantID]	36✔
127	m.lock.RUnlock()	36✔
128	if ok {	65✔
129	var cacheAttributes map[string]string	29✔
130	if reverse {	57✔
131	cacheAttributes = cache.inventoryReverse	28✔
132	} else {	29✔
133	cacheAttributes = cache.inventory	1✔
134	}	1✔
135	if len(cacheAttributes) < model.MaxMappingInventoryAttributes {	58✔
136	for i := 0; i < len(attrs); i++ {	64✔
137	if attrs[i].Scope == inventory.AttrScopeInventory {	68✔
138	if _, ok := cacheAttributes[attrs[i].Name]; !ok {	60✔
139	return nil	27✔
140	}	27✔
141	}
142	}
143	}
144	return cacheAttributes	2✔
145	}
146	return nil	7✔
147	}
148
149	func (m *mapper) updateAndGetMapping(ctx context.Context, tenantID string,
150	attrs inventory.DeviceAttributes) (*model.Mapping, error) {	4✔
151	inventoryMapping := make([]string, 0, len(attrs))	4✔
152	for i := 0; i < len(attrs); i++ {	15✔
153	if attrs[i].Scope == inventory.AttrScopeInventory {	21✔
154	inventoryMapping = append(inventoryMapping, attrs[i].Name)	10✔
155	}	10✔
156	}
157	if len(inventoryMapping) > model.MaxMappingInventoryAttributes {	4✔
NEW 158	inventoryMapping = inventoryMapping[:model.MaxMappingInventoryAttributes]	×
NEW 159	}	×
160	mapping, err := m.ds.UpdateAndGetMapping(ctx, tenantID, inventoryMapping)	4✔
161	if err != nil {	6✔
162	return nil, err	2✔
163	}	2✔
164	m.cacheMapping(tenantID, mapping)	2✔
165	return mapping, nil	2✔
166	}
167
168	func mapAttributes(attrs inventory.DeviceAttributes,
169	mapping map[string]string) inventory.DeviceAttributes {	35✔
170	mappedAttrs := make(inventory.DeviceAttributes, 0, len(attrs))	35✔
171	for i := 0; i < len(attrs); i++ {	107✔
172	var attrName string	72✔
173	if attrs[i].Scope != inventory.AttrScopeInventory {	78✔
174	attrName = attrs[i].Name	6✔
175	} else if name, ok := mapping[attrs[i].Name]; ok {	89✔
176	attrName = name	17✔
177	} else {	66✔
178	attrName = attrs[i].Name	49✔
179	}	49✔
180	mappedAttr := inventory.DeviceAttribute{	72✔
181	Name: attrName,	72✔
182	Value: attrs[i].Value,	72✔
183	Scope: attrs[i].Scope,	72✔
184	}	72✔
185	mappedAttrs = append(mappedAttrs, mappedAttr)	72✔
186	}
187	return mappedAttrs	35✔
188	}
189
190	func attributesToFields(attrs []string) map[string]string {	4✔
191	var attributesToFields = make(map[string]string, len(attrs))	4✔
192	for i := 0; i < len(attrs); i++ {	12✔
193	attributesToFields[attrs[i]] = fmt.Sprintf(inventoryAttributeTemplate, i+1)	8✔
194	}	8✔
195	return attributesToFields	4✔
196	}
197
198	func fieldsToAttributes(attrs []string) map[string]string {	30✔
199	var fieldsToAttributes = make(map[string]string, len(attrs))	30✔
200	for i := 0; i < len(attrs); i++ {	37✔
201	fieldsToAttributes[fmt.Sprintf(inventoryAttributeTemplate, i+1)] = attrs[i]	7✔
202	}	7✔
203	return fieldsToAttributes	30✔
204	}

mendersoftware / reporting / 714559549

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous