1571619767

Committed 13 Sep 2024 10:52AM UTC coverage: 85.292% (+0.06%) from 85.235%

Build # 1571619767

Build Type

push

gitlab-ci

Committed by

web-flow

Commit Message

Merge pull request #202 from mzedel/chore/deprecate

Chore/deprecate

Run Details

2998 of 3515 relevant lines covered (85.29%)

14.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.56

/mapping/mapping.go

// Copyright 2022 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package mapping

import (
        "context"
        "fmt"
        "math"
        "os"
        "path"
        "strings"
        "sync"

        "github.com/mendersoftware/reporting/client/inventory"
        "github.com/mendersoftware/reporting/model"
        "github.com/mendersoftware/reporting/store"
)

const (
        inventoryAttributeTemplate = "attribute%d"
)

// Mapping is an interface to map and reverse attributes
type Mapper interface {
        MapInventoryAttributes(ctx context.Context, tenantID string,
                attrs inventory.DeviceAttributes, update bool, passthrough bool,
        ) (inventory.DeviceAttributes, error)
        ReverseInventoryAttributes(ctx context.Context, tenantID string,
                attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error)
}

type tenantMapCache struct {
        inventory        map[string]string
        inventoryReverse map[string]string
}

type mapper struct {
        ds    store.DataStore
        cache map[string]*tenantMapCache
        lock  sync.RWMutex
}

func NewMapper(ds store.DataStore) Mapper {
        return newMapper(ds)
}

func newMapper(ds store.DataStore) *mapper {
        return &mapper{
                ds:    ds,
                cache: make(map[string]*tenantMapCache),
                lock:  sync.RWMutex{},
        }
}

// MapInventoryAttributes maps inventory attributes to ES fields
func (m *mapper) MapInventoryAttributes(ctx context.Context, tenantID string,
        attrs inventory.DeviceAttributes, update bool, passthrough bool) (
        inventory.DeviceAttributes, error) {
        attributesToFieldsMap := m.lookupMapping(tenantID, attrs, false)
        if attributesToFieldsMap == nil {
                var mapping *model.Mapping
                var err error
                if update {
                        mapping, err = m.updateAndGetMapping(ctx, tenantID, attrs)
                } else {
                        mapping, err = m.getMapping(ctx, tenantID)
                }
                if err != nil {
                        return nil, err
                }
                n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))
                attributesToFieldsMap = attributesToFields(mapping.Inventory[:n])
        }
        return mapAttributes(attrs, attributesToFieldsMap, false, passthrough), nil
}

// ReverseInventoryAttributes looks up the inventory attribute names from the ES fields
func (m *mapper) ReverseInventoryAttributes(ctx context.Context, tenantID string,
        attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error) {
        attributesToFieldsMap := m.lookupMapping(tenantID, attrs, true)
        if attributesToFieldsMap == nil {
                mapping, err := m.getMapping(ctx, tenantID)
                if err != nil {
                        return nil, err
                }
                n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))
                attributesToFieldsMap = fieldsToAttributes(mapping.Inventory[:n])
        }
        return mapAttributes(attrs, attributesToFieldsMap, true, false), nil
}

func (m *mapper) getMapping(ctx context.Context, tenantID string) (*model.Mapping, error) {
        mapping, err := m.ds.GetMapping(ctx, tenantID)
        if err == nil {
                m.cacheMapping(tenantID, mapping)
        }
        return mapping, err
}

func (m *mapper) cacheMapping(tenantID string, mapping *model.Mapping) {
        cache := &tenantMapCache{
                inventory:        make(map[string]string),
                inventoryReverse: make(map[string]string),
        }
        n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))
        for i, attr := range mapping.Inventory[:n] {
                attrName := fmt.Sprintf(inventoryAttributeTemplate, i+1)
                cache.inventory[attr] = attrName
                cache.inventoryReverse[attrName] = attr
        }
        m.lock.Lock()
        m.cache[tenantID] = cache
        m.lock.Unlock()
}

func (m *mapper) lookupMapping(tenantID string, attrs inventory.DeviceAttributes,
        reverse bool) map[string]string {
        m.lock.RLock()
        cache, ok := m.cache[tenantID]
        m.lock.RUnlock()
        if ok {
                var cacheAttributes map[string]string
                if reverse {
                        cacheAttributes = cache.inventoryReverse
                } else {
                        cacheAttributes = cache.inventory
                }
                if len(cacheAttributes) < model.MaxMappingInventoryAttributes {
                        for i := 0; i < len(attrs); i++ {
                                if shouldMapScope(attrs[i].Scope, attrs[i].Name) {
                                        var key string
                                        if reverse {
                                                key = attrs[i].Name
                                        } else {
                                                key = path.Join(attrs[i].Scope, attrs[i].Name)
                                        }
                                        if _, ok := cacheAttributes[key]; !ok {
                                                return nil
                                        }
                                }
                        }
                }
                return cacheAttributes
        }
        return nil
}

func (m *mapper) updateAndGetMapping(ctx context.Context, tenantID string,
        attrs inventory.DeviceAttributes) (*model.Mapping, error) {
        inventoryMapping := make([]string, 0, len(attrs))
        for i := 0; i < len(attrs); i++ {
                if shouldMapScope(attrs[i].Scope, attrs[i].Name) {
                        key := path.Join(attrs[i].Scope, attrs[i].Name)
                        inventoryMapping = append(inventoryMapping, key)
                }
        }
        if len(inventoryMapping) > model.MaxMappingInventoryAttributes {
                inventoryMapping = inventoryMapping[:model.MaxMappingInventoryAttributes]
        }
        mapping, err := m.ds.UpdateAndGetMapping(ctx, tenantID, inventoryMapping)
        if err != nil {
                return nil, err
        }
        m.cacheMapping(tenantID, mapping)
        return mapping, nil
}

func mapAttributes(attrs inventory.DeviceAttributes,
        mapping map[string]string, reverse bool, passthrough bool) inventory.DeviceAttributes {
        mappedAttrs := make(inventory.DeviceAttributes, 0, len(attrs))
        for i := 0; i < len(attrs); i++ {
                var attrName string
                if !shouldMapScope(attrs[i].Scope, attrs[i].Name) {
                        attrName = attrs[i].Name
                } else if reverse {
                        if name, ok := mapping[attrs[i].Name]; ok {
                                parts := strings.SplitN(name, string(os.PathSeparator), 2)
                                attrName = parts[len(parts)-1]
                        }
                } else if name, ok := mapping[path.Join(attrs[i].Scope, attrs[i].Name)]; ok {
                        attrName = name
                } else if passthrough {
                        attrName = attrs[i].Name
                }
                if attrName != "" {
                        mappedAttr := inventory.DeviceAttribute{
                                Name:        attrName,
                                Scope:       attrs[i].Scope,
                                Value:       attrs[i].Value,
                                Description: attrs[i].Description,
                        }
                        mappedAttrs = append(mappedAttrs, mappedAttr)
                }
        }
        return mappedAttrs
}

func attributesToFields(attrs []string) map[string]string {
        var attributesToFields = make(map[string]string, len(attrs))
        for i := 0; i < len(attrs); i++ {
                attributesToFields[attrs[i]] = fmt.Sprintf(inventoryAttributeTemplate, i+1)
        }
        return attributesToFields
}

func fieldsToAttributes(attrs []string) map[string]string {
        var fieldsToAttributes = make(map[string]string, len(attrs))
        for i := 0; i < len(attrs); i++ {
                fieldsToAttributes[fmt.Sprintf(inventoryAttributeTemplate, i+1)] = attrs[i]
        }
        return fieldsToAttributes
}

func shouldMapScope(scope, attribute string) bool {
        return scope != model.ScopeSystem &&
                !(scope == model.ScopeIdentity && attribute == model.AttrNameStatus)
}

1	// Copyright 2022 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package mapping
16
17	import (
18	"context"
19	"fmt"
20	"math"
21	"os"
22	"path"
23	"strings"
24	"sync"
25
26	"github.com/mendersoftware/reporting/client/inventory"
27	"github.com/mendersoftware/reporting/model"
28	"github.com/mendersoftware/reporting/store"
29	)
30
31	const (
32	inventoryAttributeTemplate = "attribute%d"
33	)
34
35	// Mapping is an interface to map and reverse attributes
36	type Mapper interface {
37	MapInventoryAttributes(ctx context.Context, tenantID string,
38	attrs inventory.DeviceAttributes, update bool, passthrough bool,
39	) (inventory.DeviceAttributes, error)
40	ReverseInventoryAttributes(ctx context.Context, tenantID string,
41	attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error)
42	}
43
44	type tenantMapCache struct {
45	inventory map[string]string
46	inventoryReverse map[string]string
47	}
48
49	type mapper struct {
50	ds store.DataStore
51	cache map[string]*tenantMapCache
52	lock sync.RWMutex
53	}
54
55	func NewMapper(ds store.DataStore) Mapper {	10✔
56	return newMapper(ds)	10✔
57	}	10✔
58
59	func newMapper(ds store.DataStore) *mapper {	12✔
60	return &mapper{	12✔
61	ds: ds,	12✔
62	cache: make(map[string]*tenantMapCache),	12✔
63	lock: sync.RWMutex{},	12✔
64	}	12✔
65	}	12✔
66
67	// MapInventoryAttributes maps inventory attributes to ES fields
68	func (m *mapper) MapInventoryAttributes(ctx context.Context, tenantID string,
69	attrs inventory.DeviceAttributes, update bool, passthrough bool) (
70	inventory.DeviceAttributes, error) {	52✔
71	attributesToFieldsMap := m.lookupMapping(tenantID, attrs, false)	52✔
72	if attributesToFieldsMap == nil {	63✔
73	var mapping *model.Mapping	11✔
74	var err error	11✔
75	if update {	16✔
76	mapping, err = m.updateAndGetMapping(ctx, tenantID, attrs)	5✔
77	} else {	11✔
78	mapping, err = m.getMapping(ctx, tenantID)	6✔
79	}	6✔
80	if err != nil {	12✔
81	return nil, err	1✔
82	}	1✔
83	n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))	10✔
84	attributesToFieldsMap = attributesToFields(mapping.Inventory[:n])	10✔
85	}
86	return mapAttributes(attrs, attributesToFieldsMap, false, passthrough), nil	51✔
87	}
88
89	// ReverseInventoryAttributes looks up the inventory attribute names from the ES fields
90	func (m *mapper) ReverseInventoryAttributes(ctx context.Context, tenantID string,
91	attrs inventory.DeviceAttributes) (inventory.DeviceAttributes, error) {	35✔
92	attributesToFieldsMap := m.lookupMapping(tenantID, attrs, true)	35✔
93	if attributesToFieldsMap == nil {	38✔
94	mapping, err := m.getMapping(ctx, tenantID)	3✔
95	if err != nil {	4✔
96	return nil, err	1✔
97	}	1✔
98	n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))	2✔
99	attributesToFieldsMap = fieldsToAttributes(mapping.Inventory[:n])	2✔
100	}
101	return mapAttributes(attrs, attributesToFieldsMap, true, false), nil	34✔
102	}
103
104	func (m mapper) getMapping(ctx context.Context, tenantID string) (model.Mapping, error) {	9✔
105	mapping, err := m.ds.GetMapping(ctx, tenantID)	9✔
106	if err == nil {	17✔
107	m.cacheMapping(tenantID, mapping)	8✔
108	}	8✔
109	return mapping, err	9✔
110	}
111
112	func (m mapper) cacheMapping(tenantID string, mapping model.Mapping) {	13✔
113	cache := &tenantMapCache{	13✔
114	inventory: make(map[string]string),	13✔
115	inventoryReverse: make(map[string]string),	13✔
116	}	13✔
117	n := int(math.Min(float64(len(mapping.Inventory)), model.MaxMappingInventoryAttributes))	13✔
118	for i, attr := range mapping.Inventory[:n] {	40✔
119	attrName := fmt.Sprintf(inventoryAttributeTemplate, i+1)	27✔
120	cache.inventory[attr] = attrName	27✔
121	cache.inventoryReverse[attrName] = attr	27✔
122	}	27✔
123	m.lock.Lock()	13✔
124	m.cache[tenantID] = cache	13✔
125	m.lock.Unlock()	13✔
126	}
127
128	func (m *mapper) lookupMapping(tenantID string, attrs inventory.DeviceAttributes,
129	reverse bool) map[string]string {	87✔
130	m.lock.RLock()	87✔
131	cache, ok := m.cache[tenantID]	87✔
132	m.lock.RUnlock()	87✔
133	if ok {	163✔
134	var cacheAttributes map[string]string	76✔
135	if reverse {	109✔
136	cacheAttributes = cache.inventoryReverse	33✔
137	} else {	76✔
138	cacheAttributes = cache.inventory	43✔
139	}	43✔
140	if len(cacheAttributes) < model.MaxMappingInventoryAttributes {	152✔
141	for i := 0; i < len(attrs); i++ {	256✔
142	if shouldMapScope(attrs[i].Scope, attrs[i].Name) {	314✔
143	var key string	134✔
144	if reverse {	211✔
145	key = attrs[i].Name	77✔
146	} else {	134✔
147	key = path.Join(attrs[i].Scope, attrs[i].Name)	57✔
148	}	57✔
149	if _, ok := cacheAttributes[key]; !ok {	137✔
150	return nil	3✔
151	}	3✔
152	}
153	}
154	}
155	return cacheAttributes	73✔
156	}
157	return nil	11✔
158	}
159
160	func (m *mapper) updateAndGetMapping(ctx context.Context, tenantID string,
161	attrs inventory.DeviceAttributes) (*model.Mapping, error) {	7✔
162	inventoryMapping := make([]string, 0, len(attrs))	7✔
163	for i := 0; i < len(attrs); i++ {	26✔
164	if shouldMapScope(attrs[i].Scope, attrs[i].Name) {	37✔
165	key := path.Join(attrs[i].Scope, attrs[i].Name)	18✔
166	inventoryMapping = append(inventoryMapping, key)	18✔
167	}	18✔
168	}
169	if len(inventoryMapping) > model.MaxMappingInventoryAttributes {	7✔
170	inventoryMapping = inventoryMapping[:model.MaxMappingInventoryAttributes]	×
171	}	×
172	mapping, err := m.ds.UpdateAndGetMapping(ctx, tenantID, inventoryMapping)	7✔
173	if err != nil {	9✔
174	return nil, err	2✔
175	}	2✔
176	m.cacheMapping(tenantID, mapping)	5✔
177	return mapping, nil	5✔
178	}
179
180	func mapAttributes(attrs inventory.DeviceAttributes,
181	mapping map[string]string, reverse bool, passthrough bool) inventory.DeviceAttributes {	86✔
182	mappedAttrs := make(inventory.DeviceAttributes, 0, len(attrs))	86✔
183	for i := 0; i < len(attrs); i++ {	291✔
184	var attrName string	205✔
185	if !shouldMapScope(attrs[i].Scope, attrs[i].Name) {	255✔
186	attrName = attrs[i].Name	50✔
187	} else if reverse {	286✔
188	if name, ok := mapping[attrs[i].Name]; ok {	162✔
189	parts := strings.SplitN(name, string(os.PathSeparator), 2)	81✔
190	attrName = parts[len(parts)-1]	81✔
191	}	81✔
192	} else if name, ok := mapping[path.Join(attrs[i].Scope, attrs[i].Name)]; ok {	144✔
193	attrName = name	70✔
194	} else if passthrough {	77✔
195	attrName = attrs[i].Name	3✔
196	}	3✔
197	if attrName != "" {	409✔
198	mappedAttr := inventory.DeviceAttribute{	204✔
199	Name: attrName,	204✔
200	Scope: attrs[i].Scope,	204✔
201	Value: attrs[i].Value,	204✔
202	Description: attrs[i].Description,	204✔
203	}	204✔
204	mappedAttrs = append(mappedAttrs, mappedAttr)	204✔
205	}	204✔
206	}
207	return mappedAttrs	86✔
208	}
209
210	func attributesToFields(attrs []string) map[string]string {	11✔
211	var attributesToFields = make(map[string]string, len(attrs))	11✔
212	for i := 0; i < len(attrs); i++ {	33✔
213	attributesToFields[attrs[i]] = fmt.Sprintf(inventoryAttributeTemplate, i+1)	22✔
214	}	22✔
215	return attributesToFields	11✔
216	}
217
218	func fieldsToAttributes(attrs []string) map[string]string {	3✔
219	var fieldsToAttributes = make(map[string]string, len(attrs))	3✔
220	for i := 0; i < len(attrs); i++ {	10✔
221	fieldsToAttributes[fmt.Sprintf(inventoryAttributeTemplate, i+1)] = attrs[i]	7✔
222	}	7✔
223	return fieldsToAttributes	3✔
224	}
225
226	func shouldMapScope(scope, attribute string) bool {	404✔
227	return scope != model.ScopeSystem &&	404✔
228	!(scope == model.ScopeIdentity && attribute == model.AttrNameStatus)	404✔
229	}	404✔

mendersoftware / reporting / 1571619767

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous