714242530

Build Type

gitlab-ci

Committed by Fabio Tranchitella

Commit Message

feat: in-memory cache the mapping to reduce the number of queries to mongodb

Pull Request Pull Request #79: MEN-5598: map inventory attributes to sequential fields

Run Details

339 of 406 new or added lines in 12 files covered. (83.5%)

6 existing lines in 3 files now uncovered.

1683 of 2121 relevant lines covered (79.35%)

12.63 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.75

/app/indexer/run.go

// Copyright 2022 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package indexer

import (
        "context"
        "os"
        "os/signal"
        "time"

        "golang.org/x/sys/unix"

        "github.com/mendersoftware/go-lib-micro/config"

        "github.com/mendersoftware/reporting/client/deviceauth"
        "github.com/mendersoftware/reporting/client/inventory"
        "github.com/mendersoftware/reporting/client/nats"
        rconfig "github.com/mendersoftware/reporting/config"
        "github.com/mendersoftware/reporting/model"
        "github.com/mendersoftware/reporting/store"
)

const jobsChanSize = 1000

// InitAndRun initializes the indexer and runs it
func InitAndRun(conf config.Reader, store store.Store, ds store.DataStore, nats nats.Client) error {
        ctx, cancel := context.WithCancel(context.Background())
        defer cancel()

        invClient := inventory.NewClient(
                conf.GetString(rconfig.SettingInventoryAddr),
        )

        devClient := deviceauth.NewClient(
                conf.GetString(rconfig.SettingDeviceAuthAddr),
        )

        indexer := NewIndexer(store, ds, nats, devClient, invClient)
        jobs := make(chan *model.Job, jobsChanSize)

        err := indexer.GetJobs(ctx, jobs)
        if err != nil {
                return err
        }

        quit := make(chan os.Signal, 1)
        signal.Notify(quit, unix.SIGINT, unix.SIGTERM)

        batchSize := conf.GetInt(rconfig.SettingReindexBatchSize)
        jobsList := make([]*model.Job, batchSize)
        jobsListSize := 0

        maxTimeMs := conf.GetInt(rconfig.SettingReindexMaxTimeMsec)
        tickerTimeout := time.Duration(maxTimeMs) * time.Millisecond
        ticker := time.NewTimer(tickerTimeout)

        for {
                select {
                case <-ticker.C:
                        if jobsListSize > 0 {
                                indexer.ProcessJobs(ctx, jobsList[0:jobsListSize])
                                jobsListSize = 0
                        }
                        ticker.Reset(tickerTimeout)

                case job := <-jobs:
                        jobsList[jobsListSize] = job
                        jobsListSize++
                        if jobsListSize == batchSize {
                                indexer.ProcessJobs(ctx, jobsList[0:jobsListSize])
                                ticker.Reset(tickerTimeout)
                                jobsListSize = 0
                        }

                case <-quit:
                        return nil
                }
        }
}

1	// Copyright 2022 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package indexer
16
17	import (
18	"context"
19	"os"
20	"os/signal"
21	"time"
22
23	"golang.org/x/sys/unix"
24
25	"github.com/mendersoftware/go-lib-micro/config"
26
27	"github.com/mendersoftware/reporting/client/deviceauth"
28	"github.com/mendersoftware/reporting/client/inventory"
29	"github.com/mendersoftware/reporting/client/nats"
30	rconfig "github.com/mendersoftware/reporting/config"
31	"github.com/mendersoftware/reporting/model"
32	"github.com/mendersoftware/reporting/store"
33	)
34
35	const jobsChanSize = 1000
36
37	// InitAndRun initializes the indexer and runs it
38	func InitAndRun(conf config.Reader, store store.Store, ds store.DataStore, nats nats.Client) error {	1✔
39	ctx, cancel := context.WithCancel(context.Background())	1✔
40	defer cancel()	1✔
41		1✔
42	invClient := inventory.NewClient(	1✔
43	conf.GetString(rconfig.SettingInventoryAddr),	1✔
44	)	1✔
45		1✔
46	devClient := deviceauth.NewClient(	1✔
47	conf.GetString(rconfig.SettingDeviceAuthAddr),	1✔
48	)	1✔
49		1✔
50	indexer := NewIndexer(store, ds, nats, devClient, invClient)	1✔
51	jobs := make(chan *model.Job, jobsChanSize)	1✔
52		1✔
53	err := indexer.GetJobs(ctx, jobs)	1✔
54	if err != nil {	1✔
55	return err	×
56	}	×
57
58	quit := make(chan os.Signal, 1)	1✔
59	signal.Notify(quit, unix.SIGINT, unix.SIGTERM)	1✔
60		1✔
61	batchSize := conf.GetInt(rconfig.SettingReindexBatchSize)	1✔
62	jobsList := make([]*model.Job, batchSize)	1✔
63	jobsListSize := 0	1✔
64		1✔
65	maxTimeMs := conf.GetInt(rconfig.SettingReindexMaxTimeMsec)	1✔
66	tickerTimeout := time.Duration(maxTimeMs) * time.Millisecond	1✔
67	ticker := time.NewTimer(tickerTimeout)	1✔
68		1✔
69	for {	25✔
70	select {	24✔
71	case <-ticker.C:	23✔
72	if jobsListSize > 0 {	23✔
73	indexer.ProcessJobs(ctx, jobsList[0:jobsListSize])	×
74	jobsListSize = 0	×
75	}	×
76	ticker.Reset(tickerTimeout)	23✔
77
78	case job := <-jobs:	×
79	jobsList[jobsListSize] = job	×
80	jobsListSize++	×
81	if jobsListSize == batchSize {	×
82	indexer.ProcessJobs(ctx, jobsList[0:jobsListSize])	×
83	ticker.Reset(tickerTimeout)	×
84	jobsListSize = 0	×
85	}	×
86
UNCOV 87	case <-quit:	×
UNCOV 88	return nil	×
89	}
90	}
91	}

mendersoftware / reporting / 714242530

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous