728455154

Build Type

gitlab-ci

Committed by Fabio Tranchitella

Commit Message

ci(tests): device inventory data aggregation acceptance tests

Pull Request Pull Request #83: feat: device inventory data aggregation

Run Details

198 of 234 new or added lines in 6 files covered. (84.62%)

2 existing lines in 1 file now uncovered.

2090 of 2464 relevant lines covered (84.82%)

16.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.82

/main.go

// Copyright 2022 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package main

import (
        "context"
        "fmt"
        "net/url"
        "os"
        "strings"
        "time"

        "github.com/pkg/errors"
        "github.com/urfave/cli"

        "github.com/mendersoftware/go-lib-micro/config"
        "github.com/mendersoftware/go-lib-micro/log"
        mlog "github.com/mendersoftware/go-lib-micro/log"

        "github.com/mendersoftware/reporting/app/indexer"
        "github.com/mendersoftware/reporting/app/server"
        "github.com/mendersoftware/reporting/client/nats"
        dconfig "github.com/mendersoftware/reporting/config"
        "github.com/mendersoftware/reporting/store"
        "github.com/mendersoftware/reporting/store/mongo"
        "github.com/mendersoftware/reporting/store/opensearch"
)

const (
        opensearchMaxWaitingTime      = 300
        opensearchRetryDelayInSeconds = 1
)

func main() {
        os.Exit(doMain(os.Args))
}

func doMain(args []string) int {
        var configPath string

        app := &cli.App{
                Flags: []cli.Flag{
                        &cli.StringFlag{
                                Name: "config",
                                Usage: "Configuration `FILE`. " +
                                        "Supports JSON, TOML, YAML and HCL formatted configs.",
                                Value:       "config.yaml",
                                Destination: &configPath,
                        },
                },
                Commands: []cli.Command{
                        {
                                Name:   "server",
                                Usage:  "Run the HTTP API server",
                                Action: cmdServer,
                                Flags: []cli.Flag{
                                        &cli.BoolFlag{
                                                Name:  "automigrate",
                                                Usage: "Run database migrations before starting.",
                                        },
                                },
                        },
                        {
                                Name:   "indexer",
                                Usage:  "Run the indexer process",
                                Action: cmdIndexer,
                                Flags: []cli.Flag{
                                        &cli.BoolFlag{
                                                Name:  "automigrate",
                                                Usage: "Run database migrations before starting.",
                                        },
                                },
                        },
                        {
                                Name:   "migrate",
                                Usage:  "Run the migrations",
                                Action: cmdMigrate,
                        },
                },
        }
        app.Usage = "Reporting"
        app.Action = cmdServer

        app.Before = func(args *cli.Context) error {
                err := config.FromConfigFile(configPath, dconfig.Defaults)
                if err != nil {
                        return cli.NewExitError(
                                fmt.Sprintf("error loading configuration: %s", err),
                                1)
                }

                // Enable setting config values by environment variables
                config.Config.SetEnvPrefix("REPORTING")
                config.Config.AutomaticEnv()
                config.Config.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))

                // setup logging
                mlog.Setup(config.Config.GetBool(dconfig.SettingDebugLog))

                return nil
        }

        err := app.Run(args)
        if err != nil {
                mlog.NewEmpty().Fatal(err)
                return 1
        }
        return 0
}

func cmdServer(args *cli.Context) error {
        store, err := getStore(args)
        if err != nil {
                return err
        }
        if args.Bool("automigrate") {
                ctx := context.Background()
                err := store.Migrate(ctx)
                if err != nil {
                        return err
                }
        }
        ctx := context.Background()
        ds, err := getDatastore(args)
        if err != nil {
                return err
        }
        defer ds.Close(ctx)
        err = ds.Migrate(ctx, mongo.DbVersion, args.Bool("automigrate"))
        if err != nil {
                return err
        }
        return server.InitAndRun(config.Config, store, ds)
}

func getNatsClient() (nats.Client, error) {
        natsURI := config.Config.GetString(dconfig.SettingNatsURI)
        nats, err := nats.NewClientWithDefaults(natsURI)
        if err != nil {
                return nil, errors.Wrap(err, "failed to connect to nats")
        }

        streamName := config.Config.GetString(dconfig.SettingNatsStreamName)
        nats = nats.WithStreamName(streamName)
        if err != nil {
                return nil, err
        }
        return nats, nil
}

func cmdIndexer(args *cli.Context) error {
        store, err := getStore(args)
        if err != nil {
                return err
        }
        if args.Bool("automigrate") {
                ctx := context.Background()
                err := store.Migrate(ctx)
                if err != nil {
                        return err
                }
        }
        nats, err := getNatsClient()
        if err != nil {
                return err
        }
        defer nats.Close()
        ctx := context.Background()
        ds, err := getDatastore(args)
        if err != nil {
                return err
        }
        defer ds.Close(ctx)
        err = ds.Migrate(ctx, mongo.DbVersion, args.Bool("automigrate"))
        if err != nil {
                return err
        }
        return indexer.InitAndRun(config.Config, store, ds, nats)
}

func cmdMigrate(args *cli.Context) error {
        store, err := getStore(args)
        if err != nil {
                return err
        }
        ctx := context.Background()
        err = store.Migrate(ctx)
        if err != nil {
                return err
        }
        ds, err := getDatastore(args)
        if err != nil {
                return err
        }
        defer ds.Close(ctx)
        return ds.Migrate(ctx, mongo.DbVersion, true)
}

func getStore(args *cli.Context) (store.Store, error) {
        addresses := config.Config.GetStringSlice(dconfig.SettingOpenSearchAddresses)
        devicesIndexName := config.Config.GetString(dconfig.SettingOpenSearchDevicesIndexName)
        deviceesIndexShards := config.Config.GetInt(dconfig.SettingOpenSearchDevicesIndexShards)
        deviceesIndexReplicas := config.Config.GetInt(
                dconfig.SettingOpenSearchDevicesIndexReplicas)
        store, err := opensearch.NewStore(
                opensearch.WithServerAddresses(addresses),
                opensearch.WithDevicesIndexName(devicesIndexName),
                opensearch.WithDevicesIndexShards(deviceesIndexShards),
                opensearch.WithDevicesIndexReplicas(deviceesIndexReplicas),
        )
        if err != nil {
                return nil, err
        }
        ctx := context.Background()
        l := log.FromContext(ctx)
        for i := 0; i < opensearchMaxWaitingTime; i++ {
                err = store.Ping(ctx)
                if err == nil {
                        break
                }
                l.Warn(err)
                time.Sleep(opensearchRetryDelayInSeconds * time.Second)
        }
        if err != nil {
                l.Error(err)
                return nil, err
        }
        l.Info("successfully connected to OpenSearch")
        return store, nil
}

func getDatastore(args *cli.Context) (store.DataStore, error) {
        mgoURL, err := url.Parse(config.Config.GetString(dconfig.SettingMongo))
        if err != nil {
                return nil, err
        }

        storeConfig := mongo.MongoStoreConfig{
                MongoURL:      mgoURL,
                SSL:           config.Config.GetBool(dconfig.SettingDbSSL),
                SSLSkipVerify: config.Config.GetBool(dconfig.SettingDbSSLSkipVerify),
                Username:      config.Config.GetString(dconfig.SettingDbUsername),
                Password:      config.Config.GetString(dconfig.SettingDbPassword),
                DbName:        mongo.DbName,
        }

        return mongo.NewMongoStore(context.Background(), storeConfig)
}

1	// Copyright 2022 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package main
16
17	import (
18	"context"
19	"fmt"
20	"net/url"
21	"os"
22	"strings"
23	"time"
24
25	"github.com/pkg/errors"
26	"github.com/urfave/cli"
27
28	"github.com/mendersoftware/go-lib-micro/config"
29	"github.com/mendersoftware/go-lib-micro/log"
30	mlog "github.com/mendersoftware/go-lib-micro/log"
31
32	"github.com/mendersoftware/reporting/app/indexer"
33	"github.com/mendersoftware/reporting/app/server"
34	"github.com/mendersoftware/reporting/client/nats"
35	dconfig "github.com/mendersoftware/reporting/config"
36	"github.com/mendersoftware/reporting/store"
37	"github.com/mendersoftware/reporting/store/mongo"
38	"github.com/mendersoftware/reporting/store/opensearch"
39	)
40
41	const (
42	opensearchMaxWaitingTime = 300
43	opensearchRetryDelayInSeconds = 1
44	)
45
46	func main() {	×
47	os.Exit(doMain(os.Args))	×
48	}	×
49
50	func doMain(args []string) int {	3✔
51	var configPath string	3✔
52		3✔
53	app := &cli.App{	3✔
54	Flags: []cli.Flag{	3✔
55	&cli.StringFlag{	3✔
56	Name: "config",	3✔
57	Usage: "Configuration `FILE`. " +	3✔
58	"Supports JSON, TOML, YAML and HCL formatted configs.",	3✔
59	Value: "config.yaml",	3✔
60	Destination: &configPath,	3✔
61	},	3✔
62	},	3✔
63	Commands: []cli.Command{	3✔
64	{	3✔
65	Name: "server",	3✔
66	Usage: "Run the HTTP API server",	3✔
67	Action: cmdServer,	3✔
68	Flags: []cli.Flag{	3✔
69	&cli.BoolFlag{	3✔
70	Name: "automigrate",	3✔
71	Usage: "Run database migrations before starting.",	3✔
72	},	3✔
73	},	3✔
74	},	3✔
75	{	3✔
76	Name: "indexer",	3✔
77	Usage: "Run the indexer process",	3✔
78	Action: cmdIndexer,	3✔
79	Flags: []cli.Flag{	3✔
80	&cli.BoolFlag{	3✔
81	Name: "automigrate",	3✔
82	Usage: "Run database migrations before starting.",	3✔
83	},	3✔
84	},	3✔
85	},	3✔
86	{	3✔
87	Name: "migrate",	3✔
88	Usage: "Run the migrations",	3✔
89	Action: cmdMigrate,	3✔
90	},	3✔
91	},	3✔
92	}	3✔
93	app.Usage = "Reporting"	3✔
94	app.Action = cmdServer	3✔
95		3✔
96	app.Before = func(args *cli.Context) error {	6✔
97	err := config.FromConfigFile(configPath, dconfig.Defaults)	3✔
98	if err != nil {	3✔
99	return cli.NewExitError(	×
100	fmt.Sprintf("error loading configuration: %s", err),	×
101	1)	×
102	}	×
103
104	// Enable setting config values by environment variables
105	config.Config.SetEnvPrefix("REPORTING")	3✔
106	config.Config.AutomaticEnv()	3✔
107	config.Config.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))	3✔
108		3✔
109	// setup logging	3✔
110	mlog.Setup(config.Config.GetBool(dconfig.SettingDebugLog))	3✔
111		3✔
112	return nil	3✔
113	}
114
115	err := app.Run(args)	3✔
116	if err != nil {	4✔
117	mlog.NewEmpty().Fatal(err)	1✔
118	return 1	1✔
119	}	1✔
120	return 0	1✔
121	}
122
123	func cmdServer(args *cli.Context) error {	2✔
124	store, err := getStore(args)	2✔
125	if err != nil {	2✔
126	return err	×
127	}	×
128	if args.Bool("automigrate") {	4✔
129	ctx := context.Background()	2✔
130	err := store.Migrate(ctx)	2✔
131	if err != nil {	3✔
132	return err	1✔
133	}	1✔
134	}
135	ctx := context.Background()	1✔
136	ds, err := getDatastore(args)	1✔
137	if err != nil {	1✔
138	return err	×
139	}	×
140	defer ds.Close(ctx)	1✔
141	err = ds.Migrate(ctx, mongo.DbVersion, args.Bool("automigrate"))	1✔
142	if err != nil {	1✔
143	return err	×
144	}	×
145	return server.InitAndRun(config.Config, store, ds)	1✔
146	}
147
148	func getNatsClient() (nats.Client, error) {	1✔
149	natsURI := config.Config.GetString(dconfig.SettingNatsURI)	1✔
150	nats, err := nats.NewClientWithDefaults(natsURI)	1✔
151	if err != nil {	1✔
152	return nil, errors.Wrap(err, "failed to connect to nats")	×
153	}	×
154
155	streamName := config.Config.GetString(dconfig.SettingNatsStreamName)	1✔
156	nats = nats.WithStreamName(streamName)	1✔
157	if err != nil {	1✔
158	return nil, err	×
159	}	×
160	return nats, nil	1✔
161	}
162
163	func cmdIndexer(args *cli.Context) error {	1✔
164	store, err := getStore(args)	1✔
165	if err != nil {	1✔
166	return err	×
167	}	×
168	if args.Bool("automigrate") {	2✔
169	ctx := context.Background()	1✔
170	err := store.Migrate(ctx)	1✔
171	if err != nil {	1✔
UNCOV 172	return err	×
UNCOV 173	}	×
174	}
175	nats, err := getNatsClient()	1✔
176	if err != nil {	1✔
177	return err	×
178	}	×
179	defer nats.Close()	1✔
180	ctx := context.Background()	1✔
181	ds, err := getDatastore(args)	1✔
182	if err != nil {	1✔
183	return err	×
184	}	×
185	defer ds.Close(ctx)	1✔
186	err = ds.Migrate(ctx, mongo.DbVersion, args.Bool("automigrate"))	1✔
187	if err != nil {	1✔
188	return err	×
189	}	×
190	return indexer.InitAndRun(config.Config, store, ds, nats)	1✔
191	}
192
193	func cmdMigrate(args *cli.Context) error {	×
194	store, err := getStore(args)	×
195	if err != nil {	×
196	return err	×
197	}	×
198	ctx := context.Background()	×
199	err = store.Migrate(ctx)	×
200	if err != nil {	×
201	return err	×
202	}	×
203	ds, err := getDatastore(args)	×
204	if err != nil {	×
205	return err	×
206	}	×
207	defer ds.Close(ctx)	×
208	return ds.Migrate(ctx, mongo.DbVersion, true)	×
209	}
210
211	func getStore(args *cli.Context) (store.Store, error) {	3✔
212	addresses := config.Config.GetStringSlice(dconfig.SettingOpenSearchAddresses)	3✔
213	devicesIndexName := config.Config.GetString(dconfig.SettingOpenSearchDevicesIndexName)	3✔
214	deviceesIndexShards := config.Config.GetInt(dconfig.SettingOpenSearchDevicesIndexShards)	3✔
215	deviceesIndexReplicas := config.Config.GetInt(	3✔
216	dconfig.SettingOpenSearchDevicesIndexReplicas)	3✔
217	store, err := opensearch.NewStore(	3✔
218	opensearch.WithServerAddresses(addresses),	3✔
219	opensearch.WithDevicesIndexName(devicesIndexName),	3✔
220	opensearch.WithDevicesIndexShards(deviceesIndexShards),	3✔
221	opensearch.WithDevicesIndexReplicas(deviceesIndexReplicas),	3✔
222	)	3✔
223	if err != nil {	3✔
224	return nil, err	×
225	}	×
226	ctx := context.Background()	3✔
227	l := log.FromContext(ctx)	3✔
228	for i := 0; i < opensearchMaxWaitingTime; i++ {	124✔
229	err = store.Ping(ctx)	121✔
230	if err == nil {	124✔
231	break	3✔
232	}
233	l.Warn(err)	118✔
234	time.Sleep(opensearchRetryDelayInSeconds * time.Second)	118✔
235	}
236	if err != nil {	3✔
237	l.Error(err)	×
238	return nil, err	×
239	}	×
240	l.Info("successfully connected to OpenSearch")	3✔
241	return store, nil	3✔
242	}
243
244	func getDatastore(args *cli.Context) (store.DataStore, error) {	2✔
245	mgoURL, err := url.Parse(config.Config.GetString(dconfig.SettingMongo))	2✔
246	if err != nil {	2✔
247	return nil, err	×
248	}	×
249
250	storeConfig := mongo.MongoStoreConfig{	2✔
251	MongoURL: mgoURL,	2✔
252	SSL: config.Config.GetBool(dconfig.SettingDbSSL),	2✔
253	SSLSkipVerify: config.Config.GetBool(dconfig.SettingDbSSLSkipVerify),	2✔
254	Username: config.Config.GetString(dconfig.SettingDbUsername),	2✔
255	Password: config.Config.GetString(dconfig.SettingDbPassword),	2✔
256	DbName: mongo.DbName,	2✔
257	}	2✔
258		2✔
259	return mongo.NewMongoStore(context.Background(), storeConfig)	2✔
260	}

mendersoftware / reporting / 728455154

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous