1282862750

Committed 08 May 2024 08:09AM UTC coverage: 85.235% (-2.5%) from 87.709%

Build # 1282862750

Build Type

Pull #196

gitlab-ci

Committed by

oldgiova

Commit Message

ci: move from public to self-hosted runners

Public runners are considered not secure; moving to self-hosted runners instead.
Additionally, the docker runner tag is no longer available.

Ticket: SEC-1133
Changelog: None

Signed-off-by: Roberto Giovanardi <roberto.giovanardi@northern.tech>

Pull Request Pull Request #196: ci: move from public to self-hosted runners

Run Details

2996 of 3515 relevant lines covered (85.23%)

14.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.73

/main.go

// Copyright 2023 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package main

import (
        "context"
        "fmt"
        "net/url"
        "os"
        "strings"
        "time"

        "github.com/pkg/errors"
        "github.com/urfave/cli"

        "github.com/mendersoftware/go-lib-micro/config"
        "github.com/mendersoftware/go-lib-micro/log"
        mlog "github.com/mendersoftware/go-lib-micro/log"

        "github.com/mendersoftware/reporting/app/indexer"
        "github.com/mendersoftware/reporting/app/server"
        "github.com/mendersoftware/reporting/client/nats"
        dconfig "github.com/mendersoftware/reporting/config"
        "github.com/mendersoftware/reporting/store"
        "github.com/mendersoftware/reporting/store/mongo"
        "github.com/mendersoftware/reporting/store/opensearch"
)

const (
        opensearchMaxWaitingTime      = 300
        opensearchRetryDelayInSeconds = 1
)

func main() {
        os.Exit(doMain(os.Args))
}

func doMain(args []string) int {
        var configPath string

        app := &cli.App{
                Flags: []cli.Flag{
                        &cli.StringFlag{
                                Name: "config",
                                Usage: "Configuration `FILE`. " +
                                        "Supports JSON, TOML, YAML and HCL formatted configs.",
                                Value:       "config.yaml",
                                Destination: &configPath,
                        },
                },
                Commands: []cli.Command{
                        {
                                Name:   "server",
                                Usage:  "Run the HTTP API server",
                                Action: cmdServer,
                                Flags: []cli.Flag{
                                        &cli.BoolFlag{
                                                Name:  "automigrate",
                                                Usage: "Run database migrations before starting.",
                                        },
                                },
                        },
                        {
                                Name:   "indexer",
                                Usage:  "Run the indexer process",
                                Action: cmdIndexer,
                                Flags: []cli.Flag{
                                        &cli.BoolFlag{
                                                Name:  "automigrate",
                                                Usage: "Run database migrations before starting.",
                                        },
                                },
                        },
                        {
                                Name:   "migrate",
                                Usage:  "Run the migrations",
                                Action: cmdMigrate,
                        },
                },
        }
        app.Usage = "Reporting"
        app.Action = cmdServer

        app.Before = func(args *cli.Context) error {
                err := config.FromConfigFile(configPath, dconfig.Defaults)
                if err != nil {
                        return cli.NewExitError(
                                fmt.Sprintf("error loading configuration: %s", err),
                                1)
                }

                // Enable setting config values by environment variables
                config.Config.SetEnvPrefix("REPORTING")
                config.Config.AutomaticEnv()
                config.Config.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))

                // setup logging
                mlog.Setup(config.Config.GetBool(dconfig.SettingDebugLog))

                return nil
        }

        err := app.Run(args)
        if err != nil {
                mlog.NewEmpty().Fatal(err)
                return 1
        }
        return 0
}

func cmdServer(args *cli.Context) error {
        store, err := getStore(args)
        if err != nil {
                return err
        }
        ctx := context.Background()
        ds, err := getDatastore(args)
        if err != nil {
                return err
        }
        defer ds.Close(ctx)
        if args.Bool("automigrate") {
                nats, err := getNatsClient()
                if err != nil {
                        return err
                }
                ctx := context.Background()
                err = migrate(ctx, store, ds, nats)
                nats.Close()
                if err != nil {
                        return err
                }
        }
        return server.InitAndRun(config.Config, store, ds)
}

func getNatsClient() (nats.Client, error) {
        natsURI := config.Config.GetString(dconfig.SettingNatsURI)
        nats, err := nats.NewClient(natsURI)
        if err != nil {
                return nil, errors.Wrap(err, "failed to connect to nats")
        }
        return nats, nil
}

func cmdIndexer(args *cli.Context) error {
        store, err := getStore(args)
        if err != nil {
                return err
        }
        nats, err := getNatsClient()
        if err != nil {
                return err
        }
        defer nats.Close()
        ctx := context.Background()
        ds, err := getDatastore(args)
        if err != nil {
                return err
        }
        defer ds.Close(ctx)
        if args.Bool("automigrate") {
                err = migrate(ctx, store, ds, nats)
                if err != nil {
                        return err
                }

        }
        return indexer.InitAndRun(config.Config, store, ds, nats)
}

func cmdMigrate(args *cli.Context) error {
        ctx := context.Background()
        store, err := getStore(args)
        if err != nil {
                return err
        }
        ds, err := getDatastore(args)
        if err != nil {
                return err
        }
        nats, err := getNatsClient()
        if err != nil {
                return err
        }
        return migrate(ctx, store, ds, nats)
}

func migrate(ctx context.Context, store store.Store, ds store.DataStore, nats nats.Client) error {
        err := store.Migrate(ctx)
        if err != nil {
                return err
        }
        err = ds.Migrate(ctx, mongo.DbVersion, true)
        if err != nil {
                return err
        }
        dur := config.Config.GetString(dconfig.SettingNatsSubscriberDurable)
        stream := config.Config.GetString(dconfig.SettingNatsStreamName)
        topic := config.Config.GetString(dconfig.SettingNatsSubscriberTopic)
        sub := stream + "." + topic
        return nats.Migrate(ctx, sub, dur, true)
}

func getStore(args *cli.Context) (store.Store, error) {
        addresses := config.Config.GetStringSlice(dconfig.SettingOpenSearchAddresses)
        devicesIndexName := config.Config.GetString(dconfig.SettingOpenSearchDevicesIndexName)
        devicesIndexShards := config.Config.GetInt(dconfig.SettingOpenSearchDevicesIndexShards)
        devicesIndexReplicas := config.Config.GetInt(
                dconfig.SettingOpenSearchDevicesIndexReplicas)
        deploymentsIndexName := config.Config.GetString(dconfig.SettingOpenSearchDeploymentsIndexName)
        deploymentsIndexShards := config.Config.GetInt(dconfig.SettingOpenSearchDeploymentsIndexShards)
        deploymentsIndexReplicas := config.Config.GetInt(
                dconfig.SettingOpenSearchDeploymentsIndexReplicas)
        store, err := opensearch.NewStore(
                opensearch.WithServerAddresses(addresses),
                opensearch.WithDevicesIndexName(devicesIndexName),
                opensearch.WithDevicesIndexShards(devicesIndexShards),
                opensearch.WithDevicesIndexReplicas(devicesIndexReplicas),
                opensearch.WithDeploymentsIndexName(deploymentsIndexName),
                opensearch.WithDeploymentsIndexShards(deploymentsIndexShards),
                opensearch.WithDeploymentsIndexReplicas(deploymentsIndexReplicas),
        )
        if err != nil {
                return nil, err
        }
        ctx := context.Background()
        l := log.FromContext(ctx)
        for i := 0; i < opensearchMaxWaitingTime; i++ {
                err = store.Ping(ctx)
                if err == nil {
                        break
                }
                l.Warn(err)
                time.Sleep(opensearchRetryDelayInSeconds * time.Second)
        }
        if err != nil {
                l.Error(err)
                return nil, err
        }
        l.Info("successfully connected to OpenSearch")
        return store, nil
}

func getDatastore(args *cli.Context) (store.DataStore, error) {
        mgoURL, err := url.Parse(config.Config.GetString(dconfig.SettingMongo))
        if err != nil {
                return nil, err
        }

        storeConfig := mongo.MongoStoreConfig{
                MongoURL:      mgoURL,
                SSL:           config.Config.GetBool(dconfig.SettingDbSSL),
                SSLSkipVerify: config.Config.GetBool(dconfig.SettingDbSSLSkipVerify),
                Username:      config.Config.GetString(dconfig.SettingDbUsername),
                Password:      config.Config.GetString(dconfig.SettingDbPassword),
                DbName:        mongo.DbName,
        }

        return mongo.NewMongoStore(context.Background(), storeConfig)
}

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package main
16
17	import (
18	"context"
19	"fmt"
20	"net/url"
21	"os"
22	"strings"
23	"time"
24
25	"github.com/pkg/errors"
26	"github.com/urfave/cli"
27
28	"github.com/mendersoftware/go-lib-micro/config"
29	"github.com/mendersoftware/go-lib-micro/log"
30	mlog "github.com/mendersoftware/go-lib-micro/log"
31
32	"github.com/mendersoftware/reporting/app/indexer"
33	"github.com/mendersoftware/reporting/app/server"
34	"github.com/mendersoftware/reporting/client/nats"
35	dconfig "github.com/mendersoftware/reporting/config"
36	"github.com/mendersoftware/reporting/store"
37	"github.com/mendersoftware/reporting/store/mongo"
38	"github.com/mendersoftware/reporting/store/opensearch"
39	)
40
41	const (
42	opensearchMaxWaitingTime = 300
43	opensearchRetryDelayInSeconds = 1
44	)
45
46	func main() {	×
47	os.Exit(doMain(os.Args))	×
48	}	×
49
50	func doMain(args []string) int {	3✔
51	var configPath string	3✔
52		3✔
53	app := &cli.App{	3✔
54	Flags: []cli.Flag{	3✔
55	&cli.StringFlag{	3✔
56	Name: "config",	3✔
57	Usage: "Configuration `FILE`. " +	3✔
58	"Supports JSON, TOML, YAML and HCL formatted configs.",	3✔
59	Value: "config.yaml",	3✔
60	Destination: &configPath,	3✔
61	},	3✔
62	},	3✔
63	Commands: []cli.Command{	3✔
64	{	3✔
65	Name: "server",	3✔
66	Usage: "Run the HTTP API server",	3✔
67	Action: cmdServer,	3✔
68	Flags: []cli.Flag{	3✔
69	&cli.BoolFlag{	3✔
70	Name: "automigrate",	3✔
71	Usage: "Run database migrations before starting.",	3✔
72	},	3✔
73	},	3✔
74	},	3✔
75	{	3✔
76	Name: "indexer",	3✔
77	Usage: "Run the indexer process",	3✔
78	Action: cmdIndexer,	3✔
79	Flags: []cli.Flag{	3✔
80	&cli.BoolFlag{	3✔
81	Name: "automigrate",	3✔
82	Usage: "Run database migrations before starting.",	3✔
83	},	3✔
84	},	3✔
85	},	3✔
86	{	3✔
87	Name: "migrate",	3✔
88	Usage: "Run the migrations",	3✔
89	Action: cmdMigrate,	3✔
90	},	3✔
91	},	3✔
92	}	3✔
93	app.Usage = "Reporting"	3✔
94	app.Action = cmdServer	3✔
95		3✔
96	app.Before = func(args *cli.Context) error {	6✔
97	err := config.FromConfigFile(configPath, dconfig.Defaults)	3✔
98	if err != nil {	3✔
99	return cli.NewExitError(	×
100	fmt.Sprintf("error loading configuration: %s", err),	×
101	1)	×
102	}	×
103
104	// Enable setting config values by environment variables
105	config.Config.SetEnvPrefix("REPORTING")	3✔
106	config.Config.AutomaticEnv()	3✔
107	config.Config.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))	3✔
108		3✔
109	// setup logging	3✔
110	mlog.Setup(config.Config.GetBool(dconfig.SettingDebugLog))	3✔
111		3✔
112	return nil	3✔
113	}
114
115	err := app.Run(args)	3✔
116	if err != nil {	4✔
117	mlog.NewEmpty().Fatal(err)	1✔
118	return 1	1✔
119	}	1✔
120	return 0	1✔
121	}
122
123	func cmdServer(args *cli.Context) error {	1✔
124	store, err := getStore(args)	1✔
125	if err != nil {	1✔
126	return err	×
127	}	×
128	ctx := context.Background()	1✔
129	ds, err := getDatastore(args)	1✔
130	if err != nil {	1✔
131	return err	×
132	}	×
133	defer ds.Close(ctx)	1✔
134	if args.Bool("automigrate") {	2✔
135	nats, err := getNatsClient()	1✔
136	if err != nil {	1✔
137	return err	×
138	}	×
139	ctx := context.Background()	1✔
140	err = migrate(ctx, store, ds, nats)	1✔
141	nats.Close()	1✔
142	if err != nil {	1✔
143	return err	×
144	}	×
145	}
146	return server.InitAndRun(config.Config, store, ds)	1✔
147	}
148
149	func getNatsClient() (nats.Client, error) {	3✔
150	natsURI := config.Config.GetString(dconfig.SettingNatsURI)	3✔
151	nats, err := nats.NewClient(natsURI)	3✔
152	if err != nil {	3✔
153	return nil, errors.Wrap(err, "failed to connect to nats")	×
154	}	×
155	return nats, nil	3✔
156	}
157
158	func cmdIndexer(args *cli.Context) error {	2✔
159	store, err := getStore(args)	2✔
160	if err != nil {	2✔
161	return err	×
162	}	×
163	nats, err := getNatsClient()	2✔
164	if err != nil {	2✔
165	return err	×
166	}	×
167	defer nats.Close()	2✔
168	ctx := context.Background()	2✔
169	ds, err := getDatastore(args)	2✔
170	if err != nil {	2✔
171	return err	×
172	}	×
173	defer ds.Close(ctx)	2✔
174	if args.Bool("automigrate") {	4✔
175	err = migrate(ctx, store, ds, nats)	2✔
176	if err != nil {	3✔
177	return err	1✔
178	}	1✔
179
180	}
181	return indexer.InitAndRun(config.Config, store, ds, nats)	1✔
182	}
183
184	func cmdMigrate(args *cli.Context) error {	×
185	ctx := context.Background()	×
186	store, err := getStore(args)	×
187	if err != nil {	×
188	return err	×
189	}	×
190	ds, err := getDatastore(args)	×
191	if err != nil {	×
192	return err	×
193	}	×
194	nats, err := getNatsClient()	×
195	if err != nil {	×
196	return err	×
197	}	×
198	return migrate(ctx, store, ds, nats)	×
199	}
200
201	func migrate(ctx context.Context, store store.Store, ds store.DataStore, nats nats.Client) error {	3✔
202	err := store.Migrate(ctx)	3✔
203	if err != nil {	4✔
204	return err	1✔
205	}	1✔
206	err = ds.Migrate(ctx, mongo.DbVersion, true)	2✔
207	if err != nil {	2✔
208	return err	×
209	}	×
210	dur := config.Config.GetString(dconfig.SettingNatsSubscriberDurable)	2✔
211	stream := config.Config.GetString(dconfig.SettingNatsStreamName)	2✔
212	topic := config.Config.GetString(dconfig.SettingNatsSubscriberTopic)	2✔
213	sub := stream + "." + topic	2✔
214	return nats.Migrate(ctx, sub, dur, true)	2✔
215	}
216
217	func getStore(args *cli.Context) (store.Store, error) {	3✔
218	addresses := config.Config.GetStringSlice(dconfig.SettingOpenSearchAddresses)	3✔
219	devicesIndexName := config.Config.GetString(dconfig.SettingOpenSearchDevicesIndexName)	3✔
220	devicesIndexShards := config.Config.GetInt(dconfig.SettingOpenSearchDevicesIndexShards)	3✔
221	devicesIndexReplicas := config.Config.GetInt(	3✔
222	dconfig.SettingOpenSearchDevicesIndexReplicas)	3✔
223	deploymentsIndexName := config.Config.GetString(dconfig.SettingOpenSearchDeploymentsIndexName)	3✔
224	deploymentsIndexShards := config.Config.GetInt(dconfig.SettingOpenSearchDeploymentsIndexShards)	3✔
225	deploymentsIndexReplicas := config.Config.GetInt(	3✔
226	dconfig.SettingOpenSearchDeploymentsIndexReplicas)	3✔
227	store, err := opensearch.NewStore(	3✔
228	opensearch.WithServerAddresses(addresses),	3✔
229	opensearch.WithDevicesIndexName(devicesIndexName),	3✔
230	opensearch.WithDevicesIndexShards(devicesIndexShards),	3✔
231	opensearch.WithDevicesIndexReplicas(devicesIndexReplicas),	3✔
232	opensearch.WithDeploymentsIndexName(deploymentsIndexName),	3✔
233	opensearch.WithDeploymentsIndexShards(deploymentsIndexShards),	3✔
234	opensearch.WithDeploymentsIndexReplicas(deploymentsIndexReplicas),	3✔
235	)	3✔
236	if err != nil {	3✔
237	return nil, err	×
238	}	×
239	ctx := context.Background()	3✔
240	l := log.FromContext(ctx)	3✔
241	for i := 0; i < opensearchMaxWaitingTime; i++ {	24✔
242	err = store.Ping(ctx)	21✔
243	if err == nil {	24✔
244	break	3✔
245	}
246	l.Warn(err)	18✔
247	time.Sleep(opensearchRetryDelayInSeconds * time.Second)	18✔
248	}
249	if err != nil {	3✔
250	l.Error(err)	×
251	return nil, err	×
252	}	×
253	l.Info("successfully connected to OpenSearch")	3✔
254	return store, nil	3✔
255	}
256
257	func getDatastore(args *cli.Context) (store.DataStore, error) {	3✔
258	mgoURL, err := url.Parse(config.Config.GetString(dconfig.SettingMongo))	3✔
259	if err != nil {	3✔
260	return nil, err	×
261	}	×
262
263	storeConfig := mongo.MongoStoreConfig{	3✔
264	MongoURL: mgoURL,	3✔
265	SSL: config.Config.GetBool(dconfig.SettingDbSSL),	3✔
266	SSLSkipVerify: config.Config.GetBool(dconfig.SettingDbSSLSkipVerify),	3✔
267	Username: config.Config.GetString(dconfig.SettingDbUsername),	3✔
268	Password: config.Config.GetString(dconfig.SettingDbPassword),	3✔
269	DbName: mongo.DbName,	3✔
270	}	3✔
271		3✔
272	return mongo.NewMongoStore(context.Background(), storeConfig)	3✔
273	}

mendersoftware / reporting / 1282862750

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous