1510130353

Committed 24 Oct 2024 07:45AM UTC coverage: 72.867% (-0.02%) from 72.887%

Build # 1510130353

Build Type

Pull #133

gitlab-ci

Committed by

mzedel

Commit Message

chore(e2e-tests): instructed auditlog terminal test to provide more context in case of failure

Signed-off-by: Manuel Zedel <manuel.zedel@northern.tech>

Pull Request Pull Request #133: debug e2e auditlogs failure

Run Details

4420 of 6386 branches covered (69.21%)

Branch coverage included in aggregate %.

42649 of 58210 relevant lines covered (73.27%)

28.52 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.25

/backend/services/workflows/main.go

// Copyright 2023 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package main

import (
        "context"
        "fmt"
        "log"
        "os"
        "strings"

        "github.com/pkg/errors"
        "github.com/urfave/cli"

        "github.com/mendersoftware/mender-server/pkg/config"

        "github.com/mendersoftware/mender-server/services/workflows/app/server"
        "github.com/mendersoftware/mender-server/services/workflows/app/worker"
        "github.com/mendersoftware/mender-server/services/workflows/client/nats"
        dconfig "github.com/mendersoftware/mender-server/services/workflows/config"
        "github.com/mendersoftware/mender-server/services/workflows/model"
        store "github.com/mendersoftware/mender-server/services/workflows/store/mongo"
)

func main() {
        doMain(os.Args)
}

func doMain(args []string) {
        var configPath string

        app := &cli.App{
                Flags: []cli.Flag{
                        &cli.StringFlag{
                                Name: "config",
                                Usage: "Configuration `FILE`." +
                                        " Supports JSON, TOML, YAML and HCL formatted configs.",
                                Destination: &configPath,
                        },
                },
                Commands: []cli.Command{
                        {
                                Name:   "server",
                                Usage:  "Run the HTTP API server",
                                Action: cmdServer,
                                Flags: []cli.Flag{
                                        &cli.BoolFlag{
                                                Name:  "automigrate",
                                                Usage: "Run database migrations before starting.",
                                        },
                                },
                        },
                        {
                                Name:   "worker",
                                Usage:  "Run the worker process",
                                Action: cmdWorker,
                                Flags: []cli.Flag{
                                        &cli.BoolFlag{
                                                Name:  "automigrate",
                                                Usage: "Run database migrations before starting.",
                                        },
                                        &cli.StringFlag{
                                                Name:  "workflows",
                                                Usage: "Comma-separated list of workflows executed by this worker",
                                        },
                                        &cli.StringFlag{
                                                Name:  "excluded-workflows",
                                                Usage: "Comma-separated list of workflows NOT executed by this worker",
                                        },
                                },
                        },
                        {
                                Name:   "migrate",
                                Usage:  "Run the migrations",
                                Action: cmdMigrate,
                                Flags: []cli.Flag{
                                        cli.BoolFlag{
                                                Name:   "skip-nats",
                                                Usage:  "Skip migrating the NATS Jetstream configuration",
                                                EnvVar: "WORKFLOWS_MIGRATION_SKIP_NATS",
                                        },
                                        cli.BoolFlag{
                                                Name:   "skip-database",
                                                Usage:  "Skip migrating the database",
                                                EnvVar: "WORKFLOWS_MIGRATION_SKIP_DATABASE",
                                        },
                                },
                        },
                        {
                                Name:   "list-jobs",
                                Usage:  "List jobs",
                                Action: cmdListJobs,
                                Flags: []cli.Flag{
                                        cli.Int64Flag{
                                                Name:  "page",
                                                Usage: "page number to show",
                                        },
                                        cli.Int64Flag{
                                                Name:  "perPage",
                                                Usage: "number of results per page",
                                        },
                                },
                        },
                },
        }
        app.Usage = "Workflows"
        app.Action = cmdServer

        app.Before = func(args *cli.Context) error {
                err := config.FromConfigFile(configPath, dconfig.Defaults)
                if err != nil {
                        return cli.NewExitError(
                                fmt.Sprintf("error loading configuration: %s", err),
                                1)
                }

                // Enable setting config values by environment variables
                config.Config.SetEnvPrefix("WORKFLOWS")
                config.Config.AutomaticEnv()
                config.Config.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))

                return nil
        }

        err := app.Run(args)
        if err != nil {
                log.Fatal(err)
        }
}

func getNatsClient() (nats.Client, error) {
        natsURI := config.Config.GetString(dconfig.SettingNatsURI)
        streamName := config.Config.GetString(dconfig.SettingNatsStreamName)
        nats, err := nats.NewClientWithDefaults(natsURI, streamName)
        if err != nil {
                return nil, errors.Wrap(err, "failed to connect to nats")
        }
        return nats, err
}

func initJetstream(nc nats.Client, producer, upsert bool) (err error) {
        durableName := config.Config.GetString(dconfig.SettingNatsSubscriberDurable)
        if producer {
                err = nc.JetStreamCreateStream(nc.StreamName())
                if err != nil {
                        return err
                }
        } else {
                var cfg nats.ConsumerConfig
                cfg, err = dconfig.GetNatsConsumerConfig(config.Config)
                if err != nil {
                        return err
                }
                err = nc.CreateConsumer(durableName, upsert, cfg)
        }
        return err
}

func cmdServer(args *cli.Context) error {
        dataStore, err := store.SetupDataStore(args.Bool("automigrate"))
        if err != nil {
                return err
        }
        defer dataStore.Close()

        nats, err := getNatsClient()
        if err != nil {
                return err
        }
        defer nats.Close()

        if err = initJetstream(nats, true, args.Bool("automigrate")); err != nil {
                return errors.WithMessage(err, "failed to apply Jetstream migrations")
        }

        return server.InitAndRun(config.Config, dataStore, nats)
}

func cmdWorker(args *cli.Context) error {
        dataStore, err := store.SetupDataStore(args.Bool("automigrate"))
        if err != nil {
                return err
        }
        defer dataStore.Close()

        nats, err := getNatsClient()
        if err != nil {
                return err
        }
        defer nats.Close()

        if err = initJetstream(nats, false, args.Bool("automigrate")); err != nil {
                return errors.WithMessage(err, "failed to apply Jetstream consumer migrations")
        }

        var included, excluded []string

        includedWorkflows := args.String("workflows")
        if includedWorkflows != "" {
                included = strings.Split(includedWorkflows, ",")
        }

        excludedWorkflows := args.String("excluded-workflows")
        if excludedWorkflows != "" {
                excluded = strings.Split(excludedWorkflows, ",")
        }

        workflows := worker.Workflows{
                Included: included,
                Excluded: excluded,
        }
        return worker.InitAndRun(config.Config, workflows, dataStore, nats)
}

func cmdMigrate(args *cli.Context) error {
        var err error
        if !args.Bool("skip-database") {
                _, err = store.SetupDataStore(true)
                if err != nil {
                        return err
                }
        }
        if !args.Bool("skip-nats") {
                var nc nats.Client
                nc, err = getNatsClient()
                if err == nil {
                        if err = initJetstream(nc, true, true); err == nil {
                                err = initJetstream(nc, false, true)
                        }
                }

        }
        return err
}

func cmdListJobs(args *cli.Context) error {
        dataStore, err := store.SetupDataStore(false)
        if err != nil {
                return err
        }
        defer dataStore.Close()

        ctx, cancel := context.WithCancel(context.Background())
        defer cancel()

        var page int64
        var perPage int64
        page = args.Int64("page")
        perPage = args.Int64("perPage")

        if page < 1 {
                page = 1
        }
        if perPage < 1 {
                perPage = 75
        }
        jobs, count, _ := dataStore.GetAllJobs(ctx, page, perPage)
        fmt.Printf("all jobs: %d; page: %d/%d perPage:%d\n%29s %24s %10s %s\n",
                count, page, count/perPage, perPage, "insert time", "id", "status", "workflow")
        for _, j := range jobs {
                format := "Mon, 2 Jan 2006 15:04:05 MST"
                fmt.Printf(
                        "%29s %24s %10s %s\n",
                        j.InsertTime.Format(format),
                        j.ID, model.StatusToString(j.Status),
                        j.WorkflowName,
                )
        }
        fmt.Printf("all jobs: %d; page: %d/%d\n", count, page, count/perPage)

        return nil
}

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package main
16
17	import (
18	"context"
19	"fmt"
20	"log"
21	"os"
22	"strings"
23
24	"github.com/pkg/errors"
25	"github.com/urfave/cli"
26
27	"github.com/mendersoftware/mender-server/pkg/config"
28
29	"github.com/mendersoftware/mender-server/services/workflows/app/server"
30	"github.com/mendersoftware/mender-server/services/workflows/app/worker"
31	"github.com/mendersoftware/mender-server/services/workflows/client/nats"
32	dconfig "github.com/mendersoftware/mender-server/services/workflows/config"
33	"github.com/mendersoftware/mender-server/services/workflows/model"
34	store "github.com/mendersoftware/mender-server/services/workflows/store/mongo"
35	)
36
37	func main() {	2✔
38	doMain(os.Args)	2✔
39	}	2✔
40
41	func doMain(args []string) {	2✔
42	var configPath string	2✔
43		2✔
44	app := &cli.App{	2✔
45	Flags: []cli.Flag{	2✔
46	&cli.StringFlag{	2✔
47	Name: "config",	2✔
48	Usage: "Configuration `FILE`." +	2✔
49	" Supports JSON, TOML, YAML and HCL formatted configs.",	2✔
50	Destination: &configPath,	2✔
51	},	2✔
52	},	2✔
53	Commands: []cli.Command{	2✔
54	{	2✔
55	Name: "server",	2✔
56	Usage: "Run the HTTP API server",	2✔
57	Action: cmdServer,	2✔
58	Flags: []cli.Flag{	2✔
59	&cli.BoolFlag{	2✔
60	Name: "automigrate",	2✔
61	Usage: "Run database migrations before starting.",	2✔
62	},	2✔
63	},	2✔
64	},	2✔
65	{	2✔
66	Name: "worker",	2✔
67	Usage: "Run the worker process",	2✔
68	Action: cmdWorker,	2✔
69	Flags: []cli.Flag{	2✔
70	&cli.BoolFlag{	2✔
71	Name: "automigrate",	2✔
72	Usage: "Run database migrations before starting.",	2✔
73	},	2✔
74	&cli.StringFlag{	2✔
75	Name: "workflows",	2✔
76	Usage: "Comma-separated list of workflows executed by this worker",	2✔
77	},	2✔
78	&cli.StringFlag{	2✔
79	Name: "excluded-workflows",	2✔
80	Usage: "Comma-separated list of workflows NOT executed by this worker",	2✔
81	},	2✔
82	},	2✔
83	},	2✔
84	{	2✔
85	Name: "migrate",	2✔
86	Usage: "Run the migrations",	2✔
87	Action: cmdMigrate,	2✔
88	Flags: []cli.Flag{	2✔
89	cli.BoolFlag{	2✔
90	Name: "skip-nats",	2✔
91	Usage: "Skip migrating the NATS Jetstream configuration",	2✔
92	EnvVar: "WORKFLOWS_MIGRATION_SKIP_NATS",	2✔
93	},	2✔
94	cli.BoolFlag{	2✔
95	Name: "skip-database",	2✔
96	Usage: "Skip migrating the database",	2✔
97	EnvVar: "WORKFLOWS_MIGRATION_SKIP_DATABASE",	2✔
98	},	2✔
99	},	2✔
100	},	2✔
101	{	2✔
102	Name: "list-jobs",	2✔
103	Usage: "List jobs",	2✔
104	Action: cmdListJobs,	2✔
105	Flags: []cli.Flag{	2✔
106	cli.Int64Flag{	2✔
107	Name: "page",	2✔
108	Usage: "page number to show",	2✔
109	},	2✔
110	cli.Int64Flag{	2✔
111	Name: "perPage",	2✔
112	Usage: "number of results per page",	2✔
113	},	2✔
114	},	2✔
115	},	2✔
116	},	2✔
117	}	2✔
118	app.Usage = "Workflows"	2✔
119	app.Action = cmdServer	2✔
120		2✔
121	app.Before = func(args *cli.Context) error {	4✔
122	err := config.FromConfigFile(configPath, dconfig.Defaults)	2✔
123	if err != nil {	2✔
124	return cli.NewExitError(	×
125	fmt.Sprintf("error loading configuration: %s", err),	×
126	1)	×
127	}	×
128
129	// Enable setting config values by environment variables
130	config.Config.SetEnvPrefix("WORKFLOWS")	2✔
131	config.Config.AutomaticEnv()	2✔
132	config.Config.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))	2✔
133		2✔
134	return nil	2✔
135	}
136
137	err := app.Run(args)	2✔
138	if err != nil {	2✔
139	log.Fatal(err)	×
140	}	×
141	}
142
143	func getNatsClient() (nats.Client, error) {	2✔
144	natsURI := config.Config.GetString(dconfig.SettingNatsURI)	2✔
145	streamName := config.Config.GetString(dconfig.SettingNatsStreamName)	2✔
146	nats, err := nats.NewClientWithDefaults(natsURI, streamName)	2✔
147	if err != nil {	2✔
148	return nil, errors.Wrap(err, "failed to connect to nats")	×
149	}	×
150	return nats, err	2✔
151	}
152
153	func initJetstream(nc nats.Client, producer, upsert bool) (err error) {	2✔
154	durableName := config.Config.GetString(dconfig.SettingNatsSubscriberDurable)	2✔
155	if producer {	4✔
156	err = nc.JetStreamCreateStream(nc.StreamName())	2✔
157	if err != nil {	2✔
158	return err	×
159	}	×
160	} else {	2✔
161	var cfg nats.ConsumerConfig	2✔
162	cfg, err = dconfig.GetNatsConsumerConfig(config.Config)	2✔
163	if err != nil {	2✔
164	return err	×
165	}	×
166	err = nc.CreateConsumer(durableName, upsert, cfg)	2✔
167	}
168	return err	2✔
169	}
170
171	func cmdServer(args *cli.Context) error {	2✔
172	dataStore, err := store.SetupDataStore(args.Bool("automigrate"))	2✔
173	if err != nil {	2✔
174	return err	×
175	}	×
176	defer dataStore.Close()	2✔
177		2✔
178	nats, err := getNatsClient()	2✔
179	if err != nil {	2✔
180	return err	×
181	}	×
182	defer nats.Close()	2✔
183		2✔
184	if err = initJetstream(nats, true, args.Bool("automigrate")); err != nil {	2✔
185	return errors.WithMessage(err, "failed to apply Jetstream migrations")	×
186	}	×
187
188	return server.InitAndRun(config.Config, dataStore, nats)	2✔
189	}
190
191	func cmdWorker(args *cli.Context) error {	2✔
192	dataStore, err := store.SetupDataStore(args.Bool("automigrate"))	2✔
193	if err != nil {	2✔
194	return err	×
195	}	×
196	defer dataStore.Close()	2✔
197		2✔
198	nats, err := getNatsClient()	2✔
199	if err != nil {	2✔
200	return err	×
201	}	×
202	defer nats.Close()	2✔
203		2✔
204	if err = initJetstream(nats, false, args.Bool("automigrate")); err != nil {	2✔
205	return errors.WithMessage(err, "failed to apply Jetstream consumer migrations")	×
206	}	×
207
208	var included, excluded []string	2✔
209		2✔
210	includedWorkflows := args.String("workflows")	2✔
211	if includedWorkflows != "" {	2✔
212	included = strings.Split(includedWorkflows, ",")	×
213	}	×
214
215	excludedWorkflows := args.String("excluded-workflows")	2✔
216	if excludedWorkflows != "" {	2✔
217	excluded = strings.Split(excludedWorkflows, ",")	×
218	}	×
219
220	workflows := worker.Workflows{	2✔
221	Included: included,	2✔
222	Excluded: excluded,	2✔
223	}	2✔
224	return worker.InitAndRun(config.Config, workflows, dataStore, nats)	2✔
225	}
226
227	func cmdMigrate(args *cli.Context) error {	×
228	var err error	×
229	if !args.Bool("skip-database") {	×
230	_, err = store.SetupDataStore(true)	×
231	if err != nil {	×
232	return err	×
233	}	×
234	}
235	if !args.Bool("skip-nats") {	×
236	var nc nats.Client	×
237	nc, err = getNatsClient()	×
238	if err == nil {	×
239	if err = initJetstream(nc, true, true); err == nil {	×
240	err = initJetstream(nc, false, true)	×
241	}	×
242	}
243
244	}
245	return err	×
246	}
247
248	func cmdListJobs(args *cli.Context) error {	×
249	dataStore, err := store.SetupDataStore(false)	×
250	if err != nil {	×
251	return err	×
252	}	×
253	defer dataStore.Close()	×
254		×
255	ctx, cancel := context.WithCancel(context.Background())	×
256	defer cancel()	×
257		×
258	var page int64	×
259	var perPage int64	×
260	page = args.Int64("page")	×
261	perPage = args.Int64("perPage")	×
262		×
263	if page < 1 {	×
264	page = 1	×
265	}	×
266	if perPage < 1 {	×
267	perPage = 75	×
268	}	×
269	jobs, count, _ := dataStore.GetAllJobs(ctx, page, perPage)	×
270	fmt.Printf("all jobs: %d; page: %d/%d perPage:%d\n%29s %24s %10s %s\n",	×
271	count, page, count/perPage, perPage, "insert time", "id", "status", "workflow")	×
272	for _, j := range jobs {	×
273	format := "Mon, 2 Jan 2006 15:04:05 MST"	×
274	fmt.Printf(	×
275	"%29s %24s %10s %s\n",	×
276	j.InsertTime.Format(format),	×
277	j.ID, model.StatusToString(j.Status),	×
278	j.WorkflowName,	×
279	)	×
280	}	×
281	fmt.Printf("all jobs: %d; page: %d/%d\n", count, page, count/perPage)	×
282		×
283	return nil	×
284	}

mendersoftware / mender-server / 1510130353

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous