757087774

Build Type

gitlab-ci

Committed by Alex Miliukov

Commit Message

test: unit tests for features enable/disable option

Pull Request Pull Request #211: QA-518: add list of repos to sync option

Run Details

31 of 42 new or added lines in 2 files covered. (73.81%)

19 existing lines in 1 file now uncovered.

1810 of 2387 relevant lines covered (75.83%)

4.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.89

/main.go

package main

import (
        "context"
        "fmt"
        "io"
        "net/http"
        "os"
        "os/signal"
        "strings"
        "time"

        "github.com/davecgh/go-spew/spew"
        "github.com/gin-gonic/gin"
        "github.com/google/go-github/v28/github"
        "github.com/sirupsen/logrus"
        "golang.org/x/sys/unix"

        clientgithub "github.com/mendersoftware/integration-test-runner/client/github"
        "github.com/mendersoftware/integration-test-runner/git"
        "github.com/mendersoftware/integration-test-runner/logger"
)

type config struct {
        dryRunMode           bool
        githubSecret         []byte
        githubProtocol       gitProtocol
        githubOrganization   string
        githubToken          string
        gitlabToken          string
        gitlabBaseURL        string
        integrationDirectory string
        isSyncRepos          bool
        isProcessPREvents    bool
        isProcessComments    bool
        reposSyncList        []string
}

type buildOptions struct {
        pr         string
        repo       string
        baseBranch string
        commitSHA  string
        makeQEMU   bool
}

// List of repos besides the ones in release_tool.py for
// which the integration pipeline shall be run
var extraWatchRepositoriesPipeline = []string{
        "meta-mender",
}

// Mapping https://github.com/<org> -> https://gitlab.com/Northern.tech/<group>
var gitHubOrganizationToGitLabGroup = map[string]string{
        "mendersoftware": "Mender",
        "cfengine":       "CFEngine",
}

// Mapping of special repos that have a custom group/project
var gitHubRepoToGitLabProjectCustom = map[string]string{
        "saas": "Northern.tech/MenderSaaS/saas",
}

var qemuBuildRepositories = []string{
        "meta-mender",
        "mender",
        "mender-artifact",
        "mender-connect",
        "monitor-client",
        "mender-auth-azure-iot",
        "mender-gateway",
}

const (
        gitOperationTimeout = 30
)

const (
        featureBranchPrefix = "feature-"
)

const (
        githubBotName = "mender-test-bot"
)

const (
        commandStartPipeline      = "start pipeline"
        commandCherryPickBranch   = "cherry-pick to:"
        commandConventionalCommit = "mark-pr as"
)

func getConfig() (*config, error) {
        var reposSyncList []string
        dryRunMode := os.Getenv("DRY_RUN") != ""
        githubSecret := os.Getenv("GITHUB_SECRET")
        githubToken := os.Getenv("GITHUB_TOKEN")
        gitlabToken := os.Getenv("GITLAB_TOKEN")
        gitlabBaseURL := os.Getenv("GITLAB_BASE_URL")
        integrationDirectory := os.Getenv("INTEGRATION_DIRECTORY")
        // default: sync repos if not explicitly disabled
        isSyncRepos := os.Getenv("NOT_SYNC_REPOS") == ""
        // default: process PR events if not explicitly disabled
        isProcessPREvents := os.Getenv("NOT_PROCESS_PR_EVENTS") == ""
        // default: process PR comments if not explicitly disabled
        isProcessComments := os.Getenv("NOT_PROCESS_COMMENTS") == ""

        logLevel, found := os.LookupEnv("INTEGRATION_TEST_RUNNER_LOG_LEVEL")
        logrus.SetLevel(logrus.InfoLevel)
        if found {
                lvl, err := logrus.ParseLevel(logLevel)
                if err != nil {
                        logrus.Infof(
                                "Failed to parse the 'INTEGRATION_TEST_RUNNER_LOG_LEVEL' variable, " +
                                        "defaulting to 'InfoLevel'",
                        )
                } else {
                        logrus.Infof("Set 'LogLevel' to %s", lvl)
                        logrus.SetLevel(lvl)
                }
        }

        // Comma separated list of repos to sync (GitHub->GitLab)
        reposSyncListRaw, found := os.LookupEnv("SYNC_REPOS_LIST")
        if found {
                reposSyncList = strings.Split(reposSyncListRaw, ",")
        }

        switch {
        case githubSecret == "" && !dryRunMode:
                return &config{}, fmt.Errorf("set GITHUB_SECRET")
        case githubToken == "":
                return &config{}, fmt.Errorf("set GITHUB_TOKEN")
        case gitlabToken == "":
                return &config{}, fmt.Errorf("set GITLAB_TOKEN")
        case gitlabBaseURL == "":
                return &config{}, fmt.Errorf("set GITLAB_BASE_URL")
        case integrationDirectory == "":
                return &config{}, fmt.Errorf("set INTEGRATION_DIRECTORY")
        }

        return &config{
                dryRunMode:           dryRunMode,
                githubSecret:         []byte(githubSecret),
                githubProtocol:       gitProtocolSSH,
                githubToken:          githubToken,
                gitlabToken:          gitlabToken,
                gitlabBaseURL:        gitlabBaseURL,
                integrationDirectory: integrationDirectory,
                isSyncRepos:          isSyncRepos,       // Is sync repos feature enabled
                isProcessPREvents:    isProcessPREvents, // Is PR events feature enabled
                isProcessComments:    isProcessComments, // Is PR comments feature enabled
                reposSyncList:        reposSyncList,     // List of repos to sync (GitHub->GitLab)
        }, nil
}

func getCustomLoggerFromContext(ctx *gin.Context) *logrus.Entry {
        deliveryID, ok := ctx.Get("delivery")
        if !ok || !isStringType(deliveryID) {
                return logrus.WithField("delivery", "nil")
        }
        return logrus.WithField("delivery", deliveryID)
}

func isStringType(i interface{}) bool {
        switch i.(type) {
        case string:
                return true
        default:
                return false
        }
}

func processGitHubWebhookRequest(
        ctx *gin.Context,
        payload []byte,
        githubClient clientgithub.Client,
        conf *config,
) {
        webhookType := github.WebHookType(ctx.Request)
        webhookEvent, _ := github.ParseWebHook(github.WebHookType(ctx.Request), payload)
        _ = processGitHubWebhook(ctx, webhookType, webhookEvent, githubClient, conf)
}

func processGitHubWebhook(
        ctx *gin.Context,
        webhookType string,
        webhookEvent interface{},
        githubClient clientgithub.Client,
        conf *config,
) error {
        githubOrganization, err := getGitHubOrganization(webhookType, webhookEvent)
        if err != nil {
                logrus.Warnln("ignoring event: ", err.Error())
                return nil
        }
        conf.githubOrganization = githubOrganization
        switch webhookType {
        case "pull_request":
                if conf.isProcessPREvents {
                        pr := webhookEvent.(*github.PullRequestEvent)
                        return processGitHubPullRequest(ctx, pr, githubClient, conf)
                }
        case "push":
                if conf.isSyncRepos {
                        push := webhookEvent.(*github.PushEvent)
                        return processGitHubPush(ctx, push, githubClient, conf)
                }
        case "issue_comment":
                if conf.isProcessComments {
                        comment := webhookEvent.(*github.IssueCommentEvent)
                        return processGitHubComment(ctx, comment, githubClient, conf)
                }
        }
        return nil
}

func setupLogging(conf *config, requestLogger logger.RequestLogger) {
        // Log to stdout and with JSON format; suitable for GKE
        formatter := &logrus.JSONFormatter{
                FieldMap: logrus.FieldMap{
                        logrus.FieldKeyTime:  "time",
                        logrus.FieldKeyLevel: "level",
                        logrus.FieldKeyMsg:   "message",
                },
        }

        if conf.dryRunMode {
                mw := io.MultiWriter(os.Stdout, requestLogger)
                logrus.SetOutput(mw)
        } else {
                logrus.SetOutput(os.Stdout)
        }
        logrus.SetFormatter(formatter)
}

func main() {
        doMain()
}

var githubClient clientgithub.Client

func doMain() {
        conf, err := getConfig()
        if err != nil {
                logrus.Fatalf("failed to load config: %s", err.Error())
        }

        requestLogger := logger.NewRequestLogger()
        logger.SetRequestLogger(requestLogger)

        setupLogging(conf, requestLogger)
        git.SetDryRunMode(conf.dryRunMode)

        logrus.Infoln("using settings: ", spew.Sdump(conf))

        githubClient = clientgithub.NewGitHubClient(conf.githubToken, conf.dryRunMode)
        r := gin.Default()
        r.Use(gin.Recovery())

        // webhook for GitHub
        r.POST("/", func(context *gin.Context) {
                payload, err := github.ValidatePayload(context.Request, conf.githubSecret)
                if err != nil {
                        logrus.Warnln("payload failed to validate, ignoring.")
                        context.Status(http.StatusForbidden)
                        return
                }
                context.Set("delivery", github.DeliveryID(context.Request))
                if conf.dryRunMode {
                        processGitHubWebhookRequest(context, payload, githubClient, conf)
                } else {
                        go processGitHubWebhookRequest(context, payload, githubClient, conf)
                }
                context.Status(http.StatusAccepted)
        })

        // 200 replay for the loadbalancer
        r.GET("/", func(_ *gin.Context) {})

        // dry-run mode, end-point to retrieve and clear logs
        if conf.dryRunMode {
                r.GET("/logs", func(context *gin.Context) {
                        logs := requestLogger.Get()
                        context.JSON(http.StatusOK, logs)
                })

                r.DELETE("/logs", func(context *gin.Context) {
                        requestLogger.Clear()
                        context.Writer.WriteHeader(http.StatusNoContent)
                })
        }

        srv := &http.Server{
                Addr:    "0.0.0.0:8080",
                Handler: r,
        }

        go func() {
                if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
                        logrus.Fatalf("Failed listening: %s\n", err)
                }
        }()

        quit := make(chan os.Signal, 1)
        signal.Notify(quit, unix.SIGINT, unix.SIGTERM)
        <-quit

        logrus.Info("Shutdown server ...")

        ctx := context.Background()
        ctxWithTimeout, cancel := context.WithTimeout(ctx, 5*time.Second)
        defer cancel()
        if err := srv.Shutdown(ctxWithTimeout); err != nil {
                logrus.Fatal("Failed to shutdown the server: ", err)
        }
}

1	package main
2
3	import (
4	"context"
5	"fmt"
6	"io"
7	"net/http"
8	"os"
9	"os/signal"
10	"strings"
11	"time"
12
13	"github.com/davecgh/go-spew/spew"
14	"github.com/gin-gonic/gin"
15	"github.com/google/go-github/v28/github"
16	"github.com/sirupsen/logrus"
17	"golang.org/x/sys/unix"
18
19	clientgithub "github.com/mendersoftware/integration-test-runner/client/github"
20	"github.com/mendersoftware/integration-test-runner/git"
21	"github.com/mendersoftware/integration-test-runner/logger"
22	)
23
24	type config struct {
25	dryRunMode bool
26	githubSecret []byte
27	githubProtocol gitProtocol
28	githubOrganization string
29	githubToken string
30	gitlabToken string
31	gitlabBaseURL string
32	integrationDirectory string
33	isSyncRepos bool
34	isProcessPREvents bool
35	isProcessComments bool
36	reposSyncList []string
37	}
38
39	type buildOptions struct {
40	pr string
41	repo string
42	baseBranch string
43	commitSHA string
44	makeQEMU bool
45	}
46
47	// List of repos besides the ones in release_tool.py for
48	// which the integration pipeline shall be run
49	var extraWatchRepositoriesPipeline = []string{
50	"meta-mender",
51	}
52
53	// Mapping https://github.com/<org> -> https://gitlab.com/Northern.tech/<group>
54	var gitHubOrganizationToGitLabGroup = map[string]string{
55	"mendersoftware": "Mender",
56	"cfengine": "CFEngine",
57	}
58
59	// Mapping of special repos that have a custom group/project
60	var gitHubRepoToGitLabProjectCustom = map[string]string{
61	"saas": "Northern.tech/MenderSaaS/saas",
62	}
63
64	var qemuBuildRepositories = []string{
65	"meta-mender",
66	"mender",
67	"mender-artifact",
68	"mender-connect",
69	"monitor-client",
70	"mender-auth-azure-iot",
71	"mender-gateway",
72	}
73
74	const (
75	gitOperationTimeout = 30
76	)
77
78	const (
79	featureBranchPrefix = "feature-"
80	)
81
82	const (
83	githubBotName = "mender-test-bot"
84	)
85
86	const (
87	commandStartPipeline = "start pipeline"
88	commandCherryPickBranch = "cherry-pick to:"
89	commandConventionalCommit = "mark-pr as"
90	)
91
92	func getConfig() (*config, error) {	1✔
93	var reposSyncList []string	1✔
94	dryRunMode := os.Getenv("DRY_RUN") != ""	1✔
95	githubSecret := os.Getenv("GITHUB_SECRET")	1✔
96	githubToken := os.Getenv("GITHUB_TOKEN")	1✔
97	gitlabToken := os.Getenv("GITLAB_TOKEN")	1✔
98	gitlabBaseURL := os.Getenv("GITLAB_BASE_URL")	1✔
99	integrationDirectory := os.Getenv("INTEGRATION_DIRECTORY")	1✔
100	// default: sync repos if not explicitly disabled	1✔
101	isSyncRepos := os.Getenv("NOT_SYNC_REPOS") == ""	1✔
102	// default: process PR events if not explicitly disabled	1✔
103	isProcessPREvents := os.Getenv("NOT_PROCESS_PR_EVENTS") == ""	1✔
104	// default: process PR comments if not explicitly disabled	1✔
105	isProcessComments := os.Getenv("NOT_PROCESS_COMMENTS") == ""	1✔
106		1✔
107	logLevel, found := os.LookupEnv("INTEGRATION_TEST_RUNNER_LOG_LEVEL")	1✔
108	logrus.SetLevel(logrus.InfoLevel)	1✔
109	if found {	2✔
110	lvl, err := logrus.ParseLevel(logLevel)	1✔
111	if err != nil {	1✔
112	logrus.Infof(	×
113	"Failed to parse the 'INTEGRATION_TEST_RUNNER_LOG_LEVEL' variable, " +	×
UNCOV 114	"defaulting to 'InfoLevel'",	×
115	)	×
116	} else {	1✔
117	logrus.Infof("Set 'LogLevel' to %s", lvl)	1✔
118	logrus.SetLevel(lvl)	1✔
119	}	1✔
120	}
121
122	// Comma separated list of repos to sync (GitHub->GitLab)
123	reposSyncListRaw, found := os.LookupEnv("SYNC_REPOS_LIST")	1✔
124	if found {	1✔
NEW UNCOV 125	reposSyncList = strings.Split(reposSyncListRaw, ",")	×
NEW UNCOV 126	}	×
127
128	switch {	1✔
UNCOV 129	case githubSecret == "" && !dryRunMode:	×
130	return &config{}, fmt.Errorf("set GITHUB_SECRET")	×
131	case githubToken == "":	×
132	return &config{}, fmt.Errorf("set GITHUB_TOKEN")	×
133	case gitlabToken == "":	×
134	return &config{}, fmt.Errorf("set GITLAB_TOKEN")	×
135	case gitlabBaseURL == "":	×
136	return &config{}, fmt.Errorf("set GITLAB_BASE_URL")	×
137	case integrationDirectory == "":	×
138	return &config{}, fmt.Errorf("set INTEGRATION_DIRECTORY")	×
139	}
140
141	return &config{	1✔
142	dryRunMode: dryRunMode,	1✔
143	githubSecret: []byte(githubSecret),	1✔
144	githubProtocol: gitProtocolSSH,	1✔
145	githubToken: githubToken,	1✔
146	gitlabToken: gitlabToken,	1✔
147	gitlabBaseURL: gitlabBaseURL,	1✔
148	integrationDirectory: integrationDirectory,	1✔
149	isSyncRepos: isSyncRepos, // Is sync repos feature enabled	1✔
150	isProcessPREvents: isProcessPREvents, // Is PR events feature enabled	1✔
151	isProcessComments: isProcessComments, // Is PR comments feature enabled	1✔
152	reposSyncList: reposSyncList, // List of repos to sync (GitHub->GitLab)	1✔
153	}, nil	1✔
154	}
155
156	func getCustomLoggerFromContext(ctx gin.Context) logrus.Entry {	25✔
157	deliveryID, ok := ctx.Get("delivery")	25✔
158	if !ok \|\| !isStringType(deliveryID) {	25✔
159	return logrus.WithField("delivery", "nil")	×
160	}	×
161	return logrus.WithField("delivery", deliveryID)	25✔
162	}
163
164	func isStringType(i interface{}) bool {	25✔
165	switch i.(type) {	25✔
166	case string:	25✔
167	return true	25✔
168	default:	×
UNCOV 169	return false	×
170	}
171	}
172
173	func processGitHubWebhookRequest(
174	ctx *gin.Context,
175	payload []byte,
176	githubClient clientgithub.Client,
177	conf *config,
178	) {	1✔
179	webhookType := github.WebHookType(ctx.Request)	1✔
180	webhookEvent, _ := github.ParseWebHook(github.WebHookType(ctx.Request), payload)	1✔
181	_ = processGitHubWebhook(ctx, webhookType, webhookEvent, githubClient, conf)	1✔
182	}	1✔
183
184	func processGitHubWebhook(
185	ctx *gin.Context,
186	webhookType string,
187	webhookEvent interface{},
188	githubClient clientgithub.Client,
189	conf *config,
190	) error {	31✔
191	githubOrganization, err := getGitHubOrganization(webhookType, webhookEvent)	31✔
192	if err != nil {	31✔
UNCOV 193	logrus.Warnln("ignoring event: ", err.Error())	×
UNCOV 194	return nil	×
UNCOV 195	}	×
196	conf.githubOrganization = githubOrganization	31✔
197	switch webhookType {	31✔
198	case "pull_request":	5✔
199	if conf.isProcessPREvents {	8✔
200	pr := webhookEvent.(*github.PullRequestEvent)	3✔
201	return processGitHubPullRequest(ctx, pr, githubClient, conf)	3✔
202	}	3✔
203	case "push":	5✔
204	if conf.isSyncRepos {	8✔
205	push := webhookEvent.(*github.PushEvent)	3✔
206	return processGitHubPush(ctx, push, githubClient, conf)	3✔
207	}	3✔
208	case "issue_comment":	23✔
209	if conf.isProcessComments {	44✔
210	comment := webhookEvent.(*github.IssueCommentEvent)	21✔
211	return processGitHubComment(ctx, comment, githubClient, conf)	21✔
212	}	21✔
213	}
214	return nil	6✔
215	}
216
217	func setupLogging(conf *config, requestLogger logger.RequestLogger) {	3✔
218	// Log to stdout and with JSON format; suitable for GKE	3✔
219	formatter := &logrus.JSONFormatter{	3✔
220	FieldMap: logrus.FieldMap{	3✔
221	logrus.FieldKeyTime: "time",	3✔
222	logrus.FieldKeyLevel: "level",	3✔
223	logrus.FieldKeyMsg: "message",	3✔
224	},	3✔
225	}	3✔
226		3✔
227	if conf.dryRunMode {	4✔
228	mw := io.MultiWriter(os.Stdout, requestLogger)	1✔
229	logrus.SetOutput(mw)	1✔
230	} else {	3✔
231	logrus.SetOutput(os.Stdout)	2✔
232	}	2✔
233	logrus.SetFormatter(formatter)	3✔
234	}
235
UNCOV 236	func main() {	×
UNCOV 237	doMain()	×
UNCOV 238	}	×
239
240	var githubClient clientgithub.Client
241
242	func doMain() {	1✔
243	conf, err := getConfig()	1✔
244	if err != nil {	1✔
245	logrus.Fatalf("failed to load config: %s", err.Error())	×
246	}	×
247
248	requestLogger := logger.NewRequestLogger()	1✔
249	logger.SetRequestLogger(requestLogger)	1✔
250		1✔
251	setupLogging(conf, requestLogger)	1✔
252	git.SetDryRunMode(conf.dryRunMode)	1✔
253		1✔
254	logrus.Infoln("using settings: ", spew.Sdump(conf))	1✔
255		1✔
256	githubClient = clientgithub.NewGitHubClient(conf.githubToken, conf.dryRunMode)	1✔
257	r := gin.Default()	1✔
258	r.Use(gin.Recovery())	1✔
259		1✔
260	// webhook for GitHub	1✔
261	r.POST("/", func(context *gin.Context) {	2✔
262	payload, err := github.ValidatePayload(context.Request, conf.githubSecret)	1✔
263	if err != nil {	1✔
UNCOV 264	logrus.Warnln("payload failed to validate, ignoring.")	×
UNCOV 265	context.Status(http.StatusForbidden)	×
UNCOV 266	return	×
UNCOV 267	}	×
268	context.Set("delivery", github.DeliveryID(context.Request))	1✔
269	if conf.dryRunMode {	2✔
270	processGitHubWebhookRequest(context, payload, githubClient, conf)	1✔
271	} else {	1✔
272	go processGitHubWebhookRequest(context, payload, githubClient, conf)	×
273	}	×
274	context.Status(http.StatusAccepted)	1✔
275	})
276
277	// 200 replay for the loadbalancer
278	r.GET("/", func(_ *gin.Context) {})	1✔
279
280	// dry-run mode, end-point to retrieve and clear logs
281	if conf.dryRunMode {	2✔
282	r.GET("/logs", func(context *gin.Context) {	2✔
283	logs := requestLogger.Get()	1✔
284	context.JSON(http.StatusOK, logs)	1✔
285	})	1✔
286
287	r.DELETE("/logs", func(context *gin.Context) {	2✔
288	requestLogger.Clear()	1✔
289	context.Writer.WriteHeader(http.StatusNoContent)	1✔
290	})	1✔
291	}
292
293	srv := &http.Server{	1✔
294	Addr: "0.0.0.0:8080",	1✔
295	Handler: r,	1✔
296	}	1✔
297		1✔
298	go func() {	2✔
299	if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {	1✔
UNCOV 300	logrus.Fatalf("Failed listening: %s\n", err)	×
UNCOV 301	}	×
302	}()
303
304	quit := make(chan os.Signal, 1)	1✔
305	signal.Notify(quit, unix.SIGINT, unix.SIGTERM)	1✔
306	<-quit	1✔
307		1✔
308	logrus.Info("Shutdown server ...")	1✔
309		1✔
310	ctx := context.Background()	1✔
311	ctxWithTimeout, cancel := context.WithTimeout(ctx, 5*time.Second)	1✔
312	defer cancel()	1✔
313	if err := srv.Shutdown(ctxWithTimeout); err != nil {	1✔
UNCOV 314	logrus.Fatal("Failed to shutdown the server: ", err)	×
UNCOV 315	}	×
316	}

mendersoftware / integration-test-runner / 757087774

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous