• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mendersoftware / integration-test-runner / 1783653474

24 Apr 2025 08:44AM UTC coverage: 65.78% (-0.08%) from 65.864%
1783653474

Pull #373

gitlab-ci

alfrunes
feat: Added feature to put option in the PR title to skip CI

If a PR title starts with `[NoCI]` (case insensitive) the CI will not
be started automatically.

Signed-off-by: Alf-Rune Siqveland <alf.rune@northern.tech>
Pull Request #373: QA-972: Add option to skip CI from PR title

20 of 36 new or added lines in 2 files covered. (55.56%)

19 existing lines in 1 file now uncovered.

1953 of 2969 relevant lines covered (65.78%)

2.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.89
/main.go
1
package main
2

3
import (
4
        "context"
5
        "errors"
6
        "fmt"
7
        "io"
8
        "net/http"
9
        "os"
10
        "os/signal"
11
        "strings"
12
        "time"
13

14
        "github.com/davecgh/go-spew/spew"
15
        "github.com/gin-gonic/gin"
16
        "github.com/google/go-github/v28/github"
17
        "github.com/sirupsen/logrus"
18
        "golang.org/x/sys/unix"
19

20
        clientgithub "github.com/mendersoftware/integration-test-runner/client/github"
21
        "github.com/mendersoftware/integration-test-runner/git"
22
        "github.com/mendersoftware/integration-test-runner/logger"
23
)
24

25
type config struct {
26
        dryRunMode             bool
27
        githubSecret           []byte
28
        githubProtocol         gitProtocol
29
        githubOrganization     string
30
        githubToken            string
31
        gitlabToken            string
32
        gitlabBaseURL          string
33
        integrationDirectory   string
34
        isProcessPushEvents    bool
35
        isProcessPREvents      bool
36
        isProcessCommentEvents bool
37
        reposSyncList          []string
38
}
39

40
type buildOptions struct {
41
        pr         string
42
        repo       string
43
        baseBranch string
44
        commitSHA  string
45
        makeQEMU   bool
46
}
47

48
// Mapping https://github.com/<org> -> https://gitlab.com/Northern.tech/<group>
49
var gitHubOrganizationToGitLabGroup = map[string]string{
50
        "mendersoftware": "Mender",
51
        "cfengine":       "CFEngine",
52
        "NorthernTechHQ": "NorthernTechHQ",
53
}
54

55
// Mapping of special repos that have a custom group/project
56
var gitHubRepoToGitLabProjectCustom = map[string]string{
57
        "saas": "Northern.tech/MenderSaaS/saas",
58
}
59

60
var qemuBuildRepositories = []string{
61
        "meta-mender",
62
        "mender",
63
        "mender-artifact",
64
        "mender-connect",
65
        "monitor-client",
66
        "mender-auth-azure-iot",
67
        "mender-gateway",
68
        "mender-snapshot",
69
}
70

71
const (
72
        KiB = 1024
73
        MiB = 1024 * KiB
74
)
75

76
const (
77
        gitOperationTimeout = 30
78
)
79

80
const (
81
        featureBranchPrefix = "feature-"
82
)
83

84
const (
85
        githubBotName = "mender-test-bot"
86
)
87

88
const (
89
        commandStartIntegrationPipeline = "start integration pipeline"
90
        commandStartClientPipeline      = "start client pipeline"
91
        commandCherryPickBranch         = "cherry-pick to:"
92
        commandConventionalCommit       = "mark-pr as"
93
        commandSyncRepos                = "sync"
94
)
95

96
func getConfig() (*config, error) {
1✔
97
        var reposSyncList []string
1✔
98
        dryRunMode := os.Getenv("DRY_RUN") != ""
1✔
99
        githubSecret := os.Getenv("GITHUB_SECRET")
1✔
100
        githubToken := os.Getenv("GITHUB_TOKEN")
1✔
101
        gitlabToken := os.Getenv("GITLAB_TOKEN")
1✔
102
        gitlabBaseURL := os.Getenv("GITLAB_BASE_URL")
1✔
103
        integrationDirectory := "/integration/"
1✔
104
        if integrationDirEnv := os.Getenv("INTEGRATION_DIRECTORY"); integrationDirEnv != "" {
1✔
105
                integrationDirectory = integrationDirEnv
×
106
        }
×
107

108
        //
109
        // Currently we don't have a distinguishment between GitHub events and features.
110
        // Different features might be implemented across different events, but in future
111
        // it's probability that we might implement proper features selection. For now the
112
        // straight goal is to being able to configure the runner to only sync repos on
113
        // push events and disable all the rest (to be used by the CFEngine team).
114
        //
115
        // default: process push events (sync repos) if not explicitly disabled
116
        isProcessPushEvents := os.Getenv("DISABLE_PUSH_EVENTS_PROCESSING") == ""
1✔
117
        // default: process PR events if not explicitly disabled
1✔
118
        isProcessPREvents := os.Getenv("DISABLE_PR_EVENTS_PROCESSING") == ""
1✔
119
        // default: process comment events if not explicitly disabled
1✔
120
        isProcessCommentEvents := os.Getenv("DISABLE_COMMENT_EVENTS_PROCESSING") == ""
1✔
121

1✔
122
        logLevel, found := os.LookupEnv("INTEGRATION_TEST_RUNNER_LOG_LEVEL")
1✔
123
        logrus.SetLevel(logrus.InfoLevel)
1✔
124
        if found {
2✔
125
                lvl, err := logrus.ParseLevel(logLevel)
1✔
126
                if err != nil {
1✔
127
                        logrus.Infof(
×
128
                                "Failed to parse the 'INTEGRATION_TEST_RUNNER_LOG_LEVEL' variable, " +
×
129
                                        "defaulting to 'InfoLevel'",
×
130
                        )
×
131
                } else {
1✔
132
                        logrus.Infof("Set 'LogLevel' to %s", lvl)
1✔
133
                        logrus.SetLevel(lvl)
1✔
134
                }
1✔
135
        }
136

137
        // Comma separated list of repos to sync (GitHub->GitLab)
138
        reposSyncListRaw, found := os.LookupEnv("SYNC_REPOS_LIST")
1✔
139
        if found {
1✔
140
                reposSyncList = strings.Split(reposSyncListRaw, ",")
×
141
        }
×
142

143
        switch {
1✔
144
        case githubSecret == "" && !dryRunMode:
×
145
                return &config{}, fmt.Errorf("set GITHUB_SECRET")
×
146
        case githubToken == "":
×
147
                return &config{}, fmt.Errorf("set GITHUB_TOKEN")
×
148
        case gitlabToken == "":
×
149
                return &config{}, fmt.Errorf("set GITLAB_TOKEN")
×
150
        case gitlabBaseURL == "":
×
151
                return &config{}, fmt.Errorf("set GITLAB_BASE_URL")
×
152
        case integrationDirectory == "":
×
153
                return &config{}, fmt.Errorf("set INTEGRATION_DIRECTORY")
×
154
        }
155

156
        return &config{
1✔
157
                dryRunMode:             dryRunMode,
1✔
158
                githubSecret:           []byte(githubSecret),
1✔
159
                githubProtocol:         gitProtocolSSH,
1✔
160
                githubToken:            githubToken,
1✔
161
                gitlabToken:            gitlabToken,
1✔
162
                gitlabBaseURL:          gitlabBaseURL,
1✔
163
                integrationDirectory:   integrationDirectory,
1✔
164
                isProcessPushEvents:    isProcessPushEvents,
1✔
165
                isProcessPREvents:      isProcessPREvents,
1✔
166
                isProcessCommentEvents: isProcessCommentEvents,
1✔
167
                reposSyncList:          reposSyncList,
1✔
168
        }, nil
1✔
169
}
170

171
func getCustomLoggerFromContext(ctx *gin.Context) *logrus.Entry {
13✔
172
        deliveryID, ok := ctx.Get("delivery")
13✔
173
        if !ok || !isStringType(deliveryID) {
13✔
174
                return logrus.WithField("delivery", "nil")
×
175
        }
×
176
        return logrus.WithField("delivery", deliveryID)
13✔
177
}
178

179
func isStringType(i interface{}) bool {
13✔
180
        switch i.(type) {
13✔
181
        case string:
13✔
182
                return true
13✔
183
        default:
×
184
                return false
×
185
        }
186
}
187

188
func processGitHubWebhookRequest(
189
        ctx *gin.Context,
190
        payload []byte,
191
        githubClient clientgithub.Client,
192
        conf *config,
193
) {
1✔
194
        webhookType := github.WebHookType(ctx.Request)
1✔
195
        webhookEvent, _ := github.ParseWebHook(webhookType, payload)
1✔
196
        _ = processGitHubWebhook(ctx, webhookType, webhookEvent, githubClient, conf)
1✔
197
}
1✔
198

199
func processGitHubWebhook(
200
        ctx *gin.Context,
201
        webhookType string,
202
        webhookEvent interface{},
203
        githubClient clientgithub.Client,
204
        conf *config,
205
) error {
16✔
206
        githubOrganization, err := getGitHubOrganization(webhookType, webhookEvent)
16✔
207
        if err != nil {
16✔
208
                logrus.Warnln("ignoring event: ", err.Error())
×
209
                return nil
×
210
        }
×
211
        conf.githubOrganization = githubOrganization
16✔
212
        switch webhookType {
16✔
213
        case "pull_request":
3✔
214
                if conf.isProcessPREvents {
5✔
215
                        pr := webhookEvent.(*github.PullRequestEvent)
2✔
216
                        return processGitHubPullRequest(ctx, pr, githubClient, conf)
2✔
217
                } else {
3✔
218
                        logrus.Infof("Webhook event %s processing is skipped", webhookType)
1✔
219
                }
1✔
220
        case "push":
3✔
221
                if conf.isProcessPushEvents {
5✔
222
                        push := webhookEvent.(*github.PushEvent)
2✔
223
                        return processGitHubPush(ctx, push, githubClient, conf)
2✔
224
                } else {
3✔
225
                        logrus.Infof("Webhook event %s processing is skipped", webhookType)
1✔
226
                }
1✔
227
        case "issue_comment":
12✔
228
                if conf.isProcessCommentEvents {
23✔
229
                        comment := webhookEvent.(*github.IssueCommentEvent)
11✔
230
                        return processGitHubComment(ctx, comment, githubClient, conf)
11✔
231
                } else {
12✔
232
                        logrus.Infof("Webhook event %s processing is skipped", webhookType)
1✔
233
                }
1✔
234
        }
235
        return nil
3✔
236
}
237

238
func setupLogging(conf *config, requestLogger logger.RequestLogger) {
2✔
239
        // Log to stdout and with JSON format; suitable for GKE
2✔
240
        formatter := &logrus.JSONFormatter{
2✔
241
                FieldMap: logrus.FieldMap{
2✔
242
                        logrus.FieldKeyTime:  "time",
2✔
243
                        logrus.FieldKeyLevel: "level",
2✔
244
                        logrus.FieldKeyMsg:   "message",
2✔
245
                },
2✔
246
        }
2✔
247

2✔
248
        if conf.dryRunMode {
3✔
249
                mw := io.MultiWriter(os.Stdout, requestLogger)
1✔
250
                logrus.SetOutput(mw)
1✔
251
        } else {
2✔
252
                logrus.SetOutput(os.Stdout)
1✔
253
        }
1✔
254
        logrus.SetFormatter(formatter)
2✔
255
}
256

257
func main() {
×
258
        doMain()
×
259
}
×
260

261
var githubClient clientgithub.Client
262

263
func doMain() {
1✔
264
        conf, err := getConfig()
1✔
265
        if err != nil {
1✔
266
                logrus.Fatalf("failed to load config: %s", err.Error())
×
267
        }
×
268

269
        requestLogger := logger.NewRequestLogger()
1✔
270
        logger.SetRequestLogger(requestLogger)
1✔
271

1✔
272
        setupLogging(conf, requestLogger)
1✔
273
        git.SetDryRunMode(conf.dryRunMode)
1✔
274

1✔
275
        logrus.Infoln("using settings: ", spew.Sdump(conf))
1✔
276

1✔
277
        githubClient = clientgithub.NewGitHubClient(conf.githubToken, conf.dryRunMode)
1✔
278

1✔
279
        r := gin.Default()
1✔
280
        filter := "/_health"
1✔
281
        if logrus.GetLevel() == logrus.DebugLevel || logrus.GetLevel() == logrus.TraceLevel {
2✔
282
                filter = ""
1✔
283
        }
1✔
284
        r.Use(gin.LoggerWithWriter(gin.DefaultWriter, filter))
1✔
285
        r.Use(gin.Recovery())
1✔
286

1✔
287
        // webhook for GitHub
1✔
288
        r.POST("/", func(context *gin.Context) {
2✔
289
                payload, err := github.ValidatePayload(context.Request, conf.githubSecret)
1✔
290
                if err != nil {
1✔
NEW
291
                        var mbErr *http.MaxBytesError
×
NEW
292
                        if errors.As(err, &mbErr) {
×
NEW
293
                                context.Status(http.StatusRequestEntityTooLarge)
×
NEW
294
                                return
×
NEW
295
                        }
×
296
                        logrus.Warnln("payload failed to validate, ignoring.")
×
297
                        context.Status(http.StatusForbidden)
×
298
                        return
×
299
                }
300
                context.Set("delivery", github.DeliveryID(context.Request))
1✔
301
                if conf.dryRunMode {
2✔
302
                        processGitHubWebhookRequest(context, payload, githubClient, conf)
1✔
303
                } else {
1✔
304
                        go processGitHubWebhookRequest(context, payload, githubClient, conf)
×
305
                }
×
306
                context.Status(http.StatusAccepted)
1✔
307
        })
308

309
        // 200 replay for the loadbalancer
310
        r.GET("/_health", func(_ *gin.Context) {})
1✔
311
        r.GET("/", func(_ *gin.Context) {})
1✔
312

313
        // dry-run mode, end-point to retrieve and clear logs
314
        if conf.dryRunMode {
2✔
315
                r.GET("/logs", func(context *gin.Context) {
2✔
316
                        logs := requestLogger.Get()
1✔
317
                        context.JSON(http.StatusOK, logs)
1✔
318
                })
1✔
319

320
                r.DELETE("/logs", func(context *gin.Context) {
2✔
321
                        requestLogger.Clear()
1✔
322
                        context.Writer.WriteHeader(http.StatusNoContent)
1✔
323
                })
1✔
324
        }
325

326
        srv := &http.Server{
1✔
327
                Addr:    "0.0.0.0:8080",
1✔
328
                Handler: http.MaxBytesHandler(r, 10*MiB),
1✔
329
        }
1✔
330

1✔
331
        go func() {
2✔
332
                if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
1✔
333
                        logrus.Fatalf("Failed listening: %s\n", err)
×
334
                }
×
335
        }()
336

337
        quit := make(chan os.Signal, 1)
1✔
338
        signal.Notify(quit, unix.SIGINT, unix.SIGTERM)
1✔
339
        <-quit
1✔
340

1✔
341
        logrus.Info("Shutdown server ...")
1✔
342

1✔
343
        ctx := context.Background()
1✔
344
        ctxWithTimeout, cancel := context.WithTimeout(ctx, 5*time.Second)
1✔
345
        defer cancel()
1✔
346
        if err := srv.Shutdown(ctxWithTimeout); err != nil {
1✔
347
                logrus.Fatal("Failed to shutdown the server: ", err)
×
348
        }
×
349
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc