• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mendersoftware / integration-test-runner / 1760438837

09 Apr 2025 02:39PM UTC coverage: 65.678% (-0.2%) from 65.864%
1760438837

Pull #373

gitlab-ci

alfrunes
feat: Added feature to put option in the PR title to skip CI

If a PR title starts with `[NoCI]` (case insensitive) the CI will not
be started automatically.

Signed-off-by: Alf-Rune Siqveland <alf.rune@northern.tech>
Pull Request #373: QA-972: Add option to skip CI from PR title

16 of 34 new or added lines in 2 files covered. (47.06%)

76 existing lines in 2 files now uncovered.

1948 of 2966 relevant lines covered (65.68%)

2.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.89
/main.go
1
package main
2

3
import (
4
        "context"
5
        "errors"
6
        "fmt"
7
        "io"
8
        "net/http"
9
        "os"
10
        "os/signal"
11
        "strings"
12
        "time"
13

14
        "github.com/davecgh/go-spew/spew"
15
        "github.com/gin-gonic/gin"
16
        "github.com/google/go-github/v28/github"
17
        "github.com/sirupsen/logrus"
18
        "golang.org/x/sys/unix"
19

20
        clientgithub "github.com/mendersoftware/integration-test-runner/client/github"
21
        "github.com/mendersoftware/integration-test-runner/git"
22
        "github.com/mendersoftware/integration-test-runner/logger"
23
)
24

25
type config struct {
26
        dryRunMode             bool
27
        githubSecret           []byte
28
        githubProtocol         gitProtocol
29
        githubOrganization     string
30
        githubToken            string
31
        gitlabToken            string
32
        gitlabBaseURL          string
33
        integrationDirectory   string
34
        isProcessPushEvents    bool
35
        isProcessPREvents      bool
36
        isProcessCommentEvents bool
37
        reposSyncList          []string
38
}
39

40
type buildOptions struct {
41
        pr         string
42
        repo       string
43
        baseBranch string
44
        commitSHA  string
45
        makeQEMU   bool
46
}
47

48
// Mapping https://github.com/<org> -> https://gitlab.com/Northern.tech/<group>
49
var gitHubOrganizationToGitLabGroup = map[string]string{
50
        "mendersoftware": "Mender",
51
        "cfengine":       "CFEngine",
52
        "NorthernTechHQ": "NorthernTechHQ",
53
}
54

55
// Mapping of special repos that have a custom group/project
56
var gitHubRepoToGitLabProjectCustom = map[string]string{
57
        "saas": "Northern.tech/MenderSaaS/saas",
58
}
59

60
var qemuBuildRepositories = []string{
61
        "meta-mender",
62
        "mender",
63
        "mender-artifact",
64
        "mender-connect",
65
        "monitor-client",
66
        "mender-auth-azure-iot",
67
        "mender-gateway",
68
        "mender-snapshot",
69
}
70

71
const (
72
        gitOperationTimeout = 30
73
)
74

75
const (
76
        featureBranchPrefix = "feature-"
77
)
78

79
const (
80
        githubBotName = "mender-test-bot"
81
)
82

83
const (
84
        commandStartIntegrationPipeline = "start integration pipeline"
85
        commandStartClientPipeline      = "start client pipeline"
86
        commandCherryPickBranch         = "cherry-pick to:"
87
        commandConventionalCommit       = "mark-pr as"
88
        commandSyncRepos                = "sync"
89
)
90

91
func getConfig() (*config, error) {
1✔
92
        var reposSyncList []string
1✔
93
        dryRunMode := os.Getenv("DRY_RUN") != ""
1✔
94
        githubSecret := os.Getenv("GITHUB_SECRET")
1✔
95
        githubToken := os.Getenv("GITHUB_TOKEN")
1✔
96
        gitlabToken := os.Getenv("GITLAB_TOKEN")
1✔
97
        gitlabBaseURL := os.Getenv("GITLAB_BASE_URL")
1✔
98
        integrationDirectory := "/integration/"
1✔
99
        if integrationDirEnv := os.Getenv("INTEGRATION_DIRECTORY"); integrationDirEnv != "" {
1✔
UNCOV
100
                integrationDirectory = integrationDirEnv
×
UNCOV
101
        }
×
102

103
        //
104
        // Currently we don't have a distinguishment between GitHub events and features.
105
        // Different features might be implemented across different events, but in future
106
        // it's probability that we might implement proper features selection. For now the
107
        // straight goal is to being able to configure the runner to only sync repos on
108
        // push events and disable all the rest (to be used by the CFEngine team).
109
        //
110
        // default: process push events (sync repos) if not explicitly disabled
111
        isProcessPushEvents := os.Getenv("DISABLE_PUSH_EVENTS_PROCESSING") == ""
1✔
112
        // default: process PR events if not explicitly disabled
1✔
113
        isProcessPREvents := os.Getenv("DISABLE_PR_EVENTS_PROCESSING") == ""
1✔
114
        // default: process comment events if not explicitly disabled
1✔
115
        isProcessCommentEvents := os.Getenv("DISABLE_COMMENT_EVENTS_PROCESSING") == ""
1✔
116

1✔
117
        logLevel, found := os.LookupEnv("INTEGRATION_TEST_RUNNER_LOG_LEVEL")
1✔
118
        logrus.SetLevel(logrus.InfoLevel)
1✔
119
        if found {
2✔
120
                lvl, err := logrus.ParseLevel(logLevel)
1✔
121
                if err != nil {
1✔
UNCOV
122
                        logrus.Infof(
×
UNCOV
123
                                "Failed to parse the 'INTEGRATION_TEST_RUNNER_LOG_LEVEL' variable, " +
×
UNCOV
124
                                        "defaulting to 'InfoLevel'",
×
UNCOV
125
                        )
×
126
                } else {
1✔
127
                        logrus.Infof("Set 'LogLevel' to %s", lvl)
1✔
128
                        logrus.SetLevel(lvl)
1✔
129
                }
1✔
130
        }
131

132
        // Comma separated list of repos to sync (GitHub->GitLab)
133
        reposSyncListRaw, found := os.LookupEnv("SYNC_REPOS_LIST")
1✔
134
        if found {
1✔
UNCOV
135
                reposSyncList = strings.Split(reposSyncListRaw, ",")
×
UNCOV
136
        }
×
137

138
        switch {
1✔
UNCOV
139
        case githubSecret == "" && !dryRunMode:
×
140
                return &config{}, fmt.Errorf("set GITHUB_SECRET")
×
141
        case githubToken == "":
×
UNCOV
142
                return &config{}, fmt.Errorf("set GITHUB_TOKEN")
×
UNCOV
143
        case gitlabToken == "":
×
144
                return &config{}, fmt.Errorf("set GITLAB_TOKEN")
×
145
        case gitlabBaseURL == "":
×
146
                return &config{}, fmt.Errorf("set GITLAB_BASE_URL")
×
147
        case integrationDirectory == "":
×
148
                return &config{}, fmt.Errorf("set INTEGRATION_DIRECTORY")
×
149
        }
150

151
        return &config{
1✔
152
                dryRunMode:             dryRunMode,
1✔
153
                githubSecret:           []byte(githubSecret),
1✔
154
                githubProtocol:         gitProtocolSSH,
1✔
155
                githubToken:            githubToken,
1✔
156
                gitlabToken:            gitlabToken,
1✔
157
                gitlabBaseURL:          gitlabBaseURL,
1✔
158
                integrationDirectory:   integrationDirectory,
1✔
159
                isProcessPushEvents:    isProcessPushEvents,
1✔
160
                isProcessPREvents:      isProcessPREvents,
1✔
161
                isProcessCommentEvents: isProcessCommentEvents,
1✔
162
                reposSyncList:          reposSyncList,
1✔
163
        }, nil
1✔
164
}
165

166
func getCustomLoggerFromContext(ctx *gin.Context) *logrus.Entry {
13✔
167
        deliveryID, ok := ctx.Get("delivery")
13✔
168
        if !ok || !isStringType(deliveryID) {
13✔
UNCOV
169
                return logrus.WithField("delivery", "nil")
×
UNCOV
170
        }
×
171
        return logrus.WithField("delivery", deliveryID)
13✔
172
}
173

174
func isStringType(i interface{}) bool {
13✔
175
        switch i.(type) {
13✔
176
        case string:
13✔
177
                return true
13✔
UNCOV
178
        default:
×
UNCOV
179
                return false
×
180
        }
181
}
182

183
func processGitHubWebhookRequest(
184
        ctx *gin.Context,
185
        payload []byte,
186
        githubClient clientgithub.Client,
187
        conf *config,
188
) {
1✔
189
        webhookType := github.WebHookType(ctx.Request)
1✔
190
        webhookEvent, _ := github.ParseWebHook(webhookType, payload)
1✔
191
        _ = processGitHubWebhook(ctx, webhookType, webhookEvent, githubClient, conf)
1✔
192
}
1✔
193

194
func processGitHubWebhook(
195
        ctx *gin.Context,
196
        webhookType string,
197
        webhookEvent interface{},
198
        githubClient clientgithub.Client,
199
        conf *config,
200
) error {
16✔
201
        githubOrganization, err := getGitHubOrganization(webhookType, webhookEvent)
16✔
202
        if err != nil {
16✔
UNCOV
203
                logrus.Warnln("ignoring event: ", err.Error())
×
UNCOV
204
                return nil
×
UNCOV
205
        }
×
206
        conf.githubOrganization = githubOrganization
16✔
207
        switch webhookType {
16✔
208
        case "pull_request":
3✔
209
                if conf.isProcessPREvents {
5✔
210
                        pr := webhookEvent.(*github.PullRequestEvent)
2✔
211
                        return processGitHubPullRequest(ctx, pr, githubClient, conf)
2✔
212
                } else {
3✔
213
                        logrus.Infof("Webhook event %s processing is skipped", webhookType)
1✔
214
                }
1✔
215
        case "push":
3✔
216
                if conf.isProcessPushEvents {
5✔
217
                        push := webhookEvent.(*github.PushEvent)
2✔
218
                        return processGitHubPush(ctx, push, githubClient, conf)
2✔
219
                } else {
3✔
220
                        logrus.Infof("Webhook event %s processing is skipped", webhookType)
1✔
221
                }
1✔
222
        case "issue_comment":
12✔
223
                if conf.isProcessCommentEvents {
23✔
224
                        comment := webhookEvent.(*github.IssueCommentEvent)
11✔
225
                        return processGitHubComment(ctx, comment, githubClient, conf)
11✔
226
                } else {
12✔
227
                        logrus.Infof("Webhook event %s processing is skipped", webhookType)
1✔
228
                }
1✔
229
        }
230
        return nil
3✔
231
}
232

233
func setupLogging(conf *config, requestLogger logger.RequestLogger) {
2✔
234
        // Log to stdout and with JSON format; suitable for GKE
2✔
235
        formatter := &logrus.JSONFormatter{
2✔
236
                FieldMap: logrus.FieldMap{
2✔
237
                        logrus.FieldKeyTime:  "time",
2✔
238
                        logrus.FieldKeyLevel: "level",
2✔
239
                        logrus.FieldKeyMsg:   "message",
2✔
240
                },
2✔
241
        }
2✔
242

2✔
243
        if conf.dryRunMode {
3✔
244
                mw := io.MultiWriter(os.Stdout, requestLogger)
1✔
245
                logrus.SetOutput(mw)
1✔
246
        } else {
2✔
247
                logrus.SetOutput(os.Stdout)
1✔
248
        }
1✔
249
        logrus.SetFormatter(formatter)
2✔
250
}
251

UNCOV
252
func main() {
×
UNCOV
253
        doMain()
×
UNCOV
254
}
×
255

256
var githubClient clientgithub.Client
257

258
func doMain() {
1✔
259
        conf, err := getConfig()
1✔
260
        if err != nil {
1✔
UNCOV
261
                logrus.Fatalf("failed to load config: %s", err.Error())
×
UNCOV
262
        }
×
263

264
        requestLogger := logger.NewRequestLogger()
1✔
265
        logger.SetRequestLogger(requestLogger)
1✔
266

1✔
267
        setupLogging(conf, requestLogger)
1✔
268
        git.SetDryRunMode(conf.dryRunMode)
1✔
269

1✔
270
        logrus.Infoln("using settings: ", spew.Sdump(conf))
1✔
271

1✔
272
        githubClient = clientgithub.NewGitHubClient(conf.githubToken, conf.dryRunMode)
1✔
273

1✔
274
        r := gin.Default()
1✔
275
        filter := "/_health"
1✔
276
        if logrus.GetLevel() == logrus.DebugLevel || logrus.GetLevel() == logrus.TraceLevel {
2✔
277
                filter = ""
1✔
278
        }
1✔
279
        r.Use(gin.LoggerWithWriter(gin.DefaultWriter, filter))
1✔
280
        r.Use(gin.Recovery())
1✔
281

1✔
282
        // webhook for GitHub
1✔
283
        r.POST("/", func(context *gin.Context) {
2✔
284
                payload, err := github.ValidatePayload(context.Request, conf.githubSecret)
1✔
285
                if err != nil {
1✔
NEW
UNCOV
286
                        var mbErr *http.MaxBytesError
×
NEW
UNCOV
287
                        if errors.As(err, &mbErr) {
×
NEW
UNCOV
288
                                context.Status(http.StatusRequestEntityTooLarge)
×
NEW
UNCOV
289
                                return
×
NEW
UNCOV
290
                        }
×
291
                        logrus.Warnln("payload failed to validate, ignoring.")
×
292
                        context.Status(http.StatusForbidden)
×
293
                        return
×
294
                }
295
                context.Set("delivery", github.DeliveryID(context.Request))
1✔
296
                if conf.dryRunMode {
2✔
297
                        processGitHubWebhookRequest(context, payload, githubClient, conf)
1✔
298
                } else {
1✔
299
                        go processGitHubWebhookRequest(context, payload, githubClient, conf)
×
UNCOV
300
                }
×
301
                context.Status(http.StatusAccepted)
1✔
302
        })
303

304
        // 200 replay for the loadbalancer
305
        r.GET("/_health", func(_ *gin.Context) {})
1✔
306
        r.GET("/", func(_ *gin.Context) {})
1✔
307

308
        // dry-run mode, end-point to retrieve and clear logs
309
        if conf.dryRunMode {
2✔
310
                r.GET("/logs", func(context *gin.Context) {
2✔
311
                        logs := requestLogger.Get()
1✔
312
                        context.JSON(http.StatusOK, logs)
1✔
313
                })
1✔
314

315
                r.DELETE("/logs", func(context *gin.Context) {
2✔
316
                        requestLogger.Clear()
1✔
317
                        context.Writer.WriteHeader(http.StatusNoContent)
1✔
318
                })
1✔
319
        }
320

321
        srv := &http.Server{
1✔
322
                Addr:    "0.0.0.0:8080",
1✔
323
                Handler: http.MaxBytesHandler(r, 10*1024*1024),
1✔
324
        }
1✔
325

1✔
326
        go func() {
2✔
327
                if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
1✔
328
                        logrus.Fatalf("Failed listening: %s\n", err)
×
UNCOV
329
                }
×
330
        }()
331

332
        quit := make(chan os.Signal, 1)
1✔
333
        signal.Notify(quit, unix.SIGINT, unix.SIGTERM)
1✔
334
        <-quit
1✔
335

1✔
336
        logrus.Info("Shutdown server ...")
1✔
337

1✔
338
        ctx := context.Background()
1✔
339
        ctxWithTimeout, cancel := context.WithTimeout(ctx, 5*time.Second)
1✔
340
        defer cancel()
1✔
341
        if err := srv.Shutdown(ctxWithTimeout); err != nil {
1✔
UNCOV
342
                logrus.Fatal("Failed to shutdown the server: ", err)
×
UNCOV
343
        }
×
344
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc