1681953125

Committed 21 Feb 2025 08:24AM UTC coverage: 59.965% (-4.4%) from 64.328%

Build # 1681953125

Build Type

Pull #357

gitlab-ci

Committed by

danielskinstad

Commit Message

feat: run integration pipeline on demand

Allow you to run the full integration pipeline on a protected branch
when a trusted member runs `start integration pipeline`.

As of now, this will only run with main - and you cannot specify PRs to
run the pipeline with.

Ticket: QA-863

Signed-off-by: Daniel Skinstad Drabitzius <daniel.drabitzius@northern.tech>

Pull Request Pull Request #357: wip: integration tests on demand

Run Details

77 of 311 new or added lines in 7 files covered. (24.76%)

2 existing lines in 1 file now uncovered.

1736 of 2895 relevant lines covered (59.97%)

2.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.84

/menderqa_pipeline.go

package main

import (
        "bytes"
        "context"
        "strconv"
        "strings"
        "text/template"

        "github.com/google/go-github/v28/github"
        "github.com/sirupsen/logrus"
        gitlab "gitlab.com/gitlab-org/api/client-go"

        clientgitlab "github.com/mendersoftware/integration-test-runner/client/gitlab"
)

const clientPipelinePath = "Northern.tech/Mender/mender-qa"
const LatestStableYoctoBranch = "kirkstone"

func parseClientPullRequest(
        log *logrus.Entry,
        conf *config,
        action string,
        pr *github.PullRequestEvent,
) []buildOptions {
        log.Info("Pull request event with action: ", action)
        var builds []buildOptions

        // github pull request events to trigger a CI job for
        if action == "opened" || action == "edited" || action == "reopened" ||
                action == "synchronize" || action == "ready_for_review" {

                return getClientBuilds(log, conf, pr)
        }

        return builds
}

func getClientBuilds(log *logrus.Entry, conf *config, pr *github.PullRequestEvent) []buildOptions {

        var builds []buildOptions

        repo := pr.GetRepo().GetName()

        commitSHA := pr.PullRequest.Head.GetSHA()
        //GetLabel returns "mendersoftware:master", we just want the branch
        baseBranch := strings.Split(pr.PullRequest.Base.GetLabel(), ":")[1]

        makeQEMU := false

        // we need to have the latest integration/master branch in order to use the release_tool.py
        if err := updateIntegrationRepo(conf); err != nil {
                log.Warn(err.Error())
        }

        allRepositories, err := getListOfAllRepositories(conf)
        if err != nil {
                log.Warn(err.Error())
        }

        for _, watchRepo := range allRepositories {
                // make sure the repo that the pull request is performed against is
                // one that we are watching.

                if watchRepo == repo {

                        // check if we need to build/test yocto
                        for _, qemuRepo := range qemuBuildRepositories {
                                if repo == qemuRepo {
                                        makeQEMU = true
                                }
                        }

                        switch repo {
                        case "meta-mender", "integration":
                                build := buildOptions{
                                        pr:         strconv.Itoa(pr.GetNumber()),
                                        repo:       repo,
                                        baseBranch: baseBranch,
                                        commitSHA:  commitSHA,
                                        makeQEMU:   makeQEMU,
                                }
                                builds = append(builds, build)

                        default:
                                var err error
                                var integrationsToTest []string

                                if integrationsToTest, err = getIntegrationVersionsUsingMicroservice(
                                        log,
                                        repo,
                                        baseBranch,
                                        conf,
                                ); err != nil {
                                        log.Errorf(
                                                "failed to get related microservices for repo: %s version: %s, failed "+
                                                        "with: %s\n",
                                                repo,
                                                baseBranch,
                                                err.Error(),
                                        )
                                        return nil
                                }
                                log.Infof(
                                        "the following integration branches: %s are using %s/%s",
                                        integrationsToTest,
                                        repo,
                                        baseBranch,
                                )

                                // one pull request can trigger multiple builds
                                for _, integrationBranch := range integrationsToTest {
                                        buildOpts := buildOptions{
                                                pr:         strconv.Itoa(pr.GetNumber()),
                                                repo:       repo,
                                                baseBranch: integrationBranch,
                                                commitSHA:  commitSHA,
                                                makeQEMU:   makeQEMU,
                                        }
                                        builds = append(builds, buildOpts)
                                }
                        }

                }
        }
        return builds
}

func isLegacyBuild(build *buildOptions, buildParameters []*gitlab.PipelineVariableOptions) bool {
        // We define a legacy build as either:
        // * A PR in integration repo with baseBranch 3.7.x
        // * A PR in any repo for which INTEGRATION_REV variable is set to 3.7.x
        if build.repo == "integration" {
                return build.baseBranch == "3.7.x"
        }
        for _, param := range buildParameters {
                if *param.Key == "INTEGRATION_REV" {
                        return *param.Value == "3.7.x"
                }
        }
        // This should never happen, INTEGRATION_REV must be found! But just in case...
        return false
}

func getMenderQARef(build *buildOptions, buildParameters []*gitlab.PipelineVariableOptions) string {
        if isLegacyBuild(build, buildParameters) {
                return "legacy-mender-3.7-lts"
        }
        return "master"
}

func triggerClientBuild(
        log *logrus.Entry,
        conf *config,
        build *buildOptions,
        pr *github.PullRequestEvent,
        buildOptions *BuildOptions,
) error {
        gitlabClient, err := clientgitlab.NewGitLabClient(
                conf.gitlabToken,
                conf.gitlabBaseURL,
                conf.dryRunMode,
        )
        if err != nil {
                return err
        }

        buildParameters, err := getClientBuildParameters(log, conf, build, buildOptions)
        if err != nil {
                return err
        }

        // first stop old pipelines with the same buildParameters
        stopStalePipelines(clientPipelinePath, log, gitlabClient, buildParameters)

        // trigger the new pipeline
        clientPipelinePath := "Northern.tech/Mender/mender-qa"
        ref := getMenderQARef(build, buildParameters)
        opt := &gitlab.CreatePipelineOptions{
                Ref:       &ref,
                Variables: &buildParameters,
        }

        variablesString := ""
        for _, variable := range *opt.Variables {
                variablesString += *variable.Key + ":" + *variable.Value + ", "
        }
        log.Infof(
                "Creating pipeline in project %s:%s with variables: %s",
                clientPipelinePath,
                *opt.Ref,
                variablesString,
        )

        pipeline, err := gitlabClient.CreatePipeline(clientPipelinePath, opt)
        if err != nil {
                log.Errorf("Could not create pipeline: %s", err.Error())
                return err
        }
        log.Infof("Created pipeline: %s", pipeline.WebURL)

        // Add the build variable matrix to the pipeline comment under a
        // drop-down tab
        // nolint:lll
        tmplString := `
Hello :smiley_cat: I created a pipeline for you here: [Pipeline-{{.Pipeline.ID}}]({{.Pipeline.WebURL}})

<details>
    <summary>Build Configuration Matrix</summary><p>

| Key   | Value |
| ----- | ----- |
{{range $i, $var := .BuildVars}}{{if $var.Value}}| {{$var.Key}} | {{$var.Value}} |{{printf "\n"}}{{end}}{{end}}

 </p></details>
`
        tmpl, err := template.New("Main").Parse(tmplString)
        if err != nil {
                log.Errorf(
                        "Failed to parse the build matrix template. Should never happen! Error: %s\n",
                        err.Error(),
                )
                return err
        }
        var buf bytes.Buffer
        if err = tmpl.Execute(&buf, struct {
                BuildVars []*gitlab.PipelineVariableOptions
                Pipeline  *gitlab.Pipeline
        }{
                BuildVars: filterOutEmptyVariables(*opt.Variables),
                Pipeline:  pipeline,
        }); err != nil {
                log.Errorf("Failed to execute the build matrix template. Error: %s\n", err.Error())
                return err
        }

        // Comment with a pipeline-link on the PR
        commentBody := buf.String()
        comment := github.IssueComment{
                Body: &commentBody,
        }

        err = githubClient.CreateComment(context.Background(),
                conf.githubOrganization, pr.GetRepo().GetName(), pr.GetNumber(), &comment)
        if err != nil {
                log.Infof("Failed to comment on the pr: %v, Error: %s", pr, err.Error())
        }

        return err
}

func getClientBuildParameters(
        log *logrus.Entry,
        conf *config,
        build *buildOptions,
        buildOptions *BuildOptions,
) ([]*gitlab.PipelineVariableOptions, error) {
        var err error
        readHead := "pull/" + build.pr + "/head"
        var buildParameters []*gitlab.PipelineVariableOptions

        var versionedRepositories []string
        if build.repo == "meta-mender" {
                // For meta-mender, pick master versions of all Mender release repos.
                versionedRepositories, err = getListOfVersionedRepositories("origin/master", conf)
        } else {
                versionedRepositories, err = getListOfVersionedRepositories(
                        "origin/"+build.baseBranch,
                        conf,
                )
        }
        if err != nil {
                log.Errorf("Could not get list of repositories: %s", err.Error())
                return nil, err
        }

        for _, versionedRepo := range versionedRepositories {
                // iterate over all the repositories (except the one we are testing) and
                // set the correct microservice versions

                // use the default "master" for both mender-qa, and meta-mender (set in CI)
                if versionedRepo != build.repo &&
                        versionedRepo != "integration" &&
                        build.repo != "meta-mender" {

                        repoKey := repoToBuildParameter(versionedRepo)

                        if _, exists := buildOptions.PullRequests[versionedRepo]; exists {
                                prVersion := buildOptions.PullRequests[versionedRepo]
                                buildParameters = append(
                                        buildParameters,
                                        &gitlab.PipelineVariableOptions{
                                                Key:   &repoKey,
                                                Value: &prVersion,
                                        },
                                )
                                continue
                        }
                        version, err := getServiceRevisionFromIntegration(
                                versionedRepo,
                                "origin/"+build.baseBranch,
                                conf,
                        )
                        if err != nil {
                                log.Errorf("failed to determine %s version: %s", versionedRepo, err.Error())
                                return nil, err
                        }
                        log.Infof("%s version %s is being used in %s", versionedRepo, version, build.baseBranch)
                        buildParameters = append(
                                buildParameters,
                                &gitlab.PipelineVariableOptions{
                                        Key:   &repoKey,
                                        Value: &version,
                                },
                        )
                }
        }

        // set the correct integration branches if we aren't performing a pull request against
        // integration
        if build.repo != "integration" && build.repo != "meta-mender" {
                integrationRevision := build.baseBranch
                if _, exists := buildOptions.PullRequests["integration"]; exists {
                        integrationRevision = buildOptions.PullRequests["integration"]
                }
                integrationRepoKey := repoToBuildParameter("integration")
                buildParameters = append(buildParameters,
                        &gitlab.PipelineVariableOptions{
                                Key:   &integrationRepoKey,
                                Value: &integrationRevision})

                if _, exists := buildOptions.PullRequests["meta-mender"]; exists {
                        metaMenderRevision := buildOptions.PullRequests["meta-mender"]
                        metaMenderRepoKey := repoToBuildParameter("meta-mender")
                        buildParameters = append(buildParameters,
                                &gitlab.PipelineVariableOptions{
                                        Key:   &metaMenderRepoKey,
                                        Value: &metaMenderRevision})
                }
        }

        // Set poky (& friends) and meta-mender revisions:
        // - If building a master PR, leave everything at defaults, which generally means
        //   meta-mender/master and poky/LatestStableYoctoBranch.
        // - If building meta-mender @ non-master, set poky branches to its baseBranch.
        // - If building any other repo @ non-master, set both meta-mender and poky to
        //   LatestStableYoctoBranch.
        if build.baseBranch != "master" {
                var pokyBranch string
                if build.repo == "meta-mender" {
                        pokyBranch = build.baseBranch
                } else {
                        pokyBranch = LatestStableYoctoBranch
                        metaMenderBranch := pokyBranch
                        metaMenderBranchKey := repoToBuildParameter("meta-mender")
                        buildParameters = append(
                                buildParameters,
                                &gitlab.PipelineVariableOptions{
                                        Key:   &metaMenderBranchKey,
                                        Value: &metaMenderBranch,
                                },
                        )
                }
                pokyBranchKey := repoToBuildParameter("poky")
                buildParameters = append(
                        buildParameters,
                        &gitlab.PipelineVariableOptions{Key: &pokyBranchKey, Value: &pokyBranch},
                )
                metaOEPokyBranchKey := repoToBuildParameter("meta-openembedded")
                buildParameters = append(
                        buildParameters,
                        &gitlab.PipelineVariableOptions{
                                Key:   &metaOEPokyBranchKey,
                                Value: &pokyBranch,
                        },
                )
                metaRPIPokyBranchKey := repoToBuildParameter("meta-raspberrypi")
                buildParameters = append(
                        buildParameters,
                        &gitlab.PipelineVariableOptions{
                                Key:   &metaRPIPokyBranchKey,
                                Value: &pokyBranch,
                        },
                )
        }

        // set the rest of the CI build parameters
        runIntegrationTests := "true"
        runIntegrationTestsKey := "RUN_INTEGRATION_TESTS"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &runIntegrationTestsKey, Value: &runIntegrationTests},
        )

        // Backend integration tests, from mender-qa pipeline, are only relevant for legacy builds
        if isLegacyBuild(build, buildParameters) {
                runBackendIntegrationTests := "true"
                if buildOptions.Fast {
                        runIntegrationTests = "false"
                }

                runBackendIntegrationTestsKey := "RUN_BACKEND_INTEGRATION_TESTS"
                buildParameters = append(
                        buildParameters,
                        &gitlab.PipelineVariableOptions{
                                Key: &runBackendIntegrationTestsKey, Value: &runBackendIntegrationTests,
                        },
                )
        }

        buildRepoKey := repoToBuildParameter(build.repo)
        buildParameters = append(buildParameters,
                &gitlab.PipelineVariableOptions{
                        Key:   &buildRepoKey,
                        Value: &readHead,
                })

        return _getClientBuildParameters(buildParameters, build)
}

func _getClientBuildParameters(
        buildParameters []*gitlab.PipelineVariableOptions,
        build *buildOptions,
) ([]*gitlab.PipelineVariableOptions, error) {
        var qemuParam string
        if build.makeQEMU {
                qemuParam = "true"
        } else {
                qemuParam = ""
        }

        buildX86UefiGrubKey := "BUILD_QEMUX86_64_UEFI_GRUB"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &buildX86UefiGrubKey, Value: &qemuParam},
        )
        testX86UefiGrubKey := "TEST_QEMUX86_64_UEFI_GRUB"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &testX86UefiGrubKey, Value: &qemuParam},
        )

        buildX86BiosGrubKey := "BUILD_QEMUX86_64_BIOS_GRUB"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &buildX86BiosGrubKey, Value: &qemuParam},
        )
        testX86BiosGrubKey := "TEST_QEMUX86_64_BIOS_GRUB"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &testX86BiosGrubKey, Value: &qemuParam},
        )

        buildX86BiosGrubGptKey := "BUILD_QEMUX86_64_BIOS_GRUB_GPT"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &buildX86BiosGrubGptKey, Value: &qemuParam},
        )
        testX86BiosGrubGptKey := "TEST_QEMUX86_64_BIOS_GRUB_GPT"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &testX86BiosGrubGptKey, Value: &qemuParam},
        )

        buildVexpress := "BUILD_VEXPRESS_QEMU"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &buildVexpress, Value: &qemuParam},
        )
        testVexpress := "TEST_VEXPRESS_QEMU"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &testVexpress, Value: &qemuParam},
        )

        buildVexpressFlash := "BUILD_VEXPRESS_QEMU_FLASH"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &buildVexpressFlash, Value: &qemuParam},
        )
        testVexpressFlash := "TEST_VEXPRESS_QEMU_FLASH"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &testVexpressFlash, Value: &qemuParam},
        )

        buildVexpressUbootUefiGrub := "BUILD_VEXPRESS_QEMU_UBOOT_UEFI_GRUB"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &buildVexpressUbootUefiGrub, Value: &qemuParam},
        )
        testVexpressUbootUefiGrub := "TEST_VEXPRESS_QEMU_UBOOT_UEFI_GRUB"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &testVexpressUbootUefiGrub, Value: &qemuParam},
        )

        buildBBBKey := "BUILD_BEAGLEBONEBLACK"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &buildBBBKey, Value: &qemuParam},
        )

        // Set BUILD_CLIENT = false, if target repo not in the qemuBuildRepositories list
        buildClientKey := "BUILD_CLIENT"
        buildClient := "false"
        for _, prebuiltClientRepo := range qemuBuildRepositories {
                if build.repo == prebuiltClientRepo {
                        buildClient = "true"
                }
        }
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &buildClientKey, Value: &buildClient},
        )

        return buildParameters, nil
}

// stopBuildsOfStaleClientPRs stops any running pipelines on a PR which has been merged.
func stopBuildsOfStaleClientPRs(
        log *logrus.Entry,
        pr *github.PullRequestEvent,
        conf *config) error {

        // If the action is "closed" the pull request was merged or just closed,
        // stop builds in both cases.
        if pr.GetAction() != "closed" {
                log.Debugf("stopBuildsOfStaleClientPRs: PR not closed, " +
                        "therefore not stopping it's pipeline")
                return nil
        }

        log.Debug("stopBuildsOfStaleClientPRs: Find any running pipelines and kill mercilessly!")

        for _, build := range getClientBuilds(log, conf, pr) {

                gitlabClient, err := clientgitlab.NewGitLabClient(
                        conf.gitlabToken,
                        conf.gitlabBaseURL,
                        conf.dryRunMode,
                )
                if err != nil {
                        return err
                }

                buildParams, err := getClientBuildParameters(log, conf, &build, NewBuildOptions())
                if err != nil {
                        log.Debug("stopBuildsOfStaleClientPRs: Failed to get the" +
                                "build-parameters for the build")
                        return err
                }

                stopStalePipelines(clientPipelinePath, log, gitlabClient, buildParams)
        }

        return nil

}

1	package main
2
3	import (
4	"bytes"
5	"context"
6	"strconv"
7	"strings"
8	"text/template"
9
10	"github.com/google/go-github/v28/github"
11	"github.com/sirupsen/logrus"
12	gitlab "gitlab.com/gitlab-org/api/client-go"
13
14	clientgitlab "github.com/mendersoftware/integration-test-runner/client/gitlab"
15	)
16
17	const clientPipelinePath = "Northern.tech/Mender/mender-qa"
18	const LatestStableYoctoBranch = "kirkstone"
19
20	func parseClientPullRequest(
21	log *logrus.Entry,
22	conf *config,
23	action string,
24	pr *github.PullRequestEvent,
25	) []buildOptions {	4✔
26	log.Info("Pull request event with action: ", action)	4✔
27	var builds []buildOptions	4✔
28		4✔
29	// github pull request events to trigger a CI job for	4✔
30	if action == "opened" \|\| action == "edited" \|\| action == "reopened" \|\|	4✔
31	action == "synchronize" \|\| action == "ready_for_review" {	7✔
32		3✔
33	return getClientBuilds(log, conf, pr)	3✔
34	}	3✔
35
36	return builds	2✔
37	}
38
39	func getClientBuilds(log logrus.Entry, conf config, pr *github.PullRequestEvent) []buildOptions {	3✔
40		3✔
41	var builds []buildOptions	3✔
42		3✔
43	repo := pr.GetRepo().GetName()	3✔
44		3✔
45	commitSHA := pr.PullRequest.Head.GetSHA()	3✔
46	//GetLabel returns "mendersoftware:master", we just want the branch	3✔
47	baseBranch := strings.Split(pr.PullRequest.Base.GetLabel(), ":")[1]	3✔
48		3✔
49	makeQEMU := false	3✔
50		3✔
51	// we need to have the latest integration/master branch in order to use the release_tool.py	3✔
52	if err := updateIntegrationRepo(conf); err != nil {	3✔
53	log.Warn(err.Error())	×
54	}	×
55
56	allRepositories, err := getListOfAllRepositories(conf)	3✔
57	if err != nil {	5✔
58	log.Warn(err.Error())	2✔
59	}	2✔
60
61	for _, watchRepo := range allRepositories {	4✔
62	// make sure the repo that the pull request is performed against is	1✔
63	// one that we are watching.	1✔
64		1✔
65	if watchRepo == repo {	2✔
66		1✔
67	// check if we need to build/test yocto	1✔
68	for _, qemuRepo := range qemuBuildRepositories {	2✔
69	if repo == qemuRepo {	2✔
70	makeQEMU = true	1✔
71	}	1✔
72	}
73
74	switch repo {	1✔
75	case "meta-mender", "integration":	×
76	build := buildOptions{	×
77	pr: strconv.Itoa(pr.GetNumber()),	×
78	repo: repo,	×
79	baseBranch: baseBranch,	×
80	commitSHA: commitSHA,	×
81	makeQEMU: makeQEMU,	×
82	}	×
83	builds = append(builds, build)	×
84
85	default:	1✔
86	var err error	1✔
87	var integrationsToTest []string	1✔
88		1✔
89	if integrationsToTest, err = getIntegrationVersionsUsingMicroservice(	1✔
90	log,	1✔
91	repo,	1✔
92	baseBranch,	1✔
93	conf,	1✔
94	); err != nil {	1✔
95	log.Errorf(	×
96	"failed to get related microservices for repo: %s version: %s, failed "+	×
97	"with: %s\n",	×
98	repo,	×
99	baseBranch,	×
100	err.Error(),	×
101	)	×
102	return nil	×
103	}	×
104	log.Infof(	1✔
105	"the following integration branches: %s are using %s/%s",	1✔
106	integrationsToTest,	1✔
107	repo,	1✔
108	baseBranch,	1✔
109	)	1✔
110		1✔
111	// one pull request can trigger multiple builds	1✔
112	for _, integrationBranch := range integrationsToTest {	2✔
113	buildOpts := buildOptions{	1✔
114	pr: strconv.Itoa(pr.GetNumber()),	1✔
115	repo: repo,	1✔
116	baseBranch: integrationBranch,	1✔
117	commitSHA: commitSHA,	1✔
118	makeQEMU: makeQEMU,	1✔
119	}	1✔
120	builds = append(builds, buildOpts)	1✔
121	}	1✔
122	}
123
124	}
125	}
126	return builds	3✔
127	}
128
129	func isLegacyBuild(build buildOptions, buildParameters []gitlab.PipelineVariableOptions) bool {	1✔
130	// We define a legacy build as either:	1✔
131	// * A PR in integration repo with baseBranch 3.7.x	1✔
132	// * A PR in any repo for which INTEGRATION_REV variable is set to 3.7.x	1✔
133	if build.repo == "integration" {	1✔
134	return build.baseBranch == "3.7.x"	×
135	}	×
136	for _, param := range buildParameters {	2✔
137	if *param.Key == "INTEGRATION_REV" {	2✔
138	return *param.Value == "3.7.x"	1✔
139	}	1✔
140	}
141	// This should never happen, INTEGRATION_REV must be found! But just in case...
142	return false	×
143	}
144
145	func getMenderQARef(build buildOptions, buildParameters []gitlab.PipelineVariableOptions) string {	1✔
146	if isLegacyBuild(build, buildParameters) {	1✔
147	return "legacy-mender-3.7-lts"	×
148	}	×
149	return "master"	1✔
150	}
151
152	func triggerClientBuild(
153	log *logrus.Entry,
154	conf *config,
155	build *buildOptions,
156	pr *github.PullRequestEvent,
157	buildOptions *BuildOptions,
158	) error {	1✔
159	gitlabClient, err := clientgitlab.NewGitLabClient(	1✔
160	conf.gitlabToken,	1✔
161	conf.gitlabBaseURL,	1✔
162	conf.dryRunMode,	1✔
163	)	1✔
164	if err != nil {	1✔
165	return err	×
166	}	×
167
168	buildParameters, err := getClientBuildParameters(log, conf, build, buildOptions)	1✔
169	if err != nil {	1✔
170	return err	×
171	}	×
172
173	// first stop old pipelines with the same buildParameters
174	stopStalePipelines(clientPipelinePath, log, gitlabClient, buildParameters)	1✔
175		1✔
176	// trigger the new pipeline	1✔
177	clientPipelinePath := "Northern.tech/Mender/mender-qa"	1✔
178	ref := getMenderQARef(build, buildParameters)	1✔
179	opt := &gitlab.CreatePipelineOptions{	1✔
180	Ref: &ref,	1✔
181	Variables: &buildParameters,	1✔
182	}	1✔
183		1✔
184	variablesString := ""	1✔
185	for _, variable := range *opt.Variables {	2✔
186	variablesString += variable.Key + ":" + variable.Value + ", "	1✔
187	}	1✔
188	log.Infof(	1✔
189	"Creating pipeline in project %s:%s with variables: %s",	1✔
190	clientPipelinePath,	1✔
191	*opt.Ref,	1✔
192	variablesString,	1✔
193	)	1✔
194		1✔
195	pipeline, err := gitlabClient.CreatePipeline(clientPipelinePath, opt)	1✔
196	if err != nil {	1✔
197	log.Errorf("Could not create pipeline: %s", err.Error())	×
198	return err	×
199	}	×
200	log.Infof("Created pipeline: %s", pipeline.WebURL)	1✔
201		1✔
202	// Add the build variable matrix to the pipeline comment under a	1✔
203	// drop-down tab	1✔
204	// nolint:lll	1✔
205	tmplString := `	1✔
206	Hello :smiley_cat: I created a pipeline for you here: [Pipeline-{{.Pipeline.ID}}]({{.Pipeline.WebURL}})	1✔
207		1✔
208	<details>	1✔
209	<summary>Build Configuration Matrix</summary><p>	1✔
210		1✔
211	\| Key \| Value \|	1✔
212	\| ----- \| ----- \|	1✔
213	{{range $i, $var := .BuildVars}}{{if $var.Value}}\| {{$var.Key}} \| {{$var.Value}} \|{{printf "\n"}}{{end}}{{end}}	1✔
214		1✔
215	</p></details>	1✔
216	`	1✔
217	tmpl, err := template.New("Main").Parse(tmplString)	1✔
218	if err != nil {	1✔
219	log.Errorf(	×
220	"Failed to parse the build matrix template. Should never happen! Error: %s\n",	×
221	err.Error(),	×
222	)	×
223	return err	×
224	}	×
225	var buf bytes.Buffer	1✔
226	if err = tmpl.Execute(&buf, struct {	1✔
227	BuildVars []*gitlab.PipelineVariableOptions	1✔
228	Pipeline *gitlab.Pipeline	1✔
229	}{	1✔
230	BuildVars: filterOutEmptyVariables(*opt.Variables),	1✔
231	Pipeline: pipeline,	1✔
232	}); err != nil {	1✔
233	log.Errorf("Failed to execute the build matrix template. Error: %s\n", err.Error())	×
234	return err	×
235	}	×
236
237	// Comment with a pipeline-link on the PR
238	commentBody := buf.String()	1✔
239	comment := github.IssueComment{	1✔
240	Body: &commentBody,	1✔
241	}	1✔
242		1✔
243	err = githubClient.CreateComment(context.Background(),	1✔
244	conf.githubOrganization, pr.GetRepo().GetName(), pr.GetNumber(), &comment)	1✔
245	if err != nil {	1✔
246	log.Infof("Failed to comment on the pr: %v, Error: %s", pr, err.Error())	×
247	}	×
248
249	return err	1✔
250	}
251
252	func getClientBuildParameters(
253	log *logrus.Entry,
254	conf *config,
255	build *buildOptions,
256	buildOptions *BuildOptions,
257	) ([]*gitlab.PipelineVariableOptions, error) {	1✔
258	var err error	1✔
259	readHead := "pull/" + build.pr + "/head"	1✔
260	var buildParameters []*gitlab.PipelineVariableOptions	1✔
261		1✔
262	var versionedRepositories []string	1✔
263	if build.repo == "meta-mender" {	1✔
264	// For meta-mender, pick master versions of all Mender release repos.	×
265	versionedRepositories, err = getListOfVersionedRepositories("origin/master", conf)	×
266	} else {	1✔
267	versionedRepositories, err = getListOfVersionedRepositories(	1✔
268	"origin/"+build.baseBranch,	1✔
269	conf,	1✔
270	)	1✔
271	}	1✔
272	if err != nil {	1✔
273	log.Errorf("Could not get list of repositories: %s", err.Error())	×
274	return nil, err	×
275	}	×
276
277	for _, versionedRepo := range versionedRepositories {	2✔
278	// iterate over all the repositories (except the one we are testing) and	1✔
279	// set the correct microservice versions	1✔
280		1✔
281	// use the default "master" for both mender-qa, and meta-mender (set in CI)	1✔
282	if versionedRepo != build.repo &&	1✔
283	versionedRepo != "integration" &&	1✔
284	build.repo != "meta-mender" {	2✔
285		1✔
286	repoKey := repoToBuildParameter(versionedRepo)	1✔
287		1✔
288	if _, exists := buildOptions.PullRequests[versionedRepo]; exists {	2✔
289	prVersion := buildOptions.PullRequests[versionedRepo]	1✔
290	buildParameters = append(	1✔
291	buildParameters,	1✔
292	&gitlab.PipelineVariableOptions{	1✔
293	Key: &repoKey,	1✔
294	Value: &prVersion,	1✔
295	},	1✔
296	)	1✔
297	continue	1✔
298	}
299	version, err := getServiceRevisionFromIntegration(	1✔
300	versionedRepo,	1✔
301	"origin/"+build.baseBranch,	1✔
302	conf,	1✔
303	)	1✔
304	if err != nil {	1✔
305	log.Errorf("failed to determine %s version: %s", versionedRepo, err.Error())	×
306	return nil, err	×
307	}	×
308	log.Infof("%s version %s is being used in %s", versionedRepo, version, build.baseBranch)	1✔
309	buildParameters = append(	1✔
310	buildParameters,	1✔
311	&gitlab.PipelineVariableOptions{	1✔
312	Key: &repoKey,	1✔
313	Value: &version,	1✔
314	},	1✔
315	)	1✔
316	}
317	}
318
319	// set the correct integration branches if we aren't performing a pull request against
320	// integration
321	if build.repo != "integration" && build.repo != "meta-mender" {	2✔
322	integrationRevision := build.baseBranch	1✔
323	if _, exists := buildOptions.PullRequests["integration"]; exists {	2✔
324	integrationRevision = buildOptions.PullRequests["integration"]	1✔
325	}	1✔
326	integrationRepoKey := repoToBuildParameter("integration")	1✔
327	buildParameters = append(buildParameters,	1✔
328	&gitlab.PipelineVariableOptions{	1✔
329	Key: &integrationRepoKey,	1✔
330	Value: &integrationRevision})	1✔
331		1✔
332	if _, exists := buildOptions.PullRequests["meta-mender"]; exists {	2✔
333	metaMenderRevision := buildOptions.PullRequests["meta-mender"]	1✔
334	metaMenderRepoKey := repoToBuildParameter("meta-mender")	1✔
335	buildParameters = append(buildParameters,	1✔
336	&gitlab.PipelineVariableOptions{	1✔
337	Key: &metaMenderRepoKey,	1✔
338	Value: &metaMenderRevision})	1✔
339	}	1✔
340	}
341
342	// Set poky (& friends) and meta-mender revisions:
343	// - If building a master PR, leave everything at defaults, which generally means
344	// meta-mender/master and poky/LatestStableYoctoBranch.
345	// - If building meta-mender @ non-master, set poky branches to its baseBranch.
346	// - If building any other repo @ non-master, set both meta-mender and poky to
347	// LatestStableYoctoBranch.
348	if build.baseBranch != "master" {	2✔
349	var pokyBranch string	1✔
350	if build.repo == "meta-mender" {	1✔
351	pokyBranch = build.baseBranch	×
352	} else {	1✔
353	pokyBranch = LatestStableYoctoBranch	1✔
354	metaMenderBranch := pokyBranch	1✔
355	metaMenderBranchKey := repoToBuildParameter("meta-mender")	1✔
356	buildParameters = append(	1✔
357	buildParameters,	1✔
358	&gitlab.PipelineVariableOptions{	1✔
359	Key: &metaMenderBranchKey,	1✔
360	Value: &metaMenderBranch,	1✔
361	},	1✔
362	)	1✔
363	}	1✔
364	pokyBranchKey := repoToBuildParameter("poky")	1✔
365	buildParameters = append(	1✔
366	buildParameters,	1✔
367	&gitlab.PipelineVariableOptions{Key: &pokyBranchKey, Value: &pokyBranch},	1✔
368	)	1✔
369	metaOEPokyBranchKey := repoToBuildParameter("meta-openembedded")	1✔
370	buildParameters = append(	1✔
371	buildParameters,	1✔
372	&gitlab.PipelineVariableOptions{	1✔
373	Key: &metaOEPokyBranchKey,	1✔
374	Value: &pokyBranch,	1✔
375	},	1✔
376	)	1✔
377	metaRPIPokyBranchKey := repoToBuildParameter("meta-raspberrypi")	1✔
378	buildParameters = append(	1✔
379	buildParameters,	1✔
380	&gitlab.PipelineVariableOptions{	1✔
381	Key: &metaRPIPokyBranchKey,	1✔
382	Value: &pokyBranch,	1✔
383	},	1✔
384	)	1✔
385	}
386
387	// set the rest of the CI build parameters
388	runIntegrationTests := "true"	1✔
389	runIntegrationTestsKey := "RUN_INTEGRATION_TESTS"	1✔
390	buildParameters = append(	1✔
391	buildParameters,	1✔
392	&gitlab.PipelineVariableOptions{Key: &runIntegrationTestsKey, Value: &runIntegrationTests},	1✔
393	)	1✔
394		1✔
395	// Backend integration tests, from mender-qa pipeline, are only relevant for legacy builds	1✔
396	if isLegacyBuild(build, buildParameters) {	1✔
397	runBackendIntegrationTests := "true"	×
398	if buildOptions.Fast {	×
399	runIntegrationTests = "false"	×
400	}	×
401
402	runBackendIntegrationTestsKey := "RUN_BACKEND_INTEGRATION_TESTS"	×
403	buildParameters = append(	×
404	buildParameters,	×
405	&gitlab.PipelineVariableOptions{	×
406	Key: &runBackendIntegrationTestsKey, Value: &runBackendIntegrationTests,	×
407	},	×
408	)	×
409	}
410
411	buildRepoKey := repoToBuildParameter(build.repo)	1✔
412	buildParameters = append(buildParameters,	1✔
413	&gitlab.PipelineVariableOptions{	1✔
414	Key: &buildRepoKey,	1✔
415	Value: &readHead,	1✔
416	})	1✔
417		1✔
418	return _getClientBuildParameters(buildParameters, build)	1✔
419	}
420
421	func _getClientBuildParameters(
422	buildParameters []*gitlab.PipelineVariableOptions,
423	build *buildOptions,
424	) ([]*gitlab.PipelineVariableOptions, error) {	1✔
425	var qemuParam string	1✔
426	if build.makeQEMU {	2✔
427	qemuParam = "true"	1✔
428	} else {	2✔
429	qemuParam = ""	1✔
430	}	1✔
431
432	buildX86UefiGrubKey := "BUILD_QEMUX86_64_UEFI_GRUB"	1✔
433	buildParameters = append(	1✔
434	buildParameters,	1✔
435	&gitlab.PipelineVariableOptions{Key: &buildX86UefiGrubKey, Value: &qemuParam},	1✔
436	)	1✔
437	testX86UefiGrubKey := "TEST_QEMUX86_64_UEFI_GRUB"	1✔
438	buildParameters = append(	1✔
439	buildParameters,	1✔
440	&gitlab.PipelineVariableOptions{Key: &testX86UefiGrubKey, Value: &qemuParam},	1✔
441	)	1✔
442		1✔
443	buildX86BiosGrubKey := "BUILD_QEMUX86_64_BIOS_GRUB"	1✔
444	buildParameters = append(	1✔
445	buildParameters,	1✔
446	&gitlab.PipelineVariableOptions{Key: &buildX86BiosGrubKey, Value: &qemuParam},	1✔
447	)	1✔
448	testX86BiosGrubKey := "TEST_QEMUX86_64_BIOS_GRUB"	1✔
449	buildParameters = append(	1✔
450	buildParameters,	1✔
451	&gitlab.PipelineVariableOptions{Key: &testX86BiosGrubKey, Value: &qemuParam},	1✔
452	)	1✔
453		1✔
454	buildX86BiosGrubGptKey := "BUILD_QEMUX86_64_BIOS_GRUB_GPT"	1✔
455	buildParameters = append(	1✔
456	buildParameters,	1✔
457	&gitlab.PipelineVariableOptions{Key: &buildX86BiosGrubGptKey, Value: &qemuParam},	1✔
458	)	1✔
459	testX86BiosGrubGptKey := "TEST_QEMUX86_64_BIOS_GRUB_GPT"	1✔
460	buildParameters = append(	1✔
461	buildParameters,	1✔
462	&gitlab.PipelineVariableOptions{Key: &testX86BiosGrubGptKey, Value: &qemuParam},	1✔
463	)	1✔
464		1✔
465	buildVexpress := "BUILD_VEXPRESS_QEMU"	1✔
466	buildParameters = append(	1✔
467	buildParameters,	1✔
468	&gitlab.PipelineVariableOptions{Key: &buildVexpress, Value: &qemuParam},	1✔
469	)	1✔
470	testVexpress := "TEST_VEXPRESS_QEMU"	1✔
471	buildParameters = append(	1✔
472	buildParameters,	1✔
473	&gitlab.PipelineVariableOptions{Key: &testVexpress, Value: &qemuParam},	1✔
474	)	1✔
475		1✔
476	buildVexpressFlash := "BUILD_VEXPRESS_QEMU_FLASH"	1✔
477	buildParameters = append(	1✔
478	buildParameters,	1✔
479	&gitlab.PipelineVariableOptions{Key: &buildVexpressFlash, Value: &qemuParam},	1✔
480	)	1✔
481	testVexpressFlash := "TEST_VEXPRESS_QEMU_FLASH"	1✔
482	buildParameters = append(	1✔
483	buildParameters,	1✔
484	&gitlab.PipelineVariableOptions{Key: &testVexpressFlash, Value: &qemuParam},	1✔
485	)	1✔
486		1✔
487	buildVexpressUbootUefiGrub := "BUILD_VEXPRESS_QEMU_UBOOT_UEFI_GRUB"	1✔
488	buildParameters = append(	1✔
489	buildParameters,	1✔
490	&gitlab.PipelineVariableOptions{Key: &buildVexpressUbootUefiGrub, Value: &qemuParam},	1✔
491	)	1✔
492	testVexpressUbootUefiGrub := "TEST_VEXPRESS_QEMU_UBOOT_UEFI_GRUB"	1✔
493	buildParameters = append(	1✔
494	buildParameters,	1✔
495	&gitlab.PipelineVariableOptions{Key: &testVexpressUbootUefiGrub, Value: &qemuParam},	1✔
496	)	1✔
497		1✔
498	buildBBBKey := "BUILD_BEAGLEBONEBLACK"	1✔
499	buildParameters = append(	1✔
500	buildParameters,	1✔
501	&gitlab.PipelineVariableOptions{Key: &buildBBBKey, Value: &qemuParam},	1✔
502	)	1✔
503		1✔
504	// Set BUILD_CLIENT = false, if target repo not in the qemuBuildRepositories list	1✔
505	buildClientKey := "BUILD_CLIENT"	1✔
506	buildClient := "false"	1✔
507	for _, prebuiltClientRepo := range qemuBuildRepositories {	2✔
508	if build.repo == prebuiltClientRepo {	2✔
509	buildClient = "true"	1✔
510	}	1✔
511	}
512	buildParameters = append(	1✔
513	buildParameters,	1✔
514	&gitlab.PipelineVariableOptions{Key: &buildClientKey, Value: &buildClient},	1✔
515	)	1✔
516		1✔
517	return buildParameters, nil	1✔
518	}
519
520	// stopBuildsOfStaleClientPRs stops any running pipelines on a PR which has been merged.
521	func stopBuildsOfStaleClientPRs(
522	log *logrus.Entry,
523	pr *github.PullRequestEvent,
524	conf *config) error {	2✔
525		2✔
526	// If the action is "closed" the pull request was merged or just closed,	2✔
527	// stop builds in both cases.	2✔
528	if pr.GetAction() != "closed" {	4✔
529	log.Debugf("stopBuildsOfStaleClientPRs: PR not closed, " +	2✔
530	"therefore not stopping it's pipeline")	2✔
531	return nil	2✔
532	}	2✔
533
534	log.Debug("stopBuildsOfStaleClientPRs: Find any running pipelines and kill mercilessly!")	1✔
535		1✔
536	for _, build := range getClientBuilds(log, conf, pr) {	2✔
537		1✔
538	gitlabClient, err := clientgitlab.NewGitLabClient(	1✔
539	conf.gitlabToken,	1✔
540	conf.gitlabBaseURL,	1✔
541	conf.dryRunMode,	1✔
542	)	1✔
543	if err != nil {	1✔
544	return err	×
545	}	×
546
547	buildParams, err := getClientBuildParameters(log, conf, &build, NewBuildOptions())	1✔
548	if err != nil {	1✔
NEW 549	log.Debug("stopBuildsOfStaleClientPRs: Failed to get the" +	×
NEW 550	"build-parameters for the build")	×
551	return err	×
552	}	×
553
554	stopStalePipelines(clientPipelinePath, log, gitlabClient, buildParams)	1✔
555	}
556
557	return nil	1✔
558
559	}

mendersoftware / integration-test-runner / 1681953125

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous