1681953125

Committed 21 Feb 2025 08:24AM UTC coverage: 59.965% (-4.4%) from 64.328%

Build # 1681953125

Build Type

Pull #357

gitlab-ci

Committed by

danielskinstad

Commit Message

feat: run integration pipeline on demand

Allow you to run the full integration pipeline on a protected branch
when a trusted member runs `start integration pipeline`.

As of now, this will only run with main - and you cannot specify PRs to
run the pipeline with.

Ticket: QA-863

Signed-off-by: Daniel Skinstad Drabitzius <daniel.drabitzius@northern.tech>

Pull Request Pull Request #357: wip: integration tests on demand

Run Details

77 of 311 new or added lines in 7 files covered. (24.76%)

2 existing lines in 1 file now uncovered.

1736 of 2895 relevant lines covered (59.97%)

2.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/integration_pipeline.go

package main

import (
        "bytes"
        "context"
        "strconv"
        "strings"
        "text/template"

        "github.com/google/go-github/v28/github"
        "github.com/sirupsen/logrus"
        gitlab "gitlab.com/gitlab-org/api/client-go"

        clientgitlab "github.com/mendersoftware/integration-test-runner/client/gitlab"
)

const integrationPipelinePath = "Northern.tech/Mender/integration"

func getIntegrationBuild(
        log *logrus.Entry,
        conf *config,
        pr *github.PullRequestEvent,
) buildOptions {

        repo := pr.GetRepo().GetName()
        commitSHA := pr.PullRequest.Base.GetSHA()

        //GetLabel returns "mendersoftware:master", we just want the branch
        baseBranch := strings.Split(pr.PullRequest.Base.GetLabel(), ":")[1]

        build := buildOptions{
                pr:         strconv.Itoa(pr.GetNumber()),
                repo:       repo,
                baseBranch: baseBranch,
                commitSHA:  commitSHA,
        }

        return build
}

func triggerIntegrationBuild(
        log *logrus.Entry,
        conf *config,
        build *buildOptions,
        pr *github.PullRequestEvent,
        buildOptions *BuildOptions,
) error {
        gitlabIntegration, err := clientgitlab.NewGitLabClient(
                conf.gitlabToken,
                conf.gitlabBaseURL,
                conf.dryRunMode,
        )
        if err != nil {
                return err
        }

        buildParameters, err := getIntegrationBuildParameters(log, conf, build, buildOptions)
        if err != nil {
                return err
        }

        // first stop old pipelines with the same buildParameters
        stopStalePipelines(integrationPipelinePath, log, gitlabIntegration, buildParameters)

        // trigger the new pipeline
        ref := "pr_" + strconv.Itoa(pr.GetNumber()) + "_protected"
        opt := &gitlab.CreatePipelineOptions{
                Ref:       &ref,
                Variables: &buildParameters,
        }

        variablesString := ""
        for _, variable := range *opt.Variables {
                variablesString += *variable.Key + ":" + *variable.Value + ", "
        }
        log.Infof(
                "Creating pipeline in project %s:%s with variables: %s",
                integrationPipelinePath,
                *opt.Ref,
                variablesString,
        )

        pipeline, err := gitlabIntegration.CreatePipeline(integrationPipelinePath, opt)
        if err != nil {
                log.Errorf("Could not create pipeline: %s", err.Error())
                return err
        }
        log.Infof("Created pipeline: %s", pipeline.WebURL)

        // Add the build variable matrix to the pipeline comment under a
        // drop-down tab
        // nolint:lll
        tmplString := `
Hello :smiley_cat: I created a pipeline for you here: [Pipeline-{{.Pipeline.ID}}]({{.Pipeline.WebURL}})

<details>
    <summary>Build Configuration Matrix</summary><p>

| Key   | Value |
| ----- | ----- |
{{range $i, $var := .BuildVars}}{{if $var.Value}}| {{$var.Key}} | {{$var.Value}} |{{printf "\n"}}{{end}}{{end}}

 </p></details>
`
        tmpl, err := template.New("Main").Parse(tmplString)
        if err != nil {
                log.Errorf(
                        "Failed to parse the build matrix template. Should never happen! Error: %s\n",
                        err.Error(),
                )
                return err
        }
        var buf bytes.Buffer
        if err = tmpl.Execute(&buf, struct {
                BuildVars []*gitlab.PipelineVariableOptions
                Pipeline  *gitlab.Pipeline
        }{
                BuildVars: filterOutEmptyVariables(*opt.Variables),
                Pipeline:  pipeline,
        }); err != nil {
                log.Errorf("Failed to execute the build matrix template. Error: %s\n", err.Error())
                return err
        }

        // Comment with a pipeline-link on the PR
        commentBody := buf.String()
        comment := github.IssueComment{
                Body: &commentBody,
        }

        err = githubClient.CreateComment(context.Background(),
                conf.githubOrganization, pr.GetRepo().GetName(), pr.GetNumber(), &comment)
        if err != nil {
                log.Infof("Failed to comment on the pr: %v, Error: %s", pr, err.Error())
        }

        return err
}

func getIntegrationBuildParameters(
        log *logrus.Entry,
        conf *config,
        build *buildOptions,
        buildOptions *BuildOptions,
) ([]*gitlab.PipelineVariableOptions, error) {
        readHead := "pull/" + build.pr + "/head"
        var buildParameters []*gitlab.PipelineVariableOptions

        runIntegrationTests := "true"
        runIntegrationTestsKey := "RUN_TESTS_FULL_INTEGRATION"
        buildParameters = append(
                buildParameters,
                &gitlab.PipelineVariableOptions{Key: &runIntegrationTestsKey, Value: &runIntegrationTests},
        )

        buildRepoKey := repoToBuildParameter(build.repo)
        buildParameters = append(buildParameters,
                &gitlab.PipelineVariableOptions{
                        Key:   &buildRepoKey,
                        Value: &readHead,
                })

        return buildParameters, nil

}

1	package main
2
3	import (
4	"bytes"
5	"context"
6	"strconv"
7	"strings"
8	"text/template"
9
10	"github.com/google/go-github/v28/github"
11	"github.com/sirupsen/logrus"
12	gitlab "gitlab.com/gitlab-org/api/client-go"
13
14	clientgitlab "github.com/mendersoftware/integration-test-runner/client/gitlab"
15	)
16
17	const integrationPipelinePath = "Northern.tech/Mender/integration"
18
19	func getIntegrationBuild(
20	log *logrus.Entry,
21	conf *config,
22	pr *github.PullRequestEvent,
NEW 23	) buildOptions {	×
NEW 24		×
NEW 25	repo := pr.GetRepo().GetName()	×
NEW 26	commitSHA := pr.PullRequest.Base.GetSHA()	×
NEW 27		×
NEW 28	//GetLabel returns "mendersoftware:master", we just want the branch	×
NEW 29	baseBranch := strings.Split(pr.PullRequest.Base.GetLabel(), ":")[1]	×
NEW 30		×
NEW 31	build := buildOptions{	×
NEW 32	pr: strconv.Itoa(pr.GetNumber()),	×
NEW 33	repo: repo,	×
NEW 34	baseBranch: baseBranch,	×
NEW 35	commitSHA: commitSHA,	×
NEW 36	}	×
NEW 37		×
NEW 38	return build	×
NEW 39	}	×
40
41	func triggerIntegrationBuild(
42	log *logrus.Entry,
43	conf *config,
44	build *buildOptions,
45	pr *github.PullRequestEvent,
46	buildOptions *BuildOptions,
NEW 47	) error {	×
NEW 48	gitlabIntegration, err := clientgitlab.NewGitLabClient(	×
NEW 49	conf.gitlabToken,	×
NEW 50	conf.gitlabBaseURL,	×
NEW 51	conf.dryRunMode,	×
NEW 52	)	×
NEW 53	if err != nil {	×
NEW 54	return err	×
NEW 55	}	×
56
NEW 57	buildParameters, err := getIntegrationBuildParameters(log, conf, build, buildOptions)	×
NEW 58	if err != nil {	×
NEW 59	return err	×
NEW 60	}	×
61
62	// first stop old pipelines with the same buildParameters
NEW 63	stopStalePipelines(integrationPipelinePath, log, gitlabIntegration, buildParameters)	×
NEW 64		×
NEW 65	// trigger the new pipeline	×
NEW 66	ref := "pr_" + strconv.Itoa(pr.GetNumber()) + "_protected"	×
NEW 67	opt := &gitlab.CreatePipelineOptions{	×
NEW 68	Ref: &ref,	×
NEW 69	Variables: &buildParameters,	×
NEW 70	}	×
NEW 71		×
NEW 72	variablesString := ""	×
NEW 73	for _, variable := range *opt.Variables {	×
NEW 74	variablesString += variable.Key + ":" + variable.Value + ", "	×
NEW 75	}	×
NEW 76	log.Infof(	×
NEW 77	"Creating pipeline in project %s:%s with variables: %s",	×
NEW 78	integrationPipelinePath,	×
NEW 79	*opt.Ref,	×
NEW 80	variablesString,	×
NEW 81	)	×
NEW 82		×
NEW 83	pipeline, err := gitlabIntegration.CreatePipeline(integrationPipelinePath, opt)	×
NEW 84	if err != nil {	×
NEW 85	log.Errorf("Could not create pipeline: %s", err.Error())	×
NEW 86	return err	×
NEW 87	}	×
NEW 88	log.Infof("Created pipeline: %s", pipeline.WebURL)	×
NEW 89		×
NEW 90	// Add the build variable matrix to the pipeline comment under a	×
NEW 91	// drop-down tab	×
NEW 92	// nolint:lll	×
NEW 93	tmplString := `	×
NEW 94	Hello :smiley_cat: I created a pipeline for you here: [Pipeline-{{.Pipeline.ID}}]({{.Pipeline.WebURL}})	×
NEW 95		×
NEW 96	<details>	×
NEW 97	<summary>Build Configuration Matrix</summary><p>	×
NEW 98		×
NEW 99	\| Key \| Value \|	×
NEW 100	\| ----- \| ----- \|	×
NEW 101	{{range $i, $var := .BuildVars}}{{if $var.Value}}\| {{$var.Key}} \| {{$var.Value}} \|{{printf "\n"}}{{end}}{{end}}	×
NEW 102		×
NEW 103	</p></details>	×
NEW 104	`	×
NEW 105	tmpl, err := template.New("Main").Parse(tmplString)	×
NEW 106	if err != nil {	×
NEW 107	log.Errorf(	×
NEW 108	"Failed to parse the build matrix template. Should never happen! Error: %s\n",	×
NEW 109	err.Error(),	×
NEW 110	)	×
NEW 111	return err	×
NEW 112	}	×
NEW 113	var buf bytes.Buffer	×
NEW 114	if err = tmpl.Execute(&buf, struct {	×
NEW 115	BuildVars []*gitlab.PipelineVariableOptions	×
NEW 116	Pipeline *gitlab.Pipeline	×
NEW 117	}{	×
NEW 118	BuildVars: filterOutEmptyVariables(*opt.Variables),	×
NEW 119	Pipeline: pipeline,	×
NEW 120	}); err != nil {	×
NEW 121	log.Errorf("Failed to execute the build matrix template. Error: %s\n", err.Error())	×
NEW 122	return err	×
NEW 123	}	×
124
125	// Comment with a pipeline-link on the PR
NEW 126	commentBody := buf.String()	×
NEW 127	comment := github.IssueComment{	×
NEW 128	Body: &commentBody,	×
NEW 129	}	×
NEW 130		×
NEW 131	err = githubClient.CreateComment(context.Background(),	×
NEW 132	conf.githubOrganization, pr.GetRepo().GetName(), pr.GetNumber(), &comment)	×
NEW 133	if err != nil {	×
NEW 134	log.Infof("Failed to comment on the pr: %v, Error: %s", pr, err.Error())	×
NEW 135	}	×
136
NEW 137	return err	×
138	}
139
140	func getIntegrationBuildParameters(
141	log *logrus.Entry,
142	conf *config,
143	build *buildOptions,
144	buildOptions *BuildOptions,
NEW 145	) ([]*gitlab.PipelineVariableOptions, error) {	×
NEW 146	readHead := "pull/" + build.pr + "/head"	×
NEW 147	var buildParameters []*gitlab.PipelineVariableOptions	×
NEW 148		×
NEW 149	runIntegrationTests := "true"	×
NEW 150	runIntegrationTestsKey := "RUN_TESTS_FULL_INTEGRATION"	×
NEW 151	buildParameters = append(	×
NEW 152	buildParameters,	×
NEW 153	&gitlab.PipelineVariableOptions{Key: &runIntegrationTestsKey, Value: &runIntegrationTests},	×
NEW 154	)	×
NEW 155		×
NEW 156	buildRepoKey := repoToBuildParameter(build.repo)	×
NEW 157	buildParameters = append(buildParameters,	×
NEW 158	&gitlab.PipelineVariableOptions{	×
NEW 159	Key: &buildRepoKey,	×
NEW 160	Value: &readHead,	×
NEW 161	})	×
NEW 162		×
NEW 163	return buildParameters, nil	×
NEW 164		×
NEW 165	}	×

mendersoftware / integration-test-runner / 1681953125

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous