1166870661

Committed 02 Feb 2024 08:35AM UTC coverage: 70.929% (-8.8%) from 79.715%

Build # 1166870661

Build Type

push

gitlab-ci

Committed by

web-flow

Commit Message

Merge pull request #305 from alfrunes/MEN-7003

MEN-7003: Add "html" encoding option

Run Details

6 of 22 new or added lines in 1 file covered. (27.27%)

234 existing lines in 8 files now uncovered.

1054 of 1486 relevant lines covered (70.93%)

5.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

17.43

/app/processor/string.go

// Copyright 2022 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package processor

import (
        "bytes"
        "html"
        "net/url"
        "os"
        "regexp"
        "strings"
        "text/template"

        "github.com/thedevsaddam/gojsonq"

        "github.com/mendersoftware/workflows/model"
        "github.com/mendersoftware/workflows/utils"
)

const (
        workflowEnvVariable   = "env."
        workflowInputVariable = "workflow.input."
        regexVariable         = `\$\{(?P<options>(?:(?:[a-zA-Z]+)=(?:[a-zA-Z0-9]+);)*)` +
                `(?P<name>[^;\}\|]+)(?:\|(?P<default>[^\}]+))?}`
        regexOutputVariable = `(.*)\.json\.(.*)`

        encodingFlag = "encoding"
        encodingURL  = "url"
        encodingHTML = "html"
)

var (
        reExpression       = regexp.MustCompile(regexVariable)
        reExpressionOutput = regexp.MustCompile(regexOutputVariable)

        reMatchIndexOptions = reExpression.SubexpIndex("options")
        reMatchIndexName    = reExpression.SubexpIndex("name")
        reMatchIndexDefault = reExpression.SubexpIndex("default")
)

type Encoding int64

const (
        EncodingPlain Encoding = iota
        EncodingURL
        EncodingHTML
)

func ParseString(s string) Encoding {
        var enc Encoding
        switch s {
        case encodingURL:
                enc = EncodingURL
        case encodingHTML:
                enc = EncodingHTML
        default:
                enc = EncodingPlain
        }
        return enc
}

func (enc Encoding) Apply(s string) string {
        switch enc {
        case EncodingURL:
                return url.QueryEscape(s)
        case EncodingHTML:
                return html.EscapeString(s)
        }
        return s
}

type JobStringProcessor struct {
        workflow *model.Workflow
        job      *model.Job
}

type Options struct {
        Encoding Encoding
}

func NewJobStringProcessor(
        workflow *model.Workflow,
        job *model.Job,
) *JobStringProcessor {
        return &JobStringProcessor{
                workflow: workflow,
                job:      job,
        }
}

func processOptionString(expression string) (opts Options) {
        const (
                flagTokenCount = 2
                lValueIndex    = 0
                rValueIndex    = 1
        )
        for _, flagToken := range strings.Split(expression, ";") {
                flagValueTokens := strings.SplitN(flagToken, "=", 2)
                if len(flagValueTokens) < flagTokenCount {
                        continue
                }
                if flagValueTokens[lValueIndex] == encodingFlag {
                        switch flagValueTokens[rValueIndex] {
                        case encodingURL:
                                opts.Encoding = EncodingURL
                        case encodingHTML:
                                opts.Encoding = EncodingHTML
                        }
                }
        }
        return
}

func (j *JobStringProcessor) ProcessJobString(data string) string {
        matches := reExpression.FindAllStringSubmatch(data, -1)

        // search for ${...} expressions in the data string
SubMatchLoop:
        for _, submatch := range matches {
                // content of the ${...} expression, without the brackets
                varName := submatch[reMatchIndexName]
                value := submatch[reMatchIndexDefault]
                options := processOptionString(submatch[reMatchIndexOptions])
                // now it is possible to override the encoding with flags: ${encoding=plain;identifier}
                // if encoding is supplied via flags, it takes precedence, we return the match
                // without the flags, otherwise fail back to original match and encoding
                if strings.HasPrefix(varName, workflowInputVariable) &&
                        len(varName) > len(workflowInputVariable) {
                        // Replace ${workflow.input.KEY} with the KEY input variable
                        paramName := varName[len(workflowInputVariable):]
                        for _, param := range j.job.InputParameters {
                                if param.Name == paramName {
                                        value = param.Value
                                        break
                                }
                        }
                } else if strings.HasPrefix(varName, workflowEnvVariable) &&
                        len(varName) > len(workflowEnvVariable) {
                        // Replace ${env.KEY} with the KEY environment variable
                        envName := varName[len(workflowEnvVariable):]
                        if envValue := os.Getenv(envName); envValue != "" {
                                value = envValue
                        }
                } else if output := reExpressionOutput.FindStringSubmatch(varName); len(output) > 0 {
                        // Replace ${TASK_NAME.json.JSONPATH} with the value of the JSONPATH expression from the
                        // JSON output of the previous task with name TASK_NAME. If the output is not a valid
                        // JSON or the JSONPATH does not resolve to a value, replace with empty string
                        for _, result := range j.job.Results {
                                if result.Name == output[1] {
                                        varKey := output[2]
                                        var output string
                                        if result.Type == model.TaskTypeHTTP {
                                                output = result.HTTPResponse.Body
                                        } else if result.Type == model.TaskTypeCLI {
                                                output = result.CLI.Output
                                        } else {
                                                continue
                                        }
                                        varValue := gojsonq.New().FromString(output).Find(varKey)
                                        if varValue == nil {
                                                varValue = ""
                                        }
                                        varValueString, err := utils.ConvertAnythingToString(varValue)
                                        if err == nil {
                                                if varValueString != "" {
                                                        value = varValueString
                                                }
                                        } else {
                                                continue SubMatchLoop
                                        }
                                        break
                                }
                        }
                }
                value = options.Encoding.Apply(value)
                data = strings.ReplaceAll(data, submatch[0], value)
        }

        return data
}

// MaybeExecuteGoTemplate tries to parse and execute data as a go template
// if it fails to do so, data is returned.
func (j *JobStringProcessor) MaybeExecuteGoTemplate(data string) string {
        input := j.job.InputParameters.Map()
        tmpl, err := template.New("go-template").Parse(data)
        if err != nil {
                return data
        }
        buf := &bytes.Buffer{}
        err = tmpl.Execute(buf, input)
        if err != nil {
                return data
        }
        return buf.String()
}

1	// Copyright 2022 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package processor
16
17	import (
18	"bytes"
19	"html"
20	"net/url"
21	"os"
22	"regexp"
23	"strings"
24	"text/template"
25
26	"github.com/thedevsaddam/gojsonq"
27
28	"github.com/mendersoftware/workflows/model"
29	"github.com/mendersoftware/workflows/utils"
30	)
31
32	const (
33	workflowEnvVariable = "env."
34	workflowInputVariable = "workflow.input."
35	regexVariable = `\$\{(?P<options>(?:(?:[a-zA-Z]+)=(?:[a-zA-Z0-9]+);)*)` +
36	`(?P<name>[^;\}\\|]+)(?:\\|(?P<default>[^\}]+))?}`
37	regexOutputVariable = `(.)\.json\.(.)`
38
39	encodingFlag = "encoding"
40	encodingURL = "url"
41	encodingHTML = "html"
42	)
43
44	var (
45	reExpression = regexp.MustCompile(regexVariable)
46	reExpressionOutput = regexp.MustCompile(regexOutputVariable)
47
48	reMatchIndexOptions = reExpression.SubexpIndex("options")
49	reMatchIndexName = reExpression.SubexpIndex("name")
50	reMatchIndexDefault = reExpression.SubexpIndex("default")
51	)
52
53	type Encoding int64
54
55	const (
56	EncodingPlain Encoding = iota
57	EncodingURL
58	EncodingHTML
59	)
60
NEW 61	func ParseString(s string) Encoding {	×
NEW 62	var enc Encoding	×
NEW 63	switch s {	×
NEW 64	case encodingURL:	×
NEW 65	enc = EncodingURL	×
NEW 66	case encodingHTML:	×
NEW 67	enc = EncodingHTML	×
NEW 68	default:	×
NEW 69	enc = EncodingPlain	×
70	}
NEW 71	return enc	×
72	}
73
74	func (enc Encoding) Apply(s string) string {	1✔
75	switch enc {	1✔
NEW 76	case EncodingURL:	×
NEW 77	return url.QueryEscape(s)	×
78	case EncodingHTML:	1✔
79	return html.EscapeString(s)	1✔
80	}
NEW 81	return s	×
82	}
83
84	type JobStringProcessor struct {
85	workflow *model.Workflow
86	job *model.Job
87	}
88
89	type Options struct {
90	Encoding Encoding
91	}
92
93	func NewJobStringProcessor(
94	workflow *model.Workflow,
95	job *model.Job,
UNCOV 96	) *JobStringProcessor {	×
UNCOV 97	return &JobStringProcessor{	×
UNCOV 98	workflow: workflow,	×
UNCOV 99	job: job,	×
UNCOV 100	}	×
UNCOV 101	}	×
102
103	func processOptionString(expression string) (opts Options) {	5✔
104	const (	5✔
105	flagTokenCount = 2	5✔
106	lValueIndex = 0	5✔
107	rValueIndex = 1	5✔
108	)	5✔
109	for _, flagToken := range strings.Split(expression, ";") {	19✔
110	flagValueTokens := strings.SplitN(flagToken, "=", 2)	14✔
111	if len(flagValueTokens) < flagTokenCount {	19✔
112	continue	5✔
113	}
114	if flagValueTokens[lValueIndex] == encodingFlag {	14✔
115	switch flagValueTokens[rValueIndex] {	5✔
116	case encodingURL:	3✔
117	opts.Encoding = EncodingURL	3✔
NEW 118	case encodingHTML:	×
NEW 119	opts.Encoding = EncodingHTML	×
120	}
121	}
122	}
123	return	5✔
124	}
125
UNCOV 126	func (j *JobStringProcessor) ProcessJobString(data string) string {	×
UNCOV 127	matches := reExpression.FindAllStringSubmatch(data, -1)	×
UNCOV 128		×
UNCOV 129	// search for ${...} expressions in the data string	×
UNCOV 130	SubMatchLoop:	×
UNCOV 131	for _, submatch := range matches {	×
UNCOV 132	// content of the ${...} expression, without the brackets	×
UNCOV 133	varName := submatch[reMatchIndexName]	×
UNCOV 134	value := submatch[reMatchIndexDefault]	×
UNCOV 135	options := processOptionString(submatch[reMatchIndexOptions])	×
UNCOV 136	// now it is possible to override the encoding with flags: ${encoding=plain;identifier}	×
UNCOV 137	// if encoding is supplied via flags, it takes precedence, we return the match	×
UNCOV 138	// without the flags, otherwise fail back to original match and encoding	×
UNCOV 139	if strings.HasPrefix(varName, workflowInputVariable) &&	×
UNCOV 140	len(varName) > len(workflowInputVariable) {	×
UNCOV 141	// Replace ${workflow.input.KEY} with the KEY input variable	×
UNCOV 142	paramName := varName[len(workflowInputVariable):]	×
UNCOV 143	for _, param := range j.job.InputParameters {	×
UNCOV 144	if param.Name == paramName {	×
UNCOV 145	value = param.Value	×
UNCOV 146	break	×
147	}
148	}
UNCOV 149	} else if strings.HasPrefix(varName, workflowEnvVariable) &&	×
UNCOV 150	len(varName) > len(workflowEnvVariable) {	×
UNCOV 151	// Replace ${env.KEY} with the KEY environment variable	×
UNCOV 152	envName := varName[len(workflowEnvVariable):]	×
UNCOV 153	if envValue := os.Getenv(envName); envValue != "" {	×
UNCOV 154	value = envValue	×
UNCOV 155	}	×
156	} else if output := reExpressionOutput.FindStringSubmatch(varName); len(output) > 0 {	×
157	// Replace ${TASK_NAME.json.JSONPATH} with the value of the JSONPATH expression from the	×
158	// JSON output of the previous task with name TASK_NAME. If the output is not a valid	×
159	// JSON or the JSONPATH does not resolve to a value, replace with empty string	×
160	for _, result := range j.job.Results {	×
161	if result.Name == output[1] {	×
162	varKey := output[2]	×
163	var output string	×
164	if result.Type == model.TaskTypeHTTP {	×
165	output = result.HTTPResponse.Body	×
166	} else if result.Type == model.TaskTypeCLI {	×
167	output = result.CLI.Output	×
168	} else {	×
169	continue	×
170	}
171	varValue := gojsonq.New().FromString(output).Find(varKey)	×
172	if varValue == nil {	×
173	varValue = ""	×
174	}	×
175	varValueString, err := utils.ConvertAnythingToString(varValue)	×
176	if err == nil {	×
177	if varValueString != "" {	×
178	value = varValueString	×
179	}	×
180	} else {	×
181	continue SubMatchLoop	×
182	}
183	break	×
184	}
185	}
186	}
NEW 187	value = options.Encoding.Apply(value)	×
UNCOV 188	data = strings.ReplaceAll(data, submatch[0], value)	×
189	}
190
UNCOV 191	return data	×
192	}
193
194	// MaybeExecuteGoTemplate tries to parse and execute data as a go template
195	// if it fails to do so, data is returned.
UNCOV 196	func (j *JobStringProcessor) MaybeExecuteGoTemplate(data string) string {	×
UNCOV 197	input := j.job.InputParameters.Map()	×
UNCOV 198	tmpl, err := template.New("go-template").Parse(data)	×
UNCOV 199	if err != nil {	×
200	return data	×
201	}	×
UNCOV 202	buf := &bytes.Buffer{}	×
UNCOV 203	err = tmpl.Execute(buf, input)	×
UNCOV 204	if err != nil {	×
205	return data	×
206	}	×
UNCOV 207	return buf.String()	×
208	}

mendersoftware / workflows / 1166870661

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous