1279561887

Committed 06 May 2024 07:18AM UTC coverage: 73.657% (-3.8%) from 77.428%

Build # 1279561887

Build Type

Pull #605

gitlab-ci

Committed by

alfrunes

Commit Message

chore(make): Refactor `coverage` target and disable pkcs11 on non-linux

Signed-off-by: Alf-Rune Siqveland <alf.rune@northern.tech>

Pull Request Pull Request #605: Allow using mender-artifact without `cgo`

Run Details

4 of 6 new or added lines in 1 file covered. (66.67%)

271 existing lines in 13 files now uncovered.

5469 of 7425 relevant lines covered (73.66%)

77.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.19

/awriter/writer.go

// Copyright 2022 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package awriter

import (
        "archive/tar"
        "fmt"
        "io"
        "io/ioutil"
        "os"
        "path/filepath"
        "regexp"

        "github.com/pkg/errors"

        "github.com/mendersoftware/mender-artifact/artifact"
        "github.com/mendersoftware/mender-artifact/artifact/stage"
        "github.com/mendersoftware/mender-artifact/handlers"
)

type ProgressWriter interface {
        Wrap(io.WriteCloser) io.Writer
        Reset(size int64, filename string, payloadNr int)
        Finish()
        io.Writer
}

// Writer provides on the fly writing of artifacts metadata file used by
// the Mender client and the server.
type Writer struct {
        w              io.Writer // underlying writer
        signer         artifact.Signer
        c              artifact.Compressor
        State          chan string    // Report progress
        ProgressWriter ProgressWriter // Report progress whilst writing
}

func NewWriter(w io.Writer, c artifact.Compressor) *Writer {
        return &Writer{
                w:     w,
                c:     c,
                State: make(chan string, 10),
        }
}

func NewWriterSigned(
        w io.Writer,
        c artifact.Compressor,
        manifestChecksumStore artifact.Signer,
) *Writer {
        nw := NewWriter(w, c)
        nw.signer = manifestChecksumStore
        return nw
}

type Updates struct {
        // Both of these are indexed the same, so Augment at index X corresponds
        // to Update at index X.
        Updates  []handlers.Composer
        Augments []handlers.Composer
}

// Iterate through all data files inside `upd` and calculate checksums.
func calcDataHash(
        manifestChecksumStore *artifact.ChecksumStore,
        upd *Updates,
        augmented bool,
) error {
        var updates []handlers.Composer
        if augmented {
                updates = upd.Augments
        } else {
                updates = upd.Updates
        }
        for i, u := range updates {
                var files [](*handlers.DataFile)
                if augmented {
                        if u == nil {
                                // Can happen if there is no augmented part.
                                continue
                        }
                        files = u.GetUpdateAugmentFiles()
                } else {
                        files = u.GetUpdateFiles()
                }
                for _, f := range files {
                        ch := artifact.NewWriterChecksum(ioutil.Discard)
                        df, err := os.Open(f.Name)
                        if err != nil {
                                return errors.Wrapf(err, "writer: can not open data file: %s", f.Name)
                        }
                        defer df.Close()
                        if _, err := io.Copy(ch, df); err != nil {
                                return errors.Wrapf(err, "writer: can not calculate checksum: %s", f.Name)
                        }
                        sum := ch.Checksum()
                        f.Checksum = sum
                        err = manifestChecksumStore.Add(
                                filepath.Join(artifact.UpdatePath(i), filepath.Base(f.Name)),
                                sum,
                        )
                        if err != nil {
                                return errors.Wrapf(err, "writer: can not calculate checksum: %s", f.Name)
                        }
                }
        }
        return nil
}

// writeTempHeader can write both the standard and the augmented header
func writeTempHeader(c artifact.Compressor, manifestChecksumStore *artifact.ChecksumStore,
        name string, args *WriteArtifactArgs, augmented bool) (*os.File, error) {

        // create temporary header file
        f, err := ioutil.TempFile("", name)
        if err != nil {
                return nil, errors.New("writer: can not create temporary header file")
        }

        ch := artifact.NewWriterChecksum(f)
        // use function to make sure to close gz and tar before
        // calculating checksum
        err = func() error {
                gz, err := c.NewWriter(ch)
                if err != nil {
                        return errors.Wrapf(err, "writer: can not open compressor")
                }
                defer gz.Close()

                htw := tar.NewWriter(gz)
                defer htw.Close()

                // Header differs in version 3 from version 2.
                if err = writeHeader(htw, args, augmented); err != nil {
                        return errors.Wrapf(err, "writer: error writing header")
                }
                return nil
        }()

        if err != nil {
                os.Remove(f.Name())
                return nil, err
        }
        fullName := fmt.Sprintf("%s.tar%s", name, c.GetFileExtension())
        err = manifestChecksumStore.Add(fullName, ch.Checksum())
        if err != nil {
                return nil, errors.Wrapf(err, "writer: can not calculate checksum: %s", fullName)
        }

        return f, nil
}

func WriteSignature(tw *tar.Writer, message []byte,
        signer artifact.Signer) error {
        if signer == nil {
                return nil
        }

        sig, err := signer.Sign(message)
        if err != nil {
                return errors.Wrap(err, "writer: can not sign artifact")
        }
        sw := artifact.NewTarWriterStream(tw)
        if err := sw.Write(sig, "manifest.sig"); err != nil {
                return errors.Wrap(err, "writer: can not tar signature")
        }
        return nil
}

type WriteArtifactArgs struct {
        Format            string
        Version           int
        Devices           []string
        Name              string
        Updates           *Updates
        Scripts           *artifact.Scripts
        Depends           *artifact.ArtifactDepends
        Provides          *artifact.ArtifactProvides
        TypeInfoV3        *artifact.TypeInfoV3
        MetaData          map[string]interface{} // Generic JSON
        AugmentTypeInfoV3 *artifact.TypeInfoV3
        AugmentMetaData   map[string]interface{} // Generic JSON
        Bootstrap         bool
}

func (aw *Writer) WriteArtifact(args *WriteArtifactArgs) (err error) {

        if args.Version == 1 {
                return errors.New(
                        "writer: The Mender-Artifact version 1 is outdated. Refusing to create artifact.",
                )
        }

        if !(args.Version == 2 || args.Version == 3) {
                return errors.New("Unsupported artifact version")
        }

        if args.Version == 3 {
                return aw.writeArtifactV3(args)
        }

        return aw.writeArtifactV2(args)
}

func (aw *Writer) writeArtifactV2(args *WriteArtifactArgs) error {

        // mender archive writer
        tw := tar.NewWriter(aw.w)
        defer tw.Close()

        aw.State <- stage.Version
        // write version file
        inf, err := artifact.ToStream(&artifact.Info{Version: args.Version, Format: args.Format})
        if err != nil {
                return err
        }
        sa := artifact.NewTarWriterStream(tw)
        if err := sa.Write(inf, "version"); err != nil {
                return errors.Wrapf(err, "writer: can not write version tar header")
        }

        aw.State <- stage.Manifest
        manifestChecksumStore := artifact.NewChecksumStore()
        // calculate checksums of all data files
        // we need this regardless of which artifact version we are writing
        if err := calcDataHash(manifestChecksumStore, args.Updates, false); err != nil {
                return err
        }
        tmpHdr, err := writeTempHeader(aw.c, manifestChecksumStore, "header", args, false)

        if err != nil {
                return err
        }
        defer os.Remove(tmpHdr.Name())

        if err = writeManifestVersion(
                args.Version,
                aw.signer,
                tw,
                manifestChecksumStore,
                nil,
                inf,
        ); err != nil {
                return errors.Wrap(err, "WriteArtifact")
        }

        // write header
        aw.State <- stage.Header
        if _, err := tmpHdr.Seek(0, 0); err != nil {
                return errors.Wrapf(err, "writer: error preparing tmp header for writing")
        }
        fw := artifact.NewTarWriterFile(tw)
        if err := fw.Write(tmpHdr, "header.tar"+aw.c.GetFileExtension()); err != nil {
                return errors.Wrapf(err, "writer: can not tar header")
        }

        // write data files
        aw.State <- stage.Data
        return writeData(tw, aw.c, args.Updates, aw.ProgressWriter)
}

func (aw *Writer) writeArtifactV3(args *WriteArtifactArgs) (err error) {
        tw := tar.NewWriter(aw.w)
        defer tw.Close()

        aw.State <- stage.Version
        ////////////////////////
        // write version file //
        ////////////////////////
        inf, err := artifact.ToStream(&artifact.Info{Version: args.Version, Format: args.Format})
        if err != nil {
                return err
        }
        sa := artifact.NewTarWriterStream(tw)
        if err := sa.Write(inf, "version"); err != nil {
                return errors.Wrapf(err, "writer: can not write version tar header")
        }

        ////////////////////////////
        // Write manifest         //
        // Write manifest.sig     //
        // Write manifest-augment //
        ////////////////////////////
        aw.State <- stage.Manifest
        augmentedDataPresent := (len(args.Updates.Augments) > 0)

        // Holds the checksum for the update, and 'header.tar.gz', and the 'version' file.
        manifestChecksumStore := artifact.NewChecksumStore()

        // Holds the checksum for 'header-augment.tar.gz'.
        augManifestChecksumStore := artifact.NewChecksumStore()
        aw.State <- stage.ManifestSignature
        if err := calcDataHash(manifestChecksumStore, args.Updates, false); err != nil {
                return err
        }
        if augmentedDataPresent {
                if err := calcDataHash(augManifestChecksumStore, args.Updates, true); err != nil {
                        return err
                }
        }
        // The header in version 3 will have the original rootfs-checksum in type-info!
        tmpHdr, err := writeTempHeader(aw.c, manifestChecksumStore, "header", args, false)
        if err != nil {
                return errors.Wrap(err, "writeArtifactV3: writing header")
        }
        defer os.Remove(tmpHdr.Name())

        var tmpAugHdr *os.File
        if augmentedDataPresent {
                tmpAugHdr, err = writeTempHeader(
                        aw.c,
                        augManifestChecksumStore,
                        "header-augment",
                        args,
                        true,
                )
                if err != nil {
                        return errors.Wrap(err, "writeArtifactV3: writing augmented header")
                }
                defer os.Remove(tmpAugHdr.Name())
        }

        if err = writeManifestVersion(
                args.Version,
                aw.signer,
                tw,
                manifestChecksumStore,
                augManifestChecksumStore,
                inf,
        ); err != nil {
                return errors.Wrap(err, "WriteArtifact")
        }

        ////////////////////
        // Write header   //
        ////////////////////
        aw.State <- stage.Header
        if _, err := tmpHdr.Seek(0, 0); err != nil {
                return errors.Wrapf(err, "writer: error preparing tmp header for writing")
        }
        fw := artifact.NewTarWriterFile(tw)
        if err := fw.Write(tmpHdr, "header.tar"+aw.c.GetFileExtension()); err != nil {
                return errors.Wrapf(err, "writer: can not tar header")
        }

        /////////////////////////////
        // Write augmented-header  //
        /////////////////////////////
        if augmentedDataPresent {
                aw.State <- stage.HeaderAugment
                if _, err := tmpAugHdr.Seek(0, 0); err != nil {
                        return errors.Wrapf(err, "writer: error preparing tmp augment-header for writing")
                }
                fw = artifact.NewTarWriterFile(tw)
                if err := fw.Write(tmpAugHdr, "header-augment.tar"+aw.c.GetFileExtension()); err != nil {
                        return errors.Wrapf(err, "writer: can not tar augmented-header")
                }
        }

        //////////////////////////
        // Write the datafiles  //
        //////////////////////////
        aw.State <- stage.Data
        return writeData(tw, aw.c, args.Updates, aw.ProgressWriter)
}

// writeArtifactVersion writes version specific artifact records.
func writeManifestVersion(
        version int,
        signer artifact.Signer,
        tw *tar.Writer,
        manifestChecksumStore,
        augmanChecksumStore *artifact.ChecksumStore,
        artifactInfoStream []byte,
) error {
        switch version {
        case 2:
                // add checksum of `version`
                ch := artifact.NewWriterChecksum(ioutil.Discard)
                _, err := ch.Write(artifactInfoStream)
                if err != nil {
                        return errors.Wrapf(err, "writer: can not write manifest stream")
                }
                err = manifestChecksumStore.Add("version", ch.Checksum())
                if err != nil {
                        return errors.Wrapf(err, "writer: can not write manifest stream")
                }
                // write `manifest` file
                sw := artifact.NewTarWriterStream(tw)
                if err := sw.Write(manifestChecksumStore.GetRaw(), "manifest"); err != nil {
                        return errors.Wrapf(err, "writer: can not write manifest stream")
                }
                // write signature
                if err := WriteSignature(tw, manifestChecksumStore.GetRaw(), signer); err != nil {
                        return err
                }
        case 3:
                // Add checksum of `version`.
                ch := artifact.NewWriterChecksum(ioutil.Discard)
                _, err := ch.Write(artifactInfoStream)
                if err != nil {
                        return errors.Wrapf(err, "writer: can not write manifest stream")
                }
                err = manifestChecksumStore.Add("version", ch.Checksum())
                if err != nil {
                        return errors.Wrapf(err, "writer: can not write manifest stream")
                }
                // Write `manifest` file.
                sw := artifact.NewTarWriterStream(tw)
                if err := sw.Write(manifestChecksumStore.GetRaw(), "manifest"); err != nil {
                        return errors.Wrapf(err, "writer: can not write manifest stream")
                }
                // Write signature.
                if err := WriteSignature(tw, manifestChecksumStore.GetRaw(), signer); err != nil {
                        return err
                }
                // Write the augmented manifest, if any.
                if len(augmanChecksumStore.GetRaw()) > 0 {
                        sw = artifact.NewTarWriterStream(tw)
                        if err := sw.Write(augmanChecksumStore.GetRaw(), "manifest-augment"); err != nil {
                                return errors.Wrapf(err, "writer: can not write manifest stream")
                        }
                }
        default:
                return fmt.Errorf("writer: unsupported artifact version: %d", version)
        }
        return nil
}

func writeScripts(tw *tar.Writer, scr *artifact.Scripts) error {
        sw := artifact.NewTarWriterFile(tw)
        for _, script := range scr.Get() {
                f, err := os.Open(script)
                if err != nil {
                        return errors.Wrapf(err, "writer: can not open script file: %s", script)
                }
                defer f.Close()

                if err :=
                        sw.Write(f, filepath.Join("scripts", filepath.Base(script))); err != nil {
                        return errors.Wrapf(err, "writer: can not store script: %s", script)
                }
        }
        return nil
}

func extractUpdateTypes(updates []handlers.Composer) []artifact.UpdateType {
        u := []artifact.UpdateType{}
        for _, upd := range updates {
                u = append(u, artifact.UpdateType{Type: upd.GetUpdateType()})
        }
        return u
}

func writeHeader(tarWriter *tar.Writer, args *WriteArtifactArgs, augmented bool) error {
        var composers []handlers.Composer
        if augmented {
                composers = args.Updates.Augments
        } else {
                composers = args.Updates.Updates
        }
        if len(composers) == 0 {
                return errors.New("writeHeader: No Payloads added")
        }

        // store header info
        var hInfo artifact.WriteValidator
        upds := extractUpdateTypes(composers)
        switch args.Version {
        case 1, 2:
                hInfo = artifact.NewHeaderInfo(args.Name, upds, args.Devices)
        case 3:
                hInfo = artifact.NewHeaderInfoV3(upds, args.Provides, args.Depends)
        }

        sa := artifact.NewTarWriterStream(tarWriter)
        stream, err := artifact.ToStream(hInfo)
        if err != nil {
                return errors.Wrap(err, "writeHeader")
        }
        if err := sa.Write(stream, "header-info"); err != nil {
                return errors.Wrap(err, "writer: can not store header-info")
        }

        // write scripts
        if !augmented && args.Scripts != nil {
                if err := writeScripts(tarWriter, args.Scripts); err != nil {
                        return err
                }
        }

        for i, upd := range composers {
                // TODO: We only have one `args` variable here, so making more
                // than one update is kind of useless. Should probably be made
                // into a list as well.
                composeHeaderArgs := handlers.ComposeHeaderArgs{
                        TarWriter: tarWriter,
                        No:        i,
                        Version:   args.Version,
                        Augmented: augmented,
                }
                if augmented {
                        composeHeaderArgs.TypeInfoV3 = args.AugmentTypeInfoV3
                        composeHeaderArgs.MetaData = args.AugmentMetaData
                } else {
                        composeHeaderArgs.TypeInfoV3 = args.TypeInfoV3
                        composeHeaderArgs.MetaData = args.MetaData
                }
                if err := upd.ComposeHeader(&composeHeaderArgs); err != nil {
                        return errors.Wrapf(err, "writer: error composing header")
                }
        }
        return nil
}

func writeData(
        tw *tar.Writer,
        comp artifact.Compressor,
        updates *Updates,
        pw ProgressWriter,
) error {
        for i, upd := range updates.Updates {
                var augment handlers.Composer = nil
                if i < len(updates.Augments) {
                        augment = updates.Augments[i]
                }
                if err := writeOneDataTar(tw, comp, i, upd, augment, pw); err != nil {
                        return errors.Wrapf(err, "writer: error writing data files")
                }
        }
        return nil
}

func writeOneDataTar(tw *tar.Writer, comp artifact.Compressor, no int,
        baseUpdate, augmentUpdate handlers.Composer, pw ProgressWriter) error {

        f, ferr := ioutil.TempFile("", "data")
        if ferr != nil {
                return errors.New("Payload: can not create temporary data file")
        }
        defer os.Remove(f.Name())

        err := func() error {
                gz, err := comp.NewWriter(f)
                if err != nil {
                        return errors.Wrap(err, "Could not open compressor")
                }
                defer gz.Close()

                var w io.Writer
                if pw != nil {
                        w = pw.Wrap(gz)
                } else {
                        w = gz
                }
                tarw := tar.NewWriter(w)
                defer tarw.Close()

                if len(baseUpdate.GetUpdateFiles()) == 0 && pw != nil {
                        pw.Reset(0, "bootstrap", 0)
                }
                for i, file := range baseUpdate.GetUpdateFiles() {
                        fi, err := os.Stat(file.Name)
                        if err != nil {
                                return err
                        }
                        if pw != nil {
                                pw.Reset(fi.Size(), file.Name, i)
                        }
                        err = writeOneDataFile(tarw, file)
                        if err != nil {
                                return err
                        }
                        if pw != nil {
                                pw.Finish()
                        }
                }
                if augmentUpdate == nil {
                        return nil
                }

                for _, file := range augmentUpdate.GetUpdateAugmentFiles() {
                        err = writeOneDataFile(tarw, file)
                        if err != nil {
                                return err
                        }
                }
                return nil
        }()

        if err != nil {
                return err
        }

        if _, err = f.Seek(0, 0); err != nil {
                return errors.Wrap(err, "Payload: can not reset file position")
        }

        dfw := artifact.NewTarWriterFile(tw)
        if err = dfw.Write(f, artifact.UpdateDataPath(no)+comp.GetFileExtension()); err != nil {
                return errors.Wrap(err, "Payload: can not write tar data header")
        }
        return nil
}

func writeOneDataFile(tarw *tar.Writer, file *handlers.DataFile) error {
        matched, err := regexp.MatchString(`^[\w\-.,]+$`, filepath.Base(file.Name))

        if err != nil {
                return errors.Wrapf(err, "Payload: invalid regular expression pattern")
        }

        if !matched {
                message := "Payload: data file " + file.Name + " contains forbidden characters"
                info := "Only letters, digits and characters in the set \".,_-\" are allowed"
                return fmt.Errorf("%s. %s", message, info)
        }

        df, err := os.Open(file.Name)
        if err != nil {
                return errors.Wrapf(err, "Payload: can not open data file: %s", file.Name)
        }
        fw := artifact.NewTarWriterFile(tarw)
        if err := fw.Write(df, filepath.Base(file.Name)); err != nil {
                df.Close()
                return errors.Wrapf(err,
                        "Payload: can not write tar temp data header: %v", file)
        }
        df.Close()
        return nil
}

1	// Copyright 2022 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package awriter
16
17	import (
18	"archive/tar"
19	"fmt"
20	"io"
21	"io/ioutil"
22	"os"
23	"path/filepath"
24	"regexp"
25
26	"github.com/pkg/errors"
27
28	"github.com/mendersoftware/mender-artifact/artifact"
29	"github.com/mendersoftware/mender-artifact/artifact/stage"
30	"github.com/mendersoftware/mender-artifact/handlers"
31	)
32
33	type ProgressWriter interface {
34	Wrap(io.WriteCloser) io.Writer
35	Reset(size int64, filename string, payloadNr int)
36	Finish()
37	io.Writer
38	}
39
40	// Writer provides on the fly writing of artifacts metadata file used by
41	// the Mender client and the server.
42	type Writer struct {
43	w io.Writer // underlying writer
44	signer artifact.Signer
45	c artifact.Compressor
46	State chan string // Report progress
47	ProgressWriter ProgressWriter // Report progress whilst writing
48	}
49
50	func NewWriter(w io.Writer, c artifact.Compressor) *Writer {	31✔
51	return &Writer{	31✔
52	w: w,	31✔
53	c: c,	31✔
54	State: make(chan string, 10),	31✔
55	}	31✔
56	}	31✔
57
58	func NewWriterSigned(
59	w io.Writer,
60	c artifact.Compressor,
61	manifestChecksumStore artifact.Signer,
62	) *Writer {	18✔
63	nw := NewWriter(w, c)	18✔
64	nw.signer = manifestChecksumStore	18✔
65	return nw	18✔
66	}	18✔
67
68	type Updates struct {
69	// Both of these are indexed the same, so Augment at index X corresponds
70	// to Update at index X.
71	Updates []handlers.Composer
72	Augments []handlers.Composer
73	}
74
75	// Iterate through all data files inside `upd` and calculate checksums.
76	func calcDataHash(
77	manifestChecksumStore *artifact.ChecksumStore,
78	upd *Updates,
79	augmented bool,
80	) error {	45✔
81	var updates []handlers.Composer	45✔
82	if augmented {	51✔
83	updates = upd.Augments	6✔
84	} else {	45✔
85	updates = upd.Updates	39✔
86	}	39✔
87	for i, u := range updates {	94✔
88	var files [](*handlers.DataFile)	49✔
89	if augmented {	55✔
90	if u == nil {	6✔
91	// Can happen if there is no augmented part.	×
92	continue	×
93	}
94	files = u.GetUpdateAugmentFiles()	6✔
95	} else {	43✔
96	files = u.GetUpdateFiles()	43✔
97	}	43✔
98	for _, f := range files {	85✔
99	ch := artifact.NewWriterChecksum(ioutil.Discard)	36✔
100	df, err := os.Open(f.Name)	36✔
101	if err != nil {	40✔
102	return errors.Wrapf(err, "writer: can not open data file: %s", f.Name)	4✔
103	}	4✔
104	defer df.Close()	32✔
105	if _, err := io.Copy(ch, df); err != nil {	32✔
106	return errors.Wrapf(err, "writer: can not calculate checksum: %s", f.Name)	×
107	}	×
108	sum := ch.Checksum()	32✔
109	f.Checksum = sum	32✔
110	err = manifestChecksumStore.Add(	32✔
111	filepath.Join(artifact.UpdatePath(i), filepath.Base(f.Name)),	32✔
112	sum,	32✔
113	)	32✔
114	if err != nil {	32✔
115	return errors.Wrapf(err, "writer: can not calculate checksum: %s", f.Name)	×
116	}	×
117	}
118	}
119	return nil	41✔
120	}
121
122	// writeTempHeader can write both the standard and the augmented header
123	func writeTempHeader(c artifact.Compressor, manifestChecksumStore *artifact.ChecksumStore,
124	name string, args WriteArtifactArgs, augmented bool) (os.File, error) {	41✔
125		41✔
126	// create temporary header file	41✔
127	f, err := ioutil.TempFile("", name)	41✔
128	if err != nil {	41✔
129	return nil, errors.New("writer: can not create temporary header file")	×
130	}	×
131
132	ch := artifact.NewWriterChecksum(f)	41✔
133	// use function to make sure to close gz and tar before	41✔
134	// calculating checksum	41✔
135	err = func() error {	82✔
136	gz, err := c.NewWriter(ch)	41✔
137	if err != nil {	41✔
138	return errors.Wrapf(err, "writer: can not open compressor")	×
139	}	×
140	defer gz.Close()	41✔
141		41✔
142	htw := tar.NewWriter(gz)	41✔
143	defer htw.Close()	41✔
144		41✔
145	// Header differs in version 3 from version 2.	41✔
146	if err = writeHeader(htw, args, augmented); err != nil {	43✔
147	return errors.Wrapf(err, "writer: error writing header")	2✔
148	}	2✔
149	return nil	39✔
150	}()
151
152	if err != nil {	43✔
153	os.Remove(f.Name())	2✔
154	return nil, err	2✔
155	}	2✔
156	fullName := fmt.Sprintf("%s.tar%s", name, c.GetFileExtension())	39✔
157	err = manifestChecksumStore.Add(fullName, ch.Checksum())	39✔
158	if err != nil {	39✔
159	return nil, errors.Wrapf(err, "writer: can not calculate checksum: %s", fullName)	×
160	}	×
161
162	return f, nil	39✔
163	}
164
165	func WriteSignature(tw *tar.Writer, message []byte,
166	signer artifact.Signer) error {	37✔
167	if signer == nil {	56✔
168	return nil	19✔
169	}	19✔
170
171	sig, err := signer.Sign(message)	18✔
172	if err != nil {	18✔
173	return errors.Wrap(err, "writer: can not sign artifact")	×
174	}	×
175	sw := artifact.NewTarWriterStream(tw)	18✔
176	if err := sw.Write(sig, "manifest.sig"); err != nil {	22✔
177	return errors.Wrap(err, "writer: can not tar signature")	4✔
178	}	4✔
179	return nil	14✔
180	}
181
182	type WriteArtifactArgs struct {
183	Format string
184	Version int
185	Devices []string
186	Name string
187	Updates *Updates
188	Scripts *artifact.Scripts
189	Depends *artifact.ArtifactDepends
190	Provides *artifact.ArtifactProvides
191	TypeInfoV3 *artifact.TypeInfoV3
192	MetaData map[string]interface{} // Generic JSON
193	AugmentTypeInfoV3 *artifact.TypeInfoV3
194	AugmentMetaData map[string]interface{} // Generic JSON
195	Bootstrap bool
196	}
197
198	func (aw Writer) WriteArtifact(args WriteArtifactArgs) (err error) {	47✔
199		47✔
200	if args.Version == 1 {	49✔
201	return errors.New(	2✔
202	"writer: The Mender-Artifact version 1 is outdated. Refusing to create artifact.",	2✔
203	)	2✔
204	}	2✔
205
206	if !(args.Version == 2 \|\| args.Version == 3) {	49✔
207	return errors.New("Unsupported artifact version")	4✔
208	}	4✔
209
210	if args.Version == 3 {	66✔
211	return aw.writeArtifactV3(args)	25✔
212	}	25✔
213
214	return aw.writeArtifactV2(args)	16✔
215	}
216
217	func (aw Writer) writeArtifactV2(args WriteArtifactArgs) error {	16✔
218		16✔
219	// mender archive writer	16✔
220	tw := tar.NewWriter(aw.w)	16✔
221	defer tw.Close()	16✔
222		16✔
223	aw.State <- stage.Version	16✔
224	// write version file	16✔
225	inf, err := artifact.ToStream(&artifact.Info{Version: args.Version, Format: args.Format})	16✔
226	if err != nil {	16✔
227	return err	×
228	}	×
229	sa := artifact.NewTarWriterStream(tw)	16✔
230	if err := sa.Write(inf, "version"); err != nil {	16✔
231	return errors.Wrapf(err, "writer: can not write version tar header")	×
232	}	×
233
234	aw.State <- stage.Manifest	16✔
235	manifestChecksumStore := artifact.NewChecksumStore()	16✔
236	// calculate checksums of all data files	16✔
237	// we need this regardless of which artifact version we are writing	16✔
238	if err := calcDataHash(manifestChecksumStore, args.Updates, false); err != nil {	16✔
239	return err	×
240	}	×
241	tmpHdr, err := writeTempHeader(aw.c, manifestChecksumStore, "header", args, false)	16✔
242		16✔
243	if err != nil {	16✔
244	return err	×
245	}	×
246	defer os.Remove(tmpHdr.Name())	16✔
247		16✔
248	if err = writeManifestVersion(	16✔
249	args.Version,	16✔
250	aw.signer,	16✔
251	tw,	16✔
252	manifestChecksumStore,	16✔
253	nil,	16✔
254	inf,	16✔
255	); err != nil {	16✔
256	return errors.Wrap(err, "WriteArtifact")	×
257	}	×
258
259	// write header
260	aw.State <- stage.Header	16✔
261	if _, err := tmpHdr.Seek(0, 0); err != nil {	16✔
262	return errors.Wrapf(err, "writer: error preparing tmp header for writing")	×
263	}	×
264	fw := artifact.NewTarWriterFile(tw)	16✔
265	if err := fw.Write(tmpHdr, "header.tar"+aw.c.GetFileExtension()); err != nil {	16✔
266	return errors.Wrapf(err, "writer: can not tar header")	×
267	}	×
268
269	// write data files
270	aw.State <- stage.Data	16✔
271	return writeData(tw, aw.c, args.Updates, aw.ProgressWriter)	16✔
272	}
273
274	func (aw Writer) writeArtifactV3(args WriteArtifactArgs) (err error) {	25✔
275	tw := tar.NewWriter(aw.w)	25✔
276	defer tw.Close()	25✔
277		25✔
278	aw.State <- stage.Version	25✔
279	////////////////////////	25✔
280	// write version file //	25✔
281	////////////////////////	25✔
282	inf, err := artifact.ToStream(&artifact.Info{Version: args.Version, Format: args.Format})	25✔
283	if err != nil {	25✔
284	return err	×
285	}	×
286	sa := artifact.NewTarWriterStream(tw)	25✔
287	if err := sa.Write(inf, "version"); err != nil {	27✔
288	return errors.Wrapf(err, "writer: can not write version tar header")	2✔
289	}	2✔
290
291	////////////////////////////
292	// Write manifest //
293	// Write manifest.sig //
294	// Write manifest-augment //
295	////////////////////////////
296	aw.State <- stage.Manifest	23✔
297	augmentedDataPresent := (len(args.Updates.Augments) > 0)	23✔
298		23✔
299	// Holds the checksum for the update, and 'header.tar.gz', and the 'version' file.	23✔
300	manifestChecksumStore := artifact.NewChecksumStore()	23✔
301		23✔
302	// Holds the checksum for 'header-augment.tar.gz'.	23✔
303	augManifestChecksumStore := artifact.NewChecksumStore()	23✔
304	aw.State <- stage.ManifestSignature	23✔
305	if err := calcDataHash(manifestChecksumStore, args.Updates, false); err != nil {	27✔
306	return err	4✔
307	}	4✔
308	if augmentedDataPresent {	25✔
309	if err := calcDataHash(augManifestChecksumStore, args.Updates, true); err != nil {	6✔
310	return err	×
311	}	×
312	}
313	// The header in version 3 will have the original rootfs-checksum in type-info!
314	tmpHdr, err := writeTempHeader(aw.c, manifestChecksumStore, "header", args, false)	19✔
315	if err != nil {	21✔
316	return errors.Wrap(err, "writeArtifactV3: writing header")	2✔
317	}	2✔
318	defer os.Remove(tmpHdr.Name())	17✔
319		17✔
320	var tmpAugHdr *os.File	17✔
321	if augmentedDataPresent {	23✔
322	tmpAugHdr, err = writeTempHeader(	6✔
323	aw.c,	6✔
324	augManifestChecksumStore,	6✔
325	"header-augment",	6✔
326	args,	6✔
327	true,	6✔
328	)	6✔
329	if err != nil {	6✔
330	return errors.Wrap(err, "writeArtifactV3: writing augmented header")	×
331	}	×
332	defer os.Remove(tmpAugHdr.Name())	6✔
333	}
334
335	if err = writeManifestVersion(	17✔
336	args.Version,	17✔
337	aw.signer,	17✔
338	tw,	17✔
339	manifestChecksumStore,	17✔
340	augManifestChecksumStore,	17✔
341	inf,	17✔
342	); err != nil {	19✔
343	return errors.Wrap(err, "WriteArtifact")	2✔
344	}	2✔
345
346	////////////////////
347	// Write header //
348	////////////////////
349	aw.State <- stage.Header	15✔
350	if _, err := tmpHdr.Seek(0, 0); err != nil {	15✔
351	return errors.Wrapf(err, "writer: error preparing tmp header for writing")	×
352	}	×
353	fw := artifact.NewTarWriterFile(tw)	15✔
354	if err := fw.Write(tmpHdr, "header.tar"+aw.c.GetFileExtension()); err != nil {	17✔
355	return errors.Wrapf(err, "writer: can not tar header")	2✔
356	}	2✔
357
358	/////////////////////////////
359	// Write augmented-header //
360	/////////////////////////////
361	if augmentedDataPresent {	19✔
362	aw.State <- stage.HeaderAugment	6✔
363	if _, err := tmpAugHdr.Seek(0, 0); err != nil {	6✔
364	return errors.Wrapf(err, "writer: error preparing tmp augment-header for writing")	×
365	}	×
366	fw = artifact.NewTarWriterFile(tw)	6✔
367	if err := fw.Write(tmpAugHdr, "header-augment.tar"+aw.c.GetFileExtension()); err != nil {	8✔
368	return errors.Wrapf(err, "writer: can not tar augmented-header")	2✔
369	}	2✔
370	}
371
372	//////////////////////////
373	// Write the datafiles //
374	//////////////////////////
375	aw.State <- stage.Data	11✔
376	return writeData(tw, aw.c, args.Updates, aw.ProgressWriter)	11✔
377	}
378
379	// writeArtifactVersion writes version specific artifact records.
380	func writeManifestVersion(
381	version int,
382	signer artifact.Signer,
383	tw *tar.Writer,
384	manifestChecksumStore,
385	augmanChecksumStore *artifact.ChecksumStore,
386	artifactInfoStream []byte,
387	) error {	45✔
388	switch version {	45✔
389	case 2:	20✔
390	// add checksum of `version`	20✔
391	ch := artifact.NewWriterChecksum(ioutil.Discard)	20✔
392	_, err := ch.Write(artifactInfoStream)	20✔
393	if err != nil {	20✔
394	return errors.Wrapf(err, "writer: can not write manifest stream")	×
395	}	×
396	err = manifestChecksumStore.Add("version", ch.Checksum())	20✔
397	if err != nil {	20✔
398	return errors.Wrapf(err, "writer: can not write manifest stream")	×
399	}	×
400	// write `manifest` file
401	sw := artifact.NewTarWriterStream(tw)	20✔
402	if err := sw.Write(manifestChecksumStore.GetRaw(), "manifest"); err != nil {	22✔
403	return errors.Wrapf(err, "writer: can not write manifest stream")	2✔
404	}	2✔
405	// write signature
406	if err := WriteSignature(tw, manifestChecksumStore.GetRaw(), signer); err != nil {	20✔
407	return err	2✔
408	}	2✔
409	case 3:	23✔
410	// Add checksum of `version`.	23✔
411	ch := artifact.NewWriterChecksum(ioutil.Discard)	23✔
412	_, err := ch.Write(artifactInfoStream)	23✔
413	if err != nil {	23✔
414	return errors.Wrapf(err, "writer: can not write manifest stream")	×
415	}	×
416	err = manifestChecksumStore.Add("version", ch.Checksum())	23✔
417	if err != nil {	23✔
418	return errors.Wrapf(err, "writer: can not write manifest stream")	×
419	}	×
420	// Write `manifest` file.
421	sw := artifact.NewTarWriterStream(tw)	23✔
422	if err := sw.Write(manifestChecksumStore.GetRaw(), "manifest"); err != nil {	27✔
423	return errors.Wrapf(err, "writer: can not write manifest stream")	4✔
424	}	4✔
425	// Write signature.
426	if err := WriteSignature(tw, manifestChecksumStore.GetRaw(), signer); err != nil {	21✔
427	return err	2✔
428	}	2✔
429	// Write the augmented manifest, if any.
430	if len(augmanChecksumStore.GetRaw()) > 0 {	25✔
431	sw = artifact.NewTarWriterStream(tw)	8✔
432	if err := sw.Write(augmanChecksumStore.GetRaw(), "manifest-augment"); err != nil {	10✔
433	return errors.Wrapf(err, "writer: can not write manifest stream")	2✔
434	}	2✔
435	}
436	default:	2✔
437	return fmt.Errorf("writer: unsupported artifact version: %d", version)	2✔
438	}
439	return nil	31✔
440	}
441
442	func writeScripts(tw tar.Writer, scr artifact.Scripts) error {	3✔
443	sw := artifact.NewTarWriterFile(tw)	3✔
444	for _, script := range scr.Get() {	5✔
445	f, err := os.Open(script)	2✔
446	if err != nil {	2✔
447	return errors.Wrapf(err, "writer: can not open script file: %s", script)	×
448	}	×
449	defer f.Close()	2✔
450		2✔
451	if err :=	2✔
452	sw.Write(f, filepath.Join("scripts", filepath.Base(script))); err != nil {	2✔
453	return errors.Wrapf(err, "writer: can not store script: %s", script)	×
454	}	×
455	}
456	return nil	3✔
457	}
458
459	func extractUpdateTypes(updates []handlers.Composer) []artifact.UpdateType {	41✔
460	u := []artifact.UpdateType{}	41✔
461	for _, upd := range updates {	86✔
462	u = append(u, artifact.UpdateType{Type: upd.GetUpdateType()})	45✔
463	}	45✔
464	return u	41✔
465	}
466
467	func writeHeader(tarWriter tar.Writer, args WriteArtifactArgs, augmented bool) error {	41✔
468	var composers []handlers.Composer	41✔
469	if augmented {	47✔
470	composers = args.Updates.Augments	6✔
471	} else {	41✔
472	composers = args.Updates.Updates	35✔
473	}	35✔
474	if len(composers) == 0 {	41✔
475	return errors.New("writeHeader: No Payloads added")	×
476	}	×
477
478	// store header info
479	var hInfo artifact.WriteValidator	41✔
480	upds := extractUpdateTypes(composers)	41✔
481	switch args.Version {	41✔
482	case 1, 2:	16✔
483	hInfo = artifact.NewHeaderInfo(args.Name, upds, args.Devices)	16✔
484	case 3:	25✔
485	hInfo = artifact.NewHeaderInfoV3(upds, args.Provides, args.Depends)	25✔
486	}
487
488	sa := artifact.NewTarWriterStream(tarWriter)	41✔
489	stream, err := artifact.ToStream(hInfo)	41✔
490	if err != nil {	43✔
491	return errors.Wrap(err, "writeHeader")	2✔
492	}	2✔
493	if err := sa.Write(stream, "header-info"); err != nil {	39✔
494	return errors.Wrap(err, "writer: can not store header-info")	×
495	}	×
496
497	// write scripts
498	if !augmented && args.Scripts != nil {	42✔
499	if err := writeScripts(tarWriter, args.Scripts); err != nil {	3✔
500	return err	×
501	}	×
502	}
503
504	for i, upd := range composers {	82✔
505	// TODO: We only have one `args` variable here, so making more	43✔
506	// than one update is kind of useless. Should probably be made	43✔
507	// into a list as well.	43✔
508	composeHeaderArgs := handlers.ComposeHeaderArgs{	43✔
509	TarWriter: tarWriter,	43✔
510	No: i,	43✔
511	Version: args.Version,	43✔
512	Augmented: augmented,	43✔
513	}	43✔
514	if augmented {	49✔
515	composeHeaderArgs.TypeInfoV3 = args.AugmentTypeInfoV3	6✔
516	composeHeaderArgs.MetaData = args.AugmentMetaData	6✔
517	} else {	43✔
518	composeHeaderArgs.TypeInfoV3 = args.TypeInfoV3	37✔
519	composeHeaderArgs.MetaData = args.MetaData	37✔
520	}	37✔
521	if err := upd.ComposeHeader(&composeHeaderArgs); err != nil {	43✔
522	return errors.Wrapf(err, "writer: error composing header")	×
523	}	×
524	}
525	return nil	39✔
526	}
527
528	func writeData(
529	tw *tar.Writer,
530	comp artifact.Compressor,
531	updates *Updates,
532	pw ProgressWriter,
533	) error {	31✔
534	for i, upd := range updates.Updates {	64✔
535	var augment handlers.Composer = nil	33✔
536	if i < len(updates.Augments) {	37✔
537	augment = updates.Augments[i]	4✔
538	}	4✔
539	if err := writeOneDataTar(tw, comp, i, upd, augment, pw); err != nil {	41✔
540	return errors.Wrapf(err, "writer: error writing data files")	8✔
541	}	8✔
542	}
543	return nil	23✔
544	}
545
546	func writeOneDataTar(tw *tar.Writer, comp artifact.Compressor, no int,
547	baseUpdate, augmentUpdate handlers.Composer, pw ProgressWriter) error {	33✔
548		33✔
549	f, ferr := ioutil.TempFile("", "data")	33✔
550	if ferr != nil {	33✔
551	return errors.New("Payload: can not create temporary data file")	×
552	}	×
553	defer os.Remove(f.Name())	33✔
554		33✔
555	err := func() error {	66✔
556	gz, err := comp.NewWriter(f)	33✔
557	if err != nil {	33✔
558	return errors.Wrap(err, "Could not open compressor")	×
559	}	×
560	defer gz.Close()	33✔
561		33✔
562	var w io.Writer	33✔
563	if pw != nil {	33✔
UNCOV 564	w = pw.Wrap(gz)	×
565	} else {	33✔
566	w = gz	33✔
567	}	33✔
568	tarw := tar.NewWriter(w)	33✔
569	defer tarw.Close()	33✔
570		33✔
571	if len(baseUpdate.GetUpdateFiles()) == 0 && pw != nil {	33✔
UNCOV 572	pw.Reset(0, "bootstrap", 0)	×
UNCOV 573	}	×
574	for i, file := range baseUpdate.GetUpdateFiles() {	61✔
575	fi, err := os.Stat(file.Name)	28✔
576	if err != nil {	30✔
577	return err	2✔
578	}	2✔
579	if pw != nil {	26✔
UNCOV 580	pw.Reset(fi.Size(), file.Name, i)	×
UNCOV 581	}	×
582	err = writeOneDataFile(tarw, file)	26✔
583	if err != nil {	32✔
584	return err	6✔
585	}	6✔
586	if pw != nil {	20✔
UNCOV 587	pw.Finish()	×
UNCOV 588	}	×
589	}
590	if augmentUpdate == nil {	46✔
591	return nil	21✔
592	}	21✔
593
594	for _, file := range augmentUpdate.GetUpdateAugmentFiles() {	4✔
UNCOV 595	err = writeOneDataFile(tarw, file)	×
UNCOV 596	if err != nil {	×
597	return err	×
598	}	×
599	}
600	return nil	4✔
601	}()
602
603	if err != nil {	41✔
604	return err	8✔
605	}	8✔
606
607	if _, err = f.Seek(0, 0); err != nil {	25✔
608	return errors.Wrap(err, "Payload: can not reset file position")	×
609	}	×
610
611	dfw := artifact.NewTarWriterFile(tw)	25✔
612	if err = dfw.Write(f, artifact.UpdateDataPath(no)+comp.GetFileExtension()); err != nil {	25✔
613	return errors.Wrap(err, "Payload: can not write tar data header")	×
614	}	×
615	return nil	25✔
616	}
617
618	func writeOneDataFile(tarw tar.Writer, file handlers.DataFile) error {	26✔
619	matched, err := regexp.MatchString(`^[\w\-.,]+$`, filepath.Base(file.Name))	26✔
620		26✔
621	if err != nil {	26✔
622	return errors.Wrapf(err, "Payload: invalid regular expression pattern")	×
623	}	×
624
625	if !matched {	32✔
626	message := "Payload: data file " + file.Name + " contains forbidden characters"	6✔
627	info := "Only letters, digits and characters in the set \".,_-\" are allowed"	6✔
628	return fmt.Errorf("%s. %s", message, info)	6✔
629	}	6✔
630
631	df, err := os.Open(file.Name)	20✔
632	if err != nil {	20✔
633	return errors.Wrapf(err, "Payload: can not open data file: %s", file.Name)	×
634	}	×
635	fw := artifact.NewTarWriterFile(tarw)	20✔
636	if err := fw.Write(df, filepath.Base(file.Name)); err != nil {	20✔
637	df.Close()	×
638	return errors.Wrapf(err,	×
639	"Payload: can not write tar temp data header: %v", file)	×
640	}	×
641	df.Close()	20✔
642	return nil	20✔
643	}

mendersoftware / mender-artifact / 1279561887

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous