1054468625

Committed 30 Oct 2023 08:04AM UTC coverage: 58.179%. First build

Build # 1054468625

Build Type

Pull #1452

gitlab-ci

Committed by

kacf

Commit Message

fix: Fix possible missing `Sync` call at the very end of the update.

We need to override the `Close` method so that we call `Sync` as the
final thing before closing. Most boards are not affected by this,
because the flushing interval is set to the native sector size of the
board, which, AFAIK, is 512 for all normal block devices except raw
Flash storage. I consider it very unlikely that a filesystem would not
be a multiple of 512 (maybe squashfs?), which means that by the time
we get to `Close()`, `Sync()` will usually have been called already
because the last write was 512 bytes or bigger.

But on raw Flash filesystems it can potentially happen, if the native
sector size is big, and the filesystem is not a multiple of this
number.

Changelog: Fix a rare bug which could corrupt the very end of a
rootfs-image update on a sudden powerloss. The circumstances where it
could happen are quite specific: The filesystem size in the update
need to *not* be a multiple of the native sector size, which is very
uncommon. The sector size is typically 512 bytes almost everywhere,
and hence filesystem also follow this block size, if not bigger. The
exception is raw Flash/UBI devices, where the sector size can be much
larger, and not a power of two, and hence these platforms may be more
susceptible.

Ticket: None

Signed-off-by: Kristian Amlie <kristian.amlie@northern.tech>
(cherry picked from commit ff0db6042)

Pull Request Pull Request #1452: fix: Fix possible missing `Sync` call at the very end of the update.

Run Details

7 of 8 new or added lines in 1 file covered. (87.5%)

6359 of 10930 relevant lines covered (58.18%)

30.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.9

/installer/block_device.go

// Copyright 2023 Northern.tech AS
//
//        Licensed under the Apache License, Version 2.0 (the "License");
//        you may not use this file except in compliance with the License.
//        You may obtain a copy of the License at
//
//            http://www.apache.org/licenses/LICENSE-2.0
//
//        Unless required by applicable law or agreed to in writing, software
//        distributed under the License is distributed on an "AS IS" BASIS,
//        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//        See the License for the specific language governing permissions and
//        limitations under the License.
package installer

import (
        "bytes"
        "io"
        "os"
        "path/filepath"
        "syscall"

        "github.com/pkg/errors"
        log "github.com/sirupsen/logrus"

        "github.com/mendersoftware/mender/system"
        "github.com/mendersoftware/mender/utils"
)

var (
        BlockDeviceGetSizeOf       BlockDeviceGetSizeFunc       = system.GetBlockDeviceSize
        BlockDeviceGetSectorSizeOf BlockDeviceGetSectorSizeFunc = system.GetBlockDeviceSectorSize
)

// BlockDevicer is a file-like interface for the block-device.
type BlockDevicer interface {
        io.Reader
        io.Writer
        io.Closer
        io.Seeker
        Sync() error // Commits previously-written data to stable storage.
}

// BlockDeviceGetSizeFunc is a helper for obtaining the size of a block device.
type BlockDeviceGetSizeFunc func(file *os.File) (uint64, error)

// BlockDeviceGetSectorSizeFunc is a helper for obtaining the sector size of a block device.
type BlockDeviceGetSectorSizeFunc func(file *os.File) (int, error)

// BlockDevice is a low-level wrapper for a block device. The wrapper implements
// the io.Writer and io.Closer interfaces, which is all that is needed by the
// mender-client.
type BlockDevice struct {
        Path string // Device path, ex. /dev/mmcblk0p1
        w    io.WriteCloser
}

type bdevice int

// Give the block-device a package-like interface,
// i.e., blockdevice.Open(partition, size)
var blockdevice bdevice

// Open tries to open the 'device' (/dev/<device> usually), and returns a
// BlockDevice.
func (bd bdevice) Open(device string, size int64) (*BlockDevice, error) {
        log.Infof("Opening device %q for writing", device)

        var out *os.File
        var err error

        log.Debugf("Installing update of size: %d", size)
        if size < 0 {
                return nil, errors.New("Have invalid update. Aborting.")
        }

        // Make sure the file system is not mounted (MEN-2084)
        if mntPt := checkMounted(device); mntPt != "" {
                log.Warnf("Inactive partition %q is mounted at %q. "+
                        "This might be caused by some \"auto mount\" service "+
                        "(e.g udisks2) that mounts all block devices. It is "+
                        "recommended to blacklist the partitions used by "+
                        "Mender to avoid any issues.", device, mntPt)
                log.Warnf("Performing umount on %q.", mntPt)
                err = syscall.Unmount(device, 0)
                if err != nil {
                        log.Errorf("Error unmounting partition %s",
                                device)
                        return nil, err
                }
        }

        typeUBI := system.IsUbiBlockDevice(device)

        log.Debugf("Device: %s is a ubi device: %t", device, typeUBI)

        var flag int

        if typeUBI {
                // UBI block devices are not prefixed with /dev due to the fact
                // that the kernel root= argument does not handle UBI block
                // devices which are prefixed with /dev
                //
                // Kernel root= only accepts:
                // - ubi0_0
                // - ubi:rootfsa
                device = filepath.Join("/dev", device)
                flag = os.O_WRONLY
        } else {
                flag = os.O_RDWR
        }

        b := &BlockDevice{
                Path: device,
        }

        if bsz, err := b.Size(); err != nil {
                log.Errorf("Failed to read size of block device %s: %v",
                        device, err)
                return nil, err
        } else if bsz < uint64(size) {
                log.Errorf("Update (%v bytes) is larger than the size of device %s (%v bytes)",
                        size, device, bsz)
                return nil, syscall.ENOSPC
        }

        nativeSsz, err := b.SectorSize()
        if err != nil {
                log.Errorf("Failed to read sector size of block device %s: %v",
                        device, err)
                return nil, err
        }

        // The size of an individual sector tends to be quite small. Rather than
        // doing a zillion small writes, do medium-size-ish writes that are
        // still sector aligned. (Doing too many small writes can put pressure
        // on the DMA subsystem (unless writes are able to be coalesced) by
        // requiring large numbers of scatter-gather descriptors to be
        // allocated.)
        chunkSize := nativeSsz

        // Pick a multiple of the sector size that's around 1 MiB.
        for chunkSize < 1*1024*1024 {
                chunkSize = chunkSize * 2
        }

        log.Infof(
                "Native sector size of block device %s is %v bytes."+
                        " Mender will write in chunks of %v bytes",
                device,
                nativeSsz,
                chunkSize,
        )

        log.Debugf("Opening device: %s for writing with flag: %d", device, flag)
        out, err = os.OpenFile(device, flag, 0)
        if err != nil {
                return nil, errors.Wrapf(err, "Failed to open the device: %q", device)
        }

        // From <mtd/ubi-user.h>
        //
        // UBI volume update
        // ~~~~~~~~~~~~~~~~~
        //
        // Volume update should be done via the UBI_IOCVOLUP ioctl command of the
        // corresponding UBI volume character device. A pointer to a 64-bit update
        // size should be passed to the ioctl. After this, UBI expects user to write
        // this number of bytes to the volume character device. The update is finished
        // when the claimed number of bytes is passed. So, the volume update sequence
        // is something like:
        //
        // fd = open("/dev/my_volume");
        // ioctl(fd, UBI_IOCVOLUP, &image_size);
        // write(fd, buf, image_size);
        // close(fd);
        if typeUBI {
                err := system.SetUbiUpdateVolume(out, uint64(size))
                if err != nil {
                        log.Errorf("Failed to write images size to UBI_IOCVOLUP: %v", err)
                        return nil, err
                }
        }

        var bdw io.WriteCloser
        if !typeUBI {
                //
                // FlushingWriter is needed due to a driver bug in the linux emmc driver
                // OOM errors.
                //
                // Implements 'BlockDevicer' interface, and is hence the owner of the file
                //
                fw := NewFlushingWriter(out, uint64(nativeSsz))

                //
                // bdw owns the block-device.
                // No-one else is allowed to touch it!
                //
                odw := &OptimizedBlockDeviceWriter{
                        blockDevice: fw,
                }

                //
                // Buffers writes, and writes to the underlying writer
                // once a buffer of size 'frameSize' is full.
                //
                bdw = &BlockFrameWriter{
                        frameSize: chunkSize,
                        buf:       bytes.NewBuffer(nil),
                        w:         odw,
                }
        } else {
                // No optimized writes possible on UBI (Mirza)
                // All the bytes have to be written
                bdw = out
        }

        //
        // The outermost writer. Makes sure that we never(!) write
        // more than the size of the image.
        //
        b.w = &utils.LimitedWriteCloser{
                W: bdw,
                N: uint64(size),
        }

        return b, nil
}

// Write writes data 'b' to the underlying writer. Although this is just one
// line, the underlying implementation is currently slightly more involved. The
// BlockDevice writer will write to a chain of writers as follows:
//
//                 LimitWriter
//        Make sure that no more than image-size
//        bytes are written to the  block-device.
//                    |
//                    |
//                    v
//               BlockFrameWriter
//        Buffers the writes into 'chunkSize' frames
//        for writing to the underlying writer.
//                    |
//                    |
//                    v
//              OptimizedBlockDeviceWriter
//        Only writes dirty frames to the underlying block-device.
//        Note: This is not done for UBI volumes
//                    |
//                    |
//                    v
//                BlockDevicer
//         This is an interface with all the main functionality
//         of a file, and is in this case a FlushingWriter,
//         which writes a chunk to the underlying file-descriptor,
//         and then calls Sync() on every 'FlushIntervalBytes' written.
//
// Due to the underlying writer caching writes, the block-device needs to be
// closed, in order to make sure that all data has been flushed to the device.
func (bd *BlockDevice) Write(b []byte) (n int, err error) {
        if bd.w == nil {
                return 0, errors.New("No device")
        }
        n, err = bd.w.Write(b)
        return n, err
}

// Close closes the underlying block device, thus automatically syncing any
// unwritten data. Othewise, behaves like io.Closer.
func (bd *BlockDevice) Close() error {
        if bd.w == nil {
                return nil
        }
        return bd.w.Close()
}

// Size queries the size of the underlying block device. Automatically opens a
// new fd in O_RDONLY mode, thus can be used in parallel to other operations.
func (bd *BlockDevice) Size() (uint64, error) {
        out, err := os.OpenFile(bd.Path, os.O_RDONLY, 0)
        if err != nil {
                return 0, err
        }
        defer out.Close()
        return BlockDeviceGetSizeOf(out)
}

// SectorSize queries the logical sector size of the underlying block device. Automatically opens a
// new fd in O_RDONLY mode, thus can be used in parallel to other operations.
func (bd *BlockDevice) SectorSize() (int, error) {
        out, err := os.OpenFile(bd.Path, os.O_RDONLY, 0)
        if err != nil {
                return 0, err
        }
        defer out.Close()
        return BlockDeviceGetSectorSizeOf(out)
}

type BlockFrameWriter struct {
        buf       *bytes.Buffer
        frameSize int
        w         io.WriteCloser
}

// Write buffers the writes into a buffer of size 'frameSize'. Then, when this
// buffer is full, it writes 'frameSize' bytes to the underlying writer.
func (bw *BlockFrameWriter) Write(b []byte) (n int, err error) {

        // Fill the frame buffer first
        n, err = bw.buf.Write(b)
        if err != nil {
                return n, err
        }

        if bw.buf.Len() < bw.frameSize {
                return n, nil // Chunk buffer not full
        }

        nFrames := bw.buf.Len() / bw.frameSize
        for i := 0; i < nFrames; i++ {
                _, err = bw.w.Write(bw.buf.Next(bw.frameSize))
                if err != nil {
                        return 0, err
                }
        }

        // Write a full frame, but report only the last byte chunk as written
        return len(b), nil
}

// Close flushes the remaining cached bytes -- if any.
func (bw *BlockFrameWriter) Close() error {
        _, err := bw.w.Write(bw.buf.Bytes())
        if cerr := bw.w.Close(); cerr != nil {
                return cerr
        }
        if err == io.EOF {
                return nil
        }
        return err
}

// OptimizedBlockDeviceWriter wraps an underlying blockDevice write, however,
// with the optimization that it compares the bytes passed in to the Write
// method, with the next len([]byte) bytes (considered a frame) on the block
// device, and if they match, the write is discarded. The lingo is that only
// dirty frames are written. Clean ones are discarded.
type OptimizedBlockDeviceWriter struct {
        blockDevice BlockDevicer
        totalFrames int
        dirtyFrames int
}

// Write only write 'dirty' frames.
// Note: a frame-size is always the size 'len(b)'
func (bd *OptimizedBlockDeviceWriter) Write(b []byte) (n int, err error) {

        frameSize := int64(len(b))
        payloadBuf := make([]byte, frameSize)

        //
        // Read len(b) bytes from the block-device
        //
        n, err = io.ReadFull(bd.blockDevice, payloadBuf)
        if err != nil {
                log.Errorf("Failed to read a full frame of size: %d from the block-device", err)
                return 0, err
        }

        //
        // Write the frame if it is dirty.
        //
        if !bytes.Equal(payloadBuf, b) {
                // In order to write, we need to seek back to
                // the start of the chunk.
                if _, err = bd.blockDevice.Seek(-int64(frameSize), io.SeekCurrent); err != nil {
                        log.Errorf("Failed to seek back to the start of the frame. Err: %v", err)
                        return 0, err
                }
                bd.totalFrames += 1
                bd.dirtyFrames += 1
                return bd.blockDevice.Write(b)
        }

        // No need to write a clean frame
        bd.totalFrames += 1
        return n, err
}

func (obw *OptimizedBlockDeviceWriter) Close() error {
        s := "The optimized block-device writer wrote a total of %d frames, " +
                "where %d frames did need to be rewritten (i.e., skipped)"
        log.Infof(s, obw.totalFrames, obw.dirtyFrames)
        return obw.blockDevice.Close()
}

// FlushingWriter is a wrapper around a BlockDevice which forces a Sync() to occur
// every FlushIntervalBytes.
type FlushingWriter struct {
        BlockDevicer
        FlushIntervalBytes    uint64
        unflushedBytesWritten uint64
}

// NewFlushingWriter returns a FlushingWriter which wraps the provided
// block-device (BlockDevicer) and automatically flushes (calls Sync()) each
// time the specified number of bytes is written. Setting flushIntervalBytes == 0
// causes Sync() to be called after every Write().
func NewFlushingWriter(wf *os.File, flushIntervalBytes uint64) *FlushingWriter {
        return &FlushingWriter{
                BlockDevicer:          wf,
                FlushIntervalBytes:    flushIntervalBytes,
                unflushedBytesWritten: 0,
        }
}

func (fw *FlushingWriter) Write(p []byte) (int, error) {
        rv, err := fw.BlockDevicer.Write(p)

        fw.unflushedBytesWritten += uint64(rv)

        if err != nil {
                return rv, err
        } else if fw.unflushedBytesWritten >= fw.FlushIntervalBytes {
                err = fw.Sync()
        }

        return rv, err
}

func (fw *FlushingWriter) Sync() error {
        err := fw.BlockDevicer.Sync()
        fw.unflushedBytesWritten = 0
        return err
}

func (fw *FlushingWriter) Close() error {
        sync_err := fw.Sync()
        close_err := fw.BlockDevicer.Close()
        if sync_err != nil {
                return sync_err
        } else {
                return close_err
        }
}

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14	package installer
15
16	import (
17	"bytes"
18	"io"
19	"os"
20	"path/filepath"
21	"syscall"
22
23	"github.com/pkg/errors"
24	log "github.com/sirupsen/logrus"
25
26	"github.com/mendersoftware/mender/system"
27	"github.com/mendersoftware/mender/utils"
28	)
29
30	var (
31	BlockDeviceGetSizeOf BlockDeviceGetSizeFunc = system.GetBlockDeviceSize
32	BlockDeviceGetSectorSizeOf BlockDeviceGetSectorSizeFunc = system.GetBlockDeviceSectorSize
33	)
34
35	// BlockDevicer is a file-like interface for the block-device.
36	type BlockDevicer interface {
37	io.Reader
38	io.Writer
39	io.Closer
40	io.Seeker
41	Sync() error // Commits previously-written data to stable storage.
42	}
43
44	// BlockDeviceGetSizeFunc is a helper for obtaining the size of a block device.
45	type BlockDeviceGetSizeFunc func(file *os.File) (uint64, error)
46
47	// BlockDeviceGetSectorSizeFunc is a helper for obtaining the sector size of a block device.
48	type BlockDeviceGetSectorSizeFunc func(file *os.File) (int, error)
49
50	// BlockDevice is a low-level wrapper for a block device. The wrapper implements
51	// the io.Writer and io.Closer interfaces, which is all that is needed by the
52	// mender-client.
53	type BlockDevice struct {
54	Path string // Device path, ex. /dev/mmcblk0p1
55	w io.WriteCloser
56	}
57
58	type bdevice int
59
60	// Give the block-device a package-like interface,
61	// i.e., blockdevice.Open(partition, size)
62	var blockdevice bdevice
63
64	// Open tries to open the 'device' (/dev/<device> usually), and returns a
65	// BlockDevice.
66	func (bd bdevice) Open(device string, size int64) (*BlockDevice, error) {	35✔
67	log.Infof("Opening device %q for writing", device)	35✔
68		35✔
69	var out *os.File	35✔
70	var err error	35✔
71		35✔
72	log.Debugf("Installing update of size: %d", size)	35✔
73	if size < 0 {	35✔
74	return nil, errors.New("Have invalid update. Aborting.")	×
75	}	×
76
77	// Make sure the file system is not mounted (MEN-2084)
78	if mntPt := checkMounted(device); mntPt != "" {	35✔
79	log.Warnf("Inactive partition %q is mounted at %q. "+	×
80	"This might be caused by some \"auto mount\" service "+	×
81	"(e.g udisks2) that mounts all block devices. It is "+	×
82	"recommended to blacklist the partitions used by "+	×
83	"Mender to avoid any issues.", device, mntPt)	×
84	log.Warnf("Performing umount on %q.", mntPt)	×
85	err = syscall.Unmount(device, 0)	×
86	if err != nil {	×
87	log.Errorf("Error unmounting partition %s",	×
88	device)	×
89	return nil, err	×
90	}	×
91	}
92
93	typeUBI := system.IsUbiBlockDevice(device)	35✔
94		35✔
95	log.Debugf("Device: %s is a ubi device: %t", device, typeUBI)	35✔
96		35✔
97	var flag int	35✔
98		35✔
99	if typeUBI {	37✔
100	// UBI block devices are not prefixed with /dev due to the fact	2✔
101	// that the kernel root= argument does not handle UBI block	2✔
102	// devices which are prefixed with /dev	2✔
103	//	2✔
104	// Kernel root= only accepts:	2✔
105	// - ubi0_0	2✔
106	// - ubi:rootfsa	2✔
107	device = filepath.Join("/dev", device)	2✔
108	flag = os.O_WRONLY	2✔
109	} else {	35✔
110	flag = os.O_RDWR	33✔
111	}	33✔
112
113	b := &BlockDevice{	35✔
114	Path: device,	35✔
115	}	35✔
116		35✔
117	if bsz, err := b.Size(); err != nil {	35✔
118	log.Errorf("Failed to read size of block device %s: %v",	×
119	device, err)	×
120	return nil, err	×
121	} else if bsz < uint64(size) {	36✔
122	log.Errorf("Update (%v bytes) is larger than the size of device %s (%v bytes)",	1✔
123	size, device, bsz)	1✔
124	return nil, syscall.ENOSPC	1✔
125	}	1✔
126
127	nativeSsz, err := b.SectorSize()	34✔
128	if err != nil {	34✔
129	log.Errorf("Failed to read sector size of block device %s: %v",	×
130	device, err)	×
131	return nil, err	×
132	}	×
133
134	// The size of an individual sector tends to be quite small. Rather than
135	// doing a zillion small writes, do medium-size-ish writes that are
136	// still sector aligned. (Doing too many small writes can put pressure
137	// on the DMA subsystem (unless writes are able to be coalesced) by
138	// requiring large numbers of scatter-gather descriptors to be
139	// allocated.)
140	chunkSize := nativeSsz	34✔
141		34✔
142	// Pick a multiple of the sector size that's around 1 MiB.	34✔
143	for chunkSize < 110241024 {	68✔
144	chunkSize = chunkSize * 2	34✔
145	}	34✔
146
147	log.Infof(	34✔
148	"Native sector size of block device %s is %v bytes."+	34✔
149	" Mender will write in chunks of %v bytes",	34✔
150	device,	34✔
151	nativeSsz,	34✔
152	chunkSize,	34✔
153	)	34✔
154		34✔
155	log.Debugf("Opening device: %s for writing with flag: %d", device, flag)	34✔
156	out, err = os.OpenFile(device, flag, 0)	34✔
157	if err != nil {	34✔
158	return nil, errors.Wrapf(err, "Failed to open the device: %q", device)	×
159	}	×
160
161	// From <mtd/ubi-user.h>
162	//
163	// UBI volume update
164	// ~~~~~~~~~~~~~~~~~
165	//
166	// Volume update should be done via the UBI_IOCVOLUP ioctl command of the
167	// corresponding UBI volume character device. A pointer to a 64-bit update
168	// size should be passed to the ioctl. After this, UBI expects user to write
169	// this number of bytes to the volume character device. The update is finished
170	// when the claimed number of bytes is passed. So, the volume update sequence
171	// is something like:
172	//
173	// fd = open("/dev/my_volume");
174	// ioctl(fd, UBI_IOCVOLUP, &image_size);
175	// write(fd, buf, image_size);
176	// close(fd);
177	if typeUBI {	36✔
178	err := system.SetUbiUpdateVolume(out, uint64(size))	2✔
179	if err != nil {	2✔
180	log.Errorf("Failed to write images size to UBI_IOCVOLUP: %v", err)	×
181	return nil, err	×
182	}	×
183	}
184
185	var bdw io.WriteCloser	34✔
186	if !typeUBI {	66✔
187	//	32✔
188	// FlushingWriter is needed due to a driver bug in the linux emmc driver	32✔
189	// OOM errors.	32✔
190	//	32✔
191	// Implements 'BlockDevicer' interface, and is hence the owner of the file	32✔
192	//	32✔
193	fw := NewFlushingWriter(out, uint64(nativeSsz))	32✔
194		32✔
195	//	32✔
196	// bdw owns the block-device.	32✔
197	// No-one else is allowed to touch it!	32✔
198	//	32✔
199	odw := &OptimizedBlockDeviceWriter{	32✔
200	blockDevice: fw,	32✔
201	}	32✔
202		32✔
203	//	32✔
204	// Buffers writes, and writes to the underlying writer	32✔
205	// once a buffer of size 'frameSize' is full.	32✔
206	//	32✔
207	bdw = &BlockFrameWriter{	32✔
208	frameSize: chunkSize,	32✔
209	buf: bytes.NewBuffer(nil),	32✔
210	w: odw,	32✔
211	}	32✔
212	} else {	34✔
213	// No optimized writes possible on UBI (Mirza)	2✔
214	// All the bytes have to be written	2✔
215	bdw = out	2✔
216	}	2✔
217
218	//
219	// The outermost writer. Makes sure that we never(!) write
220	// more than the size of the image.
221	//
222	b.w = &utils.LimitedWriteCloser{	34✔
223	W: bdw,	34✔
224	N: uint64(size),	34✔
225	}	34✔
226		34✔
227	return b, nil	34✔
228	}
229
230	// Write writes data 'b' to the underlying writer. Although this is just one
231	// line, the underlying implementation is currently slightly more involved. The
232	// BlockDevice writer will write to a chain of writers as follows:
233	//
234	// LimitWriter
235	// Make sure that no more than image-size
236	// bytes are written to the block-device.
237	// \|
238	// \|
239	// v
240	// BlockFrameWriter
241	// Buffers the writes into 'chunkSize' frames
242	// for writing to the underlying writer.
243	// \|
244	// \|
245	// v
246	// OptimizedBlockDeviceWriter
247	// Only writes dirty frames to the underlying block-device.
248	// Note: This is not done for UBI volumes
249	// \|
250	// \|
251	// v
252	// BlockDevicer
253	// This is an interface with all the main functionality
254	// of a file, and is in this case a FlushingWriter,
255	// which writes a chunk to the underlying file-descriptor,
256	// and then calls Sync() on every 'FlushIntervalBytes' written.
257	//
258	// Due to the underlying writer caching writes, the block-device needs to be
259	// closed, in order to make sure that all data has been flushed to the device.
260	func (bd *BlockDevice) Write(b []byte) (n int, err error) {	34✔
261	if bd.w == nil {	34✔
262	return 0, errors.New("No device")	×
263	}	×
264	n, err = bd.w.Write(b)	34✔
265	return n, err	34✔
266	}
267
268	// Close closes the underlying block device, thus automatically syncing any
269	// unwritten data. Othewise, behaves like io.Closer.
270	func (bd *BlockDevice) Close() error {	34✔
271	if bd.w == nil {	34✔
272	return nil	×
273	}	×
274	return bd.w.Close()	34✔
275	}
276
277	// Size queries the size of the underlying block device. Automatically opens a
278	// new fd in O_RDONLY mode, thus can be used in parallel to other operations.
279	func (bd *BlockDevice) Size() (uint64, error) {	35✔
280	out, err := os.OpenFile(bd.Path, os.O_RDONLY, 0)	35✔
281	if err != nil {	35✔
282	return 0, err	×
283	}	×
284	defer out.Close()	35✔
285	return BlockDeviceGetSizeOf(out)	35✔
286	}
287
288	// SectorSize queries the logical sector size of the underlying block device. Automatically opens a
289	// new fd in O_RDONLY mode, thus can be used in parallel to other operations.
290	func (bd *BlockDevice) SectorSize() (int, error) {	34✔
291	out, err := os.OpenFile(bd.Path, os.O_RDONLY, 0)	34✔
292	if err != nil {	34✔
293	return 0, err	×
294	}	×
295	defer out.Close()	34✔
296	return BlockDeviceGetSectorSizeOf(out)	34✔
297	}
298
299	type BlockFrameWriter struct {
300	buf *bytes.Buffer
301	frameSize int
302	w io.WriteCloser
303	}
304
305	// Write buffers the writes into a buffer of size 'frameSize'. Then, when this
306	// buffer is full, it writes 'frameSize' bytes to the underlying writer.
307	func (bw *BlockFrameWriter) Write(b []byte) (n int, err error) {	32✔
308		32✔
309	// Fill the frame buffer first	32✔
310	n, err = bw.buf.Write(b)	32✔
311	if err != nil {	32✔
312	return n, err	×
313	}	×
314
315	if bw.buf.Len() < bw.frameSize {	64✔
316	return n, nil // Chunk buffer not full	32✔
317	}	32✔
318
319	nFrames := bw.buf.Len() / bw.frameSize	32✔
320	for i := 0; i < nFrames; i++ {	64✔
321	_, err = bw.w.Write(bw.buf.Next(bw.frameSize))	32✔
322	if err != nil {	32✔
323	return 0, err	×
324	}	×
325	}
326
327	// Write a full frame, but report only the last byte chunk as written
328	return len(b), nil	32✔
329	}
330
331	// Close flushes the remaining cached bytes -- if any.
332	func (bw *BlockFrameWriter) Close() error {	32✔
333	_, err := bw.w.Write(bw.buf.Bytes())	32✔
334	if cerr := bw.w.Close(); cerr != nil {	32✔
335	return cerr	×
336	}	×
337	if err == io.EOF {	32✔
338	return nil	×
339	}	×
340	return err	32✔
341	}
342
343	// OptimizedBlockDeviceWriter wraps an underlying blockDevice write, however,
344	// with the optimization that it compares the bytes passed in to the Write
345	// method, with the next len([]byte) bytes (considered a frame) on the block
346	// device, and if they match, the write is discarded. The lingo is that only
347	// dirty frames are written. Clean ones are discarded.
348	type OptimizedBlockDeviceWriter struct {
349	blockDevice BlockDevicer
350	totalFrames int
351	dirtyFrames int
352	}
353
354	// Write only write 'dirty' frames.
355	// Note: a frame-size is always the size 'len(b)'
356	func (bd *OptimizedBlockDeviceWriter) Write(b []byte) (n int, err error) {	32✔
357		32✔
358	frameSize := int64(len(b))	32✔
359	payloadBuf := make([]byte, frameSize)	32✔
360		32✔
361	//	32✔
362	// Read len(b) bytes from the block-device	32✔
363	//	32✔
364	n, err = io.ReadFull(bd.blockDevice, payloadBuf)	32✔
365	if err != nil {	32✔
366	log.Errorf("Failed to read a full frame of size: %d from the block-device", err)	×
367	return 0, err	×
368	}	×
369
370	//
371	// Write the frame if it is dirty.
372	//
373	if !bytes.Equal(payloadBuf, b) {	64✔
374	// In order to write, we need to seek back to	32✔
375	// the start of the chunk.	32✔
376	if _, err = bd.blockDevice.Seek(-int64(frameSize), io.SeekCurrent); err != nil {	32✔
377	log.Errorf("Failed to seek back to the start of the frame. Err: %v", err)	×
378	return 0, err	×
379	}	×
380	bd.totalFrames += 1	32✔
381	bd.dirtyFrames += 1	32✔
382	return bd.blockDevice.Write(b)	32✔
383	}
384
385	// No need to write a clean frame
386	bd.totalFrames += 1	32✔
387	return n, err	32✔
388	}
389
390	func (obw *OptimizedBlockDeviceWriter) Close() error {	32✔
391	s := "The optimized block-device writer wrote a total of %d frames, " +	32✔
392	"where %d frames did need to be rewritten (i.e., skipped)"	32✔
393	log.Infof(s, obw.totalFrames, obw.dirtyFrames)	32✔
394	return obw.blockDevice.Close()	32✔
395	}	32✔
396
397	// FlushingWriter is a wrapper around a BlockDevice which forces a Sync() to occur
398	// every FlushIntervalBytes.
399	type FlushingWriter struct {
400	BlockDevicer
401	FlushIntervalBytes uint64
402	unflushedBytesWritten uint64
403	}
404
405	// NewFlushingWriter returns a FlushingWriter which wraps the provided
406	// block-device (BlockDevicer) and automatically flushes (calls Sync()) each
407	// time the specified number of bytes is written. Setting flushIntervalBytes == 0
408	// causes Sync() to be called after every Write().
409	func NewFlushingWriter(wf os.File, flushIntervalBytes uint64) FlushingWriter {	32✔
410	return &FlushingWriter{	32✔
411	BlockDevicer: wf,	32✔
412	FlushIntervalBytes: flushIntervalBytes,	32✔
413	unflushedBytesWritten: 0,	32✔
414	}	32✔
415	}	32✔
416
417	func (fw *FlushingWriter) Write(p []byte) (int, error) {	32✔
418	rv, err := fw.BlockDevicer.Write(p)	32✔
419		32✔
420	fw.unflushedBytesWritten += uint64(rv)	32✔
421		32✔
422	if err != nil {	32✔
423	return rv, err	×
424	} else if fw.unflushedBytesWritten >= fw.FlushIntervalBytes {	64✔
425	err = fw.Sync()	32✔
426	}	32✔
427
428	return rv, err	32✔
429	}
430
431	func (fw *FlushingWriter) Sync() error {	32✔
432	err := fw.BlockDevicer.Sync()	32✔
433	fw.unflushedBytesWritten = 0	32✔
434	return err	32✔
435	}	32✔
436
437	func (fw *FlushingWriter) Close() error {	32✔
438	sync_err := fw.Sync()	32✔
439	close_err := fw.BlockDevicer.Close()	32✔
440	if sync_err != nil {	32✔
NEW 441	return sync_err	×
442	} else {	32✔
443	return close_err	32✔
444	}	32✔
445	}

mendersoftware / mender / 1054468625

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous