14879619421

Committed 07 May 2025 09:11AM UTC coverage: 68.989% (-0.003%) from 68.992%

Build # 14879619421

Build Type

Pull #9788

github

Committed by

web-flow

Commit Message

Merge 2c656b4ca into 67a40c90a

Pull Request Pull Request #9788: tlv: fix SizeFunc signature

Run Details

13 of 15 new or added lines in 3 files covered. (86.67%)

76 existing lines in 18 files now uncovered.

133891 of 194077 relevant lines covered (68.99%)

22173.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.45

/tlv/stream.go

package tlv

import (
        "bytes"
        "errors"
        "fmt"
        "io"
        "math"
)

// MaxRecordSize is the maximum size of a particular record that will be parsed
// by a stream decoder. This value is currently chosen to the be equal to the
// maximum message size permitted by BOLT 1, as no record should be bigger than
// an entire message.
const MaxRecordSize = 65535 // 65KB

// ErrStreamNotCanonical signals that a decoded stream does not contain records
// sorting by monotonically-increasing type.
var ErrStreamNotCanonical = errors.New("tlv stream is not canonical")

// ErrRecordTooLarge signals that a decoded record has a length that is too
// long to parse.
var ErrRecordTooLarge = errors.New("record is too large")

// Stream defines a TLV stream that can be used for encoding or decoding a set
// of TLV Records.
type Stream struct {
        records []Record
        buf     [8]byte
}

// NewStream creates a new TLV Stream given an encoding codec, a decoding codec,
// and a set of known records.
func NewStream(records ...Record) (*Stream, error) {
        // Assert that the ordering of the Records is canonical and appear in
        // ascending order of type.
        var (
                min      Type
                overflow bool
        )
        for _, record := range records {
                if overflow || record.typ < min {
                        return nil, ErrStreamNotCanonical
                }
                if record.encoder == nil {
                        record.encoder = ENOP
                }
                if record.decoder == nil {
                        record.decoder = DNOP
                }
                if record.typ == math.MaxUint64 {
                        overflow = true
                }
                min = record.typ + 1
        }

        return &Stream{
                records: records,
        }, nil
}

// MustNewStream creates a new TLV Stream given an encoding codec, a decoding
// codec, and a set of known records. If an error is encountered in creating the
// stream, this method will panic instead of returning the error.
func MustNewStream(records ...Record) *Stream {
        stream, err := NewStream(records...)
        if err != nil {
                panic(err.Error())
        }
        return stream
}

// Encode writes a Stream to the passed io.Writer. Each of the Records known to
// the Stream is written in ascending order of their type so as to be canonical.
//
// The stream is constructed by concatenating the individual, serialized Records
// where each record has the following format:
//
//        [varint: type]
//        [varint: length]
//        [length: value]
//
// An error is returned if the io.Writer fails to accept bytes from the
// encoding, and nothing else. The ordering of the Records is asserted upon the
// creation of a Stream, and thus the output will be by definition canonical.
func (s *Stream) Encode(w io.Writer) error {
        // Iterate through all known records, if any, serializing each record's
        // type, length and value.
        for i := range s.records {
                rec := &s.records[i]

                // Write the record's type as a varint.
                err := WriteVarInt(w, uint64(rec.typ), &s.buf)
                if err != nil {
                        return err
                }

                size, err := rec.Size()
                if err != nil {
                        return fmt.Errorf("could not determine record size: %w",
                                err)
                }

                // Write the record's length as a varint.
                err = WriteVarInt(w, size, &s.buf)
                if err != nil {
                        return err
                }

                // Encode the current record's value using the stream's codec.
                err = rec.encoder(w, rec.value, &s.buf)
                if err != nil {
                        return err
                }
        }

        return nil
}

// Decode deserializes TLV Stream from the passed io.Reader for non-P2P
// settings. The Stream will inspect each record that is parsed and check to
// see if it has a corresponding Record to facilitate deserialization of that
// field. If the record is unknown, the Stream will discard the record's bytes
// and proceed to the subsequent record.
//
// Each record has the following format:
//
//        [varint: type]
//        [varint: length]
//        [length: value]
//
// A series of (possibly zero) records are concatenated into a stream, this
// example contains two records:
//
//        (t: 0x01, l: 0x04, v: 0xff, 0xff, 0xff, 0xff)
//        (t: 0x02, l: 0x01, v: 0x01)
//
// This method asserts that the byte stream is canonical, namely that each
// record is unique and that all records are sorted in ascending order. An
// ErrNotCanonicalStream error is returned if the encoded TLV stream is not.
//
// We permit an io.EOF error only when reading the type byte which signals that
// the last record was read cleanly and we should stop parsing. All other io.EOF
// or io.ErrUnexpectedEOF errors are returned.
func (s *Stream) Decode(r io.Reader) error {
        _, err := s.decode(r, nil, false)
        return err
}

// DecodeP2P is identical to Decode except that the maximum record size is
// capped at 65535.
func (s *Stream) DecodeP2P(r io.Reader) error {
        _, err := s.decode(r, nil, true)
        return err
}

// DecodeWithParsedTypes is identical to Decode, but if successful, returns a
// TypeMap containing the types of all records that were decoded or ignored from
// the stream.
func (s *Stream) DecodeWithParsedTypes(r io.Reader) (TypeMap, error) {
        return s.decode(r, make(TypeMap), false)
}

// DecodeWithParsedTypesP2P is identical to DecodeWithParsedTypes except that
// the record size is capped at 65535. This should only be called from a p2p
// setting where untrusted input is being deserialized.
func (s *Stream) DecodeWithParsedTypesP2P(r io.Reader) (TypeMap, error) {
        return s.decode(r, make(TypeMap), true)
}

// decode is a helper function that performs the basis of stream decoding. If
// the caller needs the set of parsed types, it must provide an initialized
// parsedTypes, otherwise the returned TypeMap will be nil. If the p2p bool is
// true, then the record size is capped at 65535. Otherwise, it is not capped.
func (s *Stream) decode(r io.Reader, parsedTypes TypeMap, p2p bool) (TypeMap,
        error) {

        var (
                typ       Type
                min       Type
                recordIdx int
                overflow  bool
        )

        // Iterate through all possible type identifiers. As types are read from
        // the io.Reader, min will skip forward to the last read type.
        for {
                // Read the next varint type.
                t, err := ReadVarInt(r, &s.buf)
                switch {

                // We'll silence an EOF when zero bytes remain, meaning the
                // stream was cleanly encoded.
                case err == io.EOF:
                        return parsedTypes, nil

                // Other unexpected errors.
                case err != nil:
                        return nil, err
                }

                typ = Type(t)

                // Assert that this type is greater than any previously read.
                // If we've already overflowed and we parsed another type, the
                // stream is not canonical. This check prevents us from accepts
                // encodings that have duplicate records or from accepting an
                // unsorted series.
                if overflow || typ < min {
                        return nil, ErrStreamNotCanonical
                }

                // Read the varint length.
                length, err := ReadVarInt(r, &s.buf)
                switch {

                // We'll convert any EOFs to ErrUnexpectedEOF, since this
                // results in an invalid record.
                case err == io.EOF:
                        return nil, io.ErrUnexpectedEOF

                // Other unexpected errors.
                case err != nil:
                        return nil, err
                }

                // Place a soft limit on the size of a sane record, which
                // prevents malicious encoders from causing us to allocate an
                // unbounded amount of memory when decoding variable-sized
                // fields. This is only checked when the p2p bool is true.
                if p2p && length > MaxRecordSize {
                        return nil, ErrRecordTooLarge
                }

                // Search the records known to the stream for this type. We'll
                // begin the search and recordIdx and walk forward until we find
                // it or the next record's type is larger.
                rec, newIdx, ok := s.getRecord(typ, recordIdx)
                switch {

                // We know of this record type, proceed to decode the value.
                // This method asserts that length bytes are read in the
                // process, and returns an error if the number of bytes is not
                // exactly length.
                case ok:
                        err := rec.decoder(r, rec.value, &s.buf, length)
                        switch {

                        // We'll convert any EOFs to ErrUnexpectedEOF, since this
                        // results in an invalid record.
                        case err == io.EOF:
                                return nil, io.ErrUnexpectedEOF

                        // Other unexpected errors.
                        case err != nil:
                                return nil, err
                        }

                        // Record the successfully decoded type if the caller
                        // provided an initialized TypeMap.
                        if parsedTypes != nil {
                                parsedTypes[typ] = nil
                        }

                // Otherwise, the record type is unknown and is odd, discard the
                // number of bytes specified by length.
                default:
                        // If the caller provided an initialized TypeMap, record
                        // the encoded bytes.
                        var b *bytes.Buffer
                        writer := io.Discard
                        if parsedTypes != nil {
                                b = bytes.NewBuffer(make([]byte, 0, length))
                                writer = b
                        }

                        _, err := io.CopyN(writer, r, int64(length))
                        switch {

                        // We'll convert any EOFs to ErrUnexpectedEOF, since this
                        // results in an invalid record.
                        case err == io.EOF:
                                return nil, io.ErrUnexpectedEOF

                        // Other unexpected errors.
                        case err != nil:
                                return nil, err
                        }

                        if parsedTypes != nil {
                                parsedTypes[typ] = b.Bytes()
                        }
                }

                // Update our record index so that we can begin our next search
                // from where we left off.
                recordIdx = newIdx

                // If we've parsed the largest possible type, the next loop will
                // overflow back to zero. However, we need to attempt parsing
                // the next type to ensure that the stream is empty.
                if typ == math.MaxUint64 {
                        overflow = true
                }

                // Finally, set our lower bound on the next accepted type.
                min = typ + 1
        }
}

// getRecord searches for a record matching typ known to the stream. The boolean
// return value indicates whether the record is known to the stream. The integer
// return value carries the index from where getRecord should be invoked on the
// subsequent call. The first call to getRecord should always use an idx of 0.
func (s *Stream) getRecord(typ Type, idx int) (Record, int, bool) {
        for idx < len(s.records) {
                record := s.records[idx]
                switch {

                // Found target record, return it to the caller. The next index
                // returned points to the immediately following record.
                case record.typ == typ:
                        return record, idx + 1, true

                // This record's type is lower than the target. Advance our
                // index and continue to the next record which will have a
                // strictly higher type.
                case record.typ < typ:
                        idx++
                        continue

                // This record's type is larger than the target, hence we have
                // no record matching the current type. Return the current index
                // so that we can start our search from here when processing the
                // next tlv record.
                default:
                        return Record{}, idx, false
                }
        }

        // All known records are exhausted.
        return Record{}, idx, false
}

1	package tlv
2
3	import (
4	"bytes"
5	"errors"
6	"fmt"
7	"io"
8	"math"
9	)
10
11	// MaxRecordSize is the maximum size of a particular record that will be parsed
12	// by a stream decoder. This value is currently chosen to the be equal to the
13	// maximum message size permitted by BOLT 1, as no record should be bigger than
14	// an entire message.
15	const MaxRecordSize = 65535 // 65KB
16
17	// ErrStreamNotCanonical signals that a decoded stream does not contain records
18	// sorting by monotonically-increasing type.
19	var ErrStreamNotCanonical = errors.New("tlv stream is not canonical")
20
21	// ErrRecordTooLarge signals that a decoded record has a length that is too
22	// long to parse.
23	var ErrRecordTooLarge = errors.New("record is too large")
24
25	// Stream defines a TLV stream that can be used for encoding or decoding a set
26	// of TLV Records.
27	type Stream struct {
28	records []Record
29	buf [8]byte
30	}
31
32	// NewStream creates a new TLV Stream given an encoding codec, a decoding codec,
33	// and a set of known records.	139,027✔
34	func NewStream(records ...Record) (*Stream, error) {	139,027✔
35	// Assert that the ordering of the Records is canonical and appear in	139,027✔
36	// ascending order of type.	139,027✔
37	var (	139,027✔
38	min Type	139,027✔
39	overflow bool	139,027✔
40	)	475,269✔
41	for _, record := range records {	336,242✔
UNCOV 42	if overflow \|\| record.typ < min {	×
43	return nil, ErrStreamNotCanonical	×
44	}	336,242✔
UNCOV 45	if record.encoder == nil {	×
46	record.encoder = ENOP	×
47	}	352,502✔
48	if record.decoder == nil {	16,260✔
49	record.decoder = DNOP	16,260✔
50	}	336,244✔
51	if record.typ == math.MaxUint64 {	2✔
52	overflow = true	2✔
53	}	336,242✔
54	min = record.typ + 1
55	}
56		139,027✔
57	return &Stream{	139,027✔
58	records: records,	139,027✔
59	}, nil
60	}
61
62	// MustNewStream creates a new TLV Stream given an encoding codec, a decoding
63	// codec, and a set of known records. If an error is encountered in creating the
64	// stream, this method will panic instead of returning the error.	75✔
65	func MustNewStream(records ...Record) *Stream {	75✔
66	stream, err := NewStream(records...)	75✔
UNCOV 67	if err != nil {	×
68	panic(err.Error())
69	}	75✔
70	return stream
71	}
72
73	// Encode writes a Stream to the passed io.Writer. Each of the Records known to
74	// the Stream is written in ascending order of their type so as to be canonical.
75	//
76	// The stream is constructed by concatenating the individual, serialized Records
77	// where each record has the following format:
78	//
79	// [varint: type]
80	// [varint: length]
81	// [length: value]
82	//
83	// An error is returned if the io.Writer fails to accept bytes from the
84	// encoding, and nothing else. The ordering of the Records is asserted upon the
85	// creation of a Stream, and thus the output will be by definition canonical.	79,474✔
86	func (s *Stream) Encode(w io.Writer) error {	79,474✔
87	// Iterate through all known records, if any, serializing each record's	79,474✔
88	// type, length and value.	213,472✔
89	for i := range s.records {	133,998✔
90	rec := &s.records[i]	133,998✔
91		133,998✔
92	// Write the record's type as a varint.	133,998✔
93	err := WriteVarInt(w, uint64(rec.typ), &s.buf)	133,998✔
UNCOV 94	if err != nil {	×
95	return err	×
96	}
97
98	size, err := rec.Size()	133,998✔
99	if err != nil {	133,998✔
NEW 100	return fmt.Errorf("could not determine record size: %w",	×
NEW 101	err)	×
102	}
103
104	// Write the record's length as a varint.	133,998✔
105	err = WriteVarInt(w, size, &s.buf)	133,998✔
UNCOV 106	if err != nil {	×
107	return err	×
108	}
109
110	// Encode the current record's value using the stream's codec.	79,474✔
111	err = rec.encoder(w, rec.value, &s.buf)
112	if err != nil {
113	return err
114	}
115	}
116
117	return nil
118	}
119
120	// Decode deserializes TLV Stream from the passed io.Reader for non-P2P
121	// settings. The Stream will inspect each record that is parsed and check to
122	// see if it has a corresponding Record to facilitate deserialization of that
123	// field. If the record is unknown, the Stream will discard the record's bytes
124	// and proceed to the subsequent record.
125	//
126	// Each record has the following format:
127	//
128	// [varint: type]
129	// [varint: length]
130	// [length: value]
131	//
132	// A series of (possibly zero) records are concatenated into a stream, this
133	// example contains two records:
134	//
135	// (t: 0x01, l: 0x04, v: 0xff, 0xff, 0xff, 0xff)
136	// (t: 0x02, l: 0x01, v: 0x01)
137	//
138	// This method asserts that the byte stream is canonical, namely that each	4,411✔
139	// record is unique and that all records are sorted in ascending order. An	4,411✔
140	// ErrNotCanonicalStream error is returned if the encoded TLV stream is not.	4,411✔
141	//	4,411✔
142	// We permit an io.EOF error only when reading the type byte which signals that
143	// the last record was read cleanly and we should stop parsing. All other io.EOF
144	// or io.ErrUnexpectedEOF errors are returned.
145	func (s *Stream) Decode(r io.Reader) error {	2✔
146	_, err := s.decode(r, nil, false)	2✔
147	return err	2✔
148	}	2✔
149
150	// DecodeP2P is identical to Decode except that the maximum record size is
151	// capped at 65535.
152	func (s *Stream) DecodeP2P(r io.Reader) error {
153	_, err := s.decode(r, nil, true)	34,957✔
154	return err	34,957✔
155	}	34,957✔
156
157	// DecodeWithParsedTypes is identical to Decode, but if successful, returns a
158	// TypeMap containing the types of all records that were decoded or ignored from
159	// the stream.
160	func (s *Stream) DecodeWithParsedTypes(r io.Reader) (TypeMap, error) {	16,205✔
161	return s.decode(r, make(TypeMap), false)	16,205✔
162	}	16,205✔
163
164	// DecodeWithParsedTypesP2P is identical to DecodeWithParsedTypes except that
165	// the record size is capped at 65535. This should only be called from a p2p
166	// setting where untrusted input is being deserialized.
167	func (s *Stream) DecodeWithParsedTypesP2P(r io.Reader) (TypeMap, error) {
168	return s.decode(r, make(TypeMap), true)
169	}	55,575✔
170		55,575✔
171	// decode is a helper function that performs the basis of stream decoding. If	55,575✔
172	// the caller needs the set of parsed types, it must provide an initialized	55,575✔
173	// parsedTypes, otherwise the returned TypeMap will be nil. If the p2p bool is	55,575✔
174	// true, then the record size is capped at 65535. Otherwise, it is not capped.	55,575✔
175	func (s *Stream) decode(r io.Reader, parsedTypes TypeMap, p2p bool) (TypeMap,	55,575✔
176	error) {	55,575✔
177		55,575✔
178	var (	55,575✔
179	typ Type	55,575✔
180	min Type	237,991✔
181	recordIdx int	182,416✔
182	overflow bool	182,416✔
183	)	182,416✔
184
185	// Iterate through all possible type identifiers. As types are read from
186	// the io.Reader, min will skip forward to the last read type.
187	for {	54,430✔
188	// Read the next varint type.	54,430✔
189	t, err := ReadVarInt(r, &s.buf)
190	switch {
191		188✔
192	// We'll silence an EOF when zero bytes remain, meaning the	188✔
193	// stream was cleanly encoded.
194	case err == io.EOF:
195	return parsedTypes, nil	127,798✔
196		127,798✔
197	// Other unexpected errors.	127,798✔
198	case err != nil:	127,798✔
199	return nil, err	127,798✔
200	}	127,798✔
201		127,798✔
202	typ = Type(t)	128,026✔
203		228✔
204	// Assert that this type is greater than any previously read.	228✔
205	// If we've already overflowed and we parsed another type, the
206	// stream is not canonical. This check prevents us from accepts
207	// encodings that have duplicate records or from accepting an	127,570✔
208	// unsorted series.	127,570✔
209	if overflow \|\| typ < min {
210	return nil, ErrStreamNotCanonical
211	}
212		144✔
213	// Read the varint length.	144✔
214	length, err := ReadVarInt(r, &s.buf)
215	switch {
216		34✔
217	// We'll convert any EOFs to ErrUnexpectedEOF, since this	34✔
218	// results in an invalid record.
219	case err == io.EOF:
220	return nil, io.ErrUnexpectedEOF
221
222	// Other unexpected errors.
223	case err != nil:
224	return nil, err	127,461✔
225	}	69✔
226		69✔
227	// Place a soft limit on the size of a sane record, which
228	// prevents malicious encoders from causing us to allocate an
229	// unbounded amount of memory when decoding variable-sized
230	// fields. This is only checked when the p2p bool is true.
231	if p2p && length > MaxRecordSize {	127,323✔
232	return nil, ErrRecordTooLarge	127,323✔
233	}
234
235	// Search the records known to the stream for this type. We'll
236	// begin the search and recordIdx and walk forward until we find
237	// it or the next record's type is larger.
238	rec, newIdx, ok := s.getRecord(typ, recordIdx)	114,089✔
239	switch {	114,089✔
240		114,089✔
241	// We know of this record type, proceed to decode the value.
242	// This method asserts that length bytes are read in the
243	// process, and returns an error if the number of bytes is not
244	// exactly length.	92✔
245	case ok:	92✔
246	err := rec.decoder(r, rec.value, &s.buf, length)
247	switch {
248		122✔
249	// We'll convert any EOFs to ErrUnexpectedEOF, since this	122✔
250	// results in an invalid record.
251	case err == io.EOF:
252	return nil, io.ErrUnexpectedEOF
253
254	// Other unexpected errors.	184,602✔
255	case err != nil:	70,727✔
256	return nil, err	70,727✔
257	}
258
259	// Record the successfully decoded type if the caller
260	// provided an initialized TypeMap.	13,234✔
261	if parsedTypes != nil {	13,234✔
262	parsedTypes[typ] = nil	13,234✔
263	}	13,234✔
264		13,234✔
265	// Otherwise, the record type is unknown and is odd, discard the	26,328✔
266	// number of bytes specified by length.	13,094✔
267	default:	13,094✔
268	// If the caller provided an initialized TypeMap, record	13,094✔
269	// the encoded bytes.
270	var b *bytes.Buffer	13,234✔
271	writer := io.Discard	13,234✔
272	if parsedTypes != nil {
273	b = bytes.NewBuffer(make([]byte, 0, length))
274	writer = b
275	}	268✔
276		268✔
277	_, err := io.CopyN(writer, r, int64(length))
278	switch {
UNCOV 279		×
UNCOV 280	// We'll convert any EOFs to ErrUnexpectedEOF, since this	×
281	// results in an invalid record.
282	case err == io.EOF:
283	return nil, io.ErrUnexpectedEOF	25,796✔
284		12,830✔
285	// Other unexpected errors.	12,830✔
286	case err != nil:
287	return nil, err
288	}
289
290	if parsedTypes != nil {	126,841✔
291	parsedTypes[typ] = b.Bytes()	126,841✔
292	}	126,841✔
293	}	126,841✔
294		126,841✔
295	// Update our record index so that we can begin our next search	126,867✔
296	// from where we left off.	26✔
297	recordIdx = newIdx	26✔
298
299	// If we've parsed the largest possible type, the next loop will
300	// overflow back to zero. However, we need to attempt parsing	126,841✔
301	// the next type to ensure that the stream is empty.
302	if typ == math.MaxUint64 {
303	overflow = true
304	}
305
306	// Finally, set our lower bound on the next accepted type.
307	min = typ + 1
308	}	127,323✔
309	}	251,922✔
310		124,599✔
311	// getRecord searches for a record matching typ known to the stream. The boolean	124,599✔
312	// return value indicates whether the record is known to the stream. The integer
313	// return value carries the index from where getRecord should be invoked on the
314	// subsequent call. The first call to getRecord should always use an idx of 0.
315	func (s *Stream) getRecord(typ Type, idx int) (Record, int, bool) {	114,089✔
316	for idx < len(s.records) {	114,089✔
317	record := s.records[idx]
318	switch {
319
320	// Found target record, return it to the caller. The next index
321	// returned points to the immediately following record.	9,945✔
322	case record.typ == typ:	9,945✔
323	return record, idx + 1, true	9,945✔
324
325	// This record's type is lower than the target. Advance our
326	// index and continue to the next record which will have a
327	// strictly higher type.
328	case record.typ < typ:
329	idx++	565✔
330	continue	565✔
331
332	// This record's type is larger than the target, hence we have
333	// no record matching the current type. Return the current index
334	// so that we can start our search from here when processing the
335	// next tlv record.	12,669✔
336	default:
337	return Record{}, idx, false
338	}
339	}
340
341	// All known records are exhausted.
342	return Record{}, idx, false
343	}

lightningnetwork / lnd / 14879619421

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous