• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

lightningnetwork / lnd / 13396663489

18 Feb 2025 05:34PM UTC coverage: 58.636% (-0.2%) from 58.804%
13396663489

push

github

web-flow
Merge pull request #9525 from ellemouton/graph10

graph: Restrict interface to update channel proof instead of entire channel

9 of 10 new or added lines in 2 files covered. (90.0%)

446 existing lines in 31 files now uncovered.

135736 of 231488 relevant lines covered (58.64%)

19383.26 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.4
/sweep/fee_bumper.go
1
package sweep
2

3
import (
4
        "errors"
5
        "fmt"
6
        "sync"
7
        "sync/atomic"
8

9
        "github.com/btcsuite/btcd/btcutil"
10
        "github.com/btcsuite/btcd/chaincfg/chainhash"
11
        "github.com/btcsuite/btcd/rpcclient"
12
        "github.com/btcsuite/btcd/txscript"
13
        "github.com/btcsuite/btcd/wire"
14
        "github.com/btcsuite/btcwallet/chain"
15
        "github.com/lightningnetwork/lnd/chainio"
16
        "github.com/lightningnetwork/lnd/chainntnfs"
17
        "github.com/lightningnetwork/lnd/fn/v2"
18
        "github.com/lightningnetwork/lnd/input"
19
        "github.com/lightningnetwork/lnd/labels"
20
        "github.com/lightningnetwork/lnd/lntypes"
21
        "github.com/lightningnetwork/lnd/lnutils"
22
        "github.com/lightningnetwork/lnd/lnwallet"
23
        "github.com/lightningnetwork/lnd/lnwallet/chainfee"
24
        "github.com/lightningnetwork/lnd/tlv"
25
)
26

27
var (
28
        // ErrInvalidBumpResult is returned when the bump result is invalid.
29
        ErrInvalidBumpResult = errors.New("invalid bump result")
30

31
        // ErrNotEnoughBudget is returned when the fee bumper decides the
32
        // current budget cannot cover the fee.
33
        ErrNotEnoughBudget = errors.New("not enough budget")
34

35
        // ErrLocktimeImmature is returned when sweeping an input whose
36
        // locktime is not reached.
37
        ErrLocktimeImmature = errors.New("immature input")
38

39
        // ErrTxNoOutput is returned when an output cannot be created during tx
40
        // preparation, usually due to the output being dust.
41
        ErrTxNoOutput = errors.New("tx has no output")
42

43
        // ErrThirdPartySpent is returned when a third party has spent the
44
        // input in the sweeping tx.
45
        ErrThirdPartySpent = errors.New("third party spent the output")
46
)
47

48
var (
49
        // dummyChangePkScript is a dummy tapscript change script that's used
50
        // when we don't need a real address, just something that can be used
51
        // for fee estimation.
52
        dummyChangePkScript = []byte{
53
                0x51, 0x20,
54
                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
55
                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
56
                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
57
                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
58
        }
59
)
60

61
// Bumper defines an interface that can be used by other subsystems for fee
62
// bumping.
63
type Bumper interface {
64
        // Broadcast is used to publish the tx created from the given inputs
65
        // specified in the request. It handles the tx creation, broadcasts it,
66
        // and monitors its confirmation status for potential fee bumping. It
67
        // returns a chan that the caller can use to receive updates about the
68
        // broadcast result and potential RBF attempts.
69
        Broadcast(req *BumpRequest) <-chan *BumpResult
70
}
71

72
// BumpEvent represents the event of a fee bumping attempt.
73
type BumpEvent uint8
74

75
const (
76
        // TxPublished is sent when the broadcast attempt is finished.
77
        TxPublished BumpEvent = iota
78

79
        // TxFailed is sent when the tx has encountered a fee-related error
80
        // during its creation or broadcast, or an internal error from the fee
81
        // bumper. In either case the inputs in this tx should be retried with
82
        // either a different grouping strategy or an increased budget.
83
        //
84
        // NOTE: We also send this event when there's a third party spend
85
        // event, and the sweeper will handle cleaning this up once it's
86
        // confirmed.
87
        //
88
        // TODO(yy): Remove the above usage once we remove sweeping non-CPFP
89
        // anchors.
90
        TxFailed
91

92
        // TxReplaced is sent when the original tx is replaced by a new one.
93
        TxReplaced
94

95
        // TxConfirmed is sent when the tx is confirmed.
96
        TxConfirmed
97

98
        // TxFatal is sent when the inputs in this tx cannot be retried. Txns
99
        // will end up in this state if they have encountered a non-fee related
100
        // error, which means they cannot be retried with increased budget.
101
        TxFatal
102

103
        // sentinalEvent is used to check if an event is unknown.
104
        sentinalEvent
105
)
106

107
// String returns a human-readable string for the event.
108
func (e BumpEvent) String() string {
2✔
109
        switch e {
2✔
110
        case TxPublished:
2✔
111
                return "Published"
2✔
112
        case TxFailed:
2✔
113
                return "Failed"
2✔
114
        case TxReplaced:
2✔
115
                return "Replaced"
2✔
116
        case TxConfirmed:
2✔
117
                return "Confirmed"
2✔
118
        case TxFatal:
2✔
119
                return "Fatal"
2✔
120
        default:
×
121
                return "Unknown"
×
122
        }
123
}
124

125
// Unknown returns true if the event is unknown.
126
func (e BumpEvent) Unknown() bool {
13✔
127
        return e >= sentinalEvent
13✔
128
}
13✔
129

130
// BumpRequest is used by the caller to give the Bumper the necessary info to
131
// create and manage potential fee bumps for a set of inputs.
132
type BumpRequest struct {
133
        // Budget givens the total amount that can be used as fees by these
134
        // inputs.
135
        Budget btcutil.Amount
136

137
        // Inputs is the set of inputs to sweep.
138
        Inputs []input.Input
139

140
        // DeadlineHeight is the block height at which the tx should be
141
        // confirmed.
142
        DeadlineHeight int32
143

144
        // DeliveryAddress is the script to send the change output to.
145
        DeliveryAddress lnwallet.AddrWithKey
146

147
        // MaxFeeRate is the maximum fee rate that can be used for fee bumping.
148
        MaxFeeRate chainfee.SatPerKWeight
149

150
        // StartingFeeRate is an optional parameter that can be used to specify
151
        // the initial fee rate to use for the fee function.
152
        StartingFeeRate fn.Option[chainfee.SatPerKWeight]
153

154
        // ExtraTxOut tracks if this bump request has an optional set of extra
155
        // outputs to add to the transaction.
156
        ExtraTxOut fn.Option[SweepOutput]
157

158
        // Immediate is used to specify that the tx should be broadcast
159
        // immediately.
160
        Immediate bool
161
}
162

163
// MaxFeeRateAllowed returns the maximum fee rate allowed for the given
164
// request. It calculates the feerate using the supplied budget and the weight,
165
// compares it with the specified MaxFeeRate, and returns the smaller of the
166
// two.
167
func (r *BumpRequest) MaxFeeRateAllowed() (chainfee.SatPerKWeight, error) {
11✔
168
        // We'll want to know if we have any blobs, as we need to factor this
11✔
169
        // into the max fee rate for this bump request.
11✔
170
        hasBlobs := fn.Any(r.Inputs, func(i input.Input) bool {
21✔
171
                return fn.MapOptionZ(
10✔
172
                        i.ResolutionBlob(), func(b tlv.Blob) bool {
12✔
173
                                return len(b) > 0
2✔
174
                        },
2✔
175
                )
176
        })
177

178
        sweepAddrs := [][]byte{
11✔
179
                r.DeliveryAddress.DeliveryAddress,
11✔
180
        }
11✔
181

11✔
182
        // If we have blobs, then we'll add an extra sweep addr for the size
11✔
183
        // estimate below. We know that these blobs will also always be based on
11✔
184
        // p2tr addrs.
11✔
185
        if hasBlobs {
11✔
186
                // We need to pass in a real address, so we'll use a dummy
×
187
                // tapscript change script that's used elsewhere for tests.
×
188
                sweepAddrs = append(sweepAddrs, dummyChangePkScript)
×
189
        }
×
190

191
        // Get the size of the sweep tx, which will be used to calculate the
192
        // budget fee rate.
193
        size, err := calcSweepTxWeight(
11✔
194
                r.Inputs, sweepAddrs,
11✔
195
        )
11✔
196
        if err != nil {
12✔
197
                return 0, err
1✔
198
        }
1✔
199

200
        // Use the budget and MaxFeeRate to decide the max allowed fee rate.
201
        // This is needed as, when the input has a large value and the user
202
        // sets the budget to be proportional to the input value, the fee rate
203
        // can be very high and we need to make sure it doesn't exceed the max
204
        // fee rate.
205
        maxFeeRateAllowed := chainfee.NewSatPerKWeight(r.Budget, size)
10✔
206
        if maxFeeRateAllowed > r.MaxFeeRate {
13✔
207
                log.Debugf("Budget feerate %v exceeds MaxFeeRate %v, use "+
3✔
208
                        "MaxFeeRate instead, txWeight=%v", maxFeeRateAllowed,
3✔
209
                        r.MaxFeeRate, size)
3✔
210

3✔
211
                return r.MaxFeeRate, nil
3✔
212
        }
3✔
213

214
        log.Debugf("Budget feerate %v below MaxFeeRate %v, use budget feerate "+
9✔
215
                "instead, txWeight=%v", maxFeeRateAllowed, r.MaxFeeRate, size)
9✔
216

9✔
217
        return maxFeeRateAllowed, nil
9✔
218
}
219

220
// calcSweepTxWeight calculates the weight of the sweep tx. It assumes a
221
// sweeping tx always has a single output(change).
222
func calcSweepTxWeight(inputs []input.Input,
223
        outputPkScript [][]byte) (lntypes.WeightUnit, error) {
14✔
224

14✔
225
        // Use a const fee rate as we only use the weight estimator to
14✔
226
        // calculate the size.
14✔
227
        const feeRate = 1
14✔
228

14✔
229
        // Initialize the tx weight estimator with,
14✔
230
        // - nil outputs as we only have one single change output.
14✔
231
        // - const fee rate as we don't care about the fees here.
14✔
232
        // - 0 maxfeerate as we don't care about fees here.
14✔
233
        //
14✔
234
        // TODO(yy): we should refactor the weight estimator to not require a
14✔
235
        // fee rate and max fee rate and make it a pure tx weight calculator.
14✔
236
        _, estimator, err := getWeightEstimate(
14✔
237
                inputs, nil, feeRate, 0, outputPkScript,
14✔
238
        )
14✔
239
        if err != nil {
16✔
240
                return 0, err
2✔
241
        }
2✔
242

243
        return estimator.weight(), nil
12✔
244
}
245

246
// BumpResult is used by the Bumper to send updates about the tx being
247
// broadcast.
248
type BumpResult struct {
249
        // Event is the type of event that the result is for.
250
        Event BumpEvent
251

252
        // Tx is the tx being broadcast.
253
        Tx *wire.MsgTx
254

255
        // ReplacedTx is the old, replaced tx if a fee bump is attempted.
256
        ReplacedTx *wire.MsgTx
257

258
        // FeeRate is the fee rate used for the new tx.
259
        FeeRate chainfee.SatPerKWeight
260

261
        // Fee is the fee paid by the new tx.
262
        Fee btcutil.Amount
263

264
        // Err is the error that occurred during the broadcast.
265
        Err error
266

267
        // requestID is the ID of the request that created this record.
268
        requestID uint64
269
}
270

271
// String returns a human-readable string for the result.
272
func (b *BumpResult) String() string {
2✔
273
        desc := fmt.Sprintf("Event=%v", b.Event)
2✔
274
        if b.Tx != nil {
4✔
275
                desc += fmt.Sprintf(", Tx=%v", b.Tx.TxHash())
2✔
276
        }
2✔
277

278
        return fmt.Sprintf("[%s]", desc)
2✔
279
}
280

281
// Validate validates the BumpResult so it's safe to use.
282
func (b *BumpResult) Validate() error {
14✔
283
        isFailureEvent := b.Event == TxFailed || b.Event == TxFatal
14✔
284

14✔
285
        // Every result must have a tx except the fatal or failed case.
14✔
286
        if b.Tx == nil && !isFailureEvent {
15✔
287
                return fmt.Errorf("%w: nil tx", ErrInvalidBumpResult)
1✔
288
        }
1✔
289

290
        // Every result must have a known event.
291
        if b.Event.Unknown() {
14✔
292
                return fmt.Errorf("%w: unknown event", ErrInvalidBumpResult)
1✔
293
        }
1✔
294

295
        // If it's a replacing event, it must have a replaced tx.
296
        if b.Event == TxReplaced && b.ReplacedTx == nil {
13✔
297
                return fmt.Errorf("%w: nil replacing tx", ErrInvalidBumpResult)
1✔
298
        }
1✔
299

300
        // If it's a failed or fatal event, it must have an error.
301
        if isFailureEvent && b.Err == nil {
13✔
302
                return fmt.Errorf("%w: nil error", ErrInvalidBumpResult)
2✔
303
        }
2✔
304

305
        // If it's a confirmed event, it must have a fee rate and fee.
306
        if b.Event == TxConfirmed && (b.FeeRate == 0 || b.Fee == 0) {
10✔
307
                return fmt.Errorf("%w: missing fee rate or fee",
1✔
308
                        ErrInvalidBumpResult)
1✔
309
        }
1✔
310

311
        return nil
8✔
312
}
313

314
// TxPublisherConfig is the config used to create a new TxPublisher.
315
type TxPublisherConfig struct {
316
        // Signer is used to create the tx signature.
317
        Signer input.Signer
318

319
        // Wallet is used primarily to publish the tx.
320
        Wallet Wallet
321

322
        // Estimator is used to estimate the fee rate for the new tx based on
323
        // its deadline conf target.
324
        Estimator chainfee.Estimator
325

326
        // Notifier is used to monitor the confirmation status of the tx.
327
        Notifier chainntnfs.ChainNotifier
328

329
        // AuxSweeper is an optional interface that can be used to modify the
330
        // way sweep transaction are generated.
331
        AuxSweeper fn.Option[AuxSweeper]
332
}
333

334
// TxPublisher is an implementation of the Bumper interface. It utilizes the
335
// `testmempoolaccept` RPC to bump the fee of txns it created based on
336
// different fee function selected or configed by the caller. Its purpose is to
337
// take a list of inputs specified, and create a tx that spends them to a
338
// specified output. It will then monitor the confirmation status of the tx,
339
// and if it's not confirmed within a certain time frame, it will attempt to
340
// bump the fee of the tx by creating a new tx that spends the same inputs to
341
// the same output, but with a higher fee rate. It will continue to do this
342
// until the tx is confirmed or the fee rate reaches the maximum fee rate
343
// specified by the caller.
344
type TxPublisher struct {
345
        started atomic.Bool
346
        stopped atomic.Bool
347

348
        // Embed the blockbeat consumer struct to get access to the method
349
        // `NotifyBlockProcessed` and the `BlockbeatChan`.
350
        chainio.BeatConsumer
351

352
        wg sync.WaitGroup
353

354
        // cfg specifies the configuration of the TxPublisher.
355
        cfg *TxPublisherConfig
356

357
        // currentHeight is the current block height.
358
        currentHeight atomic.Int32
359

360
        // records is a map keyed by the requestCounter and the value is the tx
361
        // being monitored.
362
        records lnutils.SyncMap[uint64, *monitorRecord]
363

364
        // requestCounter is a monotonically increasing counter used to keep
365
        // track of how many requests have been made.
366
        requestCounter atomic.Uint64
367

368
        // subscriberChans is a map keyed by the requestCounter, each item is
369
        // the chan that the publisher sends the fee bump result to.
370
        subscriberChans lnutils.SyncMap[uint64, chan *BumpResult]
371

372
        // quit is used to signal the publisher to stop.
373
        quit chan struct{}
374
}
375

376
// Compile-time constraint to ensure TxPublisher implements Bumper.
377
var _ Bumper = (*TxPublisher)(nil)
378

379
// Compile-time check for the chainio.Consumer interface.
380
var _ chainio.Consumer = (*TxPublisher)(nil)
381

382
// NewTxPublisher creates a new TxPublisher.
383
func NewTxPublisher(cfg TxPublisherConfig) *TxPublisher {
18✔
384
        tp := &TxPublisher{
18✔
385
                cfg:             &cfg,
18✔
386
                records:         lnutils.SyncMap[uint64, *monitorRecord]{},
18✔
387
                subscriberChans: lnutils.SyncMap[uint64, chan *BumpResult]{},
18✔
388
                quit:            make(chan struct{}),
18✔
389
        }
18✔
390

18✔
391
        // Mount the block consumer.
18✔
392
        tp.BeatConsumer = chainio.NewBeatConsumer(tp.quit, tp.Name())
18✔
393

18✔
394
        return tp
18✔
395
}
18✔
396

397
// isNeutrinoBackend checks if the wallet backend is neutrino.
398
func (t *TxPublisher) isNeutrinoBackend() bool {
3✔
399
        return t.cfg.Wallet.BackEnd() == "neutrino"
3✔
400
}
3✔
401

402
// Broadcast is used to publish the tx created from the given inputs. It will
403
// register the broadcast request and return a chan to the caller to subscribe
404
// the broadcast result. The initial broadcast is guaranteed to be
405
// RBF-compliant unless the budget specified cannot cover the fee.
406
//
407
// NOTE: part of the Bumper interface.
408
func (t *TxPublisher) Broadcast(req *BumpRequest) <-chan *BumpResult {
7✔
409
        log.Tracef("Received broadcast request: %s",
7✔
410
                lnutils.SpewLogClosure(req))
7✔
411

7✔
412
        // Store the request.
7✔
413
        record := t.storeInitialRecord(req)
7✔
414

7✔
415
        // Create a chan to send the result to the caller.
7✔
416
        subscriber := make(chan *BumpResult, 1)
7✔
417
        t.subscriberChans.Store(record.requestID, subscriber)
7✔
418

7✔
419
        // Publish the tx immediately if specified.
7✔
420
        if req.Immediate {
10✔
421
                t.handleInitialBroadcast(record)
3✔
422
        }
3✔
423

424
        return subscriber
7✔
425
}
426

427
// storeInitialRecord initializes a monitor record and saves it in the map.
428
func (t *TxPublisher) storeInitialRecord(req *BumpRequest) *monitorRecord {
7✔
429
        // Increase the request counter.
7✔
430
        //
7✔
431
        // NOTE: this is the only place where we increase the counter.
7✔
432
        requestID := t.requestCounter.Add(1)
7✔
433

7✔
434
        // Register the record.
7✔
435
        record := &monitorRecord{
7✔
436
                requestID: requestID,
7✔
437
                req:       req,
7✔
438
        }
7✔
439
        t.records.Store(requestID, record)
7✔
440

7✔
441
        return record
7✔
442
}
7✔
443

444
// updateRecord updates the given record's tx and fee, and saves it in the
445
// records map.
446
func (t *TxPublisher) updateRecord(r *monitorRecord,
447
        sweepCtx *sweepTxCtx) *monitorRecord {
20✔
448

20✔
449
        r.tx = sweepCtx.tx
20✔
450
        r.fee = sweepCtx.fee
20✔
451
        r.outpointToTxIndex = sweepCtx.outpointToTxIndex
20✔
452

20✔
453
        // Register the record.
20✔
454
        t.records.Store(r.requestID, r)
20✔
455

20✔
456
        return r
20✔
457
}
20✔
458

459
// NOTE: part of the `chainio.Consumer` interface.
460
func (t *TxPublisher) Name() string {
18✔
461
        return "TxPublisher"
18✔
462
}
18✔
463

464
// initializeTx initializes a fee function and creates an RBF-compliant tx. If
465
// succeeded, the initial tx is stored in the records map.
466
func (t *TxPublisher) initializeTx(r *monitorRecord) (*monitorRecord, error) {
6✔
467
        // Create a fee bumping algorithm to be used for future RBF.
6✔
468
        feeAlgo, err := t.initializeFeeFunction(r.req)
6✔
469
        if err != nil {
9✔
470
                return nil, fmt.Errorf("init fee function: %w", err)
3✔
471
        }
3✔
472

473
        // Attach the newly created fee function.
474
        //
475
        // TODO(yy): current we'd initialize a monitorRecord before creating the
476
        // fee function, while we could instead create the fee function first
477
        // then save it to the record. To make this happen we need to change the
478
        // conf target calculation below since we would be initializing the fee
479
        // function one block before.
480
        r.feeFunction = feeAlgo
5✔
481

5✔
482
        // Create the initial tx to be broadcasted. This tx is guaranteed to
5✔
483
        // comply with the RBF restrictions.
5✔
484
        record, err := t.createRBFCompliantTx(r)
5✔
485
        if err != nil {
8✔
486
                return nil, fmt.Errorf("create RBF-compliant tx: %w", err)
3✔
487
        }
3✔
488

489
        return record, nil
4✔
490
}
491

492
// initializeFeeFunction initializes a fee function to be used for this request
493
// for future fee bumping.
494
func (t *TxPublisher) initializeFeeFunction(
495
        req *BumpRequest) (FeeFunction, error) {
8✔
496

8✔
497
        // Get the max allowed feerate.
8✔
498
        maxFeeRateAllowed, err := req.MaxFeeRateAllowed()
8✔
499
        if err != nil {
8✔
500
                return nil, err
×
501
        }
×
502

503
        // Get the initial conf target.
504
        confTarget := calcCurrentConfTarget(
8✔
505
                t.currentHeight.Load(), req.DeadlineHeight,
8✔
506
        )
8✔
507

8✔
508
        log.Debugf("Initializing fee function with conf target=%v, budget=%v, "+
8✔
509
                "maxFeeRateAllowed=%v", confTarget, req.Budget,
8✔
510
                maxFeeRateAllowed)
8✔
511

8✔
512
        // Initialize the fee function and return it.
8✔
513
        //
8✔
514
        // TODO(yy): return based on differet req.Strategy?
8✔
515
        return NewLinearFeeFunction(
8✔
516
                maxFeeRateAllowed, confTarget, t.cfg.Estimator,
8✔
517
                req.StartingFeeRate,
8✔
518
        )
8✔
519
}
520

521
// createRBFCompliantTx creates a tx that is compliant with RBF rules. It does
522
// so by creating a tx, validate it using `TestMempoolAccept`, and bump its fee
523
// and redo the process until the tx is valid, or return an error when non-RBF
524
// related errors occur or the budget has been used up.
525
func (t *TxPublisher) createRBFCompliantTx(
526
        r *monitorRecord) (*monitorRecord, error) {
11✔
527

11✔
528
        f := r.feeFunction
11✔
529

11✔
530
        for {
25✔
531
                // Create a new tx with the given fee rate and check its
14✔
532
                // mempool acceptance.
14✔
533
                sweepCtx, err := t.createAndCheckTx(r.req, f)
14✔
534

14✔
535
                switch {
14✔
536
                case err == nil:
8✔
537
                        // The tx is valid, store it.
8✔
538
                        record := t.updateRecord(r, sweepCtx)
8✔
539

8✔
540
                        log.Infof("Created initial sweep tx=%v for %v inputs: "+
8✔
541
                                "feerate=%v, fee=%v, inputs:\n%v",
8✔
542
                                sweepCtx.tx.TxHash(), len(r.req.Inputs),
8✔
543
                                f.FeeRate(), sweepCtx.fee,
8✔
544
                                inputTypeSummary(r.req.Inputs))
8✔
545

8✔
546
                        return record, nil
8✔
547

548
                // If the error indicates the fees paid is not enough, we will
549
                // ask the fee function to increase the fee rate and retry.
550
                case errors.Is(err, lnwallet.ErrMempoolFee):
2✔
551
                        // We should at least start with a feerate above the
2✔
552
                        // mempool min feerate, so if we get this error, it
2✔
553
                        // means something is wrong earlier in the pipeline.
2✔
554
                        log.Errorf("Current fee=%v, feerate=%v, %v",
2✔
555
                                sweepCtx.fee, f.FeeRate(), err)
2✔
556

2✔
557
                        fallthrough
2✔
558

559
                // We are not paying enough fees so we increase it.
560
                case errors.Is(err, chain.ErrInsufficientFee):
6✔
561
                        increased := false
6✔
562

6✔
563
                        // Keep calling the fee function until the fee rate is
6✔
564
                        // increased or maxed out.
6✔
565
                        for !increased {
13✔
566
                                log.Debugf("Increasing fee for next round, "+
7✔
567
                                        "current fee=%v, feerate=%v",
7✔
568
                                        sweepCtx.fee, f.FeeRate())
7✔
569

7✔
570
                                // If the fee function tells us that we have
7✔
571
                                // used up the budget, we will return an error
7✔
572
                                // indicating this tx cannot be made. The
7✔
573
                                // sweeper should handle this error and try to
7✔
574
                                // cluster these inputs differetly.
7✔
575
                                increased, err = f.Increment()
7✔
576
                                if err != nil {
10✔
577
                                        return nil, err
3✔
578
                                }
3✔
579
                        }
580

581
                // TODO(yy): suppose there's only one bad input, we can do a
582
                // binary search to find out which input is causing this error
583
                // by recreating a tx using half of the inputs and check its
584
                // mempool acceptance.
585
                default:
4✔
586
                        log.Debugf("Failed to create RBF-compliant tx: %v", err)
4✔
587
                        return nil, err
4✔
588
                }
589
        }
590
}
591

592
// createAndCheckTx creates a tx based on the given inputs, change output
593
// script, and the fee rate. In addition, it validates the tx's mempool
594
// acceptance before returning a tx that can be published directly, along with
595
// its fee.
596
func (t *TxPublisher) createAndCheckTx(req *BumpRequest,
597
        f FeeFunction) (*sweepTxCtx, error) {
24✔
598

24✔
599
        // Create the sweep tx with max fee rate of 0 as the fee function
24✔
600
        // guarantees the fee rate used here won't exceed the max fee rate.
24✔
601
        sweepCtx, err := t.createSweepTx(
24✔
602
                req.Inputs, req.DeliveryAddress, f.FeeRate(),
24✔
603
        )
24✔
604
        if err != nil {
26✔
605
                return sweepCtx, fmt.Errorf("create sweep tx: %w", err)
2✔
606
        }
2✔
607

608
        // Sanity check the budget still covers the fee.
609
        if sweepCtx.fee > req.Budget {
26✔
610
                return sweepCtx, fmt.Errorf("%w: budget=%v, fee=%v",
2✔
611
                        ErrNotEnoughBudget, req.Budget, sweepCtx.fee)
2✔
612
        }
2✔
613

614
        // If we had an extra txOut, then we'll update the result to include
615
        // it.
616
        req.ExtraTxOut = sweepCtx.extraTxOut
22✔
617

22✔
618
        // Validate the tx's mempool acceptance.
22✔
619
        err = t.cfg.Wallet.CheckMempoolAcceptance(sweepCtx.tx)
22✔
620

22✔
621
        // Exit early if the tx is valid.
22✔
622
        if err == nil {
35✔
623
                return sweepCtx, nil
13✔
624
        }
13✔
625

626
        // Print an error log if the chain backend doesn't support the mempool
627
        // acceptance test RPC.
628
        if errors.Is(err, rpcclient.ErrBackendVersion) {
11✔
629
                log.Errorf("TestMempoolAccept not supported by backend, " +
×
630
                        "consider upgrading it to a newer version")
×
631
                return sweepCtx, nil
×
632
        }
×
633

634
        // We are running on a backend that doesn't implement the RPC
635
        // testmempoolaccept, eg, neutrino, so we'll skip the check.
636
        if errors.Is(err, chain.ErrUnimplemented) {
11✔
UNCOV
637
                log.Debug("Skipped testmempoolaccept due to not implemented")
×
UNCOV
638
                return sweepCtx, nil
×
UNCOV
639
        }
×
640

641
        return sweepCtx, fmt.Errorf("tx=%v failed mempool check: %w",
11✔
642
                sweepCtx.tx.TxHash(), err)
11✔
643
}
644

645
// broadcast takes a monitored tx and publishes it to the network. Prior to the
646
// broadcast, it will subscribe the tx's confirmation notification and attach
647
// the event channel to the record. Any broadcast-related errors will not be
648
// returned here, instead, they will be put inside the `BumpResult` and
649
// returned to the caller.
650
func (t *TxPublisher) broadcast(record *monitorRecord) (*BumpResult, error) {
10✔
651
        txid := record.tx.TxHash()
10✔
652

10✔
653
        tx := record.tx
10✔
654
        log.Debugf("Publishing sweep tx %v, num_inputs=%v, height=%v",
10✔
655
                txid, len(tx.TxIn), t.currentHeight.Load())
10✔
656

10✔
657
        // Before we go to broadcast, we'll notify the aux sweeper, if it's
10✔
658
        // present of this new broadcast attempt.
10✔
659
        err := fn.MapOptionZ(t.cfg.AuxSweeper, func(aux AuxSweeper) error {
18✔
660
                return aux.NotifyBroadcast(
8✔
661
                        record.req, tx, record.fee, record.outpointToTxIndex,
8✔
662
                )
8✔
663
        })
8✔
664
        if err != nil {
10✔
665
                return nil, fmt.Errorf("unable to notify aux sweeper: %w", err)
×
666
        }
×
667

668
        // Set the event, and change it to TxFailed if the wallet fails to
669
        // publish it.
670
        event := TxPublished
10✔
671

10✔
672
        // Publish the sweeping tx with customized label. If the publish fails,
10✔
673
        // this error will be saved in the `BumpResult` and it will be removed
10✔
674
        // from being monitored.
10✔
675
        err = t.cfg.Wallet.PublishTransaction(
10✔
676
                tx, labels.MakeLabel(labels.LabelTypeSweepTransaction, nil),
10✔
677
        )
10✔
678
        if err != nil {
13✔
679
                // NOTE: we decide to attach this error to the result instead
3✔
680
                // of returning it here because by the time the tx reaches
3✔
681
                // here, it should have passed the mempool acceptance check. If
3✔
682
                // it still fails to be broadcast, it's likely a non-RBF
3✔
683
                // related error happened. So we send this error back to the
3✔
684
                // caller so that it can handle it properly.
3✔
685
                //
3✔
686
                // TODO(yy): find out which input is causing the failure.
3✔
687
                log.Errorf("Failed to publish tx %v: %v", txid, err)
3✔
688
                event = TxFailed
3✔
689
        }
3✔
690

691
        result := &BumpResult{
10✔
692
                Event:     event,
10✔
693
                Tx:        record.tx,
10✔
694
                Fee:       record.fee,
10✔
695
                FeeRate:   record.feeFunction.FeeRate(),
10✔
696
                Err:       err,
10✔
697
                requestID: record.requestID,
10✔
698
        }
10✔
699

10✔
700
        return result, nil
10✔
701
}
702

703
// notifyResult sends the result to the resultChan specified by the requestID.
704
// This channel is expected to be read by the caller.
705
func (t *TxPublisher) notifyResult(result *BumpResult) {
13✔
706
        id := result.requestID
13✔
707
        subscriber, ok := t.subscriberChans.Load(id)
13✔
708
        if !ok {
13✔
UNCOV
709
                log.Errorf("Result chan for id=%v not found", id)
×
UNCOV
710
                return
×
UNCOV
711
        }
×
712

713
        log.Debugf("Sending result %v for requestID=%v", result, id)
13✔
714

13✔
715
        select {
13✔
716
        // Send the result to the subscriber.
717
        //
718
        // TODO(yy): Add timeout in case it's blocking?
719
        case subscriber <- result:
11✔
720
        case <-t.quit:
2✔
721
                log.Debug("Fee bumper stopped")
2✔
722
        }
723
}
724

725
// removeResult removes the tracking of the result if the result contains a
726
// non-nil error, or the tx is confirmed, the record will be removed from the
727
// maps.
728
func (t *TxPublisher) removeResult(result *BumpResult) {
13✔
729
        id := result.requestID
13✔
730

13✔
731
        var txid chainhash.Hash
13✔
732
        if result.Tx != nil {
24✔
733
                txid = result.Tx.TxHash()
11✔
734
        }
11✔
735

736
        // Remove the record from the maps if there's an error or the tx is
737
        // confirmed. When there's an error, it means this tx has failed its
738
        // broadcast and cannot be retried. There are two cases it may fail,
739
        // - when the budget cannot cover the increased fee calculated by the
740
        //   fee function, hence the budget is used up.
741
        // - when a non-fee related error returned from PublishTransaction.
742
        switch result.Event {
13✔
743
        case TxFailed:
4✔
744
                log.Errorf("Removing monitor record=%v, tx=%v, due to err: %v",
4✔
745
                        id, txid, result.Err)
4✔
746

747
        case TxConfirmed:
5✔
748
                // Remove the record if the tx is confirmed.
5✔
749
                log.Debugf("Removing confirmed monitor record=%v, tx=%v", id,
5✔
750
                        txid)
5✔
751

752
        case TxFatal:
4✔
753
                // Remove the record if there's an error.
4✔
754
                log.Debugf("Removing monitor record=%v due to fatal err: %v",
4✔
755
                        id, result.Err)
4✔
756

757
        // Do nothing if it's neither failed or confirmed.
758
        default:
6✔
759
                log.Tracef("Skipping record removal for id=%v, event=%v", id,
6✔
760
                        result.Event)
6✔
761

6✔
762
                return
6✔
763
        }
764

765
        t.records.Delete(id)
9✔
766
        t.subscriberChans.Delete(id)
9✔
767
}
768

769
// handleResult handles the result of a tx broadcast. It will notify the
770
// subscriber and remove the record if the tx is confirmed or failed to be
771
// broadcast.
772
func (t *TxPublisher) handleResult(result *BumpResult) {
10✔
773
        // Notify the subscriber.
10✔
774
        t.notifyResult(result)
10✔
775

10✔
776
        // Remove the record if it's failed or confirmed.
10✔
777
        t.removeResult(result)
10✔
778
}
10✔
779

780
// monitorRecord is used to keep track of the tx being monitored by the
781
// publisher internally.
782
type monitorRecord struct {
783
        // requestID is the ID of the request that created this record.
784
        requestID uint64
785

786
        // tx is the tx being monitored.
787
        tx *wire.MsgTx
788

789
        // req is the original request.
790
        req *BumpRequest
791

792
        // feeFunction is the fee bumping algorithm used by the publisher.
793
        feeFunction FeeFunction
794

795
        // fee is the fee paid by the tx.
796
        fee btcutil.Amount
797

798
        // outpointToTxIndex is a map of outpoint to tx index.
799
        outpointToTxIndex map[wire.OutPoint]int
800
}
801

802
// Start starts the publisher by subscribing to block epoch updates and kicking
803
// off the monitor loop.
804
func (t *TxPublisher) Start(beat chainio.Blockbeat) error {
2✔
805
        log.Info("TxPublisher starting...")
2✔
806

2✔
807
        if t.started.Swap(true) {
2✔
808
                return fmt.Errorf("TxPublisher started more than once")
×
809
        }
×
810

811
        // Set the current height.
812
        t.currentHeight.Store(beat.Height())
2✔
813

2✔
814
        t.wg.Add(1)
2✔
815
        go t.monitor()
2✔
816

2✔
817
        log.Debugf("TxPublisher started")
2✔
818

2✔
819
        return nil
2✔
820
}
821

822
// Stop stops the publisher and waits for the monitor loop to exit.
823
func (t *TxPublisher) Stop() error {
2✔
824
        log.Info("TxPublisher stopping...")
2✔
825

2✔
826
        if t.stopped.Swap(true) {
2✔
827
                return fmt.Errorf("TxPublisher stopped more than once")
×
828
        }
×
829

830
        close(t.quit)
2✔
831
        t.wg.Wait()
2✔
832

2✔
833
        log.Debug("TxPublisher stopped")
2✔
834

2✔
835
        return nil
2✔
836
}
837

838
// monitor is the main loop driven by new blocks. Whevenr a new block arrives,
839
// it will examine all the txns being monitored, and check if any of them needs
840
// to be bumped. If so, it will attempt to bump the fee of the tx.
841
//
842
// NOTE: Must be run as a goroutine.
843
func (t *TxPublisher) monitor() {
2✔
844
        defer t.wg.Done()
2✔
845

2✔
846
        for {
4✔
847
                select {
2✔
848
                case beat := <-t.BlockbeatChan:
2✔
849
                        height := beat.Height()
2✔
850
                        log.Debugf("TxPublisher received new block: %v", height)
2✔
851

2✔
852
                        // Update the best known height for the publisher.
2✔
853
                        t.currentHeight.Store(height)
2✔
854

2✔
855
                        // Check all monitored txns to see if any of them needs
2✔
856
                        // to be bumped.
2✔
857
                        t.processRecords()
2✔
858

2✔
859
                        // Notify we've processed the block.
2✔
860
                        t.NotifyBlockProcessed(beat, nil)
2✔
861

862
                case <-t.quit:
2✔
863
                        log.Debug("Fee bumper stopped, exit monitor")
2✔
864
                        return
2✔
865
                }
866
        }
867
}
868

869
// processRecords checks all the txns being monitored, and checks if any of
870
// them needs to be bumped. If so, it will attempt to bump the fee of the tx.
871
func (t *TxPublisher) processRecords() {
3✔
872
        // confirmedRecords stores a map of the records which have been
3✔
873
        // confirmed.
3✔
874
        confirmedRecords := make(map[uint64]*monitorRecord)
3✔
875

3✔
876
        // feeBumpRecords stores a map of records which need to be bumped.
3✔
877
        feeBumpRecords := make(map[uint64]*monitorRecord)
3✔
878

3✔
879
        // failedRecords stores a map of records which has inputs being spent
3✔
880
        // by a third party.
3✔
881
        //
3✔
882
        // NOTE: this is only used for neutrino backend.
3✔
883
        failedRecords := make(map[uint64]*monitorRecord)
3✔
884

3✔
885
        // initialRecords stores a map of records which are being created and
3✔
886
        // published for the first time.
3✔
887
        initialRecords := make(map[uint64]*monitorRecord)
3✔
888

3✔
889
        // visitor is a helper closure that visits each record and divides them
3✔
890
        // into two groups.
3✔
891
        visitor := func(requestID uint64, r *monitorRecord) error {
7✔
892
                if r.tx == nil {
6✔
893
                        initialRecords[requestID] = r
2✔
894
                        return nil
2✔
895
                }
2✔
896

897
                log.Tracef("Checking monitor recordID=%v for tx=%v", requestID,
4✔
898
                        r.tx.TxHash())
4✔
899

4✔
900
                // If the tx is already confirmed, we can stop monitoring it.
4✔
901
                if t.isConfirmed(r.tx.TxHash()) {
7✔
902
                        confirmedRecords[requestID] = r
3✔
903

3✔
904
                        // Move to the next record.
3✔
905
                        return nil
3✔
906
                }
3✔
907

908
                // Check whether the inputs has been spent by a third party.
909
                //
910
                // NOTE: this check is only done for neutrino backend.
911
                if t.isThirdPartySpent(r.tx.TxHash(), r.req.Inputs) {
3✔
UNCOV
912
                        failedRecords[requestID] = r
×
UNCOV
913

×
UNCOV
914
                        // Move to the next record.
×
UNCOV
915
                        return nil
×
UNCOV
916
                }
×
917

918
                feeBumpRecords[requestID] = r
3✔
919

3✔
920
                // Return nil to move to the next record.
3✔
921
                return nil
3✔
922
        }
923

924
        // Iterate through all the records and divide them into four groups.
925
        t.records.ForEach(visitor)
3✔
926

3✔
927
        // Handle the initial broadcast.
3✔
928
        for _, r := range initialRecords {
5✔
929
                t.handleInitialBroadcast(r)
2✔
930
        }
2✔
931

932
        // For records that are confirmed, we'll notify the caller about this
933
        // result.
934
        for _, r := range confirmedRecords {
6✔
935
                log.Debugf("Tx=%v is confirmed", r.tx.TxHash())
3✔
936
                t.wg.Add(1)
3✔
937
                go t.handleTxConfirmed(r)
3✔
938
        }
3✔
939

940
        // Get the current height to be used in the following goroutines.
941
        currentHeight := t.currentHeight.Load()
3✔
942

3✔
943
        // For records that are not confirmed, we perform a fee bump if needed.
3✔
944
        for _, r := range feeBumpRecords {
6✔
945
                log.Debugf("Attempting to fee bump Tx=%v", r.tx.TxHash())
3✔
946
                t.wg.Add(1)
3✔
947
                go t.handleFeeBumpTx(r, currentHeight)
3✔
948
        }
3✔
949

950
        // For records that are failed, we'll notify the caller about this
951
        // result.
952
        for _, r := range failedRecords {
3✔
UNCOV
953
                log.Debugf("Tx=%v has inputs been spent by a third party, "+
×
UNCOV
954
                        "failing it now", r.tx.TxHash())
×
UNCOV
955
                t.wg.Add(1)
×
UNCOV
956
                go t.handleThirdPartySpent(r)
×
UNCOV
957
        }
×
958
}
959

960
// handleTxConfirmed is called when a monitored tx is confirmed. It will
961
// notify the subscriber then remove the record from the maps .
962
//
963
// NOTE: Must be run as a goroutine to avoid blocking on sending the result.
964
func (t *TxPublisher) handleTxConfirmed(r *monitorRecord) {
4✔
965
        defer t.wg.Done()
4✔
966

4✔
967
        // Create a result that will be sent to the resultChan which is
4✔
968
        // listened by the caller.
4✔
969
        result := &BumpResult{
4✔
970
                Event:     TxConfirmed,
4✔
971
                Tx:        r.tx,
4✔
972
                requestID: r.requestID,
4✔
973
                Fee:       r.fee,
4✔
974
                FeeRate:   r.feeFunction.FeeRate(),
4✔
975
        }
4✔
976

4✔
977
        // Notify that this tx is confirmed and remove the record from the map.
4✔
978
        t.handleResult(result)
4✔
979
}
4✔
980

981
// handleInitialTxError takes the error from `initializeTx` and decides the
982
// bump event. It will construct a BumpResult and handles it.
983
func (t *TxPublisher) handleInitialTxError(requestID uint64, err error) {
4✔
984
        // We now decide what type of event to send.
4✔
985
        var event BumpEvent
4✔
986

4✔
987
        switch {
4✔
988
        // When the error is due to a dust output, we'll send a TxFailed so
989
        // these inputs can be retried with a different group in the next
990
        // block.
991
        case errors.Is(err, ErrTxNoOutput):
2✔
992
                event = TxFailed
2✔
993

994
        // When the error is due to budget being used up, we'll send a TxFailed
995
        // so these inputs can be retried with a different group in the next
996
        // block.
997
        case errors.Is(err, ErrMaxPosition):
2✔
998
                event = TxFailed
2✔
999

1000
        // When the error is due to zero fee rate delta, we'll send a TxFailed
1001
        // so these inputs can be retried in the next block.
1002
        case errors.Is(err, ErrZeroFeeRateDelta):
2✔
1003
                event = TxFailed
2✔
1004

1005
        // Otherwise this is not a fee-related error and the tx cannot be
1006
        // retried. In that case we will fail ALL the inputs in this tx, which
1007
        // means they will be removed from the sweeper and never be tried
1008
        // again.
1009
        //
1010
        // TODO(yy): Find out which input is causing the failure and fail that
1011
        // one only.
1012
        default:
4✔
1013
                event = TxFatal
4✔
1014
        }
1015

1016
        result := &BumpResult{
4✔
1017
                Event:     event,
4✔
1018
                Err:       err,
4✔
1019
                requestID: requestID,
4✔
1020
        }
4✔
1021

4✔
1022
        t.handleResult(result)
4✔
1023
}
1024

1025
// handleInitialBroadcast is called when a new request is received. It will
1026
// handle the initial tx creation and broadcast. In details,
1027
// 1. init a fee function based on the given strategy.
1028
// 2. create an RBF-compliant tx and monitor it for confirmation.
1029
// 3. notify the initial broadcast result back to the caller.
1030
func (t *TxPublisher) handleInitialBroadcast(r *monitorRecord) {
6✔
1031
        log.Debugf("Initial broadcast for requestID=%v", r.requestID)
6✔
1032

6✔
1033
        var (
6✔
1034
                result *BumpResult
6✔
1035
                err    error
6✔
1036
        )
6✔
1037

6✔
1038
        // Attempt an initial broadcast which is guaranteed to comply with the
6✔
1039
        // RBF rules.
6✔
1040
        //
6✔
1041
        // Create the initial tx to be broadcasted.
6✔
1042
        record, err := t.initializeTx(r)
6✔
1043
        if err != nil {
10✔
1044
                log.Errorf("Initial broadcast failed: %v", err)
4✔
1045

4✔
1046
                // We now handle the initialization error and exit.
4✔
1047
                t.handleInitialTxError(r.requestID, err)
4✔
1048

4✔
1049
                return
4✔
1050
        }
4✔
1051

1052
        // Successfully created the first tx, now broadcast it.
1053
        result, err = t.broadcast(record)
4✔
1054
        if err != nil {
4✔
1055
                // The broadcast failed, which can only happen if the tx record
×
1056
                // cannot be found or the aux sweeper returns an error. In
×
1057
                // either case, we will send back a TxFail event so these
×
1058
                // inputs can be retried.
×
1059
                result = &BumpResult{
×
1060
                        Event:     TxFailed,
×
1061
                        Err:       err,
×
1062
                        requestID: r.requestID,
×
1063
                }
×
1064
        }
×
1065

1066
        t.handleResult(result)
4✔
1067
}
1068

1069
// handleFeeBumpTx checks if the tx needs to be bumped, and if so, it will
1070
// attempt to bump the fee of the tx.
1071
//
1072
// NOTE: Must be run as a goroutine to avoid blocking on sending the result.
1073
func (t *TxPublisher) handleFeeBumpTx(r *monitorRecord, currentHeight int32) {
6✔
1074
        defer t.wg.Done()
6✔
1075

6✔
1076
        oldTxid := r.tx.TxHash()
6✔
1077

6✔
1078
        // Get the current conf target for this record.
6✔
1079
        confTarget := calcCurrentConfTarget(currentHeight, r.req.DeadlineHeight)
6✔
1080

6✔
1081
        // Ask the fee function whether a bump is needed. We expect the fee
6✔
1082
        // function to increase its returned fee rate after calling this
6✔
1083
        // method.
6✔
1084
        increased, err := r.feeFunction.IncreaseFeeRate(confTarget)
6✔
1085
        if err != nil {
9✔
1086
                // TODO(yy): send this error back to the sweeper so it can
3✔
1087
                // re-group the inputs?
3✔
1088
                log.Errorf("Failed to increase fee rate for tx %v at "+
3✔
1089
                        "height=%v: %v", oldTxid, t.currentHeight.Load(), err)
3✔
1090

3✔
1091
                return
3✔
1092
        }
3✔
1093

1094
        // If the fee rate was not increased, there's no need to bump the fee.
1095
        if !increased {
8✔
1096
                log.Tracef("Skip bumping tx %v at height=%v", oldTxid,
3✔
1097
                        t.currentHeight.Load())
3✔
1098

3✔
1099
                return
3✔
1100
        }
3✔
1101

1102
        // The fee function now has a new fee rate, we will use it to bump the
1103
        // fee of the tx.
1104
        resultOpt := t.createAndPublishTx(r)
4✔
1105

4✔
1106
        // If there's a result, we will notify the caller about the result.
4✔
1107
        resultOpt.WhenSome(func(result BumpResult) {
8✔
1108
                // Notify the new result.
4✔
1109
                t.handleResult(&result)
4✔
1110
        })
4✔
1111
}
1112

1113
// handleThirdPartySpent is called when the inputs in an unconfirmed tx is
1114
// spent. It will notify the subscriber then remove the record from the maps
1115
// and send a TxFailed event to the subscriber.
1116
//
1117
// NOTE: Must be run as a goroutine to avoid blocking on sending the result.
UNCOV
1118
func (t *TxPublisher) handleThirdPartySpent(r *monitorRecord) {
×
UNCOV
1119
        defer t.wg.Done()
×
UNCOV
1120

×
UNCOV
1121
        // Create a result that will be sent to the resultChan which is
×
UNCOV
1122
        // listened by the caller.
×
UNCOV
1123
        //
×
UNCOV
1124
        // TODO(yy): create a new state `TxThirdPartySpent` to notify the
×
UNCOV
1125
        // sweeper to remove the input, hence moving the monitoring of inputs
×
UNCOV
1126
        // spent inside the fee bumper.
×
UNCOV
1127
        result := &BumpResult{
×
UNCOV
1128
                Event:     TxFailed,
×
UNCOV
1129
                Tx:        r.tx,
×
UNCOV
1130
                requestID: r.requestID,
×
UNCOV
1131
                Err:       ErrThirdPartySpent,
×
UNCOV
1132
        }
×
UNCOV
1133

×
UNCOV
1134
        // Notify that this tx is confirmed and remove the record from the map.
×
UNCOV
1135
        t.handleResult(result)
×
UNCOV
1136
}
×
1137

1138
// createAndPublishTx creates a new tx with a higher fee rate and publishes it
1139
// to the network. It will update the record with the new tx and fee rate if
1140
// successfully created, and return the result when published successfully.
1141
func (t *TxPublisher) createAndPublishTx(
1142
        r *monitorRecord) fn.Option[BumpResult] {
9✔
1143

9✔
1144
        // Fetch the old tx.
9✔
1145
        oldTx := r.tx
9✔
1146

9✔
1147
        // Create a new tx with the new fee rate.
9✔
1148
        //
9✔
1149
        // NOTE: The fee function is expected to have increased its returned
9✔
1150
        // fee rate after calling the SkipFeeBump method. So we can use it
9✔
1151
        // directly here.
9✔
1152
        sweepCtx, err := t.createAndCheckTx(r.req, r.feeFunction)
9✔
1153

9✔
1154
        // If the error is fee related, we will return no error and let the fee
9✔
1155
        // bumper retry it at next block.
9✔
1156
        //
9✔
1157
        // NOTE: we can check the RBF error here and ask the fee function to
9✔
1158
        // recalculate the fee rate. However, this would defeat the purpose of
9✔
1159
        // using a deadline based fee function:
9✔
1160
        // - if the deadline is far away, there's no rush to RBF the tx.
9✔
1161
        // - if the deadline is close, we expect the fee function to give us a
9✔
1162
        //   higher fee rate. If the fee rate cannot satisfy the RBF rules, it
9✔
1163
        //   means the budget is not enough.
9✔
1164
        if errors.Is(err, chain.ErrInsufficientFee) ||
9✔
1165
                errors.Is(err, lnwallet.ErrMempoolFee) {
13✔
1166

4✔
1167
                log.Debugf("Failed to bump tx %v: %v", oldTx.TxHash(), err)
4✔
1168
                return fn.None[BumpResult]()
4✔
1169
        }
4✔
1170

1171
        // If the error is not fee related, we will return a `TxFailed` event
1172
        // so this input can be retried.
1173
        if err != nil {
10✔
1174
                // If the tx doesn't not have enought budget, we will return a
3✔
1175
                // result so the sweeper can handle it by re-clustering the
3✔
1176
                // utxos.
3✔
1177
                if errors.Is(err, ErrNotEnoughBudget) {
4✔
1178
                        log.Warnf("Fail to fee bump tx %v: %v", oldTx.TxHash(),
1✔
1179
                                err)
1✔
1180
                } else {
3✔
1181
                        // Otherwise, an unexpected error occurred, we will
2✔
1182
                        // fail the tx and let the sweeper retry the whole
2✔
1183
                        // process.
2✔
1184
                        log.Errorf("Failed to bump tx %v: %v", oldTx.TxHash(),
2✔
1185
                                err)
2✔
1186
                }
2✔
1187

1188
                return fn.Some(BumpResult{
3✔
1189
                        Event:     TxFailed,
3✔
1190
                        Tx:        oldTx,
3✔
1191
                        Err:       err,
3✔
1192
                        requestID: r.requestID,
3✔
1193
                })
3✔
1194
        }
1195

1196
        // The tx has been created without any errors, we now register a new
1197
        // record by overwriting the same requestID.
1198
        record := t.updateRecord(r, sweepCtx)
6✔
1199

6✔
1200
        // Attempt to broadcast this new tx.
6✔
1201
        result, err := t.broadcast(record)
6✔
1202
        if err != nil {
6✔
1203
                log.Infof("Failed to broadcast replacement tx %v: %v",
×
1204
                        sweepCtx.tx.TxHash(), err)
×
1205

×
1206
                return fn.None[BumpResult]()
×
1207
        }
×
1208

1209
        // If the result error is fee related, we will return no error and let
1210
        // the fee bumper retry it at next block.
1211
        //
1212
        // NOTE: we may get this error if we've bypassed the mempool check,
1213
        // which means we are suing neutrino backend.
1214
        if errors.Is(result.Err, chain.ErrInsufficientFee) ||
6✔
1215
                errors.Is(result.Err, lnwallet.ErrMempoolFee) {
6✔
UNCOV
1216

×
UNCOV
1217
                log.Debugf("Failed to bump tx %v: %v", oldTx.TxHash(), err)
×
UNCOV
1218
                return fn.None[BumpResult]()
×
UNCOV
1219
        }
×
1220

1221
        // A successful replacement tx is created, attach the old tx.
1222
        result.ReplacedTx = oldTx
6✔
1223

6✔
1224
        // If the new tx failed to be published, we will return the result so
6✔
1225
        // the caller can handle it.
6✔
1226
        if result.Event == TxFailed {
7✔
1227
                return fn.Some(*result)
1✔
1228
        }
1✔
1229

1230
        log.Infof("Replaced tx=%v with new tx=%v", oldTx.TxHash(),
5✔
1231
                sweepCtx.tx.TxHash())
5✔
1232

5✔
1233
        // Otherwise, it's a successful RBF, set the event and return.
5✔
1234
        result.Event = TxReplaced
5✔
1235

5✔
1236
        return fn.Some(*result)
5✔
1237
}
1238

1239
// isConfirmed checks the btcwallet to see whether the tx is confirmed.
1240
func (t *TxPublisher) isConfirmed(txid chainhash.Hash) bool {
4✔
1241
        details, err := t.cfg.Wallet.GetTransactionDetails(&txid)
4✔
1242
        if err != nil {
6✔
1243
                log.Warnf("Failed to get tx details for %v: %v", txid, err)
2✔
1244
                return false
2✔
1245
        }
2✔
1246

1247
        return details.NumConfirmations > 0
4✔
1248
}
1249

1250
// isThirdPartySpent checks whether the inputs of the tx has already been spent
1251
// by a third party. When a tx is not confirmed, yet its inputs has been spent,
1252
// then it must be spent by a different tx other than the sweeping tx here.
1253
//
1254
// NOTE: this check is only performed for neutrino backend as it has no
1255
// reliable way to tell a tx has been replaced.
1256
func (t *TxPublisher) isThirdPartySpent(txid chainhash.Hash,
1257
        inputs []input.Input) bool {
3✔
1258

3✔
1259
        // Skip this check for if this is not neutrino backend.
3✔
1260
        if !t.isNeutrinoBackend() {
6✔
1261
                return false
3✔
1262
        }
3✔
1263

1264
        // Iterate all the inputs and check if they have been spent already.
UNCOV
1265
        for _, inp := range inputs {
×
UNCOV
1266
                op := inp.OutPoint()
×
UNCOV
1267

×
UNCOV
1268
                // For wallet utxos, the height hint is not set - we don't need
×
UNCOV
1269
                // to monitor them for third party spend.
×
UNCOV
1270
                heightHint := inp.HeightHint()
×
UNCOV
1271
                if heightHint == 0 {
×
UNCOV
1272
                        log.Debugf("Skipped third party check for wallet "+
×
UNCOV
1273
                                "input %v", op)
×
UNCOV
1274

×
UNCOV
1275
                        continue
×
1276
                }
1277

1278
                // If the input has already been spent after the height hint, a
1279
                // spend event is sent back immediately.
UNCOV
1280
                spendEvent, err := t.cfg.Notifier.RegisterSpendNtfn(
×
UNCOV
1281
                        &op, inp.SignDesc().Output.PkScript, heightHint,
×
UNCOV
1282
                )
×
UNCOV
1283
                if err != nil {
×
1284
                        log.Criticalf("Failed to register spend ntfn for "+
×
1285
                                "input=%v: %v", op, err)
×
1286
                        return false
×
1287
                }
×
1288

1289
                // Remove the subscription when exit.
UNCOV
1290
                defer spendEvent.Cancel()
×
UNCOV
1291

×
UNCOV
1292
                // Do a non-blocking read to see if the output has been spent.
×
UNCOV
1293
                select {
×
UNCOV
1294
                case spend, ok := <-spendEvent.Spend:
×
UNCOV
1295
                        if !ok {
×
1296
                                log.Debugf("Spend ntfn for %v canceled", op)
×
1297
                                return false
×
1298
                        }
×
1299

UNCOV
1300
                        spendingTxID := spend.SpendingTx.TxHash()
×
UNCOV
1301

×
UNCOV
1302
                        // If the spending tx is the same as the sweeping tx
×
UNCOV
1303
                        // then we are good.
×
UNCOV
1304
                        if spendingTxID == txid {
×
UNCOV
1305
                                continue
×
1306
                        }
1307

UNCOV
1308
                        log.Warnf("Detected third party spent of output=%v "+
×
UNCOV
1309
                                "in tx=%v", op, spend.SpendingTx.TxHash())
×
UNCOV
1310

×
UNCOV
1311
                        return true
×
1312

1313
                // Move to the next input.
UNCOV
1314
                default:
×
1315
                }
1316
        }
1317

UNCOV
1318
        return false
×
1319
}
1320

1321
// calcCurrentConfTarget calculates the current confirmation target based on
1322
// the deadline height. The conf target is capped at 0 if the deadline has
1323
// already been past.
1324
func calcCurrentConfTarget(currentHeight, deadline int32) uint32 {
14✔
1325
        var confTarget uint32
14✔
1326

14✔
1327
        // Calculate how many blocks left until the deadline.
14✔
1328
        deadlineDelta := deadline - currentHeight
14✔
1329

14✔
1330
        // If we are already past the deadline, we will set the conf target to
14✔
1331
        // be 1.
14✔
1332
        if deadlineDelta < 0 {
18✔
1333
                log.Warnf("Deadline is %d blocks behind current height %v",
4✔
1334
                        -deadlineDelta, currentHeight)
4✔
1335

4✔
1336
                confTarget = 0
4✔
1337
        } else {
14✔
1338
                confTarget = uint32(deadlineDelta)
10✔
1339
        }
10✔
1340

1341
        return confTarget
14✔
1342
}
1343

1344
// sweepTxCtx houses a sweep transaction with additional context.
1345
type sweepTxCtx struct {
1346
        tx *wire.MsgTx
1347

1348
        fee btcutil.Amount
1349

1350
        extraTxOut fn.Option[SweepOutput]
1351

1352
        // outpointToTxIndex maps the outpoint of the inputs to their index in
1353
        // the sweep transaction.
1354
        outpointToTxIndex map[wire.OutPoint]int
1355
}
1356

1357
// createSweepTx creates a sweeping tx based on the given inputs, change
1358
// address and fee rate.
1359
func (t *TxPublisher) createSweepTx(inputs []input.Input,
1360
        changePkScript lnwallet.AddrWithKey,
1361
        feeRate chainfee.SatPerKWeight) (*sweepTxCtx, error) {
24✔
1362

24✔
1363
        // Validate and calculate the fee and change amount.
24✔
1364
        txFee, changeOutputsOpt, locktimeOpt, err := prepareSweepTx(
24✔
1365
                inputs, changePkScript, feeRate, t.currentHeight.Load(),
24✔
1366
                t.cfg.AuxSweeper,
24✔
1367
        )
24✔
1368
        if err != nil {
26✔
1369
                return nil, err
2✔
1370
        }
2✔
1371

1372
        var (
24✔
1373
                // Create the sweep transaction that we will be building. We
24✔
1374
                // use version 2 as it is required for CSV.
24✔
1375
                sweepTx = wire.NewMsgTx(2)
24✔
1376

24✔
1377
                // We'll add the inputs as we go so we know the final ordering
24✔
1378
                // of inputs to sign.
24✔
1379
                idxs []input.Input
24✔
1380
        )
24✔
1381

24✔
1382
        // We start by adding all inputs that commit to an output. We do this
24✔
1383
        // since the input and output index must stay the same for the
24✔
1384
        // signatures to be valid.
24✔
1385
        outpointToTxIndex := make(map[wire.OutPoint]int)
24✔
1386
        for _, o := range inputs {
48✔
1387
                if o.RequiredTxOut() == nil {
48✔
1388
                        continue
24✔
1389
                }
1390

1391
                idxs = append(idxs, o)
2✔
1392
                sweepTx.AddTxIn(&wire.TxIn{
2✔
1393
                        PreviousOutPoint: o.OutPoint(),
2✔
1394
                        Sequence:         o.BlocksToMaturity(),
2✔
1395
                })
2✔
1396
                sweepTx.AddTxOut(o.RequiredTxOut())
2✔
1397

2✔
1398
                outpointToTxIndex[o.OutPoint()] = len(sweepTx.TxOut) - 1
2✔
1399
        }
1400

1401
        // Sum up the value contained in the remaining inputs, and add them to
1402
        // the sweep transaction.
1403
        for _, o := range inputs {
48✔
1404
                if o.RequiredTxOut() != nil {
26✔
1405
                        continue
2✔
1406
                }
1407

1408
                idxs = append(idxs, o)
24✔
1409
                sweepTx.AddTxIn(&wire.TxIn{
24✔
1410
                        PreviousOutPoint: o.OutPoint(),
24✔
1411
                        Sequence:         o.BlocksToMaturity(),
24✔
1412
                })
24✔
1413
        }
1414

1415
        // If we have change outputs to add, then add it the sweep transaction
1416
        // here.
1417
        changeOutputsOpt.WhenSome(func(changeOuts []SweepOutput) {
48✔
1418
                for i := range changeOuts {
70✔
1419
                        sweepTx.AddTxOut(&changeOuts[i].TxOut)
46✔
1420
                }
46✔
1421
        })
1422

1423
        // We'll default to using the current block height as locktime, if none
1424
        // of the inputs commits to a different locktime.
1425
        sweepTx.LockTime = uint32(locktimeOpt.UnwrapOr(t.currentHeight.Load()))
24✔
1426

24✔
1427
        prevInputFetcher, err := input.MultiPrevOutFetcher(inputs)
24✔
1428
        if err != nil {
24✔
1429
                return nil, fmt.Errorf("error creating prev input fetcher "+
×
1430
                        "for hash cache: %v", err)
×
1431
        }
×
1432
        hashCache := txscript.NewTxSigHashes(sweepTx, prevInputFetcher)
24✔
1433

24✔
1434
        // With all the inputs in place, use each output's unique input script
24✔
1435
        // function to generate the final witness required for spending.
24✔
1436
        addInputScript := func(idx int, tso input.Input) error {
48✔
1437
                inputScript, err := tso.CraftInputScript(
24✔
1438
                        t.cfg.Signer, sweepTx, hashCache, prevInputFetcher, idx,
24✔
1439
                )
24✔
1440
                if err != nil {
24✔
1441
                        return err
×
1442
                }
×
1443

1444
                sweepTx.TxIn[idx].Witness = inputScript.Witness
24✔
1445

24✔
1446
                if len(inputScript.SigScript) == 0 {
48✔
1447
                        return nil
24✔
1448
                }
24✔
1449

1450
                sweepTx.TxIn[idx].SignatureScript = inputScript.SigScript
×
1451

×
1452
                return nil
×
1453
        }
1454

1455
        for idx, inp := range idxs {
48✔
1456
                if err := addInputScript(idx, inp); err != nil {
24✔
1457
                        return nil, err
×
1458
                }
×
1459
        }
1460

1461
        log.Debugf("Created sweep tx %v for inputs:\n%v", sweepTx.TxHash(),
24✔
1462
                inputTypeSummary(inputs))
24✔
1463

24✔
1464
        // Try to locate the extra change output, though there might be None.
24✔
1465
        extraTxOut := fn.MapOption(
24✔
1466
                func(sweepOuts []SweepOutput) fn.Option[SweepOutput] {
48✔
1467
                        for _, sweepOut := range sweepOuts {
70✔
1468
                                if !sweepOut.IsExtra {
92✔
1469
                                        continue
46✔
1470
                                }
1471

1472
                                // If we sweep outputs of a custom channel, the
1473
                                // custom leaves in those outputs will be merged
1474
                                // into a single output, even if we sweep
1475
                                // multiple outputs (e.g. to_remote and breached
1476
                                // to_local of a breached channel) at the same
1477
                                // time. So there will only ever be one extra
1478
                                // output.
1479
                                log.Debugf("Sweep produced extra_sweep_out=%v",
×
1480
                                        lnutils.SpewLogClosure(sweepOut))
×
1481

×
1482
                                return fn.Some(sweepOut)
×
1483
                        }
1484

1485
                        return fn.None[SweepOutput]()
24✔
1486
                },
1487
        )(changeOutputsOpt)
1488

1489
        return &sweepTxCtx{
24✔
1490
                tx:                sweepTx,
24✔
1491
                fee:               txFee,
24✔
1492
                extraTxOut:        fn.FlattenOption(extraTxOut),
24✔
1493
                outpointToTxIndex: outpointToTxIndex,
24✔
1494
        }, nil
24✔
1495
}
1496

1497
// prepareSweepTx returns the tx fee, a set of optional change outputs and an
1498
// optional locktime after a series of validations:
1499
// 1. check the locktime has been reached.
1500
// 2. check the locktimes are the same.
1501
// 3. check the inputs cover the outputs.
1502
//
1503
// NOTE: if the change amount is below dust, it will be added to the tx fee.
1504
func prepareSweepTx(inputs []input.Input, changePkScript lnwallet.AddrWithKey,
1505
        feeRate chainfee.SatPerKWeight, currentHeight int32,
1506
        auxSweeper fn.Option[AuxSweeper]) (
1507
        btcutil.Amount, fn.Option[[]SweepOutput], fn.Option[int32], error) {
24✔
1508

24✔
1509
        noChange := fn.None[[]SweepOutput]()
24✔
1510
        noLocktime := fn.None[int32]()
24✔
1511

24✔
1512
        // Given the set of inputs we have, if we have an aux sweeper, then
24✔
1513
        // we'll attempt to see if we have any other change outputs we'll need
24✔
1514
        // to add to the sweep transaction.
24✔
1515
        changePkScripts := [][]byte{changePkScript.DeliveryAddress}
24✔
1516

24✔
1517
        var extraChangeOut fn.Option[SweepOutput]
24✔
1518
        err := fn.MapOptionZ(
24✔
1519
                auxSweeper, func(aux AuxSweeper) error {
46✔
1520
                        extraOut := aux.DeriveSweepAddr(inputs, changePkScript)
22✔
1521
                        if err := extraOut.Err(); err != nil {
22✔
1522
                                return err
×
1523
                        }
×
1524

1525
                        extraChangeOut = extraOut.LeftToSome()
22✔
1526

22✔
1527
                        return nil
22✔
1528
                },
1529
        )
1530
        if err != nil {
24✔
1531
                return 0, noChange, noLocktime, err
×
1532
        }
×
1533

1534
        // Creating a weight estimator with nil outputs and zero max fee rate.
1535
        // We don't allow adding customized outputs in the sweeping tx, and the
1536
        // fee rate is already being managed before we get here.
1537
        inputs, estimator, err := getWeightEstimate(
24✔
1538
                inputs, nil, feeRate, 0, changePkScripts,
24✔
1539
        )
24✔
1540
        if err != nil {
24✔
1541
                return 0, noChange, noLocktime, err
×
1542
        }
×
1543

1544
        txFee := estimator.fee()
24✔
1545

24✔
1546
        var (
24✔
1547
                // Track whether any of the inputs require a certain locktime.
24✔
1548
                locktime = int32(-1)
24✔
1549

24✔
1550
                // We keep track of total input amount, and required output
24✔
1551
                // amount to use for calculating the change amount below.
24✔
1552
                totalInput     btcutil.Amount
24✔
1553
                requiredOutput btcutil.Amount
24✔
1554
        )
24✔
1555

24✔
1556
        // If we have an extra change output, then we'll add it as a required
24✔
1557
        // output amt.
24✔
1558
        extraChangeOut.WhenSome(func(o SweepOutput) {
46✔
1559
                requiredOutput += btcutil.Amount(o.Value)
22✔
1560
        })
22✔
1561

1562
        // Go through each input and check if the required lock times have
1563
        // reached and are the same.
1564
        for _, o := range inputs {
48✔
1565
                // If the input has a required output, we'll add it to the
24✔
1566
                // required output amount.
24✔
1567
                if o.RequiredTxOut() != nil {
26✔
1568
                        requiredOutput += btcutil.Amount(
2✔
1569
                                o.RequiredTxOut().Value,
2✔
1570
                        )
2✔
1571
                }
2✔
1572

1573
                // Update the total input amount.
1574
                totalInput += btcutil.Amount(o.SignDesc().Output.Value)
24✔
1575

24✔
1576
                lt, ok := o.RequiredLockTime()
24✔
1577

24✔
1578
                // Skip if the input doesn't require a lock time.
24✔
1579
                if !ok {
48✔
1580
                        continue
24✔
1581
                }
1582

1583
                // Check if the lock time has reached
1584
                if lt > uint32(currentHeight) {
2✔
1585
                        return 0, noChange, noLocktime,
×
1586
                                fmt.Errorf("%w: current height is %v, "+
×
1587
                                        "locktime is %v", ErrLocktimeImmature,
×
1588
                                        currentHeight, lt)
×
1589
                }
×
1590

1591
                // If another input commits to a different locktime, they
1592
                // cannot be combined in the same transaction.
1593
                if locktime != -1 && locktime != int32(lt) {
2✔
1594
                        return 0, noChange, noLocktime, ErrLocktimeConflict
×
1595
                }
×
1596

1597
                // Update the locktime for next iteration.
1598
                locktime = int32(lt)
2✔
1599
        }
1600

1601
        // Make sure total output amount is less than total input amount.
1602
        if requiredOutput+txFee > totalInput {
24✔
1603
                return 0, noChange, noLocktime, fmt.Errorf("insufficient "+
×
1604
                        "input to create sweep tx: input_sum=%v, "+
×
1605
                        "output_sum=%v", totalInput, requiredOutput+txFee)
×
1606
        }
×
1607

1608
        // The value remaining after the required output and fees is the
1609
        // change output.
1610
        changeAmt := totalInput - requiredOutput - txFee
24✔
1611
        changeOuts := make([]SweepOutput, 0, 2)
24✔
1612

24✔
1613
        extraChangeOut.WhenSome(func(o SweepOutput) {
46✔
1614
                changeOuts = append(changeOuts, o)
22✔
1615
        })
22✔
1616

1617
        // We'll calculate the dust limit for the given changePkScript since it
1618
        // is variable.
1619
        changeFloor := lnwallet.DustLimitForSize(
24✔
1620
                len(changePkScript.DeliveryAddress),
24✔
1621
        )
24✔
1622

24✔
1623
        switch {
24✔
1624
        // If the change amount is dust, we'll move it into the fees, and
1625
        // ignore it.
1626
        case changeAmt < changeFloor:
2✔
1627
                log.Infof("Change amt %v below dustlimit %v, not adding "+
2✔
1628
                        "change output", changeAmt, changeFloor)
2✔
1629

2✔
1630
                // If there's no required output, and the change output is a
2✔
1631
                // dust, it means we are creating a tx without any outputs. In
2✔
1632
                // this case we'll return an error. This could happen when
2✔
1633
                // creating a tx that has an anchor as the only input.
2✔
1634
                if requiredOutput == 0 {
4✔
1635
                        return 0, noChange, noLocktime, ErrTxNoOutput
2✔
1636
                }
2✔
1637

1638
                // The dust amount is added to the fee.
1639
                txFee += changeAmt
×
1640

1641
        // Otherwise, we'll actually recognize it as a change output.
1642
        default:
24✔
1643
                changeOuts = append(changeOuts, SweepOutput{
24✔
1644
                        TxOut: wire.TxOut{
24✔
1645
                                Value:    int64(changeAmt),
24✔
1646
                                PkScript: changePkScript.DeliveryAddress,
24✔
1647
                        },
24✔
1648
                        IsExtra:     false,
24✔
1649
                        InternalKey: changePkScript.InternalKey,
24✔
1650
                })
24✔
1651
        }
1652

1653
        // Optionally set the locktime.
1654
        locktimeOpt := fn.Some(locktime)
24✔
1655
        if locktime == -1 {
48✔
1656
                locktimeOpt = noLocktime
24✔
1657
        }
24✔
1658

1659
        var changeOutsOpt fn.Option[[]SweepOutput]
24✔
1660
        if len(changeOuts) > 0 {
48✔
1661
                changeOutsOpt = fn.Some(changeOuts)
24✔
1662
        }
24✔
1663

1664
        log.Debugf("Creating sweep tx for %v inputs (%s) using %v, "+
24✔
1665
                "tx_weight=%v, tx_fee=%v, locktime=%v, parents_count=%v, "+
24✔
1666
                "parents_fee=%v, parents_weight=%v, current_height=%v",
24✔
1667
                len(inputs), inputTypeSummary(inputs), feeRate,
24✔
1668
                estimator.weight(), txFee, locktimeOpt, len(estimator.parents),
24✔
1669
                estimator.parentsFee, estimator.parentsWeight, currentHeight)
24✔
1670

24✔
1671
        return txFee, changeOutsOpt, locktimeOpt, nil
24✔
1672
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc