• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

lightningnetwork / lnd / 11216766535

07 Oct 2024 01:37PM UTC coverage: 57.817% (-1.0%) from 58.817%
11216766535

Pull #9148

github

ProofOfKeags
lnwire: remove kickoff feerate from propose/commit
Pull Request #9148: DynComms [2/n]: lnwire: add authenticated wire messages for Dyn*

571 of 879 new or added lines in 16 files covered. (64.96%)

23253 existing lines in 251 files now uncovered.

99022 of 171268 relevant lines covered (57.82%)

38420.67 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.75
/contractcourt/channel_arbitrator.go
1
package contractcourt
2

3
import (
4
        "bytes"
5
        "context"
6
        "errors"
7
        "fmt"
8
        "math"
9
        "sync"
10
        "sync/atomic"
11
        "time"
12

13
        "github.com/btcsuite/btcd/btcutil"
14
        "github.com/btcsuite/btcd/chaincfg/chainhash"
15
        "github.com/btcsuite/btcd/txscript"
16
        "github.com/btcsuite/btcd/wire"
17
        "github.com/lightningnetwork/lnd/channeldb"
18
        "github.com/lightningnetwork/lnd/channeldb/models"
19
        "github.com/lightningnetwork/lnd/fn"
20
        "github.com/lightningnetwork/lnd/htlcswitch/hop"
21
        "github.com/lightningnetwork/lnd/input"
22
        "github.com/lightningnetwork/lnd/invoices"
23
        "github.com/lightningnetwork/lnd/kvdb"
24
        "github.com/lightningnetwork/lnd/labels"
25
        "github.com/lightningnetwork/lnd/lntypes"
26
        "github.com/lightningnetwork/lnd/lnutils"
27
        "github.com/lightningnetwork/lnd/lnwallet"
28
        "github.com/lightningnetwork/lnd/lnwire"
29
        "github.com/lightningnetwork/lnd/sweep"
30
)
31

32
var (
33
        // errAlreadyForceClosed is an error returned when we attempt to force
34
        // close a channel that's already in the process of doing so.
35
        errAlreadyForceClosed = errors.New("channel is already in the " +
36
                "process of being force closed")
37
)
38

39
const (
40
        // arbitratorBlockBufferSize is the size of the buffer we give to each
41
        // channel arbitrator.
42
        arbitratorBlockBufferSize = 20
43

44
        // AnchorOutputValue is the output value for the anchor output of an
45
        // anchor channel.
46
        // See BOLT 03 for more details:
47
        // https://github.com/lightning/bolts/blob/master/03-transactions.md
48
        AnchorOutputValue = btcutil.Amount(330)
49
)
50

51
// WitnessSubscription represents an intent to be notified once new witnesses
52
// are discovered by various active contract resolvers. A contract resolver may
53
// use this to be notified of when it can satisfy an incoming contract after we
54
// discover the witness for an outgoing contract.
55
type WitnessSubscription struct {
56
        // WitnessUpdates is a channel that newly discovered witnesses will be
57
        // sent over.
58
        //
59
        // TODO(roasbeef): couple with WitnessType?
60
        WitnessUpdates <-chan lntypes.Preimage
61

62
        // CancelSubscription is a function closure that should be used by a
63
        // client to cancel the subscription once they are no longer interested
64
        // in receiving new updates.
65
        CancelSubscription func()
66
}
67

68
// WitnessBeacon is a global beacon of witnesses. Contract resolvers will use
69
// this interface to lookup witnesses (preimages typically) of contracts
70
// they're trying to resolve, add new preimages they resolve, and finally
71
// receive new updates each new time a preimage is discovered.
72
//
73
// TODO(roasbeef): need to delete the pre-images once we've used them
74
// and have been sufficiently confirmed?
75
type WitnessBeacon interface {
76
        // SubscribeUpdates returns a channel that will be sent upon *each* time
77
        // a new preimage is discovered.
78
        SubscribeUpdates(chanID lnwire.ShortChannelID, htlc *channeldb.HTLC,
79
                payload *hop.Payload,
80
                nextHopOnionBlob []byte) (*WitnessSubscription, error)
81

82
        // LookupPreImage attempts to lookup a preimage in the global cache.
83
        // True is returned for the second argument if the preimage is found.
84
        LookupPreimage(payhash lntypes.Hash) (lntypes.Preimage, bool)
85

86
        // AddPreimages adds a batch of newly discovered preimages to the global
87
        // cache, and also signals any subscribers of the newly discovered
88
        // witness.
89
        AddPreimages(preimages ...lntypes.Preimage) error
90
}
91

92
// ArbChannel is an abstraction that allows the channel arbitrator to interact
93
// with an open channel.
94
type ArbChannel interface {
95
        // ForceCloseChan should force close the contract that this attendant
96
        // is watching over. We'll use this when we decide that we need to go
97
        // to chain. It should in addition tell the switch to remove the
98
        // corresponding link, such that we won't accept any new updates. The
99
        // returned summary contains all items needed to eventually resolve all
100
        // outputs on chain.
101
        ForceCloseChan() (*lnwallet.LocalForceCloseSummary, error)
102

103
        // NewAnchorResolutions returns the anchor resolutions for currently
104
        // valid commitment transactions.
105
        NewAnchorResolutions() (*lnwallet.AnchorResolutions, error)
106
}
107

108
// ChannelArbitratorConfig contains all the functionality that the
109
// ChannelArbitrator needs in order to properly arbitrate any contract dispute
110
// on chain.
111
type ChannelArbitratorConfig struct {
112
        // ChanPoint is the channel point that uniquely identifies this
113
        // channel.
114
        ChanPoint wire.OutPoint
115

116
        // Channel is the full channel data structure. For legacy channels, this
117
        // field may not always be set after a restart.
118
        Channel ArbChannel
119

120
        // ShortChanID describes the exact location of the channel within the
121
        // chain. We'll use this to address any messages that we need to send
122
        // to the switch during contract resolution.
123
        ShortChanID lnwire.ShortChannelID
124

125
        // ChainEvents is an active subscription to the chain watcher for this
126
        // channel to be notified of any on-chain activity related to this
127
        // channel.
128
        ChainEvents *ChainEventSubscription
129

130
        // MarkCommitmentBroadcasted should mark the channel as the commitment
131
        // being broadcast, and we are waiting for the commitment to confirm.
132
        MarkCommitmentBroadcasted func(*wire.MsgTx, lntypes.ChannelParty) error
133

134
        // MarkChannelClosed marks the channel closed in the database, with the
135
        // passed close summary. After this method successfully returns we can
136
        // no longer expect to receive chain events for this channel, and must
137
        // be able to recover from a failure without getting the close event
138
        // again. It takes an optional channel status which will update the
139
        // channel status in the record that we keep of historical channels.
140
        MarkChannelClosed func(*channeldb.ChannelCloseSummary,
141
                ...channeldb.ChannelStatus) error
142

143
        // IsPendingClose is a boolean indicating whether the channel is marked
144
        // as pending close in the database.
145
        IsPendingClose bool
146

147
        // ClosingHeight is the height at which the channel was closed. Note
148
        // that this value is only valid if IsPendingClose is true.
149
        ClosingHeight uint32
150

151
        // CloseType is the type of the close event in case IsPendingClose is
152
        // true. Otherwise this value is unset.
153
        CloseType channeldb.ClosureType
154

155
        // MarkChannelResolved is a function closure that serves to mark a
156
        // channel as "fully resolved". A channel itself can be considered
157
        // fully resolved once all active contracts have individually been
158
        // fully resolved.
159
        //
160
        // TODO(roasbeef): need RPC's to combine for pendingchannels RPC
161
        MarkChannelResolved func() error
162

163
        // PutResolverReport records a resolver report for the channel. If the
164
        // transaction provided is nil, the function should write the report
165
        // in a new transaction.
166
        PutResolverReport func(tx kvdb.RwTx,
167
                report *channeldb.ResolverReport) error
168

169
        // FetchHistoricalChannel retrieves the historical state of a channel.
170
        // This is mostly used to supplement the ContractResolvers with
171
        // additional information required for proper contract resolution.
172
        FetchHistoricalChannel func() (*channeldb.OpenChannel, error)
173

174
        // FindOutgoingHTLCDeadline returns the deadline in absolute block
175
        // height for the specified outgoing HTLC. For an outgoing HTLC, its
176
        // deadline is defined by the timeout height of its corresponding
177
        // incoming HTLC - this is the expiry height the that remote peer can
178
        // spend his/her outgoing HTLC via the timeout path.
179
        FindOutgoingHTLCDeadline func(htlc channeldb.HTLC) fn.Option[int32]
180

181
        ChainArbitratorConfig
182
}
183

184
// ReportOutputType describes the type of output that is being reported
185
// on.
186
type ReportOutputType uint8
187

188
const (
189
        // ReportOutputIncomingHtlc is an incoming hash time locked contract on
190
        // the commitment tx.
191
        ReportOutputIncomingHtlc ReportOutputType = iota
192

193
        // ReportOutputOutgoingHtlc is an outgoing hash time locked contract on
194
        // the commitment tx.
195
        ReportOutputOutgoingHtlc
196

197
        // ReportOutputUnencumbered is an uncontested output on the commitment
198
        // transaction paying to us directly.
199
        ReportOutputUnencumbered
200

201
        // ReportOutputAnchor is an anchor output on the commitment tx.
202
        ReportOutputAnchor
203
)
204

205
// ContractReport provides a summary of a commitment tx output.
206
type ContractReport struct {
207
        // Outpoint is the final output that will be swept back to the wallet.
208
        Outpoint wire.OutPoint
209

210
        // Type indicates the type of the reported output.
211
        Type ReportOutputType
212

213
        // Amount is the final value that will be swept in back to the wallet.
214
        Amount btcutil.Amount
215

216
        // MaturityHeight is the absolute block height that this output will
217
        // mature at.
218
        MaturityHeight uint32
219

220
        // Stage indicates whether the htlc is in the CLTV-timeout stage (1) or
221
        // the CSV-delay stage (2). A stage 1 htlc's maturity height will be set
222
        // to its expiry height, while a stage 2 htlc's maturity height will be
223
        // set to its confirmation height plus the maturity requirement.
224
        Stage uint32
225

226
        // LimboBalance is the total number of frozen coins within this
227
        // contract.
228
        LimboBalance btcutil.Amount
229

230
        // RecoveredBalance is the total value that has been successfully swept
231
        // back to the user's wallet.
232
        RecoveredBalance btcutil.Amount
233
}
234

235
// resolverReport creates a resolve report using some of the information in the
236
// contract report.
237
func (c *ContractReport) resolverReport(spendTx *chainhash.Hash,
238
        resolverType channeldb.ResolverType,
239
        outcome channeldb.ResolverOutcome) *channeldb.ResolverReport {
10✔
240

10✔
241
        return &channeldb.ResolverReport{
10✔
242
                OutPoint:        c.Outpoint,
10✔
243
                Amount:          c.Amount,
10✔
244
                ResolverType:    resolverType,
10✔
245
                ResolverOutcome: outcome,
10✔
246
                SpendTxID:       spendTx,
10✔
247
        }
10✔
248
}
10✔
249

250
// htlcSet represents the set of active HTLCs on a given commitment
251
// transaction.
252
type htlcSet struct {
253
        // incomingHTLCs is a map of all incoming HTLCs on the target
254
        // commitment transaction. We may potentially go onchain to claim the
255
        // funds sent to us within this set.
256
        incomingHTLCs map[uint64]channeldb.HTLC
257

258
        // outgoingHTLCs is a map of all outgoing HTLCs on the target
259
        // commitment transaction. We may potentially go onchain to reclaim the
260
        // funds that are currently in limbo.
261
        outgoingHTLCs map[uint64]channeldb.HTLC
262
}
263

264
// newHtlcSet constructs a new HTLC set from a slice of HTLC's.
265
func newHtlcSet(htlcs []channeldb.HTLC) htlcSet {
46✔
266
        outHTLCs := make(map[uint64]channeldb.HTLC)
46✔
267
        inHTLCs := make(map[uint64]channeldb.HTLC)
46✔
268
        for _, htlc := range htlcs {
77✔
269
                if htlc.Incoming {
37✔
270
                        inHTLCs[htlc.HtlcIndex] = htlc
6✔
271
                        continue
6✔
272
                }
273

274
                outHTLCs[htlc.HtlcIndex] = htlc
25✔
275
        }
276

277
        return htlcSet{
46✔
278
                incomingHTLCs: inHTLCs,
46✔
279
                outgoingHTLCs: outHTLCs,
46✔
280
        }
46✔
281
}
282

283
// HtlcSetKey is a two-tuple that uniquely identifies a set of HTLCs on a
284
// commitment transaction.
285
type HtlcSetKey struct {
286
        // IsRemote denotes if the HTLCs are on the remote commitment
287
        // transaction.
288
        IsRemote bool
289

290
        // IsPending denotes if the commitment transaction that HTLCS are on
291
        // are pending (the higher of two unrevoked commitments).
292
        IsPending bool
293
}
294

295
var (
296
        // LocalHtlcSet is the HtlcSetKey used for local commitments.
297
        LocalHtlcSet = HtlcSetKey{IsRemote: false, IsPending: false}
298

299
        // RemoteHtlcSet is the HtlcSetKey used for remote commitments.
300
        RemoteHtlcSet = HtlcSetKey{IsRemote: true, IsPending: false}
301

302
        // RemotePendingHtlcSet is the HtlcSetKey used for dangling remote
303
        // commitment transactions.
304
        RemotePendingHtlcSet = HtlcSetKey{IsRemote: true, IsPending: true}
305
)
306

307
// String returns a human readable string describing the target HtlcSetKey.
308
func (h HtlcSetKey) String() string {
8✔
309
        switch h {
8✔
310
        case LocalHtlcSet:
4✔
311
                return "LocalHtlcSet"
4✔
312
        case RemoteHtlcSet:
2✔
313
                return "RemoteHtlcSet"
2✔
314
        case RemotePendingHtlcSet:
2✔
315
                return "RemotePendingHtlcSet"
2✔
316
        default:
×
317
                return "unknown HtlcSetKey"
×
318
        }
319
}
320

321
// ChannelArbitrator is the on-chain arbitrator for a particular channel. The
322
// struct will keep in sync with the current set of HTLCs on the commitment
323
// transaction. The job of the attendant is to go on-chain to either settle or
324
// cancel an HTLC as necessary iff: an HTLC times out, or we known the
325
// pre-image to an HTLC, but it wasn't settled by the link off-chain. The
326
// ChannelArbitrator will factor in an expected confirmation delta when
327
// broadcasting to ensure that we avoid any possibility of race conditions, and
328
// sweep the output(s) without contest.
329
type ChannelArbitrator struct {
330
        started int32 // To be used atomically.
331
        stopped int32 // To be used atomically.
332

333
        // startTimestamp is the time when this ChannelArbitrator was started.
334
        startTimestamp time.Time
335

336
        // log is a persistent log that the attendant will use to checkpoint
337
        // its next action, and the state of any unresolved contracts.
338
        log ArbitratorLog
339

340
        // activeHTLCs is the set of active incoming/outgoing HTLC's on all
341
        // currently valid commitment transactions.
342
        activeHTLCs map[HtlcSetKey]htlcSet
343

344
        // unmergedSet is used to update the activeHTLCs map in two callsites:
345
        // checkLocalChainActions and sweepAnchors. It contains the latest
346
        // updates from the link. It is not deleted from, its entries may be
347
        // replaced on subsequent calls to notifyContractUpdate.
348
        unmergedSet map[HtlcSetKey]htlcSet
349
        unmergedMtx sync.RWMutex
350

351
        // cfg contains all the functionality that the ChannelArbitrator requires
352
        // to do its duty.
353
        cfg ChannelArbitratorConfig
354

355
        // blocks is a channel that the arbitrator will receive new blocks on.
356
        // This channel should be buffered by so that it does not block the
357
        // sender.
358
        blocks chan int32
359

360
        // signalUpdates is a channel that any new live signals for the channel
361
        // we're watching over will be sent.
362
        signalUpdates chan *signalUpdateMsg
363

364
        // activeResolvers is a slice of any active resolvers. This is used to
365
        // be able to signal them for shutdown in the case that we shutdown.
366
        activeResolvers []ContractResolver
367

368
        // activeResolversLock prevents simultaneous read and write to the
369
        // resolvers slice.
370
        activeResolversLock sync.RWMutex
371

372
        // resolutionSignal is a channel that will be sent upon by contract
373
        // resolvers once their contract has been fully resolved. With each
374
        // send, we'll check to see if the contract is fully resolved.
375
        resolutionSignal chan struct{}
376

377
        // forceCloseReqs is a channel that requests to forcibly close the
378
        // contract will be sent over.
379
        forceCloseReqs chan *forceCloseReq
380

381
        // state is the current state of the arbitrator. This state is examined
382
        // upon start up to decide which actions to take.
383
        state ArbitratorState
384

385
        wg   sync.WaitGroup
386
        quit chan struct{}
387
}
388

389
// NewChannelArbitrator returns a new instance of a ChannelArbitrator backed by
390
// the passed config struct.
391
func NewChannelArbitrator(cfg ChannelArbitratorConfig,
392
        htlcSets map[HtlcSetKey]htlcSet, log ArbitratorLog) *ChannelArbitrator {
50✔
393

50✔
394
        // Create a new map for unmerged HTLC's as we will overwrite the values
50✔
395
        // and want to avoid modifying activeHTLCs directly. This soft copying
50✔
396
        // is done to ensure that activeHTLCs isn't reset as an empty map later
50✔
397
        // on.
50✔
398
        unmerged := make(map[HtlcSetKey]htlcSet)
50✔
399
        unmerged[LocalHtlcSet] = htlcSets[LocalHtlcSet]
50✔
400
        unmerged[RemoteHtlcSet] = htlcSets[RemoteHtlcSet]
50✔
401

50✔
402
        // If the pending set exists, write that as well.
50✔
403
        if _, ok := htlcSets[RemotePendingHtlcSet]; ok {
50✔
404
                unmerged[RemotePendingHtlcSet] = htlcSets[RemotePendingHtlcSet]
×
405
        }
×
406

407
        return &ChannelArbitrator{
50✔
408
                log:              log,
50✔
409
                blocks:           make(chan int32, arbitratorBlockBufferSize),
50✔
410
                signalUpdates:    make(chan *signalUpdateMsg),
50✔
411
                resolutionSignal: make(chan struct{}),
50✔
412
                forceCloseReqs:   make(chan *forceCloseReq),
50✔
413
                activeHTLCs:      htlcSets,
50✔
414
                unmergedSet:      unmerged,
50✔
415
                cfg:              cfg,
50✔
416
                quit:             make(chan struct{}),
50✔
417
        }
50✔
418
}
419

420
// chanArbStartState contains the information from disk that we need to start
421
// up a channel arbitrator.
422
type chanArbStartState struct {
423
        currentState ArbitratorState
424
        commitSet    *CommitSet
425
}
426

427
// getStartState retrieves the information from disk that our channel arbitrator
428
// requires to start.
429
func (c *ChannelArbitrator) getStartState(tx kvdb.RTx) (*chanArbStartState,
430
        error) {
48✔
431

48✔
432
        // First, we'll read our last state from disk, so our internal state
48✔
433
        // machine can act accordingly.
48✔
434
        state, err := c.log.CurrentState(tx)
48✔
435
        if err != nil {
48✔
436
                return nil, err
×
437
        }
×
438

439
        // Next we'll fetch our confirmed commitment set. This will only exist
440
        // if the channel has been closed out on chain for modern nodes. For
441
        // older nodes, this won't be found at all, and will rely on the
442
        // existing written chain actions. Additionally, if this channel hasn't
443
        // logged any actions in the log, then this field won't be present.
444
        commitSet, err := c.log.FetchConfirmedCommitSet(tx)
48✔
445
        if err != nil && err != errNoCommitSet && err != errScopeBucketNoExist {
48✔
446
                return nil, err
×
447
        }
×
448

449
        return &chanArbStartState{
48✔
450
                currentState: state,
48✔
451
                commitSet:    commitSet,
48✔
452
        }, nil
48✔
453
}
454

455
// Start starts all the goroutines that the ChannelArbitrator needs to operate.
456
// If takes a start state, which will be looked up on disk if it is not
457
// provided.
458
func (c *ChannelArbitrator) Start(state *chanArbStartState) error {
48✔
459
        if !atomic.CompareAndSwapInt32(&c.started, 0, 1) {
48✔
460
                return nil
×
461
        }
×
462
        c.startTimestamp = c.cfg.Clock.Now()
48✔
463

48✔
464
        // If the state passed in is nil, we look it up now.
48✔
465
        if state == nil {
85✔
466
                var err error
37✔
467
                state, err = c.getStartState(nil)
37✔
468
                if err != nil {
37✔
469
                        return err
×
470
                }
×
471
        }
472

473
        log.Debugf("Starting ChannelArbitrator(%v), htlc_set=%v, state=%v",
48✔
474
                c.cfg.ChanPoint, lnutils.SpewLogClosure(c.activeHTLCs),
48✔
475
                state.currentState)
48✔
476

48✔
477
        // Set our state from our starting state.
48✔
478
        c.state = state.currentState
48✔
479

48✔
480
        _, bestHeight, err := c.cfg.ChainIO.GetBestBlock()
48✔
481
        if err != nil {
48✔
482
                return err
×
483
        }
×
484

485
        // If the channel has been marked pending close in the database, and we
486
        // haven't transitioned the state machine to StateContractClosed (or a
487
        // succeeding state), then a state transition most likely failed. We'll
488
        // try to recover from this by manually advancing the state by setting
489
        // the corresponding close trigger.
490
        trigger := chainTrigger
48✔
491
        triggerHeight := uint32(bestHeight)
48✔
492
        if c.cfg.IsPendingClose {
53✔
493
                switch c.state {
5✔
494
                case StateDefault:
4✔
495
                        fallthrough
4✔
496
                case StateBroadcastCommit:
5✔
497
                        fallthrough
5✔
498
                case StateCommitmentBroadcasted:
5✔
499
                        switch c.cfg.CloseType {
5✔
500

501
                        case channeldb.CooperativeClose:
1✔
502
                                trigger = coopCloseTrigger
1✔
503

504
                        case channeldb.BreachClose:
1✔
505
                                trigger = breachCloseTrigger
1✔
506

507
                        case channeldb.LocalForceClose:
1✔
508
                                trigger = localCloseTrigger
1✔
509

510
                        case channeldb.RemoteForceClose:
2✔
511
                                trigger = remoteCloseTrigger
2✔
512
                        }
513

514
                        log.Warnf("ChannelArbitrator(%v): detected stalled "+
5✔
515
                                "state=%v for closed channel",
5✔
516
                                c.cfg.ChanPoint, c.state)
5✔
517
                }
518

519
                triggerHeight = c.cfg.ClosingHeight
5✔
520
        }
521

522
        log.Infof("ChannelArbitrator(%v): starting state=%v, trigger=%v, "+
48✔
523
                "triggerHeight=%v", c.cfg.ChanPoint, c.state, trigger,
48✔
524
                triggerHeight)
48✔
525

48✔
526
        // We'll now attempt to advance our state forward based on the current
48✔
527
        // on-chain state, and our set of active contracts.
48✔
528
        startingState := c.state
48✔
529
        nextState, _, err := c.advanceState(
48✔
530
                triggerHeight, trigger, state.commitSet,
48✔
531
        )
48✔
532
        if err != nil {
50✔
533
                switch err {
2✔
534

535
                // If we detect that we tried to fetch resolutions, but failed,
536
                // this channel was marked closed in the database before
537
                // resolutions successfully written. In this case there is not
538
                // much we can do, so we don't return the error.
539
                case errScopeBucketNoExist:
×
540
                        fallthrough
×
541
                case errNoResolutions:
1✔
542
                        log.Warnf("ChannelArbitrator(%v): detected closed"+
1✔
543
                                "channel with no contract resolutions written.",
1✔
544
                                c.cfg.ChanPoint)
1✔
545

546
                default:
1✔
547
                        return err
1✔
548
                }
549
        }
550

551
        // If we start and ended at the awaiting full resolution state, then
552
        // we'll relaunch our set of unresolved contracts.
553
        if startingState == StateWaitingFullResolution &&
47✔
554
                nextState == StateWaitingFullResolution {
48✔
555

1✔
556
                // In order to relaunch the resolvers, we'll need to fetch the
1✔
557
                // set of HTLCs that were present in the commitment transaction
1✔
558
                // at the time it was confirmed. commitSet.ConfCommitKey can't
1✔
559
                // be nil at this point since we're in
1✔
560
                // StateWaitingFullResolution. We can only be in
1✔
561
                // StateWaitingFullResolution after we've transitioned from
1✔
562
                // StateContractClosed which can only be triggered by the local
1✔
563
                // or remote close trigger. This trigger is only fired when we
1✔
564
                // receive a chain event from the chain watcher that the
1✔
565
                // commitment has been confirmed on chain, and before we
1✔
566
                // advance our state step, we call InsertConfirmedCommitSet.
1✔
567
                err := c.relaunchResolvers(state.commitSet, triggerHeight)
1✔
568
                if err != nil {
1✔
569
                        return err
×
570
                }
×
571
        }
572

573
        c.wg.Add(1)
47✔
574
        go c.channelAttendant(bestHeight)
47✔
575
        return nil
47✔
576
}
577

578
// maybeAugmentTaprootResolvers will update the contract resolution information
579
// for taproot channels. This ensures that all the resolvers have the latest
580
// resolution, which may also include data such as the control block and tap
581
// tweaks.
582
func maybeAugmentTaprootResolvers(chanType channeldb.ChannelType,
583
        resolver ContractResolver,
584
        contractResolutions *ContractResolutions) {
1✔
585

1✔
586
        if !chanType.IsTaproot() {
2✔
587
                return
1✔
588
        }
1✔
589

590
        // The on disk resolutions contains all the ctrl block
591
        // information, so we'll set that now for the relevant
592
        // resolvers.
UNCOV
593
        switch r := resolver.(type) {
×
UNCOV
594
        case *commitSweepResolver:
×
UNCOV
595
                if contractResolutions.CommitResolution != nil {
×
UNCOV
596
                        //nolint:lll
×
UNCOV
597
                        r.commitResolution = *contractResolutions.CommitResolution
×
UNCOV
598
                }
×
UNCOV
599
        case *htlcOutgoingContestResolver:
×
UNCOV
600
                //nolint:lll
×
UNCOV
601
                htlcResolutions := contractResolutions.HtlcResolutions.OutgoingHTLCs
×
UNCOV
602
                for _, htlcRes := range htlcResolutions {
×
UNCOV
603
                        htlcRes := htlcRes
×
UNCOV
604

×
UNCOV
605
                        if r.htlcResolution.ClaimOutpoint ==
×
UNCOV
606
                                htlcRes.ClaimOutpoint {
×
UNCOV
607

×
UNCOV
608
                                r.htlcResolution = htlcRes
×
UNCOV
609
                        }
×
610
                }
611

UNCOV
612
        case *htlcTimeoutResolver:
×
UNCOV
613
                //nolint:lll
×
UNCOV
614
                htlcResolutions := contractResolutions.HtlcResolutions.OutgoingHTLCs
×
UNCOV
615
                for _, htlcRes := range htlcResolutions {
×
UNCOV
616
                        htlcRes := htlcRes
×
UNCOV
617

×
UNCOV
618
                        if r.htlcResolution.ClaimOutpoint ==
×
UNCOV
619
                                htlcRes.ClaimOutpoint {
×
UNCOV
620

×
UNCOV
621
                                r.htlcResolution = htlcRes
×
UNCOV
622
                        }
×
623
                }
624

UNCOV
625
        case *htlcIncomingContestResolver:
×
UNCOV
626
                //nolint:lll
×
UNCOV
627
                htlcResolutions := contractResolutions.HtlcResolutions.IncomingHTLCs
×
UNCOV
628
                for _, htlcRes := range htlcResolutions {
×
UNCOV
629
                        htlcRes := htlcRes
×
UNCOV
630

×
UNCOV
631
                        if r.htlcResolution.ClaimOutpoint ==
×
UNCOV
632
                                htlcRes.ClaimOutpoint {
×
UNCOV
633

×
UNCOV
634
                                r.htlcResolution = htlcRes
×
UNCOV
635
                        }
×
636
                }
637
        case *htlcSuccessResolver:
×
638
                //nolint:lll
×
639
                htlcResolutions := contractResolutions.HtlcResolutions.IncomingHTLCs
×
640
                for _, htlcRes := range htlcResolutions {
×
641
                        htlcRes := htlcRes
×
642

×
643
                        if r.htlcResolution.ClaimOutpoint ==
×
644
                                htlcRes.ClaimOutpoint {
×
645

×
646
                                r.htlcResolution = htlcRes
×
647
                        }
×
648
                }
649
        }
650
}
651

652
// relauchResolvers relaunches the set of resolvers for unresolved contracts in
653
// order to provide them with information that's not immediately available upon
654
// starting the ChannelArbitrator. This information should ideally be stored in
655
// the database, so this only serves as a intermediate work-around to prevent a
656
// migration.
657
func (c *ChannelArbitrator) relaunchResolvers(commitSet *CommitSet,
658
        heightHint uint32) error {
1✔
659

1✔
660
        // We'll now query our log to see if there are any active unresolved
1✔
661
        // contracts. If this is the case, then we'll relaunch all contract
1✔
662
        // resolvers.
1✔
663
        unresolvedContracts, err := c.log.FetchUnresolvedContracts()
1✔
664
        if err != nil {
1✔
665
                return err
×
666
        }
×
667

668
        // Retrieve the commitment tx hash from the log.
669
        contractResolutions, err := c.log.FetchContractResolutions()
1✔
670
        if err != nil {
1✔
671
                log.Errorf("unable to fetch contract resolutions: %v",
×
672
                        err)
×
673
                return err
×
674
        }
×
675
        commitHash := contractResolutions.CommitHash
1✔
676

1✔
677
        // In prior versions of lnd, the information needed to supplement the
1✔
678
        // resolvers (in most cases, the full amount of the HTLC) was found in
1✔
679
        // the chain action map, which is now deprecated.  As a result, if the
1✔
680
        // commitSet is nil (an older node with unresolved HTLCs at time of
1✔
681
        // upgrade), then we'll use the chain action information in place. The
1✔
682
        // chain actions may exclude some information, but we cannot recover it
1✔
683
        // for these older nodes at the moment.
1✔
684
        var confirmedHTLCs []channeldb.HTLC
1✔
685
        if commitSet != nil {
2✔
686
                confirmedHTLCs = commitSet.HtlcSets[*commitSet.ConfCommitKey]
1✔
687
        } else {
1✔
688
                chainActions, err := c.log.FetchChainActions()
×
689
                if err != nil {
×
690
                        log.Errorf("unable to fetch chain actions: %v", err)
×
691
                        return err
×
692
                }
×
693
                for _, htlcs := range chainActions {
×
694
                        confirmedHTLCs = append(confirmedHTLCs, htlcs...)
×
695
                }
×
696
        }
697

698
        // Reconstruct the htlc outpoints and data from the chain action log.
699
        // The purpose of the constructed htlc map is to supplement to
700
        // resolvers restored from database with extra data. Ideally this data
701
        // is stored as part of the resolver in the log. This is a workaround
702
        // to prevent a db migration. We use all available htlc sets here in
703
        // order to ensure we have complete coverage.
704
        htlcMap := make(map[wire.OutPoint]*channeldb.HTLC)
1✔
705
        for _, htlc := range confirmedHTLCs {
4✔
706
                htlc := htlc
3✔
707
                outpoint := wire.OutPoint{
3✔
708
                        Hash:  commitHash,
3✔
709
                        Index: uint32(htlc.OutputIndex),
3✔
710
                }
3✔
711
                htlcMap[outpoint] = &htlc
3✔
712
        }
3✔
713

714
        // We'll also fetch the historical state of this channel, as it should
715
        // have been marked as closed by now, and supplement it to each resolver
716
        // such that we can properly resolve our pending contracts.
717
        var chanState *channeldb.OpenChannel
1✔
718
        chanState, err = c.cfg.FetchHistoricalChannel()
1✔
719
        switch {
1✔
720
        // If we don't find this channel, then it may be the case that it
721
        // was closed before we started to retain the final state
722
        // information for open channels.
723
        case err == channeldb.ErrNoHistoricalBucket:
×
724
                fallthrough
×
725
        case err == channeldb.ErrChannelNotFound:
×
726
                log.Warnf("ChannelArbitrator(%v): unable to fetch historical "+
×
727
                        "state", c.cfg.ChanPoint)
×
728

729
        case err != nil:
×
730
                return err
×
731
        }
732

733
        log.Infof("ChannelArbitrator(%v): relaunching %v contract "+
1✔
734
                "resolvers", c.cfg.ChanPoint, len(unresolvedContracts))
1✔
735

1✔
736
        for i := range unresolvedContracts {
2✔
737
                resolver := unresolvedContracts[i]
1✔
738

1✔
739
                if chanState != nil {
2✔
740
                        resolver.SupplementState(chanState)
1✔
741

1✔
742
                        // For taproot channels, we'll need to also make sure
1✔
743
                        // the control block information was set properly.
1✔
744
                        maybeAugmentTaprootResolvers(
1✔
745
                                chanState.ChanType, resolver,
1✔
746
                                contractResolutions,
1✔
747
                        )
1✔
748
                }
1✔
749

750
                unresolvedContracts[i] = resolver
1✔
751

1✔
752
                htlcResolver, ok := resolver.(htlcContractResolver)
1✔
753
                if !ok {
1✔
UNCOV
754
                        continue
×
755
                }
756

757
                htlcPoint := htlcResolver.HtlcPoint()
1✔
758
                htlc, ok := htlcMap[htlcPoint]
1✔
759
                if !ok {
1✔
760
                        return fmt.Errorf(
×
761
                                "htlc resolver %T unavailable", resolver,
×
762
                        )
×
763
                }
×
764

765
                htlcResolver.Supplement(*htlc)
1✔
766

1✔
767
                // If this is an outgoing HTLC, we will also need to supplement
1✔
768
                // the resolver with the expiry block height of its
1✔
769
                // corresponding incoming HTLC.
1✔
770
                if !htlc.Incoming {
2✔
771
                        deadline := c.cfg.FindOutgoingHTLCDeadline(*htlc)
1✔
772
                        htlcResolver.SupplementDeadline(deadline)
1✔
773
                }
1✔
774
        }
775

776
        // The anchor resolver is stateless and can always be re-instantiated.
777
        if contractResolutions.AnchorResolution != nil {
1✔
UNCOV
778
                anchorResolver := newAnchorResolver(
×
UNCOV
779
                        contractResolutions.AnchorResolution.AnchorSignDescriptor,
×
UNCOV
780
                        contractResolutions.AnchorResolution.CommitAnchor,
×
UNCOV
781
                        heightHint, c.cfg.ChanPoint,
×
UNCOV
782
                        ResolverConfig{
×
UNCOV
783
                                ChannelArbitratorConfig: c.cfg,
×
UNCOV
784
                        },
×
UNCOV
785
                )
×
UNCOV
786

×
UNCOV
787
                anchorResolver.SupplementState(chanState)
×
UNCOV
788

×
UNCOV
789
                unresolvedContracts = append(unresolvedContracts, anchorResolver)
×
UNCOV
790

×
UNCOV
791
                // TODO(roasbeef): this isn't re-launched?
×
UNCOV
792
        }
×
793

794
        c.launchResolvers(unresolvedContracts, true)
1✔
795

1✔
796
        return nil
1✔
797
}
798

799
// Report returns htlc reports for the active resolvers.
UNCOV
800
func (c *ChannelArbitrator) Report() []*ContractReport {
×
UNCOV
801
        c.activeResolversLock.RLock()
×
UNCOV
802
        defer c.activeResolversLock.RUnlock()
×
UNCOV
803

×
UNCOV
804
        var reports []*ContractReport
×
UNCOV
805
        for _, resolver := range c.activeResolvers {
×
UNCOV
806
                r, ok := resolver.(reportingContractResolver)
×
UNCOV
807
                if !ok {
×
808
                        continue
×
809
                }
810

UNCOV
811
                report := r.report()
×
UNCOV
812
                if report == nil {
×
UNCOV
813
                        continue
×
814
                }
815

UNCOV
816
                reports = append(reports, report)
×
817
        }
818

UNCOV
819
        return reports
×
820
}
821

822
// Stop signals the ChannelArbitrator for a graceful shutdown.
823
func (c *ChannelArbitrator) Stop() error {
50✔
824
        if !atomic.CompareAndSwapInt32(&c.stopped, 0, 1) {
56✔
825
                return nil
6✔
826
        }
6✔
827

828
        log.Debugf("Stopping ChannelArbitrator(%v)", c.cfg.ChanPoint)
44✔
829

44✔
830
        if c.cfg.ChainEvents.Cancel != nil {
55✔
831
                go c.cfg.ChainEvents.Cancel()
11✔
832
        }
11✔
833

834
        c.activeResolversLock.RLock()
44✔
835
        for _, activeResolver := range c.activeResolvers {
50✔
836
                activeResolver.Stop()
6✔
837
        }
6✔
838
        c.activeResolversLock.RUnlock()
44✔
839

44✔
840
        close(c.quit)
44✔
841
        c.wg.Wait()
44✔
842

44✔
843
        return nil
44✔
844
}
845

846
// transitionTrigger is an enum that denotes exactly *why* a state transition
847
// was initiated. This is useful as depending on the initial trigger, we may
848
// skip certain states as those actions are expected to have already taken
849
// place as a result of the external trigger.
850
type transitionTrigger uint8
851

852
const (
853
        // chainTrigger is a transition trigger that has been attempted due to
854
        // changing on-chain conditions such as a block which times out HTLC's
855
        // being attached.
856
        chainTrigger transitionTrigger = iota
857

858
        // userTrigger is a transition trigger driven by user action. Examples
859
        // of such a trigger include a user requesting a force closure of the
860
        // channel.
861
        userTrigger
862

863
        // remoteCloseTrigger is a transition trigger driven by the remote
864
        // peer's commitment being confirmed.
865
        remoteCloseTrigger
866

867
        // localCloseTrigger is a transition trigger driven by our commitment
868
        // being confirmed.
869
        localCloseTrigger
870

871
        // coopCloseTrigger is a transition trigger driven by a cooperative
872
        // close transaction being confirmed.
873
        coopCloseTrigger
874

875
        // breachCloseTrigger is a transition trigger driven by a remote breach
876
        // being confirmed. In this case the channel arbitrator will wait for
877
        // the BreachArbitrator to finish and then clean up gracefully.
878
        breachCloseTrigger
879
)
880

881
// String returns a human readable string describing the passed
882
// transitionTrigger.
UNCOV
883
func (t transitionTrigger) String() string {
×
UNCOV
884
        switch t {
×
UNCOV
885
        case chainTrigger:
×
UNCOV
886
                return "chainTrigger"
×
887

UNCOV
888
        case remoteCloseTrigger:
×
UNCOV
889
                return "remoteCloseTrigger"
×
890

UNCOV
891
        case userTrigger:
×
UNCOV
892
                return "userTrigger"
×
893

UNCOV
894
        case localCloseTrigger:
×
UNCOV
895
                return "localCloseTrigger"
×
896

UNCOV
897
        case coopCloseTrigger:
×
UNCOV
898
                return "coopCloseTrigger"
×
899

UNCOV
900
        case breachCloseTrigger:
×
UNCOV
901
                return "breachCloseTrigger"
×
902

903
        default:
×
904
                return "unknown trigger"
×
905
        }
906
}
907

908
// stateStep is a help method that examines our internal state, and attempts
909
// the appropriate state transition if necessary. The next state we transition
910
// to is returned, Additionally, if the next transition results in a commitment
911
// broadcast, the commitment transaction itself is returned.
912
func (c *ChannelArbitrator) stateStep(
913
        triggerHeight uint32, trigger transitionTrigger,
914
        confCommitSet *CommitSet) (ArbitratorState, *wire.MsgTx, error) {
175✔
915

175✔
916
        var (
175✔
917
                nextState ArbitratorState
175✔
918
                closeTx   *wire.MsgTx
175✔
919
        )
175✔
920
        switch c.state {
175✔
921

922
        // If we're in the default state, then we'll check our set of actions
923
        // to see if while we were down, conditions have changed.
924
        case StateDefault:
64✔
925
                log.Debugf("ChannelArbitrator(%v): new block (height=%v) "+
64✔
926
                        "examining active HTLC's", c.cfg.ChanPoint,
64✔
927
                        triggerHeight)
64✔
928

64✔
929
                // As a new block has been connected to the end of the main
64✔
930
                // chain, we'll check to see if we need to make any on-chain
64✔
931
                // claims on behalf of the channel contract that we're
64✔
932
                // arbitrating for. If a commitment has confirmed, then we'll
64✔
933
                // use the set snapshot from the chain, otherwise we'll use our
64✔
934
                // current set.
64✔
935
                var htlcs map[HtlcSetKey]htlcSet
64✔
936
                if confCommitSet != nil {
73✔
937
                        htlcs = confCommitSet.toActiveHTLCSets()
9✔
938
                } else {
64✔
939
                        // Update the set of activeHTLCs so
55✔
940
                        // checkLocalChainActions has an up-to-date view of the
55✔
941
                        // commitments.
55✔
942
                        c.updateActiveHTLCs()
55✔
943
                        htlcs = c.activeHTLCs
55✔
944
                }
55✔
945
                chainActions, err := c.checkLocalChainActions(
64✔
946
                        triggerHeight, trigger, htlcs, false,
64✔
947
                )
64✔
948
                if err != nil {
64✔
949
                        return StateDefault, nil, err
×
950
                }
×
951

952
                // If there are no actions to be made, then we'll remain in the
953
                // default state. If this isn't a self initiated event (we're
954
                // checking due to a chain update), then we'll exit now.
955
                if len(chainActions) == 0 && trigger == chainTrigger {
101✔
956
                        log.Debugf("ChannelArbitrator(%v): no actions for "+
37✔
957
                                "chain trigger, terminating", c.cfg.ChanPoint)
37✔
958

37✔
959
                        return StateDefault, closeTx, nil
37✔
960
                }
37✔
961

962
                // Otherwise, we'll log that we checked the HTLC actions as the
963
                // commitment transaction has already been broadcast.
964
                log.Tracef("ChannelArbitrator(%v): logging chain_actions=%v",
27✔
965
                        c.cfg.ChanPoint, lnutils.SpewLogClosure(chainActions))
27✔
966

27✔
967
                // Depending on the type of trigger, we'll either "tunnel"
27✔
968
                // through to a farther state, or just proceed linearly to the
27✔
969
                // next state.
27✔
970
                switch trigger {
27✔
971

972
                // If this is a chain trigger, then we'll go straight to the
973
                // next state, as we still need to broadcast the commitment
974
                // transaction.
975
                case chainTrigger:
5✔
976
                        fallthrough
5✔
977
                case userTrigger:
15✔
978
                        nextState = StateBroadcastCommit
15✔
979

980
                // If the trigger is a cooperative close being confirmed, then
981
                // we can go straight to StateFullyResolved, as there won't be
982
                // any contracts to resolve.
983
                case coopCloseTrigger:
3✔
984
                        nextState = StateFullyResolved
3✔
985

986
                // Otherwise, if this state advance was triggered by a
987
                // commitment being confirmed on chain, then we'll jump
988
                // straight to the state where the contract has already been
989
                // closed, and we will inspect the set of unresolved contracts.
990
                case localCloseTrigger:
2✔
991
                        log.Errorf("ChannelArbitrator(%v): unexpected local "+
2✔
992
                                "commitment confirmed while in StateDefault",
2✔
993
                                c.cfg.ChanPoint)
2✔
994
                        fallthrough
2✔
995
                case remoteCloseTrigger:
8✔
996
                        nextState = StateContractClosed
8✔
997

998
                case breachCloseTrigger:
1✔
999
                        nextContractState, err := c.checkLegacyBreach()
1✔
1000
                        if nextContractState == StateError {
1✔
1001
                                return nextContractState, nil, err
×
1002
                        }
×
1003

1004
                        nextState = nextContractState
1✔
1005
                }
1006

1007
        // If we're in this state, then we've decided to broadcast the
1008
        // commitment transaction. We enter this state either due to an outside
1009
        // sub-system, or because an on-chain action has been triggered.
1010
        case StateBroadcastCommit:
20✔
1011
                // Under normal operation, we can only enter
20✔
1012
                // StateBroadcastCommit via a user or chain trigger. On restart,
20✔
1013
                // this state may be reexecuted after closing the channel, but
20✔
1014
                // failing to commit to StateContractClosed or
20✔
1015
                // StateFullyResolved. In that case, one of the four close
20✔
1016
                // triggers will be presented, signifying that we should skip
20✔
1017
                // rebroadcasting, and go straight to resolving the on-chain
20✔
1018
                // contract or marking the channel resolved.
20✔
1019
                switch trigger {
20✔
1020
                case localCloseTrigger, remoteCloseTrigger:
×
1021
                        log.Infof("ChannelArbitrator(%v): detected %s "+
×
1022
                                "close after closing channel, fast-forwarding "+
×
1023
                                "to %s to resolve contract",
×
1024
                                c.cfg.ChanPoint, trigger, StateContractClosed)
×
1025
                        return StateContractClosed, closeTx, nil
×
1026

1027
                case breachCloseTrigger:
1✔
1028
                        nextContractState, err := c.checkLegacyBreach()
1✔
1029
                        if nextContractState == StateError {
1✔
1030
                                log.Infof("ChannelArbitrator(%v): unable to "+
×
1031
                                        "advance breach close resolution: %v",
×
1032
                                        c.cfg.ChanPoint, nextContractState)
×
1033
                                return StateError, closeTx, err
×
1034
                        }
×
1035

1036
                        log.Infof("ChannelArbitrator(%v): detected %s close "+
1✔
1037
                                "after closing channel, fast-forwarding to %s"+
1✔
1038
                                " to resolve contract", c.cfg.ChanPoint,
1✔
1039
                                trigger, nextContractState)
1✔
1040

1✔
1041
                        return nextContractState, closeTx, nil
1✔
1042

1043
                case coopCloseTrigger:
×
1044
                        log.Infof("ChannelArbitrator(%v): detected %s "+
×
1045
                                "close after closing channel, fast-forwarding "+
×
1046
                                "to %s to resolve contract",
×
1047
                                c.cfg.ChanPoint, trigger, StateFullyResolved)
×
1048
                        return StateFullyResolved, closeTx, nil
×
1049
                }
1050

1051
                log.Infof("ChannelArbitrator(%v): force closing "+
19✔
1052
                        "chan", c.cfg.ChanPoint)
19✔
1053

19✔
1054
                // Now that we have all the actions decided for the set of
19✔
1055
                // HTLC's, we'll broadcast the commitment transaction, and
19✔
1056
                // signal the link to exit.
19✔
1057

19✔
1058
                // We'll tell the switch that it should remove the link for
19✔
1059
                // this channel, in addition to fetching the force close
19✔
1060
                // summary needed to close this channel on chain.
19✔
1061
                closeSummary, err := c.cfg.Channel.ForceCloseChan()
19✔
1062
                if err != nil {
20✔
1063
                        log.Errorf("ChannelArbitrator(%v): unable to "+
1✔
1064
                                "force close: %v", c.cfg.ChanPoint, err)
1✔
1065

1✔
1066
                        // We tried to force close (HTLC may be expiring from
1✔
1067
                        // our PoV, etc), but we think we've lost data. In this
1✔
1068
                        // case, we'll not force close, but terminate the state
1✔
1069
                        // machine here to wait to see what confirms on chain.
1✔
1070
                        if errors.Is(err, lnwallet.ErrForceCloseLocalDataLoss) {
2✔
1071
                                log.Error("ChannelArbitrator(%v): broadcast "+
1✔
1072
                                        "failed due to local data loss, "+
1✔
1073
                                        "waiting for on chain confimation...",
1✔
1074
                                        c.cfg.ChanPoint)
1✔
1075

1✔
1076
                                return StateBroadcastCommit, nil, nil
1✔
1077
                        }
1✔
1078

1079
                        return StateError, closeTx, err
×
1080
                }
1081
                closeTx = closeSummary.CloseTx
18✔
1082

18✔
1083
                // Before publishing the transaction, we store it to the
18✔
1084
                // database, such that we can re-publish later in case it
18✔
1085
                // didn't propagate. We initiated the force close, so we
18✔
1086
                // mark broadcast with local initiator set to true.
18✔
1087
                err = c.cfg.MarkCommitmentBroadcasted(closeTx, lntypes.Local)
18✔
1088
                if err != nil {
18✔
1089
                        log.Errorf("ChannelArbitrator(%v): unable to "+
×
1090
                                "mark commitment broadcasted: %v",
×
1091
                                c.cfg.ChanPoint, err)
×
1092
                        return StateError, closeTx, err
×
1093
                }
×
1094

1095
                // With the close transaction in hand, broadcast the
1096
                // transaction to the network, thereby entering the post
1097
                // channel resolution state.
1098
                log.Infof("Broadcasting force close transaction %v, "+
18✔
1099
                        "ChannelPoint(%v): %v", closeTx.TxHash(),
18✔
1100
                        c.cfg.ChanPoint, lnutils.SpewLogClosure(closeTx))
18✔
1101

18✔
1102
                // At this point, we'll now broadcast the commitment
18✔
1103
                // transaction itself.
18✔
1104
                label := labels.MakeLabel(
18✔
1105
                        labels.LabelTypeChannelClose, &c.cfg.ShortChanID,
18✔
1106
                )
18✔
1107
                if err := c.cfg.PublishTx(closeTx, label); err != nil {
23✔
1108
                        log.Errorf("ChannelArbitrator(%v): unable to broadcast "+
5✔
1109
                                "close tx: %v", c.cfg.ChanPoint, err)
5✔
1110

5✔
1111
                        // This makes sure we don't fail at startup if the
5✔
1112
                        // commitment transaction has too low fees to make it
5✔
1113
                        // into mempool. The rebroadcaster makes sure this
5✔
1114
                        // transaction is republished regularly until confirmed
5✔
1115
                        // or replaced.
5✔
1116
                        if !errors.Is(err, lnwallet.ErrDoubleSpend) &&
5✔
1117
                                !errors.Is(err, lnwallet.ErrMempoolFee) {
7✔
1118

2✔
1119
                                return StateError, closeTx, err
2✔
1120
                        }
2✔
1121
                }
1122

1123
                // We go to the StateCommitmentBroadcasted state, where we'll
1124
                // be waiting for the commitment to be confirmed.
1125
                nextState = StateCommitmentBroadcasted
16✔
1126

1127
        // In this state we have broadcasted our own commitment, and will need
1128
        // to wait for a commitment (not necessarily the one we broadcasted!)
1129
        // to be confirmed.
1130
        case StateCommitmentBroadcasted:
30✔
1131
                switch trigger {
30✔
1132

1133
                // We are waiting for a commitment to be confirmed.
1134
                case chainTrigger, userTrigger:
17✔
1135
                        // The commitment transaction has been broadcast, but it
17✔
1136
                        // doesn't necessarily need to be the commitment
17✔
1137
                        // transaction version that is going to be confirmed. To
17✔
1138
                        // be sure that any of those versions can be anchored
17✔
1139
                        // down, we now submit all anchor resolutions to the
17✔
1140
                        // sweeper. The sweeper will keep trying to sweep all of
17✔
1141
                        // them.
17✔
1142
                        //
17✔
1143
                        // Note that the sweeper is idempotent. If we ever
17✔
1144
                        // happen to end up at this point in the code again, no
17✔
1145
                        // harm is done by re-offering the anchors to the
17✔
1146
                        // sweeper.
17✔
1147
                        anchors, err := c.cfg.Channel.NewAnchorResolutions()
17✔
1148
                        if err != nil {
17✔
1149
                                return StateError, closeTx, err
×
1150
                        }
×
1151

1152
                        err = c.sweepAnchors(anchors, triggerHeight)
17✔
1153
                        if err != nil {
17✔
1154
                                return StateError, closeTx, err
×
1155
                        }
×
1156

1157
                        nextState = StateCommitmentBroadcasted
17✔
1158

1159
                // If this state advance was triggered by any of the
1160
                // commitments being confirmed, then we'll jump to the state
1161
                // where the contract has been closed.
1162
                case localCloseTrigger, remoteCloseTrigger:
13✔
1163
                        nextState = StateContractClosed
13✔
1164

1165
                // If a coop close was confirmed, jump straight to the fully
1166
                // resolved state.
1167
                case coopCloseTrigger:
×
1168
                        nextState = StateFullyResolved
×
1169

1170
                case breachCloseTrigger:
×
1171
                        nextContractState, err := c.checkLegacyBreach()
×
1172
                        if nextContractState == StateError {
×
1173
                                return nextContractState, closeTx, err
×
1174
                        }
×
1175

1176
                        nextState = nextContractState
×
1177
                }
1178

1179
                log.Infof("ChannelArbitrator(%v): trigger %v moving from "+
30✔
1180
                        "state %v to %v", c.cfg.ChanPoint, trigger, c.state,
30✔
1181
                        nextState)
30✔
1182

1183
        // If we're in this state, then the contract has been fully closed to
1184
        // outside sub-systems, so we'll process the prior set of on-chain
1185
        // contract actions and launch a set of resolvers.
1186
        case StateContractClosed:
22✔
1187
                // First, we'll fetch our chain actions, and both sets of
22✔
1188
                // resolutions so we can process them.
22✔
1189
                contractResolutions, err := c.log.FetchContractResolutions()
22✔
1190
                if err != nil {
24✔
1191
                        log.Errorf("unable to fetch contract resolutions: %v",
2✔
1192
                                err)
2✔
1193
                        return StateError, closeTx, err
2✔
1194
                }
2✔
1195

1196
                // If the resolution is empty, and we have no HTLCs at all to
1197
                // send to, then we're done here. We don't need to launch any
1198
                // resolvers, and can go straight to our final state.
1199
                if contractResolutions.IsEmpty() && confCommitSet.IsEmpty() {
28✔
1200
                        log.Infof("ChannelArbitrator(%v): contract "+
8✔
1201
                                "resolutions empty, marking channel as fully resolved!",
8✔
1202
                                c.cfg.ChanPoint)
8✔
1203
                        nextState = StateFullyResolved
8✔
1204
                        break
8✔
1205
                }
1206

1207
                // Now that we know we'll need to act, we'll process all the
1208
                // resolvers, then create the structures we need to resolve all
1209
                // outstanding contracts.
1210
                resolvers, pktsToSend, err := c.prepContractResolutions(
12✔
1211
                        contractResolutions, triggerHeight, trigger,
12✔
1212
                        confCommitSet,
12✔
1213
                )
12✔
1214
                if err != nil {
12✔
1215
                        log.Errorf("ChannelArbitrator(%v): unable to "+
×
1216
                                "resolve contracts: %v", c.cfg.ChanPoint, err)
×
1217
                        return StateError, closeTx, err
×
1218
                }
×
1219

1220
                // With the commitment broadcast, we'll then send over all
1221
                // messages we can send immediately.
1222
                if len(pktsToSend) != 0 {
22✔
1223
                        log.Debugf("ChannelArbitrator(%v): sending "+
10✔
1224
                                "resolution message=%v", c.cfg.ChanPoint,
10✔
1225
                                lnutils.SpewLogClosure(pktsToSend))
10✔
1226

10✔
1227
                        err := c.cfg.DeliverResolutionMsg(pktsToSend...)
10✔
1228
                        if err != nil {
10✔
1229
                                log.Errorf("unable to send pkts: %v", err)
×
1230
                                return StateError, closeTx, err
×
1231
                        }
×
1232
                }
1233

1234
                log.Debugf("ChannelArbitrator(%v): inserting %v contract "+
12✔
1235
                        "resolvers", c.cfg.ChanPoint, len(resolvers))
12✔
1236

12✔
1237
                err = c.log.InsertUnresolvedContracts(nil, resolvers...)
12✔
1238
                if err != nil {
12✔
1239
                        return StateError, closeTx, err
×
1240
                }
×
1241

1242
                // Finally, we'll launch all the required contract resolvers.
1243
                // Once they're all resolved, we're no longer needed.
1244
                c.launchResolvers(resolvers, false)
12✔
1245

12✔
1246
                nextState = StateWaitingFullResolution
12✔
1247

1248
        // This is our terminal state. We'll keep returning this state until
1249
        // all contracts are fully resolved.
1250
        case StateWaitingFullResolution:
17✔
1251
                log.Infof("ChannelArbitrator(%v): still awaiting contract "+
17✔
1252
                        "resolution", c.cfg.ChanPoint)
17✔
1253

17✔
1254
                unresolved, err := c.log.FetchUnresolvedContracts()
17✔
1255
                if err != nil {
17✔
1256
                        return StateError, closeTx, err
×
1257
                }
×
1258

1259
                // If we have no unresolved contracts, then we can move to the
1260
                // final state.
1261
                if len(unresolved) == 0 {
29✔
1262
                        nextState = StateFullyResolved
12✔
1263
                        break
12✔
1264
                }
1265

1266
                // Otherwise we still have unresolved contracts, then we'll
1267
                // stay alive to oversee their resolution.
1268
                nextState = StateWaitingFullResolution
5✔
1269

5✔
1270
                // Add debug logs.
5✔
1271
                for _, r := range unresolved {
10✔
1272
                        log.Debugf("ChannelArbitrator(%v): still have "+
5✔
1273
                                "unresolved contract: %T", c.cfg.ChanPoint, r)
5✔
1274
                }
5✔
1275

1276
        // If we start as fully resolved, then we'll end as fully resolved.
1277
        case StateFullyResolved:
22✔
1278
                // To ensure that the state of the contract in persistent
22✔
1279
                // storage is properly reflected, we'll mark the contract as
22✔
1280
                // fully resolved now.
22✔
1281
                nextState = StateFullyResolved
22✔
1282

22✔
1283
                log.Infof("ChannelPoint(%v) has been fully resolved "+
22✔
1284
                        "on-chain at height=%v", c.cfg.ChanPoint, triggerHeight)
22✔
1285

22✔
1286
                if err := c.cfg.MarkChannelResolved(); err != nil {
22✔
1287
                        log.Errorf("unable to mark channel resolved: %v", err)
×
1288
                        return StateError, closeTx, err
×
1289
                }
×
1290
        }
1291

1292
        log.Tracef("ChannelArbitrator(%v): next_state=%v", c.cfg.ChanPoint,
132✔
1293
                nextState)
132✔
1294

132✔
1295
        return nextState, closeTx, nil
132✔
1296
}
1297

1298
// sweepAnchors offers all given anchor resolutions to the sweeper. It requests
1299
// sweeping at the minimum fee rate. This fee rate can be upped manually by the
1300
// user via the BumpFee rpc.
1301
func (c *ChannelArbitrator) sweepAnchors(anchors *lnwallet.AnchorResolutions,
1302
        heightHint uint32) error {
18✔
1303

18✔
1304
        // Use the chan id as the exclusive group. This prevents any of the
18✔
1305
        // anchors from being batched together.
18✔
1306
        exclusiveGroup := c.cfg.ShortChanID.ToUint64()
18✔
1307

18✔
1308
        // sweepWithDeadline is a helper closure that takes an anchor
18✔
1309
        // resolution and sweeps it with its corresponding deadline.
18✔
1310
        sweepWithDeadline := func(anchor *lnwallet.AnchorResolution,
18✔
1311
                htlcs htlcSet, anchorPath string) error {
23✔
1312

5✔
1313
                // Find the deadline for this specific anchor.
5✔
1314
                deadline, value, err := c.findCommitmentDeadlineAndValue(
5✔
1315
                        heightHint, htlcs,
5✔
1316
                )
5✔
1317
                if err != nil {
5✔
1318
                        return err
×
1319
                }
×
1320

1321
                // If we cannot find a deadline, it means there's no HTLCs at
1322
                // stake, which means we can relax our anchor sweeping
1323
                // conditions as we don't have any time sensitive outputs to
1324
                // sweep. However we need to register the anchor output with the
1325
                // sweeper so we are later able to bump the close fee.
1326
                if deadline.IsNone() {
6✔
1327
                        log.Infof("ChannelArbitrator(%v): no HTLCs at stake, "+
1✔
1328
                                "sweeping anchor with default deadline",
1✔
1329
                                c.cfg.ChanPoint)
1✔
1330
                }
1✔
1331

1332
                witnessType := input.CommitmentAnchor
5✔
1333

5✔
1334
                // For taproot channels, we need to use the proper witness
5✔
1335
                // type.
5✔
1336
                if txscript.IsPayToTaproot(
5✔
1337
                        anchor.AnchorSignDescriptor.Output.PkScript,
5✔
1338
                ) {
5✔
UNCOV
1339

×
UNCOV
1340
                        witnessType = input.TaprootAnchorSweepSpend
×
UNCOV
1341
                }
×
1342

1343
                // Prepare anchor output for sweeping.
1344
                anchorInput := input.MakeBaseInput(
5✔
1345
                        &anchor.CommitAnchor,
5✔
1346
                        witnessType,
5✔
1347
                        &anchor.AnchorSignDescriptor,
5✔
1348
                        heightHint,
5✔
1349
                        &input.TxInfo{
5✔
1350
                                Fee:    anchor.CommitFee,
5✔
1351
                                Weight: anchor.CommitWeight,
5✔
1352
                        },
5✔
1353
                )
5✔
1354

5✔
1355
                // If we have a deadline, we'll use it to calculate the
5✔
1356
                // deadline height, otherwise default to none.
5✔
1357
                deadlineDesc := "None"
5✔
1358
                deadlineHeight := fn.MapOption(func(d int32) int32 {
9✔
1359
                        deadlineDesc = fmt.Sprintf("%d", d)
4✔
1360

4✔
1361
                        return d + int32(heightHint)
4✔
1362
                })(deadline)
4✔
1363

1364
                // Calculate the budget based on the value under protection,
1365
                // which is the sum of all HTLCs on this commitment subtracted
1366
                // by their budgets.
1367
                // The anchor output in itself has a small output value of 330
1368
                // sats so we also include it in the budget to pay for the
1369
                // cpfp transaction.
1370
                budget := calculateBudget(
5✔
1371
                        value, c.cfg.Budget.AnchorCPFPRatio,
5✔
1372
                        c.cfg.Budget.AnchorCPFP,
5✔
1373
                ) + AnchorOutputValue
5✔
1374

5✔
1375
                log.Infof("ChannelArbitrator(%v): offering anchor from %s "+
5✔
1376
                        "commitment %v to sweeper with deadline=%v, budget=%v",
5✔
1377
                        c.cfg.ChanPoint, anchorPath, anchor.CommitAnchor,
5✔
1378
                        deadlineDesc, budget)
5✔
1379

5✔
1380
                // Sweep anchor output with a confirmation target fee
5✔
1381
                // preference. Because this is a cpfp-operation, the anchor
5✔
1382
                // will only be attempted to sweep when the current fee
5✔
1383
                // estimate for the confirmation target exceeds the commit fee
5✔
1384
                // rate.
5✔
1385
                _, err = c.cfg.Sweeper.SweepInput(
5✔
1386
                        &anchorInput,
5✔
1387
                        sweep.Params{
5✔
1388
                                ExclusiveGroup: &exclusiveGroup,
5✔
1389
                                Budget:         budget,
5✔
1390
                                DeadlineHeight: deadlineHeight,
5✔
1391
                        },
5✔
1392
                )
5✔
1393
                if err != nil {
5✔
1394
                        return err
×
1395
                }
×
1396

1397
                return nil
5✔
1398
        }
1399

1400
        // Update the set of activeHTLCs so that the sweeping routine has an
1401
        // up-to-date view of the set of commitments.
1402
        c.updateActiveHTLCs()
18✔
1403

18✔
1404
        // Sweep anchors based on different HTLC sets. Notice the HTLC sets may
18✔
1405
        // differ across commitments, thus their deadline values could vary.
18✔
1406
        for htlcSet, htlcs := range c.activeHTLCs {
59✔
1407
                switch {
41✔
1408
                case htlcSet == LocalHtlcSet && anchors.Local != nil:
2✔
1409
                        err := sweepWithDeadline(anchors.Local, htlcs, "local")
2✔
1410
                        if err != nil {
2✔
1411
                                return err
×
1412
                        }
×
1413

1414
                case htlcSet == RemoteHtlcSet && anchors.Remote != nil:
2✔
1415
                        err := sweepWithDeadline(
2✔
1416
                                anchors.Remote, htlcs, "remote",
2✔
1417
                        )
2✔
1418
                        if err != nil {
2✔
1419
                                return err
×
1420
                        }
×
1421

1422
                case htlcSet == RemotePendingHtlcSet &&
1423
                        anchors.RemotePending != nil:
1✔
1424

1✔
1425
                        err := sweepWithDeadline(
1✔
1426
                                anchors.RemotePending, htlcs, "remote pending",
1✔
1427
                        )
1✔
1428
                        if err != nil {
1✔
1429
                                return err
×
1430
                        }
×
1431
                }
1432
        }
1433

1434
        return nil
18✔
1435
}
1436

1437
// findCommitmentDeadlineAndValue finds the deadline (relative block height)
1438
// for a commitment transaction by extracting the minimum CLTV from its HTLCs.
1439
// From our PoV, the deadline delta is defined to be the smaller of,
1440
//   - half of the least CLTV from outgoing HTLCs' corresponding incoming
1441
//     HTLCs,  or,
1442
//   - half of the least CLTV from incoming HTLCs if the preimage is available.
1443
//
1444
// We use half of the CTLV value to ensure that we have enough time to sweep
1445
// the second-level HTLCs.
1446
//
1447
// It also finds the total value that are time-sensitive, which is the sum of
1448
// all the outgoing HTLCs plus incoming HTLCs whose preimages are known. It
1449
// then returns the value left after subtracting the budget used for sweeping
1450
// the time-sensitive HTLCs.
1451
//
1452
// NOTE: when the deadline turns out to be 0 blocks, we will replace it with 1
1453
// block because our fee estimator doesn't allow a 0 conf target. This also
1454
// means we've left behind and should increase our fee to make the transaction
1455
// confirmed asap.
1456
func (c *ChannelArbitrator) findCommitmentDeadlineAndValue(heightHint uint32,
1457
        htlcs htlcSet) (fn.Option[int32], btcutil.Amount, error) {
10✔
1458

10✔
1459
        deadlineMinHeight := uint32(math.MaxUint32)
10✔
1460
        totalValue := btcutil.Amount(0)
10✔
1461

10✔
1462
        // First, iterate through the outgoingHTLCs to find the lowest CLTV
10✔
1463
        // value.
10✔
1464
        for _, htlc := range htlcs.outgoingHTLCs {
19✔
1465
                // Skip if the HTLC is dust.
9✔
1466
                if htlc.OutputIndex < 0 {
12✔
1467
                        log.Debugf("ChannelArbitrator(%v): skipped deadline "+
3✔
1468
                                "for dust htlc=%x",
3✔
1469
                                c.cfg.ChanPoint, htlc.RHash[:])
3✔
1470

3✔
1471
                        continue
3✔
1472
                }
1473

1474
                value := htlc.Amt.ToSatoshis()
6✔
1475

6✔
1476
                // Find the expiry height for this outgoing HTLC's incoming
6✔
1477
                // HTLC.
6✔
1478
                deadlineOpt := c.cfg.FindOutgoingHTLCDeadline(htlc)
6✔
1479

6✔
1480
                // The deadline is default to the current deadlineMinHeight,
6✔
1481
                // and it's overwritten when it's not none.
6✔
1482
                deadline := deadlineMinHeight
6✔
1483
                deadlineOpt.WhenSome(func(d int32) {
10✔
1484
                        deadline = uint32(d)
4✔
1485

4✔
1486
                        // We only consider the value is under protection when
4✔
1487
                        // it's time-sensitive.
4✔
1488
                        totalValue += value
4✔
1489
                })
4✔
1490

1491
                if deadline < deadlineMinHeight {
10✔
1492
                        deadlineMinHeight = deadline
4✔
1493

4✔
1494
                        log.Tracef("ChannelArbitrator(%v): outgoing HTLC has "+
4✔
1495
                                "deadline=%v, value=%v", c.cfg.ChanPoint,
4✔
1496
                                deadlineMinHeight, value)
4✔
1497
                }
4✔
1498
        }
1499

1500
        // Then going through the incomingHTLCs, and update the minHeight when
1501
        // conditions met.
1502
        for _, htlc := range htlcs.incomingHTLCs {
19✔
1503
                // Skip if the HTLC is dust.
9✔
1504
                if htlc.OutputIndex < 0 {
10✔
1505
                        log.Debugf("ChannelArbitrator(%v): skipped deadline "+
1✔
1506
                                "for dust htlc=%x",
1✔
1507
                                c.cfg.ChanPoint, htlc.RHash[:])
1✔
1508

1✔
1509
                        continue
1✔
1510
                }
1511

1512
                // Since it's an HTLC sent to us, check if we have preimage for
1513
                // this HTLC.
1514
                preimageAvailable, err := c.isPreimageAvailable(htlc.RHash)
8✔
1515
                if err != nil {
8✔
1516
                        return fn.None[int32](), 0, err
×
1517
                }
×
1518

1519
                if !preimageAvailable {
10✔
1520
                        continue
2✔
1521
                }
1522

1523
                value := htlc.Amt.ToSatoshis()
6✔
1524
                totalValue += value
6✔
1525

6✔
1526
                if htlc.RefundTimeout < deadlineMinHeight {
11✔
1527
                        deadlineMinHeight = htlc.RefundTimeout
5✔
1528

5✔
1529
                        log.Tracef("ChannelArbitrator(%v): incoming HTLC has "+
5✔
1530
                                "deadline=%v, amt=%v", c.cfg.ChanPoint,
5✔
1531
                                deadlineMinHeight, value)
5✔
1532
                }
5✔
1533
        }
1534

1535
        // Calculate the deadline. There are two cases to be handled here,
1536
        //   - when the deadlineMinHeight never gets updated, which could
1537
        //     happen when we have no outgoing HTLCs, and, for incoming HTLCs,
1538
        //       * either we have none, or,
1539
        //       * none of the HTLCs are preimageAvailable.
1540
        //   - when our deadlineMinHeight is no greater than the heightHint,
1541
        //     which means we are behind our schedule.
1542
        var deadline uint32
10✔
1543
        switch {
10✔
1544
        // When we couldn't find a deadline height from our HTLCs, we will fall
1545
        // back to the default value as there's no time pressure here.
1546
        case deadlineMinHeight == math.MaxUint32:
2✔
1547
                return fn.None[int32](), 0, nil
2✔
1548

1549
        // When the deadline is passed, we will fall back to the smallest conf
1550
        // target (1 block).
1551
        case deadlineMinHeight <= heightHint:
1✔
1552
                log.Warnf("ChannelArbitrator(%v): deadline is passed with "+
1✔
1553
                        "deadlineMinHeight=%d, heightHint=%d",
1✔
1554
                        c.cfg.ChanPoint, deadlineMinHeight, heightHint)
1✔
1555
                deadline = 1
1✔
1556

1557
        // Use half of the deadline delta, and leave the other half to be used
1558
        // to sweep the HTLCs.
1559
        default:
7✔
1560
                deadline = (deadlineMinHeight - heightHint) / 2
7✔
1561
        }
1562

1563
        // Calculate the value left after subtracting the budget used for
1564
        // sweeping the time-sensitive HTLCs.
1565
        valueLeft := totalValue - calculateBudget(
8✔
1566
                totalValue, c.cfg.Budget.DeadlineHTLCRatio,
8✔
1567
                c.cfg.Budget.DeadlineHTLC,
8✔
1568
        )
8✔
1569

8✔
1570
        log.Debugf("ChannelArbitrator(%v): calculated valueLeft=%v, "+
8✔
1571
                "deadline=%d, using deadlineMinHeight=%d, heightHint=%d",
8✔
1572
                c.cfg.ChanPoint, valueLeft, deadline, deadlineMinHeight,
8✔
1573
                heightHint)
8✔
1574

8✔
1575
        return fn.Some(int32(deadline)), valueLeft, nil
8✔
1576
}
1577

1578
// launchResolvers updates the activeResolvers list and starts the resolvers.
1579
func (c *ChannelArbitrator) launchResolvers(resolvers []ContractResolver,
1580
        immediate bool) {
13✔
1581

13✔
1582
        c.activeResolversLock.Lock()
13✔
1583
        defer c.activeResolversLock.Unlock()
13✔
1584

13✔
1585
        c.activeResolvers = resolvers
13✔
1586
        for _, contract := range resolvers {
19✔
1587
                c.wg.Add(1)
6✔
1588
                go c.resolveContract(contract, immediate)
6✔
1589
        }
6✔
1590
}
1591

1592
// advanceState is the main driver of our state machine. This method is an
1593
// iterative function which repeatedly attempts to advance the internal state
1594
// of the channel arbitrator. The state will be advanced until we reach a
1595
// redundant transition, meaning that the state transition is a noop. The final
1596
// param is a callback that allows the caller to execute an arbitrary action
1597
// after each state transition.
1598
func (c *ChannelArbitrator) advanceState(
1599
        triggerHeight uint32, trigger transitionTrigger,
1600
        confCommitSet *CommitSet) (ArbitratorState, *wire.MsgTx, error) {
89✔
1601

89✔
1602
        var (
89✔
1603
                priorState   ArbitratorState
89✔
1604
                forceCloseTx *wire.MsgTx
89✔
1605
        )
89✔
1606

89✔
1607
        // We'll continue to advance our state forward until the state we
89✔
1608
        // transition to is that same state that we started at.
89✔
1609
        for {
264✔
1610
                priorState = c.state
175✔
1611
                log.Debugf("ChannelArbitrator(%v): attempting state step with "+
175✔
1612
                        "trigger=%v from state=%v", c.cfg.ChanPoint, trigger,
175✔
1613
                        priorState)
175✔
1614

175✔
1615
                nextState, closeTx, err := c.stateStep(
175✔
1616
                        triggerHeight, trigger, confCommitSet,
175✔
1617
                )
175✔
1618
                if err != nil {
179✔
1619
                        log.Errorf("ChannelArbitrator(%v): unable to advance "+
4✔
1620
                                "state: %v", c.cfg.ChanPoint, err)
4✔
1621
                        return priorState, nil, err
4✔
1622
                }
4✔
1623

1624
                if forceCloseTx == nil && closeTx != nil {
187✔
1625
                        forceCloseTx = closeTx
16✔
1626
                }
16✔
1627

1628
                // Our termination transition is a noop transition. If we get
1629
                // our prior state back as the next state, then we'll
1630
                // terminate.
1631
                if nextState == priorState {
253✔
1632
                        log.Debugf("ChannelArbitrator(%v): terminating at "+
82✔
1633
                                "state=%v", c.cfg.ChanPoint, nextState)
82✔
1634
                        return nextState, forceCloseTx, nil
82✔
1635
                }
82✔
1636

1637
                // As the prior state was successfully executed, we can now
1638
                // commit the next state. This ensures that we will re-execute
1639
                // the prior state if anything fails.
1640
                if err := c.log.CommitState(nextState); err != nil {
92✔
1641
                        log.Errorf("ChannelArbitrator(%v): unable to commit "+
3✔
1642
                                "next state(%v): %v", c.cfg.ChanPoint,
3✔
1643
                                nextState, err)
3✔
1644
                        return priorState, nil, err
3✔
1645
                }
3✔
1646
                c.state = nextState
86✔
1647
        }
1648
}
1649

1650
// ChainAction is an enum that encompasses all possible on-chain actions
1651
// we'll take for a set of HTLC's.
1652
type ChainAction uint8
1653

1654
const (
1655
        // NoAction is the min chainAction type, indicating that no action
1656
        // needs to be taken for a given HTLC.
1657
        NoAction ChainAction = 0
1658

1659
        // HtlcTimeoutAction indicates that the HTLC will timeout soon. As a
1660
        // result, we should get ready to sweep it on chain after the timeout.
1661
        HtlcTimeoutAction = 1
1662

1663
        // HtlcClaimAction indicates that we should claim the HTLC on chain
1664
        // before its timeout period.
1665
        HtlcClaimAction = 2
1666

1667
        // HtlcFailNowAction indicates that we should fail an outgoing HTLC
1668
        // immediately by cancelling it backwards as it has no corresponding
1669
        // output in our commitment transaction.
1670
        HtlcFailNowAction = 3
1671

1672
        // HtlcOutgoingWatchAction indicates that we can't yet timeout this
1673
        // HTLC, but we had to go to chain on order to resolve an existing
1674
        // HTLC.  In this case, we'll either: time it out once it expires, or
1675
        // will learn the pre-image if the remote party claims the output. In
1676
        // this case, well add the pre-image to our global store.
1677
        HtlcOutgoingWatchAction = 4
1678

1679
        // HtlcIncomingWatchAction indicates that we don't yet have the
1680
        // pre-image to claim incoming HTLC, but we had to go to chain in order
1681
        // to resolve and existing HTLC. In this case, we'll either: let the
1682
        // other party time it out, or eventually learn of the pre-image, in
1683
        // which case we'll claim on chain.
1684
        HtlcIncomingWatchAction = 5
1685

1686
        // HtlcIncomingDustFinalAction indicates that we should mark an incoming
1687
        // dust htlc as final because it can't be claimed on-chain.
1688
        HtlcIncomingDustFinalAction = 6
1689
)
1690

1691
// String returns a human readable string describing a chain action.
1692
func (c ChainAction) String() string {
×
1693
        switch c {
×
1694
        case NoAction:
×
1695
                return "NoAction"
×
1696

1697
        case HtlcTimeoutAction:
×
1698
                return "HtlcTimeoutAction"
×
1699

1700
        case HtlcClaimAction:
×
1701
                return "HtlcClaimAction"
×
1702

1703
        case HtlcFailNowAction:
×
1704
                return "HtlcFailNowAction"
×
1705

1706
        case HtlcOutgoingWatchAction:
×
1707
                return "HtlcOutgoingWatchAction"
×
1708

1709
        case HtlcIncomingWatchAction:
×
1710
                return "HtlcIncomingWatchAction"
×
1711

1712
        case HtlcIncomingDustFinalAction:
×
1713
                return "HtlcIncomingDustFinalAction"
×
1714

1715
        default:
×
1716
                return "<unknown action>"
×
1717
        }
1718
}
1719

1720
// ChainActionMap is a map of a chain action, to the set of HTLC's that need to
1721
// be acted upon for a given action type. The channel
1722
type ChainActionMap map[ChainAction][]channeldb.HTLC
1723

1724
// Merge merges the passed chain actions with the target chain action map.
1725
func (c ChainActionMap) Merge(actions ChainActionMap) {
76✔
1726
        for chainAction, htlcs := range actions {
89✔
1727
                c[chainAction] = append(c[chainAction], htlcs...)
13✔
1728
        }
13✔
1729
}
1730

1731
// shouldGoOnChain takes into account the absolute timeout of the HTLC, if the
1732
// confirmation delta that we need is close, and returns a bool indicating if
1733
// we should go on chain to claim.  We do this rather than waiting up until the
1734
// last minute as we want to ensure that when we *need* (HTLC is timed out) to
1735
// sweep, the commitment is already confirmed.
1736
func (c *ChannelArbitrator) shouldGoOnChain(htlc channeldb.HTLC,
1737
        broadcastDelta, currentHeight uint32) bool {
27✔
1738

27✔
1739
        // We'll calculate the broadcast cut off for this HTLC. This is the
27✔
1740
        // height that (based on our current fee estimation) we should
27✔
1741
        // broadcast in order to ensure the commitment transaction is confirmed
27✔
1742
        // before the HTLC fully expires.
27✔
1743
        broadcastCutOff := htlc.RefundTimeout - broadcastDelta
27✔
1744

27✔
1745
        log.Tracef("ChannelArbitrator(%v): examining outgoing contract: "+
27✔
1746
                "expiry=%v, cutoff=%v, height=%v", c.cfg.ChanPoint, htlc.RefundTimeout,
27✔
1747
                broadcastCutOff, currentHeight)
27✔
1748

27✔
1749
        // TODO(roasbeef): take into account default HTLC delta, don't need to
27✔
1750
        // broadcast immediately
27✔
1751
        //  * can then batch with SINGLE | ANYONECANPAY
27✔
1752

27✔
1753
        // We should on-chain for this HTLC, iff we're within out broadcast
27✔
1754
        // cutoff window.
27✔
1755
        if currentHeight < broadcastCutOff {
47✔
1756
                return false
20✔
1757
        }
20✔
1758

1759
        // In case of incoming htlc we should go to chain.
1760
        if htlc.Incoming {
7✔
UNCOV
1761
                return true
×
UNCOV
1762
        }
×
1763

1764
        // For htlcs that are result of our initiated payments we give some grace
1765
        // period before force closing the channel. During this time we expect
1766
        // both nodes to connect and give a chance to the other node to send its
1767
        // updates and cancel the htlc.
1768
        // This shouldn't add any security risk as there is no incoming htlc to
1769
        // fulfill at this case and the expectation is that when the channel is
1770
        // active the other node will send update_fail_htlc to remove the htlc
1771
        // without closing the channel. It is up to the user to force close the
1772
        // channel if the peer misbehaves and doesn't send the update_fail_htlc.
1773
        // It is useful when this node is most of the time not online and is
1774
        // likely to miss the time slot where the htlc may be cancelled.
1775
        isForwarded := c.cfg.IsForwardedHTLC(c.cfg.ShortChanID, htlc.HtlcIndex)
7✔
1776
        upTime := c.cfg.Clock.Now().Sub(c.startTimestamp)
7✔
1777
        return isForwarded || upTime > c.cfg.PaymentsExpirationGracePeriod
7✔
1778
}
1779

1780
// checkCommitChainActions is called for each new block connected to the end of
1781
// the main chain. Given the new block height, this new method will examine all
1782
// active HTLC's, and determine if we need to go on-chain to claim any of them.
1783
// A map of action -> []htlc is returned, detailing what action (if any) should
1784
// be performed for each HTLC. For timed out HTLC's, once the commitment has
1785
// been sufficiently confirmed, the HTLC's should be canceled backwards. For
1786
// redeemed HTLC's, we should send the pre-image back to the incoming link.
1787
func (c *ChannelArbitrator) checkCommitChainActions(height uint32,
1788
        trigger transitionTrigger, htlcs htlcSet) (ChainActionMap, error) {
76✔
1789

76✔
1790
        // TODO(roasbeef): would need to lock channel? channel totem?
76✔
1791
        //  * race condition if adding and we broadcast, etc
76✔
1792
        //  * or would make each instance sync?
76✔
1793

76✔
1794
        log.Debugf("ChannelArbitrator(%v): checking commit chain actions at "+
76✔
1795
                "height=%v, in_htlc_count=%v, out_htlc_count=%v",
76✔
1796
                c.cfg.ChanPoint, height,
76✔
1797
                len(htlcs.incomingHTLCs), len(htlcs.outgoingHTLCs))
76✔
1798

76✔
1799
        actionMap := make(ChainActionMap)
76✔
1800

76✔
1801
        // First, we'll make an initial pass over the set of incoming and
76✔
1802
        // outgoing HTLC's to decide if we need to go on chain at all.
76✔
1803
        haveChainActions := false
76✔
1804
        for _, htlc := range htlcs.outgoingHTLCs {
83✔
1805
                // We'll need to go on-chain for an outgoing HTLC if it was
7✔
1806
                // never resolved downstream, and it's "close" to timing out.
7✔
1807
                //
7✔
1808
                // TODO(yy): If there's no corresponding incoming HTLC, it
7✔
1809
                // means we are the first hop, hence the payer. This is a
7✔
1810
                // tricky case - unlike a forwarding hop, we don't have an
7✔
1811
                // incoming HTLC that will time out, which means as long as we
7✔
1812
                // can learn the preimage, we can settle the invoice (before it
7✔
1813
                // expires?).
7✔
1814
                toChain := c.shouldGoOnChain(
7✔
1815
                        htlc, c.cfg.OutgoingBroadcastDelta, height,
7✔
1816
                )
7✔
1817

7✔
1818
                if toChain {
7✔
UNCOV
1819
                        // Convert to int64 in case of overflow.
×
UNCOV
1820
                        remainingBlocks := int64(htlc.RefundTimeout) -
×
UNCOV
1821
                                int64(height)
×
UNCOV
1822

×
UNCOV
1823
                        log.Infof("ChannelArbitrator(%v): go to chain for "+
×
UNCOV
1824
                                "outgoing htlc %x: timeout=%v, amount=%v, "+
×
UNCOV
1825
                                "blocks_until_expiry=%v, broadcast_delta=%v",
×
UNCOV
1826
                                c.cfg.ChanPoint, htlc.RHash[:],
×
UNCOV
1827
                                htlc.RefundTimeout, htlc.Amt, remainingBlocks,
×
UNCOV
1828
                                c.cfg.OutgoingBroadcastDelta,
×
UNCOV
1829
                        )
×
UNCOV
1830
                }
×
1831

1832
                haveChainActions = haveChainActions || toChain
7✔
1833
        }
1834

1835
        for _, htlc := range htlcs.incomingHTLCs {
81✔
1836
                // We'll need to go on-chain to pull an incoming HTLC iff we
5✔
1837
                // know the pre-image and it's close to timing out. We need to
5✔
1838
                // ensure that we claim the funds that are rightfully ours
5✔
1839
                // on-chain.
5✔
1840
                preimageAvailable, err := c.isPreimageAvailable(htlc.RHash)
5✔
1841
                if err != nil {
5✔
1842
                        return nil, err
×
1843
                }
×
1844

1845
                if !preimageAvailable {
8✔
1846
                        continue
3✔
1847
                }
1848

1849
                toChain := c.shouldGoOnChain(
2✔
1850
                        htlc, c.cfg.IncomingBroadcastDelta, height,
2✔
1851
                )
2✔
1852

2✔
1853
                if toChain {
2✔
UNCOV
1854
                        // Convert to int64 in case of overflow.
×
UNCOV
1855
                        remainingBlocks := int64(htlc.RefundTimeout) -
×
UNCOV
1856
                                int64(height)
×
UNCOV
1857

×
UNCOV
1858
                        log.Infof("ChannelArbitrator(%v): go to chain for "+
×
UNCOV
1859
                                "incoming htlc %x: timeout=%v, amount=%v, "+
×
UNCOV
1860
                                "blocks_until_expiry=%v, broadcast_delta=%v",
×
UNCOV
1861
                                c.cfg.ChanPoint, htlc.RHash[:],
×
UNCOV
1862
                                htlc.RefundTimeout, htlc.Amt, remainingBlocks,
×
UNCOV
1863
                                c.cfg.IncomingBroadcastDelta,
×
UNCOV
1864
                        )
×
UNCOV
1865
                }
×
1866

1867
                haveChainActions = haveChainActions || toChain
2✔
1868
        }
1869

1870
        // If we don't have any actions to make, then we'll return an empty
1871
        // action map. We only do this if this was a chain trigger though, as
1872
        // if we're going to broadcast the commitment (or the remote party did)
1873
        // we're *forced* to act on each HTLC.
1874
        if !haveChainActions && trigger == chainTrigger {
118✔
1875
                log.Tracef("ChannelArbitrator(%v): no actions to take at "+
42✔
1876
                        "height=%v", c.cfg.ChanPoint, height)
42✔
1877
                return actionMap, nil
42✔
1878
        }
42✔
1879

1880
        // Now that we know we'll need to go on-chain, we'll examine all of our
1881
        // active outgoing HTLC's to see if we either need to: sweep them after
1882
        // a timeout (then cancel backwards), cancel them backwards
1883
        // immediately, or watch them as they're still active contracts.
1884
        for _, htlc := range htlcs.outgoingHTLCs {
40✔
1885
                switch {
6✔
1886
                // If the HTLC is dust, then we can cancel it backwards
1887
                // immediately as there's no matching contract to arbitrate
1888
                // on-chain. We know the HTLC is dust, if the OutputIndex
1889
                // negative.
1890
                case htlc.OutputIndex < 0:
2✔
1891
                        log.Tracef("ChannelArbitrator(%v): immediately "+
2✔
1892
                                "failing dust htlc=%x", c.cfg.ChanPoint,
2✔
1893
                                htlc.RHash[:])
2✔
1894

2✔
1895
                        actionMap[HtlcFailNowAction] = append(
2✔
1896
                                actionMap[HtlcFailNowAction], htlc,
2✔
1897
                        )
2✔
1898

1899
                // If we don't need to immediately act on this HTLC, then we'll
1900
                // mark it still "live". After we broadcast, we'll monitor it
1901
                // until the HTLC times out to see if we can also redeem it
1902
                // on-chain.
1903
                case !c.shouldGoOnChain(htlc, c.cfg.OutgoingBroadcastDelta,
1904
                        height,
1905
                ):
4✔
1906
                        // TODO(roasbeef): also need to be able to query
4✔
1907
                        // circuit map to see if HTLC hasn't been fully
4✔
1908
                        // resolved
4✔
1909
                        //
4✔
1910
                        //  * can't fail incoming until if outgoing not yet
4✔
1911
                        //  failed
4✔
1912

4✔
1913
                        log.Tracef("ChannelArbitrator(%v): watching chain to "+
4✔
1914
                                "decide action for outgoing htlc=%x",
4✔
1915
                                c.cfg.ChanPoint, htlc.RHash[:])
4✔
1916

4✔
1917
                        actionMap[HtlcOutgoingWatchAction] = append(
4✔
1918
                                actionMap[HtlcOutgoingWatchAction], htlc,
4✔
1919
                        )
4✔
1920

1921
                // Otherwise, we'll update our actionMap to mark that we need
1922
                // to sweep this HTLC on-chain
UNCOV
1923
                default:
×
UNCOV
1924
                        log.Tracef("ChannelArbitrator(%v): going on-chain to "+
×
UNCOV
1925
                                "timeout htlc=%x", c.cfg.ChanPoint, htlc.RHash[:])
×
UNCOV
1926

×
UNCOV
1927
                        actionMap[HtlcTimeoutAction] = append(
×
UNCOV
1928
                                actionMap[HtlcTimeoutAction], htlc,
×
UNCOV
1929
                        )
×
1930
                }
1931
        }
1932

1933
        // Similarly, for each incoming HTLC, now that we need to go on-chain,
1934
        // we'll either: sweep it immediately if we know the pre-image, or
1935
        // observe the output on-chain if we don't In this last, case we'll
1936
        // either learn of it eventually from the outgoing HTLC, or the sender
1937
        // will timeout the HTLC.
1938
        for _, htlc := range htlcs.incomingHTLCs {
38✔
1939
                // If the HTLC is dust, there is no action to be taken.
4✔
1940
                if htlc.OutputIndex < 0 {
6✔
1941
                        log.Debugf("ChannelArbitrator(%v): no resolution "+
2✔
1942
                                "needed for incoming dust htlc=%x",
2✔
1943
                                c.cfg.ChanPoint, htlc.RHash[:])
2✔
1944

2✔
1945
                        actionMap[HtlcIncomingDustFinalAction] = append(
2✔
1946
                                actionMap[HtlcIncomingDustFinalAction], htlc,
2✔
1947
                        )
2✔
1948

2✔
1949
                        continue
2✔
1950
                }
1951

1952
                log.Tracef("ChannelArbitrator(%v): watching chain to decide "+
2✔
1953
                        "action for incoming htlc=%x", c.cfg.ChanPoint,
2✔
1954
                        htlc.RHash[:])
2✔
1955

2✔
1956
                actionMap[HtlcIncomingWatchAction] = append(
2✔
1957
                        actionMap[HtlcIncomingWatchAction], htlc,
2✔
1958
                )
2✔
1959
        }
1960

1961
        return actionMap, nil
34✔
1962
}
1963

1964
// isPreimageAvailable returns whether the hash preimage is available in either
1965
// the preimage cache or the invoice database.
1966
func (c *ChannelArbitrator) isPreimageAvailable(hash lntypes.Hash) (bool,
1967
        error) {
17✔
1968

17✔
1969
        // Start by checking the preimage cache for preimages of
17✔
1970
        // forwarded HTLCs.
17✔
1971
        _, preimageAvailable := c.cfg.PreimageDB.LookupPreimage(
17✔
1972
                hash,
17✔
1973
        )
17✔
1974
        if preimageAvailable {
25✔
1975
                return true, nil
8✔
1976
        }
8✔
1977

1978
        // Then check if we have an invoice that can be settled by this HTLC.
1979
        //
1980
        // TODO(joostjager): Check that there are still more blocks remaining
1981
        // than the invoice cltv delta. We don't want to go to chain only to
1982
        // have the incoming contest resolver decide that we don't want to
1983
        // settle this invoice.
1984
        invoice, err := c.cfg.Registry.LookupInvoice(context.Background(), hash)
9✔
1985
        switch {
9✔
UNCOV
1986
        case err == nil:
×
1987
        case errors.Is(err, invoices.ErrInvoiceNotFound) ||
1988
                errors.Is(err, invoices.ErrNoInvoicesCreated):
9✔
1989

9✔
1990
                return false, nil
9✔
1991
        default:
×
1992
                return false, err
×
1993
        }
1994

UNCOV
1995
        preimageAvailable = invoice.Terms.PaymentPreimage != nil
×
UNCOV
1996

×
UNCOV
1997
        return preimageAvailable, nil
×
1998
}
1999

2000
// checkLocalChainActions is similar to checkCommitChainActions, but it also
2001
// examines the set of HTLCs on the remote party's commitment. This allows us
2002
// to ensure we're able to satisfy the HTLC timeout constraints for incoming vs
2003
// outgoing HTLCs.
2004
func (c *ChannelArbitrator) checkLocalChainActions(
2005
        height uint32, trigger transitionTrigger,
2006
        activeHTLCs map[HtlcSetKey]htlcSet,
2007
        commitsConfirmed bool) (ChainActionMap, error) {
70✔
2008

70✔
2009
        // First, we'll check our local chain actions as normal. This will only
70✔
2010
        // examine HTLCs on our local commitment (timeout or settle).
70✔
2011
        localCommitActions, err := c.checkCommitChainActions(
70✔
2012
                height, trigger, activeHTLCs[LocalHtlcSet],
70✔
2013
        )
70✔
2014
        if err != nil {
70✔
2015
                return nil, err
×
2016
        }
×
2017

2018
        // Next, we'll examine the remote commitment (and maybe a dangling one)
2019
        // to see if the set difference of our HTLCs is non-empty. If so, then
2020
        // we may need to cancel back some HTLCs if we decide go to chain.
2021
        remoteDanglingActions := c.checkRemoteDanglingActions(
70✔
2022
                height, activeHTLCs, commitsConfirmed,
70✔
2023
        )
70✔
2024

70✔
2025
        // Finally, we'll merge the two set of chain actions.
70✔
2026
        localCommitActions.Merge(remoteDanglingActions)
70✔
2027

70✔
2028
        return localCommitActions, nil
70✔
2029
}
2030

2031
// checkRemoteDanglingActions examines the set of remote commitments for any
2032
// HTLCs that are close to timing out. If we find any, then we'll return a set
2033
// of chain actions for HTLCs that are on our commitment, but not theirs to
2034
// cancel immediately.
2035
func (c *ChannelArbitrator) checkRemoteDanglingActions(
2036
        height uint32, activeHTLCs map[HtlcSetKey]htlcSet,
2037
        commitsConfirmed bool) ChainActionMap {
70✔
2038

70✔
2039
        var (
70✔
2040
                pendingRemoteHTLCs []channeldb.HTLC
70✔
2041
                localHTLCs         = make(map[uint64]struct{})
70✔
2042
                remoteHTLCs        = make(map[uint64]channeldb.HTLC)
70✔
2043
                actionMap          = make(ChainActionMap)
70✔
2044
        )
70✔
2045

70✔
2046
        // First, we'll construct two sets of the outgoing HTLCs: those on our
70✔
2047
        // local commitment, and those that are on the remote commitment(s).
70✔
2048
        for htlcSetKey, htlcs := range activeHTLCs {
190✔
2049
                if htlcSetKey.IsRemote {
184✔
2050
                        for _, htlc := range htlcs.outgoingHTLCs {
80✔
2051
                                remoteHTLCs[htlc.HtlcIndex] = htlc
16✔
2052
                        }
16✔
2053
                } else {
56✔
2054
                        for _, htlc := range htlcs.outgoingHTLCs {
62✔
2055
                                localHTLCs[htlc.HtlcIndex] = struct{}{}
6✔
2056
                        }
6✔
2057
                }
2058
        }
2059

2060
        // With both sets constructed, we'll now compute the set difference of
2061
        // our two sets of HTLCs. This'll give us the HTLCs that exist on the
2062
        // remote commitment transaction, but not on ours.
2063
        for htlcIndex, htlc := range remoteHTLCs {
86✔
2064
                if _, ok := localHTLCs[htlcIndex]; ok {
18✔
2065
                        continue
2✔
2066
                }
2067

2068
                pendingRemoteHTLCs = append(pendingRemoteHTLCs, htlc)
14✔
2069
        }
2070

2071
        // Finally, we'll examine all the pending remote HTLCs for those that
2072
        // have expired. If we find any, then we'll recommend that they be
2073
        // failed now so we can free up the incoming HTLC.
2074
        for _, htlc := range pendingRemoteHTLCs {
84✔
2075
                // We'll now check if we need to go to chain in order to cancel
14✔
2076
                // the incoming HTLC.
14✔
2077
                goToChain := c.shouldGoOnChain(htlc, c.cfg.OutgoingBroadcastDelta,
14✔
2078
                        height,
14✔
2079
                )
14✔
2080

14✔
2081
                // If we don't need to go to chain, and no commitments have
14✔
2082
                // been confirmed, then we can move on. Otherwise, if
14✔
2083
                // commitments have been confirmed, then we need to cancel back
14✔
2084
                // *all* of the pending remote HTLCS.
14✔
2085
                if !goToChain && !commitsConfirmed {
19✔
2086
                        continue
5✔
2087
                }
2088

2089
                log.Infof("ChannelArbitrator(%v): fail dangling htlc=%x from "+
9✔
2090
                        "local/remote commitments diff",
9✔
2091
                        c.cfg.ChanPoint, htlc.RHash[:])
9✔
2092

9✔
2093
                actionMap[HtlcFailNowAction] = append(
9✔
2094
                        actionMap[HtlcFailNowAction], htlc,
9✔
2095
                )
9✔
2096
        }
2097

2098
        return actionMap
70✔
2099
}
2100

2101
// checkRemoteChainActions examines the two possible remote commitment chains
2102
// and returns the set of chain actions we need to carry out if the remote
2103
// commitment (non pending) confirms. The pendingConf indicates if the pending
2104
// remote commitment confirmed. This is similar to checkCommitChainActions, but
2105
// we'll immediately fail any HTLCs on the pending remote commit, but not the
2106
// remote commit (or the other way around).
2107
func (c *ChannelArbitrator) checkRemoteChainActions(
2108
        height uint32, trigger transitionTrigger,
2109
        activeHTLCs map[HtlcSetKey]htlcSet,
2110
        pendingConf bool) (ChainActionMap, error) {
6✔
2111

6✔
2112
        // First, we'll examine all the normal chain actions on the remote
6✔
2113
        // commitment that confirmed.
6✔
2114
        confHTLCs := activeHTLCs[RemoteHtlcSet]
6✔
2115
        if pendingConf {
8✔
2116
                confHTLCs = activeHTLCs[RemotePendingHtlcSet]
2✔
2117
        }
2✔
2118
        remoteCommitActions, err := c.checkCommitChainActions(
6✔
2119
                height, trigger, confHTLCs,
6✔
2120
        )
6✔
2121
        if err != nil {
6✔
2122
                return nil, err
×
2123
        }
×
2124

2125
        // With these actions computed, we'll now check the diff of the HTLCs on
2126
        // the commitments, and cancel back any that are on the pending but not
2127
        // the non-pending.
2128
        remoteDiffActions := c.checkRemoteDiffActions(
6✔
2129
                activeHTLCs, pendingConf,
6✔
2130
        )
6✔
2131

6✔
2132
        // Finally, we'll merge all the chain actions and the final set of
6✔
2133
        // chain actions.
6✔
2134
        remoteCommitActions.Merge(remoteDiffActions)
6✔
2135
        return remoteCommitActions, nil
6✔
2136
}
2137

2138
// checkRemoteDiffActions checks the set difference of the HTLCs on the remote
2139
// confirmed commit and remote dangling commit for HTLCS that we need to cancel
2140
// back. If we find any HTLCs on the remote pending but not the remote, then
2141
// we'll mark them to be failed immediately.
2142
func (c *ChannelArbitrator) checkRemoteDiffActions(
2143
        activeHTLCs map[HtlcSetKey]htlcSet,
2144
        pendingConf bool) ChainActionMap {
6✔
2145

6✔
2146
        // First, we'll partition the HTLCs into those that are present on the
6✔
2147
        // confirmed commitment, and those on the dangling commitment.
6✔
2148
        confHTLCs := activeHTLCs[RemoteHtlcSet]
6✔
2149
        danglingHTLCs := activeHTLCs[RemotePendingHtlcSet]
6✔
2150
        if pendingConf {
8✔
2151
                confHTLCs = activeHTLCs[RemotePendingHtlcSet]
2✔
2152
                danglingHTLCs = activeHTLCs[RemoteHtlcSet]
2✔
2153
        }
2✔
2154

2155
        // Next, we'll create a set of all the HTLCs confirmed commitment.
2156
        remoteHtlcs := make(map[uint64]struct{})
6✔
2157
        for _, htlc := range confHTLCs.outgoingHTLCs {
7✔
2158
                remoteHtlcs[htlc.HtlcIndex] = struct{}{}
1✔
2159
        }
1✔
2160

2161
        // With the remote HTLCs assembled, we'll mark any HTLCs only on the
2162
        // remote dangling commitment to be failed asap.
2163
        actionMap := make(ChainActionMap)
6✔
2164
        for _, htlc := range danglingHTLCs.outgoingHTLCs {
10✔
2165
                if _, ok := remoteHtlcs[htlc.HtlcIndex]; ok {
4✔
UNCOV
2166
                        continue
×
2167
                }
2168

2169
                preimageAvailable, err := c.isPreimageAvailable(htlc.RHash)
4✔
2170
                if err != nil {
4✔
2171
                        log.Errorf("ChannelArbitrator(%v): failed to query "+
×
2172
                                "preimage for dangling htlc=%x from remote "+
×
2173
                                "commitments diff", c.cfg.ChanPoint,
×
2174
                                htlc.RHash[:])
×
2175

×
2176
                        continue
×
2177
                }
2178

2179
                if preimageAvailable {
4✔
2180
                        continue
×
2181
                }
2182

2183
                actionMap[HtlcFailNowAction] = append(
4✔
2184
                        actionMap[HtlcFailNowAction], htlc,
4✔
2185
                )
4✔
2186

4✔
2187
                log.Infof("ChannelArbitrator(%v): fail dangling htlc=%x from "+
4✔
2188
                        "remote commitments diff",
4✔
2189
                        c.cfg.ChanPoint, htlc.RHash[:])
4✔
2190
        }
2191

2192
        return actionMap
6✔
2193
}
2194

2195
// constructChainActions returns the set of actions that should be taken for
2196
// confirmed HTLCs at the specified height. Our actions will depend on the set
2197
// of HTLCs that were active across all channels at the time of channel
2198
// closure.
2199
func (c *ChannelArbitrator) constructChainActions(confCommitSet *CommitSet,
2200
        height uint32, trigger transitionTrigger) (ChainActionMap, error) {
12✔
2201

12✔
2202
        // If we've reached this point and have not confirmed commitment set,
12✔
2203
        // then this is an older node that had a pending close channel before
12✔
2204
        // the CommitSet was introduced. In this case, we'll just return the
12✔
2205
        // existing ChainActionMap they had on disk.
12✔
2206
        if confCommitSet == nil {
12✔
2207
                return c.log.FetchChainActions()
×
2208
        }
×
2209

2210
        // Otherwise, we have the full commitment set written to disk, and can
2211
        // proceed as normal.
2212
        htlcSets := confCommitSet.toActiveHTLCSets()
12✔
2213
        switch *confCommitSet.ConfCommitKey {
12✔
2214

2215
        // If the local commitment transaction confirmed, then we'll examine
2216
        // that as well as their commitments to the set of chain actions.
2217
        case LocalHtlcSet:
6✔
2218
                return c.checkLocalChainActions(
6✔
2219
                        height, trigger, htlcSets, true,
6✔
2220
                )
6✔
2221

2222
        // If the remote commitment confirmed, then we'll grab all the chain
2223
        // actions for the remote commit, and check the pending commit for any
2224
        // HTLCS we need to handle immediately (dust).
2225
        case RemoteHtlcSet:
4✔
2226
                return c.checkRemoteChainActions(
4✔
2227
                        height, trigger, htlcSets, false,
4✔
2228
                )
4✔
2229

2230
        // Otherwise, the remote pending commitment confirmed, so we'll examine
2231
        // the HTLCs on that unrevoked dangling commitment.
2232
        case RemotePendingHtlcSet:
2✔
2233
                return c.checkRemoteChainActions(
2✔
2234
                        height, trigger, htlcSets, true,
2✔
2235
                )
2✔
2236
        }
2237

2238
        return nil, fmt.Errorf("unable to locate chain actions")
×
2239
}
2240

2241
// prepContractResolutions is called either in the case that we decide we need
2242
// to go to chain, or the remote party goes to chain. Given a set of actions we
2243
// need to take for each HTLC, this method will return a set of contract
2244
// resolvers that will resolve the contracts on-chain if needed, and also a set
2245
// of packets to send to the htlcswitch in order to ensure all incoming HTLC's
2246
// are properly resolved.
2247
func (c *ChannelArbitrator) prepContractResolutions(
2248
        contractResolutions *ContractResolutions, height uint32,
2249
        trigger transitionTrigger,
2250
        confCommitSet *CommitSet) ([]ContractResolver, []ResolutionMsg, error) {
12✔
2251

12✔
2252
        // First, we'll reconstruct a fresh set of chain actions as the set of
12✔
2253
        // actions we need to act on may differ based on if it was our
12✔
2254
        // commitment, or they're commitment that hit the chain.
12✔
2255
        htlcActions, err := c.constructChainActions(
12✔
2256
                confCommitSet, height, trigger,
12✔
2257
        )
12✔
2258
        if err != nil {
12✔
2259
                return nil, nil, err
×
2260
        }
×
2261

2262
        // We'll also fetch the historical state of this channel, as it should
2263
        // have been marked as closed by now, and supplement it to each resolver
2264
        // such that we can properly resolve our pending contracts.
2265
        var chanState *channeldb.OpenChannel
12✔
2266
        chanState, err = c.cfg.FetchHistoricalChannel()
12✔
2267
        switch {
12✔
2268
        // If we don't find this channel, then it may be the case that it
2269
        // was closed before we started to retain the final state
2270
        // information for open channels.
2271
        case err == channeldb.ErrNoHistoricalBucket:
×
2272
                fallthrough
×
2273
        case err == channeldb.ErrChannelNotFound:
×
2274
                log.Warnf("ChannelArbitrator(%v): unable to fetch historical "+
×
2275
                        "state", c.cfg.ChanPoint)
×
2276

2277
        case err != nil:
×
2278
                return nil, nil, err
×
2279
        }
2280

2281
        // There may be a class of HTLC's which we can fail back immediately,
2282
        // for those we'll prepare a slice of packets to add to our outbox. Any
2283
        // packets we need to send, will be cancels.
2284
        var (
12✔
2285
                msgsToSend []ResolutionMsg
12✔
2286
        )
12✔
2287

12✔
2288
        incomingResolutions := contractResolutions.HtlcResolutions.IncomingHTLCs
12✔
2289
        outgoingResolutions := contractResolutions.HtlcResolutions.OutgoingHTLCs
12✔
2290

12✔
2291
        // We'll use these two maps to quickly look up an active HTLC with its
12✔
2292
        // matching HTLC resolution.
12✔
2293
        outResolutionMap := make(map[wire.OutPoint]lnwallet.OutgoingHtlcResolution)
12✔
2294
        inResolutionMap := make(map[wire.OutPoint]lnwallet.IncomingHtlcResolution)
12✔
2295
        for i := 0; i < len(incomingResolutions); i++ {
12✔
UNCOV
2296
                inRes := incomingResolutions[i]
×
UNCOV
2297
                inResolutionMap[inRes.HtlcPoint()] = inRes
×
UNCOV
2298
        }
×
2299
        for i := 0; i < len(outgoingResolutions); i++ {
13✔
2300
                outRes := outgoingResolutions[i]
1✔
2301
                outResolutionMap[outRes.HtlcPoint()] = outRes
1✔
2302
        }
1✔
2303

2304
        // We'll create the resolver kit that we'll be cloning for each
2305
        // resolver so they each can do their duty.
2306
        resolverCfg := ResolverConfig{
12✔
2307
                ChannelArbitratorConfig: c.cfg,
12✔
2308
                Checkpoint: func(res ContractResolver,
12✔
2309
                        reports ...*channeldb.ResolverReport) error {
14✔
2310

2✔
2311
                        return c.log.InsertUnresolvedContracts(reports, res)
2✔
2312
                },
2✔
2313
        }
2314

2315
        commitHash := contractResolutions.CommitHash
12✔
2316
        failureMsg := &lnwire.FailPermanentChannelFailure{}
12✔
2317

12✔
2318
        var htlcResolvers []ContractResolver
12✔
2319

12✔
2320
        // We instantiate an anchor resolver if the commitment tx has an
12✔
2321
        // anchor.
12✔
2322
        if contractResolutions.AnchorResolution != nil {
14✔
2323
                anchorResolver := newAnchorResolver(
2✔
2324
                        contractResolutions.AnchorResolution.AnchorSignDescriptor,
2✔
2325
                        contractResolutions.AnchorResolution.CommitAnchor,
2✔
2326
                        height, c.cfg.ChanPoint, resolverCfg,
2✔
2327
                )
2✔
2328
                anchorResolver.SupplementState(chanState)
2✔
2329

2✔
2330
                htlcResolvers = append(htlcResolvers, anchorResolver)
2✔
2331
        }
2✔
2332

2333
        // If this is a breach close, we'll create a breach resolver, determine
2334
        // the htlc's to fail back, and exit. This is done because the other
2335
        // steps taken for non-breach-closes do not matter for breach-closes.
2336
        if contractResolutions.BreachResolution != nil {
14✔
2337
                breachResolver := newBreachResolver(resolverCfg)
2✔
2338
                htlcResolvers = append(htlcResolvers, breachResolver)
2✔
2339

2✔
2340
                // We'll use the CommitSet, we'll fail back all outgoing HTLC's
2✔
2341
                // that exist on either of the remote commitments. The map is
2✔
2342
                // used to deduplicate any shared htlc's.
2✔
2343
                remoteOutgoing := make(map[uint64]channeldb.HTLC)
2✔
2344
                for htlcSetKey, htlcs := range confCommitSet.HtlcSets {
4✔
2345
                        if !htlcSetKey.IsRemote {
2✔
UNCOV
2346
                                continue
×
2347
                        }
2348

2349
                        for _, htlc := range htlcs {
4✔
2350
                                if htlc.Incoming {
3✔
2351
                                        continue
1✔
2352
                                }
2353

2354
                                remoteOutgoing[htlc.HtlcIndex] = htlc
1✔
2355
                        }
2356
                }
2357

2358
                // Now we'll loop over the map and create ResolutionMsgs for
2359
                // each of them.
2360
                for _, htlc := range remoteOutgoing {
3✔
2361
                        failMsg := ResolutionMsg{
1✔
2362
                                SourceChan: c.cfg.ShortChanID,
1✔
2363
                                HtlcIndex:  htlc.HtlcIndex,
1✔
2364
                                Failure:    failureMsg,
1✔
2365
                        }
1✔
2366

1✔
2367
                        msgsToSend = append(msgsToSend, failMsg)
1✔
2368
                }
1✔
2369

2370
                return htlcResolvers, msgsToSend, nil
2✔
2371
        }
2372

2373
        // For each HTLC, we'll either act immediately, meaning we'll instantly
2374
        // fail the HTLC, or we'll act only once the transaction has been
2375
        // confirmed, in which case we'll need an HTLC resolver.
2376
        for htlcAction, htlcs := range htlcActions {
21✔
2377
                switch htlcAction {
11✔
2378

2379
                // If we can fail an HTLC immediately (an outgoing HTLC with no
2380
                // contract), then we'll assemble an HTLC fail packet to send.
2381
                case HtlcFailNowAction:
9✔
2382
                        for _, htlc := range htlcs {
18✔
2383
                                failMsg := ResolutionMsg{
9✔
2384
                                        SourceChan: c.cfg.ShortChanID,
9✔
2385
                                        HtlcIndex:  htlc.HtlcIndex,
9✔
2386
                                        Failure:    failureMsg,
9✔
2387
                                }
9✔
2388

9✔
2389
                                msgsToSend = append(msgsToSend, failMsg)
9✔
2390
                        }
9✔
2391

2392
                // If we can claim this HTLC, we'll create an HTLC resolver to
2393
                // claim the HTLC (second-level or directly), then add the pre
2394
                case HtlcClaimAction:
×
2395
                        for _, htlc := range htlcs {
×
2396
                                htlc := htlc
×
2397

×
2398
                                htlcOp := wire.OutPoint{
×
2399
                                        Hash:  commitHash,
×
2400
                                        Index: uint32(htlc.OutputIndex),
×
2401
                                }
×
2402

×
2403
                                resolution, ok := inResolutionMap[htlcOp]
×
2404
                                if !ok {
×
2405
                                        // TODO(roasbeef): panic?
×
2406
                                        log.Errorf("ChannelArbitrator(%v) unable to find "+
×
2407
                                                "incoming resolution: %v",
×
2408
                                                c.cfg.ChanPoint, htlcOp)
×
2409
                                        continue
×
2410
                                }
2411

2412
                                resolver := newSuccessResolver(
×
2413
                                        resolution, height, htlc, resolverCfg,
×
2414
                                )
×
2415
                                if chanState != nil {
×
2416
                                        resolver.SupplementState(chanState)
×
2417
                                }
×
2418
                                htlcResolvers = append(htlcResolvers, resolver)
×
2419
                        }
2420

2421
                // If we can timeout the HTLC directly, then we'll create the
2422
                // proper resolver to do so, who will then cancel the packet
2423
                // backwards.
UNCOV
2424
                case HtlcTimeoutAction:
×
UNCOV
2425
                        for _, htlc := range htlcs {
×
UNCOV
2426
                                htlc := htlc
×
UNCOV
2427

×
UNCOV
2428
                                htlcOp := wire.OutPoint{
×
UNCOV
2429
                                        Hash:  commitHash,
×
UNCOV
2430
                                        Index: uint32(htlc.OutputIndex),
×
UNCOV
2431
                                }
×
UNCOV
2432

×
UNCOV
2433
                                resolution, ok := outResolutionMap[htlcOp]
×
UNCOV
2434
                                if !ok {
×
2435
                                        log.Errorf("ChannelArbitrator(%v) unable to find "+
×
2436
                                                "outgoing resolution: %v", c.cfg.ChanPoint, htlcOp)
×
2437
                                        continue
×
2438
                                }
2439

UNCOV
2440
                                resolver := newTimeoutResolver(
×
UNCOV
2441
                                        resolution, height, htlc, resolverCfg,
×
UNCOV
2442
                                )
×
UNCOV
2443
                                if chanState != nil {
×
UNCOV
2444
                                        resolver.SupplementState(chanState)
×
UNCOV
2445
                                }
×
2446

2447
                                // For outgoing HTLCs, we will also need to
2448
                                // supplement the resolver with the expiry
2449
                                // block height of its corresponding incoming
2450
                                // HTLC.
UNCOV
2451
                                deadline := c.cfg.FindOutgoingHTLCDeadline(htlc)
×
UNCOV
2452
                                resolver.SupplementDeadline(deadline)
×
UNCOV
2453

×
UNCOV
2454
                                htlcResolvers = append(htlcResolvers, resolver)
×
2455
                        }
2456

2457
                // If this is an incoming HTLC, but we can't act yet, then
2458
                // we'll create an incoming resolver to redeem the HTLC if we
2459
                // learn of the pre-image, or let the remote party time out.
UNCOV
2460
                case HtlcIncomingWatchAction:
×
UNCOV
2461
                        for _, htlc := range htlcs {
×
UNCOV
2462
                                htlc := htlc
×
UNCOV
2463

×
UNCOV
2464
                                htlcOp := wire.OutPoint{
×
UNCOV
2465
                                        Hash:  commitHash,
×
UNCOV
2466
                                        Index: uint32(htlc.OutputIndex),
×
UNCOV
2467
                                }
×
UNCOV
2468

×
UNCOV
2469
                                // TODO(roasbeef): need to handle incoming dust...
×
UNCOV
2470

×
UNCOV
2471
                                // TODO(roasbeef): can't be negative!!!
×
UNCOV
2472
                                resolution, ok := inResolutionMap[htlcOp]
×
UNCOV
2473
                                if !ok {
×
2474
                                        log.Errorf("ChannelArbitrator(%v) unable to find "+
×
2475
                                                "incoming resolution: %v",
×
2476
                                                c.cfg.ChanPoint, htlcOp)
×
2477
                                        continue
×
2478
                                }
2479

UNCOV
2480
                                resolver := newIncomingContestResolver(
×
UNCOV
2481
                                        resolution, height, htlc,
×
UNCOV
2482
                                        resolverCfg,
×
UNCOV
2483
                                )
×
UNCOV
2484
                                if chanState != nil {
×
UNCOV
2485
                                        resolver.SupplementState(chanState)
×
UNCOV
2486
                                }
×
UNCOV
2487
                                htlcResolvers = append(htlcResolvers, resolver)
×
2488
                        }
2489

2490
                // We've lost an htlc because it isn't manifested on the
2491
                // commitment transaction that closed the channel.
2492
                case HtlcIncomingDustFinalAction:
1✔
2493
                        for _, htlc := range htlcs {
2✔
2494
                                htlc := htlc
1✔
2495

1✔
2496
                                key := models.CircuitKey{
1✔
2497
                                        ChanID: c.cfg.ShortChanID,
1✔
2498
                                        HtlcID: htlc.HtlcIndex,
1✔
2499
                                }
1✔
2500

1✔
2501
                                // Mark this dust htlc as final failed.
1✔
2502
                                chainArbCfg := c.cfg.ChainArbitratorConfig
1✔
2503
                                err := chainArbCfg.PutFinalHtlcOutcome(
1✔
2504
                                        key.ChanID, key.HtlcID, false,
1✔
2505
                                )
1✔
2506
                                if err != nil {
1✔
2507
                                        return nil, nil, err
×
2508
                                }
×
2509

2510
                                // Send notification.
2511
                                chainArbCfg.HtlcNotifier.NotifyFinalHtlcEvent(
1✔
2512
                                        key,
1✔
2513
                                        channeldb.FinalHtlcInfo{
1✔
2514
                                                Settled:  false,
1✔
2515
                                                Offchain: false,
1✔
2516
                                        },
1✔
2517
                                )
1✔
2518
                        }
2519

2520
                // Finally, if this is an outgoing HTLC we've sent, then we'll
2521
                // launch a resolver to watch for the pre-image (and settle
2522
                // backwards), or just timeout.
2523
                case HtlcOutgoingWatchAction:
1✔
2524
                        for _, htlc := range htlcs {
2✔
2525
                                htlc := htlc
1✔
2526

1✔
2527
                                htlcOp := wire.OutPoint{
1✔
2528
                                        Hash:  commitHash,
1✔
2529
                                        Index: uint32(htlc.OutputIndex),
1✔
2530
                                }
1✔
2531

1✔
2532
                                resolution, ok := outResolutionMap[htlcOp]
1✔
2533
                                if !ok {
1✔
2534
                                        log.Errorf("ChannelArbitrator(%v) unable to find "+
×
2535
                                                "outgoing resolution: %v",
×
2536
                                                c.cfg.ChanPoint, htlcOp)
×
2537
                                        continue
×
2538
                                }
2539

2540
                                resolver := newOutgoingContestResolver(
1✔
2541
                                        resolution, height, htlc, resolverCfg,
1✔
2542
                                )
1✔
2543
                                if chanState != nil {
2✔
2544
                                        resolver.SupplementState(chanState)
1✔
2545
                                }
1✔
2546

2547
                                // For outgoing HTLCs, we will also need to
2548
                                // supplement the resolver with the expiry
2549
                                // block height of its corresponding incoming
2550
                                // HTLC.
2551
                                deadline := c.cfg.FindOutgoingHTLCDeadline(htlc)
1✔
2552
                                resolver.SupplementDeadline(deadline)
1✔
2553

1✔
2554
                                htlcResolvers = append(htlcResolvers, resolver)
1✔
2555
                        }
2556
                }
2557
        }
2558

2559
        // If this is was an unilateral closure, then we'll also create a
2560
        // resolver to sweep our commitment output (but only if it wasn't
2561
        // trimmed).
2562
        if contractResolutions.CommitResolution != nil {
10✔
UNCOV
2563
                resolver := newCommitSweepResolver(
×
UNCOV
2564
                        *contractResolutions.CommitResolution, height,
×
UNCOV
2565
                        c.cfg.ChanPoint, resolverCfg,
×
UNCOV
2566
                )
×
UNCOV
2567
                if chanState != nil {
×
UNCOV
2568
                        resolver.SupplementState(chanState)
×
UNCOV
2569
                }
×
UNCOV
2570
                htlcResolvers = append(htlcResolvers, resolver)
×
2571
        }
2572

2573
        return htlcResolvers, msgsToSend, nil
10✔
2574
}
2575

2576
// replaceResolver replaces a in the list of active resolvers. If the resolver
2577
// to be replaced is not found, it returns an error.
2578
func (c *ChannelArbitrator) replaceResolver(oldResolver,
2579
        newResolver ContractResolver) error {
1✔
2580

1✔
2581
        c.activeResolversLock.Lock()
1✔
2582
        defer c.activeResolversLock.Unlock()
1✔
2583

1✔
2584
        oldKey := oldResolver.ResolverKey()
1✔
2585
        for i, r := range c.activeResolvers {
2✔
2586
                if bytes.Equal(r.ResolverKey(), oldKey) {
2✔
2587
                        c.activeResolvers[i] = newResolver
1✔
2588
                        return nil
1✔
2589
                }
1✔
2590
        }
2591

2592
        return errors.New("resolver to be replaced not found")
×
2593
}
2594

2595
// resolveContract is a goroutine tasked with fully resolving an unresolved
2596
// contract. Either the initial contract will be resolved after a single step,
2597
// or the contract will itself create another contract to be resolved. In
2598
// either case, one the contract has been fully resolved, we'll signal back to
2599
// the main goroutine so it can properly keep track of the set of unresolved
2600
// contracts.
2601
//
2602
// NOTE: This MUST be run as a goroutine.
2603
func (c *ChannelArbitrator) resolveContract(currentContract ContractResolver,
2604
        immediate bool) {
6✔
2605

6✔
2606
        defer c.wg.Done()
6✔
2607

6✔
2608
        log.Debugf("ChannelArbitrator(%v): attempting to resolve %T",
6✔
2609
                c.cfg.ChanPoint, currentContract)
6✔
2610

6✔
2611
        // Until the contract is fully resolved, we'll continue to iteratively
6✔
2612
        // resolve the contract one step at a time.
6✔
2613
        for !currentContract.IsResolved() {
13✔
2614
                log.Debugf("ChannelArbitrator(%v): contract %T not yet resolved",
7✔
2615
                        c.cfg.ChanPoint, currentContract)
7✔
2616

7✔
2617
                select {
7✔
2618

2619
                // If we've been signalled to quit, then we'll exit early.
2620
                case <-c.quit:
×
2621
                        return
×
2622

2623
                default:
7✔
2624
                        // Otherwise, we'll attempt to resolve the current
7✔
2625
                        // contract.
7✔
2626
                        nextContract, err := currentContract.Resolve(immediate)
7✔
2627
                        if err != nil {
8✔
2628
                                if err == errResolverShuttingDown {
1✔
UNCOV
2629
                                        return
×
UNCOV
2630
                                }
×
2631

2632
                                log.Errorf("ChannelArbitrator(%v): unable to "+
1✔
2633
                                        "progress %T: %v",
1✔
2634
                                        c.cfg.ChanPoint, currentContract, err)
1✔
2635
                                return
1✔
2636
                        }
2637

2638
                        switch {
6✔
2639
                        // If this contract produced another, then this means
2640
                        // the current contract was only able to be partially
2641
                        // resolved in this step. So we'll do a contract swap
2642
                        // within our logs: the new contract will take the
2643
                        // place of the old one.
2644
                        case nextContract != nil:
1✔
2645
                                log.Debugf("ChannelArbitrator(%v): swapping "+
1✔
2646
                                        "out contract %T for %T ",
1✔
2647
                                        c.cfg.ChanPoint, currentContract,
1✔
2648
                                        nextContract)
1✔
2649

1✔
2650
                                // Swap contract in log.
1✔
2651
                                err := c.log.SwapContract(
1✔
2652
                                        currentContract, nextContract,
1✔
2653
                                )
1✔
2654
                                if err != nil {
1✔
2655
                                        log.Errorf("unable to add recurse "+
×
2656
                                                "contract: %v", err)
×
2657
                                }
×
2658

2659
                                // Swap contract in resolvers list. This is to
2660
                                // make sure that reports are queried from the
2661
                                // new resolver.
2662
                                err = c.replaceResolver(
1✔
2663
                                        currentContract, nextContract,
1✔
2664
                                )
1✔
2665
                                if err != nil {
1✔
2666
                                        log.Errorf("unable to replace "+
×
2667
                                                "contract: %v", err)
×
2668
                                }
×
2669

2670
                                // As this contract produced another, we'll
2671
                                // re-assign, so we can continue our resolution
2672
                                // loop.
2673
                                currentContract = nextContract
1✔
2674

2675
                        // If this contract is actually fully resolved, then
2676
                        // we'll mark it as such within the database.
2677
                        case currentContract.IsResolved():
5✔
2678
                                log.Debugf("ChannelArbitrator(%v): marking "+
5✔
2679
                                        "contract %T fully resolved",
5✔
2680
                                        c.cfg.ChanPoint, currentContract)
5✔
2681

5✔
2682
                                err := c.log.ResolveContract(currentContract)
5✔
2683
                                if err != nil {
5✔
2684
                                        log.Errorf("unable to resolve contract: %v",
×
2685
                                                err)
×
2686
                                }
×
2687

2688
                                // Now that the contract has been resolved,
2689
                                // well signal to the main goroutine.
2690
                                select {
5✔
2691
                                case c.resolutionSignal <- struct{}{}:
4✔
2692
                                case <-c.quit:
1✔
2693
                                        return
1✔
2694
                                }
2695
                        }
2696

2697
                }
2698
        }
2699
}
2700

2701
// signalUpdateMsg is a struct that carries fresh signals to the
2702
// ChannelArbitrator. We need to receive a message like this each time the
2703
// channel becomes active, as it's internal state may change.
2704
type signalUpdateMsg struct {
2705
        // newSignals is the set of new active signals to be sent to the
2706
        // arbitrator.
2707
        newSignals *ContractSignals
2708

2709
        // doneChan is a channel that will be closed on the arbitrator has
2710
        // attached the new signals.
2711
        doneChan chan struct{}
2712
}
2713

2714
// UpdateContractSignals updates the set of signals the ChannelArbitrator needs
2715
// to receive from a channel in real-time in order to keep in sync with the
2716
// latest state of the contract.
2717
func (c *ChannelArbitrator) UpdateContractSignals(newSignals *ContractSignals) {
11✔
2718
        done := make(chan struct{})
11✔
2719

11✔
2720
        select {
11✔
2721
        case c.signalUpdates <- &signalUpdateMsg{
2722
                newSignals: newSignals,
2723
                doneChan:   done,
2724
        }:
11✔
2725
        case <-c.quit:
×
2726
        }
2727

2728
        select {
11✔
2729
        case <-done:
11✔
2730
        case <-c.quit:
×
2731
        }
2732
}
2733

2734
// notifyContractUpdate updates the ChannelArbitrator's unmerged mappings such
2735
// that it can later be merged with activeHTLCs when calling
2736
// checkLocalChainActions or sweepAnchors. These are the only two places that
2737
// activeHTLCs is used.
2738
func (c *ChannelArbitrator) notifyContractUpdate(upd *ContractUpdate) {
12✔
2739
        c.unmergedMtx.Lock()
12✔
2740
        defer c.unmergedMtx.Unlock()
12✔
2741

12✔
2742
        // Update the mapping.
12✔
2743
        c.unmergedSet[upd.HtlcKey] = newHtlcSet(upd.Htlcs)
12✔
2744

12✔
2745
        log.Tracef("ChannelArbitrator(%v): fresh set of htlcs=%v",
12✔
2746
                c.cfg.ChanPoint, lnutils.SpewLogClosure(upd))
12✔
2747
}
12✔
2748

2749
// updateActiveHTLCs merges the unmerged set of HTLCs from the link with
2750
// activeHTLCs.
2751
func (c *ChannelArbitrator) updateActiveHTLCs() {
73✔
2752
        c.unmergedMtx.RLock()
73✔
2753
        defer c.unmergedMtx.RUnlock()
73✔
2754

73✔
2755
        // Update the mapping.
73✔
2756
        c.activeHTLCs[LocalHtlcSet] = c.unmergedSet[LocalHtlcSet]
73✔
2757
        c.activeHTLCs[RemoteHtlcSet] = c.unmergedSet[RemoteHtlcSet]
73✔
2758

73✔
2759
        // If the pending set exists, update that as well.
73✔
2760
        if _, ok := c.unmergedSet[RemotePendingHtlcSet]; ok {
82✔
2761
                pendingSet := c.unmergedSet[RemotePendingHtlcSet]
9✔
2762
                c.activeHTLCs[RemotePendingHtlcSet] = pendingSet
9✔
2763
        }
9✔
2764
}
2765

2766
// channelAttendant is the primary goroutine that acts at the judicial
2767
// arbitrator between our channel state, the remote channel peer, and the
2768
// blockchain (Our judge). This goroutine will ensure that we faithfully execute
2769
// all clauses of our contract in the case that we need to go on-chain for a
2770
// dispute. Currently, two such conditions warrant our intervention: when an
2771
// outgoing HTLC is about to timeout, and when we know the pre-image for an
2772
// incoming HTLC, but it hasn't yet been settled off-chain. In these cases,
2773
// we'll: broadcast our commitment, cancel/settle any HTLC's backwards after
2774
// sufficient confirmation, and finally send our set of outputs to the UTXO
2775
// Nursery for incubation, and ultimate sweeping.
2776
//
2777
// NOTE: This MUST be run as a goroutine.
2778
func (c *ChannelArbitrator) channelAttendant(bestHeight int32) {
47✔
2779

47✔
2780
        // TODO(roasbeef): tell top chain arb we're done
47✔
2781
        defer func() {
91✔
2782
                c.wg.Done()
44✔
2783
        }()
44✔
2784

2785
        for {
152✔
2786
                select {
105✔
2787

2788
                // A new block has arrived, we'll examine all the active HTLC's
2789
                // to see if any of them have expired, and also update our
2790
                // track of the best current height.
2791
                case blockHeight, ok := <-c.blocks:
22✔
2792
                        if !ok {
29✔
2793
                                return
7✔
2794
                        }
7✔
2795
                        bestHeight = blockHeight
15✔
2796

15✔
2797
                        // If we're not in the default state, then we can
15✔
2798
                        // ignore this signal as we're waiting for contract
15✔
2799
                        // resolution.
15✔
2800
                        if c.state != StateDefault {
24✔
2801
                                continue
9✔
2802
                        }
2803

2804
                        // Now that a new block has arrived, we'll attempt to
2805
                        // advance our state forward.
2806
                        nextState, _, err := c.advanceState(
6✔
2807
                                uint32(bestHeight), chainTrigger, nil,
6✔
2808
                        )
6✔
2809
                        if err != nil {
6✔
2810
                                log.Errorf("Unable to advance state: %v", err)
×
2811
                        }
×
2812

2813
                        // If as a result of this trigger, the contract is
2814
                        // fully resolved, then well exit.
2815
                        if nextState == StateFullyResolved {
6✔
2816
                                return
×
2817
                        }
×
2818

2819
                // A new signal update was just sent. This indicates that the
2820
                // channel under watch is now live, and may modify its internal
2821
                // state, so we'll get the most up to date signals to we can
2822
                // properly do our job.
2823
                case signalUpdate := <-c.signalUpdates:
11✔
2824
                        log.Tracef("ChannelArbitrator(%v): got new signal "+
11✔
2825
                                "update!", c.cfg.ChanPoint)
11✔
2826

11✔
2827
                        // We'll update the ShortChannelID.
11✔
2828
                        c.cfg.ShortChanID = signalUpdate.newSignals.ShortChanID
11✔
2829

11✔
2830
                        // Now that the signal has been updated, we'll now
11✔
2831
                        // close the done channel to signal to the caller we've
11✔
2832
                        // registered the new ShortChannelID.
11✔
2833
                        close(signalUpdate.doneChan)
11✔
2834

2835
                // We've cooperatively closed the channel, so we're no longer
2836
                // needed. We'll mark the channel as resolved and exit.
2837
                case closeInfo := <-c.cfg.ChainEvents.CooperativeClosure:
2✔
2838
                        log.Infof("ChannelArbitrator(%v) marking channel "+
2✔
2839
                                "cooperatively closed", c.cfg.ChanPoint)
2✔
2840

2✔
2841
                        err := c.cfg.MarkChannelClosed(
2✔
2842
                                closeInfo.ChannelCloseSummary,
2✔
2843
                                channeldb.ChanStatusCoopBroadcasted,
2✔
2844
                        )
2✔
2845
                        if err != nil {
2✔
2846
                                log.Errorf("Unable to mark channel closed: "+
×
2847
                                        "%v", err)
×
2848
                                return
×
2849
                        }
×
2850

2851
                        // We'll now advance our state machine until it reaches
2852
                        // a terminal state, and the channel is marked resolved.
2853
                        _, _, err = c.advanceState(
2✔
2854
                                closeInfo.CloseHeight, coopCloseTrigger, nil,
2✔
2855
                        )
2✔
2856
                        if err != nil {
3✔
2857
                                log.Errorf("Unable to advance state: %v", err)
1✔
2858
                                return
1✔
2859
                        }
1✔
2860

2861
                // We have broadcasted our commitment, and it is now confirmed
2862
                // on-chain.
2863
                case closeInfo := <-c.cfg.ChainEvents.LocalUnilateralClosure:
12✔
2864
                        log.Infof("ChannelArbitrator(%v): local on-chain "+
12✔
2865
                                "channel close", c.cfg.ChanPoint)
12✔
2866

12✔
2867
                        if c.state != StateCommitmentBroadcasted {
13✔
2868
                                log.Errorf("ChannelArbitrator(%v): unexpected "+
1✔
2869
                                        "local on-chain channel close",
1✔
2870
                                        c.cfg.ChanPoint)
1✔
2871
                        }
1✔
2872
                        closeTx := closeInfo.CloseTx
12✔
2873

12✔
2874
                        contractRes := &ContractResolutions{
12✔
2875
                                CommitHash:       closeTx.TxHash(),
12✔
2876
                                CommitResolution: closeInfo.CommitResolution,
12✔
2877
                                HtlcResolutions:  *closeInfo.HtlcResolutions,
12✔
2878
                                AnchorResolution: closeInfo.AnchorResolution,
12✔
2879
                        }
12✔
2880

12✔
2881
                        // When processing a unilateral close event, we'll
12✔
2882
                        // transition to the ContractClosed state. We'll log
12✔
2883
                        // out the set of resolutions such that they are
12✔
2884
                        // available to fetch in that state, we'll also write
12✔
2885
                        // the commit set so we can reconstruct our chain
12✔
2886
                        // actions on restart.
12✔
2887
                        err := c.log.LogContractResolutions(contractRes)
12✔
2888
                        if err != nil {
12✔
2889
                                log.Errorf("Unable to write resolutions: %v",
×
2890
                                        err)
×
2891
                                return
×
2892
                        }
×
2893
                        err = c.log.InsertConfirmedCommitSet(
12✔
2894
                                &closeInfo.CommitSet,
12✔
2895
                        )
12✔
2896
                        if err != nil {
12✔
2897
                                log.Errorf("Unable to write commit set: %v",
×
2898
                                        err)
×
2899
                                return
×
2900
                        }
×
2901

2902
                        // After the set of resolutions are successfully
2903
                        // logged, we can safely close the channel. After this
2904
                        // succeeds we won't be getting chain events anymore,
2905
                        // so we must make sure we can recover on restart after
2906
                        // it is marked closed. If the next state transition
2907
                        // fails, we'll start up in the prior state again, and
2908
                        // we won't be longer getting chain events. In this
2909
                        // case we must manually re-trigger the state
2910
                        // transition into StateContractClosed based on the
2911
                        // close status of the channel.
2912
                        err = c.cfg.MarkChannelClosed(
12✔
2913
                                closeInfo.ChannelCloseSummary,
12✔
2914
                                channeldb.ChanStatusLocalCloseInitiator,
12✔
2915
                        )
12✔
2916
                        if err != nil {
12✔
2917
                                log.Errorf("Unable to mark "+
×
2918
                                        "channel closed: %v", err)
×
2919
                                return
×
2920
                        }
×
2921

2922
                        // We'll now advance our state machine until it reaches
2923
                        // a terminal state.
2924
                        _, _, err = c.advanceState(
12✔
2925
                                uint32(closeInfo.SpendingHeight),
12✔
2926
                                localCloseTrigger, &closeInfo.CommitSet,
12✔
2927
                        )
12✔
2928
                        if err != nil {
13✔
2929
                                log.Errorf("Unable to advance state: %v", err)
1✔
2930
                        }
1✔
2931

2932
                // The remote party has broadcast the commitment on-chain.
2933
                // We'll examine our state to determine if we need to act at
2934
                // all.
2935
                case uniClosure := <-c.cfg.ChainEvents.RemoteUnilateralClosure:
8✔
2936
                        log.Infof("ChannelArbitrator(%v): remote party has "+
8✔
2937
                                "closed channel out on-chain", c.cfg.ChanPoint)
8✔
2938

8✔
2939
                        // If we don't have a self output, and there are no
8✔
2940
                        // active HTLC's, then we can immediately mark the
8✔
2941
                        // contract as fully resolved and exit.
8✔
2942
                        contractRes := &ContractResolutions{
8✔
2943
                                CommitHash:       *uniClosure.SpenderTxHash,
8✔
2944
                                CommitResolution: uniClosure.CommitResolution,
8✔
2945
                                HtlcResolutions:  *uniClosure.HtlcResolutions,
8✔
2946
                                AnchorResolution: uniClosure.AnchorResolution,
8✔
2947
                        }
8✔
2948

8✔
2949
                        // When processing a unilateral close event, we'll
8✔
2950
                        // transition to the ContractClosed state. We'll log
8✔
2951
                        // out the set of resolutions such that they are
8✔
2952
                        // available to fetch in that state, we'll also write
8✔
2953
                        // the commit set so we can reconstruct our chain
8✔
2954
                        // actions on restart.
8✔
2955
                        err := c.log.LogContractResolutions(contractRes)
8✔
2956
                        if err != nil {
9✔
2957
                                log.Errorf("Unable to write resolutions: %v",
1✔
2958
                                        err)
1✔
2959
                                return
1✔
2960
                        }
1✔
2961
                        err = c.log.InsertConfirmedCommitSet(
7✔
2962
                                &uniClosure.CommitSet,
7✔
2963
                        )
7✔
2964
                        if err != nil {
7✔
2965
                                log.Errorf("Unable to write commit set: %v",
×
2966
                                        err)
×
2967
                                return
×
2968
                        }
×
2969

2970
                        // After the set of resolutions are successfully
2971
                        // logged, we can safely close the channel. After this
2972
                        // succeeds we won't be getting chain events anymore,
2973
                        // so we must make sure we can recover on restart after
2974
                        // it is marked closed. If the next state transition
2975
                        // fails, we'll start up in the prior state again, and
2976
                        // we won't be longer getting chain events. In this
2977
                        // case we must manually re-trigger the state
2978
                        // transition into StateContractClosed based on the
2979
                        // close status of the channel.
2980
                        closeSummary := &uniClosure.ChannelCloseSummary
7✔
2981
                        err = c.cfg.MarkChannelClosed(
7✔
2982
                                closeSummary,
7✔
2983
                                channeldb.ChanStatusRemoteCloseInitiator,
7✔
2984
                        )
7✔
2985
                        if err != nil {
8✔
2986
                                log.Errorf("Unable to mark channel closed: %v",
1✔
2987
                                        err)
1✔
2988
                                return
1✔
2989
                        }
1✔
2990

2991
                        // We'll now advance our state machine until it reaches
2992
                        // a terminal state.
2993
                        _, _, err = c.advanceState(
6✔
2994
                                uint32(uniClosure.SpendingHeight),
6✔
2995
                                remoteCloseTrigger, &uniClosure.CommitSet,
6✔
2996
                        )
6✔
2997
                        if err != nil {
8✔
2998
                                log.Errorf("Unable to advance state: %v", err)
2✔
2999
                        }
2✔
3000

3001
                // The remote has breached the channel. As this is handled by
3002
                // the ChainWatcher and BreachArbitrator, we don't have to do
3003
                // anything in particular, so just advance our state and
3004
                // gracefully exit.
3005
                case breachInfo := <-c.cfg.ChainEvents.ContractBreach:
1✔
3006
                        log.Infof("ChannelArbitrator(%v): remote party has "+
1✔
3007
                                "breached channel!", c.cfg.ChanPoint)
1✔
3008

1✔
3009
                        // In the breach case, we'll only have anchor and
1✔
3010
                        // breach resolutions.
1✔
3011
                        contractRes := &ContractResolutions{
1✔
3012
                                CommitHash:       breachInfo.CommitHash,
1✔
3013
                                BreachResolution: breachInfo.BreachResolution,
1✔
3014
                                AnchorResolution: breachInfo.AnchorResolution,
1✔
3015
                        }
1✔
3016

1✔
3017
                        // We'll transition to the ContractClosed state and log
1✔
3018
                        // the set of resolutions such that they can be turned
1✔
3019
                        // into resolvers later on. We'll also insert the
1✔
3020
                        // CommitSet of the latest set of commitments.
1✔
3021
                        err := c.log.LogContractResolutions(contractRes)
1✔
3022
                        if err != nil {
1✔
3023
                                log.Errorf("Unable to write resolutions: %v",
×
3024
                                        err)
×
3025
                                return
×
3026
                        }
×
3027
                        err = c.log.InsertConfirmedCommitSet(
1✔
3028
                                &breachInfo.CommitSet,
1✔
3029
                        )
1✔
3030
                        if err != nil {
1✔
3031
                                log.Errorf("Unable to write commit set: %v",
×
3032
                                        err)
×
3033
                                return
×
3034
                        }
×
3035

3036
                        // The channel is finally marked pending closed here as
3037
                        // the BreachArbitrator and channel arbitrator have
3038
                        // persisted the relevant states.
3039
                        closeSummary := &breachInfo.CloseSummary
1✔
3040
                        err = c.cfg.MarkChannelClosed(
1✔
3041
                                closeSummary,
1✔
3042
                                channeldb.ChanStatusRemoteCloseInitiator,
1✔
3043
                        )
1✔
3044
                        if err != nil {
1✔
3045
                                log.Errorf("Unable to mark channel closed: %v",
×
3046
                                        err)
×
3047
                                return
×
3048
                        }
×
3049

3050
                        log.Infof("Breached channel=%v marked pending-closed",
1✔
3051
                                breachInfo.BreachResolution.FundingOutPoint)
1✔
3052

1✔
3053
                        // We'll advance our state machine until it reaches a
1✔
3054
                        // terminal state.
1✔
3055
                        _, _, err = c.advanceState(
1✔
3056
                                uint32(bestHeight), breachCloseTrigger,
1✔
3057
                                &breachInfo.CommitSet,
1✔
3058
                        )
1✔
3059
                        if err != nil {
1✔
3060
                                log.Errorf("Unable to advance state: %v", err)
×
3061
                        }
×
3062

3063
                // A new contract has just been resolved, we'll now check our
3064
                // log to see if all contracts have been resolved. If so, then
3065
                // we can exit as the contract is fully resolved.
3066
                case <-c.resolutionSignal:
4✔
3067
                        log.Infof("ChannelArbitrator(%v): a contract has been "+
4✔
3068
                                "fully resolved!", c.cfg.ChanPoint)
4✔
3069

4✔
3070
                        nextState, _, err := c.advanceState(
4✔
3071
                                uint32(bestHeight), chainTrigger, nil,
4✔
3072
                        )
4✔
3073
                        if err != nil {
4✔
3074
                                log.Errorf("Unable to advance state: %v", err)
×
3075
                        }
×
3076

3077
                        // If we don't have anything further to do after
3078
                        // advancing our state, then we'll exit.
3079
                        if nextState == StateFullyResolved {
7✔
3080
                                log.Infof("ChannelArbitrator(%v): all "+
3✔
3081
                                        "contracts fully resolved, exiting",
3✔
3082
                                        c.cfg.ChanPoint)
3✔
3083

3✔
3084
                                return
3✔
3085
                        }
3✔
3086

3087
                // We've just received a request to forcibly close out the
3088
                // channel. We'll
3089
                case closeReq := <-c.forceCloseReqs:
11✔
3090
                        log.Infof("ChannelArbitrator(%v): received force "+
11✔
3091
                                "close request", c.cfg.ChanPoint)
11✔
3092

11✔
3093
                        if c.state != StateDefault {
12✔
3094
                                select {
1✔
3095
                                case closeReq.closeTx <- nil:
1✔
3096
                                case <-c.quit:
×
3097
                                }
3098

3099
                                select {
1✔
3100
                                case closeReq.errResp <- errAlreadyForceClosed:
1✔
3101
                                case <-c.quit:
×
3102
                                }
3103

3104
                                continue
1✔
3105
                        }
3106

3107
                        nextState, closeTx, err := c.advanceState(
10✔
3108
                                uint32(bestHeight), userTrigger, nil,
10✔
3109
                        )
10✔
3110
                        if err != nil {
11✔
3111
                                log.Errorf("Unable to advance state: %v", err)
1✔
3112
                        }
1✔
3113

3114
                        select {
10✔
3115
                        case closeReq.closeTx <- closeTx:
10✔
3116
                        case <-c.quit:
×
3117
                                return
×
3118
                        }
3119

3120
                        select {
10✔
3121
                        case closeReq.errResp <- err:
10✔
3122
                        case <-c.quit:
×
3123
                                return
×
3124
                        }
3125

3126
                        // If we don't have anything further to do after
3127
                        // advancing our state, then we'll exit.
3128
                        if nextState == StateFullyResolved {
10✔
3129
                                log.Infof("ChannelArbitrator(%v): all "+
×
3130
                                        "contracts resolved, exiting",
×
3131
                                        c.cfg.ChanPoint)
×
3132
                                return
×
3133
                        }
×
3134

3135
                case <-c.quit:
31✔
3136
                        return
31✔
3137
                }
3138
        }
3139
}
3140

3141
// checkLegacyBreach returns StateFullyResolved if the channel was closed with
3142
// a breach transaction before the channel arbitrator launched its own breach
3143
// resolver. StateContractClosed is returned if this is a modern breach close
3144
// with a breach resolver. StateError is returned if the log lookup failed.
3145
func (c *ChannelArbitrator) checkLegacyBreach() (ArbitratorState, error) {
2✔
3146
        // A previous version of the channel arbitrator would make the breach
2✔
3147
        // close skip to StateFullyResolved. If there are no contract
2✔
3148
        // resolutions in the bolt arbitrator log, then this is an older breach
2✔
3149
        // close. Otherwise, if there are resolutions, the state should advance
2✔
3150
        // to StateContractClosed.
2✔
3151
        _, err := c.log.FetchContractResolutions()
2✔
3152
        if err == errNoResolutions {
2✔
3153
                // This is an older breach close still in the database.
×
3154
                return StateFullyResolved, nil
×
3155
        } else if err != nil {
2✔
3156
                return StateError, err
×
3157
        }
×
3158

3159
        // This is a modern breach close with resolvers.
3160
        return StateContractClosed, nil
2✔
3161
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc