• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

zopefoundation / ZODB / 18153960591

01 Oct 2025 06:50AM UTC coverage: 83.781% (-0.03%) from 83.811%
18153960591

Pull #415

github

web-flow
Update docs/articles/old-guide/convert_zodb_guide.py

Co-authored-by: Michael Howitz <icemac@gmx.net>
Pull Request #415: Apply the latest zope.meta templates

2441 of 3542 branches covered (68.92%)

193 of 257 new or added lines in 48 files covered. (75.1%)

12 existing lines in 6 files now uncovered.

13353 of 15938 relevant lines covered (83.78%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.04
/src/ZODB/BaseStorage.py
1
##############################################################################
2
#
3
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
4
# All Rights Reserved.
5
#
6
# This software is subject to the provisions of the Zope Public License,
7
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
8
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
9
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
10
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
11
# FOR A PARTICULAR PURPOSE
12
#
13
##############################################################################
14
"""Storage base class that is mostly a mistake
15

16
The base class here is tightly coupled with its subclasses and
17
its use is not recommended.  It's still here for historical reasons.
18
"""
19

20
import logging
1✔
21
import time
1✔
22
from struct import pack as _structpack
1✔
23
from struct import unpack as _structunpack
1✔
24

25
import zope.interface
1✔
26
from persistent.TimeStamp import TimeStamp
1✔
27

28
import ZODB.interfaces
1✔
29

30
from . import POSException
1✔
31
from . import utils
1✔
32
from .Connection import TransactionMetaData
1✔
33
from .UndoLogCompatible import UndoLogCompatible
1✔
34
from .utils import byte_chr
1✔
35
from .utils import byte_ord
1✔
36
from .utils import load_current
1✔
37
from .utils import oid_repr
1✔
38
from .utils import z64
1✔
39

40

41
log = logging.getLogger("ZODB.BaseStorage")
1✔
42

43

44
class BaseStorage(UndoLogCompatible):
1✔
45
    """Base class that supports storage implementations.
46

47
    XXX Base classes like this are an attractive nuisance. They often
48
    introduce more complexity than they save.  While important logic
49
    is implemented here, we should consider exposing it as utility
50
    functions or as objects that can be used through composition.
51

52
    A subclass must define the following methods:
53
    load()
54
    store()
55
    close()
56
    cleanup()
57
    lastTransaction()
58

59
    It must override these hooks:
60
    _begin()
61
    _vote()
62
    _abort()
63
    _finish()
64
    _clear_temp()
65

66
    If it stores multiple revisions, it should implement
67
    loadSerial()
68
    loadBefore()
69

70
    Each storage will have two locks that are accessed via lock
71
    acquire and release methods bound to the instance.  (Yuck.)
72
    _lock_acquire / _lock_release (reentrant)
73
    _commit_lock_acquire / _commit_lock_release
74

75
    The commit lock is acquired in tpc_begin() and released in
76
    tpc_abort() and tpc_finish().  It is never acquired with the other
77
    lock held.
78

79
    The other lock appears to protect _oid and _transaction and
80
    perhaps other things.  It is always held when load() is called, so
81
    presumably the load() implementation should also acquire the lock.
82
    """
83
    _transaction = None  # Transaction that is being committed
1✔
84
    _tstatus = ' '      # Transaction status, used for copying data
1✔
85
    _is_read_only = False
1✔
86

87
    def __init__(self, name, base=None):
1✔
88
        self.__name__ = name
1✔
89
        log.debug("create storage %s", self.__name__)
1✔
90

91
        # Allocate locks:
92
        self._lock = utils.RLock()
1✔
93
        self._commit_lock = utils.Lock()
1✔
94

95
        # Needed by external storages that use this dumb api :(
96
        self._lock_acquire = self._lock.acquire
1✔
97
        self._lock_release = self._lock.release
1✔
98
        self._commit_lock_acquire = self._commit_lock.acquire
1✔
99
        self._commit_lock_release = self._commit_lock.release
1✔
100

101
        t = time.time()
1✔
102
        t = self._ts = TimeStamp(*(time.gmtime(t)[:5] + (t % 60,)))
1✔
103
        self._tid = t.raw()
1✔
104

105
        # ._oid is the highest oid in use (0 is always in use -- it's
106
        # a reserved oid for the root object).  Our new_oid() method
107
        # increments it by 1, and returns the result.  It's really a
108
        # 64-bit integer stored as an 8-byte big-endian string.
109
        oid = getattr(base, '_oid', None)
1✔
110
        if oid is None:
1!
111
            self._oid = z64
1✔
112
        else:
113
            self._oid = oid
×
114
        # In case that conflicts are resolved during store,
115
        # this collects oids to be returned by tpc_vote.
116
        self._resolved = []
1✔
117

118
    def sortKey(self):
1✔
119
        """Return a string that can be used to sort storage instances.
120

121
        The key must uniquely identify a storage and must be the same
122
        across multiple instantiations of the same storage.
123
        """
124
        # name may not be sufficient, e.g. ZEO has a user-definable name.
125
        return self.__name__
1✔
126

127
    def getName(self):
1✔
128
        return self.__name__
1✔
129

130
    def getSize(self):
1✔
131
        return len(self) * 300  # WAG!
1✔
132

133
    def history(self, oid, version, length=1, filter=None):
1✔
134
        return ()
×
135

136
    def new_oid(self):
1✔
137
        if self._is_read_only:
1✔
138
            raise POSException.ReadOnlyError()
1✔
139

140
        with self._lock:
1✔
141
            last = self._oid
1✔
142
            d = byte_ord(last[-1])
1✔
143
            if d < 255:  # fast path for the usual case
1✔
144
                last = last[:-1] + byte_chr(d + 1)
1✔
145
            else:        # there's a carry out of the last byte
146
                last_as_long, = _structunpack(">Q", last)
1✔
147
                last = _structpack(">Q", last_as_long + 1)
1✔
148
            self._oid = last
1✔
149
            return last
1✔
150

151
    # Update the maximum oid in use, under protection of a lock.  The
152
    # maximum-in-use attribute is changed only if possible_new_max_oid is
153
    # larger than its current value.
154
    def set_max_oid(self, possible_new_max_oid):
1✔
155
        with self._lock:
1✔
156
            if possible_new_max_oid > self._oid:
1!
157
                self._oid = possible_new_max_oid
1✔
158

159
    def registerDB(self, db):
1✔
160
        pass  # we don't care
1✔
161

162
    def isReadOnly(self):
1✔
163
        return self._is_read_only
1✔
164

165
    def tpc_abort(self, transaction):
1✔
166
        with self._lock:
1✔
167

168
            if transaction is not self._transaction:
1✔
169
                return
1✔
170

171
            try:
1✔
172
                self._abort()
1✔
173
                self._clear_temp()
1✔
174
                self._transaction = None
1✔
175
            finally:
176
                self._commit_lock_release()
1✔
177

178
    def _abort(self):
1✔
179
        """Subclasses should redefine this to supply abort actions"""
180
        pass
×
181

182
    def tpc_begin(self, transaction, tid=None, status=' '):
1✔
183
        if self._is_read_only:
1✔
184
            raise POSException.ReadOnlyError()
1✔
185

186
        with self._lock:
1✔
187
            if self._transaction is transaction:
1✔
188
                raise POSException.StorageTransactionError(
1✔
189
                    "Duplicate tpc_begin calls for same transaction")
190

191
        self._commit_lock.acquire()
1✔
192

193
        with self._lock:
1✔
194
            self._transaction = transaction
1✔
195
            self._clear_temp()
1✔
196

197
            user = transaction.user
1✔
198
            desc = transaction.description
1✔
199
            ext = transaction.extension_bytes
1✔
200

201
            self._ude = user, desc, ext
1✔
202

203
            if tid is None:
1✔
204
                now = time.time()
1✔
205
                t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
1✔
206
                self._ts = t = t.laterThan(self._ts)
1✔
207
                self._tid = t.raw()
1✔
208
            else:
209
                self._ts = TimeStamp(tid)
1✔
210
                self._tid = tid
1✔
211

212
            del self._resolved[:]
1✔
213
            self._tstatus = status
1✔
214
            self._begin(self._tid, user, desc, ext)
1✔
215

216
    def tpc_transaction(self):
1✔
217
        return self._transaction
×
218

219
    def _begin(self, tid, u, d, e):
1✔
220
        """Subclasses should redefine this to supply transaction start actions.
221
        """
222
        pass
×
223

224
    def tpc_vote(self, transaction):
1✔
225
        with self._lock:
1✔
226
            if transaction is not self._transaction:
1✔
227
                raise POSException.StorageTransactionError(
1✔
228
                    "tpc_vote called with wrong transaction")
229
            return self._vote()
1✔
230

231
    def _vote(self):
1✔
232
        """Subclasses should redefine this to supply transaction vote actions.
233
        """
234
        return self._resolved
1✔
235

236
    def tpc_finish(self, transaction, f=None):
1✔
237
        # It's important that the storage calls the function we pass
238
        # while it still has its lock.  We don't want another thread
239
        # to be able to read any updated data until we've had a chance
240
        # to send an invalidation message to all of the other
241
        # connections!
242

243
        with self._lock:
1✔
244
            if transaction is not self._transaction:
1✔
245
                raise POSException.StorageTransactionError(
1✔
246
                    "tpc_finish called with wrong transaction")
247
            try:
1✔
248
                if f is not None:
1✔
249
                    f(self._tid)
1✔
250
                u, d, e = self._ude
1✔
251
                self._finish(self._tid, u, d, e)
1✔
252
                self._clear_temp()
1✔
253
            finally:
254
                self._ude = None
1✔
255
                self._transaction = None
1✔
256
                self._commit_lock.release()
1✔
257
            return self._tid
1✔
258

259
    def _finish(self, tid, u, d, e):
1✔
260
        """Subclasses should redefine this to supply transaction finish actions
261
        """
262
        pass
×
263

264
    def lastTransaction(self):
1✔
265
        with self._lock:
1✔
266
            return self._ltid
1✔
267

268
    def getTid(self, oid):
1✔
269
        with self._lock:
1✔
270
            return load_current(self, oid)[1]
1✔
271

272
    def loadSerial(self, oid, serial):
1✔
273
        raise POSException.Unsupported(
×
274
            "Retrieval of historical revisions is not supported")
275

276
    def loadBefore(self, oid, tid):
1✔
277
        """Return most recent revision of oid before tid committed."""
278
        return None
×
279

280
    def copyTransactionsFrom(self, other, verbose=0):
1✔
281
        """Copy transactions from another storage.
282

283
        This is typically used for converting data from one storage to
284
        another.  `other` must have an .iterator() method.
285
        """
286
        copy(other, self, verbose)
1✔
287

288

289
def copy(source, dest, verbose=0):
1✔
290
    """Copy transactions from a source to a destination storage
291

292
    This is typically used for converting data from one storage to
293
    another.  `source` must have an .iterator() method.
294
    """
295
    _ts = None
1✔
296
    ok = 1
1✔
297
    preindex = {}
1✔
298
    preget = preindex.get
1✔
299
    # restore() is a new storage API method which has an identical
300
    # signature to store() except that it does not return anything.
301
    # Semantically, restore() is also identical to store() except that it
302
    # doesn't do the ConflictError or VersionLockError consistency
303
    # checks.  The reason to use restore() over store() in this method is
304
    # that store() cannot be used to copy transactions spanning a version
305
    # commit or abort, or over transactional undos.
306
    #
307
    # We'll use restore() if it's available, otherwise we'll fall back to
308
    # using store().  However, if we use store, then
309
    # copyTransactionsFrom() may fail with VersionLockError or
310
    # ConflictError.
311
    try:
1✔
312
        getattr(dest, 'restore')
1✔
313
    except:  # noqa: E722 do not use bare 'except'
1✔
314
        restoring = False
1✔
315
    else:
316
        restoring = True
1✔
317
    fiter = source.iterator()
1✔
318
    for transaction in fiter:
1✔
319
        tid = transaction.tid
1✔
320
        if _ts is None:
1✔
321
            _ts = TimeStamp(tid)
1✔
322
        else:
323
            t = TimeStamp(tid)
1✔
324
            if t <= _ts:
1!
325
                if ok:
×
NEW
326
                    print(f'Time stamps out of order {_ts}, {t}')
×
327
                ok = 0
×
328
                _ts = t.laterThan(_ts)
×
329
                tid = _ts.raw()
×
330
            else:
331
                _ts = t
1✔
332
                if not ok:
1!
333
                    print('Time stamps back in order %s' % (t))
×
334
                    ok = 1
×
335

336
        if verbose:
1!
337
            print(_ts)
×
338

339
        dest.tpc_begin(transaction, tid, transaction.status)
1✔
340
        for r in transaction:
1✔
341
            oid = r.oid
1✔
342
            if verbose:
1!
343
                print(oid_repr(oid), r.version, len(r.data))
×
344
            if restoring:
1✔
345
                dest.restore(oid, r.tid, r.data, r.version,
1✔
346
                             r.data_txn, transaction)
347
            else:
348
                pre = preget(oid, None)
1✔
349
                dest.store(oid, pre, r.data, r.version, transaction)
1✔
350
                preindex[oid] = tid
1✔
351

352
        dest.tpc_vote(transaction)
1✔
353
        dest.tpc_finish(transaction)
1✔
354

355

356
# defined outside of BaseStorage to facilitate independent reuse.
357
# just depends on _transaction attr and getTid method.
358
def checkCurrentSerialInTransaction(self, oid, serial, transaction):
1✔
359
    if transaction is not self._transaction:
1!
360
        raise POSException.StorageTransactionError(self, transaction)
×
361

362
    committed_tid = self.getTid(oid)
1✔
363
    if committed_tid != serial:
1✔
364
        raise POSException.ReadConflictError(
1✔
365
            oid=oid, serials=(committed_tid, serial))
366

367

368
BaseStorage.checkCurrentSerialInTransaction = checkCurrentSerialInTransaction
1✔
369

370

371
@zope.interface.implementer(ZODB.interfaces.IStorageTransactionInformation)
1✔
372
class TransactionRecord(TransactionMetaData):
1✔
373
    """Abstract base class for iterator protocol"""
374

375
    def __init__(self, tid, status, user, description, extension):
1✔
376
        self.tid = tid
1✔
377
        self.status = status
1✔
378
        TransactionMetaData.__init__(self, user, description, extension)
1✔
379

380

381
@zope.interface.implementer(ZODB.interfaces.IStorageRecordInformation)
1✔
382
class DataRecord:
1✔
383
    """Abstract base class for iterator protocol"""
384

385
    version = ''
1✔
386

387
    def __init__(self, oid, tid, data, prev):
1✔
388
        self.oid = oid
1✔
389
        self.tid = tid
1✔
390
        self.data = data
1✔
391
        self.data_txn = prev
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc