18153960591

Committed 01 Oct 2025 06:50AM UTC coverage: 83.781% (-0.03%) from 83.811%

Build # 18153960591

Build Type

Pull #415

github

Committed by

web-flow

Commit Message

Update docs/articles/old-guide/convert_zodb_guide.py

Co-authored-by: Michael Howitz <icemac@gmx.net>

Pull Request Pull Request #415: Apply the latest zope.meta templates

Run Details

2441 of 3542 branches covered (68.92%)

193 of 257 new or added lines in 48 files covered. (75.1%)

12 existing lines in 6 files now uncovered.

13353 of 15938 relevant lines covered (83.78%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.69

/src/ZODB/scripts/fstest.py

#!/usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Simple consistency checker for FileStorage.

usage: fstest.py [-v] data.fs

The fstest tool will scan all the data in a FileStorage and report an
error if it finds any corrupt transaction data.  The tool will print a
message when the first error is detected, then exit.

The tool accepts one or more -v arguments.  If a single -v is used, it
will print a line of text for each transaction record it encounters.
If two -v arguments are used, it will also print a line of text for
each object.  The objects for a transaction will be printed before the
transaction itself.

Note: It does not check the consistency of the object pickles.  It is
possible for the damage to occur only in the part of the file that
stores object pickles.  Those errors will go undetected.
"""

import binascii
import struct
import sys

from ZODB._compat import FILESTORAGE_MAGIC


# The implementation is based closely on the read_index() function in
# ZODB.FileStorage.  If anything about the FileStorage layout changes,
# this file will need to be udpated.


class FormatError(ValueError):
    """There is a problem with the format of the FileStorage."""


class Status:
    checkpoint = b'c'
    undone = b'u'


packed_version = FILESTORAGE_MAGIC

TREC_HDR_LEN = 23
DREC_HDR_LEN = 42

VERBOSE = 0


def hexify(s):
    r"""Format an 8-bit string as hex

        >>> hexify(b'\x00\xff\xaa\xcc')
        '0x00ffaacc'

    """
    return '0x' + binascii.hexlify(s).decode()


def chatter(msg, level=1):
    if VERBOSE >= level:
        sys.stdout.write(msg)


def U64(v):
    """Unpack an 8-byte string as a 64-bit long"""
    h, l_ = struct.unpack(">II", v)
    if h:
        return (h << 32) + l_
    else:
        return l_


def check(path):
    with open(path, 'rb') as file:
        file.seek(0, 2)
        file_size = file.tell()
        if file_size == 0:
            raise FormatError("empty file")
        file.seek(0)
        if file.read(4) != packed_version:
            raise FormatError("invalid file header")

        pos = 4
        tid = b'\000' * 8  # lowest possible tid to start
        i = 0
        while pos:
            _pos = pos
            pos, tid = check_trec(path, file, pos, tid, file_size)
            if tid is not None:
                chatter("%10d: transaction tid %s #%d \n" %
                        (_pos, hexify(tid), i))
                i = i + 1


def check_trec(path, file, pos, ltid, file_size):
    """Read an individual transaction record from file.

    Returns the pos of the next transaction and the transaction id.
    It also leaves the file pointer set to pos.  The path argument is
    used for generating error messages.
    """

    h = file.read(TREC_HDR_LEN)  # XXX must be bytes
    if not h:
        return None, None
    if len(h) != TREC_HDR_LEN:
        raise FormatError(f"{path} truncated at {pos}")

    tid, stl, status, ul, dl, el = struct.unpack(">8s8scHHH", h)
    tmeta_len = TREC_HDR_LEN + ul + dl + el

    if tid <= ltid:
        raise FormatError("%s time-stamp reduction at %s: %s <= %s" %
                          (path, pos, hexify(tid), hexify(ltid)))
    ltid = tid

    tl = U64(stl)  # transaction record length - 8
    if pos + tl + 8 > file_size:
        raise FormatError("%s truncated possibly because of"
                          " damaged records at %s" % (path, pos))
    if status == Status.checkpoint:
        raise FormatError("%s checkpoint flag was not cleared at %s"
                          % (path, pos))
    if status not in b' up':
        raise FormatError("%s has invalid status '%s' at %s" %
                          (path, status, pos))

    if tmeta_len > tl:
        raise FormatError("%s has an invalid transaction header"
                          " at %s" % (path, pos))

    tpos = pos
    tend = tpos + tl

    if status != Status.undone:
        pos = tpos + tmeta_len
        file.read(ul + dl + el)  # skip transaction metadata

        i = 0
        while pos < tend:
            _pos = pos
            pos, oid = check_drec(path, file, pos, tpos, tid)
            if pos > tend:
                raise FormatError("%s has data records that extend beyond"
                                  " the transaction record; end at %s" %
                                  (path, pos))
            chatter("%10d: object oid %s #%d\n" % (_pos, hexify(oid), i),
                    level=2)
            i = i + 1

    file.seek(tend)
    rtl = file.read(8)
    if rtl != stl:
        raise FormatError("%s has inconsistent transaction length"
                          " for undone transaction at %s" % (path, pos))
    pos = tend + 8
    return pos, tid


def check_drec(path, file, pos, tpos, tid):
    """Check a data record for the current transaction record"""

    h = file.read(DREC_HDR_LEN)
    if len(h) != DREC_HDR_LEN:
        raise FormatError(f"{path} truncated at {pos}")
    oid, serial, _prev, _tloc, vlen, _plen = (
        struct.unpack(">8s8s8s8sH8s", h))
    U64(_prev)
    tloc = U64(_tloc)
    plen = U64(_plen)
    dlen = DREC_HDR_LEN + (plen or 8)

    if vlen:
        dlen = dlen + 16 + vlen
        file.seek(8, 1)
        U64(file.read(8))
        file.seek(vlen, 1)  # skip the version data

    if tloc != tpos:
        raise FormatError("%s data record exceeds transaction record "
                          "at %s: tloc %d != tpos %d" %
                          (path, pos, tloc, tpos))

    pos = pos + dlen
    if plen:
        file.seek(plen, 1)
    else:
        file.seek(8, 1)
        # _loadBack() ?

    return pos, oid


def usage():
    sys.exit(__doc__)


def main(args=None):
    if args is None:
        args = sys.argv[1:]
    import getopt

    global VERBOSE
    try:
        opts, args = getopt.getopt(args, 'v')
        if len(args) != 1:
            raise ValueError("expected one argument")
        for k, v in opts:
            if k == '-v':
                VERBOSE = VERBOSE + 1
    except (getopt.error, ValueError):
        usage()

    try:
        check(args[0])
    except FormatError as msg:
        sys.exit(msg)

    chatter("no errors detected")


if __name__ == "__main__":
    main()

1	#!/usr/bin/env python
2	##############################################################################
3	#
4	# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
5	# All Rights Reserved.
6	#
7	# This software is subject to the provisions of the Zope Public License,
8	# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
9	# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
10	# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
11	# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
12	# FOR A PARTICULAR PURPOSE
13	#
14	##############################################################################
15	"""Simple consistency checker for FileStorage.
16
17	usage: fstest.py [-v] data.fs
18
19	The fstest tool will scan all the data in a FileStorage and report an
20	error if it finds any corrupt transaction data. The tool will print a
21	message when the first error is detected, then exit.
22
23	The tool accepts one or more -v arguments. If a single -v is used, it
24	will print a line of text for each transaction record it encounters.
25	If two -v arguments are used, it will also print a line of text for
26	each object. The objects for a transaction will be printed before the
27	transaction itself.
28
29	Note: It does not check the consistency of the object pickles. It is
30	possible for the damage to occur only in the part of the file that
31	stores object pickles. Those errors will go undetected.
32	"""
33
34	import binascii	1✔
35	import struct	1✔
36	import sys	1✔
37
38	from ZODB._compat import FILESTORAGE_MAGIC	1✔
39
40
41	# The implementation is based closely on the read_index() function in
42	# ZODB.FileStorage. If anything about the FileStorage layout changes,
43	# this file will need to be udpated.
44
45
46	class FormatError(ValueError):	1✔
47	"""There is a problem with the format of the FileStorage."""
48
49
50	class Status:	1✔
51	checkpoint = b'c'	1✔
52	undone = b'u'	1✔
53
54
55	packed_version = FILESTORAGE_MAGIC	1✔
56
57	TREC_HDR_LEN = 23	1✔
58	DREC_HDR_LEN = 42	1✔
59
60	VERBOSE = 0	1✔
61
62
63	def hexify(s):	1✔
64	r"""Format an 8-bit string as hex
65
66	>>> hexify(b'\x00\xff\xaa\xcc')
67	'0x00ffaacc'
68
69	"""
70	return '0x' + binascii.hexlify(s).decode()	1✔
71
72
73	def chatter(msg, level=1):	1✔
74	if VERBOSE >= level:	1✔
75	sys.stdout.write(msg)	1✔
76
77
78	def U64(v):	1✔
79	"""Unpack an 8-byte string as a 64-bit long"""
80	h, l_ = struct.unpack(">II", v)	1✔
81	if h:	1!
82	return (h << 32) + l_	×
83	else:
84	return l_	1✔
85
86
87	def check(path):	1✔
88	with open(path, 'rb') as file:	1✔
89	file.seek(0, 2)	1✔
90	file_size = file.tell()	1✔
91	if file_size == 0:	1!
92	raise FormatError("empty file")	×
93	file.seek(0)	1✔
94	if file.read(4) != packed_version:	1!
95	raise FormatError("invalid file header")	×
96
97	pos = 4	1✔
98	tid = b'\000' * 8 # lowest possible tid to start	1✔
99	i = 0	1✔
100	while pos:	1✔
101	_pos = pos	1✔
102	pos, tid = check_trec(path, file, pos, tid, file_size)	1✔
103	if tid is not None:	1✔
104	chatter("%10d: transaction tid %s #%d \n" %	1✔
105	(_pos, hexify(tid), i))
106	i = i + 1	1✔
107
108
109	def check_trec(path, file, pos, ltid, file_size):	1✔
110	"""Read an individual transaction record from file.
111
112	Returns the pos of the next transaction and the transaction id.
113	It also leaves the file pointer set to pos. The path argument is
114	used for generating error messages.
115	"""
116
117	h = file.read(TREC_HDR_LEN) # XXX must be bytes	1✔
118	if not h:	1✔
119	return None, None	1✔
120	if len(h) != TREC_HDR_LEN:	1!
NEW 121	raise FormatError(f"{path} truncated at {pos}")	×
122
123	tid, stl, status, ul, dl, el = struct.unpack(">8s8scHHH", h)	1✔
124	tmeta_len = TREC_HDR_LEN + ul + dl + el	1✔
125
126	if tid <= ltid:	1!
127	raise FormatError("%s time-stamp reduction at %s: %s <= %s" %	×
128	(path, pos, hexify(tid), hexify(ltid)))
129	ltid = tid	1✔
130
131	tl = U64(stl) # transaction record length - 8	1✔
132	if pos + tl + 8 > file_size:	1!
133	raise FormatError("%s truncated possibly because of"	×
134	" damaged records at %s" % (path, pos))
135	if status == Status.checkpoint:	1!
136	raise FormatError("%s checkpoint flag was not cleared at %s"	×
137	% (path, pos))
138	if status not in b' up':	1!
139	raise FormatError("%s has invalid status '%s' at %s" %	×
140	(path, status, pos))
141
142	if tmeta_len > tl:	1!
143	raise FormatError("%s has an invalid transaction header"	×
144	" at %s" % (path, pos))
145
146	tpos = pos	1✔
147	tend = tpos + tl	1✔
148
149	if status != Status.undone:	1!
150	pos = tpos + tmeta_len	1✔
151	file.read(ul + dl + el) # skip transaction metadata	1✔
152
153	i = 0	1✔
154	while pos < tend:	1✔
155	_pos = pos	1✔
156	pos, oid = check_drec(path, file, pos, tpos, tid)	1✔
157	if pos > tend:	1!
158	raise FormatError("%s has data records that extend beyond"	×
159	" the transaction record; end at %s" %
160	(path, pos))
161	chatter("%10d: object oid %s #%d\n" % (_pos, hexify(oid), i),	1✔
162	level=2)
163	i = i + 1	1✔
164
165	file.seek(tend)	1✔
166	rtl = file.read(8)	1✔
167	if rtl != stl:	1!
168	raise FormatError("%s has inconsistent transaction length"	×
169	" for undone transaction at %s" % (path, pos))
170	pos = tend + 8	1✔
171	return pos, tid	1✔
172
173
174	def check_drec(path, file, pos, tpos, tid):	1✔
175	"""Check a data record for the current transaction record"""
176
177	h = file.read(DREC_HDR_LEN)	1✔
178	if len(h) != DREC_HDR_LEN:	1!
NEW 179	raise FormatError(f"{path} truncated at {pos}")	×
180	oid, serial, _prev, _tloc, vlen, _plen = (	1✔
181	struct.unpack(">8s8s8s8sH8s", h))
182	U64(_prev)	1✔
183	tloc = U64(_tloc)	1✔
184	plen = U64(_plen)	1✔
185	dlen = DREC_HDR_LEN + (plen or 8)	1✔
186
187	if vlen:	1!
188	dlen = dlen + 16 + vlen	×
189	file.seek(8, 1)	×
190	U64(file.read(8))	×
191	file.seek(vlen, 1) # skip the version data	×
192
193	if tloc != tpos:	1!
194	raise FormatError("%s data record exceeds transaction record "	×
195	"at %s: tloc %d != tpos %d" %
196	(path, pos, tloc, tpos))
197
198	pos = pos + dlen	1✔
199	if plen:	1!
200	file.seek(plen, 1)	1✔
201	else:
202	file.seek(8, 1)	×
203	# _loadBack() ?
204
205	return pos, oid	1✔
206
207
208	def usage():	1✔
209	sys.exit(__doc__)	×
210
211
212	def main(args=None):	1✔
213	if args is None:	1!
214	args = sys.argv[1:]	×
215	import getopt	1✔
216
217	global VERBOSE
218	try:	1✔
219	opts, args = getopt.getopt(args, 'v')	1✔
220	if len(args) != 1:	1!
221	raise ValueError("expected one argument")	×
222	for k, v in opts:	1✔
223	if k == '-v':	1!
224	VERBOSE = VERBOSE + 1	1✔
225	except (getopt.error, ValueError):	×
226	usage()	×
227
228	try:	1✔
229	check(args[0])	1✔
230	except FormatError as msg:	×
231	sys.exit(msg)	×
232
233	chatter("no errors detected")	1✔
234
235
236	if __name__ == "__main__":	1!
237	main()	×

zopefoundation / ZODB / 18153960591

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous