18153960591

Committed 01 Oct 2025 06:50AM UTC coverage: 83.781% (-0.03%) from 83.811%

Build # 18153960591

Build Type

Pull #415

github

Committed by

web-flow

Commit Message

Update docs/articles/old-guide/convert_zodb_guide.py

Co-authored-by: Michael Howitz <icemac@gmx.net>

Pull Request Pull Request #415: Apply the latest zope.meta templates

Run Details

2441 of 3542 branches covered (68.92%)

193 of 257 new or added lines in 48 files covered. (75.1%)

12 existing lines in 6 files now uncovered.

13353 of 15938 relevant lines covered (83.78%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

65.96

/src/ZODB/fsrecover.py

##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Simple script for repairing damaged FileStorage files.

Usage: %s [-f] [-v level] [-p] [-P seconds] input output

Recover data from a FileStorage data file, skipping over damaged data.  Any
damaged data will be lost.  This could lead to useless output if critical
data is lost.

Options:

    -f
       Overwrite output file even if it exists.

    -v level

       Set the verbosity level:

         0 -- show progress indicator (default)

         1 -- show transaction times and sizes

         2 -- show transaction times and sizes, and show object (record)
              ids, versions, and sizes

    -p

       Copy partial transactions.  If a data record in the middle of a
       transaction is bad, the data up to the bad data are packed.  The
       output record is marked as packed.  If this option is not used,
       transactions with any bad data are skipped.

    -P t

       Pack data to t seconds in the past.  Note that if the "-p" option is
       used, then t should be 0.


Important:  The ZODB package must be importable.  You may need to adjust
            PYTHONPATH accordingly.
"""

import getopt
import os
import sys
import time
from struct import unpack


# Algorithm:
#
#     position to start of input
#     while 1:
#         if end of file:
#             break
#         try:
#             copy_transaction
#          except:
#             scan for transaction
#             continue


try:
    import ZODB
except ImportError:
    if os.path.exists('ZODB'):
        sys.path.append('.')
    elif os.path.exists('FileStorage.py'):
        sys.path.append('..')
    import ZODB

from persistent.TimeStamp import TimeStamp

import ZODB.FileStorage
from ZODB.FileStorage import TransactionRecord
from ZODB.utils import as_text
from ZODB.utils import u64


def die(mess='', show_docstring=False):
    if mess:
        print(mess + '\n', file=sys.stderr)
    if show_docstring:
        print(__doc__ % sys.argv[0], file=sys.stderr)
    sys.exit(1)


class ErrorFound(Exception):
    pass


def error(mess, *args):
    raise ErrorFound(mess % args)


def read_txn_header(f, pos, file_size, outp, ltid):
    # Read the transaction record
    f.seek(pos)
    h = f.read(23)
    if len(h) < 23:
        raise EOFError

    tid, stl, status, ul, dl, el = unpack(">8s8scHHH", h)
    status = as_text(status)
    tl = u64(stl)

    if pos + (tl + 8) > file_size:
        error("bad transaction length at %s", pos)

    if tl < (23 + ul + dl + el):
        error("invalid transaction length, %s, at %s", tl, pos)

    if ltid and tid < ltid:
        error("time-stamp reducation %s < %s, at %s", u64(tid), u64(ltid), pos)

    if status == "c":
        truncate(f, pos, file_size, outp)
        raise EOFError

    if status not in " up":
        error("invalid status, %r, at %s", status, pos)

    tpos = pos
    tend = tpos + tl

    if status == "u":
        # Undone transaction, skip it
        f.seek(tend)
        h = f.read(8)
        if h != stl:
            error("inconsistent transaction length at %s", pos)
        pos = tend + 8
        return pos, None, tid

    pos = tpos + (23 + ul + dl + el)
    user = f.read(ul)
    description = f.read(dl)
    ext = f.read(el)

    result = TransactionRecord(tid, status, user, description, ext, pos, tend,
                               f, tpos)
    pos = tend

    # Read the (intentionally redundant) transaction length
    f.seek(pos)
    h = f.read(8)
    if h != stl:
        error("redundant transaction length check failed at %s", pos)
    pos += 8

    return pos, result, tid


def truncate(f, pos, file_size, outp):
    """Copy data from pos to end of f to a .trNNN file."""

    # _trname is global so that the test suite can know the path too (in
    # order to delete the file when the test ends).
    global _trname

    i = 0
    while 1:
        _trname = outp + ".tr%d" % i
        if os.path.exists(_trname):
            i += 1
        else:
            break
    tr = open(_trname, "wb")
    copy(f, tr, file_size - pos)
    f.seek(pos)
    tr.close()


def copy(src, dst, n):
    while n:
        buf = src.read(8096)
        if not buf:
            break
        if len(buf) > n:
            buf = buf[:n]
        dst.write(buf)
        n -= len(buf)


def scan(f, pos):
    """Return a potential transaction location following pos in f.

    This routine scans forward from pos looking for the last data
    record in a transaction.  A period '.' always occurs at the end of
    a pickle, and an 8-byte transaction length follows the last
    pickle.  If a period is followed by a plausible 8-byte transaction
    length, assume that we have found the end of a transaction.

    The caller should try to verify that the returned location is
    actually a transaction header.
    """
    while 1:
        f.seek(pos)
        data = f.read(8096)
        if not data:
            return 0

        s = 0
        while 1:
            l_ = data.find(b".", s)
            if l_ < 0:
                pos += len(data)
                break
            # If we are less than 8 bytes from the end of the
            # string, we need to read more data.
            s = l_ + 1
            if s > len(data) - 8:
                pos += l_
                break
            tl = u64(data[s:s + 8])
            if tl < pos:
                return pos + s + 8


def iprogress(i):
    if i % 2:
        print(".", end=' ')
    else:
        print((i / 2) % 10, end=' ')
    sys.stdout.flush()


def progress(p):
    for i in range(p):
        iprogress(i)


def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "fv:pP:")
    except getopt.error as msg:
        die(str(msg), show_docstring=True)

    if len(args) != 2:
        die("two positional arguments required", show_docstring=True)
    inp, outp = args

    force = partial = False
    verbose = 0
    pack = None
    for opt, v in opts:
        if opt == "-v":
            verbose = int(v)
        elif opt == "-p":
            partial = True
        elif opt == "-f":
            force = True
        elif opt == "-P":
            pack = time.time() - float(v)

    recover(inp, outp, verbose, partial, force, pack)


def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
    print("Recovering", inp, "into", outp)

    if os.path.exists(outp) and not force:
        die("%s exists" % outp)

    f = open(inp, "rb")
    if f.read(4) != ZODB.FileStorage.packed_version:
        die("input is not a file storage")

    f.seek(0, 2)
    file_size = f.tell()

    ofs = ZODB.FileStorage.FileStorage(outp, create=1)
    _ts = None
    ok = 1
    prog1 = 0
    undone = 0

    pos = 4
    ltid = None
    while pos:
        try:
            npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
        except EOFError:
            break
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as err:
            print("error reading txn header:", err)
            if not verbose:
                progress(prog1)
            pos = scan(f, pos)
            if verbose > 1:
                print("looking for valid txn header at", pos)
            continue
        ltid = tid

        if txn is None:
            undone = undone + npos - pos
            pos = npos
            continue
        else:
            pos = npos

        tid = txn.tid

        if _ts is None:
            _ts = TimeStamp(tid)
        else:
            t = TimeStamp(tid)
            if t <= _ts:
                if ok:
                    print(f"Time stamps out of order {_ts}, {t}")
                ok = 0
                _ts = t.laterThan(_ts)
                tid = _ts.raw()
            else:
                _ts = t
                if not ok:
                    print("Time stamps back in order %s" % (t))
                    ok = 1

        ofs.tpc_begin(txn, tid, txn.status)

        if verbose:
            print("begin", pos, _ts, end=' ')
            if verbose > 1:
                print()
            sys.stdout.flush()

        nrec = 0
        try:
            for r in txn:
                if verbose > 1:
                    if r.data is None:
                        l_ = "bp"
                    else:
                        l_ = len(r.data)

                    print("%7d %s" % (u64(r.oid), l_))
                ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
                            txn)
                nrec += 1
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as err:
            if partial and nrec:
                ofs._status = "p"
                ofs.tpc_vote(txn)
                ofs.tpc_finish(txn)
                if verbose:
                    print("partial")
            else:
                ofs.tpc_abort(txn)
            print("error copying transaction:", err)
            if not verbose:
                progress(prog1)
            pos = scan(f, pos)
            if verbose > 1:
                print("looking for valid txn header at", pos)
        else:
            ofs.tpc_vote(txn)
            ofs.tpc_finish(txn)
            if verbose:
                print("finish")
                sys.stdout.flush()

        if not verbose:
            prog = pos * 20 / file_size
            while prog > prog1:
                prog1 = prog1 + 1
                iprogress(prog1)

    bad = file_size - undone - ofs._pos

    print("\n%s bytes removed during recovery" % bad)
    if undone:
        print("%s bytes of undone transaction data were skipped" % undone)

    if pack is not None:
        print("Packing ...")
        from ZODB.serialize import referencesf
        ofs.pack(pack, referencesf)

    ofs.close()
    f.close()


if __name__ == "__main__":
    main()

1	##############################################################################
2	#
3	# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
4	# All Rights Reserved.
5	#
6	# This software is subject to the provisions of the Zope Public License,
7	# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
8	# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
9	# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
10	# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
11	# FOR A PARTICULAR PURPOSE
12	#
13	##############################################################################
14	"""Simple script for repairing damaged FileStorage files.
15
16	Usage: %s [-f] [-v level] [-p] [-P seconds] input output
17
18	Recover data from a FileStorage data file, skipping over damaged data. Any
19	damaged data will be lost. This could lead to useless output if critical
20	data is lost.
21
22	Options:
23
24	-f
25	Overwrite output file even if it exists.
26
27	-v level
28
29	Set the verbosity level:
30
31	0 -- show progress indicator (default)
32
33	1 -- show transaction times and sizes
34
35	2 -- show transaction times and sizes, and show object (record)
36	ids, versions, and sizes
37
38	-p
39
40	Copy partial transactions. If a data record in the middle of a
41	transaction is bad, the data up to the bad data are packed. The
42	output record is marked as packed. If this option is not used,
43	transactions with any bad data are skipped.
44
45	-P t
46
47	Pack data to t seconds in the past. Note that if the "-p" option is
48	used, then t should be 0.
49
50
51	Important: The ZODB package must be importable. You may need to adjust
52	PYTHONPATH accordingly.
53	"""
54
55	import getopt	1✔
56	import os	1✔
57	import sys	1✔
58	import time	1✔
59	from struct import unpack	1✔
60
61
62	# Algorithm:
63	#
64	# position to start of input
65	# while 1:
66	# if end of file:
67	# break
68	# try:
69	# copy_transaction
70	# except:
71	# scan for transaction
72	# continue
73
74
75	try:	1✔
76	import ZODB	1✔
77	except ImportError:
78	if os.path.exists('ZODB'):
79	sys.path.append('.')
80	elif os.path.exists('FileStorage.py'):
81	sys.path.append('..')
82	import ZODB
83
84	from persistent.TimeStamp import TimeStamp	1✔
85
86	import ZODB.FileStorage	1✔
87	from ZODB.FileStorage import TransactionRecord	1✔
88	from ZODB.utils import as_text	1✔
89	from ZODB.utils import u64	1✔
90
91
92	def die(mess='', show_docstring=False):	1✔
93	if mess:	×
94	print(mess + '\n', file=sys.stderr)	×
95	if show_docstring:	×
96	print(__doc__ % sys.argv[0], file=sys.stderr)	×
97	sys.exit(1)	×
98
99
100	class ErrorFound(Exception):	1✔
101	pass	1✔
102
103
104	def error(mess, *args):	1✔
105	raise ErrorFound(mess % args)	1✔
106
107
108	def read_txn_header(f, pos, file_size, outp, ltid):	1✔
109	# Read the transaction record
110	f.seek(pos)	1✔
111	h = f.read(23)	1✔
112	if len(h) < 23:	1✔
113	raise EOFError	1✔
114
115	tid, stl, status, ul, dl, el = unpack(">8s8scHHH", h)	1✔
116	status = as_text(status)	1✔
117	tl = u64(stl)	1✔
118
119	if pos + (tl + 8) > file_size:	1!
120	error("bad transaction length at %s", pos)	×
121
122	if tl < (23 + ul + dl + el):	1✔
123	error("invalid transaction length, %s, at %s", tl, pos)	1✔
124
125	if ltid and tid < ltid:	1!
126	error("time-stamp reducation %s < %s, at %s", u64(tid), u64(ltid), pos)	×
127
128	if status == "c":	1✔
129	truncate(f, pos, file_size, outp)	1✔
130	raise EOFError	1✔
131
132	if status not in " up":	1!
133	error("invalid status, %r, at %s", status, pos)	×
134
135	tpos = pos	1✔
136	tend = tpos + tl	1✔
137
138	if status == "u":	1!
139	# Undone transaction, skip it
140	f.seek(tend)	×
141	h = f.read(8)	×
142	if h != stl:	×
143	error("inconsistent transaction length at %s", pos)	×
144	pos = tend + 8	×
145	return pos, None, tid	×
146
147	pos = tpos + (23 + ul + dl + el)	1✔
148	user = f.read(ul)	1✔
149	description = f.read(dl)	1✔
150	ext = f.read(el)	1✔
151
152	result = TransactionRecord(tid, status, user, description, ext, pos, tend,	1✔
153	f, tpos)
154	pos = tend	1✔
155
156	# Read the (intentionally redundant) transaction length
157	f.seek(pos)	1✔
158	h = f.read(8)	1✔
159	if h != stl:	1✔
160	error("redundant transaction length check failed at %s", pos)	1✔
161	pos += 8	1✔
162
163	return pos, result, tid	1✔
164
165
166	def truncate(f, pos, file_size, outp):	1✔
167	"""Copy data from pos to end of f to a .trNNN file."""
168
169	# _trname is global so that the test suite can know the path too (in
170	# order to delete the file when the test ends).
171	global _trname
172
173	i = 0	1✔
174	while 1:	1✔
175	_trname = outp + ".tr%d" % i	1✔
176	if os.path.exists(_trname):	1!
177	i += 1	×
178	else:
179	break	1✔
180	tr = open(_trname, "wb")	1✔
181	copy(f, tr, file_size - pos)	1✔
182	f.seek(pos)	1✔
183	tr.close()	1✔
184
185
186	def copy(src, dst, n):	1✔
187	while n:	1!
188	buf = src.read(8096)	1✔
189	if not buf:	1✔
190	break	1✔
191	if len(buf) > n:	1!
192	buf = buf[:n]	×
193	dst.write(buf)	1✔
194	n -= len(buf)	1✔
195
196
197	def scan(f, pos):	1✔
198	"""Return a potential transaction location following pos in f.
199
200	This routine scans forward from pos looking for the last data
201	record in a transaction. A period '.' always occurs at the end of
202	a pickle, and an 8-byte transaction length follows the last
203	pickle. If a period is followed by a plausible 8-byte transaction
204	length, assume that we have found the end of a transaction.
205
206	The caller should try to verify that the returned location is
207	actually a transaction header.
208	"""
209	while 1:	1✔
210	f.seek(pos)	1✔
211	data = f.read(8096)	1✔
212	if not data:	1✔
213	return 0	1✔
214
215	s = 0	1✔
216	while 1:	1✔
217	l_ = data.find(b".", s)	1✔
218	if l_ < 0:	1✔
219	pos += len(data)	1✔
220	break	1✔
221	# If we are less than 8 bytes from the end of the
222	# string, we need to read more data.
223	s = l_ + 1	1✔
224	if s > len(data) - 8:	1!
225	pos += l_	×
226	break	×
227	tl = u64(data[s:s + 8])	1✔
228	if tl < pos:	1✔
229	return pos + s + 8	1✔
230
231
232	def iprogress(i):	1✔
233	if i % 2:	1✔
234	print(".", end=' ')	1✔
235	else:
236	print((i / 2) % 10, end=' ')	1✔
237	sys.stdout.flush()	1✔
238
239
240	def progress(p):	1✔
241	for i in range(p):	1✔
242	iprogress(i)	1✔
243
244
245	def main():	1✔
246	try:	×
247	opts, args = getopt.getopt(sys.argv[1:], "fv:pP:")	×
248	except getopt.error as msg:	×
249	die(str(msg), show_docstring=True)	×
250
251	if len(args) != 2:	×
252	die("two positional arguments required", show_docstring=True)	×
253	inp, outp = args	×
254
255	force = partial = False	×
256	verbose = 0	×
257	pack = None	×
258	for opt, v in opts:	×
259	if opt == "-v":	×
260	verbose = int(v)	×
261	elif opt == "-p":	×
262	partial = True	×
263	elif opt == "-f":	×
264	force = True	×
265	elif opt == "-P":	×
266	pack = time.time() - float(v)	×
267
268	recover(inp, outp, verbose, partial, force, pack)	×
269
270
271	def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):	1✔
272	print("Recovering", inp, "into", outp)	1✔
273
274	if os.path.exists(outp) and not force:	1!
275	die("%s exists" % outp)	×
276
277	f = open(inp, "rb")	1✔
278	if f.read(4) != ZODB.FileStorage.packed_version:	1!
279	die("input is not a file storage")	×
280
281	f.seek(0, 2)	1✔
282	file_size = f.tell()	1✔
283
284	ofs = ZODB.FileStorage.FileStorage(outp, create=1)	1✔
285	_ts = None	1✔
286	ok = 1	1✔
287	prog1 = 0	1✔
288	undone = 0	1✔
289
290	pos = 4	1✔
291	ltid = None	1✔
292	while pos:	1✔
293	try:	1✔
294	npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)	1✔
295	except EOFError:	1✔
296	break	1✔
297	except (KeyboardInterrupt, SystemExit):	1✔
298	raise	×
299	except Exception as err:	1✔
300	print("error reading txn header:", err)	1✔
301	if not verbose:	1!
302	progress(prog1)	1✔
303	pos = scan(f, pos)	1✔
304	if verbose > 1:	1!
305	print("looking for valid txn header at", pos)	×
306	continue	1✔
307	ltid = tid	1✔
308
309	if txn is None:	1!
310	undone = undone + npos - pos	×
311	pos = npos	×
312	continue	×
313	else:
314	pos = npos	1✔
315
316	tid = txn.tid	1✔
317
318	if _ts is None:	1✔
319	_ts = TimeStamp(tid)	1✔
320	else:
321	t = TimeStamp(tid)	1✔
322	if t <= _ts:	1!
323	if ok:	×
NEW 324	print(f"Time stamps out of order {_ts}, {t}")	×
325	ok = 0	×
326	_ts = t.laterThan(_ts)	×
327	tid = _ts.raw()	×
328	else:
329	_ts = t	1✔
330	if not ok:	1!
331	print("Time stamps back in order %s" % (t))	×
332	ok = 1	×
333
334	ofs.tpc_begin(txn, tid, txn.status)	1✔
335
336	if verbose:	1!
337	print("begin", pos, _ts, end=' ')	×
338	if verbose > 1:	×
339	print()	×
340	sys.stdout.flush()	×
341
342	nrec = 0	1✔
343	try:	1✔
344	for r in txn:	1✔
345	if verbose > 1:	1!
346	if r.data is None:	×
347	l_ = "bp"	×
348	else:
349	l_ = len(r.data)	×
350
351	print("%7d %s" % (u64(r.oid), l_))	×
352	ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,	1✔
353	txn)
354	nrec += 1	1✔
355	except (KeyboardInterrupt, SystemExit):	×
356	raise	×
357	except Exception as err:	×
358	if partial and nrec:	×
359	ofs._status = "p"	×
360	ofs.tpc_vote(txn)	×
361	ofs.tpc_finish(txn)	×
362	if verbose:	×
363	print("partial")	×
364	else:
365	ofs.tpc_abort(txn)	×
366	print("error copying transaction:", err)	×
367	if not verbose:	×
368	progress(prog1)	×
369	pos = scan(f, pos)	×
370	if verbose > 1:	×
371	print("looking for valid txn header at", pos)	×
372	else:
373	ofs.tpc_vote(txn)	1✔
374	ofs.tpc_finish(txn)	1✔
375	if verbose:	1!
376	print("finish")	×
377	sys.stdout.flush()	×
378
379	if not verbose:	1!
380	prog = pos * 20 / file_size	1✔
381	while prog > prog1:	1✔
382	prog1 = prog1 + 1	1✔
383	iprogress(prog1)	1✔
384
385	bad = file_size - undone - ofs._pos	1✔
386
387	print("\n%s bytes removed during recovery" % bad)	1✔
388	if undone:	1!
389	print("%s bytes of undone transaction data were skipped" % undone)	×
390
391	if pack is not None:	1!
392	print("Packing ...")	1✔
393	from ZODB.serialize import referencesf	1✔
394	ofs.pack(pack, referencesf)	1✔
395
396	ofs.close()	1✔
397	f.close()	1✔
398
399
400	if __name__ == "__main__":	1!
401	main()	×

zopefoundation / ZODB / 18153960591

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous