18153960591

Committed 01 Oct 2025 06:50AM UTC coverage: 83.781% (-0.03%) from 83.811%

Build # 18153960591

Build Type

Pull #415

github

Committed by

web-flow

Commit Message

Update docs/articles/old-guide/convert_zodb_guide.py

Co-authored-by: Michael Howitz <icemac@gmx.net>

Pull Request Pull Request #415: Apply the latest zope.meta templates

Run Details

2441 of 3542 branches covered (68.92%)

193 of 257 new or added lines in 48 files covered. (75.1%)

12 existing lines in 6 files now uncovered.

13353 of 15938 relevant lines covered (83.78%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.91

/src/ZODB/scripts/analyze.py

#!/usr/bin/env python

# Based on a transaction analyzer by Matt Kromer.

import sys
from io import BytesIO

from ZODB._compat import PersistentUnpickler
from ZODB.FileStorage import FileStorage


class FakeError(Exception):
    def __init__(self, module, name):
        Exception.__init__(self)
        self.module = module
        self.name = name


def fake_find_class(module, name):
    raise FakeError(module, name)


def FakeUnpickler(f):
    unpickler = PersistentUnpickler(fake_find_class, None, f)
    return unpickler


class Report:
    def __init__(self):
        self.OIDMAP = {}
        self.TYPEMAP = {}
        self.TYPESIZE = {}
        self.FREEMAP = {}
        self.USEDMAP = {}
        self.TIDS = 0
        self.OIDS = 0
        self.DBYTES = 0
        self.COIDS = 0
        self.CBYTES = 0
        self.FOIDS = 0
        self.FBYTES = 0


def shorten(s, n):
    length = len(s)
    if length <= n:
        return s
    while len(s) + 3 > n:  # account for ...
        i = s.find(".")
        if i == -1:
            # In the worst case, just return the rightmost n bytes
            return s[-n:]
        else:
            s = s[i + 1:]
            length = len(s)
    return "..." + s


def report(rep):
    print(f"Processed {rep.OIDS} records in {rep.TIDS} transactions")
    print(f"Average record size is {rep.DBYTES * 1.0 / rep.OIDS:7.2f} bytes")
    print("Average transaction size is"
          f" {rep.DBYTES * 1.0 / rep.TIDS:7.2f} bytes")

    print("Types used:")
    fmt = "%-46s %7s %9s %6s %7s"
    fmtp = "%-46s %7d %9d %5.1f%% %7.2f"  # per-class format
    fmts = "%46s %7d %8dk %5.1f%% %7.2f"  # summary format
    print(fmt % ("Class Name", "Count", "TBytes", "Pct", "AvgSize"))
    print(fmt % ('-' * 46, '-' * 7, '-' * 9, '-' * 5, '-' * 7))
    typemap = sorted(rep.TYPEMAP)
    cumpct = 0.0
    for t in typemap:
        pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
        cumpct += pct
        print(fmtp % (shorten(t, 46), rep.TYPEMAP[t], rep.TYPESIZE[t],
                      pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t]))

    print(fmt % ('=' * 46, '=' * 7, '=' * 9, '=' * 5, '=' * 7))
    print("%46s %7d %9s %6s %6.2fk" % (
        'Total Transactions', rep.TIDS, ' ', ' ',
        rep.DBYTES * 1.0 / rep.TIDS / 1024.0))
    print(fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
                  rep.DBYTES * 1.0 / rep.OIDS))

    print(fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,
                  rep.CBYTES * 100.0 / rep.DBYTES,
                  rep.CBYTES * 1.0 / rep.COIDS))
    if rep.FOIDS:
        print(fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,
                      rep.FBYTES * 100.0 / rep.DBYTES,
                      rep.FBYTES * 1.0 / rep.FOIDS))


def analyze(path):
    fs = FileStorage(path, read_only=1)
    fsi = fs.iterator()
    report = Report()
    for txn in fsi:
        analyze_trans(report, txn)
    return report


def analyze_trans(report, txn):
    report.TIDS += 1
    for rec in txn:
        analyze_rec(report, rec)


def get_type(record):
    try:
        unpickled = FakeUnpickler(BytesIO(record.data)).load()
    except FakeError as err:
        return f"{err.module}.{err.name}"
    classinfo = unpickled[0]
    if isinstance(classinfo, tuple):
        mod, klass = classinfo
        return f"{mod}.{klass}"
    else:
        return str(classinfo)


def analyze_rec(report, record):
    oid = record.oid
    report.OIDS += 1
    if record.data is None:
        # No pickle -- aborted version or undo of object creation.
        return
    try:
        size = len(record.data)  # Ignores various overhead
        report.DBYTES += size
        if oid not in report.OIDMAP:
            type = get_type(record)
            report.OIDMAP[oid] = type
            report.USEDMAP[oid] = size
            report.COIDS += 1
            report.CBYTES += size
        else:
            type = report.OIDMAP[oid]
            fsize = report.USEDMAP[oid]
            report.FREEMAP[oid] = report.FREEMAP.get(oid, 0) + fsize
            report.USEDMAP[oid] = size
            report.FOIDS += 1
            report.FBYTES += fsize
            report.CBYTES += size - fsize
        report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1
        report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size
    except Exception as err:
        print(err)


if __name__ == "__main__":
    path = sys.argv[1]
    report(analyze(path))

1	#!/usr/bin/env python
2
3	# Based on a transaction analyzer by Matt Kromer.
4
5	import sys	1✔
6	from io import BytesIO	1✔
7
8	from ZODB._compat import PersistentUnpickler	1✔
9	from ZODB.FileStorage import FileStorage	1✔
10
11
12	class FakeError(Exception):	1✔
13	def __init__(self, module, name):	1✔
14	Exception.__init__(self)	1✔
15	self.module = module	1✔
16	self.name = name	1✔
17
18
19	def fake_find_class(module, name):	1✔
20	raise FakeError(module, name)	1✔
21
22
23	def FakeUnpickler(f):	1✔
24	unpickler = PersistentUnpickler(fake_find_class, None, f)	1✔
25	return unpickler	1✔
26
27
28	class Report:	1✔
29	def __init__(self):	1✔
30	self.OIDMAP = {}	1✔
31	self.TYPEMAP = {}	1✔
32	self.TYPESIZE = {}	1✔
33	self.FREEMAP = {}	1✔
34	self.USEDMAP = {}	1✔
35	self.TIDS = 0	1✔
36	self.OIDS = 0	1✔
37	self.DBYTES = 0	1✔
38	self.COIDS = 0	1✔
39	self.CBYTES = 0	1✔
40	self.FOIDS = 0	1✔
41	self.FBYTES = 0	1✔
42
43
44	def shorten(s, n):	1✔
45	length = len(s)	×
46	if length <= n:	×
47	return s	×
48	while len(s) + 3 > n: # account for ...	×
49	i = s.find(".")	×
50	if i == -1:	×
51	# In the worst case, just return the rightmost n bytes
52	return s[-n:]	×
53	else:
54	s = s[i + 1:]	×
55	length = len(s)	×
56	return "..." + s	×
57
58
59	def report(rep):	1✔
60	print(f"Processed {rep.OIDS} records in {rep.TIDS} transactions")	×
61	print(f"Average record size is {rep.DBYTES * 1.0 / rep.OIDS:7.2f} bytes")	×
62	print("Average transaction size is"	×
63	f" {rep.DBYTES * 1.0 / rep.TIDS:7.2f} bytes")
64
65	print("Types used:")	×
66	fmt = "%-46s %7s %9s %6s %7s"	×
67	fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format	×
68	fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format	×
69	print(fmt % ("Class Name", "Count", "TBytes", "Pct", "AvgSize"))	×
NEW 70	print(fmt % ('-' * 46, '-' * 7, '-' * 9, '-' * 5, '-' * 7))	×
71	typemap = sorted(rep.TYPEMAP)	×
72	cumpct = 0.0	×
73	for t in typemap:	×
74	pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES	×
75	cumpct += pct	×
76	print(fmtp % (shorten(t, 46), rep.TYPEMAP[t], rep.TYPESIZE[t],	×
77	pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t]))
78
NEW 79	print(fmt % ('=' * 46, '=' * 7, '=' * 9, '=' * 5, '=' * 7))	×
80	print("%46s %7d %9s %6s %6.2fk" % (	×
81	'Total Transactions', rep.TIDS, ' ', ' ',
82	rep.DBYTES * 1.0 / rep.TIDS / 1024.0))
83	print(fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,	×
84	rep.DBYTES * 1.0 / rep.OIDS))
85
86	print(fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,	×
87	rep.CBYTES * 100.0 / rep.DBYTES,
88	rep.CBYTES * 1.0 / rep.COIDS))
89	if rep.FOIDS:	×
90	print(fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,	×
91	rep.FBYTES * 100.0 / rep.DBYTES,
92	rep.FBYTES * 1.0 / rep.FOIDS))
93
94
95	def analyze(path):	1✔
96	fs = FileStorage(path, read_only=1)	×
97	fsi = fs.iterator()	×
98	report = Report()	×
99	for txn in fsi:	×
100	analyze_trans(report, txn)	×
101	return report	×
102
103
104	def analyze_trans(report, txn):	1✔
105	report.TIDS += 1	1✔
106	for rec in txn:	1✔
107	analyze_rec(report, rec)	1✔
108
109
110	def get_type(record):	1✔
111	try:	1✔
112	unpickled = FakeUnpickler(BytesIO(record.data)).load()	1✔
113	except FakeError as err:	1✔
114	return f"{err.module}.{err.name}"	1✔
115	classinfo = unpickled[0]	1✔
116	if isinstance(classinfo, tuple):	1!
117	mod, klass = classinfo	1✔
118	return f"{mod}.{klass}"	1✔
119	else:
120	return str(classinfo)	×
121
122
123	def analyze_rec(report, record):	1✔
124	oid = record.oid	1✔
125	report.OIDS += 1	1✔
126	if record.data is None:	1!
127	# No pickle -- aborted version or undo of object creation.
128	return	×
129	try:	1✔
130	size = len(record.data) # Ignores various overhead	1✔
131	report.DBYTES += size	1✔
132	if oid not in report.OIDMAP:	1✔
133	type = get_type(record)	1✔
134	report.OIDMAP[oid] = type	1✔
135	report.USEDMAP[oid] = size	1✔
136	report.COIDS += 1	1✔
137	report.CBYTES += size	1✔
138	else:
139	type = report.OIDMAP[oid]	1✔
140	fsize = report.USEDMAP[oid]	1✔
141	report.FREEMAP[oid] = report.FREEMAP.get(oid, 0) + fsize	1✔
142	report.USEDMAP[oid] = size	1✔
143	report.FOIDS += 1	1✔
144	report.FBYTES += fsize	1✔
145	report.CBYTES += size - fsize	1✔
146	report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1	1✔
147	report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size	1✔
148	except Exception as err:	×
149	print(err)	×
150
151
152	if __name__ == "__main__":	1!
153	path = sys.argv[1]	×
154	report(analyze(path))	×

zopefoundation / ZODB / 18153960591

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous