13228537945

Committed 09 Feb 2025 06:34PM UTC coverage: 35.131%. Remained the same

Build # 13228537945

Build Type

Pull #644

github

Committed by

web-flow

Commit Message

Merge 8b3aeb3fa into 16f1fc1f8

Pull Request Pull Request #644: Ignore archived projects

Run Details

437 of 1188 branches covered (36.78%)

645 of 1836 relevant lines covered (35.13%)

3.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.16

/tools/gen_addon_readme.py

# License AGPLv3 (https://www.gnu.org/licenses/agpl-3.0-standalone.html)
# Copyright (c) 2018 ACSONE SA/NV
# Copyright (c) 2018 GRAP (http://www.grap.coop)

import atexit
import functools
import os
import re
import sys
import tempfile
from pathlib import Path
from typing import Union
from urllib.parse import urljoin

import click
import pypandoc
from docutils.core import publish_file
from jinja2 import Template

from ._hash import hash
from .gitutils import commit_if_needed
from .manifest import NoManifestFound, find_addons, get_manifest_path, read_manifest

if sys.version_info >= (3, 8):
    from typing import Literal
else:
    from typing_extensions import Literal


class FragmentProperties:
    def __init__(self, level: int):
        self.level = level


FragmentFormat = Literal[".rst", ".md"]

FRAGMENTS_DIR = "readme"

FRAGMENTS = {
    "DESCRIPTION": FragmentProperties(level=2),
    "CONTEXT": FragmentProperties(level=2),
    "INSTALL": FragmentProperties(level=2),
    "CONFIGURE": FragmentProperties(level=2),
    "USAGE": FragmentProperties(level=2),
    "ROADMAP": FragmentProperties(level=2),
    "DEVELOP": FragmentProperties(level=2),
    "CONTRIBUTORS": FragmentProperties(level=3),
    "CREDITS": FragmentProperties(level=3),
    "HISTORY": FragmentProperties(level=2),
}

LICENSE_BADGES = {
    "AGPL-3": (
        "https://img.shields.io/badge/licence-AGPL--3-blue.png",
        "http://www.gnu.org/licenses/agpl-3.0-standalone.html",
        "License: AGPL-3",
    ),
    "LGPL-3": (
        "https://img.shields.io/badge/licence-LGPL--3-blue.png",
        "http://www.gnu.org/licenses/lgpl-3.0-standalone.html",
        "License: LGPL-3",
    ),
    "GPL-3": (
        "https://img.shields.io/badge/licence-GPL--3-blue.png",
        "http://www.gnu.org/licenses/gpl-3.0-standalone.html",
        "License: GPL-3",
    ),
}

DEVELOPMENT_STATUS_BADGES = {
    "mature": (
        "https://img.shields.io/badge/maturity-Mature-brightgreen.png",
        "https://odoo-community.org/page/development-status",
        "Mature",
    ),
    "production/stable": (
        "https://img.shields.io/badge/maturity-Production%2FStable-green.png",
        "https://odoo-community.org/page/development-status",
        "Production/Stable",
    ),
    "beta": (
        "https://img.shields.io/badge/maturity-Beta-yellow.png",
        "https://odoo-community.org/page/development-status",
        "Beta",
    ),
    "alpha": (
        "https://img.shields.io/badge/maturity-Alpha-red.png",
        "https://odoo-community.org/page/development-status",
        "Alpha",
    ),
}

# this comes from pypa/readme_renderer
RST2HTML_SETTINGS = {
    # Prevent local files from being included into the rendered output.
    # This is a security concern because people can insert files
    # that are part of the system, such as /etc/passwd.
    "file_insertion_enabled": False,
    # Halt rendering and throw an exception if there was any errors or
    # warnings from docutils.
    "halt_level": 2,
    # Output math blocks as LaTeX that can be interpreted by MathJax for
    # a prettier display of Math formulas.
    "math_output": "MathJax",
    # Disable raw html as enabling it is a security risk, we do not want
    # people to be able to include any old HTML in the final output.
    "raw_enabled": False,
    # Use typographic quotes, and transform --, ---, and ... into their
    # typographic counterparts.
    "smart_quotes": True,
    # Use the short form of syntax highlighting so that the generated
    # Pygments CSS can be used to style the output.
    "syntax_highlight": "short",
    # Since odoo/odoo@8d06889, Odoo emits a warning
    # if index.html contains an xml declaration
    "xml_declaration": False,
    # ...but even for previous versions we don't need
    # the xml declaration as docutils adds a <meta> tag:
    # <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    # utf-8 is default value for output_encoding
    # but let's make it explicit here:
    "output_encoding": "utf-8",
}

# GitHub Flavored Markdown
# - raw html is disabled
# - auto identifiers is disabled because pylint-odoo complains about them (Hyperlink
#   target "..;" is not referenced.)
PANDOC_MARKDOWN_FORMAT = "gfm-raw_html-gfm_auto_identifiers"


@functools.lru_cache(maxsize=None)
def ensure_pandoc_installed() -> None:
    pypandoc.ensure_pandoc_installed(delete_installer=True)


def make_runboat_badge(repo, branch):
    return (
        "https://img.shields.io/badge/runboat-Try%20me-875A7B.png",
        "https://runboat.odoo-community.org/builds?"
        "repo=OCA/{repo}&target_branch={branch}".format(**locals()),
        "Try me on Runboat",
    )


def make_weblate_badge(repo_name, branch, addon_name):
    branch = branch.replace(".", "-")
    return (
        "https://img.shields.io/badge/weblate-Translate%20me-F47D42.png",
        "https://translation.odoo-community.org/projects/"
        "{repo_name}-{branch}/{repo_name}-{branch}-{addon_name}".format(**locals()),
        "Translate me on Weblate",
    )


def make_repo_badge(org_name, repo_name, branch, addon_name):
    badge_repo_name = repo_name.replace("-", "--")
    badge_org_name = org_name.replace("-", "--")
    return (
        "https://img.shields.io/badge/github-{badge_org_name}%2F{badge_repo_name}"
        "-lightgray.png?logo=github".format(**locals()),
        "https://github.com/{org_name}/{repo_name}/tree/"
        "{branch}/{addon_name}".format(**locals()),
        "{org_name}/{repo_name}".format(**locals()),
    )


def generate_fragment(org_name, repo_name, branch, addon_name, file):
    fragment_lines = file.readlines()
    if not fragment_lines:
        return False

    # Replace relative path by absolute path for figures
    image_path_re = re.compile(r".*\s*\.\..* (figure|image)::\s+(?P<path>.*?)\s*$")
    module_url = (
        "https://raw.githubusercontent.com/{org_name}/{repo_name}"
        "/{branch}/{addon_name}/".format(**locals())
    )
    for index, fragment_line in enumerate(fragment_lines):
        mo = image_path_re.match(fragment_line)
        if not mo:
            continue
        path = mo.group("path")

        if path.startswith("http"):
            # It is already an absolute path
            continue
        else:
            # remove '../' if exists that make the fragment working
            # on github interface, in the 'readme' subfolder
            relative_path = path.replace("../", "")
            fragment_lines[index] = fragment_line.replace(
                path, urljoin(module_url, relative_path)
            )
    fragment = "".join(fragment_lines)

    # ensure that there is a new empty line at the end of the fragment
    if fragment[-1] != "\n":
        fragment += "\n"
    return fragment


def get_fragment_format(
    addon_dir: str, fragment_name: str
) -> Union[FragmentFormat, None]:
    """Return the format of the named fragment of the given addon.

    Raise an exception if several formats are found.
    """
    fragment_rst_filename = make_fragment_filename(addon_dir, fragment_name, ".rst")
    fragment_md_filename = make_fragment_filename(addon_dir, fragment_name, ".md")
    if os.path.exists(fragment_rst_filename):
        if os.path.exists(fragment_md_filename):
            raise SystemExit(
                f"Both .md and .rst found for {fragment_name}. Please remove one"
                f" of {fragment_rst_filename} or {fragment_md_filename}."
            )
        return ".rst"
    if os.path.exists(fragment_md_filename):
        return ".md"
    return None


def get_fragments_format(addon_dir: str) -> FragmentFormat:
    """Return the format of the fragments of the given addon.

    Raise an exception if several formats are found.
    """
    fragments_format = None
    for fragment_name in FRAGMENTS:
        this_fragment_format = get_fragment_format(addon_dir, fragment_name)
        if this_fragment_format is None:
            # fragment does not exist
            continue
        if fragments_format and this_fragment_format != fragments_format:
            raise SystemExit(
                f"Both .md and .rst fragments found in {addon_dir}/readme. "
                f"Please ensure the same format is used for all fragments."
            )
        fragments_format = this_fragment_format
    return fragments_format


def make_fragment_filename(
    addon_dir: str, fragment_name: str, format: FragmentFormat
) -> str:
    return os.path.join(
        addon_dir,
        FRAGMENTS_DIR,
        fragment_name + format,
    )


def safe_remove(filename: str) -> None:
    try:
        os.remove(filename)
    except Exception:
        pass


def prepare_rst_fragment(addon_dir: str, fragment_name: str) -> Union[str, None]:
    fragment_rst_filename = make_fragment_filename(addon_dir, fragment_name, ".rst")
    fragment_md_filename = make_fragment_filename(addon_dir, fragment_name, ".md")
    if os.path.exists(fragment_rst_filename):
        if os.path.exists(fragment_md_filename):
            raise SystemExit(
                f"Both .md and .rst fragment found. Please remove one of "
                f"{fragment_rst_filename} or {fragment_md_filename}."
            )
        return fragment_rst_filename
    if not os.path.exists(fragment_md_filename):
        # no .rst nor .md fragment found
        return None
    # convert .md to .rst
    fragment_properties = FRAGMENTS[fragment_name]
    ensure_pandoc_installed()
    atexit.register(safe_remove, fragment_rst_filename)
    pypandoc.convert_file(
        fragment_md_filename,
        format=PANDOC_MARKDOWN_FORMAT,
        to="rst",
        outputfile=fragment_rst_filename,
        extra_args=[f"--shift-heading-level-by={fragment_properties.level-2}"],
        sandbox=True,
    )
    return fragment_rst_filename


def fragment_exists(addon_dir: str, fragment_name: str) -> bool:
    return os.path.exists(
        make_fragment_filename(
            addon_dir,
            fragment_name,
            ".rst",
        )
    ) or os.path.exists(
        make_fragment_filename(
            addon_dir,
            fragment_name,
            ".md",
        )
    )


def convert_fragments_to_md(addon_dir: str) -> None:
    """Convert all fragments from .rst to .md format."""
    for fragment_name in FRAGMENTS:
        fragment_rst_filename = make_fragment_filename(
            addon_dir,
            fragment_name,
            ".rst",
        )
        if not os.path.exists(fragment_rst_filename):
            continue
        fragment_md_filename = make_fragment_filename(
            addon_dir,
            fragment_name,
            ".md",
        )
        if os.path.exists(fragment_md_filename):
            continue
        ensure_pandoc_installed()
        pypandoc.convert_file(
            fragment_rst_filename,
            format="rst",
            to=PANDOC_MARKDOWN_FORMAT,
            outputfile=fragment_md_filename,
            extra_args=["--shift-heading-level=1"],
            sandbox=True,
        )
        os.remove(fragment_rst_filename)


def gen_one_addon_readme(
    org_name,
    repo_name,
    branch,
    addon_name,
    addon_dir,
    manifest,
    template_filename,
    readme_filename,
    source_digest,
):
    fragments_format = get_fragments_format(addon_dir)
    fragments = {}
    for fragment_name in FRAGMENTS:
        fragment_filename = prepare_rst_fragment(addon_dir, fragment_name)
        if fragment_filename:
            with open(fragment_filename, "r", encoding="utf8") as f:
                fragment = generate_fragment(org_name, repo_name, branch, addon_name, f)
                if fragment:
                    fragments[fragment_name] = fragment
    badges = []
    development_status = manifest.get("development_status", "Beta").lower()
    if development_status in DEVELOPMENT_STATUS_BADGES:
        badges.append(DEVELOPMENT_STATUS_BADGES[development_status])
    license = manifest.get("license")
    if license in LICENSE_BADGES:
        badges.append(LICENSE_BADGES[license])
    badges.append(make_repo_badge(org_name, repo_name, branch, addon_name))
    if org_name == "OCA":
        badges.append(make_weblate_badge(repo_name, branch, addon_name))
    if org_name == "OCA":
        badges.append(make_runboat_badge(repo_name, branch))
    authors = [
        a.strip()
        for a in manifest.get("author", "").split(",")
        if "(OCA)" not in a
        # remove OCA because it's in authors for the purpose
        # of finding OCA addons in apps.odoo.com, OCA is not
        # a real author, but is rather referenced in the
        # maintainers section
    ]
    # generate
    with open(template_filename, "r", encoding="utf8") as tf:
        template = Template(tf.read())
    with open(readme_filename, "w", encoding="utf8") as rf:
        rf.write(
            template.render(
                addon_name=addon_name,
                authors=authors,
                badges=badges,
                branch=branch,
                fragments=fragments,
                manifest=manifest,
                org_name=org_name,
                repo_name=repo_name,
                development_status=development_status,
                source_digest=source_digest,
                level3_underline="~" if fragments_format == ".rst" else "-",
            )
        )


def check_rst(readme_filename):
    with tempfile.NamedTemporaryFile() as f:
        publish_file(
            source_path=readme_filename,
            destination=f,
            writer_name="html4css1",
            settings_overrides=RST2HTML_SETTINGS,
        )


def gen_one_addon_index(readme_filename):
    addon_dir = os.path.dirname(readme_filename)
    index_dir = os.path.join(addon_dir, "static", "description")
    index_filename = os.path.join(index_dir, "index.html")
    if os.path.exists(index_filename):
        with open(index_filename) as f:
            if "oca-gen-addon-readme" not in f.read():
                # index was created manually
                return
    if not os.path.isdir(index_dir):
        os.makedirs(index_dir)
    publish_file(
        source_path=readme_filename,
        destination_path=index_filename,
        writer_name="html4css1",
        settings_overrides=RST2HTML_SETTINGS,
    )
    with open(index_filename, "rb") as f:
        index = f.read()
    # remove the docutils version from generated html, to avoid
    # useless changes in the readme
    index = re.sub(
        rb"(<meta.*generator.*Docutils)\s*[\d.]+", rb"\1", index, re.MULTILINE
    )
    with open(index_filename, "wb") as f:
        f.write(index)
    return index_filename


def _source_digest_match(readme_filename, source_digest):
    if not os.path.isfile(readme_filename):
        return False
    digest_comment = f"!! source digest: {source_digest}"
    with open(readme_filename, "r", encoding="utf8") as f:
        for line in f:
            if digest_comment in line:
                return True
    return False


def _get_source_digest(readme_filename: str) -> Union[str, None]:
    """Get the source digest from the given readme file.

    Return None if the file does not exist, or if the digest is not found.
    """
    readme_path = Path(readme_filename)
    if not readme_path.is_file():
        return None
    digest_re = re.compile(r"!! source digest: (?P<digest>sha256:\w+)")
    mo = digest_re.search(readme_path.read_text(encoding="utf8"))
    if not mo:
        return None
    return mo.group("digest")


@click.command()
@click.option("--org-name", default="OCA", help="Organization name, eg. OCA.")
@click.option("--repo-name", required=True, help="Repository name, eg. server-tools.")
@click.option("--branch", required=True, help="Odoo series. eg 11.0.")
@click.option(
    "--addon-dir",
    "addon_dirs",
    type=click.Path(dir_okay=True, file_okay=False, exists=True),
    multiple=True,
    help="Directory where addon manifest is located. This option " "may be repeated.",
)
@click.option(
    "--addons-dir",
    type=click.Path(dir_okay=True, file_okay=False, exists=True),
    help="Directory containing several addons, the README will be "
    "generated for all installable addons found there.",
)
@click.option(
    "--if-source-changed",
    "--if-fragments-changed",
    "if_fragments_changed",
    is_flag=True,
    default=False,
    help="Only generate if source fragments or manifest changed.",
)
@click.option(
    "--keep-source-digest",
    is_flag=True,
    default=False,
    help=(
        "Do not update the source digest in the generated file. "
        "Useful to avoid merge conflicts when changes that do not impact "
        "the generated file are made to the manifest."
    ),
)
@click.option(
    "--commit/--no-commit",
    help="git commit changes to README.rst and index.html, if any.",
)
@click.option(
    "--gen-html/--no-gen-html",
    default=True,
    help="Generate index html file.",
)
@click.option(
    "--template-filename",
    default=os.path.join(
        os.path.dirname(__file__),
        "gen_addon_readme.rst.jinja",
    ),
    help="Template file to use.",
)
@click.option(
    "--convert-fragments-to-markdown",
    is_flag=True,
    default=False,
)
def gen_addon_readme(
    org_name,
    repo_name,
    branch,
    addon_dirs,
    addons_dir,
    commit,
    gen_html,
    template_filename,
    if_fragments_changed,
    convert_fragments_to_markdown,
    keep_source_digest,
):
    """Generate README.rst from fragments.

    Do nothing if readme/DESCRIPTION(.rst|.md) is absent, otherwise overwrite
    existing README.rst with content generated from the template,
    fragments (DESCRIPTION(.rst|.md), USAGE(.rst|.md), etc) and the addon manifest.
    """
    addons = []
    if addons_dir:
        addons.extend(find_addons(addons_dir))
    for addon_dir in addon_dirs:
        addon_name = os.path.basename(os.path.abspath(addon_dir))
        try:
            manifest = read_manifest(addon_dir)
        except NoManifestFound:
            continue
        addons.append((addon_name, addon_dir, manifest))
    readme_filenames = []
    for addon_name, addon_dir, manifest in addons:
        if convert_fragments_to_markdown:
            convert_fragments_to_md(addon_dir)
        if not fragment_exists(addon_dir, "DESCRIPTION"):
            continue
        readme_filename = os.path.join(addon_dir, "README.rst")
        source_digest = hash(
            get_manifest_path(addon_dir),
            os.path.join(addon_dir, FRAGMENTS_DIR),
            relative_to=addon_dir,
        )
        if if_fragments_changed:
            if _source_digest_match(readme_filename, source_digest):
                continue
        if keep_source_digest:
            source_digest = _get_source_digest(readme_filename) or source_digest
        gen_one_addon_readme(
            org_name,
            repo_name,
            branch,
            addon_name,
            addon_dir,
            manifest,
            template_filename,
            readme_filename,
            source_digest,
        )
        check_rst(readme_filename)
        readme_filenames.append(readme_filename)
        if gen_html:
            if not manifest.get("preloadable", True):
                continue
            index_filename = gen_one_addon_index(readme_filename)
            if index_filename:
                readme_filenames.append(index_filename)
    if commit:
        commit_if_needed(readme_filenames, "[UPD] README.rst")


if __name__ == "__main__":
    gen_addon_readme()

1	# License AGPLv3 (https://www.gnu.org/licenses/agpl-3.0-standalone.html)
2	# Copyright (c) 2018 ACSONE SA/NV
3	# Copyright (c) 2018 GRAP (http://www.grap.coop)
4
5	import atexit	10✔
6	import functools	10✔
7	import os	10✔
8	import re	10✔
9	import sys	10✔
10	import tempfile	10✔
11	from pathlib import Path	10✔
12	from typing import Union	10✔
13	from urllib.parse import urljoin	10✔
14
15	import click	10✔
16	import pypandoc	10✔
17	from docutils.core import publish_file	10✔
18	from jinja2 import Template	10✔
19
20	from ._hash import hash	10✔
21	from .gitutils import commit_if_needed	10✔
22	from .manifest import NoManifestFound, find_addons, get_manifest_path, read_manifest	10✔
23
24	if sys.version_info >= (3, 8):	10✔
25	from typing import Literal	8!
26	else:
27	from typing_extensions import Literal	2!
28
29
30	class FragmentProperties:	10✔
31	def __init__(self, level: int):	10✔
32	self.level = level	10✔
33
34
35	FragmentFormat = Literal[".rst", ".md"]	10✔
36
37	FRAGMENTS_DIR = "readme"	10✔
38
39	FRAGMENTS = {	10✔
40	"DESCRIPTION": FragmentProperties(level=2),
41	"CONTEXT": FragmentProperties(level=2),
42	"INSTALL": FragmentProperties(level=2),
43	"CONFIGURE": FragmentProperties(level=2),
44	"USAGE": FragmentProperties(level=2),
45	"ROADMAP": FragmentProperties(level=2),
46	"DEVELOP": FragmentProperties(level=2),
47	"CONTRIBUTORS": FragmentProperties(level=3),
48	"CREDITS": FragmentProperties(level=3),
49	"HISTORY": FragmentProperties(level=2),
50	}
51
52	LICENSE_BADGES = {	10✔
53	"AGPL-3": (
54	"https://img.shields.io/badge/licence-AGPL--3-blue.png",
55	"http://www.gnu.org/licenses/agpl-3.0-standalone.html",
56	"License: AGPL-3",
57	),
58	"LGPL-3": (
59	"https://img.shields.io/badge/licence-LGPL--3-blue.png",
60	"http://www.gnu.org/licenses/lgpl-3.0-standalone.html",
61	"License: LGPL-3",
62	),
63	"GPL-3": (
64	"https://img.shields.io/badge/licence-GPL--3-blue.png",
65	"http://www.gnu.org/licenses/gpl-3.0-standalone.html",
66	"License: GPL-3",
67	),
68	}
69
70	DEVELOPMENT_STATUS_BADGES = {	10✔
71	"mature": (
72	"https://img.shields.io/badge/maturity-Mature-brightgreen.png",
73	"https://odoo-community.org/page/development-status",
74	"Mature",
75	),
76	"production/stable": (
77	"https://img.shields.io/badge/maturity-Production%2FStable-green.png",
78	"https://odoo-community.org/page/development-status",
79	"Production/Stable",
80	),
81	"beta": (
82	"https://img.shields.io/badge/maturity-Beta-yellow.png",
83	"https://odoo-community.org/page/development-status",
84	"Beta",
85	),
86	"alpha": (
87	"https://img.shields.io/badge/maturity-Alpha-red.png",
88	"https://odoo-community.org/page/development-status",
89	"Alpha",
90	),
91	}
92
93	# this comes from pypa/readme_renderer
94	RST2HTML_SETTINGS = {	10✔
95	# Prevent local files from being included into the rendered output.
96	# This is a security concern because people can insert files
97	# that are part of the system, such as /etc/passwd.
98	"file_insertion_enabled": False,
99	# Halt rendering and throw an exception if there was any errors or
100	# warnings from docutils.
101	"halt_level": 2,
102	# Output math blocks as LaTeX that can be interpreted by MathJax for
103	# a prettier display of Math formulas.
104	"math_output": "MathJax",
105	# Disable raw html as enabling it is a security risk, we do not want
106	# people to be able to include any old HTML in the final output.
107	"raw_enabled": False,
108	# Use typographic quotes, and transform --, ---, and ... into their
109	# typographic counterparts.
110	"smart_quotes": True,
111	# Use the short form of syntax highlighting so that the generated
112	# Pygments CSS can be used to style the output.
113	"syntax_highlight": "short",
114	# Since odoo/odoo@8d06889, Odoo emits a warning
115	# if index.html contains an xml declaration
116	"xml_declaration": False,
117	# ...but even for previous versions we don't need
118	# the xml declaration as docutils adds a <meta> tag:
119	# <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
120	# utf-8 is default value for output_encoding
121	# but let's make it explicit here:
122	"output_encoding": "utf-8",
123	}
124
125	# GitHub Flavored Markdown
126	# - raw html is disabled
127	# - auto identifiers is disabled because pylint-odoo complains about them (Hyperlink
128	# target "..;" is not referenced.)
129	PANDOC_MARKDOWN_FORMAT = "gfm-raw_html-gfm_auto_identifiers"	10✔
130
131
132	@functools.lru_cache(maxsize=None)	10✔
133	def ensure_pandoc_installed() -> None:	10✔
134	pypandoc.ensure_pandoc_installed(delete_installer=True)	×
135
136
137	def make_runboat_badge(repo, branch):	10✔
138	return (	10✔
139	"https://img.shields.io/badge/runboat-Try%20me-875A7B.png",
140	"https://runboat.odoo-community.org/builds?"
141	"repo=OCA/{repo}&target_branch={branch}".format(**locals()),
142	"Try me on Runboat",
143	)
144
145
146	def make_weblate_badge(repo_name, branch, addon_name):	10✔
147	branch = branch.replace(".", "-")	10✔
148	return (	10✔
149	"https://img.shields.io/badge/weblate-Translate%20me-F47D42.png",
150	"https://translation.odoo-community.org/projects/"
151	"{repo_name}-{branch}/{repo_name}-{branch}-{addon_name}".format(**locals()),
152	"Translate me on Weblate",
153	)
154
155
156	def make_repo_badge(org_name, repo_name, branch, addon_name):	10✔
157	badge_repo_name = repo_name.replace("-", "--")	10✔
158	badge_org_name = org_name.replace("-", "--")	10✔
159	return (	10✔
160	"https://img.shields.io/badge/github-{badge_org_name}%2F{badge_repo_name}"
161	"-lightgray.png?logo=github".format(**locals()),
162	"https://github.com/{org_name}/{repo_name}/tree/"
163	"{branch}/{addon_name}".format(**locals()),
164	"{org_name}/{repo_name}".format(**locals()),
165	)
166
167
168	def generate_fragment(org_name, repo_name, branch, addon_name, file):	10✔
169	fragment_lines = file.readlines()	10✔
170	if not fragment_lines:	10✔
171	return False	×
172
173	# Replace relative path by absolute path for figures
174	image_path_re = re.compile(r".\s\.\..* (figure\|image)::\s+(?P<path>.?)\s$")	10✔
175	module_url = (	10✔
176	"https://raw.githubusercontent.com/{org_name}/{repo_name}"
177	"/{branch}/{addon_name}/".format(**locals())
178	)
179	for index, fragment_line in enumerate(fragment_lines):	10✔
180	mo = image_path_re.match(fragment_line)	10✔
181	if not mo:	10✔
182	continue	10✔
183	path = mo.group("path")	10✔
184
185	if path.startswith("http"):	10✔
186	# It is already an absolute path
187	continue	×
188	else:
189	# remove '../' if exists that make the fragment working
190	# on github interface, in the 'readme' subfolder
191	relative_path = path.replace("../", "")	10✔
192	fragment_lines[index] = fragment_line.replace(	10✔
193	path, urljoin(module_url, relative_path)
194	)
195	fragment = "".join(fragment_lines)	10✔
196
197	# ensure that there is a new empty line at the end of the fragment
198	if fragment[-1] != "\n":	10✔
199	fragment += "\n"	10✔
200	return fragment	10✔
201
202
203	def get_fragment_format(	10✔
204	addon_dir: str, fragment_name: str
205	) -> Union[FragmentFormat, None]:
206	"""Return the format of the named fragment of the given addon.
207
208	Raise an exception if several formats are found.
209	"""
210	fragment_rst_filename = make_fragment_filename(addon_dir, fragment_name, ".rst")	10✔
211	fragment_md_filename = make_fragment_filename(addon_dir, fragment_name, ".md")	10✔
212	if os.path.exists(fragment_rst_filename):	10✔
213	if os.path.exists(fragment_md_filename):	10✔
214	raise SystemExit(	10✔
215	f"Both .md and .rst found for {fragment_name}. Please remove one"
216	f" of {fragment_rst_filename} or {fragment_md_filename}."
217	)
218	return ".rst"	10✔
219	if os.path.exists(fragment_md_filename):	10✔
220	return ".md"	10✔
221	return None	10✔
222
223
224	def get_fragments_format(addon_dir: str) -> FragmentFormat:	10✔
225	"""Return the format of the fragments of the given addon.
226
227	Raise an exception if several formats are found.
228	"""
229	fragments_format = None	10✔
230	for fragment_name in FRAGMENTS:	10✔
231	this_fragment_format = get_fragment_format(addon_dir, fragment_name)	10✔
232	if this_fragment_format is None:	10✔
233	# fragment does not exist
234	continue	10✔
235	if fragments_format and this_fragment_format != fragments_format:	10✔
236	raise SystemExit(	10✔
237	f"Both .md and .rst fragments found in {addon_dir}/readme. "
238	f"Please ensure the same format is used for all fragments."
239	)
240	fragments_format = this_fragment_format	10✔
241	return fragments_format	10✔
242
243
244	def make_fragment_filename(	10✔
245	addon_dir: str, fragment_name: str, format: FragmentFormat
246	) -> str:
247	return os.path.join(	10✔
248	addon_dir,
249	FRAGMENTS_DIR,
250	fragment_name + format,
251	)
252
253
254	def safe_remove(filename: str) -> None:	10✔
255	try:	10✔
256	os.remove(filename)	10✔
257	except Exception:	10✔
258	pass	10✔
259
260
261	def prepare_rst_fragment(addon_dir: str, fragment_name: str) -> Union[str, None]:	10✔
262	fragment_rst_filename = make_fragment_filename(addon_dir, fragment_name, ".rst")	10✔
263	fragment_md_filename = make_fragment_filename(addon_dir, fragment_name, ".md")	10✔
264	if os.path.exists(fragment_rst_filename):	10✔
265	if os.path.exists(fragment_md_filename):	10✔
266	raise SystemExit(	×
267	f"Both .md and .rst fragment found. Please remove one of "
268	f"{fragment_rst_filename} or {fragment_md_filename}."
269	)
270	return fragment_rst_filename	10✔
271	if not os.path.exists(fragment_md_filename):	10✔
272	# no .rst nor .md fragment found
273	return None	10✔
274	# convert .md to .rst
275	fragment_properties = FRAGMENTS[fragment_name]	×
276	ensure_pandoc_installed()	×
277	atexit.register(safe_remove, fragment_rst_filename)	×
278	pypandoc.convert_file(	×
279	fragment_md_filename,
280	format=PANDOC_MARKDOWN_FORMAT,
281	to="rst",
282	outputfile=fragment_rst_filename,
283	extra_args=[f"--shift-heading-level-by={fragment_properties.level-2}"],
284	sandbox=True,
285	)
286	return fragment_rst_filename	×
287
288
289	def fragment_exists(addon_dir: str, fragment_name: str) -> bool:	10✔
290	return os.path.exists(	10✔
291	make_fragment_filename(
292	addon_dir,
293	fragment_name,
294	".rst",
295	)
296	) or os.path.exists(
297	make_fragment_filename(
298	addon_dir,
299	fragment_name,
300	".md",
301	)
302	)
303
304
305	def convert_fragments_to_md(addon_dir: str) -> None:	10✔
306	"""Convert all fragments from .rst to .md format."""
307	for fragment_name in FRAGMENTS:	×
308	fragment_rst_filename = make_fragment_filename(	×
309	addon_dir,
310	fragment_name,
311	".rst",
312	)
313	if not os.path.exists(fragment_rst_filename):	×
314	continue	×
315	fragment_md_filename = make_fragment_filename(	×
316	addon_dir,
317	fragment_name,
318	".md",
319	)
320	if os.path.exists(fragment_md_filename):	×
321	continue	×
322	ensure_pandoc_installed()	×
323	pypandoc.convert_file(	×
324	fragment_rst_filename,
325	format="rst",
326	to=PANDOC_MARKDOWN_FORMAT,
327	outputfile=fragment_md_filename,
328	extra_args=["--shift-heading-level=1"],
329	sandbox=True,
330	)
331	os.remove(fragment_rst_filename)	×
332
333
334	def gen_one_addon_readme(	10✔
335	org_name,
336	repo_name,
337	branch,
338	addon_name,
339	addon_dir,
340	manifest,
341	template_filename,
342	readme_filename,
343	source_digest,
344	):
345	fragments_format = get_fragments_format(addon_dir)	10✔
346	fragments = {}	10✔
347	for fragment_name in FRAGMENTS:	10!
348	fragment_filename = prepare_rst_fragment(addon_dir, fragment_name)	10✔
349	if fragment_filename:	10✔
350	with open(fragment_filename, "r", encoding="utf8") as f:	10✔
351	fragment = generate_fragment(org_name, repo_name, branch, addon_name, f)	10✔
352	if fragment:	10✔
353	fragments[fragment_name] = fragment	10✔
354	badges = []	10✔
355	development_status = manifest.get("development_status", "Beta").lower()	10✔
356	if development_status in DEVELOPMENT_STATUS_BADGES:	10✔
357	badges.append(DEVELOPMENT_STATUS_BADGES[development_status])	10✔
358	license = manifest.get("license")	10!
359	if license in LICENSE_BADGES:	10✔
360	badges.append(LICENSE_BADGES[license])	×
361	badges.append(make_repo_badge(org_name, repo_name, branch, addon_name))	10✔
362	if org_name == "OCA":	10✔
363	badges.append(make_weblate_badge(repo_name, branch, addon_name))	10✔
364	if org_name == "OCA":	10✔
365	badges.append(make_runboat_badge(repo_name, branch))	10✔
366	authors = [	10✔
367	a.strip()
368	for a in manifest.get("author", "").split(",")
369	if "(OCA)" not in a
370	# remove OCA because it's in authors for the purpose
371	# of finding OCA addons in apps.odoo.com, OCA is not
372	# a real author, but is rather referenced in the
373	# maintainers section
374	]
375	# generate
376	with open(template_filename, "r", encoding="utf8") as tf:	10✔
377	template = Template(tf.read())	10✔
378	with open(readme_filename, "w", encoding="utf8") as rf:	10✔
379	rf.write(	10✔
380	template.render(
381	addon_name=addon_name,
382	authors=authors,
383	badges=badges,
384	branch=branch,
385	fragments=fragments,
386	manifest=manifest,
387	org_name=org_name,
388	repo_name=repo_name,
389	development_status=development_status,
390	source_digest=source_digest,
391	level3_underline="~" if fragments_format == ".rst" else "-",
392	)
393	)
394
395
396	def check_rst(readme_filename):	10✔
397	with tempfile.NamedTemporaryFile() as f:	10✔
398	publish_file(	10✔
399	source_path=readme_filename,
400	destination=f,
401	writer_name="html4css1",
402	settings_overrides=RST2HTML_SETTINGS,
403	)
404
405
406	def gen_one_addon_index(readme_filename):	10✔
407	addon_dir = os.path.dirname(readme_filename)	10✔
408	index_dir = os.path.join(addon_dir, "static", "description")	10✔
409	index_filename = os.path.join(index_dir, "index.html")	10✔
410	if os.path.exists(index_filename):	10✔
411	with open(index_filename) as f:	10✔
412	if "oca-gen-addon-readme" not in f.read():	10✔
413	# index was created manually
414	return	×
415	if not os.path.isdir(index_dir):	10✔
416	os.makedirs(index_dir)	10✔
417	publish_file(	10✔
418	source_path=readme_filename,
419	destination_path=index_filename,
420	writer_name="html4css1",
421	settings_overrides=RST2HTML_SETTINGS,
422	)
423	with open(index_filename, "rb") as f:	10✔
424	index = f.read()	10✔
425	# remove the docutils version from generated html, to avoid
426	# useless changes in the readme
427	index = re.sub(	10✔
428	rb"(<meta.generator.Docutils)\s*[\d.]+", rb"\1", index, re.MULTILINE
429	)
430	with open(index_filename, "wb") as f:	10✔
431	f.write(index)	10✔
432	return index_filename	10✔
433
434
435	def _source_digest_match(readme_filename, source_digest):	10✔
436	if not os.path.isfile(readme_filename):	10✔
437	return False	10✔
438	digest_comment = f"!! source digest: {source_digest}"	10✔
439	with open(readme_filename, "r", encoding="utf8") as f:	10✔
440	for line in f:	10✔
441	if digest_comment in line:	10✔
442	return True	10✔
443	return False	10✔
444
445
446	def _get_source_digest(readme_filename: str) -> Union[str, None]:	10✔
447	"""Get the source digest from the given readme file.
448
449	Return None if the file does not exist, or if the digest is not found.
450	"""
451	readme_path = Path(readme_filename)	10✔
452	if not readme_path.is_file():	10✔
453	return None	10✔
454	digest_re = re.compile(r"!! source digest: (?P<digest>sha256:\w+)")	10✔
455	mo = digest_re.search(readme_path.read_text(encoding="utf8"))	10✔
456	if not mo:	10✔
457	return None	10✔
458	return mo.group("digest")	10✔
459
460
461	@click.command()	10✔
462	@click.option("--org-name", default="OCA", help="Organization name, eg. OCA.")	10✔
463	@click.option("--repo-name", required=True, help="Repository name, eg. server-tools.")	10✔
464	@click.option("--branch", required=True, help="Odoo series. eg 11.0.")	10✔
465	@click.option(	10✔
466	"--addon-dir",
467	"addon_dirs",
468	type=click.Path(dir_okay=True, file_okay=False, exists=True),
469	multiple=True,
470	help="Directory where addon manifest is located. This option " "may be repeated.",
471	)
472	@click.option(	10✔
473	"--addons-dir",
474	type=click.Path(dir_okay=True, file_okay=False, exists=True),
475	help="Directory containing several addons, the README will be "
476	"generated for all installable addons found there.",
477	)
478	@click.option(	10✔
479	"--if-source-changed",
480	"--if-fragments-changed",
481	"if_fragments_changed",
482	is_flag=True,
483	default=False,
484	help="Only generate if source fragments or manifest changed.",
485	)
486	@click.option(	10✔
487	"--keep-source-digest",
488	is_flag=True,
489	default=False,
490	help=(
491	"Do not update the source digest in the generated file. "
492	"Useful to avoid merge conflicts when changes that do not impact "
493	"the generated file are made to the manifest."
494	),
495	)
496	@click.option(	10✔
497	"--commit/--no-commit",
498	help="git commit changes to README.rst and index.html, if any.",
499	)
500	@click.option(	10✔
501	"--gen-html/--no-gen-html",
502	default=True,
503	help="Generate index html file.",
504	)
505	@click.option(	10✔
506	"--template-filename",
507	default=os.path.join(
508	os.path.dirname(__file__),
509	"gen_addon_readme.rst.jinja",
510	),
511	help="Template file to use.",
512	)
513	@click.option(	10✔
514	"--convert-fragments-to-markdown",
515	is_flag=True,
516	default=False,
517	)
518	def gen_addon_readme(	8✔
519	org_name,
520	repo_name,
521	branch,
522	addon_dirs,
523	addons_dir,
524	commit,
525	gen_html,
526	template_filename,
527	if_fragments_changed,
528	convert_fragments_to_markdown,
529	keep_source_digest,
530	):
531	"""Generate README.rst from fragments.
532
533	Do nothing if readme/DESCRIPTION(.rst\|.md) is absent, otherwise overwrite
534	existing README.rst with content generated from the template,
535	fragments (DESCRIPTION(.rst\|.md), USAGE(.rst\|.md), etc) and the addon manifest.
536	"""
537	addons = []	10✔
538	if addons_dir:	10✔
539	addons.extend(find_addons(addons_dir))	10✔
540	for addon_dir in addon_dirs:	10✔
541	addon_name = os.path.basename(os.path.abspath(addon_dir))	10✔
542	try:	10✔
543	manifest = read_manifest(addon_dir)	10✔
544	except NoManifestFound:	×
545	continue	×
546	addons.append((addon_name, addon_dir, manifest))	10✔
547	readme_filenames = []	10✔
548	for addon_name, addon_dir, manifest in addons:	10!
549	if convert_fragments_to_markdown:	10✔
550	convert_fragments_to_md(addon_dir)	×
551	if not fragment_exists(addon_dir, "DESCRIPTION"):	10✔
552	continue	×
553	readme_filename = os.path.join(addon_dir, "README.rst")	10✔
554	source_digest = hash(	10✔
555	get_manifest_path(addon_dir),
556	os.path.join(addon_dir, FRAGMENTS_DIR),
557	relative_to=addon_dir,
558	)
559	if if_fragments_changed:	10✔
560	if _source_digest_match(readme_filename, source_digest):	10✔
561	continue	10✔
562	if keep_source_digest:	10✔
563	source_digest = _get_source_digest(readme_filename) or source_digest	10✔
564	gen_one_addon_readme(	10✔
565	org_name,
566	repo_name,
567	branch,
568	addon_name,
569	addon_dir,
570	manifest,
571	template_filename,
572	readme_filename,
573	source_digest,
574	)
575	check_rst(readme_filename)	10✔
576	readme_filenames.append(readme_filename)	10✔
577	if gen_html:	10✔
578	if not manifest.get("preloadable", True):	10✔
579	continue	×
580	index_filename = gen_one_addon_index(readme_filename)	10✔
581	if index_filename:	10✔
582	readme_filenames.append(index_filename)	10✔
583	if commit:	10✔
584	commit_if_needed(readme_filenames, "[UPD] README.rst")	×
585
586
587	if __name__ == "__main__":	10✔
588	gen_addon_readme()	10✔

OCA / maintainer-tools / 13228537945

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous