14429819143

Committed 13 Apr 2025 01:13PM UTC coverage: 90.562% (-0.9%) from 91.486%

Build # 14429819143

Build Type

Pull #793

github

Committed by

eliashaeussler

Commit Message

[!!!][FEATURE] Allow to exclude sites and languages from warming

Resolves: #777

Pull Request Pull Request #793: [!!!][FEATURE] Allow to exclude sites and languages from warming

Run Details

116 of 135 new or added lines in 10 files covered. (85.93%)

7 existing lines in 3 files now uncovered.

1161 of 1282 relevant lines covered (90.56%)

8.47 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.41

/Classes/Command/WarmupCommand.php

<?php

declare(strict_types=1);

/*
 * This file is part of the TYPO3 CMS extension "warming".
 *
 * Copyright (C) 2021-2025 Elias Häußler <elias@haeussler.dev>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

namespace EliasHaeussler\Typo3Warming\Command;

use EliasHaeussler\CacheWarmup;
use EliasHaeussler\Typo3SitemapLocator;
use EliasHaeussler\Typo3Warming\Configuration;
use EliasHaeussler\Typo3Warming\Crawler;
use EliasHaeussler\Typo3Warming\Domain;
use EliasHaeussler\Typo3Warming\Http;
use Psr\EventDispatcher;
use Symfony\Component\Console;
use TYPO3\CMS\Core;

/**
 * WarmupCommand
 *
 * @author Elias Häußler <elias@haeussler.dev>
 * @license GPL-2.0-or-later
 */
#[Console\Attribute\AsCommand(
    name: 'warming:cachewarmup',
    description: 'Warm up Frontend caches of single pages and/or whole sites using their XML sitemaps.',
)]
final class WarmupCommand extends Console\Command\Command
{
    private const ALL_LANGUAGES = -1;
    private const ALL_SITES = 'all';

    public function __construct(
        private readonly Http\Client\ClientFactory $clientFactory,
        private readonly Configuration\Configuration $configuration,
        private readonly Crawler\Strategy\CrawlingStrategyFactory $crawlingStrategyFactory,
        private readonly Typo3SitemapLocator\Sitemap\SitemapLocator $sitemapLocator,
        private readonly Domain\Repository\SiteRepository $siteRepository,
        private readonly Domain\Repository\SiteLanguageRepository $siteLanguageRepository,
        private readonly EventDispatcher\EventDispatcherInterface $eventDispatcher,
        private readonly Core\Package\PackageManager $packageManager,
        private readonly Http\Message\PageUriBuilder $pageUriBuilder,
    ) {
        parent::__construct();
    }

    protected function configure(): void
    {
        $v = fn(mixed $value) => $value;
        $decoratedCrawlingStrategies = \implode(PHP_EOL, array_map(
            static fn(string $strategy) => '  │  * <info>' . $strategy . '</info>',
            array_keys($this->crawlingStrategyFactory->getAll()),
        ));

        $this->setDescription('Warm up Frontend caches of single pages and/or whole sites using their XML sitemaps.');
        $this->setHelp(
            <<<HELP
This command can be used in many ways to warm up frontend caches.
Some possible combinations and options are shown below.

<info>Sites and pages</info>
<info>===============</info>

To warm up caches, either <info>pages</info> or <info>sites</info> can be specified.
Both types can also be combined or extended by the specification of one or more <info>languages</info>.
If you omit the language option, the caches of all languages of the requested pages and sites
will be warmed up.

You can also use the special keyword <info>all</info> for <info>sites</info>.
This will cause all available sites to be warmed up.

Examples:

* <comment>warming:cachewarmup -p 1,2,3</comment>
  ├─ Pages: <info>1, 2 and 3</info>
  └─ Languages: <info>all</info>

* <comment>warming:cachewarmup -s 1</comment>
* <comment>warming:cachewarmup -s main</comment>
  ├─ Sites: <info>Root page ID 1</info> or <info>identifier "main"</info>
  └─ Languages: <info>all</info>

* <comment>warming:cachewarmup -p 1 -s 1</comment>
* <comment>warming:cachewarmup -p 1 -s main</comment>
  ├─ Pages: <info>1</info>
  ├─ Sites: <info>Root page ID 1</info> or <info>identifier "main"</info>
  └─ Languages: <info>all</info>

* <comment>warming:cachewarmup -s 1 -l 0,1</comment>
  ├─ Sites: <info>Root page ID 1</info> or <info>identifier "main"</info>
  └─ Languages: <info>0 and 1</info>

* <comment>warming:cachewarmup -s all</comment>
  ├─ Sites: <info>all</info>
  └─ Languages: <info>all</info>

<info>Additional options</info>
<info>==================</info>

* <comment>Configuration file</comment>
  ├─ A preconfigured set of configuration options can be written to a configuration file.
  │  This file can be passed using the <info>--config</info> option.
  │  It may also contain extension paths, e.g. <info>EXT:sitepackage/Configuration/cache-warmup.json</info>.
  │  The following file formats are currently supported:
  │  * <info>json</info>
  │  * <info>php</info>
  │  * <info>yaml</info>
  │  * <info>yml</info>
  └─ Example: <comment>warming:cachewarmup --config path/to/cache-warmup.json</comment>

* <comment>Strict mode</comment>
  ├─ You can pass the <info>--strict</info> (or <info>-x</info>) option to terminate execution with an error code
  │  if individual caches warm up incorrectly.
  │  This is especially useful for automated execution of cache warmups.
  ├─ Default: <info>false</info>
  └─ Example: <comment>warming:cachewarmup -s 1 --strict</comment>

* <comment>Crawl limit</comment>
  ├─ The maximum number of pages to be warmed up can be defined via the extension configuration <info>limit</info>.
  │  It can be overridden by using the <info>--limit</info> option.
  │  The value <info>0</info> deactivates the crawl limit.
  ├─ Default: <info>{$v($this->configuration->getLimit())}</info>
  ├─ Example: <comment>warming:cachewarmup -s 1 --limit 100</comment> (limits crawling to 100 pages)
  └─ Example: <comment>warming:cachewarmup -s 1 --limit 0</comment> (no limit)

* <comment>Crawling strategy</comment>
  ├─ A crawling strategy defines how URLs will be crawled, e.g. by sorting them by a specific property.
  │  It can be defined via the extension configuration <info>strategy</info> or by using the <info>--strategy</info> option.
  │  The following strategies are currently available:
{$decoratedCrawlingStrategies}
  ├─ Default: <info>{$v($this->configuration->getStrategy() ?? 'none')}</info>
  └─ Example: <comment>warming:cachewarmup --strategy {$v(CacheWarmup\Crawler\Strategy\SortByPriorityStrategy::getName())}</comment>

* <comment>Format output</comment>
  ├─ By default, all user-oriented output is printed as plain text to the console.
  │  However, you can use other formatters by using the <info>--format</info> (or <info>-f</info>) option.
  ├─ Default: <info>{$v(CacheWarmup\Formatter\TextFormatter::getType())}</info>
  ├─ Example: <comment>warming:cachewarmup --format {$v(CacheWarmup\Formatter\TextFormatter::getType())}</comment> (normal output as plaintext)
  └─ Example: <comment>warming:cachewarmup --format {$v(CacheWarmup\Formatter\JsonFormatter::getType())}</comment> (displays output as JSON)

<info>Crawling configuration</info>
<info>======================</info>

* <comment>Alternative crawler</comment>
  ├─ Use the extension configuration <info>verboseCrawler</info> to use an alternative crawler for
  │  command-line requests. For warmup requests triggered via the TYPO3 backend, you can use the
  │  extension configuration <info>crawler</info>.
  ├─ Currently used default crawler: <info>{$v($this->configuration->getCrawler())}</info>
  └─ Currently used verbose crawler: <info>{$v($this->configuration->getVerboseCrawler())}</info>

* <comment>Custom User-Agent header</comment>
  ├─ When the default crawler is used, each warmup request is executed with a special User-Agent header.
  │  This header is generated from the encryption key of the TYPO3 installation.
  │  It can be used, for example, to exclude warmup requests from your search statistics.
  └─ Current User-Agent: <info>{$v($this->configuration->getUserAgent())}</info>
HELP
        );

        $this->addOption(
            'pages',
            'p',
            Console\Input\InputOption::VALUE_REQUIRED | Console\Input\InputOption::VALUE_IS_ARRAY,
            'Pages whose Frontend caches are to be warmed up.',
        );
        $this->addOption(
            'sites',
            's',
            Console\Input\InputOption::VALUE_REQUIRED | Console\Input\InputOption::VALUE_IS_ARRAY,
            'Site identifiers or root page IDs of sites whose caches are to be warmed up.',
        );
        $this->addOption(
            'languages',
            'l',
            Console\Input\InputOption::VALUE_REQUIRED | Console\Input\InputOption::VALUE_IS_ARRAY,
            'Optional identifiers of languages for which caches are to be warmed up.',
        );
        $this->addOption(
            'config',
            'c',
            Console\Input\InputOption::VALUE_REQUIRED,
            'Path to optional configuration file',
        );
        $this->addOption(
            'limit',
            null,
            Console\Input\InputOption::VALUE_REQUIRED,
            'Maximum number of pages to be crawled. Set to <info>0</info> to disable the limit.',
            $this->configuration->getLimit(),
        );
        $this->addOption(
            'strategy',
            null,
            Console\Input\InputOption::VALUE_REQUIRED,
            'Optional strategy to prepare URLs before crawling them.',
            $this->configuration->getStrategy(),
        );
        $this->addOption(
            'format',
            'f',
            Console\Input\InputOption::VALUE_REQUIRED,
            'Formatter used to print the cache warmup result',
            CacheWarmup\Formatter\TextFormatter::getType(),
        );
        $this->addOption(
            'strict',
            'x',
            Console\Input\InputOption::VALUE_NONE,
            'Fail if an error occurred during cache warmup.',
        );
    }

    protected function initialize(Console\Input\InputInterface $input, Console\Output\OutputInterface $output): void
    {
        Core\Core\Bootstrap::initializeBackendAuthentication();
    }

    /**
     * @throws CacheWarmup\Exception\CrawlerOptionIsInvalid
     * @throws CacheWarmup\Exception\LocalFilePathIsMissingInUrl
     * @throws CacheWarmup\Exception\UrlIsEmpty
     * @throws CacheWarmup\Exception\UrlIsInvalid
     * @throws Console\Exception\ExceptionInterface
     * @throws Core\Exception\SiteNotFoundException
     * @throws Core\Package\Exception\UnknownPackageException
     * @throws Core\Package\Exception\UnknownPackagePathException
     * @throws Typo3SitemapLocator\Exception\BaseUrlIsNotSupported
     * @throws Typo3SitemapLocator\Exception\SitemapIsMissing
     * @throws \JsonException
     */
    protected function execute(Console\Input\InputInterface $input, Console\Output\OutputInterface $output): int
    {
        // Initialize client
        $clientOptions = $this->configuration->getParserClientOptions();
        $client = $this->clientFactory->get($clientOptions);

        // Initialize sub command
        $subCommand = new CacheWarmup\Command\CacheWarmupCommand($client, $this->eventDispatcher);
        $subCommand->setApplication($this->getApplication() ?? new Console\Application());

        // Initialize sub command input
        $subCommandInput = new Console\Input\ArrayInput(
            $this->prepareCommandParameters($input),
            $subCommand->getDefinition(),
        );
        $subCommandInput->setInteractive(false);

        // Run cache warmup in sub command from eliashaeussler/cache-warmup
        $statusCode = $subCommand->run($subCommandInput, $output);

        // Fail if strict mode is enabled and at least one crawl was erroneous
        if ($input->getOption('strict') && $statusCode > 0) {
            return $statusCode;
        }

        return self::SUCCESS;
    }

    /**
     * @return array<string, mixed>
     * @throws CacheWarmup\Exception\CrawlerOptionIsInvalid
     * @throws CacheWarmup\Exception\LocalFilePathIsMissingInUrl
     * @throws CacheWarmup\Exception\UrlIsEmpty
     * @throws CacheWarmup\Exception\UrlIsInvalid
     * @throws Core\Exception\SiteNotFoundException
     * @throws Core\Package\Exception\UnknownPackageException
     * @throws Core\Package\Exception\UnknownPackagePathException
     * @throws Typo3SitemapLocator\Exception\BaseUrlIsNotSupported
     * @throws Typo3SitemapLocator\Exception\SitemapIsMissing
     * @throws \JsonException
     */
    private function prepareCommandParameters(Console\Input\InputInterface $input): array
    {
        // Resolve input options
        $languages = $this->resolveLanguages($input->getOption('languages'));
        $urls = array_unique($this->resolvePages($input->getOption('pages'), $languages));
        $sitemaps = array_unique($this->resolveSites($input->getOption('sites'), $languages));
        $config = $input->getOption('config');
        $limit = max(0, (int)$input->getOption('limit'));
        $strategy = $input->getOption('strategy');
        $format = $input->getOption('format');
        $excludePatterns = $this->configuration->getExcludePatterns();

        // Fetch crawler and crawler options
        $crawler = $this->configuration->getVerboseCrawler();
        $crawlerOptions = $this->configuration->getVerboseCrawlerOptions();

        // Initialize sub-command parameters
        $subCommandParameters = [
            'sitemaps' => $sitemaps,
            '--urls' => $urls,
            '--limit' => $limit,
            '--crawler' => $crawler,
            '--format' => $format,
        ];

        // Add crawler options to sub-command parameters
        if ($crawlerOptions !== []) {
            $subCommandParameters['--crawler-options'] = json_encode($crawlerOptions, JSON_THROW_ON_ERROR);
        }

        // Add exclude patterns
        if ($excludePatterns !== []) {
            $subCommandParameters['--exclude'] = $excludePatterns;
        }

        // Add crawling strategy
        if ($strategy !== null) {
            $subCommandParameters['--strategy'] = $strategy;
        }

        // Add config file
        if ($config !== null) {
            if (Core\Utility\PathUtility::isExtensionPath($config)) {
                $config = $this->packageManager->resolvePackagePath($config);
            }

            $subCommandParameters['--config'] = $config;
        }

        return $subCommandParameters;
    }

    /**
     * @param array<string> $pages
     * @param list<int<-1, max>> $languages
     * @return list<string>
     * @throws Core\Exception\SiteNotFoundException
     */
    private function resolvePages(array $pages, array $languages): array
    {
        $resolvedUrls = [];

        foreach ($pages as $pageList) {
            $normalizedPages = Core\Utility\GeneralUtility::intExplode(',', $pageList, true);

            /** @var positive-int $page */
            foreach ($normalizedPages as $page) {
                $languageIds = $languages;

                if (\in_array(self::ALL_LANGUAGES, $languageIds, true)) {
                    $site = $this->siteRepository->findOneByPageId($page);
                    $languageIds = array_keys($site?->getLanguages() ?? []);
                }

                foreach ($languageIds as $languageId) {
                    $uri = $this->pageUriBuilder->build($page, $languageId);

                    if ($uri !== null) {
                        $resolvedUrls[] = (string)$uri;
                    }
                }
            }
        }

        return $resolvedUrls;
    }

    /**
     * @param array<string> $sites
     * @param list<int<-1, max>> $languages
     * @return list<Domain\Model\SiteAwareSitemap>
     * @throws CacheWarmup\Exception\LocalFilePathIsMissingInUrl
     * @throws CacheWarmup\Exception\UrlIsEmpty
     * @throws CacheWarmup\Exception\UrlIsInvalid
     * @throws Core\Exception\SiteNotFoundException
     * @throws Typo3SitemapLocator\Exception\BaseUrlIsNotSupported
     * @throws Typo3SitemapLocator\Exception\SitemapIsMissing
     */
    private function resolveSites(array $sites, array $languages): array
    {
        $requestedSites = [];
        $resolvedSitemaps = [];

        foreach ($sites as $siteList) {
            $requestedSites += Core\Utility\GeneralUtility::trimExplode(',', $siteList, true);

            if (\in_array(self::ALL_SITES, $requestedSites, true)) {
                $requestedSites = $this->siteRepository->findAll();

                break;
            }
        }

        foreach ($requestedSites as $site) {
            if (Core\Utility\MathUtility::canBeInterpretedAsInteger($site)) {
                /** @var positive-int $rootPageId */
                $rootPageId = (int)$site;
                $site = $this->siteRepository->findOneByRootPageId($rootPageId);
            } elseif (is_string($site)) {
                $site = $this->siteRepository->findOneByIdentifier($site);
            }

            // Skip inaccessible sites
            if ($site === null) {
                continue;
            }

            $languageIds = $languages;

            if ([self::ALL_LANGUAGES] === $languageIds) {
                $languageIds = array_keys($site->getLanguages());
            }

            foreach ($languageIds as $languageId) {
                $siteLanguage = $this->siteLanguageRepository->findOneByLanguageId($site, $languageId);

                // Skip inaccessible site languages
                if ($siteLanguage === null) {
                    continue;
                }

                $sitemaps = $this->sitemapLocator->locateBySite($site, $siteLanguage);

                foreach ($sitemaps as $sitemap) {
                    $resolvedSitemaps[] = Domain\Model\SiteAwareSitemap::fromLocatedSitemap($sitemap);
                }
            }
        }

        return $resolvedSitemaps;
    }

    /**
     * @param array<string> $languages
     * @return list<int<-1, max>>
     */
    private function resolveLanguages(array $languages): array
    {
        $resolvedLanguages = [];

        if ($languages === []) {
            // Run cache warmup for all languages by default
            return [self::ALL_LANGUAGES];
        }

        foreach ($languages as $languageList) {
            $normalizedLanguages = Core\Utility\GeneralUtility::intExplode(',', $languageList, true);

            if (\in_array(self::ALL_LANGUAGES, $normalizedLanguages, true)) {
                return [self::ALL_LANGUAGES];
            }

            foreach ($normalizedLanguages as $languageId) {
                if ($languageId >= 0) {
                    $resolvedLanguages[] = $languageId;
                }
            }
        }

        return $resolvedLanguages;
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	/*
6	* This file is part of the TYPO3 CMS extension "warming".
7	*
8	* Copyright (C) 2021-2025 Elias Häußler <elias@haeussler.dev>
9	*
10	* This program is free software: you can redistribute it and/or modify
11	* it under the terms of the GNU General Public License as published by
12	* the Free Software Foundation, either version 2 of the License, or
13	* (at your option) any later version.
14	*
15	* This program is distributed in the hope that it will be useful,
16	* but WITHOUT ANY WARRANTY; without even the implied warranty of
17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18	* GNU General Public License for more details.
19	*
20	* You should have received a copy of the GNU General Public License
21	* along with this program. If not, see <https://www.gnu.org/licenses/>.
22	*/
23
24	namespace EliasHaeussler\Typo3Warming\Command;
25
26	use EliasHaeussler\CacheWarmup;
27	use EliasHaeussler\Typo3SitemapLocator;
28	use EliasHaeussler\Typo3Warming\Configuration;
29	use EliasHaeussler\Typo3Warming\Crawler;
30	use EliasHaeussler\Typo3Warming\Domain;
31	use EliasHaeussler\Typo3Warming\Http;
32	use Psr\EventDispatcher;
33	use Symfony\Component\Console;
34	use TYPO3\CMS\Core;
35
36	/**
37	* WarmupCommand
38	*
39	* @author Elias Häußler <elias@haeussler.dev>
40	* @license GPL-2.0-or-later
41	*/
42	#[Console\Attribute\AsCommand(
43	name: 'warming:cachewarmup',
44	description: 'Warm up Frontend caches of single pages and/or whole sites using their XML sitemaps.',
45	)]
46	final class WarmupCommand extends Console\Command\Command
47	{
48	private const ALL_LANGUAGES = -1;
49	private const ALL_SITES = 'all';
50
51	public function __construct(	14✔
52	private readonly Http\Client\ClientFactory $clientFactory,
53	private readonly Configuration\Configuration $configuration,
54	private readonly Crawler\Strategy\CrawlingStrategyFactory $crawlingStrategyFactory,
55	private readonly Typo3SitemapLocator\Sitemap\SitemapLocator $sitemapLocator,
56	private readonly Domain\Repository\SiteRepository $siteRepository,
57	private readonly Domain\Repository\SiteLanguageRepository $siteLanguageRepository,
58	private readonly EventDispatcher\EventDispatcherInterface $eventDispatcher,
59	private readonly Core\Package\PackageManager $packageManager,
60	private readonly Http\Message\PageUriBuilder $pageUriBuilder,
61	) {
62	parent::__construct();	14✔
63	}
64
65	protected function configure(): void	14✔
66	{
67	$v = fn(mixed $value) => $value;	14✔
68	$decoratedCrawlingStrategies = \implode(PHP_EOL, array_map(	14✔
69	static fn(string $strategy) => ' │ * <info>' . $strategy . '</info>',	14✔
70	array_keys($this->crawlingStrategyFactory->getAll()),	14✔
71	));	14✔
72
73	$this->setDescription('Warm up Frontend caches of single pages and/or whole sites using their XML sitemaps.');	14✔
74	$this->setHelp(	14✔
75	<<<HELP	14✔
76	This command can be used in many ways to warm up frontend caches.	14✔
77	Some possible combinations and options are shown below.
78
79	<info>Sites and pages</info>
80	<info>===============</info>
81
82	To warm up caches, either <info>pages</info> or <info>sites</info> can be specified.
83	Both types can also be combined or extended by the specification of one or more <info>languages</info>.
84	If you omit the language option, the caches of all languages of the requested pages and sites
85	will be warmed up.
86
87	You can also use the special keyword <info>all</info> for <info>sites</info>.
88	This will cause all available sites to be warmed up.
89
90	Examples:
91
92	* <comment>warming:cachewarmup -p 1,2,3</comment>
93	├─ Pages: <info>1, 2 and 3</info>
94	└─ Languages: <info>all</info>
95
96	* <comment>warming:cachewarmup -s 1</comment>
97	* <comment>warming:cachewarmup -s main</comment>
98	├─ Sites: <info>Root page ID 1</info> or <info>identifier "main"</info>
99	└─ Languages: <info>all</info>
100
101	* <comment>warming:cachewarmup -p 1 -s 1</comment>
102	* <comment>warming:cachewarmup -p 1 -s main</comment>
103	├─ Pages: <info>1</info>
104	├─ Sites: <info>Root page ID 1</info> or <info>identifier "main"</info>
105	└─ Languages: <info>all</info>
106
107	* <comment>warming:cachewarmup -s 1 -l 0,1</comment>
108	├─ Sites: <info>Root page ID 1</info> or <info>identifier "main"</info>
109	└─ Languages: <info>0 and 1</info>
110
111	* <comment>warming:cachewarmup -s all</comment>
112	├─ Sites: <info>all</info>
113	└─ Languages: <info>all</info>
114
115	<info>Additional options</info>
116	<info>==================</info>
117
118	* <comment>Configuration file</comment>
119	├─ A preconfigured set of configuration options can be written to a configuration file.
120	│ This file can be passed using the <info>--config</info> option.
121	│ It may also contain extension paths, e.g. <info>EXT:sitepackage/Configuration/cache-warmup.json</info>.
122	│ The following file formats are currently supported:
123	│ * <info>json</info>
124	│ * <info>php</info>
125	│ * <info>yaml</info>
126	│ * <info>yml</info>
127	└─ Example: <comment>warming:cachewarmup --config path/to/cache-warmup.json</comment>
128
129	* <comment>Strict mode</comment>
130	├─ You can pass the <info>--strict</info> (or <info>-x</info>) option to terminate execution with an error code
131	│ if individual caches warm up incorrectly.
132	│ This is especially useful for automated execution of cache warmups.
133	├─ Default: <info>false</info>
134	└─ Example: <comment>warming:cachewarmup -s 1 --strict</comment>
135
136	* <comment>Crawl limit</comment>
137	├─ The maximum number of pages to be warmed up can be defined via the extension configuration <info>limit</info>.
138	│ It can be overridden by using the <info>--limit</info> option.
139	│ The value <info>0</info> deactivates the crawl limit.
140	├─ Default: <info>{$v($this->configuration->getLimit())}</info>	14✔
141	├─ Example: <comment>warming:cachewarmup -s 1 --limit 100</comment> (limits crawling to 100 pages)
142	└─ Example: <comment>warming:cachewarmup -s 1 --limit 0</comment> (no limit)
143
144	* <comment>Crawling strategy</comment>
145	├─ A crawling strategy defines how URLs will be crawled, e.g. by sorting them by a specific property.
146	│ It can be defined via the extension configuration <info>strategy</info> or by using the <info>--strategy</info> option.
147	│ The following strategies are currently available:
148	{$decoratedCrawlingStrategies}	14✔
149	├─ Default: <info>{$v($this->configuration->getStrategy() ?? 'none')}</info>	14✔
150	└─ Example: <comment>warming:cachewarmup --strategy {$v(CacheWarmup\Crawler\Strategy\SortByPriorityStrategy::getName())}</comment>	14✔
151
152	* <comment>Format output</comment>
153	├─ By default, all user-oriented output is printed as plain text to the console.
154	│ However, you can use other formatters by using the <info>--format</info> (or <info>-f</info>) option.
155	├─ Default: <info>{$v(CacheWarmup\Formatter\TextFormatter::getType())}</info>	14✔
156	├─ Example: <comment>warming:cachewarmup --format {$v(CacheWarmup\Formatter\TextFormatter::getType())}</comment> (normal output as plaintext)	14✔
157	└─ Example: <comment>warming:cachewarmup --format {$v(CacheWarmup\Formatter\JsonFormatter::getType())}</comment> (displays output as JSON)	14✔
158
159	<info>Crawling configuration</info>
160	<info>======================</info>
161
162	* <comment>Alternative crawler</comment>
163	├─ Use the extension configuration <info>verboseCrawler</info> to use an alternative crawler for
164	│ command-line requests. For warmup requests triggered via the TYPO3 backend, you can use the
165	│ extension configuration <info>crawler</info>.
166	├─ Currently used default crawler: <info>{$v($this->configuration->getCrawler())}</info>	14✔
167	└─ Currently used verbose crawler: <info>{$v($this->configuration->getVerboseCrawler())}</info>	14✔
168
169	* <comment>Custom User-Agent header</comment>
170	├─ When the default crawler is used, each warmup request is executed with a special User-Agent header.
171	│ This header is generated from the encryption key of the TYPO3 installation.
172	│ It can be used, for example, to exclude warmup requests from your search statistics.
173	└─ Current User-Agent: <info>{$v($this->configuration->getUserAgent())}</info>	14✔
174	HELP	14✔
175	);	14✔
176
177	$this->addOption(	14✔
178	'pages',	14✔
179	'p',	14✔
180	Console\Input\InputOption::VALUE_REQUIRED \| Console\Input\InputOption::VALUE_IS_ARRAY,	14✔
181	'Pages whose Frontend caches are to be warmed up.',	14✔
182	);	14✔
183	$this->addOption(	14✔
184	'sites',	14✔
185	's',	14✔
186	Console\Input\InputOption::VALUE_REQUIRED \| Console\Input\InputOption::VALUE_IS_ARRAY,	14✔
187	'Site identifiers or root page IDs of sites whose caches are to be warmed up.',	14✔
188	);	14✔
189	$this->addOption(	14✔
190	'languages',	14✔
191	'l',	14✔
192	Console\Input\InputOption::VALUE_REQUIRED \| Console\Input\InputOption::VALUE_IS_ARRAY,	14✔
193	'Optional identifiers of languages for which caches are to be warmed up.',	14✔
194	);	14✔
195	$this->addOption(	14✔
196	'config',	14✔
197	'c',	14✔
198	Console\Input\InputOption::VALUE_REQUIRED,	14✔
199	'Path to optional configuration file',	14✔
200	);	14✔
201	$this->addOption(	14✔
202	'limit',	14✔
203	null,	14✔
204	Console\Input\InputOption::VALUE_REQUIRED,	14✔
205	'Maximum number of pages to be crawled. Set to <info>0</info> to disable the limit.',	14✔
206	$this->configuration->getLimit(),	14✔
207	);	14✔
208	$this->addOption(	14✔
209	'strategy',	14✔
210	null,	14✔
211	Console\Input\InputOption::VALUE_REQUIRED,	14✔
212	'Optional strategy to prepare URLs before crawling them.',	14✔
213	$this->configuration->getStrategy(),	14✔
214	);	14✔
215	$this->addOption(	14✔
216	'format',	14✔
217	'f',	14✔
218	Console\Input\InputOption::VALUE_REQUIRED,	14✔
219	'Formatter used to print the cache warmup result',	14✔
220	CacheWarmup\Formatter\TextFormatter::getType(),	14✔
221	);	14✔
222	$this->addOption(	14✔
223	'strict',	14✔
224	'x',	14✔
225	Console\Input\InputOption::VALUE_NONE,	14✔
226	'Fail if an error occurred during cache warmup.',	14✔
227	);	14✔
228	}
229
230	protected function initialize(Console\Input\InputInterface $input, Console\Output\OutputInterface $output): void	14✔
231	{
232	Core\Core\Bootstrap::initializeBackendAuthentication();	14✔
233	}
234
235	/**
236	* @throws CacheWarmup\Exception\CrawlerOptionIsInvalid
237	* @throws CacheWarmup\Exception\LocalFilePathIsMissingInUrl
238	* @throws CacheWarmup\Exception\UrlIsEmpty
239	* @throws CacheWarmup\Exception\UrlIsInvalid
240	* @throws Console\Exception\ExceptionInterface
241	* @throws Core\Exception\SiteNotFoundException
242	* @throws Core\Package\Exception\UnknownPackageException
243	* @throws Core\Package\Exception\UnknownPackagePathException
244	* @throws Typo3SitemapLocator\Exception\BaseUrlIsNotSupported
245	* @throws Typo3SitemapLocator\Exception\SitemapIsMissing
246	* @throws \JsonException
247	*/
248	protected function execute(Console\Input\InputInterface $input, Console\Output\OutputInterface $output): int	14✔
249	{
250	// Initialize client
251	$clientOptions = $this->configuration->getParserClientOptions();	14✔
252	$client = $this->clientFactory->get($clientOptions);	14✔
253
254	// Initialize sub command
255	$subCommand = new CacheWarmup\Command\CacheWarmupCommand($client, $this->eventDispatcher);	14✔
256	$subCommand->setApplication($this->getApplication() ?? new Console\Application());	14✔
257
258	// Initialize sub command input
259	$subCommandInput = new Console\Input\ArrayInput(	14✔
260	$this->prepareCommandParameters($input),	14✔
261	$subCommand->getDefinition(),	14✔
262	);	14✔
263	$subCommandInput->setInteractive(false);	14✔
264
265	// Run cache warmup in sub command from eliashaeussler/cache-warmup
266	$statusCode = $subCommand->run($subCommandInput, $output);	14✔
267
268	// Fail if strict mode is enabled and at least one crawl was erroneous
269	if ($input->getOption('strict') && $statusCode > 0) {	13✔
270	return $statusCode;	1✔
271	}
272
273	return self::SUCCESS;	12✔
274	}
275
276	/**
277	* @return array<string, mixed>
278	* @throws CacheWarmup\Exception\CrawlerOptionIsInvalid
279	* @throws CacheWarmup\Exception\LocalFilePathIsMissingInUrl
280	* @throws CacheWarmup\Exception\UrlIsEmpty
281	* @throws CacheWarmup\Exception\UrlIsInvalid
282	* @throws Core\Exception\SiteNotFoundException
283	* @throws Core\Package\Exception\UnknownPackageException
284	* @throws Core\Package\Exception\UnknownPackagePathException
285	* @throws Typo3SitemapLocator\Exception\BaseUrlIsNotSupported
286	* @throws Typo3SitemapLocator\Exception\SitemapIsMissing
287	* @throws \JsonException
288	*/
289	private function prepareCommandParameters(Console\Input\InputInterface $input): array	14✔
290	{
291	// Resolve input options
292	$languages = $this->resolveLanguages($input->getOption('languages'));	14✔
293	$urls = array_unique($this->resolvePages($input->getOption('pages'), $languages));	14✔
294	$sitemaps = array_unique($this->resolveSites($input->getOption('sites'), $languages));	14✔
295	$config = $input->getOption('config');	14✔
296	$limit = max(0, (int)$input->getOption('limit'));	14✔
297	$strategy = $input->getOption('strategy');	14✔
298	$format = $input->getOption('format');	14✔
299	$excludePatterns = $this->configuration->getExcludePatterns();	14✔
300
301	// Fetch crawler and crawler options
302	$crawler = $this->configuration->getVerboseCrawler();	14✔
303	$crawlerOptions = $this->configuration->getVerboseCrawlerOptions();	14✔
304
305	// Initialize sub-command parameters
306	$subCommandParameters = [	14✔
307	'sitemaps' => $sitemaps,	14✔
308	'--urls' => $urls,	14✔
309	'--limit' => $limit,	14✔
310	'--crawler' => $crawler,	14✔
311	'--format' => $format,	14✔
312	];	14✔
313
314	// Add crawler options to sub-command parameters
315	if ($crawlerOptions !== []) {	14✔
316	$subCommandParameters['--crawler-options'] = json_encode($crawlerOptions, JSON_THROW_ON_ERROR);	1✔
317	}
318
319	// Add exclude patterns
320	if ($excludePatterns !== []) {	14✔
321	$subCommandParameters['--exclude'] = $excludePatterns;	1✔
322	}
323
324	// Add crawling strategy
325	if ($strategy !== null) {	14✔
326	$subCommandParameters['--strategy'] = $strategy;	1✔
327	}
328
329	// Add config file
330	if ($config !== null) {	14✔
331	if (Core\Utility\PathUtility::isExtensionPath($config)) {	1✔
332	$config = $this->packageManager->resolvePackagePath($config);	1✔
333	}
334
335	$subCommandParameters['--config'] = $config;	1✔
336	}
337
338	return $subCommandParameters;	14✔
339	}
340
341	/**
342	* @param array<string> $pages
343	* @param list<int<-1, max>> $languages
344	* @return list<string>
345	* @throws Core\Exception\SiteNotFoundException
346	*/
347	private function resolvePages(array $pages, array $languages): array	14✔
348	{
349	$resolvedUrls = [];	14✔
350
351	foreach ($pages as $pageList) {	14✔
352	$normalizedPages = Core\Utility\GeneralUtility::intExplode(',', $pageList, true);	9✔
353
354	/** @var positive-int $page */
355	foreach ($normalizedPages as $page) {	9✔
356	$languageIds = $languages;	9✔
357
358	if (\in_array(self::ALL_LANGUAGES, $languageIds, true)) {	9✔
359	$site = $this->siteRepository->findOneByPageId($page);	8✔
360	$languageIds = array_keys($site?->getLanguages() ?? []);	8✔
361	}
362
363	foreach ($languageIds as $languageId) {	9✔
364	$uri = $this->pageUriBuilder->build($page, $languageId);	9✔
365
366	if ($uri !== null) {	9✔
367	$resolvedUrls[] = (string)$uri;	9✔
368	}
369	}
370	}
371	}
372
373	return $resolvedUrls;	14✔
374	}
375
376	/**
377	* @param array<string> $sites
378	* @param list<int<-1, max>> $languages
379	* @return list<Domain\Model\SiteAwareSitemap>
380	* @throws CacheWarmup\Exception\LocalFilePathIsMissingInUrl
381	* @throws CacheWarmup\Exception\UrlIsEmpty
382	* @throws CacheWarmup\Exception\UrlIsInvalid
383	* @throws Core\Exception\SiteNotFoundException
384	* @throws Typo3SitemapLocator\Exception\BaseUrlIsNotSupported
385	* @throws Typo3SitemapLocator\Exception\SitemapIsMissing
386	*/
387	private function resolveSites(array $sites, array $languages): array	14✔
388	{
389	$requestedSites = [];	14✔
390	$resolvedSitemaps = [];	14✔
391
392	foreach ($sites as $siteList) {	14✔
393	$requestedSites += Core\Utility\GeneralUtility::trimExplode(',', $siteList, true);	4✔
394
395	if (\in_array(self::ALL_SITES, $requestedSites, true)) {	4✔
396	$requestedSites = $this->siteRepository->findAll();	1✔
397
398	break;
399	}
400	}
401
402	foreach ($requestedSites as $site) {	14✔
403	if (Core\Utility\MathUtility::canBeInterpretedAsInteger($site)) {	4✔
404	/** @var positive-int $rootPageId */
405	$rootPageId = (int)$site;	2✔
406	$site = $this->siteRepository->findOneByRootPageId($rootPageId);	2✔
407	} elseif (is_string($site)) {	2✔
408	$site = $this->siteRepository->findOneByIdentifier($site);	1✔
409	}
410
411	// Skip inaccessible sites
412	if ($site === null) {	4✔
413	continue;
414	}
415
416	$languageIds = $languages;	4✔
417
418	if ([self::ALL_LANGUAGES] === $languageIds) {	4✔
419	$languageIds = array_keys($site->getLanguages());	4✔
420	}
421
422	foreach ($languageIds as $languageId) {	4✔
423	$siteLanguage = $this->siteLanguageRepository->findOneByLanguageId($site, $languageId);	4✔
424
425	// Skip inaccessible site languages
426	if ($siteLanguage === null) {	4✔
427	continue;
428	}
429
430	$sitemaps = $this->sitemapLocator->locateBySite($site, $siteLanguage);	4✔
431
432	foreach ($sitemaps as $sitemap) {	4✔
433	$resolvedSitemaps[] = Domain\Model\SiteAwareSitemap::fromLocatedSitemap($sitemap);	4✔
434	}
435	}
436	}
437
438	return $resolvedSitemaps;	14✔
439	}
440
441	/**
442	* @param array<string> $languages
443	* @return list<int<-1, max>>
444	*/
445	private function resolveLanguages(array $languages): array	14✔
446	{
447	$resolvedLanguages = [];	14✔
448
449	if ($languages === []) {	14✔
450	// Run cache warmup for all languages by default
451	return [self::ALL_LANGUAGES];	13✔
452	}
453
454	foreach ($languages as $languageList) {	1✔
455	$normalizedLanguages = Core\Utility\GeneralUtility::intExplode(',', $languageList, true);	1✔
456
457	if (\in_array(self::ALL_LANGUAGES, $normalizedLanguages, true)) {	1✔
NEW 458	return [self::ALL_LANGUAGES];	×
459	}
460
461	foreach ($normalizedLanguages as $languageId) {	1✔
462	if ($languageId >= 0) {	1✔
463	$resolvedLanguages[] = $languageId;	1✔
464	}
465	}
466	}
467
468	return $resolvedLanguages;	1✔
469	}
470	}

eliashaeussler / typo3-warming / 14429819143

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous