diff --git a/Classes/Aspects/CacheUrlMappingAspect.php b/Classes/Aspects/CacheUrlMappingAspect.php
index a8e0db4..b6d8476 100644
--- a/Classes/Aspects/CacheUrlMappingAspect.php
+++ b/Classes/Aspects/CacheUrlMappingAspect.php
@@ -37,6 +37,11 @@ class CacheUrlMappingAspect
*/
protected $isActive = false;
+ /**
+ * @var null | int
+ */
+ protected $renderTimestamp = null;
+
/**
* @var array
*/
@@ -102,9 +107,6 @@ public function storeRootCacheIdentifier(JoinPointInterface $joinPoint)
/** @var NodeInterface $node */
$node = $this->currentEvaluateContext['cacheIdentifierValues']['node'];
- if (!$node->getContext()->getWorkspace()->isPublicWorkspace()) {
- return;
- }
$url = $this->getCurrentUrl();
@@ -144,8 +146,10 @@ public function storeRootCacheIdentifier(JoinPointInterface $joinPoint)
$logger->debug('Mapping URL ' . $url . ' to ' . $rootIdentifier . ' with tags ' . implode(', ', $rootTags));
$arguments = $this->getCurrentArguments($node);
+ // TODO: To make parallel rendering possible, we need to make sure that the cache key also includes the currently rendered workspace, as the node might originate from a base workspace (usually live). See `DocumentNodeCacheKey`.
$rootKey = DocumentNodeCacheKey::fromNodeAndArguments($node, $arguments);
- $rootCacheValues = DocumentNodeCacheValues::create($rootIdentifier, $url);
+ $rootCacheValues = DocumentNodeCacheValues::create($rootIdentifier, $url)
+ ->withMetadata('renderTime', (int)(microtime(true) * 1000) - $this->renderTimestamp);
// allow other document metadata generators here
$rootCacheValues = $this->nodeRenderingExtensionManager->runDocumentMetadataGenerators($node, $arguments, $this->controllerContext, $rootCacheValues);
$this->contentCacheFrontend->set($rootKey->redisKeyName(), json_encode($rootCacheValues), $rootTags);
@@ -202,6 +206,7 @@ public function beforeDocumentRendering(ContentReleaseLogger $contentReleaseLogg
{
$this->isActive = true;
$this->contentReleaseLogger = $contentReleaseLogger;
+ $this->renderTimestamp = (int)(microtime(true) * 1000);
}
public function afterDocumentRendering(): void
diff --git a/Classes/BackendUi/BackendUiDataService.php b/Classes/BackendUi/BackendUiDataService.php
index ebea4dd..165939e 100644
--- a/Classes/BackendUi/BackendUiDataService.php
+++ b/Classes/BackendUi/BackendUiDataService.php
@@ -14,6 +14,7 @@
use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderingStatistics;
use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingErrorManager;
use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingStatisticsStore;
+use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata;
use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
use Flowpack\DecoupledContentStore\ReleaseSwitch\Infrastructure\RedisReleaseSwitchService;
use Neos\Flow\Annotations as Flow;
@@ -81,12 +82,17 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId
$lastRendering = RenderingStatistics::fromJsonString($lastRenderingStatisticsEntries->getResultForContentRelease($contentReleaseId));
$firstRendering = RenderingStatistics::fromJsonString($firstRenderingStatisticsEntries->getResultForContentRelease($contentReleaseId));
+ $metadataForContentRelease = $metadata->getResultForContentRelease($contentReleaseId);
+ $countForContentRelease = $counts->getResultForContentRelease($contentReleaseId);
+ $iterationsCountForContentRelease = $iterationsCounts->getResultForContentRelease($contentReleaseId);
+ $errorCountForContentRelease = $errorCounts->getResultForContentRelease($contentReleaseId);
+
$result[] = new ContentReleaseOverviewRow(
$contentReleaseId,
- $metadata->getResultForContentRelease($contentReleaseId),
- $counts->getResultForContentRelease($contentReleaseId),
- $iterationsCounts->getResultForContentRelease($contentReleaseId),
- $errorCounts->getResultForContentRelease($contentReleaseId),
+ $metadataForContentRelease instanceof ContentReleaseMetadata ? $metadataForContentRelease : null,
+ is_int($countForContentRelease) ? $countForContentRelease : 0,
+ is_int($iterationsCountForContentRelease) ? $iterationsCountForContentRelease : 0,
+ is_int($errorCountForContentRelease) ? $errorCountForContentRelease : 0,
$lastRendering->getTotalJobs() > 0 ? round($lastRendering->getRenderedJobs()
/ $lastRendering->getTotalJobs() * 100) : 100,
$firstRendering->getRenderedJobs(),
@@ -114,9 +120,14 @@ private function calculateReleaseSize(RedisInstanceIdentifier $redisInstanceIden
return round($size / 1000000, 2);
}
- public function loadDetailsData(ContentReleaseIdentifier $contentReleaseIdentifier, RedisInstanceIdentifier $redisInstanceIdentifier): ContentReleaseDetails
+ public function loadDetailsData(ContentReleaseIdentifier $contentReleaseIdentifier, RedisInstanceIdentifier $redisInstanceIdentifier): ?ContentReleaseDetails
{
$contentReleaseMetadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier, $redisInstanceIdentifier);
+
+ if (!$contentReleaseMetadata) {
+ return null;
+ }
+
$contentReleaseJob = $this->prunnerApiService->loadJobDetail($contentReleaseMetadata->getPrunnerJobId()->toJobId());
$manualTransferJobs = count($contentReleaseMetadata->getManualTransferJobIds()) ? array_map(function (PrunnerJobId $item) {
diff --git a/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php b/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php
index d125bda..990f39d 100644
--- a/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php
+++ b/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php
@@ -14,7 +14,7 @@
class ContentReleaseOverviewRow
{
private ContentReleaseIdentifier $contentReleaseIdentifier;
- private ContentReleaseMetadata $metadata;
+ private ?ContentReleaseMetadata $metadata;
private int $enumeratedDocumentNodesCount;
private int $iterationsCount;
private int $errorCount;
@@ -23,7 +23,7 @@ class ContentReleaseOverviewRow
private bool $isActive;
private float $releaseSize;
- public function __construct(ContentReleaseIdentifier $contentReleaseIdentifier, ContentReleaseMetadata $metadata,
+ public function __construct(ContentReleaseIdentifier $contentReleaseIdentifier, ?ContentReleaseMetadata $metadata,
int $enumeratedDocumentNodesCount, int $iterationsCount, int $errorCount,
float $progress, int $renderedUrlCount, bool $isActive, float $releaseSize)
{
@@ -38,73 +38,46 @@ public function __construct(ContentReleaseIdentifier $contentReleaseIdentifier,
$this->releaseSize = $releaseSize;
}
- /**
- * @return ContentReleaseMetadata
- */
- public function getMetadata(): ContentReleaseMetadata
+ public function getMetadata(): ?ContentReleaseMetadata
{
return $this->metadata;
}
- /**
- * @return ContentReleaseIdentifier
- */
public function getContentReleaseIdentifier(): ContentReleaseIdentifier
{
return $this->contentReleaseIdentifier;
}
- /**
- * @return int
- */
public function getEnumeratedDocumentNodesCount(): int
{
return $this->enumeratedDocumentNodesCount;
}
- /**
- * @return int
- */
public function getIterationsCount(): int
{
return $this->iterationsCount;
}
- /**
- * @return int
- */
public function getErrorCount(): int
{
return $this->errorCount;
}
- /**
- * @return float
- */
public function getProgress(): float
{
return $this->progress;
}
- /**
- * @return int
- */
public function getRenderedUrlCount(): int
{
return $this->renderedUrlCount;
}
- /**
- * @return bool
- */
public function isActive(): bool
{
return $this->isActive;
}
- /**
- * @return float
- */
public function getReleaseSize(): float
{
return $this->releaseSize;
diff --git a/Classes/Command/ContentReleasePrepareCommandController.php b/Classes/Command/ContentReleasePrepareCommandController.php
index e48aa3c..3648869 100644
--- a/Classes/Command/ContentReleasePrepareCommandController.php
+++ b/Classes/Command/ContentReleasePrepareCommandController.php
@@ -10,6 +10,7 @@
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger;
use Neos\Flow\Cli\CommandController;
+use Neos\Fusion\Core\Cache\ContentCache;
/**
* Commands for the PREPARE stage in the pipeline. Not meant to be called manually.
@@ -28,6 +29,12 @@ class ContentReleasePrepareCommandController extends CommandController
*/
protected $concurrentBuildLock;
+ /**
+ * @Flow\Inject
+ * @var ContentCache
+ */
+ protected $contentCache;
+
public function createContentReleaseCommand(string $contentReleaseIdentifier, string $prunnerJobId, string $workspaceName = 'live'): void
{
$contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);
@@ -52,4 +59,15 @@ public function registerManualTransferJobCommand(string $contentReleaseIdentifie
$this->redisContentReleaseService->registerManualTransferJob($contentReleaseIdentifier, $prunnerJobId, $logger);
}
+
+ public function flushContentCacheIfRequiredCommand(string $contentReleaseIdentifier, bool $flushContentCache = false): void
+ {
+ $logger = ContentReleaseLogger::fromConsoleOutput($this->output, ContentReleaseIdentifier::fromString($contentReleaseIdentifier));
+ if (!$flushContentCache) {
+ $logger->info('Not flushing content cache');
+ return;
+ }
+ $logger->info('Flushing content cache');
+ $this->contentCache->flush();
+ }
}
diff --git a/Classes/ContentReleaseManager.php b/Classes/ContentReleaseManager.php
index d84e558..7b23cf8 100644
--- a/Classes/ContentReleaseManager.php
+++ b/Classes/ContentReleaseManager.php
@@ -12,18 +12,12 @@
use Neos\Flow\Annotations as Flow;
use Flowpack\Prunner\PrunnerApiService;
use Flowpack\Prunner\ValueObject\PipelineName;
-use Neos\Fusion\Core\Cache\ContentCache;
/**
* @Flow\Scope("singleton")
*/
class ContentReleaseManager
{
- /**
- * @Flow\Inject
- * @var ContentCache
- */
- protected $contentCache;
/**
* @Flow\Inject
@@ -61,6 +55,7 @@ public function startIncrementalContentRelease(string $currentContentReleaseId =
'currentContentReleaseId' => $currentContentReleaseId ?: self::NO_PREVIOUS_RELEASE,
'validate' => true,
'workspaceName' => $workspace ? $workspace->getName() : 'live',
+ 'flushContentCache' => false,
]));
return $contentReleaseId;
}
@@ -74,12 +69,12 @@ public function startFullContentRelease(bool $validate = true, string $currentCo
}
$contentReleaseId = ContentReleaseIdentifier::create();
- $this->contentCache->flush();
$this->prunnerApiService->schedulePipeline(PipelineName::create('do_content_release'), array_merge($additionalVariables, [
'contentReleaseId' => $contentReleaseId,
'currentContentReleaseId' => $currentContentReleaseId ?: self::NO_PREVIOUS_RELEASE,
'validate' => $validate,
'workspaceName' => $workspace ? $workspace->getName() : 'live',
+ 'flushContentCache' => true,
]));
return $contentReleaseId;
}
diff --git a/Classes/Controller/BackendController.php b/Classes/Controller/BackendController.php
index 60931fc..2622f67 100644
--- a/Classes/Controller/BackendController.php
+++ b/Classes/Controller/BackendController.php
@@ -114,6 +114,7 @@ public function detailsAction(string $contentReleaseIdentifier, ?string $content
$contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);
$contentStore = $contentStore ? RedisInstanceIdentifier::fromString($contentStore) : RedisInstanceIdentifier::primary();
+ $this->view->assign('contentReleaseIdentifier', $contentReleaseIdentifier);
$this->view->assign('contentStore', $contentStore->getIdentifier());
$detailsData = $this->backendUiDataService->loadDetailsData($contentReleaseIdentifier, $contentStore);
diff --git a/Classes/Core/ConcurrentBuildLockService.php b/Classes/Core/ConcurrentBuildLockService.php
index 6ddcdc8..0d91187 100644
--- a/Classes/Core/ConcurrentBuildLockService.php
+++ b/Classes/Core/ConcurrentBuildLockService.php
@@ -4,10 +4,11 @@
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
use Flowpack\DecoupledContentStore\Core\Infrastructure\RedisClientManager;
+use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
use Neos\Flow\Annotations as Flow;
/**
- * We usually rely on prunner to ensure that only one build is running at any given time.
+ * We usually rely on prunner to ensure that only one build per workspace is running at any given time.
*
* However, when running in a cloud environment with no shared storage, the prunner data folder is not shared between
* instances. In this case, during a deployment, two containers run concurrently, with two separate prunner instances
@@ -27,6 +28,7 @@
*/
class ConcurrentBuildLockService
{
+ private const CONTENT_STORE_CONCURRENT_BUILD_LOCK = 'contentStore:concurrentBuildLocks';
/**
* @Flow\Inject
@@ -34,28 +36,35 @@ class ConcurrentBuildLockService
*/
protected $redisClientManager;
- public function ensureAllOtherInProgressContentReleasesWillBeTerminated(ContentReleaseIdentifier $contentReleaseIdentifier)
+ /**
+ * @Flow\Inject
+ * @var RedisContentReleaseService
+ */
+ protected $redisContentReleaseService;
+
+ public function ensureAllOtherInProgressContentReleasesWillBeTerminated(ContentReleaseIdentifier $contentReleaseIdentifier): void
{
- $this->redisClientManager->getPrimaryRedis()->set('contentStore:concurrentBuildLock', $contentReleaseIdentifier->getIdentifier());
+ $metadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier);
+ $this->redisClientManager->getPrimaryRedis()->hSet(self::CONTENT_STORE_CONCURRENT_BUILD_LOCK, $metadata->getWorkspaceName(), (string)$contentReleaseIdentifier);
}
- public function assertNoOtherContentReleaseWasStarted(ContentReleaseIdentifier $contentReleaseIdentifier)
+ public function assertNoOtherContentReleaseWasStarted(ContentReleaseIdentifier $contentReleaseIdentifier): void
{
- $concurrentBuildLockString = $this->redisClientManager->getPrimaryRedis()->get('contentStore:concurrentBuildLock');
+ $metadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier);
+ $concurrentBuildLockStrings = $this->redisClientManager->getPrimaryRedis()->hGetAll(self::CONTENT_STORE_CONCURRENT_BUILD_LOCK);
+ $concurrentBuildLockStringForWorkspace = $concurrentBuildLockStrings[$metadata->getWorkspaceName()] ?? null;
- if (empty($concurrentBuildLockString)) {
+ if (!$concurrentBuildLockStringForWorkspace) {
echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock does not exist.' . "\n\n";
echo "This should never happen for correctly configured jobs (that run after prepare_finished).\n\n";
exit(1);
}
-
- $concurrentBuildLock = ContentReleaseIdentifier::fromString($concurrentBuildLockString);
+ $concurrentBuildLock = ContentReleaseIdentifier::fromString($concurrentBuildLockStringForWorkspace);
if (!$contentReleaseIdentifier->equals($concurrentBuildLock)) {
// the concurrent build lock is different (i.e. newer) than our currently-running content release.
// Thus, we abort the in-progress content release as quickly as we can - by DYING.
-
- echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock contains ' . $concurrentBuildLock->getIdentifier() . "\n\n";
+ echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock for workspace "' . $metadata->getWorkspaceName() . '" contains ' . $concurrentBuildLock->getIdentifier() . "\n\n";
echo "This is no error during deployment, but should never happen outside a deployment.\n\n It can only happen if two prunner instances run concurrently.\n\n";
exit(1);
}
diff --git a/Classes/Core/Domain/Dto/ContentReleaseBatchResult.php b/Classes/Core/Domain/Dto/ContentReleaseBatchResult.php
index 491f6ea..673f654 100644
--- a/Classes/Core/Domain/Dto/ContentReleaseBatchResult.php
+++ b/Classes/Core/Domain/Dto/ContentReleaseBatchResult.php
@@ -3,6 +3,7 @@
namespace Flowpack\DecoupledContentStore\Core\Domain\Dto;
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
+use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata;
use Neos\Flow\Annotations as Flow;
/**
@@ -20,7 +21,6 @@ private function __construct(array $results)
$this->results = $results;
}
-
public static function createFromArray(array $in): self
{
return new self($in);
@@ -28,7 +28,7 @@ public static function createFromArray(array $in): self
public function getResultForContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier)
{
- return $this->results[$contentReleaseIdentifier->jsonSerialize()];
+ return $this->results[(string)$contentReleaseIdentifier] ?? null;
}
-}
\ No newline at end of file
+}
diff --git a/Classes/Core/Infrastructure/ContentReleaseLogger.php b/Classes/Core/Infrastructure/ContentReleaseLogger.php
index 3531115..01400f0 100644
--- a/Classes/Core/Infrastructure/ContentReleaseLogger.php
+++ b/Classes/Core/Infrastructure/ContentReleaseLogger.php
@@ -19,10 +19,9 @@ class ContentReleaseLogger
*/
protected $contentReleaseIdentifier;
- /**
- * @var string
- */
- protected $logPrefix = '';
+ protected string $logPrefix = '';
+
+ protected ?RendererIdentifier $rendererIdentifier;
protected function __construct(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, ?RendererIdentifier $rendererIdentifier)
{
@@ -47,24 +46,30 @@ public static function fromSymfonyOutput(OutputInterface $output, ContentRelease
return new static($output, $contentReleaseIdentifier, null);
}
- public function debug($message, array $additionalPayload = [])
+ public function debug(string $message, array $additionalPayload = []): void
+ {
+ $this->logToOutput('DEBUG', $message, $additionalPayload);
+ }
+
+ public function info(string $message, array $additionalPayload = []): void
{
- $this->output->writeln($this->logPrefix . $message . json_encode($additionalPayload));
+ $this->logToOutput('INFO', $message, $additionalPayload);
}
- public function info($message, array $additionalPayload = [])
+ public function warn(string $message, array $additionalPayload = []): void
{
- $this->output->writeln($this->logPrefix . $message . json_encode($additionalPayload));
+ $this->logToOutput('WARNING', $message, $additionalPayload);
}
- public function warn($message, array $additionalPayload = [])
+ public function error(string $message, array $additionalPayload = []): void
{
- $this->output->writeln($this->logPrefix . $message . json_encode($additionalPayload));
+ $this->logToOutput('ERROR', $message, $additionalPayload);
}
- public function error($message, array $additionalPayload = [])
+ protected function logToOutput(string $level, string $message, array $additionalPayload = []): void
{
- $this->output->writeln($this->logPrefix . $message . json_encode($additionalPayload));
+ $formattedPayload = $additionalPayload ? json_encode($additionalPayload) : '';
+ $this->output->writeln($this->logPrefix . $level . ': ' . $message . $formattedPayload);
}
public function logException(\Exception $exception, string $message, array $additionalPayload)
diff --git a/Classes/Eel/ModuleHelper.php b/Classes/Eel/ModuleHelper.php
new file mode 100644
index 0000000..6aedf61
--- /dev/null
+++ b/Classes/Eel/ModuleHelper.php
@@ -0,0 +1,63 @@
+$1: $2", htmlSpecialChars($line));
+
+ // Add line numbers
+ $line = ($lineCount - $index) . ': ' . $line;
+
+ // Insert formatted JSON data
+ if ($jsonData) {
+ $isDebug = strpos($line, 'DEBUG:') !== false;
+ $jsonString = "\n" . json_encode($jsonData, JSON_PRETTY_PRINT) . "";
+ $line = $isDebug ? '' . '' . $line . '
' . $jsonString . ' ' : $line . $jsonString;
+ }
+
+ // Wrap in tags
+ return '' . $line . '
';
+ }, $lines, range(1, $lineCount));
+
+ return implode("\n", $formattedLines);
+ }
+
+ public function allowsCallOfMethod($methodName): bool
+ {
+ return true;
+ }
+}
diff --git a/Classes/IncrementalContentReleaseHandler.php b/Classes/IncrementalContentReleaseHandler.php
index 7f081da..f0b1aaa 100644
--- a/Classes/IncrementalContentReleaseHandler.php
+++ b/Classes/IncrementalContentReleaseHandler.php
@@ -4,9 +4,6 @@
namespace Flowpack\DecoupledContentStore;
use Neos\Flow\Annotations as Flow;
-use Flowpack\Prunner\PrunnerApiService;
-use Flowpack\Prunner\ValueObject\PipelineName;
-use Neos\Fusion\Core\Cache\ContentCache;
/**
* @Flow\Scope("singleton")
@@ -19,6 +16,11 @@ class IncrementalContentReleaseHandler
*/
protected $contentReleaseManager;
+ /**
+ * @Flow\InjectConfiguration("startIncrementalReleaseOnWorkspacePublish")
+ */
+ protected $startIncrementalReleaseOnWorkspacePublish;
+
protected $nodePublishedInThisRequest = false;
public function nodePublished()
@@ -28,8 +30,8 @@ public function nodePublished()
public function startContentReleaseIfNodesWerePublishedBefore()
{
- if ($this->nodePublishedInThisRequest === true) {
+ if ($this->startIncrementalReleaseOnWorkspacePublish === true && $this->nodePublishedInThisRequest === true) {
$this->contentReleaseManager->startIncrementalContentRelease();
}
}
-}
\ No newline at end of file
+}
diff --git a/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php b/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php
index 329cb78..850da33 100644
--- a/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php
+++ b/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php
@@ -56,12 +56,14 @@ static public function fromJsonString(string $enumeratedNodeString): self
return new self($tmp['contextPath'], $tmp['nodeIdentifier'], $tmp['arguments']);
}
- public function jsonSerialize()
+ public function jsonSerialize(): array
{
return [
'contextPath' => $this->contextPath,
'nodeIdentifier' => $this->nodeIdentifier,
- 'arguments' => $this->arguments
+ 'dimensions' => $this->getDimensionsFromContextPath(),
+ 'workspaceName' => $this->getWorkspaceNameFromContextPath(),
+ 'arguments' => $this->arguments,
];
}
@@ -80,9 +82,12 @@ public function getDimensionsFromContextPath(): array
return $nodePathAndContext['dimensions'];
}
- /**
- * @return string
- */
+ public function getWorkspaceNameFromContextPath(): string
+ {
+ $nodePathAndContext = NodePaths::explodeContextPath($this->contextPath);
+ return $nodePathAndContext['workspaceName'];
+ }
+
public function getNodeIdentifier(): string
{
return $this->nodeIdentifier;
@@ -97,4 +102,4 @@ public function debugString(): string
{
return sprintf('Node %s (%s)', $this->nodeIdentifier, $this->contextPath);
}
-}
\ No newline at end of file
+}
diff --git a/Classes/NodeEnumeration/Domain/Service/NodeContextCombinator.php b/Classes/NodeEnumeration/Domain/Service/NodeContextCombinator.php
index 5e11da4..7ae2fa6 100644
--- a/Classes/NodeEnumeration/Domain/Service/NodeContextCombinator.php
+++ b/Classes/NodeEnumeration/Domain/Service/NodeContextCombinator.php
@@ -33,6 +33,12 @@ class NodeContextCombinator
*/
protected $contextFactory;
+ /**
+ * @Flow\InjectConfiguration(path="nodeRendering.recurseHiddenContent", package="Flowpack.DecoupledContentStore")
+ * @var ContextFactoryInterface
+ */
+ protected $recurseHiddenContent;
+
/**
* Iterate over the node with the given identifier and site in contexts for all available presets (if it exists as a variant)
*
@@ -86,7 +92,8 @@ public function siteNodeInContexts(Site $site, string $workspaceName = 'live'):
'currentSite' => $site,
'workspaceName' => $workspaceName,
'dimensions' => [],
- 'targetDimensions' => []
+ 'targetDimensions' => [],
+ 'invisibleContentShown' => $this->recurseHiddenContent,
));
$siteNode = $contentContext->getNode('/sites/' . $site->getNodeName());
@@ -101,7 +108,8 @@ public function siteNodeInContexts(Site $site, string $workspaceName = 'live'):
'currentSite' => $site,
'workspaceName' => $workspaceName,
'dimensions' => $dimensions,
- 'targetDimensions' => []
+ 'targetDimensions' => [],
+ 'invisibleContentShown' => $this->recurseHiddenContent,
));
$siteNode = $contentContext->getNode('/sites/' . $site->getNodeName());
diff --git a/Classes/NodeEnumeration/NodeEnumerator.php b/Classes/NodeEnumeration/NodeEnumerator.php
index ac8c157..a022bb8 100644
--- a/Classes/NodeEnumeration/NodeEnumerator.php
+++ b/Classes/NodeEnumeration/NodeEnumerator.php
@@ -13,18 +13,14 @@
use Flowpack\DecoupledContentStore\NodeRendering\Dto\NodeRenderingCompletionStatus;
use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
use Flowpack\DecoupledContentStore\Utility\GeneratorUtility;
-use Neos\ContentRepository\Domain\NodeType\NodeTypeConstraintFactory;
-use Neos\ContentRepository\Domain\NodeType\NodeTypeName;
+use Neos\ContentRepository\Domain\Model\NodeInterface;
+use Neos\Eel\Exception;
+use Neos\Eel\FlowQuery\FlowQuery;
use Neos\Flow\Annotations as Flow;
use Neos\Neos\Domain\Model\Site;
class NodeEnumerator
{
- /**
- * @Flow\Inject
- * @var NodeTypeConstraintFactory
- */
- protected $nodeTypeConstraintFactory;
/**
* @Flow\Inject
@@ -48,19 +44,34 @@ class NodeEnumerator
* @Flow\InjectConfiguration("nodeRendering.nodeTypeWhitelist")
* @var string
*/
- protected $nodeTypeWhitelist;
-
- public function enumerateAndStoreInRedis(?Site $site, ContentReleaseLogger $contentReleaseLogger, ContentReleaseIdentifier $releaseIdentifier): void
- {
- $contentReleaseLogger->info('Starting content release', ['contentReleaseIdentifier' => $releaseIdentifier->jsonSerialize()]);
+ protected $nodeTypeList;
+
+ public function enumerateAndStoreInRedis(
+ ?Site $site,
+ ContentReleaseLogger $contentReleaseLogger,
+ ContentReleaseIdentifier $releaseIdentifier
+ ): void {
+ $contentReleaseLogger->info(
+ 'Starting content release',
+ ['contentReleaseIdentifier' => $releaseIdentifier->jsonSerialize()]
+ );
// set content release status to running
$currentMetadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($releaseIdentifier);
$newMetadata = $currentMetadata->withStatus(NodeRenderingCompletionStatus::running());
- $this->redisContentReleaseService->setContentReleaseMetadata($releaseIdentifier, $newMetadata, RedisInstanceIdentifier::primary());
+ $this->redisContentReleaseService->setContentReleaseMetadata(
+ $releaseIdentifier,
+ $newMetadata,
+ RedisInstanceIdentifier::primary()
+ );
$this->redisEnumerationRepository->clearDocumentNodesEnumeration($releaseIdentifier);
- foreach (GeneratorUtility::createArrayBatch($this->enumerateAll($site, $contentReleaseLogger, $newMetadata->getWorkspaceName()), 100) as $enumeration) {
+ foreach (
+ GeneratorUtility::createArrayBatch(
+ $this->enumerateAll($site, $contentReleaseLogger, $newMetadata->getWorkspaceName()),
+ 100
+ ) as $enumeration
+ ) {
$this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($releaseIdentifier);
// $enumeration is an array of EnumeratedNode, with at most 100 elements in it.
// TODO: EXTENSION POINT HERE, TO ADD ADDITIONAL ENUMERATIONS (.metadata.json f.e.)
@@ -71,48 +82,85 @@ public function enumerateAndStoreInRedis(?Site $site, ContentReleaseLogger $cont
/**
* @return iterable
+ * @throws Exception
*/
- private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseLogger, string $workspaceName): iterable
- {
+ private function enumerateAll(
+ ?Site $site,
+ ContentReleaseLogger $contentReleaseLogger,
+ string $workspaceName
+ ): iterable {
$combinator = new NodeContextCombinator();
- $nodeTypeWhitelist = $this->nodeTypeConstraintFactory->parseFilterString($this->nodeTypeWhitelist);
-
- $queueSite = function (Site $site) use ($combinator, &$documentNodeVariantsToRender, $nodeTypeWhitelist, $contentReleaseLogger, $workspaceName) {
+ // Build filter from allowed/disallowed nodetypes
+ $nodeTypeList = explode(',', $this->nodeTypeList ?: 'Neos.Neos:Document');
+ $nodeTypeFilter = implode(
+ ',',
+ array_map(static function ($nodeType) {
+ if ($nodeType[0] === '!') {
+ return '[!instanceof ' . substr($nodeType, 1) . ']';
+ }
+ return '[instanceof ' . $nodeType . ']';
+ }, $nodeTypeList)
+ );
+
+ $queueSite = static function (Site $site) use (
+ $combinator,
+ $nodeTypeFilter,
+ $contentReleaseLogger,
+ $workspaceName
+ ) {
$contentReleaseLogger->debug('Publishing site', [
'name' => $site->getName(),
'domain' => $site->getFirstActiveDomain()
]);
foreach ($combinator->siteNodeInContexts($site, $workspaceName) as $siteNode) {
+ $startTime = microtime(true);
$dimensionValues = $siteNode->getContext()->getDimensions();
$contentReleaseLogger->debug('Publishing dimension combination', [
'dimensionValues' => $dimensionValues
]);
- foreach ($combinator->recurseDocumentChildNodes($siteNode) as $documentNode) {
- $contextPath = $documentNode->getContextPath();
-
- if ($nodeTypeWhitelist->matches(NodeTypeName::fromString($documentNode->getNodeType()->getName()))) {
+ $nodeQuery = new FlowQuery([$siteNode]);
+ /** @var NodeInterface[] $matchingNodes */
+ $matchingNodes = $nodeQuery->find($nodeTypeFilter)->add($siteNode)->get();
+
+ foreach ($matchingNodes as $nodeToEnumerate) {
+ $contextPath = $nodeToEnumerate->getContextPath();
+
+ // Verify that the node is not orphaned
+ $parentNode = $nodeToEnumerate->getParent();
+ while ($parentNode !== $siteNode) {
+ if ($parentNode === null) {
+ $contentReleaseLogger->debug('Skipping node from publishing, because it is orphaned', [
+ 'node' => $contextPath,
+ ]);
+ // Continue with the next document
+ continue 2;
+ }
+ $parentNode = $parentNode->getParent();
+ }
- $contentReleaseLogger->debug('Registering node for publishing', [
- 'node' => $contextPath
+ if ($nodeToEnumerate->isHidden()) {
+ $contentReleaseLogger->debug('Skipping node from publishing, because it is hidden', [
+ 'node' => $contextPath,
]);
-
- yield EnumeratedNode::fromNode($documentNode);
} else {
- $contentReleaseLogger->debug('Skipping node from publishing, because it did not match the configured nodeTypeWhitelist', [
- 'node' => $contextPath,
- 'nodeTypeWhitelist' => $this->nodeTypeWhitelist
+ $contentReleaseLogger->debug('Registering node for publishing', [
+ 'node' => $contextPath
]);
+ yield EnumeratedNode::fromNode($nodeToEnumerate);
}
}
}
+ $contentReleaseLogger->debug(
+ sprintf('Finished enumerating site %s in %dms', $site->getName(), (microtime(true) - $startTime) * 1000)
+ );
};
if ($site === null) {
- foreach ($combinator->sites() as $site) {
- yield from $queueSite($site);
+ foreach ($combinator->sites() as $siteInList) {
+ yield from $queueSite($siteInList);
}
} else {
yield from $queueSite($site);
diff --git a/Classes/NodeRendering/Dto/DocumentNodeCacheKey.php b/Classes/NodeRendering/Dto/DocumentNodeCacheKey.php
index 9d63dc2..2727b71 100644
--- a/Classes/NodeRendering/Dto/DocumentNodeCacheKey.php
+++ b/Classes/NodeRendering/Dto/DocumentNodeCacheKey.php
@@ -25,12 +25,17 @@ final class DocumentNodeCacheKey
*/
protected $dimensions;
+ /**
+ * @var string
+ */
+ protected $workspaceName;
+
/**
* @var array
*/
protected $arguments;
- private function __construct(string $nodeIdentifier, array $dimensions, array $arguments)
+ private function __construct(string $nodeIdentifier, array $dimensions, string $workspaceName, array $arguments)
{
// we need to make this deterministic
ksort($arguments);
@@ -38,22 +43,24 @@ private function __construct(string $nodeIdentifier, array $dimensions, array $a
$this->nodeIdentifier = $nodeIdentifier;
$this->dimensions = $dimensions;
+ $this->workspaceName = $workspaceName;
$this->arguments = $arguments;
}
public static function fromNodeAndArguments(NodeInterface $node, array $arguments): self
{
- return new self($node->getIdentifier(), $node->getContext()->getDimensions(), $arguments);
+ return new self($node->getIdentifier(), $node->getContext()->getDimensions(), $node->getWorkspace()->getName(), $arguments);
}
public static function fromEnumeratedNode(EnumeratedNode $enumeratedNode)
{
- return new self($enumeratedNode->getNodeIdentifier(), $enumeratedNode->getDimensionsFromContextPath(), $enumeratedNode->getArguments());
+ return new self($enumeratedNode->getNodeIdentifier(), $enumeratedNode->getDimensionsFromContextPath(), $enumeratedNode->getWorkspaceNameFromContextPath(), $enumeratedNode->getArguments());
}
public function redisKeyName(): string
{
+ // TODO: Add workspace name to cache entry to allow parallel releases, but `CacheUrlMappingAspect` has to provide node in correct workspace during rendering
return preg_replace('/[^a-zA-Z0-9-]/', '_', sprintf('doc--%s-%s-%s', $this->nodeIdentifier, json_encode($this->dimensions), json_encode($this->arguments)));
}
diff --git a/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php b/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php
index 97e01f7..1a77cbf 100644
--- a/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php
+++ b/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php
@@ -10,6 +10,7 @@
use Neos\Cache\Frontend\StringFrontend;
use Neos\Flow\Annotations as Flow;
use Neos\Flow\ObjectManagement\ObjectManagerInterface;
+use Neos\Flow\Package\Exception\UnknownPackageException;
use Neos\Flow\Package\PackageManager;
use Neos\Fusion\Core\Cache\ContentCache;
@@ -37,41 +38,85 @@ class RedisContentCacheReader
*/
protected $applicationIdentifier;
- public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentNodeCacheKey $documentNodeCacheKey): RenderedDocumentFromContentCache
- {
- $maxNestLevel = ContentCache::MAXIMUM_NESTING_LEVEL;
- $contentCacheStartToken = ContentCache::CACHE_SEGMENT_START_TOKEN;
- $contentCacheEndToken = ContentCache::CACHE_SEGMENT_END_TOKEN;
- $contentCacheMarker = ContentCache::CACHE_SEGMENT_MARKER;
+ protected ?string $scriptSha1 = null;
+ protected ?\Redis $redis = null;
+
+ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentNodeCacheKey $documentNodeCacheKey
+ ): RenderedDocumentFromContentCache {
/**
* @see AbstractBackend::setCache()
*/
$identifierPrefix = md5($this->applicationIdentifier) . ':';
- $packageManager = $this->objectManager->get(PackageManager::class);
- $flowPackage = $packageManager->getPackage('Neos.Flow');
- preg_match('/^(\d+\.\d+)/', $flowPackage->getInstalledVersion(), $versionMatches);
- $flowMajorVersion = (int)($versionMatches[1] ?? '0');
+ $redis = $this->getRedis();
- $backend = $this->contentCache->getBackend();
- if ($flowMajorVersion >= 8 && $backend instanceof RedisBackend) {
- $reflProp = new \ReflectionProperty(RedisBackend::class, 'redis');
- $reflProp->setAccessible(true);
- $redis = $reflProp->getValue($backend);
- } elseif (get_class($backend) === 'Sandstorm\OptimizedRedisCacheBackend\OptimizedRedisCacheBackend') {
- $reflProp = new \ReflectionProperty(\Sandstorm\OptimizedRedisCacheBackend\OptimizedRedisCacheBackend::class, 'redis');
- $reflProp->setAccessible(true);
- $redis = $reflProp->getValue($backend);
- } else {
- throw new \RuntimeException('The cache backend for "Neos_Fusion_Content" must be an OptimizedRedisCacheBackend, but is ' . get_class($backend), 1622570000);
- }
$serializedCacheValues = $redis->get($documentNodeCacheKey->fullyQualifiedRedisKeyName($identifierPrefix));
if ($serializedCacheValues === false) {
- return RenderedDocumentFromContentCache::createIncomplete('No Redis Key "' . $documentNodeCacheKey->redisKeyName() . '" found.');
+ return RenderedDocumentFromContentCache::createIncomplete(
+ 'No Redis Key "' . $documentNodeCacheKey->redisKeyName() . '" found.'
+ );
}
$documentNodeCacheValues = DocumentNodeCacheValues::fromJsonString($serializedCacheValues);
+ $res = $this->executeRedisCall([$documentNodeCacheValues->getRootIdentifier(), $identifierPrefix]);
+ [$content, $error] = $res;
- $script = "
+ if (strlen($error) > 0) {
+ return RenderedDocumentFromContentCache::createIncomplete($error);
+ }
+ return RenderedDocumentFromContentCache::createWithFullContent($content, $documentNodeCacheValues);
+ }
+
+ protected function executeRedisCall(array $params): array
+ {
+ $redis = $this->getRedis();
+
+ $this->prepareRedisScript();
+ try {
+ // starting with Lua 7, eval_ro can be used.
+ $res = $redis->evalSha($this->scriptSha1, $params);
+
+ $error = $redis->getLastError();
+ if ($error !== null) {
+ throw new \RuntimeException('Redis error: ' . $error);
+ }
+ } catch (\Exception $e) {
+ throw new \RuntimeException('Redis script execution error: ' . $e->getMessage());
+ }
+
+ if (!is_array($res) || count($res) !== 2) {
+ throw new \RuntimeException('Result is no array of length 2, but: ' . count($res));
+ }
+
+ return $res;
+ }
+
+ protected function prepareRedisScript(): void
+ {
+ $redis = $this->getRedis();
+
+ try {
+ // Load script into redis if it is not already loaded
+ $scriptExists = false;
+ if ($this->scriptSha1) {
+ $scriptExists = $redis->script('exists', $this->scriptSha1)[0] ?? false;
+ }
+ if (!$scriptExists) {
+ $script = $this->buildRedisScript();
+ $this->scriptSha1 = $redis->script('load', $script);
+ }
+ } catch (\RedisException $e) {
+ throw new \RuntimeException('Redis Error: ' . $e->getMessage());
+ }
+ }
+
+ protected function buildRedisScript(): string
+ {
+ $maxNestLevel = ContentCache::MAXIMUM_NESTING_LEVEL;
+ $contentCacheStartToken = ContentCache::CACHE_SEGMENT_START_TOKEN;
+ $contentCacheEndToken = ContentCache::CACHE_SEGMENT_END_TOKEN;
+ $contentCacheMarker = ContentCache::CACHE_SEGMENT_MARKER;
+
+ return "
local rootIdentifier = ARGV[1]
local identifierPrefix = ARGV[2]
@@ -120,22 +165,45 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentN
return {content, error}
";
- // starting with Lua 7, eval_ro can be used.
- $res = $redis->eval($script, [$documentNodeCacheValues->getRootIdentifier(), $identifierPrefix], 0);
- $error = $redis->getLastError();
- if ($error !== null) {
- throw new \RuntimeException('Redis Error: ' . $error);
+ }
+
+ /**
+ * @throws UnknownPackageException
+ */
+ protected function getRedis(): \Redis
+ {
+ if ($this->redis) {
+ return $this->redis;
}
- if (count($res) !== 2) {
- throw new \RuntimeException('Result is no array of length 2, but: ' . count($res));
+ $packageManager = $this->objectManager->get(PackageManager::class);
+ $flowPackage = $packageManager->getPackage('Neos.Flow');
+ preg_match('/^(\d+\.\d+)/', $flowPackage->getInstalledVersion(), $versionMatches);
+ $flowMajorVersion = (int)($versionMatches[1] ?? '0');
+
+ $backend = $this->contentCache->getBackend();
+
+ if ($flowMajorVersion >= 8 && $backend instanceof RedisBackend) {
+ $reflProp = new \ReflectionProperty(RedisBackend::class, 'redis');
+ $reflProp->setAccessible(true);
+ $this->redis = $reflProp->getValue($backend);
+ return $this->redis;
}
- $content = $res[0];
- $error = $res[1];
- if (strlen($error) > 0) {
- return RenderedDocumentFromContentCache::createIncomplete($error);
+ if (get_class($backend) === 'Sandstorm\OptimizedRedisCacheBackend\OptimizedRedisCacheBackend') {
+ $reflProp = new \ReflectionProperty(
+ \Sandstorm\OptimizedRedisCacheBackend\OptimizedRedisCacheBackend::class,
+ 'redis'
+ );
+ $reflProp->setAccessible(true);
+ $this->redis = $reflProp->getValue($backend);
+ return $this->redis;
}
- return RenderedDocumentFromContentCache::createWithFullContent($content, $documentNodeCacheValues);
+
+ throw new \RuntimeException(
+ 'The cache backend for "Neos_Fusion_Content" must be an OptimizedRedisCacheBackend, but is ' . get_class(
+ $backend
+ ), 1622570000
+ );
}
}
diff --git a/Classes/NodeRendering/NodeRenderOrchestrator.php b/Classes/NodeRendering/NodeRenderOrchestrator.php
index d0ada05..6bf0289 100644
--- a/Classes/NodeRendering/NodeRenderOrchestrator.php
+++ b/Classes/NodeRendering/NodeRenderOrchestrator.php
@@ -122,6 +122,8 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde
return;
}
+ $startTime = time();
+
// Ensure we start with an empty queue here, in case this command is called multiple times.
$this->redisRenderingQueue->flush($contentReleaseIdentifier);
$this->redisRenderingErrorManager->flush($contentReleaseIdentifier);
@@ -172,7 +174,7 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde
if (count($nodesScheduledForRendering) === 0) {
// we have NO nodes scheduled for rendering anymore, so that means we FINISHED successfully.
- $contentReleaseLogger->info('Everything rendered completely. Finishing RenderOrchestrator');
+ $contentReleaseLogger->info(sprintf('Everything rendered completely in %d seconds. Finishing RenderOrchestrator', time() - $startTime));
// info to all renderers that we finished, and they should terminate themselves gracefully.
$this->redisContentReleaseService->setContentReleaseMetadata($contentReleaseIdentifier, $releaseMetadata->withStatus(NodeRenderingCompletionStatus::success())->withEndTime(new \DateTimeImmutable()), RedisInstanceIdentifier::primary());
@@ -223,10 +225,11 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde
// This is to gain better visibility into all errors currently happening; and thus maybe being able to see
// patterns among the errors.
// We also do NOT start a new incremental release, as this would lead very likely to the same errors.
- $amountOfRenderingErrors = count($this->redisRenderingErrorManager->getRenderingErrors($contentReleaseIdentifier));
+ $renderingErrors = $this->redisRenderingErrorManager->getRenderingErrors($contentReleaseIdentifier);
+ $amountOfRenderingErrors = count($renderingErrors);
if ($amountOfRenderingErrors > 0) {
$this->redisContentReleaseService->setContentReleaseMetadata($contentReleaseIdentifier, $releaseMetadata->withStatus(NodeRenderingCompletionStatus::failed()), RedisInstanceIdentifier::primary());
- $contentReleaseLogger->error('In this iteration, there happened ' . $amountOfRenderingErrors . ' rendering errors. EXITING now, as there is no chance of completing the content release successfully.');
+ $contentReleaseLogger->error('In this iteration, there happened ' . $amountOfRenderingErrors . ' rendering errors. EXITING now, as there is no chance of completing the content release successfully.', [$renderingErrors]);
yield ExitEvent::createWithStatusCode(self::EXIT_ERRORSTATUSCODE_RENDERING_ERRORS);
return;
}
diff --git a/Classes/NodeRendering/NodeRenderer.php b/Classes/NodeRendering/NodeRenderer.php
index 9baffae..b58a13d 100644
--- a/Classes/NodeRendering/NodeRenderer.php
+++ b/Classes/NodeRendering/NodeRenderer.php
@@ -51,6 +51,8 @@
*/
class NodeRenderer
{
+ protected const RESTART_AFTER_RENDER_COUNT = 20;
+ protected const CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT = 5;
/**
* @Flow\Inject
@@ -158,12 +160,12 @@ public function render(ContentReleaseIdentifier $contentReleaseIdentifier, Conte
$i++;
- if ($i % 5 === 0) {
+ if (static::CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT > 0 && $i % static::CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT === 0) {
$this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($contentReleaseIdentifier);
}
- if ($i % 20 === 0) {
- $contentReleaseLogger->info('Restarting after 20 renders.');
+ if ($i % static::RESTART_AFTER_RENDER_COUNT === 0) {
+ $contentReleaseLogger->info(sprintf('Restarting after %d renders.', static::RESTART_AFTER_RENDER_COUNT));
yield ExitEvent::createWithStatusCode(193);
return;
}
@@ -203,7 +205,9 @@ protected function renderDocumentNodeVariant(EnumeratedNode $enumeratedNode, Con
$contentReleaseLogger->debug('Rendering document node variant', [
'node' => $node->getContextPath(),
'nodeIdentifier' => $node->getIdentifier(),
- 'arguments' => $enumeratedNode->getArguments()
+ 'workspaceName' => $enumeratedNode->getWorkspaceNameFromContextPath(),
+ 'dimensions' => $enumeratedNode->getDimensionsFromContextPath(),
+ 'arguments' => $enumeratedNode->getArguments(),
]);
$this->documentRenderer->renderDocumentNodeVariant($node, $enumeratedNode->getArguments(), $contentReleaseLogger);
@@ -255,7 +259,7 @@ protected function fetchRenderableNode(EnumeratedNode $enumeratedNode): ?NodeInt
$site = $this->siteRepository->findOneByNodeName($enumeratedNode->getSiteNodeNameFromContextPath());
$context = $this->contextFactory->create([
- 'workspaceName' => 'live',
+ 'workspaceName' => $enumeratedNode->getWorkspaceNameFromContextPath(),
'currentSite' => $site,
'currentDomain' => $site->getFirstActiveDomain(),
'dimensions' => $enumeratedNode->getDimensionsFromContextPath()
diff --git a/Classes/NodeRendering/Render/RenderExceptionExtractor.php b/Classes/NodeRendering/Render/RenderExceptionExtractor.php
index e4ecc19..181d820 100644
--- a/Classes/NodeRendering/Render/RenderExceptionExtractor.php
+++ b/Classes/NodeRendering/Render/RenderExceptionExtractor.php
@@ -47,7 +47,7 @@ public static function extractRenderingException($content)
preg_match(self::HTML_MESSAGE_HANDLER_PATTERN, $content, $matches) ||
preg_match(self::XML_COMMENT_HANDLER_PATTERN, $content, $matches)
) {
- return new ExtractedExceptionDto($matches['message'], $matches['stackTrace'], $matches['referenceCode']);
+ return new ExtractedExceptionDto($matches['message'], $matches['stackTrace'], $matches['referenceCode'] ?? '');
}
return null;
}
diff --git a/Classes/PrepareContentRelease/Dto/ContentReleaseMetadata.php b/Classes/PrepareContentRelease/Dto/ContentReleaseMetadata.php
index 195fab7..94ff76f 100644
--- a/Classes/PrepareContentRelease/Dto/ContentReleaseMetadata.php
+++ b/Classes/PrepareContentRelease/Dto/ContentReleaseMetadata.php
@@ -60,10 +60,9 @@ private function __construct(
$this->workspaceName = $workspaceName;
}
-
- public static function create(PrunnerJobId $prunnerJobId, \DateTimeInterface $startTime, string $workspace = 'live'): self
+ public static function create(PrunnerJobId $prunnerJobId, \DateTimeInterface $startTime, string $workspaceName = 'live'): self
{
- return new self($prunnerJobId, $startTime, null, null, NodeRenderingCompletionStatus::scheduled(), [], $workspace);
+ return new self($prunnerJobId, $startTime, null, null, NodeRenderingCompletionStatus::scheduled(), [], $workspaceName);
}
public static function fromJsonString($metadataEncoded, ContentReleaseIdentifier $contentReleaseIdentifier): self
@@ -85,7 +84,7 @@ public static function fromJsonString($metadataEncoded, ContentReleaseIdentifier
isset($tmp['manualTransferJobIds']) ? array_map(function (string $item) {
return PrunnerJobId::fromString($item);
}, json_decode($tmp['manualTransferJobIds'])) : [],
- $tmp['workspace'] ?? 'live'
+ $tmp['workspaceName'] ?? 'live'
);
}
@@ -105,24 +104,24 @@ public function jsonSerialize(): array
public function withEndTime(\DateTimeInterface $endTime): self
{
- return new self($this->prunnerJobId, $this->startTime, $endTime, $this->switchTime, $this->status, $this->manualTransferJobIds);
+ return new self($this->prunnerJobId, $this->startTime, $endTime, $this->switchTime, $this->status, $this->manualTransferJobIds, $this->workspaceName);
}
public function withSwitchTime(\DateTimeInterface $switchTime): self
{
- return new self($this->prunnerJobId, $this->startTime, $this->endTime, $switchTime, $this->status, $this->manualTransferJobIds);
+ return new self($this->prunnerJobId, $this->startTime, $this->endTime, $switchTime, $this->status, $this->manualTransferJobIds, $this->workspaceName);
}
public function withStatus(NodeRenderingCompletionStatus $status): self
{
- return new self($this->prunnerJobId, $this->startTime, $this->endTime, $this->switchTime, $status, $this->manualTransferJobIds);
+ return new self($this->prunnerJobId, $this->startTime, $this->endTime, $this->switchTime, $status, $this->manualTransferJobIds, $this->workspaceName);
}
public function withAdditionalManualTransferJobId(PrunnerJobId $prunnerJobId): self
{
- $manualTransferIdArray = self::getManualTransferJobIds();
+ $manualTransferIdArray = $this->getManualTransferJobIds();
$manualTransferIdArray[] = $prunnerJobId;
- return new self($this->prunnerJobId, $this->startTime, $this->endTime, $this->switchTime, $this->status, $manualTransferIdArray);
+ return new self($this->prunnerJobId, $this->startTime, $this->endTime, $this->switchTime, $this->status, $manualTransferIdArray, $this->workspaceName);
}
public function getPrunnerJobId(): PrunnerJobId
diff --git a/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php b/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php
index ff78cdd..282757b 100644
--- a/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php
+++ b/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php
@@ -32,15 +32,17 @@ class RedisContentReleaseService
*/
protected $redisKeyService;
- /**
- * @Flow\Inject
- * @var RedisContentReleaseService
- */
- protected $redisContentReleaseService;
-
public function createContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, PrunnerJobId $prunnerJobId, ContentReleaseLogger $contentReleaseLogger, string $workspaceName = 'live'): void
{
$redis = $this->redisClientManager->getPrimaryRedis();
+
+ // Check there is no existing release with the same identifier
+ $existingRelease = $redis->get($this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'meta:info'));
+ if ($existingRelease) {
+ $contentReleaseLogger->error(sprintf('Content Release "%s" already exists', $contentReleaseIdentifier->getIdentifier()));
+ throw new \RuntimeException(sprintf('Content Release "%s" already exists, cannot create a release with the same identifier', $contentReleaseIdentifier->getIdentifier()), 1689750292);
+ }
+
$metadata = ContentReleaseMetadata::create($prunnerJobId, new \DateTimeImmutable(), $workspaceName);
$redis->multi();
try {
@@ -63,8 +65,8 @@ public function setContentReleaseMetadata(ContentReleaseIdentifier $contentRelea
public function registerManualTransferJob(ContentReleaseIdentifier $contentReleaseIdentifier, PrunnerJobId $prunnerJobId, ContentReleaseLogger $contentReleaseLogger): void
{
- $releaseMetadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier);
- $this->redisContentReleaseService->setContentReleaseMetadata($contentReleaseIdentifier, $releaseMetadata->withAdditionalManualTransferJobId($prunnerJobId), RedisInstanceIdentifier::primary());
+ $releaseMetadata = $this->fetchMetadataForContentRelease($contentReleaseIdentifier);
+ $this->setContentReleaseMetadata($contentReleaseIdentifier, $releaseMetadata->withAdditionalManualTransferJobId($prunnerJobId), RedisInstanceIdentifier::primary());
$contentReleaseLogger->info(sprintf('Register new pipeline for release %s', $contentReleaseIdentifier->getIdentifier()));
}
@@ -85,11 +87,14 @@ public function fetchAllReleaseIds(RedisInstanceIdentifier $redisInstanceIdentif
return $result;
}
- public function fetchMetadataForContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, ?RedisInstanceIdentifier $redisInstanceIdentifier = null): ContentReleaseMetadata
+ public function fetchMetadataForContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, ?RedisInstanceIdentifier $redisInstanceIdentifier = null): ?ContentReleaseMetadata
{
$redisInstanceIdentifier = $redisInstanceIdentifier ?: RedisInstanceIdentifier::primary();
$redis = $this->redisClientManager->getRedis($redisInstanceIdentifier);
$metadataEncoded = $redis->get($this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'meta:info'));
+ if (!$metadataEncoded) {
+ return null;
+ }
return ContentReleaseMetadata::fromJsonString($metadataEncoded, $contentReleaseIdentifier);
}
@@ -104,7 +109,7 @@ public function fetchMetadataForContentReleases(RedisInstanceIdentifier $redisIn
}
$res = $redisPipeline->exec();
foreach ($batchedReleaseIdentifiers as $i => $releaseIdentifier) {
- $result[$releaseIdentifier->jsonSerialize()] = ContentReleaseMetadata::fromJsonString($res[$i], $releaseIdentifier);
+ $result[(string)$releaseIdentifier] = $res[$i] ? ContentReleaseMetadata::fromJsonString($res[$i], $releaseIdentifier) : null;
}
}
return ContentReleaseBatchResult::createFromArray($result);
diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml
index 09f7d35..1a57722 100644
--- a/Configuration/Settings.yaml
+++ b/Configuration/Settings.yaml
@@ -1,5 +1,10 @@
Flowpack:
DecoupledContentStore:
+
+ # auto-start an incremental release whenever a workspace publish happens
+ # if you want to disable auto-start of incremental releases, set this to false
+ startIncrementalReleaseOnWorkspacePublish: true
+
redisContentStores:
# the "Primary" content store is the one Neos writes to during building the content release.
primary:
@@ -41,6 +46,9 @@ Flowpack:
# Add full HTTP message to the output of the content store
addHttpMessage: false
+ # Recurse to child nodes of hidden nodes
+ recurseHiddenContent: false
+
extensions:
@@ -148,7 +156,7 @@ Neos:
label: 'Content Store'
controller: 'Flowpack\DecoupledContentStore\Controller\BackendController'
description: 'Decoupled Content Publishing'
- icon: 'fas fa-exchange'
+ icon: 'fas fa-exchange-alt'
mainStylesheet: 'Lite'
Flow:
@@ -172,3 +180,7 @@ Neos:
routes:
'Flowpack.DecoupledContentStore':
position: 'before Neos.Neos'
+
+ Fusion:
+ defaultContext:
+ Flowpack.DecoupledContentStore: Flowpack\DecoupledContentStore\Eel\ModuleHelper
diff --git a/README.md b/README.md
index 36360ed..ab5e986 100644
--- a/README.md
+++ b/README.md
@@ -239,6 +239,20 @@ As a big improvement for stability (compared to v1), the rendering pipeline does
it is a full or an incremental render. To trigger a full render, the content cache is flushed before
the rendering is started.
+### auto-starting of Incremental Releases on workspace changes
+
+By default, the Package is configured to automatically start an incremental release whenever any workspace is published.
+This behavior can be disabled via Flow configuration in your Settings.yaml
+
+```yaml
+Flowpack:
+ DecoupledContentStore:
+ # if you want to disable auto-start of incremental releases, set this to false
+ # (default value: true)
+ startIncrementalReleaseOnWorkspacePublish: false
+ # ...
+```
+
### What happens if edits happen during a rendering?
If a change by an editor happens during a rendering, the content cache is flushed (by tag) as a result of
diff --git a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion
index a3735ee..a06310f 100644
--- a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion
+++ b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion
@@ -8,41 +8,52 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component
// - redisContentStores: array of all configured content store identifiers
// - isPrimary: bool
+ stdOutLines = ${Flowpack.DecoupledContentStore.formatStdOutput(jobLogs.stdout)}
+
renderer = afx`
-
-
-
- Content Release {detailsData.contentReleaseIdentifier.identifier}
-
- {" "}
- {contentStore}
-
-
-
-
-
- Log Output for {detailTaskName}
-
- {jobLogs.stderr}
-
-
- {jobLogs.stdout}
-
+ />
+
+
+
+ Content Release {contentReleaseIdentifier.identifier}
+
+
+ {" "}
+ {contentStore}
+
+
+
+
+
+
+
Error output for {detailTaskName}
+
+ {String.htmlSpecialChars(jobLogs.stderr)}
+
+
Log output for {detailTaskName}
+ {props.stdOutLines}
+
+
+ No data exists for this release in Redis.
+
+
`
}
@@ -400,7 +411,11 @@ prototype(Flowpack.DecoupledContentStore:DetailsFooter) < prototype(Neos.Fusion:
}
renderer = afx`
-