From b77c2b709af24fd82bb8db2c4a118c4e7a4838b6 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Thu, 19 Jan 2023 14:51:48 +0100 Subject: [PATCH 01/34] BUGFIX: Show module icon in module list --- Configuration/Settings.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 09f7d35..baa071d 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -148,7 +148,7 @@ Neos: label: 'Content Store' controller: 'Flowpack\DecoupledContentStore\Controller\BackendController' description: 'Decoupled Content Publishing' - icon: 'fas fa-exchange' + icon: 'fas fa-exchange-alt' mainStylesheet: 'Lite' Flow: From fb3ee84ea9905e9fd1f3772cdf6b88da26064230 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Thu, 19 Jan 2023 14:53:13 +0100 Subject: [PATCH 02/34] BUGFIX: Prevent tailwind from breaking neos backend styles With this change the tailwind styles are only applied to the app container of the decoupled content store --- .../Integration/Backend.Details.fusion | 68 +-- .../Integration/Backend.Index.fusion | 104 ++-- Resources/Private/Css/index.css | 1 - Resources/Private/Js/foo.css | 3 - Resources/Private/Js/index.js | 4 +- Resources/Private/Js/styles.css | 25 + Resources/Public/BackendCompiled/out.css | 444 ++++-------------- Resources/Public/BackendCompiled/out.js | 2 +- tailwind.config.js | 1 + 9 files changed, 208 insertions(+), 444 deletions(-) delete mode 100644 Resources/Private/Css/index.css delete mode 100644 Resources/Private/Js/foo.css create mode 100644 Resources/Private/Js/styles.css diff --git a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion index a3735ee..9f1fe88 100644 --- a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion +++ b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion @@ -9,40 +9,42 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component // - isPrimary: bool renderer = afx` -
- -
- -

- Content Release {detailsData.contentReleaseIdentifier.identifier} -

- {" "} - {contentStore} - - +
+
+ +
+ +

+ Content Release {detailsData.contentReleaseIdentifier.identifier} +

+ {" "} + {contentStore} - - -

Log Output for {detailTaskName}

- 
-            {jobLogs.stderr}
-
- 
-            {jobLogs.stdout}
-
+ + + + +

Log Output for {detailTaskName}

+ 
+                {jobLogs.stderr}
+
+ 
+                {jobLogs.stdout}
+
+
` } @@ -400,7 +402,7 @@ prototype(Flowpack.DecoupledContentStore:DetailsFooter) < prototype(Neos.Fusion: } renderer = afx` -
+
Reload diff --git a/Resources/Private/BackendFusion/Integration/Backend.Index.fusion b/Resources/Private/BackendFusion/Integration/Backend.Index.fusion index e6b45d6..8cc9d32 100644 --- a/Resources/Private/BackendFusion/Integration/Backend.Index.fusion +++ b/Resources/Private/BackendFusion/Integration/Backend.Index.fusion @@ -77,61 +77,63 @@ Flowpack.DecoupledContentStore.BackendController.index = Neos.Fusion:Component { } } renderer = afx` - -
- -
-
- - Content Store Releases -
- - - - - - - - - - - - - - - {props._renderedTableBody} - -
IdentifierPage CountRender ProgressIterationsStartStopSwitchRelease Size
+
+ +
+ +
+
+ + Content Store Releases +
+ + + + + + + + + + + + + + + {props._renderedTableBody} + +
IdentifierPage CountRender ProgressIterationsStartStopSwitchRelease Size
-
- Store size: - {storeSize} -
- -
- - - - -
+
+ Store size: + {storeSize} +
-
- +
+ + + + +
+ +
+ +
` } } @@ -143,7 +145,7 @@ prototype(Flowpack.DecoupledContentStore:ListFooter) < prototype(Neos.Fusion:Joi ` switchContentStore = afx` - + Active Content Store:   :not([hidden]) ~ :not([hidden]) { +#app .divide-y > :not([hidden]) ~ :not([hidden]) { --tw-divide-y-reverse: 0; border-top-width: calc(1px * calc(1 - var(--tw-divide-y-reverse))); border-bottom-width: calc(1px * var(--tw-divide-y-reverse)); } -.divide-gray-300 > :not([hidden]) ~ :not([hidden]) { +#app .divide-gray-300 > :not([hidden]) ~ :not([hidden]) { --tw-divide-opacity: 1; border-color: rgb(209 213 219 / var(--tw-divide-opacity)); } -.overflow-hidden { +#app .overflow-hidden { overflow: hidden; } -.truncate { +#app .truncate { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } -.rounded-md { +#app .rounded-md { border-radius: 0.375rem; } -.rounded-full { +#app .rounded-full { border-radius: 9999px; } -.rounded-lg { +#app .rounded-lg { border-radius: 0.5rem; } -.border { +#app .border { border-width: 1px; } -.border-2 { +#app .border-2 { border-width: 2px; } -.border-gray-300 { +#app .border-gray-300 { --tw-border-opacity: 1; border-color: rgb(209 213 219 / var(--tw-border-opacity)); } -.border-green-400 { +#app .border-green-400 { --tw-border-opacity: 1; border-color: rgb(74 222 128 / var(--tw-border-opacity)); } -.border-yellow-400 { +#app .border-yellow-400 { --tw-border-opacity: 1; border-color: rgb(250 204 21 / var(--tw-border-opacity)); } -.border-red-500 { +#app .border-red-500 { --tw-border-opacity: 1; border-color: rgb(239 68 68 / var(--tw-border-opacity)); } -.border-gray-500 { +#app .border-gray-500 { --tw-border-opacity: 1; border-color: rgb(107 114 128 / var(--tw-border-opacity)); } -.bg-gray-700 { +#app .bg-gray-700 { --tw-bg-opacity: 1; background-color: rgb(55 65 81 / var(--tw-bg-opacity)); } -.bg-white { +#app .bg-white { --tw-bg-opacity: 1; background-color: rgb(255 255 255 / var(--tw-bg-opacity)); } -.bg-green-400 { +#app .bg-green-400 { --tw-bg-opacity: 1; background-color: rgb(74 222 128 / var(--tw-bg-opacity)); } -.bg-yellow-400 { +#app .bg-yellow-400 { --tw-bg-opacity: 1; background-color: rgb(250 204 21 / var(--tw-bg-opacity)); } -.bg-red-500 { +#app .bg-red-500 { --tw-bg-opacity: 1; background-color: rgb(239 68 68 / var(--tw-bg-opacity)); } -.bg-gray-300 { +#app .bg-gray-300 { --tw-bg-opacity: 1; background-color: rgb(209 213 219 / var(--tw-bg-opacity)); } -.bg-blue-300 { +#app .bg-blue-300 { --tw-bg-opacity: 1; background-color: rgb(147 197 253 / var(--tw-bg-opacity)); } -.py-5 { +#app .py-5 { padding-top: 1.25rem; padding-bottom: 1.25rem; } -.py-2 { +#app .py-2 { padding-top: 0.5rem; padding-bottom: 0.5rem; } -.px-6 { +#app .px-6 { padding-left: 1.5rem; padding-right: 1.5rem; } -.py-4 { +#app .py-4 { padding-top: 1rem; padding-bottom: 1rem; } -.px-2\.5 { +#app .px-2\.5 { padding-left: 0.625rem; padding-right: 0.625rem; } -.py-0\.5 { +#app .py-0\.5 { padding-top: 0.125rem; padding-bottom: 0.125rem; } -.px-2 { +#app .px-2 { padding-left: 0.5rem; padding-right: 0.5rem; } -.py-0 { +#app .py-0 { padding-top: 0px; padding-bottom: 0px; } -.py-3 { +#app .py-3 { padding-top: 0.75rem; padding-bottom: 0.75rem; } -.pl-5 { +#app .pl-5 { padding-left: 1.25rem; } -.pr-10 { +#app .pr-10 { padding-right: 2.5rem; } -.pt-2 { +#app .pt-2 { padding-top: 0.5rem; } -.pr-4 { - padding-right: 1rem; -} -.pl-16 { - padding-left: 4rem; -} -.align-middle { - vertical-align: middle; -} -.text-3xl { +#app .text-3xl { font-size: 1.875rem; line-height: 2.25rem; } -.text-sm { +#app .text-sm { font-size: 0.875rem; line-height: 1.25rem; } -.text-xs { +#app .text-xs { font-size: 0.75rem; line-height: 1rem; } -.font-medium { +#app .font-medium { font-weight: 500; } -.font-semibold { +#app .font-semibold { font-weight: 600; } -.leading-6 { +#app .leading-6 { line-height: 1.5rem; } -.text-white { +#app .text-white { --tw-text-opacity: 1; color: rgb(255 255 255 / var(--tw-text-opacity)); } -.text-gray-300 { +#app .text-gray-300 { --tw-text-opacity: 1; color: rgb(209 213 219 / var(--tw-text-opacity)); } -.text-gray-500 { +#app .text-gray-500 { --tw-text-opacity: 1; color: rgb(107 114 128 / var(--tw-text-opacity)); } -.text-gray-900 { +#app .text-gray-900 { --tw-text-opacity: 1; color: rgb(17 24 39 / var(--tw-text-opacity)); } -.shadow { +#app .shadow { --tw-shadow: 0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1); --tw-shadow-colored: 0 1px 3px 0 var(--tw-shadow-color), 0 1px 2px -1px var(--tw-shadow-color); box-shadow: @@ -551,17 +281,25 @@ video { var(--tw-ring-shadow, 0 0 #0000), var(--tw-shadow); } +.switchContentStore { + vertical-align: middle; + display: inline-flex; + font-size: 0.9rem; + gap: 0.5rem; + margin-left: 1rem; + align-items: center; +} @media (min-width: 768px) { - .md\:block { + #app .md\:block { display: block; } - .md\:flex { + #app .md\:flex { display: flex; } - .md\:flex-1 { + #app .md\:flex-1 { flex: 1 1 0%; } - .md\:divide-y-0 > :not([hidden]) ~ :not([hidden]) { + #app .md\:divide-y-0 > :not([hidden]) ~ :not([hidden]) { --tw-divide-y-reverse: 0; border-top-width: calc(0px * calc(1 - var(--tw-divide-y-reverse))); border-bottom-width: calc(0px * var(--tw-divide-y-reverse)); diff --git a/Resources/Public/BackendCompiled/out.js b/Resources/Public/BackendCompiled/out.js index 711d977..2716914 100644 --- a/Resources/Public/BackendCompiled/out.js +++ b/Resources/Public/BackendCompiled/out.js @@ -1,4 +1,4 @@ (() => { // Resources/Private/Js/index.js - console.log("HI"); + console.log("Initialized DecoupledContentStore"); })(); diff --git a/tailwind.config.js b/tailwind.config.js index 84f5076..fb8504d 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -4,4 +4,5 @@ module.exports = { extend: {}, }, plugins: [], + important: '#app' } From 5e53d53144920ffb6e105eeb2e6e49a6b81ef09d Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Fri, 20 Jan 2023 10:22:26 +0100 Subject: [PATCH 03/34] BUGFIX: Prevent metadata from loosing its workspacename --- .../Dto/ContentReleaseMetadata.php | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Classes/PrepareContentRelease/Dto/ContentReleaseMetadata.php b/Classes/PrepareContentRelease/Dto/ContentReleaseMetadata.php index 195fab7..94ff76f 100644 --- a/Classes/PrepareContentRelease/Dto/ContentReleaseMetadata.php +++ b/Classes/PrepareContentRelease/Dto/ContentReleaseMetadata.php @@ -60,10 +60,9 @@ private function __construct( $this->workspaceName = $workspaceName; } - - public static function create(PrunnerJobId $prunnerJobId, \DateTimeInterface $startTime, string $workspace = 'live'): self + public static function create(PrunnerJobId $prunnerJobId, \DateTimeInterface $startTime, string $workspaceName = 'live'): self { - return new self($prunnerJobId, $startTime, null, null, NodeRenderingCompletionStatus::scheduled(), [], $workspace); + return new self($prunnerJobId, $startTime, null, null, NodeRenderingCompletionStatus::scheduled(), [], $workspaceName); } public static function fromJsonString($metadataEncoded, ContentReleaseIdentifier $contentReleaseIdentifier): self @@ -85,7 +84,7 @@ public static function fromJsonString($metadataEncoded, ContentReleaseIdentifier isset($tmp['manualTransferJobIds']) ? array_map(function (string $item) { return PrunnerJobId::fromString($item); }, json_decode($tmp['manualTransferJobIds'])) : [], - $tmp['workspace'] ?? 'live' + $tmp['workspaceName'] ?? 'live' ); } @@ -105,24 +104,24 @@ public function jsonSerialize(): array public function withEndTime(\DateTimeInterface $endTime): self { - return new self($this->prunnerJobId, $this->startTime, $endTime, $this->switchTime, $this->status, $this->manualTransferJobIds); + return new self($this->prunnerJobId, $this->startTime, $endTime, $this->switchTime, $this->status, $this->manualTransferJobIds, $this->workspaceName); } public function withSwitchTime(\DateTimeInterface $switchTime): self { - return new self($this->prunnerJobId, $this->startTime, $this->endTime, $switchTime, $this->status, $this->manualTransferJobIds); + return new self($this->prunnerJobId, $this->startTime, $this->endTime, $switchTime, $this->status, $this->manualTransferJobIds, $this->workspaceName); } public function withStatus(NodeRenderingCompletionStatus $status): self { - return new self($this->prunnerJobId, $this->startTime, $this->endTime, $this->switchTime, $status, $this->manualTransferJobIds); + return new self($this->prunnerJobId, $this->startTime, $this->endTime, $this->switchTime, $status, $this->manualTransferJobIds, $this->workspaceName); } public function withAdditionalManualTransferJobId(PrunnerJobId $prunnerJobId): self { - $manualTransferIdArray = self::getManualTransferJobIds(); + $manualTransferIdArray = $this->getManualTransferJobIds(); $manualTransferIdArray[] = $prunnerJobId; - return new self($this->prunnerJobId, $this->startTime, $this->endTime, $this->switchTime, $this->status, $manualTransferIdArray); + return new self($this->prunnerJobId, $this->startTime, $this->endTime, $this->switchTime, $this->status, $manualTransferIdArray, $this->workspaceName); } public function getPrunnerJobId(): PrunnerJobId From 6b25d92a72746afb916c5b435fb5fdd0d815495c Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Fri, 20 Jan 2023 10:24:56 +0100 Subject: [PATCH 04/34] FEATURE: Set build lock by workspace to allow parallel releases --- Classes/Core/ConcurrentBuildLockService.php | 29 ++++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/Classes/Core/ConcurrentBuildLockService.php b/Classes/Core/ConcurrentBuildLockService.php index 6ddcdc8..0d91187 100644 --- a/Classes/Core/ConcurrentBuildLockService.php +++ b/Classes/Core/ConcurrentBuildLockService.php @@ -4,10 +4,11 @@ use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier; use Flowpack\DecoupledContentStore\Core\Infrastructure\RedisClientManager; +use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService; use Neos\Flow\Annotations as Flow; /** - * We usually rely on prunner to ensure that only one build is running at any given time. + * We usually rely on prunner to ensure that only one build per workspace is running at any given time. * * However, when running in a cloud environment with no shared storage, the prunner data folder is not shared between * instances. In this case, during a deployment, two containers run concurrently, with two separate prunner instances @@ -27,6 +28,7 @@ */ class ConcurrentBuildLockService { + private const CONTENT_STORE_CONCURRENT_BUILD_LOCK = 'contentStore:concurrentBuildLocks'; /** * @Flow\Inject @@ -34,28 +36,35 @@ class ConcurrentBuildLockService */ protected $redisClientManager; - public function ensureAllOtherInProgressContentReleasesWillBeTerminated(ContentReleaseIdentifier $contentReleaseIdentifier) + /** + * @Flow\Inject + * @var RedisContentReleaseService + */ + protected $redisContentReleaseService; + + public function ensureAllOtherInProgressContentReleasesWillBeTerminated(ContentReleaseIdentifier $contentReleaseIdentifier): void { - $this->redisClientManager->getPrimaryRedis()->set('contentStore:concurrentBuildLock', $contentReleaseIdentifier->getIdentifier()); + $metadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier); + $this->redisClientManager->getPrimaryRedis()->hSet(self::CONTENT_STORE_CONCURRENT_BUILD_LOCK, $metadata->getWorkspaceName(), (string)$contentReleaseIdentifier); } - public function assertNoOtherContentReleaseWasStarted(ContentReleaseIdentifier $contentReleaseIdentifier) + public function assertNoOtherContentReleaseWasStarted(ContentReleaseIdentifier $contentReleaseIdentifier): void { - $concurrentBuildLockString = $this->redisClientManager->getPrimaryRedis()->get('contentStore:concurrentBuildLock'); + $metadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier); + $concurrentBuildLockStrings = $this->redisClientManager->getPrimaryRedis()->hGetAll(self::CONTENT_STORE_CONCURRENT_BUILD_LOCK); + $concurrentBuildLockStringForWorkspace = $concurrentBuildLockStrings[$metadata->getWorkspaceName()] ?? null; - if (empty($concurrentBuildLockString)) { + if (!$concurrentBuildLockStringForWorkspace) { echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock does not exist.' . "\n\n"; echo "This should never happen for correctly configured jobs (that run after prepare_finished).\n\n"; exit(1); } - - $concurrentBuildLock = ContentReleaseIdentifier::fromString($concurrentBuildLockString); + $concurrentBuildLock = ContentReleaseIdentifier::fromString($concurrentBuildLockStringForWorkspace); if (!$contentReleaseIdentifier->equals($concurrentBuildLock)) { // the concurrent build lock is different (i.e. newer) than our currently-running content release. // Thus, we abort the in-progress content release as quickly as we can - by DYING. - - echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock contains ' . $concurrentBuildLock->getIdentifier() . "\n\n"; + echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock for workspace "' . $metadata->getWorkspaceName() . '" contains ' . $concurrentBuildLock->getIdentifier() . "\n\n"; echo "This is no error during deployment, but should never happen outside a deployment.\n\n It can only happen if two prunner instances run concurrently.\n\n"; exit(1); } From 8b5f110cf457c01b507f7629bae0db4354f2eb1e Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Fri, 20 Jan 2023 13:31:52 +0100 Subject: [PATCH 05/34] TASK: Show rendering error details in render_orchestrator output This stops having to go through all individual worker outputs --- Classes/NodeRendering/NodeRenderOrchestrator.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Classes/NodeRendering/NodeRenderOrchestrator.php b/Classes/NodeRendering/NodeRenderOrchestrator.php index d0ada05..a1e72e3 100644 --- a/Classes/NodeRendering/NodeRenderOrchestrator.php +++ b/Classes/NodeRendering/NodeRenderOrchestrator.php @@ -223,10 +223,11 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde // This is to gain better visibility into all errors currently happening; and thus maybe being able to see // patterns among the errors. // We also do NOT start a new incremental release, as this would lead very likely to the same errors. - $amountOfRenderingErrors = count($this->redisRenderingErrorManager->getRenderingErrors($contentReleaseIdentifier)); + $renderingErrors = $this->redisRenderingErrorManager->getRenderingErrors($contentReleaseIdentifier); + $amountOfRenderingErrors = count($renderingErrors); if ($amountOfRenderingErrors > 0) { $this->redisContentReleaseService->setContentReleaseMetadata($contentReleaseIdentifier, $releaseMetadata->withStatus(NodeRenderingCompletionStatus::failed()), RedisInstanceIdentifier::primary()); - $contentReleaseLogger->error('In this iteration, there happened ' . $amountOfRenderingErrors . ' rendering errors. EXITING now, as there is no chance of completing the content release successfully.'); + $contentReleaseLogger->error('In this iteration, there happened ' . $amountOfRenderingErrors . ' rendering errors. EXITING now, as there is no chance of completing the content release successfully.', [$renderingErrors]); yield ExitEvent::createWithStatusCode(self::EXIT_ERRORSTATUSCODE_RENDERING_ERRORS); return; } From d39ef93ac8f4b0a0d46fc3e901ee1ff92896cb73 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Wed, 1 Feb 2023 14:37:21 +0100 Subject: [PATCH 06/34] BUGFIX: Flush content cache when the pipeline actually starts --- .../ContentReleasePrepareCommandController.php | 18 ++++++++++++++++++ Classes/ContentReleaseManager.php | 9 ++------- pipelines_template.yml | 3 ++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Classes/Command/ContentReleasePrepareCommandController.php b/Classes/Command/ContentReleasePrepareCommandController.php index e48aa3c..3648869 100644 --- a/Classes/Command/ContentReleasePrepareCommandController.php +++ b/Classes/Command/ContentReleasePrepareCommandController.php @@ -10,6 +10,7 @@ use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier; use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger; use Neos\Flow\Cli\CommandController; +use Neos\Fusion\Core\Cache\ContentCache; /** * Commands for the PREPARE stage in the pipeline. Not meant to be called manually. @@ -28,6 +29,12 @@ class ContentReleasePrepareCommandController extends CommandController */ protected $concurrentBuildLock; + /** + * @Flow\Inject + * @var ContentCache + */ + protected $contentCache; + public function createContentReleaseCommand(string $contentReleaseIdentifier, string $prunnerJobId, string $workspaceName = 'live'): void { $contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier); @@ -52,4 +59,15 @@ public function registerManualTransferJobCommand(string $contentReleaseIdentifie $this->redisContentReleaseService->registerManualTransferJob($contentReleaseIdentifier, $prunnerJobId, $logger); } + + public function flushContentCacheIfRequiredCommand(string $contentReleaseIdentifier, bool $flushContentCache = false): void + { + $logger = ContentReleaseLogger::fromConsoleOutput($this->output, ContentReleaseIdentifier::fromString($contentReleaseIdentifier)); + if (!$flushContentCache) { + $logger->info('Not flushing content cache'); + return; + } + $logger->info('Flushing content cache'); + $this->contentCache->flush(); + } } diff --git a/Classes/ContentReleaseManager.php b/Classes/ContentReleaseManager.php index d84e558..7b23cf8 100644 --- a/Classes/ContentReleaseManager.php +++ b/Classes/ContentReleaseManager.php @@ -12,18 +12,12 @@ use Neos\Flow\Annotations as Flow; use Flowpack\Prunner\PrunnerApiService; use Flowpack\Prunner\ValueObject\PipelineName; -use Neos\Fusion\Core\Cache\ContentCache; /** * @Flow\Scope("singleton") */ class ContentReleaseManager { - /** - * @Flow\Inject - * @var ContentCache - */ - protected $contentCache; /** * @Flow\Inject @@ -61,6 +55,7 @@ public function startIncrementalContentRelease(string $currentContentReleaseId = 'currentContentReleaseId' => $currentContentReleaseId ?: self::NO_PREVIOUS_RELEASE, 'validate' => true, 'workspaceName' => $workspace ? $workspace->getName() : 'live', + 'flushContentCache' => false, ])); return $contentReleaseId; } @@ -74,12 +69,12 @@ public function startFullContentRelease(bool $validate = true, string $currentCo } $contentReleaseId = ContentReleaseIdentifier::create(); - $this->contentCache->flush(); $this->prunnerApiService->schedulePipeline(PipelineName::create('do_content_release'), array_merge($additionalVariables, [ 'contentReleaseId' => $contentReleaseId, 'currentContentReleaseId' => $currentContentReleaseId ?: self::NO_PREVIOUS_RELEASE, 'validate' => $validate, 'workspaceName' => $workspace ? $workspace->getName() : 'live', + 'flushContentCache' => true, ])); return $contentReleaseId; } diff --git a/pipelines_template.yml b/pipelines_template.yml index 854bf41..f343e6f 100644 --- a/pipelines_template.yml +++ b/pipelines_template.yml @@ -34,8 +34,9 @@ pipelines: ################################################################################ prepare_finished: script: - - ./flow contentReleasePrepare:createContentRelease {{ .contentReleaseId }} {{ .__jobID }} {{ .workspaceName }} + - ./flow contentReleasePrepare:createContentRelease {{ .contentReleaseId }} {{ .__jobID }} --workspaceName={{ .workspaceName }} - ./flow contentReleasePrepare:ensureAllOtherInProgressContentReleasesWillBeTerminated {{ .contentReleaseId }} + - ./flow contentReleasePrepare:flushContentCacheIfRequired {{ .contentReleaseId }} --flushContentCache={{ .flushContentCache }} ################################################################################ # 1) ENUMERATION From 28f7d2f4ee61511dca01a5fe2537686747a52000 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 14 Feb 2023 14:24:29 +0100 Subject: [PATCH 07/34] FEATURE: Allow recursing through hidden documents --- .../Domain/Service/NodeContextCombinator.php | 12 ++++++++++-- Classes/NodeEnumeration/NodeEnumerator.php | 7 +++++-- Configuration/Settings.yaml | 3 +++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Classes/NodeEnumeration/Domain/Service/NodeContextCombinator.php b/Classes/NodeEnumeration/Domain/Service/NodeContextCombinator.php index 5e11da4..7ae2fa6 100644 --- a/Classes/NodeEnumeration/Domain/Service/NodeContextCombinator.php +++ b/Classes/NodeEnumeration/Domain/Service/NodeContextCombinator.php @@ -33,6 +33,12 @@ class NodeContextCombinator */ protected $contextFactory; + /** + * @Flow\InjectConfiguration(path="nodeRendering.recurseHiddenContent", package="Flowpack.DecoupledContentStore") + * @var ContextFactoryInterface + */ + protected $recurseHiddenContent; + /** * Iterate over the node with the given identifier and site in contexts for all available presets (if it exists as a variant) * @@ -86,7 +92,8 @@ public function siteNodeInContexts(Site $site, string $workspaceName = 'live'): 'currentSite' => $site, 'workspaceName' => $workspaceName, 'dimensions' => [], - 'targetDimensions' => [] + 'targetDimensions' => [], + 'invisibleContentShown' => $this->recurseHiddenContent, )); $siteNode = $contentContext->getNode('/sites/' . $site->getNodeName()); @@ -101,7 +108,8 @@ public function siteNodeInContexts(Site $site, string $workspaceName = 'live'): 'currentSite' => $site, 'workspaceName' => $workspaceName, 'dimensions' => $dimensions, - 'targetDimensions' => [] + 'targetDimensions' => [], + 'invisibleContentShown' => $this->recurseHiddenContent, )); $siteNode = $contentContext->getNode('/sites/' . $site->getNodeName()); diff --git a/Classes/NodeEnumeration/NodeEnumerator.php b/Classes/NodeEnumeration/NodeEnumerator.php index ac8c157..eb213f6 100644 --- a/Classes/NodeEnumeration/NodeEnumerator.php +++ b/Classes/NodeEnumeration/NodeEnumerator.php @@ -93,8 +93,11 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL foreach ($combinator->recurseDocumentChildNodes($siteNode) as $documentNode) { $contextPath = $documentNode->getContextPath(); - if ($nodeTypeWhitelist->matches(NodeTypeName::fromString($documentNode->getNodeType()->getName()))) { - + if ($documentNode->isHidden()) { + $contentReleaseLogger->debug('Skipping node from publishing, because it is hidden', [ + 'node' => $contextPath, + ]); + } else if ($nodeTypeWhitelist->matches(NodeTypeName::fromString($documentNode->getNodeType()->getName()))) { $contentReleaseLogger->debug('Registering node for publishing', [ 'node' => $contextPath ]); diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index baa071d..d82a2fa 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -41,6 +41,9 @@ Flowpack: # Add full HTTP message to the output of the content store addHttpMessage: false + # Recurse to child nodes of hidden nodes + recurseHiddenContent: false + extensions: From efc5811fae1a355b41d8448384871b8f05563b6d Mon Sep 17 00:00:00 2001 From: Eric Kloss Date: Wed, 22 Feb 2023 11:44:49 +0100 Subject: [PATCH 08/34] add configuration to enable/disable auto-start of incremental releases --- Classes/IncrementalContentReleaseHandler.php | 12 +++++++----- Configuration/Settings.yaml | 5 +++++ README.md | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Classes/IncrementalContentReleaseHandler.php b/Classes/IncrementalContentReleaseHandler.php index 7f081da..f0b1aaa 100644 --- a/Classes/IncrementalContentReleaseHandler.php +++ b/Classes/IncrementalContentReleaseHandler.php @@ -4,9 +4,6 @@ namespace Flowpack\DecoupledContentStore; use Neos\Flow\Annotations as Flow; -use Flowpack\Prunner\PrunnerApiService; -use Flowpack\Prunner\ValueObject\PipelineName; -use Neos\Fusion\Core\Cache\ContentCache; /** * @Flow\Scope("singleton") @@ -19,6 +16,11 @@ class IncrementalContentReleaseHandler */ protected $contentReleaseManager; + /** + * @Flow\InjectConfiguration("startIncrementalReleaseOnWorkspacePublish") + */ + protected $startIncrementalReleaseOnWorkspacePublish; + protected $nodePublishedInThisRequest = false; public function nodePublished() @@ -28,8 +30,8 @@ public function nodePublished() public function startContentReleaseIfNodesWerePublishedBefore() { - if ($this->nodePublishedInThisRequest === true) { + if ($this->startIncrementalReleaseOnWorkspacePublish === true && $this->nodePublishedInThisRequest === true) { $this->contentReleaseManager->startIncrementalContentRelease(); } } -} \ No newline at end of file +} diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index d82a2fa..adb8901 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -1,5 +1,10 @@ Flowpack: DecoupledContentStore: + + # auto-start an incremental release whenever a workspace publish happens + # if you want to disable auto-start of incremental releases, set this to false + startIncrementalReleaseOnWorkspacePublish: true + redisContentStores: # the "Primary" content store is the one Neos writes to during building the content release. primary: diff --git a/README.md b/README.md index 36360ed..ab5e986 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,20 @@ As a big improvement for stability (compared to v1), the rendering pipeline does it is a full or an incremental render. To trigger a full render, the content cache is flushed before the rendering is started. +### auto-starting of Incremental Releases on workspace changes + +By default, the Package is configured to automatically start an incremental release whenever any workspace is published. +This behavior can be disabled via Flow configuration in your Settings.yaml + +```yaml +Flowpack: + DecoupledContentStore: + # if you want to disable auto-start of incremental releases, set this to false + # (default value: true) + startIncrementalReleaseOnWorkspacePublish: false + # ... +``` + ### What happens if edits happen during a rendering? If a change by an editor happens during a rendering, the content cache is flushed (by tag) as a result of From eb2bca87fc1c111e804ecd429a606a983520e614 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Wed, 19 Apr 2023 09:30:06 +0200 Subject: [PATCH 09/34] BUGFIX: Use workspace name from enumerated node during rendering --- Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php | 8 +++++++- Classes/NodeRendering/NodeRenderer.php | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php b/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php index 329cb78..50a5325 100644 --- a/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php +++ b/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php @@ -80,6 +80,12 @@ public function getDimensionsFromContextPath(): array return $nodePathAndContext['dimensions']; } + public function getWorkspaceNameFromContextPath(): string + { + $nodePathAndContext = NodePaths::explodeContextPath($this->contextPath); + return $nodePathAndContext['workspaceName']; + } + /** * @return string */ @@ -97,4 +103,4 @@ public function debugString(): string { return sprintf('Node %s (%s)', $this->nodeIdentifier, $this->contextPath); } -} \ No newline at end of file +} diff --git a/Classes/NodeRendering/NodeRenderer.php b/Classes/NodeRendering/NodeRenderer.php index 9baffae..d0467ed 100644 --- a/Classes/NodeRendering/NodeRenderer.php +++ b/Classes/NodeRendering/NodeRenderer.php @@ -255,7 +255,7 @@ protected function fetchRenderableNode(EnumeratedNode $enumeratedNode): ?NodeInt $site = $this->siteRepository->findOneByNodeName($enumeratedNode->getSiteNodeNameFromContextPath()); $context = $this->contextFactory->create([ - 'workspaceName' => 'live', + 'workspaceName' => $enumeratedNode->getWorkspaceNameFromContextPath(), 'currentSite' => $site, 'currentDomain' => $site->getFirstActiveDomain(), 'dimensions' => $enumeratedNode->getDimensionsFromContextPath() From b0aabd77942d4bcf550255a7e7d08681ed37ed1f Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Wed, 19 Apr 2023 14:26:19 +0200 Subject: [PATCH 10/34] BUGFIX: Enable storing rendered documents in non public workspaces --- Classes/Aspects/CacheUrlMappingAspect.php | 4 +--- .../NodeEnumeration/Domain/Dto/EnumeratedNode.php | 9 ++++----- Classes/NodeRendering/Dto/DocumentNodeCacheKey.php | 13 ++++++++++--- Classes/NodeRendering/NodeRenderer.php | 4 +++- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Classes/Aspects/CacheUrlMappingAspect.php b/Classes/Aspects/CacheUrlMappingAspect.php index a8e0db4..6804082 100644 --- a/Classes/Aspects/CacheUrlMappingAspect.php +++ b/Classes/Aspects/CacheUrlMappingAspect.php @@ -102,9 +102,6 @@ public function storeRootCacheIdentifier(JoinPointInterface $joinPoint) /** @var NodeInterface $node */ $node = $this->currentEvaluateContext['cacheIdentifierValues']['node']; - if (!$node->getContext()->getWorkspace()->isPublicWorkspace()) { - return; - } $url = $this->getCurrentUrl(); @@ -144,6 +141,7 @@ public function storeRootCacheIdentifier(JoinPointInterface $joinPoint) $logger->debug('Mapping URL ' . $url . ' to ' . $rootIdentifier . ' with tags ' . implode(', ', $rootTags)); $arguments = $this->getCurrentArguments($node); + // TODO: To make parallel rendering possible, we need to make sure that the cache key also includes the currently rendered workspace, as the node might originate from a base workspace (usually live). See `DocumentNodeCacheKey`. $rootKey = DocumentNodeCacheKey::fromNodeAndArguments($node, $arguments); $rootCacheValues = DocumentNodeCacheValues::create($rootIdentifier, $url); // allow other document metadata generators here diff --git a/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php b/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php index 50a5325..850da33 100644 --- a/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php +++ b/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php @@ -56,12 +56,14 @@ static public function fromJsonString(string $enumeratedNodeString): self return new self($tmp['contextPath'], $tmp['nodeIdentifier'], $tmp['arguments']); } - public function jsonSerialize() + public function jsonSerialize(): array { return [ 'contextPath' => $this->contextPath, 'nodeIdentifier' => $this->nodeIdentifier, - 'arguments' => $this->arguments + 'dimensions' => $this->getDimensionsFromContextPath(), + 'workspaceName' => $this->getWorkspaceNameFromContextPath(), + 'arguments' => $this->arguments, ]; } @@ -86,9 +88,6 @@ public function getWorkspaceNameFromContextPath(): string return $nodePathAndContext['workspaceName']; } - /** - * @return string - */ public function getNodeIdentifier(): string { return $this->nodeIdentifier; diff --git a/Classes/NodeRendering/Dto/DocumentNodeCacheKey.php b/Classes/NodeRendering/Dto/DocumentNodeCacheKey.php index 9d63dc2..2727b71 100644 --- a/Classes/NodeRendering/Dto/DocumentNodeCacheKey.php +++ b/Classes/NodeRendering/Dto/DocumentNodeCacheKey.php @@ -25,12 +25,17 @@ final class DocumentNodeCacheKey */ protected $dimensions; + /** + * @var string + */ + protected $workspaceName; + /** * @var array */ protected $arguments; - private function __construct(string $nodeIdentifier, array $dimensions, array $arguments) + private function __construct(string $nodeIdentifier, array $dimensions, string $workspaceName, array $arguments) { // we need to make this deterministic ksort($arguments); @@ -38,22 +43,24 @@ private function __construct(string $nodeIdentifier, array $dimensions, array $a $this->nodeIdentifier = $nodeIdentifier; $this->dimensions = $dimensions; + $this->workspaceName = $workspaceName; $this->arguments = $arguments; } public static function fromNodeAndArguments(NodeInterface $node, array $arguments): self { - return new self($node->getIdentifier(), $node->getContext()->getDimensions(), $arguments); + return new self($node->getIdentifier(), $node->getContext()->getDimensions(), $node->getWorkspace()->getName(), $arguments); } public static function fromEnumeratedNode(EnumeratedNode $enumeratedNode) { - return new self($enumeratedNode->getNodeIdentifier(), $enumeratedNode->getDimensionsFromContextPath(), $enumeratedNode->getArguments()); + return new self($enumeratedNode->getNodeIdentifier(), $enumeratedNode->getDimensionsFromContextPath(), $enumeratedNode->getWorkspaceNameFromContextPath(), $enumeratedNode->getArguments()); } public function redisKeyName(): string { + // TODO: Add workspace name to cache entry to allow parallel releases, but `CacheUrlMappingAspect` has to provide node in correct workspace during rendering return preg_replace('/[^a-zA-Z0-9-]/', '_', sprintf('doc--%s-%s-%s', $this->nodeIdentifier, json_encode($this->dimensions), json_encode($this->arguments))); } diff --git a/Classes/NodeRendering/NodeRenderer.php b/Classes/NodeRendering/NodeRenderer.php index d0467ed..9316e6d 100644 --- a/Classes/NodeRendering/NodeRenderer.php +++ b/Classes/NodeRendering/NodeRenderer.php @@ -203,7 +203,9 @@ protected function renderDocumentNodeVariant(EnumeratedNode $enumeratedNode, Con $contentReleaseLogger->debug('Rendering document node variant', [ 'node' => $node->getContextPath(), 'nodeIdentifier' => $node->getIdentifier(), - 'arguments' => $enumeratedNode->getArguments() + 'workspaceName' => $enumeratedNode->getWorkspaceNameFromContextPath(), + 'dimensions' => $enumeratedNode->getDimensionsFromContextPath(), + 'arguments' => $enumeratedNode->getArguments(), ]); $this->documentRenderer->renderDocumentNodeVariant($node, $enumeratedNode->getArguments(), $contentReleaseLogger); From a0de60dc09d9450e57fcfd42ce815810f90f8010 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 2 May 2023 14:52:38 +0200 Subject: [PATCH 11/34] BUGFIX: Prevent exception in overview when ContentReleaseMetadata is not available --- .../Dto/ContentReleaseOverviewRow.php | 33 ++----------------- .../Domain/Dto/ContentReleaseBatchResult.php | 6 ++-- .../RedisContentReleaseService.php | 2 +- 3 files changed, 7 insertions(+), 34 deletions(-) diff --git a/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php b/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php index d125bda..990f39d 100644 --- a/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php +++ b/Classes/BackendUi/Dto/ContentReleaseOverviewRow.php @@ -14,7 +14,7 @@ class ContentReleaseOverviewRow { private ContentReleaseIdentifier $contentReleaseIdentifier; - private ContentReleaseMetadata $metadata; + private ?ContentReleaseMetadata $metadata; private int $enumeratedDocumentNodesCount; private int $iterationsCount; private int $errorCount; @@ -23,7 +23,7 @@ class ContentReleaseOverviewRow private bool $isActive; private float $releaseSize; - public function __construct(ContentReleaseIdentifier $contentReleaseIdentifier, ContentReleaseMetadata $metadata, + public function __construct(ContentReleaseIdentifier $contentReleaseIdentifier, ?ContentReleaseMetadata $metadata, int $enumeratedDocumentNodesCount, int $iterationsCount, int $errorCount, float $progress, int $renderedUrlCount, bool $isActive, float $releaseSize) { @@ -38,73 +38,46 @@ public function __construct(ContentReleaseIdentifier $contentReleaseIdentifier, $this->releaseSize = $releaseSize; } - /** - * @return ContentReleaseMetadata - */ - public function getMetadata(): ContentReleaseMetadata + public function getMetadata(): ?ContentReleaseMetadata { return $this->metadata; } - /** - * @return ContentReleaseIdentifier - */ public function getContentReleaseIdentifier(): ContentReleaseIdentifier { return $this->contentReleaseIdentifier; } - /** - * @return int - */ public function getEnumeratedDocumentNodesCount(): int { return $this->enumeratedDocumentNodesCount; } - /** - * @return int - */ public function getIterationsCount(): int { return $this->iterationsCount; } - /** - * @return int - */ public function getErrorCount(): int { return $this->errorCount; } - /** - * @return float - */ public function getProgress(): float { return $this->progress; } - /** - * @return int - */ public function getRenderedUrlCount(): int { return $this->renderedUrlCount; } - /** - * @return bool - */ public function isActive(): bool { return $this->isActive; } - /** - * @return float - */ public function getReleaseSize(): float { return $this->releaseSize; diff --git a/Classes/Core/Domain/Dto/ContentReleaseBatchResult.php b/Classes/Core/Domain/Dto/ContentReleaseBatchResult.php index 491f6ea..673f654 100644 --- a/Classes/Core/Domain/Dto/ContentReleaseBatchResult.php +++ b/Classes/Core/Domain/Dto/ContentReleaseBatchResult.php @@ -3,6 +3,7 @@ namespace Flowpack\DecoupledContentStore\Core\Domain\Dto; use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier; +use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata; use Neos\Flow\Annotations as Flow; /** @@ -20,7 +21,6 @@ private function __construct(array $results) $this->results = $results; } - public static function createFromArray(array $in): self { return new self($in); @@ -28,7 +28,7 @@ public static function createFromArray(array $in): self public function getResultForContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier) { - return $this->results[$contentReleaseIdentifier->jsonSerialize()]; + return $this->results[(string)$contentReleaseIdentifier] ?? null; } -} \ No newline at end of file +} diff --git a/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php b/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php index ff78cdd..bff144d 100644 --- a/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php +++ b/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php @@ -104,7 +104,7 @@ public function fetchMetadataForContentReleases(RedisInstanceIdentifier $redisIn } $res = $redisPipeline->exec(); foreach ($batchedReleaseIdentifiers as $i => $releaseIdentifier) { - $result[$releaseIdentifier->jsonSerialize()] = ContentReleaseMetadata::fromJsonString($res[$i], $releaseIdentifier); + $result[(string)$releaseIdentifier] = $res[$i] ? ContentReleaseMetadata::fromJsonString($res[$i], $releaseIdentifier) : null; } } return ContentReleaseBatchResult::createFromArray($result); From 477108a47bbe961466df34bffd9d39946a9ac148 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 16 May 2023 15:39:06 +0200 Subject: [PATCH 12/34] TASK: Add rendertime to render orchestrator output --- Classes/NodeRendering/NodeRenderOrchestrator.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Classes/NodeRendering/NodeRenderOrchestrator.php b/Classes/NodeRendering/NodeRenderOrchestrator.php index a1e72e3..6bf0289 100644 --- a/Classes/NodeRendering/NodeRenderOrchestrator.php +++ b/Classes/NodeRendering/NodeRenderOrchestrator.php @@ -122,6 +122,8 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde return; } + $startTime = time(); + // Ensure we start with an empty queue here, in case this command is called multiple times. $this->redisRenderingQueue->flush($contentReleaseIdentifier); $this->redisRenderingErrorManager->flush($contentReleaseIdentifier); @@ -172,7 +174,7 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde if (count($nodesScheduledForRendering) === 0) { // we have NO nodes scheduled for rendering anymore, so that means we FINISHED successfully. - $contentReleaseLogger->info('Everything rendered completely. Finishing RenderOrchestrator'); + $contentReleaseLogger->info(sprintf('Everything rendered completely in %d seconds. Finishing RenderOrchestrator', time() - $startTime)); // info to all renderers that we finished, and they should terminate themselves gracefully. $this->redisContentReleaseService->setContentReleaseMetadata($contentReleaseIdentifier, $releaseMetadata->withStatus(NodeRenderingCompletionStatus::success())->withEndTime(new \DateTimeImmutable()), RedisInstanceIdentifier::primary()); From 4876ca0c06ca628eb08c763e26d154b6712c4faa Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Mon, 10 Jul 2023 10:50:17 +0200 Subject: [PATCH 13/34] BUGFIX: Prevent error when exception has no referenceCode --- Classes/NodeRendering/Render/RenderExceptionExtractor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/NodeRendering/Render/RenderExceptionExtractor.php b/Classes/NodeRendering/Render/RenderExceptionExtractor.php index e4ecc19..181d820 100644 --- a/Classes/NodeRendering/Render/RenderExceptionExtractor.php +++ b/Classes/NodeRendering/Render/RenderExceptionExtractor.php @@ -47,7 +47,7 @@ public static function extractRenderingException($content) preg_match(self::HTML_MESSAGE_HANDLER_PATTERN, $content, $matches) || preg_match(self::XML_COMMENT_HANDLER_PATTERN, $content, $matches) ) { - return new ExtractedExceptionDto($matches['message'], $matches['stackTrace'], $matches['referenceCode']); + return new ExtractedExceptionDto($matches['message'], $matches['stackTrace'], $matches['referenceCode'] ?? ''); } return null; } From f3f71104f51e86112b15e637a746f5e81bcb30ec Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Wed, 19 Jul 2023 11:25:40 +0200 Subject: [PATCH 14/34] BUGFIX: Prevent creating a second content release with the same id --- .../Infrastructure/RedisContentReleaseService.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php b/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php index bff144d..4a8cf25 100644 --- a/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php +++ b/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php @@ -41,6 +41,14 @@ class RedisContentReleaseService public function createContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, PrunnerJobId $prunnerJobId, ContentReleaseLogger $contentReleaseLogger, string $workspaceName = 'live'): void { $redis = $this->redisClientManager->getPrimaryRedis(); + + // Check there is no existing release with the same identifier + $existingRelease = $redis->get($this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'meta:info')); + if ($existingRelease) { + $contentReleaseLogger->error(sprintf('Content Release "%s" already exists', $contentReleaseIdentifier->getIdentifier())); + throw new \RuntimeException(sprintf('Content Release "%s" already exists, cannot create a release with the same identifier', $contentReleaseIdentifier->getIdentifier()), 1689750292); + } + $metadata = ContentReleaseMetadata::create($prunnerJobId, new \DateTimeImmutable(), $workspaceName); $redis->multi(); try { From 7ce17099466cb474c35f9d6a84b79d3ec955b443 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Mon, 9 Oct 2023 09:02:27 +0200 Subject: [PATCH 15/34] FEATURE: Store rendertime of documents in metadata --- Classes/Aspects/CacheUrlMappingAspect.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Classes/Aspects/CacheUrlMappingAspect.php b/Classes/Aspects/CacheUrlMappingAspect.php index 6804082..b6d8476 100644 --- a/Classes/Aspects/CacheUrlMappingAspect.php +++ b/Classes/Aspects/CacheUrlMappingAspect.php @@ -37,6 +37,11 @@ class CacheUrlMappingAspect */ protected $isActive = false; + /** + * @var null | int + */ + protected $renderTimestamp = null; + /** * @var array */ @@ -143,7 +148,8 @@ public function storeRootCacheIdentifier(JoinPointInterface $joinPoint) $arguments = $this->getCurrentArguments($node); // TODO: To make parallel rendering possible, we need to make sure that the cache key also includes the currently rendered workspace, as the node might originate from a base workspace (usually live). See `DocumentNodeCacheKey`. $rootKey = DocumentNodeCacheKey::fromNodeAndArguments($node, $arguments); - $rootCacheValues = DocumentNodeCacheValues::create($rootIdentifier, $url); + $rootCacheValues = DocumentNodeCacheValues::create($rootIdentifier, $url) + ->withMetadata('renderTime', (int)(microtime(true) * 1000) - $this->renderTimestamp); // allow other document metadata generators here $rootCacheValues = $this->nodeRenderingExtensionManager->runDocumentMetadataGenerators($node, $arguments, $this->controllerContext, $rootCacheValues); $this->contentCacheFrontend->set($rootKey->redisKeyName(), json_encode($rootCacheValues), $rootTags); @@ -200,6 +206,7 @@ public function beforeDocumentRendering(ContentReleaseLogger $contentReleaseLogg { $this->isActive = true; $this->contentReleaseLogger = $contentReleaseLogger; + $this->renderTimestamp = (int)(microtime(true) * 1000); } public function afterDocumentRendering(): void From 8bafbaeeec5a21e5e7bcdd37256584c89112ce41 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Thu, 9 Nov 2023 10:24:15 +0100 Subject: [PATCH 16/34] BUGFIX: Prevent error when viewing content releases with no Redis metadata --- Classes/BackendUi/BackendUiDataService.php | 7 +- Classes/Controller/BackendController.php | 1 + .../RedisContentReleaseService.php | 15 ++--- .../Integration/Backend.Details.fusion | 67 +++++++++++-------- 4 files changed, 51 insertions(+), 39 deletions(-) diff --git a/Classes/BackendUi/BackendUiDataService.php b/Classes/BackendUi/BackendUiDataService.php index ebea4dd..145daa9 100644 --- a/Classes/BackendUi/BackendUiDataService.php +++ b/Classes/BackendUi/BackendUiDataService.php @@ -114,9 +114,14 @@ private function calculateReleaseSize(RedisInstanceIdentifier $redisInstanceIden return round($size / 1000000, 2); } - public function loadDetailsData(ContentReleaseIdentifier $contentReleaseIdentifier, RedisInstanceIdentifier $redisInstanceIdentifier): ContentReleaseDetails + public function loadDetailsData(ContentReleaseIdentifier $contentReleaseIdentifier, RedisInstanceIdentifier $redisInstanceIdentifier): ?ContentReleaseDetails { $contentReleaseMetadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier, $redisInstanceIdentifier); + + if (!$contentReleaseMetadata) { + return null; + } + $contentReleaseJob = $this->prunnerApiService->loadJobDetail($contentReleaseMetadata->getPrunnerJobId()->toJobId()); $manualTransferJobs = count($contentReleaseMetadata->getManualTransferJobIds()) ? array_map(function (PrunnerJobId $item) { diff --git a/Classes/Controller/BackendController.php b/Classes/Controller/BackendController.php index 60931fc..2622f67 100644 --- a/Classes/Controller/BackendController.php +++ b/Classes/Controller/BackendController.php @@ -114,6 +114,7 @@ public function detailsAction(string $contentReleaseIdentifier, ?string $content $contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier); $contentStore = $contentStore ? RedisInstanceIdentifier::fromString($contentStore) : RedisInstanceIdentifier::primary(); + $this->view->assign('contentReleaseIdentifier', $contentReleaseIdentifier); $this->view->assign('contentStore', $contentStore->getIdentifier()); $detailsData = $this->backendUiDataService->loadDetailsData($contentReleaseIdentifier, $contentStore); diff --git a/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php b/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php index 4a8cf25..282757b 100644 --- a/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php +++ b/Classes/PrepareContentRelease/Infrastructure/RedisContentReleaseService.php @@ -32,12 +32,6 @@ class RedisContentReleaseService */ protected $redisKeyService; - /** - * @Flow\Inject - * @var RedisContentReleaseService - */ - protected $redisContentReleaseService; - public function createContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, PrunnerJobId $prunnerJobId, ContentReleaseLogger $contentReleaseLogger, string $workspaceName = 'live'): void { $redis = $this->redisClientManager->getPrimaryRedis(); @@ -71,8 +65,8 @@ public function setContentReleaseMetadata(ContentReleaseIdentifier $contentRelea public function registerManualTransferJob(ContentReleaseIdentifier $contentReleaseIdentifier, PrunnerJobId $prunnerJobId, ContentReleaseLogger $contentReleaseLogger): void { - $releaseMetadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier); - $this->redisContentReleaseService->setContentReleaseMetadata($contentReleaseIdentifier, $releaseMetadata->withAdditionalManualTransferJobId($prunnerJobId), RedisInstanceIdentifier::primary()); + $releaseMetadata = $this->fetchMetadataForContentRelease($contentReleaseIdentifier); + $this->setContentReleaseMetadata($contentReleaseIdentifier, $releaseMetadata->withAdditionalManualTransferJobId($prunnerJobId), RedisInstanceIdentifier::primary()); $contentReleaseLogger->info(sprintf('Register new pipeline for release %s', $contentReleaseIdentifier->getIdentifier())); } @@ -93,11 +87,14 @@ public function fetchAllReleaseIds(RedisInstanceIdentifier $redisInstanceIdentif return $result; } - public function fetchMetadataForContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, ?RedisInstanceIdentifier $redisInstanceIdentifier = null): ContentReleaseMetadata + public function fetchMetadataForContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, ?RedisInstanceIdentifier $redisInstanceIdentifier = null): ?ContentReleaseMetadata { $redisInstanceIdentifier = $redisInstanceIdentifier ?: RedisInstanceIdentifier::primary(); $redis = $this->redisClientManager->getRedis($redisInstanceIdentifier); $metadataEncoded = $redis->get($this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'meta:info')); + if (!$metadataEncoded) { + return null; + } return ContentReleaseMetadata::fromJsonString($metadataEncoded, $contentReleaseIdentifier); } diff --git a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion index 9f1fe88..4b9db79 100644 --- a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion +++ b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion @@ -11,39 +11,44 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component renderer = afx`
- +

- Content Release {detailsData.contentReleaseIdentifier.identifier} + Content Release {contentReleaseIdentifier.identifier}

- {" "} - {contentStore} - - - - - -

Log Output for {detailTaskName}

- 
-                {jobLogs.stderr}
-
- 
-                {jobLogs.stdout}
-
+ + {" "} + {contentStore} + + + + + +

Log Output for {detailTaskName}

+ 
+                  {jobLogs.stderr}
+
+ 
+                  {jobLogs.stdout}
+
+ +

+ No data exists for this release in Redis. +

` @@ -403,6 +408,10 @@ prototype(Flowpack.DecoupledContentStore:DetailsFooter) < prototype(Neos.Fusion: renderer = afx`
+ + Back + + Reload From a15d22eefc6b436bbff2a4055a7772bd0432890b Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 14 Nov 2023 13:39:42 +0100 Subject: [PATCH 17/34] FEATURE: Log enumeration duration for each site --- Classes/NodeEnumeration/NodeEnumerator.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Classes/NodeEnumeration/NodeEnumerator.php b/Classes/NodeEnumeration/NodeEnumerator.php index eb213f6..a319bfc 100644 --- a/Classes/NodeEnumeration/NodeEnumerator.php +++ b/Classes/NodeEnumeration/NodeEnumerator.php @@ -84,6 +84,7 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL 'domain' => $site->getFirstActiveDomain() ]); foreach ($combinator->siteNodeInContexts($site, $workspaceName) as $siteNode) { + $startTime = microtime(true); $dimensionValues = $siteNode->getContext()->getDimensions(); $contentReleaseLogger->debug('Publishing dimension combination', [ @@ -111,6 +112,7 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL } } } + $contentReleaseLogger->debug(sprintf('Finished enumerating site %s in %dms', $site->getName(), (microtime(true) - $startTime) * 1000)); }; if ($site === null) { From 8e20be9bcd4dce19927ca1a48222c4f2578f7e08 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 14 Nov 2023 13:44:47 +0100 Subject: [PATCH 18/34] BUGFIX: Speed up enumeration by fetching all documents with one query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each document needs to be checked whether it’s still connected to the page tree though. --- Classes/NodeEnumeration/NodeEnumerator.php | 28 ++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/Classes/NodeEnumeration/NodeEnumerator.php b/Classes/NodeEnumeration/NodeEnumerator.php index a319bfc..d6eb0a0 100644 --- a/Classes/NodeEnumeration/NodeEnumerator.php +++ b/Classes/NodeEnumeration/NodeEnumerator.php @@ -13,8 +13,11 @@ use Flowpack\DecoupledContentStore\NodeRendering\Dto\NodeRenderingCompletionStatus; use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService; use Flowpack\DecoupledContentStore\Utility\GeneratorUtility; +use Neos\ContentRepository\Domain\Model\NodeInterface; use Neos\ContentRepository\Domain\NodeType\NodeTypeConstraintFactory; use Neos\ContentRepository\Domain\NodeType\NodeTypeName; +use Neos\ContentRepository\Domain\Utility\NodePaths; +use Neos\Eel\FlowQuery\FlowQuery; use Neos\Flow\Annotations as Flow; use Neos\Neos\Domain\Model\Site; @@ -91,10 +94,31 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL 'dimensionValues' => $dimensionValues ]); - foreach ($combinator->recurseDocumentChildNodes($siteNode) as $documentNode) { + $documentQuery = new FlowQuery([$siteNode]); + /** @var NodeInterface[] $documents */ + $documents = $documentQuery->find('[instanceof Neos.Neos:Document]')->add($siteNode)->get(); + + foreach ($documents as $documentNode) { $contextPath = $documentNode->getContextPath(); - if ($documentNode->isHidden()) { + // Verify that the node is not orphaned + $parentNode = $documentNode->getParent(); + while ($parentNode !== $siteNode) { + if ($parentNode === null) { + $contentReleaseLogger->debug('Skipping node from publishing, because it is orphaned', [ + 'node' => $contextPath, + ]); + // Continue with the next document + continue 2; + } + $parentNode = $parentNode->getParent(); + } + + if (!$documentNode->getParent()) { + $contentReleaseLogger->debug('Skipping node from publishing, because it is orphaned', [ + 'node' => $contextPath, + ]); + } else if ($documentNode->isHidden()) { $contentReleaseLogger->debug('Skipping node from publishing, because it is hidden', [ 'node' => $contextPath, ]); From 9e988aa1d3d90140fc1f09f205dbb24ed09859de Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Wed, 15 Nov 2023 17:04:05 +0100 Subject: [PATCH 19/34] TASK: Cache redis instance for repeated calls in CacheReader --- .../RedisContentCacheReader.php | 68 +++++++++++++------ 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php b/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php index 97e01f7..64a0f5e 100644 --- a/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php +++ b/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php @@ -10,6 +10,7 @@ use Neos\Cache\Frontend\StringFrontend; use Neos\Flow\Annotations as Flow; use Neos\Flow\ObjectManagement\ObjectManagerInterface; +use Neos\Flow\Package\Exception\UnknownPackageException; use Neos\Flow\Package\PackageManager; use Neos\Fusion\Core\Cache\ContentCache; @@ -37,8 +38,10 @@ class RedisContentCacheReader */ protected $applicationIdentifier; - public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentNodeCacheKey $documentNodeCacheKey): RenderedDocumentFromContentCache - { + protected $redis; + + public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentNodeCacheKey $documentNodeCacheKey + ): RenderedDocumentFromContentCache { $maxNestLevel = ContentCache::MAXIMUM_NESTING_LEVEL; $contentCacheStartToken = ContentCache::CACHE_SEGMENT_START_TOKEN; $contentCacheEndToken = ContentCache::CACHE_SEGMENT_END_TOKEN; @@ -48,26 +51,13 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentN */ $identifierPrefix = md5($this->applicationIdentifier) . ':'; - $packageManager = $this->objectManager->get(PackageManager::class); - $flowPackage = $packageManager->getPackage('Neos.Flow'); - preg_match('/^(\d+\.\d+)/', $flowPackage->getInstalledVersion(), $versionMatches); - $flowMajorVersion = (int)($versionMatches[1] ?? '0'); + $redis = $this->getRedis(); - $backend = $this->contentCache->getBackend(); - if ($flowMajorVersion >= 8 && $backend instanceof RedisBackend) { - $reflProp = new \ReflectionProperty(RedisBackend::class, 'redis'); - $reflProp->setAccessible(true); - $redis = $reflProp->getValue($backend); - } elseif (get_class($backend) === 'Sandstorm\OptimizedRedisCacheBackend\OptimizedRedisCacheBackend') { - $reflProp = new \ReflectionProperty(\Sandstorm\OptimizedRedisCacheBackend\OptimizedRedisCacheBackend::class, 'redis'); - $reflProp->setAccessible(true); - $redis = $reflProp->getValue($backend); - } else { - throw new \RuntimeException('The cache backend for "Neos_Fusion_Content" must be an OptimizedRedisCacheBackend, but is ' . get_class($backend), 1622570000); - } $serializedCacheValues = $redis->get($documentNodeCacheKey->fullyQualifiedRedisKeyName($identifierPrefix)); if ($serializedCacheValues === false) { - return RenderedDocumentFromContentCache::createIncomplete('No Redis Key "' . $documentNodeCacheKey->redisKeyName() . '" found.'); + return RenderedDocumentFromContentCache::createIncomplete( + 'No Redis Key "' . $documentNodeCacheKey->redisKeyName() . '" found.' + ); } $documentNodeCacheValues = DocumentNodeCacheValues::fromJsonString($serializedCacheValues); @@ -138,4 +128,44 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentN } return RenderedDocumentFromContentCache::createWithFullContent($content, $documentNodeCacheValues); } + + /** + * @throws UnknownPackageException + */ + protected function getRedis() + { + if ($this->redis) { + return $this->redis; + } + + $packageManager = $this->objectManager->get(PackageManager::class); + $flowPackage = $packageManager->getPackage('Neos.Flow'); + preg_match('/^(\d+\.\d+)/', $flowPackage->getInstalledVersion(), $versionMatches); + $flowMajorVersion = (int)($versionMatches[1] ?? '0'); + + $backend = $this->contentCache->getBackend(); + + if ($flowMajorVersion >= 8 && $backend instanceof RedisBackend) { + $reflProp = new \ReflectionProperty(RedisBackend::class, 'redis'); + $reflProp->setAccessible(true); + $this->redis = $reflProp->getValue($backend); + return $this->redis; + } + + if (get_class($backend) === 'Sandstorm\OptimizedRedisCacheBackend\OptimizedRedisCacheBackend') { + $reflProp = new \ReflectionProperty( + \Sandstorm\OptimizedRedisCacheBackend\OptimizedRedisCacheBackend::class, + 'redis' + ); + $reflProp->setAccessible(true); + $this->redis = $reflProp->getValue($backend); + return $this->redis; + } + + throw new \RuntimeException( + 'The cache backend for "Neos_Fusion_Content" must be an OptimizedRedisCacheBackend, but is ' . get_class( + $backend + ), 1622570000 + ); + } } From 3816e27638bcba8b20a388c71975b704cde09688 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Wed, 15 Nov 2023 17:05:00 +0100 Subject: [PATCH 20/34] FEATURE: Allow enumeration of any nodetype --- Classes/NodeEnumeration/NodeEnumerator.php | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Classes/NodeEnumeration/NodeEnumerator.php b/Classes/NodeEnumeration/NodeEnumerator.php index d6eb0a0..9bb6d82 100644 --- a/Classes/NodeEnumeration/NodeEnumerator.php +++ b/Classes/NodeEnumeration/NodeEnumerator.php @@ -81,7 +81,7 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL $nodeTypeWhitelist = $this->nodeTypeConstraintFactory->parseFilterString($this->nodeTypeWhitelist); - $queueSite = function (Site $site) use ($combinator, &$documentNodeVariantsToRender, $nodeTypeWhitelist, $contentReleaseLogger, $workspaceName) { + $queueSite = function (Site $site) use ($combinator, $nodeTypeWhitelist, $contentReleaseLogger, $workspaceName) { $contentReleaseLogger->debug('Publishing site', [ 'name' => $site->getName(), 'domain' => $site->getFirstActiveDomain() @@ -94,9 +94,18 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL 'dimensionValues' => $dimensionValues ]); + // Build filter from white listed nodetypes + $nodeTypeWhitelist = explode(',', $this->nodeTypeWhitelist ?: 'Neos.Neos:Document'); + $nodeTypeFilter = implode(',', array_map(static function ($nodeType) { + if ($nodeType[0] === '!') { + return '[!instanceof ' . substr($nodeType, 1) . ']'; + } + return '[instanceof ' . $nodeType . ']'; + }, $nodeTypeWhitelist)); + $documentQuery = new FlowQuery([$siteNode]); /** @var NodeInterface[] $documents */ - $documents = $documentQuery->find('[instanceof Neos.Neos:Document]')->add($siteNode)->get(); + $documents = $documentQuery->find($nodeTypeFilter)->add($siteNode)->get(); foreach ($documents as $documentNode) { $contextPath = $documentNode->getContextPath(); @@ -122,17 +131,11 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL $contentReleaseLogger->debug('Skipping node from publishing, because it is hidden', [ 'node' => $contextPath, ]); - } else if ($nodeTypeWhitelist->matches(NodeTypeName::fromString($documentNode->getNodeType()->getName()))) { + } else { $contentReleaseLogger->debug('Registering node for publishing', [ 'node' => $contextPath ]); - yield EnumeratedNode::fromNode($documentNode); - } else { - $contentReleaseLogger->debug('Skipping node from publishing, because it did not match the configured nodeTypeWhitelist', [ - 'node' => $contextPath, - 'nodeTypeWhitelist' => $this->nodeTypeWhitelist - ]); } } } From a59129f4c349c891bd87eb6fd6ded9c12928ec4f Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Thu, 16 Nov 2023 11:50:38 +0100 Subject: [PATCH 21/34] TASK: Optimise redis content cache reader Only upload script once and reuse script and redis instance --- .../RedisContentCacheReader.php | 86 +++++++++++++------ 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php b/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php index 64a0f5e..1a77cbf 100644 --- a/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php +++ b/Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php @@ -38,14 +38,11 @@ class RedisContentCacheReader */ protected $applicationIdentifier; - protected $redis; + protected ?string $scriptSha1 = null; + protected ?\Redis $redis = null; public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentNodeCacheKey $documentNodeCacheKey ): RenderedDocumentFromContentCache { - $maxNestLevel = ContentCache::MAXIMUM_NESTING_LEVEL; - $contentCacheStartToken = ContentCache::CACHE_SEGMENT_START_TOKEN; - $contentCacheEndToken = ContentCache::CACHE_SEGMENT_END_TOKEN; - $contentCacheMarker = ContentCache::CACHE_SEGMENT_MARKER; /** * @see AbstractBackend::setCache() */ @@ -60,8 +57,66 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentN ); } $documentNodeCacheValues = DocumentNodeCacheValues::fromJsonString($serializedCacheValues); + $res = $this->executeRedisCall([$documentNodeCacheValues->getRootIdentifier(), $identifierPrefix]); + [$content, $error] = $res; - $script = " + if (strlen($error) > 0) { + return RenderedDocumentFromContentCache::createIncomplete($error); + } + return RenderedDocumentFromContentCache::createWithFullContent($content, $documentNodeCacheValues); + } + + protected function executeRedisCall(array $params): array + { + $redis = $this->getRedis(); + + $this->prepareRedisScript(); + try { + // starting with Lua 7, eval_ro can be used. + $res = $redis->evalSha($this->scriptSha1, $params); + + $error = $redis->getLastError(); + if ($error !== null) { + throw new \RuntimeException('Redis error: ' . $error); + } + } catch (\Exception $e) { + throw new \RuntimeException('Redis script execution error: ' . $e->getMessage()); + } + + if (!is_array($res) || count($res) !== 2) { + throw new \RuntimeException('Result is no array of length 2, but: ' . count($res)); + } + + return $res; + } + + protected function prepareRedisScript(): void + { + $redis = $this->getRedis(); + + try { + // Load script into redis if it is not already loaded + $scriptExists = false; + if ($this->scriptSha1) { + $scriptExists = $redis->script('exists', $this->scriptSha1)[0] ?? false; + } + if (!$scriptExists) { + $script = $this->buildRedisScript(); + $this->scriptSha1 = $redis->script('load', $script); + } + } catch (\RedisException $e) { + throw new \RuntimeException('Redis Error: ' . $e->getMessage()); + } + } + + protected function buildRedisScript(): string + { + $maxNestLevel = ContentCache::MAXIMUM_NESTING_LEVEL; + $contentCacheStartToken = ContentCache::CACHE_SEGMENT_START_TOKEN; + $contentCacheEndToken = ContentCache::CACHE_SEGMENT_END_TOKEN; + $contentCacheMarker = ContentCache::CACHE_SEGMENT_MARKER; + + return " local rootIdentifier = ARGV[1] local identifierPrefix = ARGV[2] @@ -110,29 +165,12 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentN return {content, error} "; - // starting with Lua 7, eval_ro can be used. - $res = $redis->eval($script, [$documentNodeCacheValues->getRootIdentifier(), $identifierPrefix], 0); - $error = $redis->getLastError(); - if ($error !== null) { - throw new \RuntimeException('Redis Error: ' . $error); - } - - if (count($res) !== 2) { - throw new \RuntimeException('Result is no array of length 2, but: ' . count($res)); - } - $content = $res[0]; - $error = $res[1]; - - if (strlen($error) > 0) { - return RenderedDocumentFromContentCache::createIncomplete($error); - } - return RenderedDocumentFromContentCache::createWithFullContent($content, $documentNodeCacheValues); } /** * @throws UnknownPackageException */ - protected function getRedis() + protected function getRedis(): \Redis { if ($this->redis) { return $this->redis; From b07abbc95084eb5799ae478072397747c294d4d2 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Wed, 22 Nov 2023 09:53:50 +0100 Subject: [PATCH 22/34] TASK: Cleanup enumeration code --- Classes/NodeEnumeration/NodeEnumerator.php | 52 ++++++++-------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/Classes/NodeEnumeration/NodeEnumerator.php b/Classes/NodeEnumeration/NodeEnumerator.php index 9bb6d82..f6e5d61 100644 --- a/Classes/NodeEnumeration/NodeEnumerator.php +++ b/Classes/NodeEnumeration/NodeEnumerator.php @@ -14,20 +14,12 @@ use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService; use Flowpack\DecoupledContentStore\Utility\GeneratorUtility; use Neos\ContentRepository\Domain\Model\NodeInterface; -use Neos\ContentRepository\Domain\NodeType\NodeTypeConstraintFactory; -use Neos\ContentRepository\Domain\NodeType\NodeTypeName; -use Neos\ContentRepository\Domain\Utility\NodePaths; use Neos\Eel\FlowQuery\FlowQuery; use Neos\Flow\Annotations as Flow; use Neos\Neos\Domain\Model\Site; class NodeEnumerator { - /** - * @Flow\Inject - * @var NodeTypeConstraintFactory - */ - protected $nodeTypeConstraintFactory; /** * @Flow\Inject @@ -79,9 +71,16 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL { $combinator = new NodeContextCombinator(); - $nodeTypeWhitelist = $this->nodeTypeConstraintFactory->parseFilterString($this->nodeTypeWhitelist); + // Build filter from white listed nodetypes + $nodeTypeWhitelist = explode(',', $this->nodeTypeWhitelist ?: 'Neos.Neos:Document'); + $nodeTypeFilter = implode(',', array_map(static function ($nodeType) { + if ($nodeType[0] === '!') { + return '[!instanceof ' . substr($nodeType, 1) . ']'; + } + return '[instanceof ' . $nodeType . ']'; + }, $nodeTypeWhitelist)); - $queueSite = function (Site $site) use ($combinator, $nodeTypeWhitelist, $contentReleaseLogger, $workspaceName) { + $queueSite = static function (Site $site) use ($combinator, $nodeTypeFilter, $contentReleaseLogger, $workspaceName) { $contentReleaseLogger->debug('Publishing site', [ 'name' => $site->getName(), 'domain' => $site->getFirstActiveDomain() @@ -94,24 +93,15 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL 'dimensionValues' => $dimensionValues ]); - // Build filter from white listed nodetypes - $nodeTypeWhitelist = explode(',', $this->nodeTypeWhitelist ?: 'Neos.Neos:Document'); - $nodeTypeFilter = implode(',', array_map(static function ($nodeType) { - if ($nodeType[0] === '!') { - return '[!instanceof ' . substr($nodeType, 1) . ']'; - } - return '[instanceof ' . $nodeType . ']'; - }, $nodeTypeWhitelist)); - - $documentQuery = new FlowQuery([$siteNode]); - /** @var NodeInterface[] $documents */ - $documents = $documentQuery->find($nodeTypeFilter)->add($siteNode)->get(); + $nodeQuery = new FlowQuery([$siteNode]); + /** @var NodeInterface[] $matchingNodes */ + $matchingNodes = $nodeQuery->find($nodeTypeFilter)->add($siteNode)->get(); - foreach ($documents as $documentNode) { - $contextPath = $documentNode->getContextPath(); + foreach ($matchingNodes as $nodeToEnumerate) { + $contextPath = $nodeToEnumerate->getContextPath(); // Verify that the node is not orphaned - $parentNode = $documentNode->getParent(); + $parentNode = $nodeToEnumerate->getParent(); while ($parentNode !== $siteNode) { if ($parentNode === null) { $contentReleaseLogger->debug('Skipping node from publishing, because it is orphaned', [ @@ -123,11 +113,7 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL $parentNode = $parentNode->getParent(); } - if (!$documentNode->getParent()) { - $contentReleaseLogger->debug('Skipping node from publishing, because it is orphaned', [ - 'node' => $contextPath, - ]); - } else if ($documentNode->isHidden()) { + if ($nodeToEnumerate->isHidden()) { $contentReleaseLogger->debug('Skipping node from publishing, because it is hidden', [ 'node' => $contextPath, ]); @@ -135,7 +121,7 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL $contentReleaseLogger->debug('Registering node for publishing', [ 'node' => $contextPath ]); - yield EnumeratedNode::fromNode($documentNode); + yield EnumeratedNode::fromNode($nodeToEnumerate); } } } @@ -143,8 +129,8 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL }; if ($site === null) { - foreach ($combinator->sites() as $site) { - yield from $queueSite($site); + foreach ($combinator->sites() as $siteInList) { + yield from $queueSite($siteInList); } } else { yield from $queueSite($site); From ba2248f5ddc210bba14d68d931b0d9939dd31b91 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 28 Nov 2023 11:09:01 +0100 Subject: [PATCH 23/34] TASK: Reformat NodeEnumerator code --- Classes/NodeEnumeration/NodeEnumerator.php | 70 +++++++++++++++------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/Classes/NodeEnumeration/NodeEnumerator.php b/Classes/NodeEnumeration/NodeEnumerator.php index f6e5d61..a022bb8 100644 --- a/Classes/NodeEnumeration/NodeEnumerator.php +++ b/Classes/NodeEnumeration/NodeEnumerator.php @@ -14,6 +14,7 @@ use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService; use Flowpack\DecoupledContentStore\Utility\GeneratorUtility; use Neos\ContentRepository\Domain\Model\NodeInterface; +use Neos\Eel\Exception; use Neos\Eel\FlowQuery\FlowQuery; use Neos\Flow\Annotations as Flow; use Neos\Neos\Domain\Model\Site; @@ -43,19 +44,34 @@ class NodeEnumerator * @Flow\InjectConfiguration("nodeRendering.nodeTypeWhitelist") * @var string */ - protected $nodeTypeWhitelist; - - public function enumerateAndStoreInRedis(?Site $site, ContentReleaseLogger $contentReleaseLogger, ContentReleaseIdentifier $releaseIdentifier): void - { - $contentReleaseLogger->info('Starting content release', ['contentReleaseIdentifier' => $releaseIdentifier->jsonSerialize()]); + protected $nodeTypeList; + + public function enumerateAndStoreInRedis( + ?Site $site, + ContentReleaseLogger $contentReleaseLogger, + ContentReleaseIdentifier $releaseIdentifier + ): void { + $contentReleaseLogger->info( + 'Starting content release', + ['contentReleaseIdentifier' => $releaseIdentifier->jsonSerialize()] + ); // set content release status to running $currentMetadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($releaseIdentifier); $newMetadata = $currentMetadata->withStatus(NodeRenderingCompletionStatus::running()); - $this->redisContentReleaseService->setContentReleaseMetadata($releaseIdentifier, $newMetadata, RedisInstanceIdentifier::primary()); + $this->redisContentReleaseService->setContentReleaseMetadata( + $releaseIdentifier, + $newMetadata, + RedisInstanceIdentifier::primary() + ); $this->redisEnumerationRepository->clearDocumentNodesEnumeration($releaseIdentifier); - foreach (GeneratorUtility::createArrayBatch($this->enumerateAll($site, $contentReleaseLogger, $newMetadata->getWorkspaceName()), 100) as $enumeration) { + foreach ( + GeneratorUtility::createArrayBatch( + $this->enumerateAll($site, $contentReleaseLogger, $newMetadata->getWorkspaceName()), + 100 + ) as $enumeration + ) { $this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($releaseIdentifier); // $enumeration is an array of EnumeratedNode, with at most 100 elements in it. // TODO: EXTENSION POINT HERE, TO ADD ADDITIONAL ENUMERATIONS (.metadata.json f.e.) @@ -66,21 +82,33 @@ public function enumerateAndStoreInRedis(?Site $site, ContentReleaseLogger $cont /** * @return iterable + * @throws Exception */ - private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseLogger, string $workspaceName): iterable - { + private function enumerateAll( + ?Site $site, + ContentReleaseLogger $contentReleaseLogger, + string $workspaceName + ): iterable { $combinator = new NodeContextCombinator(); - // Build filter from white listed nodetypes - $nodeTypeWhitelist = explode(',', $this->nodeTypeWhitelist ?: 'Neos.Neos:Document'); - $nodeTypeFilter = implode(',', array_map(static function ($nodeType) { - if ($nodeType[0] === '!') { - return '[!instanceof ' . substr($nodeType, 1) . ']'; - } - return '[instanceof ' . $nodeType . ']'; - }, $nodeTypeWhitelist)); - - $queueSite = static function (Site $site) use ($combinator, $nodeTypeFilter, $contentReleaseLogger, $workspaceName) { + // Build filter from allowed/disallowed nodetypes + $nodeTypeList = explode(',', $this->nodeTypeList ?: 'Neos.Neos:Document'); + $nodeTypeFilter = implode( + ',', + array_map(static function ($nodeType) { + if ($nodeType[0] === '!') { + return '[!instanceof ' . substr($nodeType, 1) . ']'; + } + return '[instanceof ' . $nodeType . ']'; + }, $nodeTypeList) + ); + + $queueSite = static function (Site $site) use ( + $combinator, + $nodeTypeFilter, + $contentReleaseLogger, + $workspaceName + ) { $contentReleaseLogger->debug('Publishing site', [ 'name' => $site->getName(), 'domain' => $site->getFirstActiveDomain() @@ -125,7 +153,9 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL } } } - $contentReleaseLogger->debug(sprintf('Finished enumerating site %s in %dms', $site->getName(), (microtime(true) - $startTime) * 1000)); + $contentReleaseLogger->debug( + sprintf('Finished enumerating site %s in %dms', $site->getName(), (microtime(true) - $startTime) * 1000) + ); }; if ($site === null) { From 74157bf204e33f17274ef64daacbd74676e95a16 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 7 Dec 2023 15:15:59 +0100 Subject: [PATCH 24/34] BUGFIX: Add html escaping to stderr and stdout of job logs --- .../Private/BackendFusion/Integration/Backend.Details.fusion | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion index 4b9db79..777c967 100644 --- a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion +++ b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion @@ -40,10 +40,10 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component

Log Output for {detailTaskName}

-                  {jobLogs.stderr}
+                  {String.htmlSpecialChars(jobLogs.stderr)}
               
-                  {jobLogs.stdout}
+                  {String.htmlSpecialChars(jobLogs.stdout)}

From ce641702d158debe40afbc824e44f93afa1e75e1 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Mon, 11 Dec 2023 17:29:07 +0100 Subject: [PATCH 25/34] FEATURE: Make iteration checks in NodeRenderer configurable --- Classes/NodeRendering/NodeRenderer.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Classes/NodeRendering/NodeRenderer.php b/Classes/NodeRendering/NodeRenderer.php index 9316e6d..2e91198 100644 --- a/Classes/NodeRendering/NodeRenderer.php +++ b/Classes/NodeRendering/NodeRenderer.php @@ -51,6 +51,8 @@ */ class NodeRenderer { + protected const RESTART_AFTER_RENDER_COUNT = 100; + protected const CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT = 5; /** * @Flow\Inject @@ -158,12 +160,12 @@ public function render(ContentReleaseIdentifier $contentReleaseIdentifier, Conte $i++; - if ($i % 5 === 0) { + if (self::CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT > 0 && $i % self::CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT === 0) { $this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($contentReleaseIdentifier); } - if ($i % 20 === 0) { - $contentReleaseLogger->info('Restarting after 20 renders.'); + if ($i % self::RESTART_AFTER_RENDER_COUNT === 0) { + $contentReleaseLogger->info(sprintf('Restarting after %d renders.', self::RESTART_AFTER_RENDER_COUNT)); yield ExitEvent::createWithStatusCode(193); return; } From dbb37059ba19d02afdf9256b77cf7101989dc5c7 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 12 Dec 2023 08:58:29 +0100 Subject: [PATCH 26/34] TASK: Append payload only in logging if set --- .../Infrastructure/ContentReleaseLogger.php | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Classes/Core/Infrastructure/ContentReleaseLogger.php b/Classes/Core/Infrastructure/ContentReleaseLogger.php index 3531115..35e9f9c 100644 --- a/Classes/Core/Infrastructure/ContentReleaseLogger.php +++ b/Classes/Core/Infrastructure/ContentReleaseLogger.php @@ -47,24 +47,30 @@ public static function fromSymfonyOutput(OutputInterface $output, ContentRelease return new static($output, $contentReleaseIdentifier, null); } - public function debug($message, array $additionalPayload = []) + public function debug($message, array $additionalPayload = []): void { - $this->output->writeln($this->logPrefix . $message . json_encode($additionalPayload)); + $this->logToOutput($message, $additionalPayload); } - public function info($message, array $additionalPayload = []) + public function info($message, array $additionalPayload = []): void { - $this->output->writeln($this->logPrefix . $message . json_encode($additionalPayload)); + $this->logToOutput($message, $additionalPayload); } - public function warn($message, array $additionalPayload = []) + public function warn($message, array $additionalPayload = []): void { - $this->output->writeln($this->logPrefix . $message . json_encode($additionalPayload)); + $this->logToOutput($message, $additionalPayload); } - public function error($message, array $additionalPayload = []) + public function error($message, array $additionalPayload = []): void { - $this->output->writeln($this->logPrefix . $message . json_encode($additionalPayload)); + $this->logToOutput($message, $additionalPayload); + } + + protected function logToOutput($message, array $additionalPayload = []): void + { + $formattedPayload = $additionalPayload ? json_encode($additionalPayload, JSON_PRETTY_PRINT) : ''; + $this->output->writeln($this->logPrefix . $message . $formattedPayload); } public function logException(\Exception $exception, string $message, array $additionalPayload) From 9e887c9e8c52727a5d34a3639dcb119cc5536d4b Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 12 Dec 2023 09:04:30 +0100 Subject: [PATCH 27/34] TASK: Add newlines to log output --- .../Private/BackendFusion/Integration/Backend.Details.fusion | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion index 777c967..0fc6d6d 100644 --- a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion +++ b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion @@ -40,10 +40,10 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component

Log Output for {detailTaskName}

-                  {String.htmlSpecialChars(jobLogs.stderr)}
+                  {String.nl2br(String.htmlSpecialChars(jobLogs.stderr))}
               
-                  {String.htmlSpecialChars(jobLogs.stdout)}
+                  {String.nl2br(String.htmlSpecialChars(jobLogs.stdout))}

From a63980072b23c5231b1f45056b74610d9964dd81 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 12 Dec 2023 09:47:03 +0100 Subject: [PATCH 28/34] BUGFIX: Remove pretty formatting for payloads again --- Classes/Core/Infrastructure/ContentReleaseLogger.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Core/Infrastructure/ContentReleaseLogger.php b/Classes/Core/Infrastructure/ContentReleaseLogger.php index 35e9f9c..c5bdca3 100644 --- a/Classes/Core/Infrastructure/ContentReleaseLogger.php +++ b/Classes/Core/Infrastructure/ContentReleaseLogger.php @@ -69,7 +69,7 @@ public function error($message, array $additionalPayload = []): void protected function logToOutput($message, array $additionalPayload = []): void { - $formattedPayload = $additionalPayload ? json_encode($additionalPayload, JSON_PRETTY_PRINT) : ''; + $formattedPayload = $additionalPayload ? json_encode($additionalPayload) : ''; $this->output->writeln($this->logPrefix . $message . $formattedPayload); } From f68938e41c6ca6bd61095b3938b12161d65e2df4 Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Tue, 12 Dec 2023 16:03:32 +0100 Subject: [PATCH 29/34] BUGFIX: Make NodeRenderer constants overridable --- Classes/NodeRendering/NodeRenderer.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Classes/NodeRendering/NodeRenderer.php b/Classes/NodeRendering/NodeRenderer.php index 2e91198..b58a13d 100644 --- a/Classes/NodeRendering/NodeRenderer.php +++ b/Classes/NodeRendering/NodeRenderer.php @@ -51,7 +51,7 @@ */ class NodeRenderer { - protected const RESTART_AFTER_RENDER_COUNT = 100; + protected const RESTART_AFTER_RENDER_COUNT = 20; protected const CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT = 5; /** @@ -160,12 +160,12 @@ public function render(ContentReleaseIdentifier $contentReleaseIdentifier, Conte $i++; - if (self::CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT > 0 && $i % self::CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT === 0) { + if (static::CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT > 0 && $i % static::CHECK_FOR_CONCURRENT_RELEASES_RENDER_COUNT === 0) { $this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($contentReleaseIdentifier); } - if ($i % self::RESTART_AFTER_RENDER_COUNT === 0) { - $contentReleaseLogger->info(sprintf('Restarting after %d renders.', self::RESTART_AFTER_RENDER_COUNT)); + if ($i % static::RESTART_AFTER_RENDER_COUNT === 0) { + $contentReleaseLogger->info(sprintf('Restarting after %d renders.', static::RESTART_AFTER_RENDER_COUNT)); yield ExitEvent::createWithStatusCode(193); return; } From 53db941e5fcd825717bb88e3d6751c09e5de0a84 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Dec 2023 10:47:24 +0100 Subject: [PATCH 30/34] TASK: Forgiving data handling in ui service Previously if `ContentReleaseBatchResult` contained a wrong value or the fallback null the view would crash. --- Classes/BackendUi/BackendUiDataService.php | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Classes/BackendUi/BackendUiDataService.php b/Classes/BackendUi/BackendUiDataService.php index 145daa9..165939e 100644 --- a/Classes/BackendUi/BackendUiDataService.php +++ b/Classes/BackendUi/BackendUiDataService.php @@ -14,6 +14,7 @@ use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderingStatistics; use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingErrorManager; use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingStatisticsStore; +use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata; use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService; use Flowpack\DecoupledContentStore\ReleaseSwitch\Infrastructure\RedisReleaseSwitchService; use Neos\Flow\Annotations as Flow; @@ -81,12 +82,17 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId $lastRendering = RenderingStatistics::fromJsonString($lastRenderingStatisticsEntries->getResultForContentRelease($contentReleaseId)); $firstRendering = RenderingStatistics::fromJsonString($firstRenderingStatisticsEntries->getResultForContentRelease($contentReleaseId)); + $metadataForContentRelease = $metadata->getResultForContentRelease($contentReleaseId); + $countForContentRelease = $counts->getResultForContentRelease($contentReleaseId); + $iterationsCountForContentRelease = $iterationsCounts->getResultForContentRelease($contentReleaseId); + $errorCountForContentRelease = $errorCounts->getResultForContentRelease($contentReleaseId); + $result[] = new ContentReleaseOverviewRow( $contentReleaseId, - $metadata->getResultForContentRelease($contentReleaseId), - $counts->getResultForContentRelease($contentReleaseId), - $iterationsCounts->getResultForContentRelease($contentReleaseId), - $errorCounts->getResultForContentRelease($contentReleaseId), + $metadataForContentRelease instanceof ContentReleaseMetadata ? $metadataForContentRelease : null, + is_int($countForContentRelease) ? $countForContentRelease : 0, + is_int($iterationsCountForContentRelease) ? $iterationsCountForContentRelease : 0, + is_int($errorCountForContentRelease) ? $errorCountForContentRelease : 0, $lastRendering->getTotalJobs() > 0 ? round($lastRendering->getRenderedJobs() / $lastRendering->getTotalJobs() * 100) : 100, $firstRendering->getRenderedJobs(), From aab5a6dcd3e3630bb3ed55103f481d6190038176 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 17 Jan 2024 09:17:15 +0100 Subject: [PATCH 31/34] BUGFIX: Joblogs preserve whitespace and highlight errors Related: https://github.com/Flowpack/Flowpack.DecoupledContentStore/pull/35 --- .../Integration/Backend.Details.fusion | 8 +++---- Resources/Public/BackendCompiled/out.css | 21 ++++++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion index 0fc6d6d..938b938 100644 --- a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion +++ b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion @@ -39,11 +39,11 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component

Log Output for {detailTaskName}

- 
-                  {String.nl2br(String.htmlSpecialChars(jobLogs.stderr))}
+              
+                {String.htmlSpecialChars(jobLogs.stderr)}
               

-              
-                  {String.nl2br(String.htmlSpecialChars(jobLogs.stdout))}
+              
+                {String.htmlSpecialChars(jobLogs.stdout)}
               

             
             

diff --git a/Resources/Public/BackendCompiled/out.css b/Resources/Public/BackendCompiled/out.css
index 2fb6cbc..e135e8a 100644
--- a/Resources/Public/BackendCompiled/out.css
+++ b/Resources/Public/BackendCompiled/out.css
@@ -1,4 +1,4 @@
-/* ../../../../../../../../../var/folders/1z/rq1_kd710hn00r35j5rzqs5h0000gn/T/tmp-53849-7B7Y00uLTAH0/Flowpack.DecoupledContentStore/Resources/Private/Js/1674136282935-styles.css */
+/* ../../../../../../../../../var/folders/jx/0n3dgdj52yd7lhy7jz8m_lyc0000gn/T/tmp-76784-4SF1kWZd2RxP/Flowpack.DecoupledContentStore/Resources/Private/Js/1705479026689-styles.css */
 #app *,
 #app ::before,
 #app ::after {
@@ -34,6 +34,9 @@
   margin-top: 1.25rem;
   margin-bottom: 1.25rem;
 }
+#app .mb-6 {
+  margin-bottom: 1.5rem;
+}
 #app .mt-4 {
   margin-top: 1rem;
 }
@@ -49,6 +52,9 @@
 #app .mb-2 {
   margin-bottom: 0.5rem;
 }
+#app .block {
+  display: block;
+}
 #app .inline-block {
   display: inline-block;
 }
@@ -123,11 +129,17 @@
 #app .overflow-hidden {
   overflow: hidden;
 }
+#app .overflow-x-scroll {
+  overflow-x: scroll;
+}
 #app .truncate {
   overflow: hidden;
   text-overflow: ellipsis;
   white-space: nowrap;
 }
+#app .whitespace-pre {
+  white-space: pre;
+}
 #app .rounded-md {
   border-radius: 0.375rem;
 }
@@ -143,6 +155,10 @@
 #app .border-2 {
   border-width: 2px;
 }
+#app .border-red-800 {
+  --tw-border-opacity: 1;
+  border-color: rgb(153 27 27 / var(--tw-border-opacity));
+}
 #app .border-gray-300 {
   --tw-border-opacity: 1;
   border-color: rgb(209 213 219 / var(--tw-border-opacity));
@@ -191,6 +207,9 @@
   --tw-bg-opacity: 1;
   background-color: rgb(147 197 253 / var(--tw-bg-opacity));
 }
+#app .p-2 {
+  padding: 0.5rem;
+}
 #app .py-5 {
   padding-top: 1.25rem;
   padding-bottom: 1.25rem;

From 11a2a2b3bba76cd77571d066e18018a803e50b69 Mon Sep 17 00:00:00 2001
From: Sebastian Helzle 
Date: Wed, 17 Jan 2024 13:14:43 +0100
Subject: [PATCH 32/34] TASK: Reverse stdout log and improve styling

---
 .../Infrastructure/ContentReleaseLogger.php   | 27 +++++++++----------
 .../Integration/Backend.Details.fusion        | 22 ++++++++++-----
 Resources/Public/BackendCompiled/out.css      |  8 +++++-
 3 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/Classes/Core/Infrastructure/ContentReleaseLogger.php b/Classes/Core/Infrastructure/ContentReleaseLogger.php
index c5bdca3..01400f0 100644
--- a/Classes/Core/Infrastructure/ContentReleaseLogger.php
+++ b/Classes/Core/Infrastructure/ContentReleaseLogger.php
@@ -19,10 +19,9 @@ class ContentReleaseLogger
      */
     protected $contentReleaseIdentifier;
 
-    /**
-     * @var string
-     */
-    protected $logPrefix = '';
+    protected string $logPrefix = '';
+
+    protected ?RendererIdentifier $rendererIdentifier;
 
     protected function __construct(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, ?RendererIdentifier $rendererIdentifier)
     {
@@ -47,30 +46,30 @@ public static function fromSymfonyOutput(OutputInterface $output, ContentRelease
         return new static($output, $contentReleaseIdentifier, null);
     }
 
-    public function debug($message, array $additionalPayload = []): void
+    public function debug(string $message, array $additionalPayload = []): void
     {
-        $this->logToOutput($message, $additionalPayload);
+        $this->logToOutput('DEBUG', $message, $additionalPayload);
     }
 
-    public function info($message, array $additionalPayload = []): void
+    public function info(string $message, array $additionalPayload = []): void
     {
-        $this->logToOutput($message, $additionalPayload);
+        $this->logToOutput('INFO', $message, $additionalPayload);
     }
 
-    public function warn($message, array $additionalPayload = []): void
+    public function warn(string $message, array $additionalPayload = []): void
     {
-        $this->logToOutput($message, $additionalPayload);
+        $this->logToOutput('WARNING', $message, $additionalPayload);
     }
 
-    public function error($message, array $additionalPayload = []): void
+    public function error(string $message, array $additionalPayload = []): void
     {
-        $this->logToOutput($message, $additionalPayload);
+        $this->logToOutput('ERROR', $message, $additionalPayload);
     }
 
-    protected function logToOutput($message, array $additionalPayload = []): void
+    protected function logToOutput(string $level, string $message, array $additionalPayload = []): void
     {
         $formattedPayload = $additionalPayload ? json_encode($additionalPayload) : '';
-        $this->output->writeln($this->logPrefix . $message . $formattedPayload);
+        $this->output->writeln($this->logPrefix . $level . ': ' . $message . $formattedPayload);
     }
 
     public function logException(\Exception $exception, string $message, array $additionalPayload)
diff --git a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion
index 938b938..2c37323 100644
--- a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion
+++ b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion
@@ -8,6 +8,8 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component
     // - redisContentStores: array of all configured content store identifiers
     // - isPrimary: bool
 
+    stdOutLines = ${Array.filter(String.pregSplit(String.htmlSpecialChars(jobLogs.stdout), "/\r\n|\n|\r/"))}
+
     renderer = afx`
         

             

@@ -38,13 +40,19 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component
                   />
               
 
-              
Log Output for {detailTaskName}

-              
-                {String.htmlSpecialChars(jobLogs.stderr)}
-              

-              
-                {String.htmlSpecialChars(jobLogs.stdout)}
-              

+              

+                
Error output for {detailTaskName}

+                
+                  {String.htmlSpecialChars(jobLogs.stderr)}
+                

+              

+
+              
Log output for {detailTaskName}

+              

+                
+                  
{Array.length(props.stdOutLines) - iteration.index}: {String.pregReplace(item, "/(DEBUG|WARNING|ERROR|INFO): (.*)/", "$1: $2")}

+                
+              

             
             

               No data exists for this release in Redis.
diff --git a/Resources/Public/BackendCompiled/out.css b/Resources/Public/BackendCompiled/out.css
index e135e8a..ae6e69e 100644
--- a/Resources/Public/BackendCompiled/out.css
+++ b/Resources/Public/BackendCompiled/out.css
@@ -1,4 +1,4 @@
-/* ../../../../../../../../../var/folders/jx/0n3dgdj52yd7lhy7jz8m_lyc0000gn/T/tmp-76784-4SF1kWZd2RxP/Flowpack.DecoupledContentStore/Resources/Private/Js/1705479026689-styles.css */
+/* ../../../../../../../../../var/folders/1z/rq1_kd710hn00r35j5rzqs5h0000gn/T/tmp-24630-ZfKI51gf2y1c/Flowpack.DecoupledContentStore/Resources/Private/Js/1705488391983-styles.css */
 #app *,
 #app ::before,
 #app ::after {
@@ -70,6 +70,9 @@
 #app .table {
   display: table;
 }
+#app .grid {
+  display: grid;
+}
 #app .hidden {
   display: none;
 }
@@ -117,6 +120,9 @@
 #app .justify-center {
   justify-content: center;
 }
+#app .gap-2 {
+  gap: 0.5rem;
+}
 #app .divide-y > :not([hidden]) ~ :not([hidden]) {
   --tw-divide-y-reverse: 0;
   border-top-width: calc(1px * calc(1 - var(--tw-divide-y-reverse)));

From d62e01339e3f15f46f5f51e272a8ca2929da9948 Mon Sep 17 00:00:00 2001
From: Sebastian Helzle 
Date: Mon, 15 Apr 2024 09:29:15 +0200
Subject: [PATCH 33/34] FEATURE: Format stdout with eel helper

If a line contains json output it is rendered as toggable details element
---
 Classes/Eel/ModuleHelper.php                  | 59 +++++++++++++++++++
 Configuration/Settings.yaml                   |  4 ++
 .../Integration/Backend.Details.fusion        |  8 +--
 3 files changed, 65 insertions(+), 6 deletions(-)
 create mode 100644 Classes/Eel/ModuleHelper.php

diff --git a/Classes/Eel/ModuleHelper.php b/Classes/Eel/ModuleHelper.php
new file mode 100644
index 0000000..5ac9c0f
--- /dev/null
+++ b/Classes/Eel/ModuleHelper.php
@@ -0,0 +1,59 @@
+$1: $2", htmlSpecialChars($line));
+
+            // Add line numbers
+            $line = ($lineCount - $index) . ': ' . $line;
+
+            // Insert formatted JSON data
+            if ($jsonData) {
+                $isDebug = strpos($line, 'DEBUG:') !== false;
+                $jsonString = "\n" . json_encode($jsonData, JSON_PRETTY_PRINT) . "";
+                $line = $isDebug ? '
' . '' . $line . '' . $jsonString . '
' : $line . $jsonString;
+            }
+
+            // Wrap in 
 tags
+            return '
' . $line . '
';
+        }, $lines, range(1, $lineCount));
+
+        return implode("\n", $formattedLines);
+    }
+
+    public function allowsCallOfMethod($methodName): bool
+    {
+        return true;
+    }
+}
diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml
index adb8901..1a57722 100644
--- a/Configuration/Settings.yaml
+++ b/Configuration/Settings.yaml
@@ -180,3 +180,7 @@ Neos:
       routes:
         'Flowpack.DecoupledContentStore':
           position: 'before Neos.Neos'
+
+  Fusion:
+    defaultContext:
+      Flowpack.DecoupledContentStore: Flowpack\DecoupledContentStore\Eel\ModuleHelper
diff --git a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion
index 2c37323..a06310f 100644
--- a/Resources/Private/BackendFusion/Integration/Backend.Details.fusion
+++ b/Resources/Private/BackendFusion/Integration/Backend.Details.fusion
@@ -8,7 +8,7 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component
     // - redisContentStores: array of all configured content store identifiers
     // - isPrimary: bool
 
-    stdOutLines = ${Array.filter(String.pregSplit(String.htmlSpecialChars(jobLogs.stdout), "/\r\n|\n|\r/"))}
+    stdOutLines = ${Flowpack.DecoupledContentStore.formatStdOutput(jobLogs.stdout)}
 
     renderer = afx`
         

@@ -48,11 +48,7 @@ Flowpack.DecoupledContentStore.BackendController.details = Neos.Fusion:Component
               

 
               
Log output for {detailTaskName}

-              

-                
-                  
{Array.length(props.stdOutLines) - iteration.index}: {String.pregReplace(item, "/(DEBUG|WARNING|ERROR|INFO): (.*)/", "$1: $2")}

-                
-              

+              
{props.stdOutLines}

             
             

               No data exists for this release in Redis.

From 7d576fe6e93b39fec8515832d0ca961a8e92d220 Mon Sep 17 00:00:00 2001
From: Sebastian Helzle 
Date: Wed, 24 Apr 2024 09:58:44 +0200
Subject: [PATCH 34/34] BUGFIX: Handle empty stdout when formatting logs

---
 Classes/Eel/ModuleHelper.php | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Classes/Eel/ModuleHelper.php b/Classes/Eel/ModuleHelper.php
index 5ac9c0f..6aedf61 100644
--- a/Classes/Eel/ModuleHelper.php
+++ b/Classes/Eel/ModuleHelper.php
@@ -12,8 +12,12 @@
  */
 class ModuleHelper implements ProtectedContextAwareInterface
 {
-    public function formatStdOutput(string $stdOut): string
+    public function formatStdOutput(?string $stdOut): string
     {
+        if (!$stdOut) {
+            return '';
+        }
+
         $escapedString = $stdOut;
         $lines = array_reverse(array_filter(preg_split("/\r\n|\n|\r/", $escapedString)));
         $lineCount = count($lines);