WIP TASK: Further improvements for the 1and1 project

Classes/Aspects/CacheUrlMappingAspect.php

@@ -37,6 +37,11 @@ class CacheUrlMappingAspect
      */
     protected $isActive = false;
 
+    /**
+     * @var null | int
+     */
+    protected $renderTimestamp = null;
+
     /**
      * @var array
      */
@@ -102,9 +107,6 @@ public function storeRootCacheIdentifier(JoinPointInterface $joinPoint)
 
         /** @var NodeInterface $node */
         $node = $this->currentEvaluateContext['cacheIdentifierValues']['node'];
-        if (!$node->getContext()->getWorkspace()->isPublicWorkspace()) {
-            return;
-        }
 
         $url = $this->getCurrentUrl();
 
@@ -144,8 +146,10 @@ public function storeRootCacheIdentifier(JoinPointInterface $joinPoint)
         $logger->debug('Mapping URL ' . $url . ' to ' . $rootIdentifier . ' with tags ' . implode(', ', $rootTags));
 
         $arguments = $this->getCurrentArguments($node);
+        // TODO: To make parallel rendering possible, we need to make sure that the cache key also includes the currently rendered workspace, as the node might originate from a base workspace (usually live). See `DocumentNodeCacheKey`.
         $rootKey = DocumentNodeCacheKey::fromNodeAndArguments($node, $arguments);
-        $rootCacheValues = DocumentNodeCacheValues::create($rootIdentifier, $url);
+        $rootCacheValues = DocumentNodeCacheValues::create($rootIdentifier, $url)
+            ->withMetadata('renderTime', (int)(microtime(true) * 1000) - $this->renderTimestamp);
         // allow other document metadata generators here
         $rootCacheValues = $this->nodeRenderingExtensionManager->runDocumentMetadataGenerators($node, $arguments, $this->controllerContext, $rootCacheValues);
         $this->contentCacheFrontend->set($rootKey->redisKeyName(), json_encode($rootCacheValues), $rootTags);
@@ -202,6 +206,7 @@ public function beforeDocumentRendering(ContentReleaseLogger $contentReleaseLogg
     {
         $this->isActive = true;
         $this->contentReleaseLogger = $contentReleaseLogger;
+        $this->renderTimestamp = (int)(microtime(true) * 1000);
     }
 
     public function afterDocumentRendering(): void

Classes/BackendUi/BackendUiDataService.php

@@ -14,6 +14,7 @@
 use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderingStatistics;
 use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingErrorManager;
 use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingStatisticsStore;
+use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata;
 use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
 use Flowpack\DecoupledContentStore\ReleaseSwitch\Infrastructure\RedisReleaseSwitchService;
 use Neos\Flow\Annotations as Flow;
@@ -81,12 +82,17 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId
             $lastRendering = RenderingStatistics::fromJsonString($lastRenderingStatisticsEntries->getResultForContentRelease($contentReleaseId));
             $firstRendering = RenderingStatistics::fromJsonString($firstRenderingStatisticsEntries->getResultForContentRelease($contentReleaseId));
 
+            $metadataForContentRelease = $metadata->getResultForContentRelease($contentReleaseId);
+            $countForContentRelease = $counts->getResultForContentRelease($contentReleaseId);
+            $iterationsCountForContentRelease = $iterationsCounts->getResultForContentRelease($contentReleaseId);
+            $errorCountForContentRelease = $errorCounts->getResultForContentRelease($contentReleaseId);
+
             $result[] = new ContentReleaseOverviewRow(
                 $contentReleaseId,
-                $metadata->getResultForContentRelease($contentReleaseId),
-                $counts->getResultForContentRelease($contentReleaseId),
-                $iterationsCounts->getResultForContentRelease($contentReleaseId),
-                $errorCounts->getResultForContentRelease($contentReleaseId),
+                $metadataForContentRelease instanceof ContentReleaseMetadata ? $metadataForContentRelease : null,
+                is_int($countForContentRelease) ? $countForContentRelease : 0,
+                is_int($iterationsCountForContentRelease) ? $iterationsCountForContentRelease : 0,
+                is_int($errorCountForContentRelease) ? $errorCountForContentRelease : 0,
                 $lastRendering->getTotalJobs() > 0 ? round($lastRendering->getRenderedJobs()
                     / $lastRendering->getTotalJobs() * 100) : 100,
                 $firstRendering->getRenderedJobs(),
@@ -114,9 +120,14 @@ private function calculateReleaseSize(RedisInstanceIdentifier $redisInstanceIden
         return round($size / 1000000, 2);
     }
 
-    public function loadDetailsData(ContentReleaseIdentifier $contentReleaseIdentifier, RedisInstanceIdentifier $redisInstanceIdentifier): ContentReleaseDetails
+    public function loadDetailsData(ContentReleaseIdentifier $contentReleaseIdentifier, RedisInstanceIdentifier $redisInstanceIdentifier): ?ContentReleaseDetails
     {
         $contentReleaseMetadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier, $redisInstanceIdentifier);
+
+        if (!$contentReleaseMetadata) {
+            return null;
+        }
+
         $contentReleaseJob = $this->prunnerApiService->loadJobDetail($contentReleaseMetadata->getPrunnerJobId()->toJobId());
 
         $manualTransferJobs = count($contentReleaseMetadata->getManualTransferJobIds()) ? array_map(function (PrunnerJobId $item) {

Classes/BackendUi/Dto/ContentReleaseOverviewRow.php

@@ -14,7 +14,7 @@
 class ContentReleaseOverviewRow
 {
     private ContentReleaseIdentifier $contentReleaseIdentifier;
-    private ContentReleaseMetadata $metadata;
+    private ?ContentReleaseMetadata $metadata;
     private int $enumeratedDocumentNodesCount;
     private int $iterationsCount;
     private int $errorCount;
@@ -23,7 +23,7 @@ class ContentReleaseOverviewRow
     private bool $isActive;
     private float $releaseSize;
 
-    public function __construct(ContentReleaseIdentifier $contentReleaseIdentifier, ContentReleaseMetadata $metadata,
+    public function __construct(ContentReleaseIdentifier $contentReleaseIdentifier, ?ContentReleaseMetadata $metadata,
                                 int $enumeratedDocumentNodesCount, int $iterationsCount, int $errorCount,
                                 float $progress, int $renderedUrlCount, bool $isActive, float $releaseSize)
     {
@@ -38,73 +38,46 @@ public function __construct(ContentReleaseIdentifier $contentReleaseIdentifier,
         $this->releaseSize = $releaseSize;
     }
 
-    /**
-     * @return ContentReleaseMetadata
-     */
-    public function getMetadata(): ContentReleaseMetadata
+    public function getMetadata(): ?ContentReleaseMetadata
     {
         return $this->metadata;
     }
 
-    /**
-     * @return ContentReleaseIdentifier
-     */
     public function getContentReleaseIdentifier(): ContentReleaseIdentifier
     {
         return $this->contentReleaseIdentifier;
     }
 
-    /**
-     * @return int
-     */
     public function getEnumeratedDocumentNodesCount(): int
     {
         return $this->enumeratedDocumentNodesCount;
     }
 
-    /**
-     * @return int
-     */
     public function getIterationsCount(): int
     {
         return $this->iterationsCount;
     }
 
-    /**
-     * @return int
-     */
     public function getErrorCount(): int
     {
         return $this->errorCount;
     }
 
-    /**
-     * @return float
-     */
     public function getProgress(): float
     {
         return $this->progress;
     }
 
-    /**
-     * @return int
-     */
     public function getRenderedUrlCount(): int
     {
         return $this->renderedUrlCount;
     }
 
-    /**
-     * @return bool
-     */
     public function isActive(): bool
     {
         return $this->isActive;
     }
 
-    /**
-     * @return float
-     */
     public function getReleaseSize(): float
     {
         return $this->releaseSize;

Classes/Command/ContentReleasePrepareCommandController.php

@@ -10,6 +10,7 @@
 use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
 use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger;
 use Neos\Flow\Cli\CommandController;
+use Neos\Fusion\Core\Cache\ContentCache;
 
 /**
  * Commands for the PREPARE stage in the pipeline. Not meant to be called manually.
@@ -28,6 +29,12 @@ class ContentReleasePrepareCommandController extends CommandController
      */
     protected $concurrentBuildLock;
 
+    /**
+     * @Flow\Inject
+     * @var ContentCache
+     */
+    protected $contentCache;
+
     public function createContentReleaseCommand(string $contentReleaseIdentifier, string $prunnerJobId, string $workspaceName = 'live'): void
     {
         $contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);
@@ -52,4 +59,15 @@ public function registerManualTransferJobCommand(string $contentReleaseIdentifie
 
         $this->redisContentReleaseService->registerManualTransferJob($contentReleaseIdentifier, $prunnerJobId, $logger);
     }
+
+    public function flushContentCacheIfRequiredCommand(string $contentReleaseIdentifier, bool $flushContentCache = false): void
+    {
+        $logger = ContentReleaseLogger::fromConsoleOutput($this->output, ContentReleaseIdentifier::fromString($contentReleaseIdentifier));
+        if (!$flushContentCache) {
+            $logger->info('Not flushing content cache');
+            return;
+        }
+        $logger->info('Flushing content cache');
+        $this->contentCache->flush();
+    }
 }

Classes/ContentReleaseManager.php

@@ -12,18 +12,12 @@
 use Neos\Flow\Annotations as Flow;
 use Flowpack\Prunner\PrunnerApiService;
 use Flowpack\Prunner\ValueObject\PipelineName;
-use Neos\Fusion\Core\Cache\ContentCache;
 
 /**
  * @Flow\Scope("singleton")
  */
 class ContentReleaseManager
 {
-    /**
-     * @Flow\Inject
-     * @var ContentCache
-     */
-    protected $contentCache;
 
     /**
      * @Flow\Inject
@@ -61,6 +55,7 @@ public function startIncrementalContentRelease(string $currentContentReleaseId =
             'currentContentReleaseId' => $currentContentReleaseId ?: self::NO_PREVIOUS_RELEASE,
             'validate' => true,
             'workspaceName' => $workspace ? $workspace->getName() : 'live',
+            'flushContentCache' => false,
         ]));
         return $contentReleaseId;
     }
@@ -74,12 +69,12 @@ public function startFullContentRelease(bool $validate = true, string $currentCo
         }
 
         $contentReleaseId = ContentReleaseIdentifier::create();
-        $this->contentCache->flush();
         $this->prunnerApiService->schedulePipeline(PipelineName::create('do_content_release'), array_merge($additionalVariables, [
             'contentReleaseId' => $contentReleaseId,
             'currentContentReleaseId' => $currentContentReleaseId ?: self::NO_PREVIOUS_RELEASE,
             'validate' => $validate,
             'workspaceName' => $workspace ? $workspace->getName() : 'live',
+            'flushContentCache' => true,
         ]));
         return $contentReleaseId;
     }

Classes/Controller/BackendController.php

@@ -114,6 +114,7 @@ public function detailsAction(string $contentReleaseIdentifier, ?string $content
         $contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);
         $contentStore = $contentStore ? RedisInstanceIdentifier::fromString($contentStore) : RedisInstanceIdentifier::primary();
 
+        $this->view->assign('contentReleaseIdentifier', $contentReleaseIdentifier);
         $this->view->assign('contentStore', $contentStore->getIdentifier());
 
         $detailsData = $this->backendUiDataService->loadDetailsData($contentReleaseIdentifier, $contentStore);

Classes/Core/ConcurrentBuildLockService.php

@@ -4,10 +4,11 @@
 
 use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
 use Flowpack\DecoupledContentStore\Core\Infrastructure\RedisClientManager;
+use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
 use Neos\Flow\Annotations as Flow;
 
 /**
- * We usually rely on prunner to ensure that only one build is running at any given time.
+ * We usually rely on prunner to ensure that only one build per workspace is running at any given time.
  *
  * However, when running in a cloud environment with no shared storage, the prunner data folder is not shared between
  * instances. In this case, during a deployment, two containers run concurrently, with two separate prunner instances
@@ -27,35 +28,43 @@
  */
 class ConcurrentBuildLockService
 {
+    private const CONTENT_STORE_CONCURRENT_BUILD_LOCK = 'contentStore:concurrentBuildLocks';
 
     /**
      * @Flow\Inject
      * @var RedisClientManager
      */
     protected $redisClientManager;
 
-    public function ensureAllOtherInProgressContentReleasesWillBeTerminated(ContentReleaseIdentifier $contentReleaseIdentifier)
+    /**
+     * @Flow\Inject
+     * @var RedisContentReleaseService
+     */
+    protected $redisContentReleaseService;
+
+    public function ensureAllOtherInProgressContentReleasesWillBeTerminated(ContentReleaseIdentifier $contentReleaseIdentifier): void
     {
-        $this->redisClientManager->getPrimaryRedis()->set('contentStore:concurrentBuildLock', $contentReleaseIdentifier->getIdentifier());
+        $metadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier);
+        $this->redisClientManager->getPrimaryRedis()->hSet(self::CONTENT_STORE_CONCURRENT_BUILD_LOCK, $metadata->getWorkspaceName(), (string)$contentReleaseIdentifier);
     }
 
-    public function assertNoOtherContentReleaseWasStarted(ContentReleaseIdentifier $contentReleaseIdentifier)
+    public function assertNoOtherContentReleaseWasStarted(ContentReleaseIdentifier $contentReleaseIdentifier): void
     {
-        $concurrentBuildLockString = $this->redisClientManager->getPrimaryRedis()->get('contentStore:concurrentBuildLock');
+        $metadata = $this->redisContentReleaseService->fetchMetadataForContentRelease($contentReleaseIdentifier);
+        $concurrentBuildLockStrings = $this->redisClientManager->getPrimaryRedis()->hGetAll(self::CONTENT_STORE_CONCURRENT_BUILD_LOCK);
+        $concurrentBuildLockStringForWorkspace = $concurrentBuildLockStrings[$metadata->getWorkspaceName()] ?? null;
 
-        if (empty($concurrentBuildLockString)) {
+        if (!$concurrentBuildLockStringForWorkspace) {
             echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock does not exist.' . "\n\n";
             echo "This should never happen for correctly configured jobs (that run after prepare_finished).\n\n";
             exit(1);
         }
-
-        $concurrentBuildLock = ContentReleaseIdentifier::fromString($concurrentBuildLockString);
 
+        $concurrentBuildLock = ContentReleaseIdentifier::fromString($concurrentBuildLockStringForWorkspace);
         if (!$contentReleaseIdentifier->equals($concurrentBuildLock)) {
             // the concurrent build lock is different (i.e. newer) than our currently-running content release.
             // Thus, we abort the in-progress content release as quickly as we can - by DYING.
-
-            echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock contains ' . $concurrentBuildLock->getIdentifier() . "\n\n";
+            echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock for workspace "' . $metadata->getWorkspaceName() . '" contains ' . $concurrentBuildLock->getIdentifier() . "\n\n";
             echo "This is no error during deployment, but should never happen outside a deployment.\n\n It can only happen if two prunner instances run concurrently.\n\n";
             exit(1);
         }

Classes/Core/Domain/Dto/ContentReleaseBatchResult.php

@@ -3,6 +3,7 @@
 namespace Flowpack\DecoupledContentStore\Core\Domain\Dto;
 
 use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
+use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata;
 use Neos\Flow\Annotations as Flow;
 
 /**
@@ -20,15 +21,14 @@ private function __construct(array $results)
         $this->results = $results;
     }
 
-
     public static function createFromArray(array $in): self
     {
         return new self($in);
     }
 
     public function getResultForContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier)
     {
-        return $this->results[$contentReleaseIdentifier->jsonSerialize()];
+        return $this->results[(string)$contentReleaseIdentifier] ?? null;
     }
 
-}
+}