From 8638de87f735565d41ea7d7262ef637cf5ec7db8 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:36:44 +0200 Subject: [PATCH 01/28] TASK: Remove `removed` state from ContentStream model and database Followup to https://github.com/neos/neos-development-collection/commit/041f23926f41fa0ad2dad6e62f16efb45aa2e836 By diffing `findAllContentStreamEventNames` with the `$transitiveUsedStreams` we can remove the `removed` flag from content streams, which would just introduce to much complexity. --- .../src/ContentRepositoryReadModelAdapter.php | 7 ++- .../DoctrineDbalContentGraphSchemaBuilder.php | 1 - .../Projection/Feature/ContentStream.php | 4 +- .../Feature/ContentStreamEventStreamName.php | 2 +- .../Classes/Service/ContentStreamPruner.php | 46 +++++++++++++------ .../SharedModel/Workspace/ContentStream.php | 1 - .../Features/Bootstrap/CRTestSuiteTrait.php | 3 +- .../ContentStreamCommandController.php | 6 +-- 8 files changed, 41 insertions(+), 29 deletions(-) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentRepositoryReadModelAdapter.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentRepositoryReadModelAdapter.php index 56ad85a9d63..afad0f1022a 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentRepositoryReadModelAdapter.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentRepositoryReadModelAdapter.php @@ -95,7 +95,7 @@ public function findContentStreamById(ContentStreamId $contentStreamId): ?Conten { $contentStreamByIdStatement = <<tableNames->contentStream()} WHERE @@ -119,7 +119,7 @@ public function findContentStreams(): ContentStreams { $contentStreamsStatement = <<tableNames->contentStream()} SQL; @@ -153,8 +153,7 @@ private static function contentStreamFromDatabaseRow(array $row): ContentStream ContentStreamId::fromString($row['id']), isset($row['sourceContentStreamId']) ? ContentStreamId::fromString($row['sourceContentStreamId']) : null, ContentStreamStatus::from($row['status']), - Version::fromInteger((int)$row['version']), - (bool)$row['removed'] + Version::fromInteger((int)$row['version']) ); } } diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php index e10b4f16c62..9d7e66a9185 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php @@ -129,7 +129,6 @@ private function createContentStreamTable(): Table DbalSchemaFactory::columnForContentStreamId('sourceContentStreamId')->setNotnull(false), // Should become a DB ENUM (unclear how to configure with DBAL) or int (latter needs adaption to code) (new Column('status', Type::getType(Types::BINARY)))->setLength(20)->setNotnull(true), - (new Column('removed', Type::getType(Types::BOOLEAN)))->setDefault(false)->setNotnull(false) ]); } diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Projection/Feature/ContentStream.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Projection/Feature/ContentStream.php index dfe1f7f46d5..c0b3c180292 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Projection/Feature/ContentStream.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Projection/Feature/ContentStream.php @@ -36,9 +36,7 @@ private function updateContentStreamStatus(ContentStreamId $contentStreamId, Con private function removeContentStream(ContentStreamId $contentStreamId): void { - $this->dbal->update($this->tableNames->contentStream(), [ - 'removed' => true, - ], [ + $this->dbal->delete($this->tableNames->contentStream(), [ 'id' => $contentStreamId->value ]); } diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php index 5e3a7acad57..c0cdbb0a4f3 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php +++ b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php @@ -24,7 +24,7 @@ */ final readonly class ContentStreamEventStreamName { - private const EVENT_STREAM_NAME_PREFIX = 'ContentStream:'; + public const EVENT_STREAM_NAME_PREFIX = 'ContentStream:'; private function __construct( public string $value diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 15d7667ae5e..3ca21a9b9ca 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -10,9 +10,10 @@ use Neos\ContentRepository\Core\Feature\ContentStreamRemoval\Command\RemoveContentStream; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStream; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreams; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\EventStore\EventStoreInterface; +use Neos\EventStore\Model\Event\StreamName; +use Neos\EventStore\Model\EventStream\VirtualStreamName; /** * For implementation details of the content stream states and removed state, see {@see ContentStream}. @@ -73,15 +74,13 @@ public function prune(bool $removeTemporary = false): iterable * * - Otherwise, we cannot replay the other content streams correctly (if the base content streams are missing). * - * @return ContentStreams the removed content streams + * @return list the removed content streams */ - public function pruneRemovedFromEventStream(): ContentStreams + public function pruneRemovedFromEventStream(): array { - $removedContentStreams = $this->findUnusedAndRemovedContentStreams(); - foreach ($removedContentStreams as $removedContentStream) { - $streamName = ContentStreamEventStreamName::fromContentStreamId($removedContentStream->id) - ->getEventStreamName(); - $this->eventStore->deleteStream($streamName); + $removedContentStreams = $this->findUnusedAndRemovedContentStreamIds(); + foreach ($removedContentStreams as $removedContentStreamName) { + $this->eventStore->deleteStream($removedContentStreamName); } return $removedContentStreams; } @@ -94,7 +93,10 @@ public function pruneAll(): void } } - private function findUnusedAndRemovedContentStreams(): ContentStreams + /** + * @return list + */ + private function findUnusedAndRemovedContentStreamIds(): array { $allContentStreams = $this->contentRepository->findContentStreams(); @@ -105,7 +107,7 @@ private function findUnusedAndRemovedContentStreams(): ContentStreams // Step 1: Find all content streams currently in direct use by a workspace foreach ($allContentStreams as $stream) { - if ($stream->status === ContentStreamStatus::IN_USE_BY_WORKSPACE && !$stream->removed) { + if ($stream->status === ContentStreamStatus::IN_USE_BY_WORKSPACE) { $contentStreamIdsStack[] = $stream->id; } } @@ -129,13 +131,29 @@ private function findUnusedAndRemovedContentStreams(): ContentStreams } // Step 3: Check for removed content streams which we do not need anymore transitively + $allContentStreamEventStreamNames = $this->findAllContentStreamEventNames(); + $removedContentStreams = []; - foreach ($allContentStreams as $contentStream) { - if ($contentStream->removed && !array_key_exists($contentStream->id->value, $transitiveUsedStreams)) { - $removedContentStreams[] = $contentStream; + foreach ($allContentStreamEventStreamNames as $streamName) { + $removedContentStream = substr($streamName->value, strlen(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX)); + if (!array_key_exists($removedContentStream, $transitiveUsedStreams)) { + $removedContentStreams[] = $streamName; } } - return ContentStreams::fromArray($removedContentStreams); + return $removedContentStreams; + } + + /** + * @return list + */ + private function findAllContentStreamEventNames(): array + { + $events = $this->eventStore->load(VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX)); + $allContentStreamEventStreamNames = []; + foreach ($events as $eventEnvelope) { + $allContentStreamEventStreamNames[$eventEnvelope->streamName->value] = true; + } + return array_map(StreamName::fromString(...), array_keys($allContentStreamEventStreamNames)); } } diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php index def543c48b0..38b79f67255 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php @@ -79,7 +79,6 @@ public function __construct( public ?ContentStreamId $sourceContentStreamId, public ContentStreamStatus $status, public Version $version, - public bool $removed ) { } } diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index ab550c6bd35..6dbe16daf40 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -135,8 +135,7 @@ protected function readPayloadTable(TableNode $payloadTable): array public function iExpectTheContentStreamToNotExist(string $rawContentStreamId): void { $contentStream = $this->currentContentRepository->findContentStreamById(ContentStreamId::fromString($rawContentStreamId)); - $contentStreamExists = $contentStream !== null && !$contentStream->removed; - Assert::assertFalse($contentStreamExists, sprintf('Content stream "%s" was not expected to exist, but it does', $rawContentStreamId)); + Assert::assertNull($contentStream, sprintf('Content stream "%s" was not expected to exist, but it does', $rawContentStreamId)); } /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index 40d4330c168..9e1fe40d3b5 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -60,10 +60,10 @@ public function pruneRemovedFromEventStreamCommand(string $contentRepository = ' $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); - $unusedContentStreams = $contentStreamPruner->pruneRemovedFromEventStream(); + $unusedContentStreamNames = $contentStreamPruner->pruneRemovedFromEventStream(); $unusedContentStreamsPresent = false; - foreach ($unusedContentStreams as $contentStreamId) { - $this->outputFormatted('Removed events for %s', [$contentStreamId->id->value]); + foreach ($unusedContentStreamNames as $contentStreamName) { + $this->outputFormatted('Removed events for %s', [$contentStreamName->value]); $unusedContentStreamsPresent = true; } if (!$unusedContentStreamsPresent) { From 0d0ffa5b2dcb2c0b3420634c6a66d343c91624b2 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:37:38 +0200 Subject: [PATCH 02/28] BUGFIX: 4913 make `pruneAll` independent of projection --- .../Classes/Service/ContentStreamPruner.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 3ca21a9b9ca..5c8de669518 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -87,8 +87,7 @@ public function pruneRemovedFromEventStream(): array public function pruneAll(): void { - foreach ($this->contentRepository->findContentStreams() as $contentStream) { - $streamName = ContentStreamEventStreamName::fromContentStreamId($contentStream->id)->getEventStreamName(); + foreach ($this->findAllContentStreamEventNames() as $streamName) { $this->eventStore->deleteStream($streamName); } } From 58581c82822a212837227cfbd2753b989f9b54a9 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:38:49 +0200 Subject: [PATCH 03/28] BUGFIX: Also remove closed content streams in prune --- .../Classes/Service/ContentStreamPruner.php | 1 + 1 file changed, 1 insertion(+) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 5c8de669518..6ea45d8c380 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -50,6 +50,7 @@ public function prune(bool $removeTemporary = false): iterable if ($removeTemporary) { $status[] = ContentStreamStatus::CREATED; $status[] = ContentStreamStatus::FORKED; + $status[] = ContentStreamStatus::CLOSED; } $unusedContentStreams = $this->contentRepository->findContentStreams()->filter( static fn (ContentStream $contentStream) => in_array($contentStream->status, $status, true), From 1750160b73d829d672fc5c3473a4629416c3a562 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 17 Oct 2024 23:33:08 +0200 Subject: [PATCH 04/28] TASK: Remove obsolete `REBASE_ERROR` content stream state Via #4965 the status REBASE_ERROR is obsolete. Instead of still using this status on replay we mimic the new logic: > In case of a [rebase error}, reopen the old content stream and remove the newly created Additionally, to not break `findContentStreams` when fetching all content streams we make sure that the streams with `REBASE_ERROR` are transformed to `NO_LONGER_IN_USE` so they can be cleaned up. Related https://github.com/neos/neos-development-collection/issues/5101 --- .../src/DoctrineDbalContentGraphProjection.php | 14 ++++++++++++-- .../Classes/Service/ContentStreamPruner.php | 6 +++--- .../SharedModel/Workspace/ContentStreamStatus.php | 8 -------- .../Command/ContentStreamCommandController.php | 2 +- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index ecd410b0a07..b6567a69d26 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -125,7 +125,10 @@ public function setUp(): void $this->dbal->getSchemaManager()->tablesExist([$legacyContentStreamTableName]) && !$this->dbal->getSchemaManager()->tablesExist([$this->tableNames->contentStream()]) ) { - $statements[] = 'INSERT INTO ' . $this->tableNames->contentStream() . ' (id, version, sourceContentStreamId, status, removed) SELECT contentStreamId AS id, version, sourceContentStreamId, state AS status, removed FROM ' . $legacyContentStreamTableName; + // special migration regarding content stream table: + // 1.) don't copy removed content streams as they would not be flagged but actually removed + // 2.) transform legacy `REBASE_ERROR` content streams into `NO_LONGER_IN_USE`, as they should be removed and wouldn't exist today as well. (We cannot reopen the content stream that was attempted to be rebased like we would today in the migration here, for that a replay must be executed see whenWorkspaceRebaseFailed) + $statements[] = 'INSERT INTO ' . $this->tableNames->contentStream() . ' (id, version, sourceContentStreamId, status) SELECT contentStreamId AS id, version, sourceContentStreamId, REPLACE(state, \'REBASE_ERROR\', \'NO_LONGER_IN_USE\') AS status FROM ' . $legacyContentStreamTableName . ' WHERE NOT removed = 1'; } // /MIGRATION @@ -733,8 +736,15 @@ private function whenWorkspaceBaseWorkspaceWasChanged(WorkspaceBaseWorkspaceWasC private function whenWorkspaceRebaseFailed(WorkspaceRebaseFailed $event): void { + // legacy handling: + // before https://github.com/neos/neos-development-collection/pull/4965 this event was emitted and set the content stream status to `REBASE_ERROR` + // instead of setting the error state on replay for old events we make it behave like if the rebase had failed today: + // 4.E) In case of a [rebase error}, reopen the old content stream and remove the newly created + // The ContentStreamWasReopened event would be emitted with `IN_USE_BY_WORKSPACE` (because that _must_ be the previous state as a workspace was rebased) + $this->whenContentStreamWasReopened(new ContentStreamWasReopened($event->sourceContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE)); + $this->whenContentStreamWasRemoved(new ContentStreamWasRemoved($event->candidateContentStreamId)); + $this->markWorkspaceAsOutdatedConflict($event->workspaceName); - $this->updateContentStreamStatus($event->candidateContentStreamId, ContentStreamStatus::REBASE_ERROR); } private function whenWorkspaceWasCreated(WorkspaceWasCreated $event): void diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 6ea45d8c380..159cc729009 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -37,16 +37,16 @@ public function __construct( * To remove the deleted Content Streams, * call {@see ContentStreamPruner::pruneRemovedFromEventStream()} afterwards. * - * By default, only content streams in STATE_NO_LONGER_IN_USE and STATE_REBASE_ERROR will be removed. + * By default, only content streams that are NO_LONGER_IN_USE will be removed. * If you also call with $removeTemporary=true, will delete ALL content streams which are currently not assigned - * to a workspace (f.e. dangling ones in FORKED or CREATED.). + * to a workspace (f.e. dangling ones in FORKED, CLOSED or CREATED.). * * @param bool $removeTemporary if TRUE, will delete ALL content streams not bound to a workspace * @return iterable the identifiers of the removed content streams */ public function prune(bool $removeTemporary = false): iterable { - $status = [ContentStreamStatus::NO_LONGER_IN_USE, ContentStreamStatus::REBASE_ERROR]; + $status = [ContentStreamStatus::NO_LONGER_IN_USE]; if ($removeTemporary) { $status[] = ContentStreamStatus::CREATED; $status[] = ContentStreamStatus::FORKED; diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStreamStatus.php b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStreamStatus.php index ac21aeaa4cc..1bdb87e45a4 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStreamStatus.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStreamStatus.php @@ -39,14 +39,6 @@ enum ContentStreamStatus: string implements \JsonSerializable */ case IN_USE_BY_WORKSPACE = 'IN_USE_BY_WORKSPACE'; - /** - * a workspace was tried to be rebased, and during the rebase an error occured. This is the content stream - * which contains the errored state - so that we can recover content from it (probably manually) - * - * @deprecated legacy status, FIXME clean up! https://github.com/neos/neos-development-collection/issues/5101 - */ - case REBASE_ERROR = 'REBASE_ERROR'; - /** * the content stream was closed and must no longer accept new events */ diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index 9e1fe40d3b5..c36226b87b8 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -27,7 +27,7 @@ class ContentStreamCommandController extends CommandController * To remove the deleted Content Streams, use `./flow contentStream:pruneRemovedFromEventStream` after running * `./flow contentStream:prune`. * - * By default, only content streams in STATE_NO_LONGER_IN_USE and STATE_REBASE_ERROR will be removed. + * By default, only content streams that are NO_LONGER_IN_USE will be removed. * If you also call with "--removeTemporary", will delete ALL content streams which are currently not assigned * to a workspace (f.e. dangling ones in FORKED or CREATED.). * From 6989b34f54b9c5b6616df8db958e0327bd1a650b Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:13:06 +0200 Subject: [PATCH 05/28] TASK: Use `ContentStreamWasRemoved` event instead of `RemoveContentStream` the command will be removed in the future --- .../Classes/Service/ContentStreamPruner.php | 21 ++++++++++++++++--- .../Service/ContentStreamPrunerFactory.php | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 159cc729009..be437941b39 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -5,14 +5,18 @@ namespace Neos\ContentRepository\Core\Service; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\EventStore\EventPersister; +use Neos\ContentRepository\Core\EventStore\Events; +use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; -use Neos\ContentRepository\Core\Feature\ContentStreamRemoval\Command\RemoveContentStream; +use Neos\ContentRepository\Core\Feature\ContentStreamRemoval\Event\ContentStreamWasRemoved; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStream; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\StreamName; +use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\EventStore\Model\EventStream\VirtualStreamName; /** @@ -25,6 +29,7 @@ class ContentStreamPruner implements ContentRepositoryServiceInterface public function __construct( private readonly ContentRepository $contentRepository, private readonly EventStoreInterface $eventStore, + private readonly EventPersister $eventPersister, ) { } @@ -57,9 +62,19 @@ public function prune(bool $removeTemporary = false): iterable ); $unusedContentStreamIds = []; foreach ($unusedContentStreams as $contentStream) { - $this->contentRepository->handle( - RemoveContentStream::create($contentStream->id) + $removeContentStream = new EventsToPublish( + ContentStreamEventStreamName::fromContentStreamId($contentStream->id)->getEventStreamName(), + Events::with(new ContentStreamWasRemoved( + $contentStream->id + )), + ExpectedVersion::fromVersion($contentStream->version) ); + + $this->eventPersister->publishEvents( + $this->contentRepository, + $removeContentStream + ); + $unusedContentStreamIds[] = $contentStream->id; } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php index 66acf840a9e..ac7912e9bdc 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php @@ -18,6 +18,7 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor return new ContentStreamPruner( $serviceFactoryDependencies->contentRepository, $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->eventPersister ); } } From cacc70c7ef9e1c23a0e30c408eb1a5ea332470f7 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:41:54 +0200 Subject: [PATCH 06/28] TASK: Build content stream for pruning state by reading the event stream Otherwise, its not possible to prune if the projections are out of date or broken --- .../Feature/WorkspaceEventStreamName.php | 2 +- .../Classes/Service/ContentStreamPruner.php | 168 ++++++++++++++++-- .../ContentStreamForPruning.php | 55 ++++++ .../Service/ContentStreamPrunerFactory.php | 3 +- 4 files changed, 212 insertions(+), 16 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php diff --git a/Neos.ContentRepository.Core/Classes/Feature/WorkspaceEventStreamName.php b/Neos.ContentRepository.Core/Classes/Feature/WorkspaceEventStreamName.php index 8d873a2a96a..1a14bf70895 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/WorkspaceEventStreamName.php +++ b/Neos.ContentRepository.Core/Classes/Feature/WorkspaceEventStreamName.php @@ -14,7 +14,7 @@ */ final readonly class WorkspaceEventStreamName { - private const EVENT_STREAM_NAME_PREFIX = 'Workspace:'; + public const EVENT_STREAM_NAME_PREFIX = 'Workspace:'; private function __construct( public string $eventStreamName, diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index be437941b39..af651b070c4 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -5,17 +5,30 @@ namespace Neos\ContentRepository\Core\Service; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\EventStore\Events; use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; +use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; +use Neos\ContentRepository\Core\Feature\ContentStreamForking\Event\ContentStreamWasForked; use Neos\ContentRepository\Core\Feature\ContentStreamRemoval\Event\ContentStreamWasRemoved; +use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Event\WorkspaceWasCreated; +use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; +use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasDiscarded; +use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyDiscarded; +use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyPublished; +use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; +use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamForPruning; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStream; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\EventStore\EventStoreInterface; +use Neos\EventStore\Model\Event\EventType; +use Neos\EventStore\Model\Event\EventTypes; use Neos\EventStore\Model\Event\StreamName; +use Neos\EventStore\Model\EventStream\EventStreamFilter; use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\EventStore\Model\EventStream\VirtualStreamName; @@ -30,6 +43,7 @@ public function __construct( private readonly ContentRepository $contentRepository, private readonly EventStoreInterface $eventStore, private readonly EventPersister $eventPersister, + private readonly EventNormalizer $eventNormalizer ) { } @@ -95,8 +109,12 @@ public function prune(bool $removeTemporary = false): iterable public function pruneRemovedFromEventStream(): array { $removedContentStreams = $this->findUnusedAndRemovedContentStreamIds(); - foreach ($removedContentStreams as $removedContentStreamName) { - $this->eventStore->deleteStream($removedContentStreamName); + foreach ($removedContentStreams as $removedContentStream) { + $this->eventStore->deleteStream( + ContentStreamEventStreamName::fromContentStreamId( + $removedContentStream + )->getEventStreamName() + ); } return $removedContentStreams; } @@ -109,11 +127,11 @@ public function pruneAll(): void } /** - * @return list + * @return list */ private function findUnusedAndRemovedContentStreamIds(): array { - $allContentStreams = $this->contentRepository->findContentStreams(); + $allContentStreams = $this->getContentStreamsForPruning(); /** @var array $transitiveUsedStreams */ $transitiveUsedStreams = []; @@ -122,8 +140,8 @@ private function findUnusedAndRemovedContentStreamIds(): array // Step 1: Find all content streams currently in direct use by a workspace foreach ($allContentStreams as $stream) { - if ($stream->status === ContentStreamStatus::IN_USE_BY_WORKSPACE) { - $contentStreamIdsStack[] = $stream->id; + if ($stream->status === ContentStreamStatus::IN_USE_BY_WORKSPACE && !$stream->removed) { + $contentStreamIdsStack[] = $stream->contentStreamId; } } @@ -135,7 +153,7 @@ private function findUnusedAndRemovedContentStreamIds(): array // Find source content streams for the current stream foreach ($allContentStreams as $stream) { - if ($stream->id === $currentStreamId && $stream->sourceContentStreamId !== null) { + if ($stream->contentStreamId === $currentStreamId && $stream->sourceContentStreamId !== null) { $sourceStreamId = $stream->sourceContentStreamId; if (!array_key_exists($sourceStreamId->value, $transitiveUsedStreams)) { $contentStreamIdsStack[] = $sourceStreamId; @@ -146,25 +164,147 @@ private function findUnusedAndRemovedContentStreamIds(): array } // Step 3: Check for removed content streams which we do not need anymore transitively - $allContentStreamEventStreamNames = $this->findAllContentStreamEventNames(); - $removedContentStreams = []; - foreach ($allContentStreamEventStreamNames as $streamName) { - $removedContentStream = substr($streamName->value, strlen(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX)); - if (!array_key_exists($removedContentStream, $transitiveUsedStreams)) { - $removedContentStreams[] = $streamName; + foreach ($allContentStreams as $contentStream) { + if ($contentStream->removed && !array_key_exists($contentStream->contentStreamId->value, $transitiveUsedStreams)) { + $removedContentStreams[] = $contentStream->contentStreamId; } } return $removedContentStreams; } + /** + * @return array + */ + private function getContentStreamsForPruning(): array + { + $events = $this->eventStore->load( + VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + EventType::fromString('ContentStreamWasCreated'), + EventType::fromString('ContentStreamWasForked'), + EventType::fromString('ContentStreamWasRemoved'), + ) + ) + ); + + /** @var array $status */ + $status = []; + foreach ($events as $eventEnvelope) { + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + + switch ($domainEvent::class) { + case ContentStreamWasCreated::class: + $status[$domainEvent->contentStreamId->value] = ContentStreamForPruning::create( + $domainEvent->contentStreamId, + ContentStreamStatus::CREATED, + null + ); break; + case ContentStreamWasForked::class: + $status[$domainEvent->newContentStreamId->value] = ContentStreamForPruning::create( + $domainEvent->newContentStreamId, + ContentStreamStatus::FORKED, + $domainEvent->sourceContentStreamId + ); + break; + case ContentStreamWasRemoved::class: + if (isset($status[$domainEvent->contentStreamId->value])) { + $status[$domainEvent->contentStreamId->value] = $status[$domainEvent->contentStreamId->value] + ->withRemoved(); + } + break; + default: + throw new \RuntimeException(sprintf('Unhandled event %s', $eventEnvelope->event->type->value)); + } + } + + $workspaceEvents = $this->eventStore->load( + VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + EventType::fromString('WorkspaceWasCreated'), + EventType::fromString('WorkspaceWasDiscarded'), + EventType::fromString('WorkspaceWasPartiallyDiscarded'), + EventType::fromString('WorkspaceWasPartiallyPublished'), + EventType::fromString('WorkspaceWasPublished'), + EventType::fromString('WorkspaceWasRebased'), + // we don't need to track WorkspaceWasRemoved as a ContentStreamWasRemoved event would be emitted before + ) + ) + ); + foreach ($workspaceEvents as $eventEnvelope) { + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + + switch ($domainEvent::class) { + case WorkspaceWasCreated::class: + if (isset($status[$domainEvent->newContentStreamId->value])) { + $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); + } + break; + case WorkspaceWasDiscarded::class: + if (isset($status[$domainEvent->newContentStreamId->value])) { + $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); + } + if (isset($status[$domainEvent->previousContentStreamId->value])) { + $status[$domainEvent->previousContentStreamId->value] = $status[$domainEvent->previousContentStreamId->value] + ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); + } + break; + case WorkspaceWasPartiallyDiscarded::class: + if (isset($status[$domainEvent->newContentStreamId->value])) { + $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); + } + if (isset($status[$domainEvent->previousContentStreamId->value])) { + $status[$domainEvent->previousContentStreamId->value] = $status[$domainEvent->previousContentStreamId->value] + ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); + } + break; + case WorkspaceWasPartiallyPublished::class: + if (isset($status[$domainEvent->newSourceContentStreamId->value])) { + $status[$domainEvent->newSourceContentStreamId->value] = $status[$domainEvent->newSourceContentStreamId->value] + ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); + } + if (isset($status[$domainEvent->previousSourceContentStreamId->value])) { + $status[$domainEvent->previousSourceContentStreamId->value] = $status[$domainEvent->previousSourceContentStreamId->value] + ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); + } + break; + case WorkspaceWasRebased::class: + if (isset($status[$domainEvent->newContentStreamId->value])) { + $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); + } + if (isset($status[$domainEvent->previousContentStreamId->value])) { + $status[$domainEvent->previousContentStreamId->value] = $status[$domainEvent->previousContentStreamId->value] + ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); + } + break; + default: + throw new \RuntimeException(sprintf('Unhandled event %s', $eventEnvelope->event->type->value)); + } + } + return $status; + } + /** * @return list */ private function findAllContentStreamEventNames(): array { - $events = $this->eventStore->load(VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX)); + $events = $this->eventStore->load( + VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + EventType::fromString('ContentStreamWasCreated'), + EventType::fromString('ContentStreamWasForked') + ) + ) + ); $allContentStreamEventStreamNames = []; foreach ($events as $eventEnvelope) { $allContentStreamEventStreamNames[$eventEnvelope->streamName->value] = true; diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php new file mode 100644 index 00000000000..8d8bdb07acc --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php @@ -0,0 +1,55 @@ +contentStreamId, + $status, + $this->sourceContentStreamId, + $this->removed + ); + } + + public function withRemoved(): self + { + return new self( + $this->contentStreamId, + $this->status, + $this->sourceContentStreamId, + true + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php index ac7912e9bdc..c29071e5d79 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php @@ -18,7 +18,8 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor return new ContentStreamPruner( $serviceFactoryDependencies->contentRepository, $serviceFactoryDependencies->eventStore, - $serviceFactoryDependencies->eventPersister + $serviceFactoryDependencies->eventPersister, + $serviceFactoryDependencies->eventNormalizer ); } } From 7aa6b6c4a72530e2e09b6e685702ad27320f2dd9 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:55:16 +0200 Subject: [PATCH 07/28] TASK: Harden `contentstream:prune` to work on event stream and catch failed catchup --- .../Classes/Service/ContentStreamPruner.php | 61 ++++++++++--------- .../ContentStreamForPruning.php | 10 +-- .../Service/ContentStreamPrunerFactory.php | 1 - .../Features/Bootstrap/CRTestSuiteTrait.php | 2 +- .../ContentStreamCommandController.php | 13 ++-- 5 files changed, 43 insertions(+), 44 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index af651b070c4..92bb17b2b36 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -6,9 +6,6 @@ use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; -use Neos\ContentRepository\Core\EventStore\Events; -use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; @@ -21,7 +18,6 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyPublished; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamForPruning; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStream; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\EventStore\EventStoreInterface; @@ -33,7 +29,7 @@ use Neos\EventStore\Model\EventStream\VirtualStreamName; /** - * For implementation details of the content stream states and removed state, see {@see ContentStream}. + * For implementation details of the content stream states and removed state, see {@see ContentStreamForPruning}. * * @api */ @@ -42,7 +38,6 @@ class ContentStreamPruner implements ContentRepositoryServiceInterface public function __construct( private readonly ContentRepository $contentRepository, private readonly EventStoreInterface $eventStore, - private readonly EventPersister $eventPersister, private readonly EventNormalizer $eventNormalizer ) { } @@ -61,9 +56,8 @@ public function __construct( * to a workspace (f.e. dangling ones in FORKED, CLOSED or CREATED.). * * @param bool $removeTemporary if TRUE, will delete ALL content streams not bound to a workspace - * @return iterable the identifiers of the removed content streams */ - public function prune(bool $removeTemporary = false): iterable + public function prune(bool $removeTemporary, \Closure $outputFn): void { $status = [ContentStreamStatus::NO_LONGER_IN_USE]; if ($removeTemporary) { @@ -71,28 +65,39 @@ public function prune(bool $removeTemporary = false): iterable $status[] = ContentStreamStatus::FORKED; $status[] = ContentStreamStatus::CLOSED; } - $unusedContentStreams = $this->contentRepository->findContentStreams()->filter( - static fn (ContentStream $contentStream) => in_array($contentStream->status, $status, true), - ); - $unusedContentStreamIds = []; - foreach ($unusedContentStreams as $contentStream) { - $removeContentStream = new EventsToPublish( + + $allContentStreams = $this->getContentStreamsForPruning(); + + $unusedContentStreamsPresent = false; + foreach ($allContentStreams as $contentStream) { + if (!in_array($contentStream->status, $status, true)) { + continue; + } + + $this->eventStore->commit( ContentStreamEventStreamName::fromContentStreamId($contentStream->id)->getEventStreamName(), - Events::with(new ContentStreamWasRemoved( - $contentStream->id - )), - ExpectedVersion::fromVersion($contentStream->version) + $this->eventNormalizer->normalize( + new ContentStreamWasRemoved( + $contentStream->id + ) + ), + ExpectedVersion::STREAM_EXISTS() ); - $this->eventPersister->publishEvents( - $this->contentRepository, - $removeContentStream - ); + $outputFn(sprintf('Removed %s', $contentStream->id)); - $unusedContentStreamIds[] = $contentStream->id; + $unusedContentStreamsPresent = true; } - return $unusedContentStreamIds; + if ($unusedContentStreamsPresent) { + try { + $this->contentRepository->catchUpProjections(); + } catch (\Exception $e) { + $outputFn(sprintf('Could not catchup after removing unused content streams: %s. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.', $e->getMessage())); + } + } else { + $outputFn('There are no unused content streams.'); + } } /** @@ -141,7 +146,7 @@ private function findUnusedAndRemovedContentStreamIds(): array // Step 1: Find all content streams currently in direct use by a workspace foreach ($allContentStreams as $stream) { if ($stream->status === ContentStreamStatus::IN_USE_BY_WORKSPACE && !$stream->removed) { - $contentStreamIdsStack[] = $stream->contentStreamId; + $contentStreamIdsStack[] = $stream->id; } } @@ -153,7 +158,7 @@ private function findUnusedAndRemovedContentStreamIds(): array // Find source content streams for the current stream foreach ($allContentStreams as $stream) { - if ($stream->contentStreamId === $currentStreamId && $stream->sourceContentStreamId !== null) { + if ($stream->id === $currentStreamId && $stream->sourceContentStreamId !== null) { $sourceStreamId = $stream->sourceContentStreamId; if (!array_key_exists($sourceStreamId->value, $transitiveUsedStreams)) { $contentStreamIdsStack[] = $sourceStreamId; @@ -166,8 +171,8 @@ private function findUnusedAndRemovedContentStreamIds(): array // Step 3: Check for removed content streams which we do not need anymore transitively $removedContentStreams = []; foreach ($allContentStreams as $contentStream) { - if ($contentStream->removed && !array_key_exists($contentStream->contentStreamId->value, $transitiveUsedStreams)) { - $removedContentStreams[] = $contentStream->contentStreamId; + if ($contentStream->removed && !array_key_exists($contentStream->id->value, $transitiveUsedStreams)) { + $removedContentStreams[] = $contentStream->id; } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php index 8d8bdb07acc..0f405fe5b71 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php @@ -13,7 +13,7 @@ final readonly class ContentStreamForPruning { private function __construct( - public ContentStreamId $contentStreamId, + public ContentStreamId $id, public ContentStreamStatus $status, public ?ContentStreamId $sourceContentStreamId, public bool $removed, @@ -21,12 +21,12 @@ private function __construct( } public static function create( - ContentStreamId $contentStreamId, + ContentStreamId $id, ContentStreamStatus $status, ?ContentStreamId $sourceContentStreamId ): self { return new self( - $contentStreamId, + $id, $status, $sourceContentStreamId, false @@ -36,7 +36,7 @@ public static function create( public function withStatus(ContentStreamStatus $status): self { return new self( - $this->contentStreamId, + $this->id, $status, $this->sourceContentStreamId, $this->removed @@ -46,7 +46,7 @@ public function withStatus(ContentStreamStatus $status): self public function withRemoved(): self { return new self( - $this->contentStreamId, + $this->id, $this->status, $this->sourceContentStreamId, true diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php index c29071e5d79..f9940f8f56a 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php @@ -18,7 +18,6 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor return new ContentStreamPruner( $serviceFactoryDependencies->contentRepository, $serviceFactoryDependencies->eventStore, - $serviceFactoryDependencies->eventPersister, $serviceFactoryDependencies->eventNormalizer ); } diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index 29cc5deadb2..b93930b0f12 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -272,7 +272,7 @@ public function iPruneUnusedContentStreams(): void { /** @var ContentStreamPruner $contentStreamPruner */ $contentStreamPruner = $this->getContentRepositoryService(new ContentStreamPrunerFactory()); - $contentStreamPruner->prune(); + $contentStreamPruner->prune(false, fn () => null); } /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index c36226b87b8..866f5cc2274 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -39,15 +39,10 @@ public function pruneCommand(string $contentRepository = 'default', bool $remove $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); - $unusedContentStreams = $contentStreamPruner->prune($removeTemporary); - $unusedContentStreamsPresent = false; - foreach ($unusedContentStreams as $contentStreamId) { - $this->outputFormatted('Removed %s', [$contentStreamId->value]); - $unusedContentStreamsPresent = true; - } - if (!$unusedContentStreamsPresent) { - $this->outputLine('There are no unused content streams.'); - } + $contentStreamPruner->prune( + $removeTemporary, + $this->outputLine(...) + ); } /** From b8a93cfe19f6f6eec81efbb2401bca2eb3c4b35c Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:06:18 +0200 Subject: [PATCH 08/28] Linte is basically ente with i --- .../Classes/Service/ContentStreamPruner.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 92bb17b2b36..af5efdff588 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -109,7 +109,7 @@ public function prune(bool $removeTemporary, \Closure $outputFn): void * * - Otherwise, we cannot replay the other content streams correctly (if the base content streams are missing). * - * @return list the removed content streams + * @return list the removed content streams */ public function pruneRemovedFromEventStream(): array { @@ -206,7 +206,8 @@ private function getContentStreamsForPruning(): array $domainEvent->contentStreamId, ContentStreamStatus::CREATED, null - ); break; + ); + break; case ContentStreamWasForked::class: $status[$domainEvent->newContentStreamId->value] = ContentStreamForPruning::create( $domainEvent->newContentStreamId, From 733490e60d1497ec6f505d52af75cac242524d4d Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:16:42 +0200 Subject: [PATCH 09/28] TASK: Also handle `WorkspaceWasPublished` event --- .../Classes/Service/ContentStreamPruner.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index af5efdff588..6e5dd639ca9 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -16,6 +16,7 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasDiscarded; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyDiscarded; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyPublished; +use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPublished; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamForPruning; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; @@ -280,6 +281,16 @@ private function getContentStreamsForPruning(): array ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); } break; + case WorkspaceWasPublished::class: + if (isset($status[$domainEvent->newSourceContentStreamId->value])) { + $status[$domainEvent->newSourceContentStreamId->value] = $status[$domainEvent->newSourceContentStreamId->value] + ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); + } + if (isset($status[$domainEvent->previousSourceContentStreamId->value])) { + $status[$domainEvent->previousSourceContentStreamId->value] = $status[$domainEvent->previousSourceContentStreamId->value] + ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); + } + break; case WorkspaceWasRebased::class: if (isset($status[$domainEvent->newContentStreamId->value])) { $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] From 3dec67f6cc6baa7d70133839c45637c474b9c0e6 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 24 Oct 2024 17:11:00 +0200 Subject: [PATCH 10/28] TASK: Prune also dangling streams of legacy `WorkspaceRebaseFailed` if existing --- .../src/DoctrineDbalContentGraphProjection.php | 8 +++----- .../Classes/Service/ContentStreamPruner.php | 9 +++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index 4c8f8459dda..3eb1819d01a 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -714,13 +714,11 @@ private function whenWorkspaceBaseWorkspaceWasChanged(WorkspaceBaseWorkspaceWasC private function whenWorkspaceRebaseFailed(WorkspaceRebaseFailed $event): void { - // legacy handling: todo + // legacy handling: // before https://github.com/neos/neos-development-collection/pull/4965 this event was emitted and set the content stream status to `REBASE_ERROR` - // instead of setting the error state on replay for old events we make it behave like if the rebase had failed today: - // 4.E) In case of a [rebase error}, reopen the old content stream and remove the newly created - // The ContentStreamWasReopened event would be emitted with `IN_USE_BY_WORKSPACE` (because that _must_ be the previous state as a workspace was rebased) + // instead of setting the error state on replay for old events we make it almost behave like if the rebase had failed today: reopen the workspaces content stream id + // the candidateContentStreamId will be removed by the ContentStreamPruner $this->whenContentStreamWasReopened(new ContentStreamWasReopened($event->sourceContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE)); - $this->whenContentStreamWasRemoved(new ContentStreamWasRemoved($event->candidateContentStreamId)); } private function whenWorkspaceWasCreated(WorkspaceWasCreated $event): void diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 6e5dd639ca9..be6618ad8c8 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -17,6 +17,7 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyDiscarded; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyPublished; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPublished; +use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamForPruning; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; @@ -237,6 +238,7 @@ private function getContentStreamsForPruning(): array EventType::fromString('WorkspaceWasPartiallyPublished'), EventType::fromString('WorkspaceWasPublished'), EventType::fromString('WorkspaceWasRebased'), + EventType::fromString('WorkspaceRebaseFailed'), // we don't need to track WorkspaceWasRemoved as a ContentStreamWasRemoved event would be emitted before ) ) @@ -301,6 +303,13 @@ private function getContentStreamsForPruning(): array ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); } break; + case WorkspaceRebaseFailed::class: + // legacy handling, as we previously kept failed candidateContentStreamId we make it behave like a ContentStreamWasRemoved event to clean up: + if (isset($status[$domainEvent->candidateContentStreamId->value])) { + $status[$domainEvent->candidateContentStreamId->value] = $status[$domainEvent->candidateContentStreamId->value] + ->withRemoved(); + } + break; default: throw new \RuntimeException(sprintf('Unhandled event %s', $eventEnvelope->event->type->value)); } From b279a3802c992e7a7665f4cab38dc79f411a1f65 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 26 Oct 2024 20:05:49 +0200 Subject: [PATCH 11/28] TASK: Remove now obsolete content stream status from projection With the pruner now building up the state from scratch we dont need to project it any further. The only exception is the content stream state that wasnt one to begin with: the closed state. It will get its own column. --- .../src/ContentGraphReadModelAdapter.php | 11 ++- .../DoctrineDbalContentGraphProjection.php | 52 ++----------- .../DoctrineDbalContentGraphSchemaBuilder.php | 3 +- .../Projection/Feature/ContentStream.php | 19 +++-- .../Workspaces/PruneContentStreams.feature | 22 ++++-- .../CommandHandlingDependencies.php | 10 +-- .../Feature/Common/ConstraintChecks.php | 5 +- .../Command/ReopenContentStream.php | 15 +--- .../Event/ContentStreamWasReopened.php | 10 +-- .../Feature/ContentStreamCommandHandler.php | 4 +- .../Feature/ContentStreamEventStreamName.php | 9 +++ .../Classes/Feature/ContentStreamHandling.php | 10 +-- .../Feature/WorkspaceCommandHandler.php | 15 +--- .../Classes/Service/ContentStreamPruner.php | 11 ++- .../ContentStreamForPruning.php | 48 +++++++++++- .../ContentStreamStatus.monopic} | Bin .../ContentStreamStatus.php | 19 +---- .../SharedModel/Workspace/ContentStream.php | 72 +++++------------- .../Features/Bootstrap/CRTestSuiteTrait.php | 40 +++------- .../ContentStreamCommandController.php | 6 +- 20 files changed, 158 insertions(+), 223 deletions(-) rename Neos.ContentRepository.Core/Classes/{SharedModel/Workspace/ContentStream.monopic => Service/ContentStreamPruner/ContentStreamStatus.monopic} (100%) rename Neos.ContentRepository.Core/Classes/{SharedModel/Workspace => Service/ContentStreamPruner}/ContentStreamStatus.php (73%) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php index 9de6a5aa251..ee14a8239d5 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php @@ -26,7 +26,6 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStream; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreams; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspaces; @@ -104,7 +103,7 @@ public function findContentStreamById(ContentStreamId $contentStreamId): ?Conten { $contentStreamByIdStatement = <<tableNames->contentStream()} WHERE @@ -128,7 +127,7 @@ public function findContentStreams(): ContentStreams { $contentStreamsStatement = <<tableNames->contentStream()} SQL; @@ -193,11 +192,11 @@ private static function workspaceFromDatabaseRow(array $row): Workspace */ private static function contentStreamFromDatabaseRow(array $row): ContentStream { - return new ContentStream( + return ContentStream::create( ContentStreamId::fromString($row['id']), isset($row['sourceContentStreamId']) ? ContentStreamId::fromString($row['sourceContentStreamId']) : null, - ContentStreamStatus::from($row['status']), - Version::fromInteger((int)$row['version']) + Version::fromInteger((int)$row['version']), + (bool)$row['closed'], ); } } diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index 62109349166..477453656a9 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -17,12 +17,11 @@ use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Projection\NodeRelationAnchorPoint; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\DimensionSpacePointsRepository; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\ProjectionContentGraph; -use Neos\ContentRepository\Core\EventStore\InitiatingEventMetadata; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePointSet; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; use Neos\ContentRepository\Core\EventStore\EventInterface; +use Neos\ContentRepository\Core\EventStore\InitiatingEventMetadata; use Neos\ContentRepository\Core\Feature\Common\EmbedsContentStreamId; use Neos\ContentRepository\Core\Feature\Common\InterdimensionalSiblings; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Event\ContentStreamWasClosed; @@ -64,15 +63,15 @@ use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\NodeTags; use Neos\ContentRepository\Core\Projection\ContentGraph\Timestamps; use Neos\ContentRepository\Core\Projection\ProjectionStatus; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateClassification; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Node\NodeName; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; @@ -259,12 +258,12 @@ public function inSimulation(\Closure $fn): mixed private function whenContentStreamWasClosed(ContentStreamWasClosed $event): void { - $this->updateContentStreamStatus($event->contentStreamId, ContentStreamStatus::CLOSED); + $this->closeContentStream($event->contentStreamId); } private function whenContentStreamWasCreated(ContentStreamWasCreated $event): void { - $this->createContentStream($event->contentStreamId, ContentStreamStatus::CREATED); + $this->createContentStream($event->contentStreamId); } private function whenContentStreamWasForked(ContentStreamWasForked $event): void @@ -303,7 +302,7 @@ private function whenContentStreamWasForked(ContentStreamWasForked $event): void // NOTE: as reference edges are attached to Relation Anchor Points (and they are lazily copy-on-written), // we do not need to copy reference edges here (but we need to do it during copy on write). - $this->createContentStream($event->newContentStreamId, ContentStreamStatus::FORKED, $event->sourceContentStreamId, $event->versionOfSourceContentStream); + $this->createContentStream($event->newContentStreamId, $event->sourceContentStreamId, $event->versionOfSourceContentStream); } private function whenContentStreamWasRemoved(ContentStreamWasRemoved $event): void @@ -353,7 +352,7 @@ private function whenContentStreamWasRemoved(ContentStreamWasRemoved $event): vo private function whenContentStreamWasReopened(ContentStreamWasReopened $event): void { - $this->updateContentStreamStatus($event->contentStreamId, $event->previousState); + $this->reopenContentStream($event->contentStreamId); } private function whenDimensionShineThroughWasAdded(DimensionShineThroughWasAdded $event): void @@ -708,9 +707,6 @@ private function whenRootNodeAggregateWithNodeWasCreated(RootNodeAggregateWithNo private function whenRootWorkspaceWasCreated(RootWorkspaceWasCreated $event): void { $this->createWorkspace($event->workspaceName, null, $event->newContentStreamId); - - // the content stream is in use now - $this->updateContentStreamStatus($event->newContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE); } private function whenSubtreeWasTagged(SubtreeWasTagged $event): void @@ -734,69 +730,37 @@ private function whenWorkspaceRebaseFailed(WorkspaceRebaseFailed $event): void // before https://github.com/neos/neos-development-collection/pull/4965 this event was emitted and set the content stream status to `REBASE_ERROR` // instead of setting the error state on replay for old events we make it almost behave like if the rebase had failed today: reopen the workspaces content stream id // the candidateContentStreamId will be removed by the ContentStreamPruner - $this->whenContentStreamWasReopened(new ContentStreamWasReopened($event->sourceContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE)); + $this->reopenContentStream($event->sourceContentStreamId); } private function whenWorkspaceWasCreated(WorkspaceWasCreated $event): void { $this->createWorkspace($event->workspaceName, $event->baseWorkspaceName, $event->newContentStreamId); - - // the content stream is in use now - $this->updateContentStreamStatus($event->newContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE); } private function whenWorkspaceWasDiscarded(WorkspaceWasDiscarded $event): void { $this->updateWorkspaceContentStreamId($event->workspaceName, $event->newContentStreamId); - - // the new content stream is in use now - $this->updateContentStreamStatus($event->newContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE); - // the previous content stream is no longer in use - $this->updateContentStreamStatus($event->previousContentStreamId, ContentStreamStatus::NO_LONGER_IN_USE); } private function whenWorkspaceWasPartiallyDiscarded(WorkspaceWasPartiallyDiscarded $event): void { $this->updateWorkspaceContentStreamId($event->workspaceName, $event->newContentStreamId); - - // the new content stream is in use now - $this->updateContentStreamStatus($event->newContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE); - - // the previous content stream is no longer in use - $this->updateContentStreamStatus($event->previousContentStreamId, ContentStreamStatus::NO_LONGER_IN_USE); } private function whenWorkspaceWasPartiallyPublished(WorkspaceWasPartiallyPublished $event): void { $this->updateWorkspaceContentStreamId($event->sourceWorkspaceName, $event->newSourceContentStreamId); - - // the new content stream is in use now - $this->updateContentStreamStatus($event->newSourceContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE); - - // the previous content stream is no longer in use - $this->updateContentStreamStatus($event->previousSourceContentStreamId, ContentStreamStatus::NO_LONGER_IN_USE); } private function whenWorkspaceWasPublished(WorkspaceWasPublished $event): void { $this->updateWorkspaceContentStreamId($event->sourceWorkspaceName, $event->newSourceContentStreamId); - - // the new content stream is in use now - $this->updateContentStreamStatus($event->newSourceContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE); - - // the previous content stream is no longer in use - $this->updateContentStreamStatus($event->previousSourceContentStreamId, ContentStreamStatus::NO_LONGER_IN_USE); } private function whenWorkspaceWasRebased(WorkspaceWasRebased $event): void { $this->updateWorkspaceContentStreamId($event->workspaceName, $event->newContentStreamId); - - // the new content stream is in use now - $this->updateContentStreamStatus($event->newContentStreamId, ContentStreamStatus::IN_USE_BY_WORKSPACE); - - // the previous content stream is no longer in use - $this->updateContentStreamStatus($event->previousContentStreamId, ContentStreamStatus::NO_LONGER_IN_USE); } private function whenWorkspaceWasRemoved(WorkspaceWasRemoved $event): void diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php index 5b8e6421e28..0d13ff5ef91 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php @@ -127,8 +127,7 @@ private function createContentStreamTable(): Table (new Column('version', Type::getType(Types::INTEGER)))->setNotnull(true), DbalSchemaFactory::columnForContentStreamId('sourceContentStreamId')->setNotnull(false), (new Column('sourceContentStreamVersion', Type::getType(Types::INTEGER)))->setNotnull(false), - // Should become a DB ENUM (unclear how to configure with DBAL) or int (latter needs adaption to code) - (new Column('status', Type::getType(Types::BINARY)))->setLength(20)->setNotnull(true), + (new Column('closed', Type::getType(Types::BOOLEAN)))->setDefault(false)->setNotnull(true), ]); } diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Projection/Feature/ContentStream.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Projection/Feature/ContentStream.php index de69d00d5ed..50cd71bb02e 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Projection/Feature/ContentStream.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/Domain/Projection/Feature/ContentStream.php @@ -5,7 +5,6 @@ namespace Neos\ContentGraph\DoctrineDbalAdapter\Domain\Projection\Feature; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\EventStore\Model\Event\Version; /** @@ -15,21 +14,29 @@ */ trait ContentStream { - private function createContentStream(ContentStreamId $contentStreamId, ContentStreamStatus $status, ?ContentStreamId $sourceContentStreamId = null, ?Version $sourceVersion = null): void + private function createContentStream(ContentStreamId $contentStreamId, ?ContentStreamId $sourceContentStreamId = null, ?Version $sourceVersion = null): void { $this->dbal->insert($this->tableNames->contentStream(), [ 'id' => $contentStreamId->value, 'version' => 0, 'sourceContentStreamId' => $sourceContentStreamId?->value, - 'sourceContentStreamVersion' => $sourceVersion?->value, - 'status' => $status->value, + 'sourceContentStreamVersion' => $sourceVersion?->value ]); } - private function updateContentStreamStatus(ContentStreamId $contentStreamId, ContentStreamStatus $status): void + private function closeContentStream(ContentStreamId $contentStreamId): void { $this->dbal->update($this->tableNames->contentStream(), [ - 'status' => $status->value, + 'closed' => 1, + ], [ + 'id' => $contentStreamId->value + ]); + } + + private function reopenContentStream(ContentStreamId $contentStreamId): void + { + $this->dbal->update($this->tableNames->contentStream(), [ + 'closed' => 0, ], [ 'id' => $contentStreamId->value ]); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature index dadd9e7e112..c470848e37b 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature @@ -20,32 +20,38 @@ Feature: If content streams are not in use anymore by the workspace, they can be | nodeTypeName | "Neos.ContentRepository:Root" | Scenario: content streams are marked as IN_USE_BY_WORKSPACE properly after creation - Then the content stream "cs-identifier" has status "IN_USE_BY_WORKSPACE" Then I expect the content stream "non-existing" to not exist - Scenario: on creating a nested workspace, the new content stream is marked as IN_USE_BY_WORKSPACE. + When I prune unused content streams + Then I expect the content stream "cs-identifier" to exist + + Scenario: on creating a nested workspace, the new content stream is not pruned. When the command CreateWorkspace is executed with payload: | Key | Value | | workspaceName | "user-test" | | baseWorkspaceName | "live" | | newContentStreamId | "user-cs-identifier" | - Then the content stream "user-cs-identifier" has status "IN_USE_BY_WORKSPACE" + When I prune unused content streams + Then I expect the content stream "user-cs-identifier" to exist - Scenario: when rebasing a nested workspace, the new content stream will be marked as IN_USE_BY_WORKSPACE; and the old content stream is NO_LONGER_IN_USE. + Scenario: when rebasing a nested workspace, the new content stream will not be pruned; but the old content stream is pruned. When the command CreateWorkspace is executed with payload: | Key | Value | | workspaceName | "user-test" | | baseWorkspaceName | "live" | | newContentStreamId | "user-cs-identifier" | When the command RebaseWorkspace is executed with payload: - | Key | Value | - | workspaceName | "user-test" | + | Key | Value | + | workspaceName | "user-test" | + | rebasedContentStreamId | "user-cs-identifier-rebased" | | rebaseErrorHandlingStrategy | "force" | When I am in workspace "user-test" and dimension space point {} - Then the current content stream has status "IN_USE_BY_WORKSPACE" - And the content stream "user-cs-identifier" has status "NO_LONGER_IN_USE" + Then I expect the content stream "user-cs-identifier-rebased" to exist + + When I prune unused content streams + Then I expect the content stream "user-cs-identifier" to not exist Scenario: when pruning content streams, NO_LONGER_IN_USE content streams will be properly cleaned from the graph projection. diff --git a/Neos.ContentRepository.Core/Classes/CommandHandler/CommandHandlingDependencies.php b/Neos.ContentRepository.Core/Classes/CommandHandler/CommandHandlingDependencies.php index 09b97dc9293..9e726cacea1 100644 --- a/Neos.ContentRepository.Core/Classes/CommandHandler/CommandHandlingDependencies.php +++ b/Neos.ContentRepository.Core/Classes/CommandHandler/CommandHandlingDependencies.php @@ -18,7 +18,6 @@ use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\EventStore\Model\Event\Version; @@ -46,17 +45,16 @@ public function getContentStreamVersion(ContentStreamId $contentStreamId): Versi public function contentStreamExists(ContentStreamId $contentStreamId): bool { - $cs = $this->contentGraphReadModel->findContentStreamById($contentStreamId); - return $cs !== null && !$cs->removed; + return $this->contentGraphReadModel->findContentStreamById($contentStreamId) !== null; } - public function getContentStreamStatus(ContentStreamId $contentStreamId): ContentStreamStatus + public function isContentStreamClosed(ContentStreamId $contentStreamId): bool { $contentStream = $this->contentGraphReadModel->findContentStreamById($contentStreamId); if ($contentStream === null) { - throw new \InvalidArgumentException(sprintf('Failed to find content stream with id "%s"', $contentStreamId->value), 1716902219); + throw new \InvalidArgumentException(sprintf('Failed to find content stream with id "%s"', $contentStreamId->value), 1729863973); } - return $contentStream->status; + return $contentStream->isClosed; } public function findWorkspaceByName(WorkspaceName $workspaceName): ?Workspace diff --git a/Neos.ContentRepository.Core/Classes/Feature/Common/ConstraintChecks.php b/Neos.ContentRepository.Core/Classes/Feature/Common/ConstraintChecks.php index 30b67c1de78..7957ab4de30 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/Common/ConstraintChecks.php +++ b/Neos.ContentRepository.Core/Classes/Feature/Common/ConstraintChecks.php @@ -61,7 +61,6 @@ use Neos\ContentRepository\Core\SharedModel\Node\PropertyName; use Neos\ContentRepository\Core\SharedModel\Node\ReferenceName; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\EventStore\Model\EventStream\ExpectedVersion; @@ -88,8 +87,8 @@ protected function requireContentStream( 1521386692 ); } - $state = $commandHandlingDependencies->getContentStreamStatus($contentStreamId); - if ($state === ContentStreamStatus::CLOSED) { + + if ($commandHandlingDependencies->isContentStreamClosed($contentStreamId)) { throw new ContentStreamIsClosed( 'Content stream "' . $contentStreamId->value . '" is closed.', 1710260081 diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/ReopenContentStream.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/ReopenContentStream.php index 10280ae9890..056d4f08be7 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/ReopenContentStream.php +++ b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/ReopenContentStream.php @@ -16,7 +16,6 @@ use Neos\ContentRepository\Core\CommandHandler\CommandInterface; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; /** * @internal implementation detail. You must not use this command directly. @@ -27,25 +26,20 @@ { /** * @param ContentStreamId $contentStreamId The id of the content stream to reopen - * @param ContentStreamStatus $previousState The state the content stream was in before closing and is to be reset to */ private function __construct( - public ContentStreamId $contentStreamId, - public ContentStreamStatus $previousState + public ContentStreamId $contentStreamId ) { } /** * @param ContentStreamId $contentStreamId The id of the content stream to reopen - * @param ContentStreamStatus $previousState The state the content stream was in before closing and is to be reset to */ public static function create( - ContentStreamId $contentStreamId, - ContentStreamStatus $previousState + ContentStreamId $contentStreamId ): self { return new self( - $contentStreamId, - $previousState + $contentStreamId ); } @@ -56,8 +50,7 @@ public static function create( public static function fromArray(array $array): self { return new self( - ContentStreamId::fromString($array['contentStreamId']), - ContentStreamStatus::from($array['previousState']), + ContentStreamId::fromString($array['contentStreamId']) ); } } diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Event/ContentStreamWasReopened.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Event/ContentStreamWasReopened.php index 4b0248803f7..b783ee99fb5 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Event/ContentStreamWasReopened.php +++ b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Event/ContentStreamWasReopened.php @@ -17,7 +17,6 @@ use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Feature\Common\EmbedsContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; /** * @api events are the persistence-API of the content repository @@ -25,8 +24,7 @@ final readonly class ContentStreamWasReopened implements EventInterface, EmbedsContentStreamId { public function __construct( - public ContentStreamId $contentStreamId, - public ContentStreamStatus $previousState, + public ContentStreamId $contentStreamId ) { } @@ -38,16 +36,14 @@ public function getContentStreamId(): ContentStreamId public static function fromArray(array $values): self { return new self( - ContentStreamId::fromString($values['contentStreamId']), - ContentStreamStatus::from($values['previousState']), + ContentStreamId::fromString($values['contentStreamId']) ); } public function jsonSerialize(): array { return [ - 'contentStreamId' => $this->contentStreamId, - 'previousState' => $this->previousState, + 'contentStreamId' => $this->contentStreamId ]; } } diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamCommandHandler.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamCommandHandler.php index 1b8400833d1..fbd0c5cc918 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamCommandHandler.php +++ b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamCommandHandler.php @@ -15,8 +15,8 @@ namespace Neos\ContentRepository\Core\Feature; use Neos\ContentRepository\Core\CommandHandler\CommandHandlerInterface; -use Neos\ContentRepository\Core\CommandHandler\CommandInterface; use Neos\ContentRepository\Core\CommandHandler\CommandHandlingDependencies; +use Neos\ContentRepository\Core\CommandHandler\CommandInterface; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Command\CloseContentStream; @@ -60,7 +60,7 @@ private function handleReopenContentStream( ReopenContentStream $command, CommandHandlingDependencies $commandHandlingDependencies, ): EventsToPublish { - return $this->reopenContentStream($command->contentStreamId, $command->previousState, $commandHandlingDependencies); + return $this->reopenContentStream($command->contentStreamId, $commandHandlingDependencies); } private function handleRemoveContentStream( diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php index c0cdbb0a4f3..99c55157882 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php +++ b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php @@ -41,6 +41,15 @@ public static function isContentStreamStreamName(StreamName $streamName): bool return str_starts_with($streamName->value, self::EVENT_STREAM_NAME_PREFIX); } + public static function extractContentStreamId(StreamName $streamName): ContentStreamId + { + [$prefix, $contentStreamId] = explode(':', $streamName->value, 2); + if (($prefix . ':') !== self::EVENT_STREAM_NAME_PREFIX) { + throw new \InvalidArgumentException(sprintf('Stream name is not a content stream event stream "%s"', $streamName->value)); + } + return ContentStreamId::fromString($contentStreamId); + } + public function getEventStreamName(): StreamName { return StreamName::fromString($this->value); diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php index 16d191cfc10..c3027d71147 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php +++ b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php @@ -17,7 +17,6 @@ use Neos\ContentRepository\Core\SharedModel\Exception\ContentStreamIsClosed; use Neos\ContentRepository\Core\SharedModel\Exception\ContentStreamIsNotClosed; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\EventStore\Model\EventStream\ExpectedVersion; trait ContentStreamHandling @@ -74,12 +73,10 @@ private function closeContentStream( /** * @param ContentStreamId $contentStreamId The id of the content stream to reopen - * @param ContentStreamStatus $previousState The state the content stream was in before closing and is to be reset to * @phpstan-pure this method is pure, to persist the events they must be handled outside */ private function reopenContentStream( ContentStreamId $contentStreamId, - ContentStreamStatus $previousState, CommandHandlingDependencies $commandHandlingDependencies, ): EventsToPublish { $this->requireContentStreamToExist($contentStreamId, $commandHandlingDependencies); @@ -90,8 +87,7 @@ private function reopenContentStream( $streamName, Events::with( new ContentStreamWasReopened( - $contentStreamId, - $previousState, + $contentStreamId ), ), ExpectedVersion::ANY() @@ -197,7 +193,7 @@ private function requireContentStreamToNotBeClosed( ContentStreamId $contentStreamId, CommandHandlingDependencies $commandHandlingDependencies ): void { - if ($commandHandlingDependencies->getContentStreamStatus($contentStreamId) === ContentStreamStatus::CLOSED) { + if ($commandHandlingDependencies->isContentStreamClosed($contentStreamId)) { throw new ContentStreamIsClosed( 'Content stream "' . $contentStreamId->value . '" is closed.', 1710260081 @@ -209,7 +205,7 @@ private function requireContentStreamToBeClosed( ContentStreamId $contentStreamId, CommandHandlingDependencies $commandHandlingDependencies ): void { - if ($commandHandlingDependencies->getContentStreamStatus($contentStreamId) !== ContentStreamStatus::CLOSED) { + if (!$commandHandlingDependencies->isContentStreamClosed($contentStreamId)) { throw new ContentStreamIsNotClosed( 'Content stream "' . $contentStreamId->value . '" is not closed.', 1710405911 diff --git a/Neos.ContentRepository.Core/Classes/Feature/WorkspaceCommandHandler.php b/Neos.ContentRepository.Core/Classes/Feature/WorkspaceCommandHandler.php index 0cea05dc7e3..9d4511ce755 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/WorkspaceCommandHandler.php +++ b/Neos.ContentRepository.Core/Classes/Feature/WorkspaceCommandHandler.php @@ -58,7 +58,6 @@ use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceHasNoBaseWorkspaceName; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceStatus; @@ -202,7 +201,6 @@ private function handlePublishWorkspace( // we have no changes, we just reopen; partial no-op yield $this->reopenContentStream( $workspace->currentContentStreamId, - ContentStreamStatus::IN_USE_BY_WORKSPACE, // todo will be removed $commandHandlingDependencies ); return; @@ -220,7 +218,6 @@ private function handlePublishWorkspace( } catch (WorkspaceRebaseFailed $workspaceRebaseFailed) { yield $this->reopenContentStream( $workspace->currentContentStreamId, - ContentStreamStatus::IN_USE_BY_WORKSPACE, // todo will be removed $commandHandlingDependencies ); throw $workspaceRebaseFailed; @@ -347,7 +344,6 @@ private function handleRebaseWorkspace( if (!$commandHandlingDependencies->contentStreamExists($workspace->currentContentStreamId)) { throw new \RuntimeException('Cannot rebase a workspace with a stateless content stream', 1711718314); } - $currentWorkspaceContentStreamState = $commandHandlingDependencies->getContentStreamStatus($workspace->currentContentStreamId); if ( $workspace->status === WorkspaceStatus::UP_TO_DATE @@ -396,7 +392,6 @@ static function ($handle) use ($rebaseableCommands): void { ) { yield $this->reopenContentStream( $workspace->currentContentStreamId, - $currentWorkspaceContentStreamState, $commandHandlingDependencies ); @@ -452,7 +447,6 @@ private function handlePublishIndividualNodesFromWorkspace( if (!$commandHandlingDependencies->contentStreamExists($workspace->currentContentStreamId)) { throw new \RuntimeException('Cannot publish nodes on a workspace with a stateless content stream', 1710410114); } - $currentWorkspaceContentStreamState = $commandHandlingDependencies->getContentStreamStatus($workspace->currentContentStreamId); $baseWorkspace = $this->requireBaseWorkspace($workspace, $commandHandlingDependencies); $this->requireContentStreamToNotBeClosed($baseWorkspace->currentContentStreamId, $commandHandlingDependencies); $baseContentStreamVersion = $commandHandlingDependencies->getContentStreamVersion($baseWorkspace->currentContentStreamId); @@ -475,7 +469,6 @@ private function handlePublishIndividualNodesFromWorkspace( // almost a noop (e.g. random node ids were specified) ;) yield $this->reopenContentStream( $workspace->currentContentStreamId, - $currentWorkspaceContentStreamState, $commandHandlingDependencies ); return; @@ -496,7 +489,6 @@ private function handlePublishIndividualNodesFromWorkspace( } catch (WorkspaceRebaseFailed $workspaceRebaseFailed) { yield $this->reopenContentStream( $workspace->currentContentStreamId, - ContentStreamStatus::IN_USE_BY_WORKSPACE, // todo will be removed $commandHandlingDependencies ); throw $workspaceRebaseFailed; @@ -521,7 +513,6 @@ static function ($handle) use ($commandSimulator, $matchingCommands, $remainingC if ($commandSimulator->hasCommandsThatFailed()) { yield $this->reopenContentStream( $workspace->currentContentStreamId, - $currentWorkspaceContentStreamState, $commandHandlingDependencies ); @@ -590,7 +581,6 @@ private function handleDiscardIndividualNodesFromWorkspace( if (!$commandHandlingDependencies->contentStreamExists($workspace->currentContentStreamId)) { throw new \RuntimeException('Cannot discard nodes on a workspace with a stateless content stream', 1710408112); } - $currentWorkspaceContentStreamState = $commandHandlingDependencies->getContentStreamStatus($workspace->currentContentStreamId); $baseWorkspace = $this->requireBaseWorkspace($workspace, $commandHandlingDependencies); yield $this->closeContentStream( @@ -611,7 +601,6 @@ private function handleDiscardIndividualNodesFromWorkspace( // if we have nothing to discard, we can just keep all. (e.g. random node ids were specified) It's almost a noop ;) yield $this->reopenContentStream( $workspace->currentContentStreamId, - $currentWorkspaceContentStreamState, $commandHandlingDependencies ); return; @@ -641,7 +630,6 @@ static function ($handle) use ($commandsToKeep): void { if ($commandSimulator->hasCommandsThatFailed()) { yield $this->reopenContentStream( $workspace->currentContentStreamId, - $currentWorkspaceContentStreamState, $commandHandlingDependencies ); throw WorkspaceRebaseFailed::duringDiscard($commandSimulator->getCommandsThatFailed()); @@ -819,8 +807,7 @@ private function forkNewContentStreamAndApplyEvents( $eventsToApplyOnNewContentStream->withAppendedEvents( Events::with( new ContentStreamWasReopened( - $newContentStreamId, - ContentStreamStatus::IN_USE_BY_WORKSPACE // todo remove just temporary + $newContentStreamId ) ) ), diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index be6618ad8c8..1d14173639c 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -20,8 +20,8 @@ use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamForPruning; +use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\EventType; use Neos\EventStore\Model\Event\EventTypes; @@ -65,13 +65,16 @@ public function prune(bool $removeTemporary, \Closure $outputFn): void if ($removeTemporary) { $status[] = ContentStreamStatus::CREATED; $status[] = ContentStreamStatus::FORKED; - $status[] = ContentStreamStatus::CLOSED; } $allContentStreams = $this->getContentStreamsForPruning(); $unusedContentStreamsPresent = false; foreach ($allContentStreams as $contentStream) { + if ($contentStream->removed) { + continue; + } + if (!in_array($contentStream->status, $status, true)) { continue; } @@ -333,8 +336,8 @@ private function findAllContentStreamEventNames(): array ); $allContentStreamEventStreamNames = []; foreach ($events as $eventEnvelope) { - $allContentStreamEventStreamNames[$eventEnvelope->streamName->value] = true; + $allContentStreamEventStreamNames[] = $eventEnvelope->streamName; } - return array_map(StreamName::fromString(...), array_keys($allContentStreamEventStreamNames)); + return array_unique($allContentStreamEventStreamNames, SORT_REGULAR); } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php index 0f405fe5b71..64ca64a005a 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php @@ -5,9 +5,55 @@ namespace Neos\ContentRepository\Core\Service\ContentStreamPruner; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; /** + * This model reflects if content streams are currently in use or not. Each content stream + * is first CREATED or FORKED, and then moves to the IN_USE or REBASE_ERROR states; or is removed directly + * in case of temporary content streams. + * + * FORKING: Content streams are forked from a base content stream. It can happen that the base content + * stream is NO_LONGER_IN_USE, but the child content stream is still IN_USE_BY_WORKSPACE. In this case, + * the base content stream can go to removed=true (removed from the content graph), but needs to be retained + * in the event store: If we do a full replay, we need the events of the base content stream before the + * fork happened to rebuild the child content stream. + * This logic is done in {@see ContentStreamPruner::findUnusedAndRemovedContentStreamIds()}. + * + * TEMPORARY content streams: Projections should take care to dispose their temporary content streams, + * by triggering a ContentStreamWasRemoved event after the content stream is no longer used. + * + * The different status a content stream can be in + * + * │ │ + * │(for root │during + * │ content │rebase + * ▼ stream) ▼ + * ┌──────────┐ ┌──────────┐ Temporary + * │ CREATED │ │ FORKED │────┐ status + * └──────────┘ └──────────┘ for + * │ │ temporary + * ├───────────────────────┤ content + * ▼ │ streams + * ┌───────────────────┐ │ │ + * │IN_USE_BY_WORKSPACE│ │ │ + * └───────────────────┘ │ │ Persistent + * │ │ │ status + * ▼ │ │ + * ┌───────────────────┐ │ │ + * │ NO_LONGER_IN_USE │ │ │ + * └───────────────────┘ │ │ + * │ │ │ + * └──────────┬────────────┘ │ + * ▼ │ + * ┌────────────────────────────────────────┐ │ + * │ removed=true │ │ + * │ => removed from content graph │◀─┘ + * └────────────────────────────────────────┘ Cleanup + * │ + * ▼ + * ┌────────────────────────────────────────┐ + * │ completely deleted from event stream │ + * └────────────────────────────────────────┘ + * * @internal */ final readonly class ContentStreamForPruning diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.monopic b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.monopic similarity index 100% rename from Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.monopic rename to Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.monopic diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStreamStatus.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php similarity index 73% rename from Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStreamStatus.php rename to Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php index 1bdb87e45a4..8a7b3071512 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStreamStatus.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php @@ -12,12 +12,12 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\SharedModel\Workspace; +namespace Neos\ContentRepository\Core\Service\ContentStreamPruner; /** * @api */ -enum ContentStreamStatus: string implements \JsonSerializable +enum ContentStreamStatus: string { /** * the content stream was created, but not yet assigned to a workspace. @@ -39,23 +39,8 @@ enum ContentStreamStatus: string implements \JsonSerializable */ case IN_USE_BY_WORKSPACE = 'IN_USE_BY_WORKSPACE'; - /** - * the content stream was closed and must no longer accept new events - */ - case CLOSED = 'CLOSED'; - /** * the content stream is not used anymore, and can be removed. */ case NO_LONGER_IN_USE = 'NO_LONGER_IN_USE'; - - public static function fromString(string $value): self - { - return self::from($value); - } - - public function jsonSerialize(): string - { - return $this->value; - } } diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php index 38b79f67255..8bc2c0134d4 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php @@ -14,71 +14,37 @@ namespace Neos\ContentRepository\Core\SharedModel\Workspace; -use Neos\ContentRepository\Core\Service\ContentStreamPruner; use Neos\EventStore\Model\Event\Version; /** * Content Stream Read Model * - * This model reflects if content streams are currently in use or not. Each content stream - * is first CREATED or FORKED, and then moves to the IN_USE or REBASE_ERROR states; or is removed directly - * in case of temporary content streams. - * - * FORKING: Content streams are forked from a base content stream. It can happen that the base content - * stream is NO_LONGER_IN_USE, but the child content stream is still IN_USE_BY_WORKSPACE. In this case, - * the base content stream can go to removed=true (removed from the content graph), but needs to be retained - * in the event store: If we do a full replay, we need the events of the base content stream before the - * fork happened to rebuild the child content stream. - * This logic is done in {@see ContentStreamPruner::findUnusedAndRemovedContentStreamIds()}. - * - * TEMPORARY content streams: Projections should take care to dispose their temporary content streams, - * by triggering a ContentStreamWasRemoved event after the content stream is no longer used. - * - * The different status a content stream can be in - * - * │ │ - * │(for root │during - * │ content │rebase - * ▼ stream) ▼ - * ┌──────────┐ ┌──────────┐ Temporary - * │ CREATED │ │ FORKED │────┐ status - * └──────────┘ └──────────┘ for - * │ │ temporary - * ├───────────────────────┤ content - * ▼ │ streams - * ┌───────────────────┐ │ │ - * │IN_USE_BY_WORKSPACE│ │ │ - * └───────────────────┘ │ │ Persistent - * │ │ │ status - * ▼ │ │ - * ┌───────────────────┐ │ │ - * │ NO_LONGER_IN_USE │ │ │ - * └───────────────────┘ │ │ - * │ │ │ - * └──────────┬────────────┘ │ - * ▼ │ - * ┌────────────────────────────────────────┐ │ - * │ removed=true │ │ - * │ => removed from content graph │◀─┘ - * └────────────────────────────────────────┘ Cleanup - * │ - * ▼ - * ┌────────────────────────────────────────┐ - * │ completely deleted from event stream │ - * └────────────────────────────────────────┘ - * * @api */ final readonly class ContentStream { - /** - * @internal - */ - public function __construct( + private function __construct( public ContentStreamId $id, public ?ContentStreamId $sourceContentStreamId, - public ContentStreamStatus $status, public Version $version, + public bool $isClosed ) { } + + /** + * @internal + */ + public static function create( + ContentStreamId $id, + ?ContentStreamId $sourceContentStreamId, + Version $version, + bool $isClosed + ): self { + return new self( + $id, + $sourceContentStreamId, + $version, + $isClosed + ); + } } diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index 8ea25a95c3b..c7c62d03330 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -16,23 +16,21 @@ use Behat\Behat\Hook\Scope\BeforeScenarioScope; use Behat\Gherkin\Node\TableNode; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\NodeModification\Dto\PropertyValuesToWrite; use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\CatchUpOptions; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\FindSubtreeFilter; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\NodeType\NodeTypeCriteria; use Neos\ContentRepository\Core\Projection\ContentGraph\Subtree; use Neos\ContentRepository\Core\Projection\ContentGraph\VisibilityConstraints; -use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\Service\ContentStreamPruner; use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\ContentStreamClosing; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeCopying; @@ -129,10 +127,19 @@ protected function readPayloadTable(TableNode $payloadTable): array return $eventPayload; } + /** + * @Then /^I expect the content stream "([^"]*)" to exist$/ + */ + public function iExpectTheContentStreamToExist(string $rawContentStreamId): void + { + $contentStream = $this->currentContentRepository->findContentStreamById(ContentStreamId::fromString($rawContentStreamId)); + Assert::assertNotNull($contentStream, sprintf('Content stream "%s" was expected to exist, but it does not', $rawContentStreamId)); + } + /** * @Then /^I expect the content stream "([^"]*)" to not exist$/ */ - public function iExpectTheContentStreamToNotExist(string $rawContentStreamId): void + public function iExpectTheContentStreamToNotExist(string $rawContentStreamId, string $not = ''): void { $contentStream = $this->currentContentRepository->findContentStreamById(ContentStreamId::fromString($rawContentStreamId)); Assert::assertNull($contentStream, sprintf('Content stream "%s" was not expected to exist, but it does', $rawContentStreamId)); @@ -243,31 +250,6 @@ protected function getRootNodeAggregateId(): ?NodeAggregateId )->nodeAggregateId; } - /** - * @Then the content stream :contentStreamId has status :expectedState - */ - public function theContentStreamHasStatus(string $contentStreamId, string $expectedStatus): void - { - $contentStream = $this->currentContentRepository->findContentStreamById(ContentStreamId::fromString($contentStreamId)); - if ($contentStream === null) { - Assert::fail(sprintf('Expected content stream "%s" to have status "%s" but it does not exist', $contentStreamId, $expectedStatus)); - } - Assert::assertSame(ContentStreamStatus::tryFrom($expectedStatus), $contentStream->status); - } - - /** - * @Then the current content stream has status :expectedStatus - */ - public function theCurrentContentStreamHasStatus(string $expectedStatus): void - { - $this->theContentStreamHasStatus( - $this->currentContentRepository - ->findWorkspaceByName($this->currentWorkspaceName) - ->currentContentStreamId->value, - $expectedStatus - ); - } - /** * @When I prune unused content streams */ diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index 866f5cc2274..262d2dd9aa9 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -55,10 +55,10 @@ public function pruneRemovedFromEventStreamCommand(string $contentRepository = ' $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); - $unusedContentStreamNames = $contentStreamPruner->pruneRemovedFromEventStream(); + $unusedContentStreamIds = $contentStreamPruner->pruneRemovedFromEventStream(); $unusedContentStreamsPresent = false; - foreach ($unusedContentStreamNames as $contentStreamName) { - $this->outputFormatted('Removed events for %s', [$contentStreamName->value]); + foreach ($unusedContentStreamIds as $contentStreamId) { + $this->outputFormatted('Removed events for %s', [$contentStreamId->value]); $unusedContentStreamsPresent = true; } if (!$unusedContentStreamsPresent) { From 2541e7c00ff4bb957ee15c9aa1e0898e63c183b3 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 26 Oct 2024 20:34:52 +0200 Subject: [PATCH 12/28] TASK: Avoid use of `CloseContentStream` command in tests and use event Also as we dont expose the command anymore we can get rid of the constraint checks and: > the command CloseContentStream is executed with payload and exceptions are caught --- ...AggregateWithNode_ConstraintChecks.feature | 2 +- ...AggregateWithNode_ConstraintChecks.feature | 2 +- ...CreateNodeVariant_ConstraintChecks.feature | 2 +- ...SetNodeProperties_ConstraintChecks.feature | 2 +- ...SetNodeReferences_ConstraintChecks.feature | 2 +- ...ableNodeAggregate_ConstraintChecks.feature | 2 +- ...moveNodeAggregate_ConstraintChecks.feature | 2 +- .../01-MoveNodes_ConstraintChecks.feature | 2 +- ...NodeAggregateName_ConstraintChecks.feature | 2 +- ...NodeAggregateType_ConstraintChecks.feature | 2 +- ...loseContentStream_ConstraintChecks.feature | 30 ------------------- ...ForkContentStream_ConstraintChecks.feature | 2 +- .../Features/ContentStreamClosing.php | 27 ++++++----------- 13 files changed, 20 insertions(+), 59 deletions(-) delete mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/ContentStreamClosing/01-CloseContentStream_ConstraintChecks.feature diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/01-RootNodeCreation/01-CreateRootNodeAggregateWithNode_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/01-RootNodeCreation/01-CreateRootNodeAggregateWithNode_ConstraintChecks.feature index be7df0bd441..10960530bf2 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/01-RootNodeCreation/01-CreateRootNodeAggregateWithNode_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/01-RootNodeCreation/01-CreateRootNodeAggregateWithNode_ConstraintChecks.feature @@ -39,7 +39,7 @@ Feature: Create a root node aggregate Then the last command should have thrown an exception of type "WorkspaceDoesNotExist" Scenario: Try to create a root node aggregate in a closed content stream: - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | And the command CreateRootNodeAggregateWithNode is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/02-NodeCreation/01-CreateNodeAggregateWithNode_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/02-NodeCreation/01-CreateNodeAggregateWithNode_ConstraintChecks.feature index 6e608301510..19a5500ab86 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/02-NodeCreation/01-CreateNodeAggregateWithNode_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/02-NodeCreation/01-CreateNodeAggregateWithNode_ConstraintChecks.feature @@ -53,7 +53,7 @@ Feature: Create node aggregate with node Then the last command should have thrown an exception of type "WorkspaceDoesNotExist" Scenario: Try to create a node aggregate in a workspace whose content stream is closed: - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | And the command CreateNodeAggregateWithNode is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/03-NodeVariation/01-CreateNodeVariant_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/03-NodeVariation/01-CreateNodeVariant_ConstraintChecks.feature index cbe2098d6da..5df29d7fe45 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/03-NodeVariation/01-CreateNodeVariant_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/03-NodeVariation/01-CreateNodeVariant_ConstraintChecks.feature @@ -47,7 +47,7 @@ Feature: Create node variant Then the last command should have thrown an exception of type "WorkspaceDoesNotExist" Scenario: Try to create a variant in a workspace that does not exist - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | And the command CreateNodeVariant is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/04-NodeModification/01-SetNodeProperties_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/04-NodeModification/01-SetNodeProperties_ConstraintChecks.feature index 3dd1295af19..968798de225 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/04-NodeModification/01-SetNodeProperties_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/04-NodeModification/01-SetNodeProperties_ConstraintChecks.feature @@ -44,7 +44,7 @@ Feature: Set node properties: Constraint checks Then the last command should have thrown an exception of type "WorkspaceDoesNotExist" Scenario: Try to set properties in a workspace whose content stream is closed - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | When the command SetNodeProperties is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/05-NodeReferencing/01-SetNodeReferences_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/05-NodeReferencing/01-SetNodeReferences_ConstraintChecks.feature index 7f6aa1c37ca..7dc5687b611 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/05-NodeReferencing/01-SetNodeReferences_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/05-NodeReferencing/01-SetNodeReferences_ConstraintChecks.feature @@ -55,7 +55,7 @@ Feature: Constraint checks on SetNodeReferences | berta-destinode | Neos.ContentRepository.Testing:ReferencedNode | lady-eleonode-rootford | Scenario: Try to reference nodes in a workspace whose content stream is closed - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | When the command SetNodeReferences is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/06-NodeDisabling/01-DisableNodeAggregate_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/06-NodeDisabling/01-DisableNodeAggregate_ConstraintChecks.feature index 4a353636c61..91c845e2b27 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/06-NodeDisabling/01-DisableNodeAggregate_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/06-NodeDisabling/01-DisableNodeAggregate_ConstraintChecks.feature @@ -38,7 +38,7 @@ Feature: Constraint checks on node aggregate disabling Then the last command should have thrown an exception of type "WorkspaceDoesNotExist" Scenario: Try to disable a node aggregate in a workspace whose content stream is closed - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | When the command DisableNodeAggregate is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/07-NodeRemoval/01-RemoveNodeAggregate_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/07-NodeRemoval/01-RemoveNodeAggregate_ConstraintChecks.feature index c11417b9ad0..e7e35cb4a5a 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/07-NodeRemoval/01-RemoveNodeAggregate_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/07-NodeRemoval/01-RemoveNodeAggregate_ConstraintChecks.feature @@ -43,7 +43,7 @@ Feature: Remove NodeAggregate Then the last command should have thrown an exception of type "WorkspaceDoesNotExist" Scenario: Try to remove a node aggregate in a workspace whose content stream is closed - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | When the command RemoveNodeAggregate is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/08-NodeMove/01-MoveNodes_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/08-NodeMove/01-MoveNodes_ConstraintChecks.feature index 3ebce961613..37f9a1c8523 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/08-NodeMove/01-MoveNodes_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/08-NodeMove/01-MoveNodes_ConstraintChecks.feature @@ -59,7 +59,7 @@ Feature: Move node to a new parent / within the current parent before a sibling Then the last command should have thrown an exception of type "WorkspaceDoesNotExist" Scenario: Try to move a node in a workspace whose content stream is closed: - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | When the command MoveNodeAggregate is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/09-NodeRenaming/01-ChangeNodeAggregateName_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/09-NodeRenaming/01-ChangeNodeAggregateName_ConstraintChecks.feature index 50c499912b4..b8a003c26a4 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/09-NodeRenaming/01-ChangeNodeAggregateName_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/09-NodeRenaming/01-ChangeNodeAggregateName_ConstraintChecks.feature @@ -43,7 +43,7 @@ Feature: Change node name Then the last command should have thrown an exception of type "WorkspaceDoesNotExist" Scenario: Try to rename a node aggregate in a workspace whose content stream is closed: - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | When the command ChangeNodeAggregateName is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/11-NodeTypeChange/01-ChangeNodeAggregateType_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/11-NodeTypeChange/01-ChangeNodeAggregateType_ConstraintChecks.feature index 55148afde37..9ee1fafd150 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/11-NodeTypeChange/01-ChangeNodeAggregateType_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/11-NodeTypeChange/01-ChangeNodeAggregateType_ConstraintChecks.feature @@ -78,7 +78,7 @@ Feature: Change node aggregate type - basic error cases Then the last command should have thrown an exception of type "WorkspaceDoesNotExist" Scenario: Try to change the node aggregate type in a workspace whose content stream is closed - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | And the command ChangeNodeAggregateType is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/ContentStreamClosing/01-CloseContentStream_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/ContentStreamClosing/01-CloseContentStream_ConstraintChecks.feature deleted file mode 100644 index 595c2e53055..00000000000 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/ContentStreamClosing/01-CloseContentStream_ConstraintChecks.feature +++ /dev/null @@ -1,30 +0,0 @@ -@contentrepository @adapters=DoctrineDBAL -Feature: Constraint check test cases for closing content streams - - Background: - Given using no content dimensions - And using the following node types: - """yaml - Neos.ContentRepository:Root: {} - """ - And using identifier "default", I define a content repository - And I am in content repository "default" - And the command CreateRootWorkspace is executed with payload: - | Key | Value | - | workspaceName | "live" | - | newContentStreamId | "cs-identifier" | - - Scenario: Try to close a non-existing content stream: - And the command CloseContentStream is executed with payload and exceptions are caught: - | Key | Value | - | contentStreamId | "i-do-not-exist" | - Then the last command should have thrown an exception of type "ContentStreamDoesNotExistYet" - - Scenario: Try to close a content stream that is already closed: - When the command CloseContentStream is executed with payload: - | Key | Value | - | contentStreamId | "cs-identifier" | - And the command CloseContentStream is executed with payload and exceptions are caught: - | Key | Value | - | contentStreamId | "cs-identifier" | - Then the last command should have thrown an exception of type "ContentStreamIsClosed" diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/ContentStreamForking/01-ForkContentStream_ConstraintChecks.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/ContentStreamForking/01-ForkContentStream_ConstraintChecks.feature index 5b3a1fd7723..2aa54f9856f 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/ContentStreamForking/01-ForkContentStream_ConstraintChecks.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/ContentStreamForking/01-ForkContentStream_ConstraintChecks.feature @@ -50,7 +50,7 @@ Feature: ForkContentStream Without Dimensions | propertiesToUnset | {} | Scenario: Try to create a workspace with the base workspace referring to a closed content stream - When the command CloseContentStream is executed with payload: + When the event ContentStreamWasClosed was published with payload: | Key | Value | | contentStreamId | "cs-identifier" | When the command CreateWorkspace is executed with payload and exceptions are caught: diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/Features/ContentStreamClosing.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/Features/ContentStreamClosing.php index 81c21a91567..05c7546c0b2 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/Features/ContentStreamClosing.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/Features/ContentStreamClosing.php @@ -15,7 +15,7 @@ namespace Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features; use Behat\Gherkin\Node\TableNode; -use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Command\CloseContentStream; +use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\CRTestSuiteRuntimeVariables; @@ -29,26 +29,17 @@ trait ContentStreamClosing abstract protected function readPayloadTable(TableNode $payloadTable): array; /** - * @Given /^the command CloseContentStream is executed with payload:$/ + * @Given /^the event ContentStreamWasClosed was published with payload:$/ + * @param TableNode $payloadTable * @throws \Exception */ - public function theCommandCloseContentStreamIsExecutedWithPayload(TableNode $payloadTable): void + public function theEventContentStreamWasClosedWasPublishedWithPayload(TableNode $payloadTable) { - $commandArguments = $this->readPayloadTable($payloadTable); - $command = CloseContentStream::create(ContentStreamId::fromString($commandArguments['contentStreamId'])); + $eventPayload = $this->readPayloadTable($payloadTable); + $streamName = ContentStreamEventStreamName::fromContentStreamId( + ContentStreamId::fromString($eventPayload['contentStreamId']) + ); - $this->currentContentRepository->handle($command); - } - - /** - * @Given /^the command CloseContentStream is executed with payload and exceptions are caught:$/ - */ - public function theCommandCloseContentStreamIsExecutedWithPayloadAndExceptionsAreCaught(TableNode $payloadTable): void - { - try { - $this->theCommandCloseContentStreamIsExecutedWithPayload($payloadTable); - } catch (\Exception $exception) { - $this->lastCommandException = $exception; - } + $this->publishEvent('ContentStreamWasClosed', $streamName->getEventStreamName(), $eventPayload); } } From d715cf75e55d7b77d374ff3de139a6ca0aef8d68 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 26 Oct 2024 20:35:31 +0200 Subject: [PATCH 13/28] TASK: Remove legacy low level ContentStream commands and command handler :D --- .../Factory/ContentRepositoryFactory.php | 2 - .../Command/CloseContentStream.php | 52 -------------- .../Command/ReopenContentStream.php | 56 --------------- .../Feature/ContentStreamCommandHandler.php | 72 ------------------- .../Command/RemoveContentStream.php | 44 ------------ .../WorkspaceMaintenanceServiceFactory.php | 3 +- 6 files changed, 1 insertion(+), 228 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/CloseContentStream.php delete mode 100644 Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/ReopenContentStream.php delete mode 100644 Neos.ContentRepository.Core/Classes/Feature/ContentStreamCommandHandler.php delete mode 100644 Neos.ContentRepository.Core/Classes/Feature/ContentStreamRemoval/Command/RemoveContentStream.php diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index f8244afea53..d424f39138e 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -23,7 +23,6 @@ use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\EventStore\EventPersister; -use Neos\ContentRepository\Core\Feature\ContentStreamCommandHandler; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\DimensionSpaceCommandHandler; use Neos\ContentRepository\Core\Feature\NodeAggregateCommandHandler; use Neos\ContentRepository\Core\Feature\NodeDuplication\NodeDuplicationCommandHandler; @@ -121,7 +120,6 @@ public function getOrBuild(): ContentRepository ); $publicCommandBus = $commandBusForRebaseableCommands->withAdditionalHandlers( - new ContentStreamCommandHandler(), new WorkspaceCommandHandler( $commandSimulatorFactory, $this->projectionFactoryDependencies->eventStore, diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/CloseContentStream.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/CloseContentStream.php deleted file mode 100644 index 3234b633586..00000000000 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/CloseContentStream.php +++ /dev/null @@ -1,52 +0,0 @@ - $array - * @internal only used for testcases - */ - public static function fromArray(array $array): self - { - return new self( - ContentStreamId::fromString($array['contentStreamId']), - ); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/ReopenContentStream.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/ReopenContentStream.php deleted file mode 100644 index 056d4f08be7..00000000000 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamClosing/Command/ReopenContentStream.php +++ /dev/null @@ -1,56 +0,0 @@ - $array - * @internal only used for testcases - */ - public static function fromArray(array $array): self - { - return new self( - ContentStreamId::fromString($array['contentStreamId']) - ); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamCommandHandler.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamCommandHandler.php deleted file mode 100644 index fbd0c5cc918..00000000000 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamCommandHandler.php +++ /dev/null @@ -1,72 +0,0 @@ -getShortName()); - } - - public function handle(CommandInterface $command, CommandHandlingDependencies $commandHandlingDependencies): EventsToPublish - { - return match ($command::class) { - CloseContentStream::class => $this->handleCloseContentStream($command, $commandHandlingDependencies), - ReopenContentStream::class => $this->handleReopenContentStream($command, $commandHandlingDependencies), - RemoveContentStream::class => $this->handleRemoveContentStream($command, $commandHandlingDependencies), - default => throw new \DomainException('Cannot handle commands of class ' . get_class($command), 1710408206), - }; - } - - private function handleCloseContentStream( - CloseContentStream $command, - CommandHandlingDependencies $commandHandlingDependencies, - ): EventsToPublish { - return $this->closeContentStream($command->contentStreamId, $commandHandlingDependencies); - } - - private function handleReopenContentStream( - ReopenContentStream $command, - CommandHandlingDependencies $commandHandlingDependencies, - ): EventsToPublish { - return $this->reopenContentStream($command->contentStreamId, $commandHandlingDependencies); - } - - private function handleRemoveContentStream( - RemoveContentStream $command, - CommandHandlingDependencies $commandHandlingDependencies - ): EventsToPublish { - return $this->removeContentStream($command->contentStreamId, $commandHandlingDependencies); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamRemoval/Command/RemoveContentStream.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamRemoval/Command/RemoveContentStream.php deleted file mode 100644 index 932a488a842..00000000000 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamRemoval/Command/RemoveContentStream.php +++ /dev/null @@ -1,44 +0,0 @@ -contentRepository, - $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->contentRepository ); } } From 4d64cbd8461d71724340bd853f14cc0f8be4ff78 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 26 Oct 2024 20:13:50 +0200 Subject: [PATCH 14/28] TASK: Ensure that `cr:prune` prunes also workspaces cleanly --- .../Classes/Service/ContentStreamPruner.php | 40 +++++++++++++++---- .../Service/WorkspaceMaintenanceService.php | 15 +------ .../Classes/Command/CrCommandController.php | 3 +- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 1d14173639c..6e723a1c094 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -129,10 +129,13 @@ public function pruneRemovedFromEventStream(): array return $removedContentStreams; } - public function pruneAll(): void + public function pruneAllWorkspacesAndContentStreamsFromEventStream(): void { - foreach ($this->findAllContentStreamEventNames() as $streamName) { - $this->eventStore->deleteStream($streamName); + foreach ($this->findAllContentStreamStreamNames() as $contentStreamStreamName) { + $this->eventStore->deleteStream($contentStreamStreamName); + } + foreach ($this->findAllWorkspaceStreamNames() as $workspaceStreamName) { + $this->eventStore->deleteStream($workspaceStreamName); } } @@ -323,21 +326,44 @@ private function getContentStreamsForPruning(): array /** * @return list */ - private function findAllContentStreamEventNames(): array + private function findAllContentStreamStreamNames(): array { $events = $this->eventStore->load( VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), EventStreamFilter::create( EventTypes::create( + // we are only interested in the creation events to limit the amount of events to fetch EventType::fromString('ContentStreamWasCreated'), EventType::fromString('ContentStreamWasForked') ) ) ); - $allContentStreamEventStreamNames = []; + $allStreamNames = []; + foreach ($events as $eventEnvelope) { + $allStreamNames[] = $eventEnvelope->streamName; + } + return array_unique($allStreamNames, SORT_REGULAR); + } + + /** + * @return list + */ + private function findAllWorkspaceStreamNames(): array + { + $events = $this->eventStore->load( + VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + // we are only interested in the creation events to limit the amount of events to fetch + EventType::fromString('RootWorkspaceWasCreated'), + EventType::fromString('WorkspaceWasCreated') + ) + ) + ); + $allStreamNames = []; foreach ($events as $eventEnvelope) { - $allContentStreamEventStreamNames[] = $eventEnvelope->streamName; + $allStreamNames[] = $eventEnvelope->streamName; } - return array_unique($allContentStreamEventStreamNames, SORT_REGULAR); + return array_unique($allStreamNames, SORT_REGULAR); } } diff --git a/Neos.ContentRepository.Core/Classes/Service/WorkspaceMaintenanceService.php b/Neos.ContentRepository.Core/Classes/Service/WorkspaceMaintenanceService.php index 0c2ee14feab..b166f0d1271 100644 --- a/Neos.ContentRepository.Core/Classes/Service/WorkspaceMaintenanceService.php +++ b/Neos.ContentRepository.Core/Classes/Service/WorkspaceMaintenanceService.php @@ -6,13 +6,11 @@ use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Command\RebaseWorkspace; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Dto\RebaseErrorHandlingStrategy; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspaces; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceStatus; -use Neos\EventStore\EventStoreInterface; /** * @api @@ -20,8 +18,7 @@ class WorkspaceMaintenanceService implements ContentRepositoryServiceInterface { public function __construct( - private readonly ContentRepository $contentRepository, - private readonly EventStoreInterface $eventStore, + private readonly ContentRepository $contentRepository ) { } @@ -33,7 +30,7 @@ public function rebaseOutdatedWorkspaces(?RebaseErrorHandlingStrategy $strategy $outdatedWorkspaces = $this->contentRepository->findWorkspaces()->filter( fn (Workspace $workspace) => $workspace->status === WorkspaceStatus::OUTDATED ); - /** @var Workspace $workspace */ + // todo we need to loop through the workspaces from root level first foreach ($outdatedWorkspaces as $workspace) { if ($workspace->status !== WorkspaceStatus::OUTDATED) { continue; @@ -49,12 +46,4 @@ public function rebaseOutdatedWorkspaces(?RebaseErrorHandlingStrategy $strategy return $outdatedWorkspaces; } - - public function pruneAll(): void - { - foreach ($this->contentRepository->findWorkspaces() as $workspace) { - $streamName = WorkspaceEventStreamName::fromWorkspaceName($workspace->workspaceName)->getEventStreamName(); - $this->eventStore->deleteStream($streamName); - } - } } diff --git a/Neos.Neos/Classes/Command/CrCommandController.php b/Neos.Neos/Classes/Command/CrCommandController.php index ed2ab33e522..d94cbeac1c9 100644 --- a/Neos.Neos/Classes/Command/CrCommandController.php +++ b/Neos.Neos/Classes/Command/CrCommandController.php @@ -165,8 +165,7 @@ public function pruneCommand(string $contentRepository = 'default', bool $force $this->workspaceService->pruneWorkspaceMetadata($contentRepositoryId); // reset the events table - $contentStreamPruner->pruneAll(); - $workspaceMaintenanceService->pruneAll(); + $contentStreamPruner->pruneAllWorkspacesAndContentStreamsFromEventStream(); // reset the projections state $projectionService->resetAllProjections(); From a2b3c580658d084dc8f16b06f9a42ab4c5738e67 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:28:42 +0100 Subject: [PATCH 15/28] BUGFIX: ContentStream pruner mark `RootWorkspaceWasCreated` as in use --- .../Classes/Service/ContentStreamPruner.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 6e723a1c094..e04a50ae5ac 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -11,6 +11,7 @@ use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; use Neos\ContentRepository\Core\Feature\ContentStreamForking\Event\ContentStreamWasForked; use Neos\ContentRepository\Core\Feature\ContentStreamRemoval\Event\ContentStreamWasRemoved; +use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Event\RootWorkspaceWasCreated; use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Event\WorkspaceWasCreated; use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasDiscarded; @@ -238,6 +239,7 @@ private function getContentStreamsForPruning(): array VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), EventStreamFilter::create( EventTypes::create( + EventType::fromString('RootWorkspaceWasCreated'), EventType::fromString('WorkspaceWasCreated'), EventType::fromString('WorkspaceWasDiscarded'), EventType::fromString('WorkspaceWasPartiallyDiscarded'), @@ -253,6 +255,12 @@ private function getContentStreamsForPruning(): array $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); switch ($domainEvent::class) { + case RootWorkspaceWasCreated::class: + if (isset($status[$domainEvent->newContentStreamId->value])) { + $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); + } + break; case WorkspaceWasCreated::class: if (isset($status[$domainEvent->newContentStreamId->value])) { $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] From 41d2941724ab2e30b9e68845c5c4ad5fde4f011d Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:09:52 +0100 Subject: [PATCH 16/28] TASK: Always remove all dangling content streams --- .../Classes/Service/ContentStreamPruner.php | 26 ++++++------------- .../ContentStreamStatus.php | 8 +++--- .../Features/Bootstrap/CRTestSuiteTrait.php | 2 +- .../ContentStreamCommandController.php | 3 +-- 4 files changed, 14 insertions(+), 25 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index e04a50ae5ac..90bea68c51c 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -46,28 +46,18 @@ public function __construct( } /** - * Remove all content streams which are not needed anymore from the projections. + * Before publishing version 3 (#5301) dangling content streams were not removed during publishing, discard or rebase * - * NOTE: This still **keeps** the event stream as is; so it would be possible to re-construct the content stream - * at a later point in time (though we currently do not provide any API for it). + * Removes all nodes, hierarchy relations and content stream entries which are not needed anymore from the projections. * - * To remove the deleted Content Streams, - * call {@see ContentStreamPruner::pruneRemovedFromEventStream()} afterwards. + * NOTE: This still **keeps** the event stream as is; so it would be possible to re-construct the content stream at a later point in time. * - * By default, only content streams that are NO_LONGER_IN_USE will be removed. - * If you also call with $removeTemporary=true, will delete ALL content streams which are currently not assigned - * to a workspace (f.e. dangling ones in FORKED, CLOSED or CREATED.). + * To prune the removed content streams from the event store, call {@see ContentStreamPruner::pruneRemovedFromEventStream()} afterwards. * - * @param bool $removeTemporary if TRUE, will delete ALL content streams not bound to a workspace + * @deprecated with Neos 9 beta 15, only used to migrate from earlier versions */ - public function prune(bool $removeTemporary, \Closure $outputFn): void + public function removeDangelingContentStreams(\Closure $outputFn): void { - $status = [ContentStreamStatus::NO_LONGER_IN_USE]; - if ($removeTemporary) { - $status[] = ContentStreamStatus::CREATED; - $status[] = ContentStreamStatus::FORKED; - } - $allContentStreams = $this->getContentStreamsForPruning(); $unusedContentStreamsPresent = false; @@ -76,7 +66,7 @@ public function prune(bool $removeTemporary, \Closure $outputFn): void continue; } - if (!in_array($contentStream->status, $status, true)) { + if ($contentStream->status === ContentStreamStatus::IN_USE_BY_WORKSPACE) { continue; } @@ -90,7 +80,7 @@ public function prune(bool $removeTemporary, \Closure $outputFn): void ExpectedVersion::STREAM_EXISTS() ); - $outputFn(sprintf('Removed %s', $contentStream->id)); + $outputFn(sprintf('Removed %s with status %s', $contentStream->id, $contentStream->status->value)); $unusedContentStreamsPresent = true; } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php index 8a7b3071512..b6b855eb5f3 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php @@ -24,7 +24,7 @@ enum ContentStreamStatus: string * * **temporary state** which should not appear if the system is idle (for content streams which are used with workspaces). */ - case CREATED = 'CREATED'; + case CREATED = 'created (temporary)'; /** * FORKED means the content stream was forked from an existing content stream, but not yet assigned @@ -32,15 +32,15 @@ enum ContentStreamStatus: string * * **temporary state** which should not appear if the system is idle (for content streams which are used with workspaces). */ - case FORKED = 'FORKED'; + case FORKED = 'forked (temporary)'; /** * the content stream is currently referenced as the "active" content stream by a workspace. */ - case IN_USE_BY_WORKSPACE = 'IN_USE_BY_WORKSPACE'; + case IN_USE_BY_WORKSPACE = 'in use by workspace'; /** * the content stream is not used anymore, and can be removed. */ - case NO_LONGER_IN_USE = 'NO_LONGER_IN_USE'; + case NO_LONGER_IN_USE = 'no longer in use'; } diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index c7c62d03330..3fbc1506c81 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -257,7 +257,7 @@ public function iPruneUnusedContentStreams(): void { /** @var ContentStreamPruner $contentStreamPruner */ $contentStreamPruner = $this->getContentRepositoryService(new ContentStreamPrunerFactory()); - $contentStreamPruner->prune(false, fn () => null); + $contentStreamPruner->removeDangelingContentStreams(fn () => null); } /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index 262d2dd9aa9..1e2b93f77e1 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -39,8 +39,7 @@ public function pruneCommand(string $contentRepository = 'default', bool $remove $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); - $contentStreamPruner->prune( - $removeTemporary, + $contentStreamPruner->removeDangelingContentStreams( $this->outputLine(...) ); } From baf56ffca947e6687eee9fb57861d25a8e1bad0c Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:24:29 +0100 Subject: [PATCH 17/28] BUGFIX: Mark workspace outdated if its source content stream was removed During rebase of the base workspace, its content stream will be now fully removed from the projection (no soft removal) thats why we have to adjust the calculation to not rely on the select being `null` ONLY for when it has no base. Its also null if the source content stream was removed. ``` 001 Scenario: Publishing to the root workspace render dependents outdated # Features/Workspaces/WorkspaceState.feature:71 Then workspace user-ws-two has status OUTDATED ``` --- .../src/ContentGraphReadModelAdapter.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php index ee14a8239d5..041659607ca 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php @@ -159,7 +159,7 @@ private function getBasicWorkspaceQuery(): QueryBuilder $queryBuilder = $this->dbal->createQueryBuilder(); return $queryBuilder - ->select('ws.name, ws.baseWorkspaceName, ws.currentContentStreamId, cs.sourceContentStreamVersion != scs.version as baseWorkspaceChanged') + ->select('ws.name, ws.baseWorkspaceName, ws.currentContentStreamId, cs.sourceContentStreamVersion = scs.version as upToDateWithBase') ->from($this->tableNames->workspace(), 'ws') ->join('ws', $this->tableNames->contentStream(), 'cs', 'cs.id = ws.currentcontentstreamid') ->leftJoin('cs', $this->tableNames->contentStream(), 'scs', 'scs.id = cs.sourceContentStreamId'); @@ -171,13 +171,17 @@ private function getBasicWorkspaceQuery(): QueryBuilder private static function workspaceFromDatabaseRow(array $row): Workspace { $baseWorkspaceName = $row['baseWorkspaceName'] !== null ? WorkspaceName::fromString($row['baseWorkspaceName']) : null; - $status = match ($row['baseWorkspaceChanged']) { + + if ($baseWorkspaceName === null) { // no base workspace, a root is always up-to-date - null => WorkspaceStatus::UP_TO_DATE, - // base workspace didnt change (sql 0 is _false_) - 0 => WorkspaceStatus::UP_TO_DATE, - default => WorkspaceStatus::OUTDATED, - }; + $status = WorkspaceStatus::UP_TO_DATE; + } elseif ($row['upToDateWithBase'] === 1) { + // base workspace didnt change + $status = WorkspaceStatus::UP_TO_DATE; + } else { + // base content stream was removed or contains newer changes + $status = WorkspaceStatus::OUTDATED; + } return new Workspace( WorkspaceName::fromString($row['name']), From da89494d0eb9a716fad335961c926dae2d58bb42 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:38:09 +0100 Subject: [PATCH 18/28] TASK: `Workspace` to use static `create` like `Node` and other read models --- .../src/ContentGraphReadModelAdapter.php | 2 +- .../SharedModel/Workspace/ContentStream.php | 2 +- .../SharedModel/Workspace/Workspace.php | 39 +++++++------------ .../SharedModel/Workspace/Workspaces.php | 1 - 4 files changed, 17 insertions(+), 27 deletions(-) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php index 041659607ca..fc9c6f4ddc7 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphReadModelAdapter.php @@ -183,7 +183,7 @@ private static function workspaceFromDatabaseRow(array $row): Workspace $status = WorkspaceStatus::OUTDATED; } - return new Workspace( + return Workspace::create( WorkspaceName::fromString($row['name']), $baseWorkspaceName, ContentStreamId::fromString($row['currentContentStreamId']), diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php index 8bc2c0134d4..c52109de269 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/ContentStream.php @@ -19,7 +19,7 @@ /** * Content Stream Read Model * - * @api + * @api Note: The constructor is not part of the public API */ final readonly class ContentStream { diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/Workspace.php b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/Workspace.php index 7f51c161123..b3fd69556fe 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/Workspace.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/Workspace.php @@ -17,43 +17,34 @@ /** * Workspace Read Model * - * @api + * @api Note: The constructor is not part of the public API */ final readonly class Workspace { /** - * @var WorkspaceName Workspace identifier, unique within one Content Repository instance + * @param WorkspaceName $workspaceName Workspace identifier, unique within one Content Repository instance + * @param WorkspaceName|null $baseWorkspaceName Workspace identifier of the base workspace (i.e. the target when publishing changes) – if null this instance is considered a root (aka public) workspace + * @param ContentStreamId $currentContentStreamId The Content Stream this workspace currently points to – usually it is set to a new, empty content stream after publishing/rebasing the workspace + * @param WorkspaceStatus $status The current status of this workspace */ - public WorkspaceName $workspaceName; - - /** - * @var WorkspaceName|null Workspace identifier of the base workspace (i.e. the target when publishing changes) – if null this instance is considered a root (aka public) workspace - */ - public ?WorkspaceName $baseWorkspaceName; - - /** - * The Content Stream this workspace currently points to – usually it is set to a new, empty content stream after publishing/rebasing the workspace - */ - public ContentStreamId $currentContentStreamId; - - /** - * The current status of this workspace - */ - public WorkspaceStatus $status; + private function __construct( + public WorkspaceName $workspaceName, + public ?WorkspaceName $baseWorkspaceName, + public ContentStreamId $currentContentStreamId, + public WorkspaceStatus $status, + ) { + } /** * @internal */ - public function __construct( + public static function create( WorkspaceName $workspaceName, ?WorkspaceName $baseWorkspaceName, ContentStreamId $currentContentStreamId, WorkspaceStatus $status, - ) { - $this->workspaceName = $workspaceName; - $this->baseWorkspaceName = $baseWorkspaceName; - $this->currentContentStreamId = $currentContentStreamId; - $this->status = $status; + ): self { + return new self($workspaceName, $baseWorkspaceName, $currentContentStreamId, $status); } /** diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/Workspaces.php b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/Workspaces.php index f6af24a86ec..be0969b591d 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/Workspaces.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/Workspace/Workspaces.php @@ -21,7 +21,6 @@ * * @api */ - final class Workspaces implements \IteratorAggregate, \Countable { /** From a7ee5254ae87b6e56c38989a91039b3da2135751 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:03:32 +0100 Subject: [PATCH 19/28] TASK: Introduce `contentstream:status` --- .../Classes/Service/ContentStreamPruner.php | 97 ++++++++++++++++--- .../ContentStreamStatus.php | 4 +- .../ContentStreamCommandController.php | 28 ++++-- 3 files changed, 106 insertions(+), 23 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 90bea68c51c..79e633753fd 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -45,6 +45,73 @@ public function __construct( ) { } + /** + * Detects if dangling content streams exists and which content streams could be pruned from the event store + * + * Dangling content streams + * ------------------------ + * + * Content streams that are not removed via the event ContentStreamWasRemoved and are not in use by a workspace: + * Not a current's workspace content stream. + * + * Pruneable content streams + * ------------------------- + * + * Content streams that were removed ContentStreamWasRemoved e.g. after publishing, and are not required for a full + * replay to reconstruct the current projections state. The ability to reconstitute a previous state will be lost. + * + * @return bool if dangling content streams exist + */ + public function status(\Closure $outputFn): bool + { + $allContentStreams = $this->getContentStreamsForPruning(); + + $danglingContentStreamPresent = false; + foreach ($allContentStreams as $contentStream) { + if ($contentStream->removed) { + continue; + } + if ($contentStream->status === ContentStreamStatus::IN_USE_BY_WORKSPACE) { + continue; + } + if ($danglingContentStreamPresent === false) { + $outputFn(sprintf('Dangling content streams that are not removed (ContentStreamWasRemoved) and not %s:', ContentStreamStatus::IN_USE_BY_WORKSPACE->value)); + } + + $outputFn(sprintf(' id: %s reason for removal: %s', $contentStream->id->value, $contentStream->status->value)); + $danglingContentStreamPresent = true; + } + + if ($danglingContentStreamPresent === true) { + $outputFn('To remove the dangling streams from the projections please run ./flow contentstream:prune'); + $outputFn('Then they are ready for removal from the event store'); + $outputFn(); + } else { + $outputFn('Okay. No dangling streams found'); + $outputFn(); + } + + $removedContentStreams = $this->findUnusedAndRemovedContentStreamIds($allContentStreams); + + $pruneableContentStreamPresent = false; + foreach ($removedContentStreams as $removedContentStream) { + if ($pruneableContentStreamPresent === false) { + $outputFn('Removed content streams that can be pruned from the event store'); + } + $pruneableContentStreamPresent = true; + $outputFn(sprintf(' id: %s previous state: %s', $removedContentStream->id->value, $removedContentStream->status->value)); + } + + if ($pruneableContentStreamPresent === true) { + $outputFn('To prune the removed streams from the event store run ./flow contentstream:pruneremovedfromeventstream'); + $outputFn('Then they are indefinitely pruned from the event store'); + } else { + $outputFn('Okay. No pruneable streams in the event store'); + } + + return $danglingContentStreamPresent; + } + /** * Before publishing version 3 (#5301) dangling content streams were not removed during publishing, discard or rebase * @@ -88,7 +155,7 @@ public function removeDangelingContentStreams(\Closure $outputFn): void if ($unusedContentStreamsPresent) { try { $this->contentRepository->catchUpProjections(); - } catch (\Exception $e) { + } catch (\Throwable $e) { $outputFn(sprintf('Could not catchup after removing unused content streams: %s. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.', $e->getMessage())); } } else { @@ -104,20 +171,27 @@ public function removeDangelingContentStreams(\Closure $outputFn): void * these dependent Content Streams are not allowed to be removed in the event store. * * - Otherwise, we cannot replay the other content streams correctly (if the base content streams are missing). - * - * @return list the removed content streams */ - public function pruneRemovedFromEventStream(): array + public function pruneRemovedFromEventStream(\Closure $outputFn): void { - $removedContentStreams = $this->findUnusedAndRemovedContentStreamIds(); + $allContentStreams = $this->getContentStreamsForPruning(); + + $removedContentStreams = $this->findUnusedAndRemovedContentStreamIds($allContentStreams); + + $unusedContentStreamsPresent = false; foreach ($removedContentStreams as $removedContentStream) { $this->eventStore->deleteStream( ContentStreamEventStreamName::fromContentStreamId( - $removedContentStream + $removedContentStream->id )->getEventStreamName() ); + $unusedContentStreamsPresent = true; + $outputFn(sprintf('Removed events for %s (previous state %s)', $removedContentStream->id->value, $removedContentStream->status->value)); + } + + if ($unusedContentStreamsPresent === false) { + $outputFn('There are no unused content streams.'); } - return $removedContentStreams; } public function pruneAllWorkspacesAndContentStreamsFromEventStream(): void @@ -131,12 +205,11 @@ public function pruneAllWorkspacesAndContentStreamsFromEventStream(): void } /** - * @return list + * @param array $allContentStreams + * @return list */ - private function findUnusedAndRemovedContentStreamIds(): array + private function findUnusedAndRemovedContentStreamIds(array $allContentStreams): array { - $allContentStreams = $this->getContentStreamsForPruning(); - /** @var array $transitiveUsedStreams */ $transitiveUsedStreams = []; /** @var list $contentStreamIdsStack */ @@ -171,7 +244,7 @@ private function findUnusedAndRemovedContentStreamIds(): array $removedContentStreams = []; foreach ($allContentStreams as $contentStream) { if ($contentStream->removed && !array_key_exists($contentStream->id->value, $transitiveUsedStreams)) { - $removedContentStreams[] = $contentStream->id; + $removedContentStreams[] = $contentStream; } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php index b6b855eb5f3..5b2ea6b1cf6 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php @@ -24,7 +24,7 @@ enum ContentStreamStatus: string * * **temporary state** which should not appear if the system is idle (for content streams which are used with workspaces). */ - case CREATED = 'created (temporary)'; + case CREATED = 'created (temporary state)'; /** * FORKED means the content stream was forked from an existing content stream, but not yet assigned @@ -32,7 +32,7 @@ enum ContentStreamStatus: string * * **temporary state** which should not appear if the system is idle (for content streams which are used with workspaces). */ - case FORKED = 'forked (temporary)'; + case FORKED = 'forked (temporary state)'; /** * the content stream is currently referenced as the "active" content stream by a workspace. diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index 1e2b93f77e1..7e1358457c8 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -18,6 +18,22 @@ class ContentStreamCommandController extends CommandController */ protected $contentRepositoryRegistry; + /** + * @param string $contentRepository Identifier of the content repository. (Default: 'default') + */ + public function statusCommand(string $contentRepository = 'default'): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); + + $status = $contentStreamPruner->status( + $this->outputLine(...) + ); + if ($status === false) { + $this->quit(1); + } + } + /** * Remove all content streams which are not needed anymore from the projections. * @@ -54,14 +70,8 @@ public function pruneRemovedFromEventStreamCommand(string $contentRepository = ' $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); - $unusedContentStreamIds = $contentStreamPruner->pruneRemovedFromEventStream(); - $unusedContentStreamsPresent = false; - foreach ($unusedContentStreamIds as $contentStreamId) { - $this->outputFormatted('Removed events for %s', [$contentStreamId->value]); - $unusedContentStreamsPresent = true; - } - if (!$unusedContentStreamsPresent) { - $this->outputLine('There are no unused content streams.'); - } + $contentStreamPruner->pruneRemovedFromEventStream( + $this->outputLine(...) + ); } } From 40f41fd545c4b8b88d6df2a620628d0b3153a65f Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:04:50 +0100 Subject: [PATCH 20/28] TASK: Only remove dangling streams `contentstream:removeDangling` that are older than x That way we ensure this command can be run during production and doent kill any workspaces that are *just* created --- .../Classes/Service/ContentStreamPruner.php | 33 +++++++++++++------ .../ContentStreamForPruning.php | 7 +++- .../ContentStreamStatus.php | 12 +++++-- .../ContentStreamCommandController.php | 21 +++++------- 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 79e633753fd..b9cf4116bc8 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -78,12 +78,17 @@ public function status(\Closure $outputFn): bool $outputFn(sprintf('Dangling content streams that are not removed (ContentStreamWasRemoved) and not %s:', ContentStreamStatus::IN_USE_BY_WORKSPACE->value)); } - $outputFn(sprintf(' id: %s reason for removal: %s', $contentStream->id->value, $contentStream->status->value)); + if ($contentStream->status->isTemporary()) { + $outputFn(sprintf(' id: %s temporary %s at %s', $contentStream->id->value, $contentStream->status->value, $contentStream->created->format('Y-m-d H:i'))); + } else { + $outputFn(sprintf(' id: %s %s', $contentStream->id->value, $contentStream->status->value)); + } + $danglingContentStreamPresent = true; } if ($danglingContentStreamPresent === true) { - $outputFn('To remove the dangling streams from the projections please run ./flow contentstream:prune'); + $outputFn('To remove the dangling streams from the projections please run ./flow contentStream:removeDangling'); $outputFn('Then they are ready for removal from the event store'); $outputFn(); } else { @@ -103,7 +108,7 @@ public function status(\Closure $outputFn): bool } if ($pruneableContentStreamPresent === true) { - $outputFn('To prune the removed streams from the event store run ./flow contentstream:pruneremovedfromeventstream'); + $outputFn('To prune the removed streams from the event store run ./flow contentStream:pruneRemovedFromEventstream'); $outputFn('Then they are indefinitely pruned from the event store'); } else { $outputFn('Okay. No pruneable streams in the event store'); @@ -113,17 +118,15 @@ public function status(\Closure $outputFn): bool } /** - * Before publishing version 3 (#5301) dangling content streams were not removed during publishing, discard or rebase - * * Removes all nodes, hierarchy relations and content stream entries which are not needed anymore from the projections. * * NOTE: This still **keeps** the event stream as is; so it would be possible to re-construct the content stream at a later point in time. * * To prune the removed content streams from the event store, call {@see ContentStreamPruner::pruneRemovedFromEventStream()} afterwards. * - * @deprecated with Neos 9 beta 15, only used to migrate from earlier versions + * @param \DateTimeImmutable $removeTemporaryBefore includes all temporary content streams like FORKED or CREATED older than that in the removal */ - public function removeDangelingContentStreams(\Closure $outputFn): void + public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmutable $removeTemporaryBefore): void { $allContentStreams = $this->getContentStreamsForPruning(); @@ -137,6 +140,14 @@ public function removeDangelingContentStreams(\Closure $outputFn): void continue; } + if ( + $contentStream->status->isTemporary() + && $removeTemporaryBefore < $contentStream->created + ) { + $outputFn(sprintf('Did not remove %s temporary %s at %s', $contentStream->id->value, $contentStream->status->value, $contentStream->created->format('Y-m-d H:i'))); + continue; + } + $this->eventStore->commit( ContentStreamEventStreamName::fromContentStreamId($contentStream->id)->getEventStreamName(), $this->eventNormalizer->normalize( @@ -186,7 +197,7 @@ public function pruneRemovedFromEventStream(\Closure $outputFn): void )->getEventStreamName() ); $unusedContentStreamsPresent = true; - $outputFn(sprintf('Removed events for %s (previous state %s)', $removedContentStream->id->value, $removedContentStream->status->value)); + $outputFn(sprintf('Removed events for %s', $removedContentStream->id->value)); } if ($unusedContentStreamsPresent === false) { @@ -277,14 +288,16 @@ private function getContentStreamsForPruning(): array $status[$domainEvent->contentStreamId->value] = ContentStreamForPruning::create( $domainEvent->contentStreamId, ContentStreamStatus::CREATED, - null + null, + $eventEnvelope->recordedAt ); break; case ContentStreamWasForked::class: $status[$domainEvent->newContentStreamId->value] = ContentStreamForPruning::create( $domainEvent->newContentStreamId, ContentStreamStatus::FORKED, - $domainEvent->sourceContentStreamId + $domainEvent->sourceContentStreamId, + $eventEnvelope->recordedAt ); break; case ContentStreamWasRemoved::class: diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php index 64ca64a005a..bf3e99eac98 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php @@ -62,6 +62,7 @@ private function __construct( public ContentStreamId $id, public ContentStreamStatus $status, public ?ContentStreamId $sourceContentStreamId, + public \DateTimeImmutable $created, public bool $removed, ) { } @@ -69,12 +70,14 @@ private function __construct( public static function create( ContentStreamId $id, ContentStreamStatus $status, - ?ContentStreamId $sourceContentStreamId + ?ContentStreamId $sourceContentStreamId, + \DateTimeImmutable $create, ): self { return new self( $id, $status, $sourceContentStreamId, + $create, false ); } @@ -85,6 +88,7 @@ public function withStatus(ContentStreamStatus $status): self $this->id, $status, $this->sourceContentStreamId, + $this->created, $this->removed ); } @@ -95,6 +99,7 @@ public function withRemoved(): self $this->id, $this->status, $this->sourceContentStreamId, + $this->created, true ); } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php index 5b2ea6b1cf6..1c96d2f62e0 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamStatus.php @@ -24,7 +24,7 @@ enum ContentStreamStatus: string * * **temporary state** which should not appear if the system is idle (for content streams which are used with workspaces). */ - case CREATED = 'created (temporary state)'; + case CREATED = 'created'; /** * FORKED means the content stream was forked from an existing content stream, but not yet assigned @@ -32,7 +32,7 @@ enum ContentStreamStatus: string * * **temporary state** which should not appear if the system is idle (for content streams which are used with workspaces). */ - case FORKED = 'forked (temporary state)'; + case FORKED = 'forked'; /** * the content stream is currently referenced as the "active" content stream by a workspace. @@ -43,4 +43,12 @@ enum ContentStreamStatus: string * the content stream is not used anymore, and can be removed. */ case NO_LONGER_IN_USE = 'no longer in use'; + + public function isTemporary(): bool + { + return match ($this) { + self::CREATED, self::FORKED => true, + default => false + }; + } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index 7e1358457c8..fa65d26f994 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -35,28 +35,25 @@ public function statusCommand(string $contentRepository = 'default'): void } /** - * Remove all content streams which are not needed anymore from the projections. + * Before Neos 9 beta 15 (#5301), dangling content streams were not removed during publishing, discard or rebase. * - * NOTE: This still **keeps** the event stream as is; so it would be possible to re-construct the content stream - * at a later point in time (though we currently do not provide any API for it). + * Removes all nodes, hierarchy relations and content stream entries which are not needed anymore from the projections. * - * To remove the deleted Content Streams, use `./flow contentStream:pruneRemovedFromEventStream` after running - * `./flow contentStream:prune`. + * NOTE: This still **keeps** the event stream as is; so it would be possible to re-construct the content stream at a later point in time. * - * By default, only content streams that are NO_LONGER_IN_USE will be removed. - * If you also call with "--removeTemporary", will delete ALL content streams which are currently not assigned - * to a workspace (f.e. dangling ones in FORKED or CREATED.). + * To prune the removed content streams from the event store, call ./flow contentStream:pruneRemovedFromEventStream afterwards. * * @param string $contentRepository Identifier of the content repository. (Default: 'default') - * @param boolean $removeTemporary Will delete all content streams which are currently not assigned (Default: false) + * @param string $removeTemporaryBefore includes all temporary content streams like FORKED or CREATED older than that in the removal */ - public function pruneCommand(string $contentRepository = 'default', bool $removeTemporary = false): void + public function removeDanglingCommand(string $contentRepository = 'default', string $removeTemporaryBefore = '-1day'): void { $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); - $contentStreamPruner->removeDangelingContentStreams( - $this->outputLine(...) + $contentStreamPruner->removeDanglingContentStreams( + $this->outputLine(...), + new \DateTimeImmutable($removeTemporaryBefore) ); } From 27b72d421f2b1429ff8245ffce4d773d419e5646 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:05:59 +0100 Subject: [PATCH 21/28] TASK: Remove obsolete explicit pruning (I prune unused content streams) from tests that is part of the publishing now --- .../Workspaces/PruneContentStreams.feature | 26 +++++++++---------- .../Features/Bootstrap/CRTestSuiteTrait.php | 13 +--------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature index c470848e37b..d630716b542 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature @@ -19,20 +19,21 @@ Feature: If content streams are not in use anymore by the workspace, they can be | nodeAggregateId | "root-node" | | nodeTypeName | "Neos.ContentRepository:Root" | - Scenario: content streams are marked as IN_USE_BY_WORKSPACE properly after creation - Then I expect the content stream "non-existing" to not exist + # + # Before Neos 9 beta 15 (publishing version 3 #5301), dangling content streams were not removed during publishing, discard or rebase + # The first scenarios assert that the automatic deletion works correctly + # - When I prune unused content streams + Scenario: content streams are in use after creation + Then I expect the content stream "non-existing" to not exist Then I expect the content stream "cs-identifier" to exist - Scenario: on creating a nested workspace, the new content stream is not pruned. + Scenario: on creating a nested workspace, the new content stream is not pruned When the command CreateWorkspace is executed with payload: | Key | Value | | workspaceName | "user-test" | | baseWorkspaceName | "live" | | newContentStreamId | "user-cs-identifier" | - - When I prune unused content streams Then I expect the content stream "user-cs-identifier" to exist Scenario: when rebasing a nested workspace, the new content stream will not be pruned; but the old content stream is pruned. @@ -50,11 +51,10 @@ Feature: If content streams are not in use anymore by the workspace, they can be When I am in workspace "user-test" and dimension space point {} Then I expect the content stream "user-cs-identifier-rebased" to exist - When I prune unused content streams Then I expect the content stream "user-cs-identifier" to not exist - Scenario: when pruning content streams, NO_LONGER_IN_USE content streams will be properly cleaned from the graph projection. + Scenario: no longer in use content streams will be properly cleaned from the graph projection. When the command CreateWorkspace is executed with payload: | Key | Value | | workspaceName | "user-test" | @@ -71,13 +71,13 @@ Feature: If content streams are not in use anymore by the workspace, they can be | rebaseErrorHandlingStrategy | "force" | # now, we have one unused content stream (the old content stream of the user-test workspace) - When I prune unused content streams Then I expect the content stream "user-cs-identifier" to not exist When I am in workspace "user-test" and dimension space point {} + # todo test that the graph projection really is cleaned up and that no hierarchy stil exist? Then I expect node aggregate identifier "root-node" to lead to node user-cs-identifier-rebased;root-node;{} - Scenario: NO_LONGER_IN_USE content streams can be cleaned up completely (simple case) + Scenario: no longer in use content streams can be cleaned up completely (simple case) When the command CreateWorkspace is executed with payload: | Key | Value | @@ -91,13 +91,12 @@ Feature: If content streams are not in use anymore by the workspace, they can be # now, we have one unused content stream (the old content stream of the user-test workspace) - When I prune unused content streams And I prune removed content streams from the event stream Then I expect exactly 0 events to be published on stream "ContentStream:user-cs-identifier" - Scenario: NO_LONGER_IN_USE content streams are only cleaned up if no other content stream which is still in use depends on it + Scenario: no longer in use content streams are only cleaned up if no other content stream which is still in use depends on it # we build a "review" workspace, and then a "user-test" workspace depending on the review workspace. When the command CreateWorkspace is executed with payload: | Key | Value | @@ -110,7 +109,7 @@ Feature: If content streams are not in use anymore by the workspace, they can be | baseWorkspaceName | "review" | | newContentStreamId | "user-cs-identifier" | - # now, we rebase the "review" workspace, effectively marking the "review-cs-identifier" content stream as NO_LONGER_IN_USE. + # now, we rebase the "review" workspace, effectively marking the "review-cs-identifier" content stream as no longer in use. # however, we are not allowed to drop the content stream from the event store yet, because the "user-cs-identifier" is based # on the (no-longer-in-direct-use) review-cs-identifier. When the command RebaseWorkspace is executed with payload: @@ -118,7 +117,6 @@ Feature: If content streams are not in use anymore by the workspace, they can be | workspaceName | "review" | | rebaseErrorHandlingStrategy | "force" | - When I prune unused content streams And I prune removed content streams from the event stream # the events should still exist diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index 3fbc1506c81..92e58e02ace 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -27,7 +27,6 @@ use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\NodeType\NodeTypeCriteria; use Neos\ContentRepository\Core\Projection\ContentGraph\Subtree; use Neos\ContentRepository\Core\Projection\ContentGraph\VisibilityConstraints; -use Neos\ContentRepository\Core\Service\ContentStreamPruner; use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; @@ -250,22 +249,12 @@ protected function getRootNodeAggregateId(): ?NodeAggregateId )->nodeAggregateId; } - /** - * @When I prune unused content streams - */ - public function iPruneUnusedContentStreams(): void - { - /** @var ContentStreamPruner $contentStreamPruner */ - $contentStreamPruner = $this->getContentRepositoryService(new ContentStreamPrunerFactory()); - $contentStreamPruner->removeDangelingContentStreams(fn () => null); - } - /** * @When I prune removed content streams from the event stream */ public function iPruneRemovedContentStreamsFromTheEventStream(): void { - $this->getContentRepositoryService(new ContentStreamPrunerFactory())->pruneRemovedFromEventStream(); + $this->getContentRepositoryService(new ContentStreamPrunerFactory())->pruneRemovedFromEventStream(fn () => null); } abstract protected function getContentRepositoryService( From d1d56bb567046f511f0f1d3f59135dcc3e0da109 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:21:21 +0100 Subject: [PATCH 22/28] TASK: Adjust documentation and add force flag --- .../Classes/Service/ContentStreamPruner.php | 16 +++++--- .../ContentStreamCommandController.php | 38 ++++++++++++++++--- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index b9cf4116bc8..0f925b0b696 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -51,8 +51,12 @@ public function __construct( * Dangling content streams * ------------------------ * - * Content streams that are not removed via the event ContentStreamWasRemoved and are not in use by a workspace: - * Not a current's workspace content stream. + * Content streams that are not removed via the event ContentStreamWasRemoved and are not in use by a workspace + * (not a current's workspace content stream). + * + * Previously before Neos 9 beta 15 (#5301), dangling content streams were not removed during publishing, discard or rebase. + * + * {@see removeDanglingContentStreams} * * Pruneable content streams * ------------------------- @@ -60,7 +64,9 @@ public function __construct( * Content streams that were removed ContentStreamWasRemoved e.g. after publishing, and are not required for a full * replay to reconstruct the current projections state. The ability to reconstitute a previous state will be lost. * - * @return bool if dangling content streams exist + * {@see pruneRemovedFromEventStream} + * + * @return bool false if dangling content streams exist because they should not */ public function status(\Closure $outputFn): bool { @@ -114,7 +120,7 @@ public function status(\Closure $outputFn): bool $outputFn('Okay. No pruneable streams in the event store'); } - return $danglingContentStreamPresent; + return !$danglingContentStreamPresent; } /** @@ -175,7 +181,7 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta } /** - * Remove unused and deleted content streams from the event stream; effectively REMOVING information completely. + * Prune removed content streams that are unused from the event stream; effectively REMOVING information completely. * * This is not so easy for nested workspaces / content streams: * - As long as content streams are used as basis for others which are IN_USE_BY_WORKSPACE, diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index fa65d26f994..cebaae5e84c 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -19,6 +19,26 @@ class ContentStreamCommandController extends CommandController protected $contentRepositoryRegistry; /** + * Detects if dangling content streams exists and which content streams could be pruned from the event store + * + * Dangling content streams + * ------------------------ + * + * Content streams that are not removed via the event ContentStreamWasRemoved and are not in use by a workspace + * (not a current's workspace content stream). + * + * Previously before Neos 9 beta 15 (#5301), dangling content streams were not removed during publishing, discard or rebase. + * + * ./flow contentStream:removeDangling + * + * Pruneable content streams + * ------------------------- + * + * Content streams that were removed ContentStreamWasRemoved e.g. after publishing, and are not required for a full + * replay to reconstruct the current projections state. The ability to reconstitute a previous state will be lost. + * + * ./flow contentStream:pruneRemovedFromEventStream + * * @param string $contentRepository Identifier of the content repository. (Default: 'default') */ public function statusCommand(string $contentRepository = 'default'): void @@ -35,13 +55,13 @@ public function statusCommand(string $contentRepository = 'default'): void } /** - * Before Neos 9 beta 15 (#5301), dangling content streams were not removed during publishing, discard or rebase. - * * Removes all nodes, hierarchy relations and content stream entries which are not needed anymore from the projections. * * NOTE: This still **keeps** the event stream as is; so it would be possible to re-construct the content stream at a later point in time. * - * To prune the removed content streams from the event store, call ./flow contentStream:pruneRemovedFromEventStream afterwards. + * HINT: ./flow contentStream:status gives information what is about to be removed + * + * To prune the removed content streams from the event store, run ./flow contentStream:pruneRemovedFromEventStream afterwards. * * @param string $contentRepository Identifier of the content repository. (Default: 'default') * @param string $removeTemporaryBefore includes all temporary content streams like FORKED or CREATED older than that in the removal @@ -58,12 +78,20 @@ public function removeDanglingCommand(string $contentRepository = 'default', str } /** - * Remove unused and deleted content streams from the event stream; effectively REMOVING information completely + * Prune removed content streams that are unused from the event stream; effectively REMOVING information completely + * + * HINT: ./flow contentStream:status gives information what is about to be pruned * * @param string $contentRepository Identifier of the content repository. (Default: 'default') + * @param bool $force Prune the unused content streams without confirmation. This cannot be reverted! */ - public function pruneRemovedFromEventStreamCommand(string $contentRepository = 'default'): void + public function pruneRemovedFromEventStreamCommand(string $contentRepository = 'default', bool $force = false): void { + if (!$force && !$this->output->askConfirmation(sprintf('> This will prune removed content streams that are unused from the event stream in content repository "%s" (see contentStream:status). Are you sure to proceed? (y/n) ', $contentRepository), false)) { + $this->outputLine('Abort.'); + return; + } + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); From 3d808c0b5d93f6f47faaee46a83a33d5de2a9fa8 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:22:36 +0100 Subject: [PATCH 23/28] TASK: Remove obsolete `extractContentStreamId` --- .../Classes/Feature/ContentStreamEventStreamName.php | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php index 99c55157882..c0cdbb0a4f3 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php +++ b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamEventStreamName.php @@ -41,15 +41,6 @@ public static function isContentStreamStreamName(StreamName $streamName): bool return str_starts_with($streamName->value, self::EVENT_STREAM_NAME_PREFIX); } - public static function extractContentStreamId(StreamName $streamName): ContentStreamId - { - [$prefix, $contentStreamId] = explode(':', $streamName->value, 2); - if (($prefix . ':') !== self::EVENT_STREAM_NAME_PREFIX) { - throw new \InvalidArgumentException(sprintf('Stream name is not a content stream event stream "%s"', $streamName->value)); - } - return ContentStreamId::fromString($contentStreamId); - } - public function getEventStreamName(): StreamName { return StreamName::fromString($this->value); From 52c54a2f76630e5d0676392e21d2c2e8e052080e Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:40:36 +0100 Subject: [PATCH 24/28] TASK: Minor cleanup in docs and introduce `isDangling` --- .../Classes/Service/ContentStreamPruner.php | 32 +++++++------------ .../ContentStreamForPruning.php | 5 +++ .../ContentStreamCommandController.php | 6 ++-- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 0f925b0b696..2893ade2dda 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -46,7 +46,7 @@ public function __construct( } /** - * Detects if dangling content streams exists and which content streams could be pruned from the event store + * Detects if dangling content streams exists and which content streams could be pruned from the event stream * * Dangling content streams * ------------------------ @@ -74,10 +74,7 @@ public function status(\Closure $outputFn): bool $danglingContentStreamPresent = false; foreach ($allContentStreams as $contentStream) { - if ($contentStream->removed) { - continue; - } - if ($contentStream->status === ContentStreamStatus::IN_USE_BY_WORKSPACE) { + if (!$contentStream->isDangling()) { continue; } if ($danglingContentStreamPresent === false) { @@ -95,7 +92,7 @@ public function status(\Closure $outputFn): bool if ($danglingContentStreamPresent === true) { $outputFn('To remove the dangling streams from the projections please run ./flow contentStream:removeDangling'); - $outputFn('Then they are ready for removal from the event store'); + $outputFn('Then they are ready for removal from the event stream'); $outputFn(); } else { $outputFn('Okay. No dangling streams found'); @@ -107,17 +104,17 @@ public function status(\Closure $outputFn): bool $pruneableContentStreamPresent = false; foreach ($removedContentStreams as $removedContentStream) { if ($pruneableContentStreamPresent === false) { - $outputFn('Removed content streams that can be pruned from the event store'); + $outputFn('Removed content streams that can be pruned from the event stream'); } $pruneableContentStreamPresent = true; $outputFn(sprintf(' id: %s previous state: %s', $removedContentStream->id->value, $removedContentStream->status->value)); } if ($pruneableContentStreamPresent === true) { - $outputFn('To prune the removed streams from the event store run ./flow contentStream:pruneRemovedFromEventstream'); - $outputFn('Then they are indefinitely pruned from the event store'); + $outputFn('To prune the removed streams from the event stream run ./flow contentStream:pruneRemovedFromEventstream'); + $outputFn('Then they are indefinitely pruned from the event stream'); } else { - $outputFn('Okay. No pruneable streams in the event store'); + $outputFn('Okay. No pruneable streams in the event stream'); } return !$danglingContentStreamPresent; @@ -128,7 +125,7 @@ public function status(\Closure $outputFn): bool * * NOTE: This still **keeps** the event stream as is; so it would be possible to re-construct the content stream at a later point in time. * - * To prune the removed content streams from the event store, call {@see ContentStreamPruner::pruneRemovedFromEventStream()} afterwards. + * To prune the removed content streams from the event stream, call {@see ContentStreamPruner::pruneRemovedFromEventStream()} afterwards. * * @param \DateTimeImmutable $removeTemporaryBefore includes all temporary content streams like FORKED or CREATED older than that in the removal */ @@ -138,14 +135,9 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta $unusedContentStreamsPresent = false; foreach ($allContentStreams as $contentStream) { - if ($contentStream->removed) { - continue; - } - - if ($contentStream->status === ContentStreamStatus::IN_USE_BY_WORKSPACE) { + if (!$contentStream->isDangling()) { continue; } - if ( $contentStream->status->isTemporary() && $removeTemporaryBefore < $contentStream->created @@ -176,7 +168,7 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta $outputFn(sprintf('Could not catchup after removing unused content streams: %s. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.', $e->getMessage())); } } else { - $outputFn('There are no unused content streams.'); + $outputFn('Okay. No pruneable streams in the event stream'); } } @@ -185,7 +177,7 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta * * This is not so easy for nested workspaces / content streams: * - As long as content streams are used as basis for others which are IN_USE_BY_WORKSPACE, - * these dependent Content Streams are not allowed to be removed in the event store. + * these dependent Content Streams are not allowed to be removed in the event stream. * * - Otherwise, we cannot replay the other content streams correctly (if the base content streams are missing). */ @@ -207,7 +199,7 @@ public function pruneRemovedFromEventStream(\Closure $outputFn): void } if ($unusedContentStreamsPresent === false) { - $outputFn('There are no unused content streams.'); + $outputFn('Okay. There are no pruneable content streams.'); } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php index bf3e99eac98..9264fc8b9f7 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner/ContentStreamForPruning.php @@ -103,4 +103,9 @@ public function withRemoved(): self true ); } + + public function isDangling(): bool + { + return !$this->removed && $this->status !== ContentStreamStatus::IN_USE_BY_WORKSPACE; + } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index cebaae5e84c..0670457b5ef 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -19,7 +19,7 @@ class ContentStreamCommandController extends CommandController protected $contentRepositoryRegistry; /** - * Detects if dangling content streams exists and which content streams could be pruned from the event store + * Detects if dangling content streams exists and which content streams could be pruned from the event stream * * Dangling content streams * ------------------------ @@ -61,7 +61,7 @@ public function statusCommand(string $contentRepository = 'default'): void * * HINT: ./flow contentStream:status gives information what is about to be removed * - * To prune the removed content streams from the event store, run ./flow contentStream:pruneRemovedFromEventStream afterwards. + * To prune the removed content streams from the event stream, run ./flow contentStream:pruneRemovedFromEventStream afterwards. * * @param string $contentRepository Identifier of the content repository. (Default: 'default') * @param string $removeTemporaryBefore includes all temporary content streams like FORKED or CREATED older than that in the removal @@ -87,7 +87,7 @@ public function removeDanglingCommand(string $contentRepository = 'default', str */ public function pruneRemovedFromEventStreamCommand(string $contentRepository = 'default', bool $force = false): void { - if (!$force && !$this->output->askConfirmation(sprintf('> This will prune removed content streams that are unused from the event stream in content repository "%s" (see contentStream:status). Are you sure to proceed? (y/n) ', $contentRepository), false)) { + if (!$force && !$this->output->askConfirmation(sprintf('> This will prune removed content streams that are unused from the event stream in content repository "%s" (see flow contentStream:status). Are you sure to proceed? (y/n) ', $contentRepository), false)) { $this->outputLine('Abort.'); return; } From e665abab91ffe29249fcc1e4775c903bec72a91f Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:05:21 +0100 Subject: [PATCH 25/28] TASK: Test content stream pruner status output --- .../Workspaces/PruneContentStreams.feature | 58 +++++++++++++------ .../Classes/Service/ContentStreamPruner.php | 3 +- .../Features/Bootstrap/CRTestSuiteTrait.php | 15 +++++ .../ContentStreamCommandController.php | 2 +- 4 files changed, 57 insertions(+), 21 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature index d630716b542..c64ec997a14 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature +++ b/Neos.ContentRepository.BehavioralTests/Tests/Behavior/Features/Workspaces/PruneContentStreams.feature @@ -28,6 +28,13 @@ Feature: If content streams are not in use anymore by the workspace, they can be Then I expect the content stream "non-existing" to not exist Then I expect the content stream "cs-identifier" to exist + Then I expect the content stream pruner status output: + """ + Okay. No dangling streams found + + Okay. No pruneable streams in the event stream + """ + Scenario: on creating a nested workspace, the new content stream is not pruned When the command CreateWorkspace is executed with payload: | Key | Value | @@ -36,23 +43,12 @@ Feature: If content streams are not in use anymore by the workspace, they can be | newContentStreamId | "user-cs-identifier" | Then I expect the content stream "user-cs-identifier" to exist - Scenario: when rebasing a nested workspace, the new content stream will not be pruned; but the old content stream is pruned. - When the command CreateWorkspace is executed with payload: - | Key | Value | - | workspaceName | "user-test" | - | baseWorkspaceName | "live" | - | newContentStreamId | "user-cs-identifier" | - When the command RebaseWorkspace is executed with payload: - | Key | Value | - | workspaceName | "user-test" | - | rebasedContentStreamId | "user-cs-identifier-rebased" | - | rebaseErrorHandlingStrategy | "force" | - - When I am in workspace "user-test" and dimension space point {} - Then I expect the content stream "user-cs-identifier-rebased" to exist - - Then I expect the content stream "user-cs-identifier" to not exist + Then I expect the content stream pruner status output: + """ + Okay. No dangling streams found + Okay. No pruneable streams in the event stream + """ Scenario: no longer in use content streams will be properly cleaned from the graph projection. When the command CreateWorkspace is executed with payload: @@ -85,16 +81,35 @@ Feature: If content streams are not in use anymore by the workspace, they can be | baseWorkspaceName | "live" | | newContentStreamId | "user-cs-identifier" | When the command RebaseWorkspace is executed with payload: - | Key | Value | - | workspaceName | "user-test" | + | Key | Value | + | workspaceName | "user-test" | + | rebasedContentStreamId | "user-cs-identifier-rebased" | | rebaseErrorHandlingStrategy | "force" | + Then I expect the content stream "user-cs-identifier-rebased" to exist + Then I expect the content stream "user-cs-identifier" to not exist + # now, we have one unused content stream (the old content stream of the user-test workspace) + Then I expect the content stream pruner status output: + """ + Okay. No dangling streams found + + Removed content streams that can be pruned from the event stream + id: user-cs-identifier previous state: no longer in use + To prune the removed streams from the event stream run ./flow contentStream:pruneRemovedFromEventstream + """ + And I prune removed content streams from the event stream Then I expect exactly 0 events to be published on stream "ContentStream:user-cs-identifier" + Then I expect the content stream pruner status output: + """ + Okay. No dangling streams found + + Okay. No pruneable streams in the event stream + """ Scenario: no longer in use content streams are only cleaned up if no other content stream which is still in use depends on it # we build a "review" workspace, and then a "user-test" workspace depending on the review workspace. @@ -121,3 +136,10 @@ Feature: If content streams are not in use anymore by the workspace, they can be # the events should still exist Then I expect exactly 3 events to be published on stream "ContentStream:review-cs-identifier" + + Then I expect the content stream pruner status output: + """ + Okay. No dangling streams found + + Okay. No pruneable streams in the event stream + """ diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 2893ade2dda..a6afcdbf81e 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -68,7 +68,7 @@ public function __construct( * * @return bool false if dangling content streams exist because they should not */ - public function status(\Closure $outputFn): bool + public function outputStatus(\Closure $outputFn): bool { $allContentStreams = $this->getContentStreamsForPruning(); @@ -112,7 +112,6 @@ public function status(\Closure $outputFn): bool if ($pruneableContentStreamPresent === true) { $outputFn('To prune the removed streams from the event stream run ./flow contentStream:pruneRemovedFromEventstream'); - $outputFn('Then they are indefinitely pruned from the event stream'); } else { $outputFn('Okay. No pruneable streams in the event stream'); } diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index 92e58e02ace..18fddc56f68 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -15,6 +15,7 @@ namespace Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap; use Behat\Behat\Hook\Scope\BeforeScenarioScope; +use Behat\Gherkin\Node\PyStringNode; use Behat\Gherkin\Node\TableNode; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; @@ -257,6 +258,20 @@ public function iPruneRemovedContentStreamsFromTheEventStream(): void $this->getContentRepositoryService(new ContentStreamPrunerFactory())->pruneRemovedFromEventStream(fn () => null); } + /** + * @When I expect the content stream pruner status output: + */ + public function iExpectTheContentStreamStatus(PyStringNode $pyStringNode): void + { + // todo a little dirty to compare the cli output here :D + $lines = []; + $this->getContentRepositoryService(new ContentStreamPrunerFactory())->outputStatus(function ($line = '') use (&$lines) { + $lines[] = $line; + }); + Assert::assertSame($pyStringNode->getRaw(), join("\n", $lines)); + } + + abstract protected function getContentRepositoryService( ContentRepositoryServiceFactoryInterface $factory ): ContentRepositoryServiceInterface; diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index 0670457b5ef..4a40dd2ea47 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -46,7 +46,7 @@ public function statusCommand(string $contentRepository = 'default'): void $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); - $status = $contentStreamPruner->status( + $status = $contentStreamPruner->outputStatus( $this->outputLine(...) ); if ($status === false) { From 00f6af77730b20d36f51856fb177c902b0667192 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:10:58 +0100 Subject: [PATCH 26/28] TASK: Introduce simple `flow migrateevents:backup` --- .../Command/MigrateEventsCommandController.php | 12 ++++++++++++ .../Classes/Service/EventMigrationService.php | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php index c0401b67d1c..b43c6023464 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php @@ -18,6 +18,18 @@ public function __construct( parent::__construct(); } + /** + * Temporary low level backup to ensure the prune migration https://github.com/neos/neos-development-collection/pull/5297 is safe + * + * @param string $contentRepository Identifier of the Content Repository to backup + */ + public function backupCommand(string $contentRepository = 'default'): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $eventMigrationService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->eventMigrationServiceFactory); + $eventMigrationService->backup($this->outputLine(...)); + } + /** * Migrates initial metadata & roles from the CR core workspaces to the corresponding Neos database tables * diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php index 9cd7d315f80..e88a6139e2b 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php @@ -56,6 +56,14 @@ public function __construct( ) { } + public function backup(\Closure $outputFn): void + { + $backupEventTableName = DoctrineEventStoreFactory::databaseTableName($this->contentRepositoryId) + . '_bkp_' . date('Y_m_d_H_i_s'); + $this->copyEventTable($backupEventTableName); + $outputFn(sprintf('Backup. Copied events table to %s', $backupEventTableName)); + } + /** * The following things have to be migrated: * From aa6a6b5a019e85f8c55c90ab1b4143441359a687 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:00:19 +0100 Subject: [PATCH 27/28] TASK: Adjust variables and namings --- .../Classes/Service/ContentStreamPruner.php | 126 ++++++++++-------- 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index a6afcdbf81e..5afbcddc590 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -70,7 +70,7 @@ public function __construct( */ public function outputStatus(\Closure $outputFn): bool { - $allContentStreams = $this->getContentStreamsForPruning(); + $allContentStreams = $this->findAllContentStreams(); $danglingContentStreamPresent = false; foreach ($allContentStreams as $contentStream) { @@ -99,15 +99,15 @@ public function outputStatus(\Closure $outputFn): bool $outputFn(); } - $removedContentStreams = $this->findUnusedAndRemovedContentStreamIds($allContentStreams); + $pruneableContentStreams = $this->findRemovedContentStreamsThatAreUnused($allContentStreams); $pruneableContentStreamPresent = false; - foreach ($removedContentStreams as $removedContentStream) { + foreach ($pruneableContentStreams as $pruneableContentStream) { if ($pruneableContentStreamPresent === false) { $outputFn('Removed content streams that can be pruned from the event stream'); } $pruneableContentStreamPresent = true; - $outputFn(sprintf(' id: %s previous state: %s', $removedContentStream->id->value, $removedContentStream->status->value)); + $outputFn(sprintf(' id: %s previous state: %s', $pruneableContentStream->id->value, $pruneableContentStream->status->value)); } if ($pruneableContentStreamPresent === true) { @@ -130,9 +130,9 @@ public function outputStatus(\Closure $outputFn): bool */ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmutable $removeTemporaryBefore): void { - $allContentStreams = $this->getContentStreamsForPruning(); + $allContentStreams = $this->findAllContentStreams(); - $unusedContentStreamsPresent = false; + $danglingContentStreamsPresent = false; foreach ($allContentStreams as $contentStream) { if (!$contentStream->isDangling()) { continue; @@ -157,10 +157,10 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta $outputFn(sprintf('Removed %s with status %s', $contentStream->id, $contentStream->status->value)); - $unusedContentStreamsPresent = true; + $danglingContentStreamsPresent = true; } - if ($unusedContentStreamsPresent) { + if ($danglingContentStreamsPresent) { try { $this->contentRepository->catchUpProjections(); } catch (\Throwable $e) { @@ -174,30 +174,28 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta /** * Prune removed content streams that are unused from the event stream; effectively REMOVING information completely. * - * This is not so easy for nested workspaces / content streams: - * - As long as content streams are used as basis for others which are IN_USE_BY_WORKSPACE, - * these dependent Content Streams are not allowed to be removed in the event stream. + * Note that replaying to only a previous point in time would not be possible anymore as workspace would reference non-existing content streams. * - * - Otherwise, we cannot replay the other content streams correctly (if the base content streams are missing). + * @see findRemovedContentStreamsThatAreUnused for implementation */ public function pruneRemovedFromEventStream(\Closure $outputFn): void { - $allContentStreams = $this->getContentStreamsForPruning(); + $allContentStreams = $this->findAllContentStreams(); - $removedContentStreams = $this->findUnusedAndRemovedContentStreamIds($allContentStreams); + $pruneableContentStreams = $this->findRemovedContentStreamsThatAreUnused($allContentStreams); - $unusedContentStreamsPresent = false; - foreach ($removedContentStreams as $removedContentStream) { + $pruneableContentStreamsPresent = false; + foreach ($pruneableContentStreams as $pruneableContentStream) { $this->eventStore->deleteStream( ContentStreamEventStreamName::fromContentStreamId( - $removedContentStream->id + $pruneableContentStream->id )->getEventStreamName() ); - $unusedContentStreamsPresent = true; - $outputFn(sprintf('Removed events for %s', $removedContentStream->id->value)); + $pruneableContentStreamsPresent = true; + $outputFn(sprintf('Removed events for %s', $pruneableContentStream->id->value)); } - if ($unusedContentStreamsPresent === false) { + if ($pruneableContentStreamsPresent === false) { $outputFn('Okay. There are no pruneable content streams.'); } } @@ -213,14 +211,24 @@ public function pruneAllWorkspacesAndContentStreamsFromEventStream(): void } /** + * Find all removed content streams that are unused in the event stream + * + * This is not so easy for nested workspaces / content streams: + * - As long as content streams are used as basis for others which are IN_USE_BY_WORKSPACE, + * these dependent Content Streams are not allowed to be removed in the event stream. + * - Otherwise, we cannot replay the other content streams correctly (if the base content streams are missing). + * * @param array $allContentStreams * @return list */ - private function findUnusedAndRemovedContentStreamIds(array $allContentStreams): array + private function findRemovedContentStreamsThatAreUnused(array $allContentStreams): array { /** @var array $transitiveUsedStreams */ $transitiveUsedStreams = []; - /** @var list $contentStreamIdsStack */ + /** + * Collection of content streams we iterate through to build up all streams that are in use transitively (by being a source content stream) or because it is in use + * @var list $contentStreamIdsStack + */ $contentStreamIdsStack = []; // Step 1: Find all content streams currently in direct use by a workspace @@ -249,20 +257,20 @@ private function findUnusedAndRemovedContentStreamIds(array $allContentStreams): } // Step 3: Check for removed content streams which we do not need anymore transitively - $removedContentStreams = []; + $removedContentStreamsThatAreUnused = []; foreach ($allContentStreams as $contentStream) { if ($contentStream->removed && !array_key_exists($contentStream->id->value, $transitiveUsedStreams)) { - $removedContentStreams[] = $contentStream; + $removedContentStreamsThatAreUnused[] = $contentStream; } } - return $removedContentStreams; + return $removedContentStreamsThatAreUnused; } /** * @return array */ - private function getContentStreamsForPruning(): array + private function findAllContentStreams(): array { $events = $this->eventStore->load( VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), @@ -275,14 +283,14 @@ private function getContentStreamsForPruning(): array ) ); - /** @var array $status */ - $status = []; + /** @var array $cs */ + $cs = []; foreach ($events as $eventEnvelope) { $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); switch ($domainEvent::class) { case ContentStreamWasCreated::class: - $status[$domainEvent->contentStreamId->value] = ContentStreamForPruning::create( + $cs[$domainEvent->contentStreamId->value] = ContentStreamForPruning::create( $domainEvent->contentStreamId, ContentStreamStatus::CREATED, null, @@ -290,7 +298,7 @@ private function getContentStreamsForPruning(): array ); break; case ContentStreamWasForked::class: - $status[$domainEvent->newContentStreamId->value] = ContentStreamForPruning::create( + $cs[$domainEvent->newContentStreamId->value] = ContentStreamForPruning::create( $domainEvent->newContentStreamId, ContentStreamStatus::FORKED, $domainEvent->sourceContentStreamId, @@ -298,8 +306,8 @@ private function getContentStreamsForPruning(): array ); break; case ContentStreamWasRemoved::class: - if (isset($status[$domainEvent->contentStreamId->value])) { - $status[$domainEvent->contentStreamId->value] = $status[$domainEvent->contentStreamId->value] + if (isset($cs[$domainEvent->contentStreamId->value])) { + $cs[$domainEvent->contentStreamId->value] = $cs[$domainEvent->contentStreamId->value] ->withRemoved(); } break; @@ -329,71 +337,71 @@ private function getContentStreamsForPruning(): array switch ($domainEvent::class) { case RootWorkspaceWasCreated::class: - if (isset($status[$domainEvent->newContentStreamId->value])) { - $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + if (isset($cs[$domainEvent->newContentStreamId->value])) { + $cs[$domainEvent->newContentStreamId->value] = $cs[$domainEvent->newContentStreamId->value] ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); } break; case WorkspaceWasCreated::class: - if (isset($status[$domainEvent->newContentStreamId->value])) { - $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + if (isset($cs[$domainEvent->newContentStreamId->value])) { + $cs[$domainEvent->newContentStreamId->value] = $cs[$domainEvent->newContentStreamId->value] ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); } break; case WorkspaceWasDiscarded::class: - if (isset($status[$domainEvent->newContentStreamId->value])) { - $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + if (isset($cs[$domainEvent->newContentStreamId->value])) { + $cs[$domainEvent->newContentStreamId->value] = $cs[$domainEvent->newContentStreamId->value] ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); } - if (isset($status[$domainEvent->previousContentStreamId->value])) { - $status[$domainEvent->previousContentStreamId->value] = $status[$domainEvent->previousContentStreamId->value] + if (isset($cs[$domainEvent->previousContentStreamId->value])) { + $cs[$domainEvent->previousContentStreamId->value] = $cs[$domainEvent->previousContentStreamId->value] ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); } break; case WorkspaceWasPartiallyDiscarded::class: - if (isset($status[$domainEvent->newContentStreamId->value])) { - $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + if (isset($cs[$domainEvent->newContentStreamId->value])) { + $cs[$domainEvent->newContentStreamId->value] = $cs[$domainEvent->newContentStreamId->value] ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); } - if (isset($status[$domainEvent->previousContentStreamId->value])) { - $status[$domainEvent->previousContentStreamId->value] = $status[$domainEvent->previousContentStreamId->value] + if (isset($cs[$domainEvent->previousContentStreamId->value])) { + $cs[$domainEvent->previousContentStreamId->value] = $cs[$domainEvent->previousContentStreamId->value] ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); } break; case WorkspaceWasPartiallyPublished::class: - if (isset($status[$domainEvent->newSourceContentStreamId->value])) { - $status[$domainEvent->newSourceContentStreamId->value] = $status[$domainEvent->newSourceContentStreamId->value] + if (isset($cs[$domainEvent->newSourceContentStreamId->value])) { + $cs[$domainEvent->newSourceContentStreamId->value] = $cs[$domainEvent->newSourceContentStreamId->value] ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); } - if (isset($status[$domainEvent->previousSourceContentStreamId->value])) { - $status[$domainEvent->previousSourceContentStreamId->value] = $status[$domainEvent->previousSourceContentStreamId->value] + if (isset($cs[$domainEvent->previousSourceContentStreamId->value])) { + $cs[$domainEvent->previousSourceContentStreamId->value] = $cs[$domainEvent->previousSourceContentStreamId->value] ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); } break; case WorkspaceWasPublished::class: - if (isset($status[$domainEvent->newSourceContentStreamId->value])) { - $status[$domainEvent->newSourceContentStreamId->value] = $status[$domainEvent->newSourceContentStreamId->value] + if (isset($cs[$domainEvent->newSourceContentStreamId->value])) { + $cs[$domainEvent->newSourceContentStreamId->value] = $cs[$domainEvent->newSourceContentStreamId->value] ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); } - if (isset($status[$domainEvent->previousSourceContentStreamId->value])) { - $status[$domainEvent->previousSourceContentStreamId->value] = $status[$domainEvent->previousSourceContentStreamId->value] + if (isset($cs[$domainEvent->previousSourceContentStreamId->value])) { + $cs[$domainEvent->previousSourceContentStreamId->value] = $cs[$domainEvent->previousSourceContentStreamId->value] ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); } break; case WorkspaceWasRebased::class: - if (isset($status[$domainEvent->newContentStreamId->value])) { - $status[$domainEvent->newContentStreamId->value] = $status[$domainEvent->newContentStreamId->value] + if (isset($cs[$domainEvent->newContentStreamId->value])) { + $cs[$domainEvent->newContentStreamId->value] = $cs[$domainEvent->newContentStreamId->value] ->withStatus(ContentStreamStatus::IN_USE_BY_WORKSPACE); } - if (isset($status[$domainEvent->previousContentStreamId->value])) { - $status[$domainEvent->previousContentStreamId->value] = $status[$domainEvent->previousContentStreamId->value] + if (isset($cs[$domainEvent->previousContentStreamId->value])) { + $cs[$domainEvent->previousContentStreamId->value] = $cs[$domainEvent->previousContentStreamId->value] ->withStatus(ContentStreamStatus::NO_LONGER_IN_USE); } break; case WorkspaceRebaseFailed::class: // legacy handling, as we previously kept failed candidateContentStreamId we make it behave like a ContentStreamWasRemoved event to clean up: - if (isset($status[$domainEvent->candidateContentStreamId->value])) { - $status[$domainEvent->candidateContentStreamId->value] = $status[$domainEvent->candidateContentStreamId->value] + if (isset($cs[$domainEvent->candidateContentStreamId->value])) { + $cs[$domainEvent->candidateContentStreamId->value] = $cs[$domainEvent->candidateContentStreamId->value] ->withRemoved(); } break; @@ -401,7 +409,7 @@ private function getContentStreamsForPruning(): array throw new \RuntimeException(sprintf('Unhandled event %s', $eventEnvelope->event->type->value)); } } - return $status; + return $cs; } /** From d6e7306e838bf3490cd9a81f5e099e0b3a7a1aff Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:44:34 +0100 Subject: [PATCH 28/28] TASK: Validate `--remove-temporary-before` --- .../ContentStreamCommandController.php | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php index 4a40dd2ea47..61325fb6a0e 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/ContentStreamCommandController.php @@ -63,17 +63,32 @@ public function statusCommand(string $contentRepository = 'default'): void * * To prune the removed content streams from the event stream, run ./flow contentStream:pruneRemovedFromEventStream afterwards. * + * NOTE: To ensure that no temporary content streams of the *current* moment are removed, a time threshold is configurable via --remove-temporary-before + * * @param string $contentRepository Identifier of the content repository. (Default: 'default') - * @param string $removeTemporaryBefore includes all temporary content streams like FORKED or CREATED older than that in the removal + * @param string $removeTemporaryBefore includes all temporary content streams like FORKED or CREATED older than that in the removal. To remove all use --remove-temporary-before=-1sec */ public function removeDanglingCommand(string $contentRepository = 'default', string $removeTemporaryBefore = '-1day'): void { $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentStreamPruner = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentStreamPrunerFactory()); + try { + $removeTemporaryBeforeDate = new \DateTimeImmutable($removeTemporaryBefore); + } catch (\Exception $exception) { + $this->outputLine(sprintf('--remove-temporary-before=%s is not a valid date: %s', $removeTemporaryBefore, $exception->getMessage())); + $this->quit(1); + } + + $now = new \DateTimeImmutable('now'); + if ($removeTemporaryBeforeDate > $now) { + $this->outputLine(sprintf('--remove-temporary-before=%s must be in the past', $removeTemporaryBefore)); + $this->quit(1); + } + $contentStreamPruner->removeDanglingContentStreams( $this->outputLine(...), - new \DateTimeImmutable($removeTemporaryBefore) + $removeTemporaryBeforeDate ); }