From 8a954495097c1b992a1786919599119dab607d4c Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 21 Oct 2024 13:53:46 +0200 Subject: [PATCH 01/56] WIP: FEATURE: Site import Resolves: #4448 --- .../Bootstrap/CrImportExportTrait.php | 209 ------------------ .../Features/Bootstrap/FeatureContext.php | 78 ------- .../Behavior/Features/Export/Export.feature | 43 ---- .../Behavior/Features/Import/Import.feature | 81 ------- .../Tests/Behavior/behat.yml.dist | 11 - .../src/Asset/AssetExporter.php | 5 +- .../src/ExportService.php | 63 ------ .../src/ExportServiceFactory.php | 39 ---- .../src/ImportService.php | 86 ------- .../src/ImportServiceFactory.php | 45 ---- .../src/ProcessingContext.php | 24 ++ .../src/ProcessorInterface.php | 13 +- .../src/ProcessorResult.php | 23 -- .../src/Processors.php | 37 ++++ .../src/Processors/AssetExportProcessor.php | 68 ++---- .../AssetRepositoryImportProcessor.php | 65 ++---- .../ContentRepositorySetupProcessor.php | 28 +++ .../src/Processors/EventExportProcessor.php | 52 ++--- .../Processors/EventStoreImportProcessor.php | 38 +--- .../Classes/LegacyMigrationService.php | 33 ++- .../Classes/NodeDataToAssetsProcessor.php | 44 +--- .../Classes/NodeDataToEventsProcessor.php | 82 +++---- .../Behavior/Bootstrap/FeatureContext.php | 38 ++-- .../Classes/Command/CrCommandController.php | 128 ----------- .../Classes/Command/SiteCommandController.php | 41 ++++ .../Import/DoctrineMigrateProcessor.php | 32 +++ .../Domain/Import/SiteCreationProcessor.php | 104 +++++++++ .../Domain/Service/SiteImportService.php | 53 +++++ .../Service/SiteImportServiceFactory.php | 67 ++++++ .../Classes/Domain/Service/SiteService.php | 2 +- 30 files changed, 535 insertions(+), 1097 deletions(-) delete mode 100644 Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php delete mode 100644 Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/FeatureContext.php delete mode 100644 Neos.ContentRepository.Export/Tests/Behavior/Features/Export/Export.feature delete mode 100644 Neos.ContentRepository.Export/Tests/Behavior/Features/Import/Import.feature delete mode 100644 Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist delete mode 100644 Neos.ContentRepository.Export/src/ExportService.php delete mode 100644 Neos.ContentRepository.Export/src/ExportServiceFactory.php delete mode 100644 Neos.ContentRepository.Export/src/ImportService.php delete mode 100644 Neos.ContentRepository.Export/src/ImportServiceFactory.php create mode 100644 Neos.ContentRepository.Export/src/ProcessingContext.php delete mode 100644 Neos.ContentRepository.Export/src/ProcessorResult.php create mode 100644 Neos.ContentRepository.Export/src/Processors.php create mode 100644 Neos.ContentRepository.Export/src/Processors/ContentRepositorySetupProcessor.php delete mode 100644 Neos.Neos/Classes/Command/CrCommandController.php create mode 100644 Neos.Neos/Classes/Domain/Import/DoctrineMigrateProcessor.php create mode 100644 Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php create mode 100644 Neos.Neos/Classes/Domain/Service/SiteImportService.php create mode 100644 Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php deleted file mode 100644 index 0aa571b20ea..00000000000 --- a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php +++ /dev/null @@ -1,209 +0,0 @@ - */ - private array $crImportExportTrait_loggedErrors = []; - - /** @var array */ - private array $crImportExportTrait_loggedWarnings = []; - - public function setupCrImportExportTrait() - { - $this->crImportExportTrait_filesystem = new Filesystem(new InMemoryFilesystemAdapter()); - } - - /** - * @When /^the events are exported$/ - */ - public function theEventsAreExportedIExpectTheFollowingJsonl() - { - $eventExporter = $this->getContentRepositoryService( - new class ($this->crImportExportTrait_filesystem) implements ContentRepositoryServiceFactoryInterface { - public function __construct(private readonly Filesystem $filesystem) - { - } - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): EventExportProcessor { - return new EventExportProcessor( - $this->filesystem, - $serviceFactoryDependencies->contentRepository->findWorkspaceByName(WorkspaceName::forLive()), - $serviceFactoryDependencies->eventStore - ); - } - } - ); - assert($eventExporter instanceof EventExportProcessor); - - $eventExporter->onMessage(function (Severity $severity, string $message) { - if ($severity === Severity::ERROR) { - $this->crImportExportTrait_loggedErrors[] = $message; - } elseif ($severity === Severity::WARNING) { - $this->crImportExportTrait_loggedWarnings[] = $message; - } - }); - $this->crImportExportTrait_lastMigrationResult = $eventExporter->run(); - } - - /** - * @When /^I import the events\.jsonl(?: into "([^"]*)")?$/ - */ - public function iImportTheFollowingJson(?string $contentStreamId = null) - { - $eventImporter = $this->getContentRepositoryService( - new class ($this->crImportExportTrait_filesystem, $contentStreamId ? ContentStreamId::fromString($contentStreamId) : null) implements ContentRepositoryServiceFactoryInterface { - public function __construct( - private readonly Filesystem $filesystem, - private readonly ?ContentStreamId $contentStreamId - ) { - } - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): EventStoreImportProcessor { - return new EventStoreImportProcessor( - false, - $this->filesystem, - $serviceFactoryDependencies->eventStore, - $serviceFactoryDependencies->eventNormalizer, - $this->contentStreamId - ); - } - } - ); - assert($eventImporter instanceof EventStoreImportProcessor); - - $eventImporter->onMessage(function (Severity $severity, string $message) { - if ($severity === Severity::ERROR) { - $this->crImportExportTrait_loggedErrors[] = $message; - } elseif ($severity === Severity::WARNING) { - $this->crImportExportTrait_loggedWarnings[] = $message; - } - }); - $this->crImportExportTrait_lastMigrationResult = $eventImporter->run(); - } - - /** - * @Given /^using the following events\.jsonl:$/ - */ - public function usingTheFollowingEventsJsonl(PyStringNode $string) - { - $this->crImportExportTrait_filesystem->write('events.jsonl', $string->getRaw()); - } - - /** - * @AfterScenario - */ - public function failIfLastMigrationHasErrors(): void - { - if ($this->crImportExportTrait_lastMigrationResult !== null && $this->crImportExportTrait_lastMigrationResult->severity === Severity::ERROR) { - throw new \RuntimeException(sprintf('The last migration run led to an error: %s', $this->crImportExportTrait_lastMigrationResult->message)); - } - if ($this->crImportExportTrait_loggedErrors !== []) { - throw new \RuntimeException(sprintf('The last migration run logged %d error%s', count($this->crImportExportTrait_loggedErrors), count($this->crImportExportTrait_loggedErrors) === 1 ? '' : 's')); - } - } - - /** - * @Then I expect the following jsonl: - */ - public function iExpectTheFollowingJsonL(PyStringNode $string): void - { - if (!$this->crImportExportTrait_filesystem->has('events.jsonl')) { - Assert::fail('No events were exported'); - } - - $jsonL = $this->crImportExportTrait_filesystem->read('events.jsonl'); - - $exportedEvents = ExportedEvents::fromJsonl($jsonL); - $eventsWithoutRandomIds = []; - - foreach ($exportedEvents as $exportedEvent) { - // we have to remove the event id in \Neos\ContentRepository\Core\Feature\Common\NodeAggregateEventPublisher::enrichWithCommand - // and the initiatingTimestamp to make the events diff able - $eventsWithoutRandomIds[] = $exportedEvent - ->withIdentifier('random-event-uuid') - ->processMetadata(function (array $metadata) { - $metadata['initiatingTimestamp'] = 'random-time'; - return $metadata; - }); - } - - Assert::assertSame($string->getRaw(), ExportedEvents::fromIterable($eventsWithoutRandomIds)->toJsonl()); - } - - /** - * @Then I expect the following errors to be logged - */ - public function iExpectTheFollowingErrorsToBeLogged(TableNode $table): void - { - Assert::assertSame($table->getColumn(0), $this->crImportExportTrait_loggedErrors, 'Expected logged errors do not match'); - $this->crImportExportTrait_loggedErrors = []; - } - - /** - * @Then I expect the following warnings to be logged - */ - public function iExpectTheFollowingWarningsToBeLogged(TableNode $table): void - { - Assert::assertSame($table->getColumn(0), $this->crImportExportTrait_loggedWarnings, 'Expected logged warnings do not match'); - $this->crImportExportTrait_loggedWarnings = []; - } - - /** - * @Then I expect a MigrationError - * @Then I expect a MigrationError with the message - */ - public function iExpectAMigrationErrorWithTheMessage(PyStringNode $expectedMessage = null): void - { - Assert::assertNotNull($this->crImportExportTrait_lastMigrationResult, 'Expected the previous migration to contain errors, but no migration has been executed'); - Assert::assertSame(Severity::ERROR, $this->crImportExportTrait_lastMigrationResult->severity, sprintf('Expected the previous migration to contain errors, but it ended with severity "%s"', $this->crImportExportTrait_lastMigrationResult->severity->name)); - if ($expectedMessage !== null) { - Assert::assertSame($expectedMessage->getRaw(), $this->crImportExportTrait_lastMigrationResult->message); - } - $this->crImportExportTrait_lastMigrationResult = null; - } - - /** - * @template T of object - * @param class-string $className - * - * @return T - */ - abstract private function getObject(string $className): object; -} diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/FeatureContext.php b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/FeatureContext.php deleted file mode 100644 index 01310c7beff..00000000000 --- a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/FeatureContext.php +++ /dev/null @@ -1,78 +0,0 @@ -contentRepositoryRegistry = $this->getObject(ContentRepositoryRegistry::class); - - $this->setupCrImportExportTrait(); - } - - /** - * @BeforeScenario - */ - public function resetContentRepositoryComponents(BeforeScenarioScope $scope): void - { - GherkinTableNodeBasedContentDimensionSourceFactory::reset(); - GherkinPyStringNodeBasedNodeTypeManagerFactory::reset(); - } - - protected function getContentRepositoryService( - ContentRepositoryServiceFactoryInterface $factory - ): ContentRepositoryServiceInterface { - return $this->contentRepositoryRegistry->buildService( - $this->currentContentRepository->id, - $factory - ); - } - - protected function createContentRepository( - ContentRepositoryId $contentRepositoryId - ): ContentRepository { - $this->contentRepositoryRegistry->resetFactoryInstance($contentRepositoryId); - $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); - GherkinTableNodeBasedContentDimensionSourceFactory::reset(); - GherkinPyStringNodeBasedNodeTypeManagerFactory::reset(); - - return $contentRepository; - } -} diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/Export/Export.feature b/Neos.ContentRepository.Export/Tests/Behavior/Features/Export/Export.feature deleted file mode 100644 index 99ecc22b627..00000000000 --- a/Neos.ContentRepository.Export/Tests/Behavior/Features/Export/Export.feature +++ /dev/null @@ -1,43 +0,0 @@ -@contentrepository -Feature: As a user of the CR I want to export the event stream - - Background: - Given using the following content dimensions: - | Identifier | Values | Generalizations | - | language | de, gsw, fr | gsw->de | - And using the following node types: - """yaml - 'Neos.ContentRepository.Testing:Document': [] - """ - And using identifier "default", I define a content repository - And I am in content repository "default" - And the command CreateRootWorkspace is executed with payload: - | Key | Value | - | workspaceName | "live" | - | newContentStreamId | "cs-identifier" | - And I am in workspace "live" - And the command CreateRootNodeAggregateWithNode is executed with payload: - | Key | Value | - | nodeAggregateId | "lady-eleonode-rootford" | - | nodeTypeName | "Neos.ContentRepository:Root" | - And the event NodeAggregateWithNodeWasCreated was published with payload: - | Key | Value | - | workspaceName | "live" | - | contentStreamId | "cs-identifier" | - | nodeAggregateId | "nody-mc-nodeface" | - | nodeTypeName | "Neos.ContentRepository.Testing:Document" | - | originDimensionSpacePoint | {"language":"de"} | - | coveredDimensionSpacePoints | [{"language":"de"},{"language":"gsw"},{"language":"fr"}] | - | parentNodeAggregateId | "lady-eleonode-rootford" | - | nodeName | "child-document" | - | nodeAggregateClassification | "regular" | - - Scenario: Export the event stream - Then I expect exactly 3 events to be published on stream with prefix "ContentStream:cs-identifier" - When the events are exported - Then I expect the following jsonl: - """ - {"identifier":"random-event-uuid","type":"RootNodeAggregateWithNodeWasCreated","payload":{"workspaceName":"live","contentStreamId":"cs-identifier","nodeAggregateId":"lady-eleonode-rootford","nodeTypeName":"Neos.ContentRepository:Root","coveredDimensionSpacePoints":[{"language":"de"},{"language":"gsw"},{"language":"fr"}],"nodeAggregateClassification":"root"},"metadata":{"commandClass":"Neos\\ContentRepository\\Core\\Feature\\RootNodeCreation\\Command\\CreateRootNodeAggregateWithNode","commandPayload":{"workspaceName":"live","nodeAggregateId":"lady-eleonode-rootford","nodeTypeName":"Neos.ContentRepository:Root","tetheredDescendantNodeAggregateIds":[]},"initiatingUserId":"system","initiatingTimestamp":"random-time"}} - {"identifier":"random-event-uuid","type":"NodeAggregateWithNodeWasCreated","payload":{"workspaceName":"live","contentStreamId":"cs-identifier","nodeAggregateId":"nody-mc-nodeface","nodeTypeName":"Neos.ContentRepository.Testing:Document","originDimensionSpacePoint":{"language":"de"},"succeedingSiblingsForCoverage":[{"dimensionSpacePoint":{"language":"de"},"nodeAggregateId":null},{"dimensionSpacePoint":{"language":"gsw"},"nodeAggregateId":null},{"dimensionSpacePoint":{"language":"fr"},"nodeAggregateId":null}],"parentNodeAggregateId":"lady-eleonode-rootford","nodeName":"child-document","initialPropertyValues":[],"nodeAggregateClassification":"regular"},"metadata":{"initiatingTimestamp":"random-time"}} - - """ diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/Import/Import.feature b/Neos.ContentRepository.Export/Tests/Behavior/Features/Import/Import.feature deleted file mode 100644 index 6c61f644b57..00000000000 --- a/Neos.ContentRepository.Export/Tests/Behavior/Features/Import/Import.feature +++ /dev/null @@ -1,81 +0,0 @@ -@contentrepository -Feature: As a user of the CR I want to export the event stream - - Background: - Given using no content dimensions - And using the following node types: - """yaml - Vendor.Site:HomePage': - superTypes: - Neos.Neos:Site: true - """ - And using identifier "default", I define a content repository - And I am in content repository "default" - - Scenario: Import the event stream into a specific content stream - Then I expect exactly 0 events to be published on stream with prefix "ContentStream:cs-identifier" - Given using the following events.jsonl: - """ - {"identifier":"9f64c281-e5b0-48d9-900b-288a8faf92a9","type":"RootNodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site-sites","nodeTypeName":"Neos.Neos:Sites","coveredDimensionSpacePoints":[[]],"nodeAggregateClassification":"root"},"metadata":[]} - {"identifier":"1640ebbf-7ffe-4526-b0f4-7575cefabfab","type":"NodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site","nodeTypeName":"Vendor.Site:HomePage","originDimensionSpacePoint":[],"succeedingSiblingsForCoverage":[{"dimensionSpacePoint":[],"nodeAggregateId":null}],"parentNodeAggregateId":"acme-site-sites","nodeName":"acme-site","initialPropertyValues":{"title":{"value":"My Site","type":"string"},"uriPathSegment":{"value":"my-site","type":"string"}},"nodeAggregateClassification":"regular"},"metadata":[]} - """ - And I import the events.jsonl into "cs-identifier" - Then I expect exactly 3 events to be published on stream with prefix "ContentStream:cs-identifier" - And event at index 0 is of type "ContentStreamWasCreated" with payload: - | Key | Expected | - | contentStreamId | "cs-identifier" | - And event at index 1 is of type "RootNodeAggregateWithNodeWasCreated" with payload: - | Key | Expected | - | workspaceName | "workspace-name" | - | contentStreamId | "cs-identifier" | - | nodeAggregateId | "acme-site-sites" | - | nodeTypeName | "Neos.Neos:Sites" | - And event at index 2 is of type "NodeAggregateWithNodeWasCreated" with payload: - | Key | Expected | - | workspaceName | "workspace-name" | - | contentStreamId | "cs-identifier" | - | nodeAggregateId | "acme-site" | - | nodeTypeName | "Vendor.Site:HomePage" | - - Scenario: Import the event stream - Then I expect exactly 0 events to be published on stream with prefix "ContentStream:cs-imported-identifier" - Given using the following events.jsonl: - """ - {"identifier":"9f64c281-e5b0-48d9-900b-288a8faf92a9","type":"RootNodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site-sites","nodeTypeName":"Neos.Neos:Sites","coveredDimensionSpacePoints":[[]],"nodeAggregateClassification":"root"},"metadata":[]} - {"identifier":"1640ebbf-7ffe-4526-b0f4-7575cefabfab","type":"NodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site","nodeTypeName":"Vendor.Site:HomePage","originDimensionSpacePoint":[],"succeedingSiblingsForCoverage":[{"dimensionSpacePoint":[],"nodeAggregateId":null}],"parentNodeAggregateId":"acme-site-sites","nodeName":"acme-site","initialPropertyValues":{"title":{"value":"My Site","type":"string"},"uriPathSegment":{"value":"my-site","type":"string"}},"nodeAggregateClassification":"regular"},"metadata":[]} - """ - And I import the events.jsonl - Then I expect exactly 3 events to be published on stream with prefix "ContentStream:cs-imported-identifier" - And event at index 0 is of type "ContentStreamWasCreated" with payload: - | Key | Expected | - | contentStreamId | "cs-imported-identifier" | - And event at index 1 is of type "RootNodeAggregateWithNodeWasCreated" with payload: - | Key | Expected | - | workspaceName | "workspace-name" | - | contentStreamId | "cs-imported-identifier" | - | nodeAggregateId | "acme-site-sites" | - | nodeTypeName | "Neos.Neos:Sites" | - And event at index 2 is of type "NodeAggregateWithNodeWasCreated" with payload: - | Key | Expected | - | workspaceName | "workspace-name" | - | contentStreamId | "cs-imported-identifier" | - | nodeAggregateId | "acme-site" | - | nodeTypeName | "Vendor.Site:HomePage" | - - Scenario: Import faulty event stream with explicit "ContentStreamWasCreated" does not duplicate content-stream - see issue https://github.com/neos/neos-development-collection/issues/4298 - - Given using the following events.jsonl: - """ - {"identifier":"5f2da12d-7037-4524-acb0-d52037342c77","type":"ContentStreamWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier"},"metadata":[]} - {"identifier":"9f64c281-e5b0-48d9-900b-288a8faf92a9","type":"RootNodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site-sites","nodeTypeName":"Neos.Neos:Sites","coveredDimensionSpacePoints":[[]],"nodeAggregateClassification":"root"},"metadata":[]} - {"identifier":"1640ebbf-7ffe-4526-b0f4-7575cefabfab","type":"NodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site","nodeTypeName":"Vendor.Site:HomePage","originDimensionSpacePoint":[],"coveredDimensionSpacePoints":[[]],"parentNodeAggregateId":"acme-site-sites","nodeName":"acme-site","initialPropertyValues":{"title":{"value":"My Site","type":"string"},"uriPathSegment":{"value":"my-site","type":"string"}},"nodeAggregateClassification":"regular","succeedingNodeAggregateId":null},"metadata":[]} - """ - And I import the events.jsonl - - And I expect a MigrationError with the message - """ - Failed to read events. ContentStreamWasCreated is not expected in imported event stream. - """ - - Then I expect exactly 0 events to be published on stream with prefix "ContentStream:cs-imported-identifier" diff --git a/Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist b/Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist deleted file mode 100644 index 11e2845b599..00000000000 --- a/Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist +++ /dev/null @@ -1,11 +0,0 @@ - -default: - autoload: - '': "%paths.base%/Features/Bootstrap" - suites: - cr: - paths: - - "%paths.base%/Features" - - contexts: - - FeatureContext diff --git a/Neos.ContentRepository.Export/src/Asset/AssetExporter.php b/Neos.ContentRepository.Export/src/Asset/AssetExporter.php index 3ce343eebb2..0dc915dbf03 100644 --- a/Neos.ContentRepository.Export/src/Asset/AssetExporter.php +++ b/Neos.ContentRepository.Export/src/Asset/AssetExporter.php @@ -1,5 +1,7 @@ $processors */ - $processors = [ - 'Exporting events' => new EventExportProcessor( - $this->filesystem, - $this->targetWorkspace, - $this->eventStore - ), - 'Exporting assets' => new AssetExportProcessor( - $this->contentRepositoryId, - $this->filesystem, - $this->assetRepository, - $this->targetWorkspace, - $this->assetUsageService - ) - ]; - - foreach ($processors as $label => $processor) { - $outputLineFn($label . '...'); - $verbose && $processor->onMessage( - fn(Severity $severity, string $message) => $outputLineFn('<%1$s>%2$s', [$severity === Severity::ERROR ? 'error' : 'comment', $message]) - ); - $result = $processor->run(); - if ($result->severity === Severity::ERROR) { - throw new \RuntimeException($label . ': ' . ($result->message ?? '')); - } - $outputLineFn(' ' . $result->message); - $outputLineFn(); - } - } -} diff --git a/Neos.ContentRepository.Export/src/ExportServiceFactory.php b/Neos.ContentRepository.Export/src/ExportServiceFactory.php deleted file mode 100644 index e36d50be983..00000000000 --- a/Neos.ContentRepository.Export/src/ExportServiceFactory.php +++ /dev/null @@ -1,39 +0,0 @@ - - */ -class ExportServiceFactory implements ContentRepositoryServiceFactoryInterface -{ - - public function __construct( - private readonly Filesystem $filesystem, - private readonly Workspace $targetWorkspace, - private readonly AssetRepository $assetRepository, - private readonly AssetUsageService $assetUsageService, - ) { - } - - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ExportService - { - return new ExportService( - $serviceFactoryDependencies->contentRepositoryId, - $this->filesystem, - $this->targetWorkspace, - $this->assetRepository, - $this->assetUsageService, - $serviceFactoryDependencies->eventStore, - ); - } -} diff --git a/Neos.ContentRepository.Export/src/ImportService.php b/Neos.ContentRepository.Export/src/ImportService.php deleted file mode 100644 index de06ced5b3c..00000000000 --- a/Neos.ContentRepository.Export/src/ImportService.php +++ /dev/null @@ -1,86 +0,0 @@ -liveWorkspaceContentStreamExists()) { - throw new LiveWorkspaceContentStreamExistsException(); - } - - /** @var ProcessorInterface[] $processors */ - $processors = [ - 'Importing assets' => new AssetRepositoryImportProcessor( - $this->filesystem, - $this->assetRepository, - $this->resourceRepository, - $this->resourceManager, - $this->persistenceManager, - ), - 'Importing events' => new EventStoreImportProcessor( - false, - $this->filesystem, - $this->eventStore, - $this->eventNormalizer, - $this->contentStreamIdentifier, - ) - ]; - - foreach ($processors as $label => $processor) { - $outputLineFn($label . '...'); - $verbose && $processor->onMessage( - fn(Severity $severity, string $message) => $outputLineFn('<%1$s>%2$s', [$severity === Severity::ERROR ? 'error' : 'comment', $message]) - ); - $result = $processor->run(); - if ($result->severity === Severity::ERROR) { - throw new \RuntimeException($label . ': ' . ($result->message ?? '')); - } - $outputLineFn(' ' . $result->message); - $outputLineFn(); - } - } - - private function liveWorkspaceContentStreamExists(): bool - { - $workspaceStreamName = WorkspaceEventStreamName::fromWorkspaceName(WorkspaceName::forLive())->getEventStreamName(); - $eventStream = $this->eventStore->load($workspaceStreamName); - foreach ($eventStream as $event) { - return true; - } - return false; - } -} diff --git a/Neos.ContentRepository.Export/src/ImportServiceFactory.php b/Neos.ContentRepository.Export/src/ImportServiceFactory.php deleted file mode 100644 index b5504818664..00000000000 --- a/Neos.ContentRepository.Export/src/ImportServiceFactory.php +++ /dev/null @@ -1,45 +0,0 @@ - - */ -class ImportServiceFactory implements ContentRepositoryServiceFactoryInterface -{ - - public function __construct( - private readonly Filesystem $filesystem, - private readonly ContentStreamId $contentStreamIdentifier, - private readonly AssetRepository $assetRepository, - private readonly ResourceRepository $resourceRepository, - private readonly ResourceManager $resourceManager, - private readonly PersistenceManagerInterface $persistenceManager, - ) { - } - - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ImportService - { - return new ImportService( - $this->filesystem, - $this->contentStreamIdentifier, - $this->assetRepository, - $this->resourceRepository, - $this->resourceManager, - $this->persistenceManager, - $serviceFactoryDependencies->eventNormalizer, - $serviceFactoryDependencies->eventStore, - ); - } -} diff --git a/Neos.ContentRepository.Export/src/ProcessingContext.php b/Neos.ContentRepository.Export/src/ProcessingContext.php new file mode 100644 index 00000000000..cd6fd16bba0 --- /dev/null +++ b/Neos.ContentRepository.Export/src/ProcessingContext.php @@ -0,0 +1,24 @@ +onEvent)($severity, $message); + } +} diff --git a/Neos.ContentRepository.Export/src/ProcessorInterface.php b/Neos.ContentRepository.Export/src/ProcessorInterface.php index 6ee2a5246d7..e033c200256 100644 --- a/Neos.ContentRepository.Export/src/ProcessorInterface.php +++ b/Neos.ContentRepository.Export/src/ProcessorInterface.php @@ -1,14 +1,13 @@ + */ +final readonly class Processors implements \IteratorAggregate, \Countable +{ + /** + * @param array $processors + */ + private function __construct( + private array $processors + ) { + } + + /** + * @param array $processors + */ + public static function fromArray(array $processors): self + { + return new self($processors); + } + + public function getIterator(): \Traversable + { + yield from $this->processors; + } + + public function count(): int + { + return count($this->processors); + } +} diff --git a/Neos.ContentRepository.Export/src/Processors/AssetExportProcessor.php b/Neos.ContentRepository.Export/src/Processors/AssetExportProcessor.php index 3bceb6dc77f..78deda81ee3 100644 --- a/Neos.ContentRepository.Export/src/Processors/AssetExportProcessor.php +++ b/Neos.ContentRepository.Export/src/Processors/AssetExportProcessor.php @@ -1,14 +1,15 @@ */ - private array $callbacks = []; - public function __construct( private readonly ContentRepositoryId $contentRepositoryId, - private readonly Filesystem $files, private readonly AssetRepository $assetRepository, private readonly Workspace $targetWorkspace, private readonly AssetUsageService $assetUsageService, - ) {} - - public function onMessage(\Closure $callback): void - { - $this->callbacks[] = $callback; + ) { } - - public function run(): ProcessorResult + public function run(ProcessingContext $context): void { $assetFilter = AssetUsageFilter::create()->withWorkspaceName($this->targetWorkspace->workspaceName)->groupByAsset(); - $numberOfExportedAssets = 0; - $numberOfExportedImageVariants = 0; - $numberOfErrors = 0; - foreach ($this->assetUsageService->findByFilter($this->contentRepositoryId, $assetFilter) as $assetUsage) { /** @var Asset|null $asset */ $asset = $this->assetRepository->findByIdentifier($assetUsage->assetId); if ($asset === null) { - $numberOfErrors ++; - $this->dispatch(Severity::ERROR, 'Skipping asset "%s" because it does not exist in the database', $assetUsage->assetId); + $context->dispatch(Severity::ERROR, "Skipping asset \"{$assetUsage->assetId}\" because it does not exist in the database"); continue; } @@ -63,64 +50,47 @@ public function run(): ProcessorResult /** @var Asset $originalAsset */ $originalAsset = $asset->getOriginalAsset(); try { - $this->exportAsset($originalAsset); - $numberOfExportedAssets ++; + $this->exportAsset($context, $originalAsset); } catch (\Throwable $e) { - $numberOfErrors ++; - $this->dispatch(Severity::ERROR, 'Failed to export original asset "%s" (for variant "%s"): %s', $originalAsset->getIdentifier(), $asset->getIdentifier(), $e->getMessage()); + $context->dispatch(Severity::ERROR, "Failed to export original asset \"{$originalAsset->getIdentifier()}\" (for variant \"{$asset->getIdentifier()}\"): {$e->getMessage()}"); } } try { - $this->exportAsset($asset); - if ($asset instanceof AssetVariantInterface) { - $numberOfExportedImageVariants ++; - } else { - $numberOfExportedAssets ++; - } + $this->exportAsset($context, $asset); } catch (\Throwable $e) { - $numberOfErrors ++; - $this->dispatch(Severity::ERROR, 'Failed to export asset "%s": %s', $asset->getIdentifier(), $e->getMessage()); + $context->dispatch(Severity::ERROR, "Failed to export asset \"{$asset->getIdentifier()}\": {$e->getMessage()}"); } } - return ProcessorResult::success(sprintf('Exported %d Asset%s and %d Image Variant%s. Errors: %d', $numberOfExportedAssets, $numberOfExportedAssets === 1 ? '' : 's', $numberOfExportedImageVariants, $numberOfExportedImageVariants === 1 ? '' : 's', $numberOfErrors)); } /** --------------------------------------- */ - private function exportAsset(Asset $asset): void + private function exportAsset(ProcessingContext $context, Asset $asset): void { $fileLocation = $asset instanceof ImageVariant ? "ImageVariants/{$asset->getIdentifier()}.json" : "Assets/{$asset->getIdentifier()}.json"; - if ($this->files->has($fileLocation)) { + if ($context->files->has($fileLocation)) { return; } if ($asset instanceof ImageVariant) { - $this->files->write($fileLocation, SerializedImageVariant::fromImageVariant($asset)->toJson()); + $context->files->write($fileLocation, SerializedImageVariant::fromImageVariant($asset)->toJson()); return; } /** @var PersistentResource|null $resource */ $resource = $asset->getResource(); if ($resource === null) { - $this->dispatch(Severity::ERROR, 'Skipping asset "%s" because the corresponding PersistentResource does not exist in the database', $asset->getIdentifier()); + $context->dispatch(Severity::ERROR, "Skipping asset \"{$asset->getIdentifier()}\" because the corresponding PersistentResource does not exist in the database"); return; } - $this->files->write($fileLocation, SerializedAsset::fromAsset($asset)->toJson()); - $this->exportResource($resource); + $context->files->write($fileLocation, SerializedAsset::fromAsset($asset)->toJson()); + $this->exportResource($context, $resource); } - private function exportResource(PersistentResource $resource): void + private function exportResource(ProcessingContext $context, PersistentResource $resource): void { $fileLocation = "Resources/{$resource->getSha1()}"; - if ($this->files->has($fileLocation)) { + if ($context->files->has($fileLocation)) { return; } - $this->files->writeStream($fileLocation, $resource->getStream()); - } - - private function dispatch(Severity $severity, string $message, mixed ...$args): void - { - $renderedMessage = sprintf($message, ...$args); - foreach ($this->callbacks as $callback) { - $callback($severity, $renderedMessage); - } + $context->files->writeStream($fileLocation, $resource->getStream()); } } diff --git a/Neos.ContentRepository.Export/src/Processors/AssetRepositoryImportProcessor.php b/Neos.ContentRepository.Export/src/Processors/AssetRepositoryImportProcessor.php index b389adababa..2021098fb19 100644 --- a/Neos.ContentRepository.Export/src/Processors/AssetRepositoryImportProcessor.php +++ b/Neos.ContentRepository.Export/src/Processors/AssetRepositoryImportProcessor.php @@ -1,15 +1,16 @@ */ - private array $callbacks = []; - public function __construct( - private readonly Filesystem $files, private readonly AssetRepository $assetRepository, private readonly ResourceRepository $resourceRepository, private readonly ResourceManager $resourceManager, private readonly PersistenceManagerInterface $persistenceManager, - ) {} - - public function onMessage(\Closure $callback): void - { - $this->callbacks[] = $callback; + ) { } - public function run(): ProcessorResult + public function run(ProcessingContext $context): void { $this->persistenceManager->clearState(); - $numberOfErrors = 0; - $numberOfImportedAssets = 0; - foreach ($this->files->listContents('/Assets') as $file) { + foreach ($context->files->listContents('/Assets') as $file) { if (!$file->isFile()) { continue; } try { - $this->importAsset($file); - $numberOfImportedAssets ++; + $this->importAsset($context, $file); } catch (\Throwable $e) { - $numberOfErrors ++; - $this->dispatch(Severity::ERROR, 'Failed to import asset from file "%s": %s', $file->path(), $e->getMessage()); + $context->dispatch(Severity::ERROR, "Failed to import asset from file \"{$file->path()}\": {$e->getMessage()}"); } } - $numberOfImportedImageVariants = 0; - foreach ($this->files->listContents('/ImageVariants') as $file) { + foreach ($context->files->listContents('/ImageVariants') as $file) { if (!$file->isFile()) { continue; } try { - $this->importImageVariant($file); - $numberOfImportedImageVariants ++; + $this->importImageVariant($context, $file); } catch (\Throwable $e) { - $numberOfErrors ++; - $this->dispatch(Severity::ERROR, 'Failed to import image variant from file "%s": %s', $file->path(), $e->getMessage()); + $context->dispatch(Severity::ERROR, "Failed to import image variant from file \"{$file->path()}\": {$e->getMessage()}"); } } - return ProcessorResult::success(sprintf('Imported %d Asset%s and %d Image Variant%s. Errors: %d', $numberOfImportedAssets, $numberOfImportedAssets === 1 ? '' : 's', $numberOfImportedImageVariants, $numberOfImportedImageVariants === 1 ? '' : 's', $numberOfErrors)); } /** --------------------------------------- */ - private function importAsset(StorageAttributes $file): void + private function importAsset(ProcessingContext $context, StorageAttributes $file): void { - $fileContents = $this->files->read($file->path()); + $fileContents = $context->files->read($file->path()); $serializedAsset = SerializedAsset::fromJson($fileContents); /** @var Asset|null $existingAsset */ $existingAsset = $this->assetRepository->findByIdentifier($serializedAsset->identifier); if ($existingAsset !== null) { if ($serializedAsset->matches($existingAsset)) { - $this->dispatch(Severity::NOTICE, 'Asset "%s" was skipped because it already exists!', $serializedAsset->identifier); + $context->dispatch(Severity::NOTICE, "Asset \"{$serializedAsset->identifier}\" was skipped because it already exists!"); } else { - $this->dispatch(Severity::ERROR, 'Asset "%s" has been changed in the meantime, it was NOT updated!', $serializedAsset->identifier); + $context->dispatch(Severity::ERROR, "Asset \"{$serializedAsset->identifier}\" has been changed in the meantime, it was NOT updated!"); } return; } /** @var PersistentResource|null $resource */ $resource = $this->resourceRepository->findBySha1AndCollectionName($serializedAsset->resource->sha1, $serializedAsset->resource->collectionName)[0] ?? null; if ($resource === null) { - $content = $this->files->read('/Resources/' . $serializedAsset->resource->sha1); + $content = $context->files->read('/Resources/' . $serializedAsset->resource->sha1); $resource = $this->resourceManager->importResourceFromContent($content, $serializedAsset->resource->filename, $serializedAsset->resource->collectionName); $resource->setMediaType($serializedAsset->resource->mediaType); } @@ -120,23 +105,23 @@ private function importAsset(StorageAttributes $file): void $this->persistenceManager->persistAll(); } - private function importImageVariant(StorageAttributes $file): void + private function importImageVariant(ProcessingContext $context, StorageAttributes $file): void { - $fileContents = $this->files->read($file->path()); + $fileContents = $context->files->read($file->path()); $serializedImageVariant = SerializedImageVariant::fromJson($fileContents); $existingImageVariant = $this->assetRepository->findByIdentifier($serializedImageVariant->identifier); assert($existingImageVariant === null || $existingImageVariant instanceof ImageVariant); if ($existingImageVariant !== null) { if ($serializedImageVariant->matches($existingImageVariant)) { - $this->dispatch(Severity::NOTICE, 'Image Variant "%s" was skipped because it already exists!', $serializedImageVariant->identifier); + $context->dispatch(Severity::NOTICE, "Image Variant \"{$serializedImageVariant->identifier}\" was skipped because it already exists!"); } else { - $this->dispatch(Severity::ERROR, 'Image Variant "%s" has been changed in the meantime, it was NOT updated!', $serializedImageVariant->identifier); + $context->dispatch(Severity::ERROR, "Image Variant \"{$serializedImageVariant->identifier}\" has been changed in the meantime, it was NOT updated!"); } return; } $originalImage = $this->assetRepository->findByIdentifier($serializedImageVariant->originalAssetIdentifier); if ($originalImage === null) { - $this->dispatch(Severity::ERROR, 'Failed to find original asset "%s", skipping image variant "%s"', $serializedImageVariant->originalAssetIdentifier, $serializedImageVariant->identifier); + $context->dispatch(Severity::ERROR, "Failed to find original asset \"{$serializedImageVariant->originalAssetIdentifier}\", skipping image variant \"{$serializedImageVariant->identifier}\""); return; } assert($originalImage instanceof Image); @@ -154,12 +139,4 @@ private function importImageVariant(StorageAttributes $file): void $this->assetRepository->add($imageVariant); $this->persistenceManager->persistAll(); } - - private function dispatch(Severity $severity, string $message, mixed ...$args): void - { - $renderedMessage = sprintf($message, ...$args); - foreach ($this->callbacks as $callback) { - $callback($severity, $renderedMessage); - } - } } diff --git a/Neos.ContentRepository.Export/src/Processors/ContentRepositorySetupProcessor.php b/Neos.ContentRepository.Export/src/Processors/ContentRepositorySetupProcessor.php new file mode 100644 index 00000000000..57d322921da --- /dev/null +++ b/Neos.ContentRepository.Export/src/Processors/ContentRepositorySetupProcessor.php @@ -0,0 +1,28 @@ +dispatch(Severity::NOTICE, "Setting up content repository \"{$this->contentRepository->id->value}\""); + $this->contentRepository->setUp(); + } +} diff --git a/Neos.ContentRepository.Export/src/Processors/EventExportProcessor.php b/Neos.ContentRepository.Export/src/Processors/EventExportProcessor.php index b7d0b486188..f32019a6e65 100644 --- a/Neos.ContentRepository.Export/src/Processors/EventExportProcessor.php +++ b/Neos.ContentRepository.Export/src/Processors/EventExportProcessor.php @@ -1,49 +1,43 @@ */ - private array $callbacks = []; - + /** + * @param ContentStreamId $contentStreamId Identifier of the content stream to export + */ public function __construct( - private readonly Filesystem $files, - private readonly Workspace $targetWorkspace, - private readonly EventStoreInterface $eventStore, + private ContentStreamId $contentStreamId, + private EventStoreInterface $eventStore, ) { } - public function onMessage(\Closure $callback): void - { - $this->callbacks[] = $callback; - } - - public function run(): ProcessorResult + public function run(ProcessingContext $context): void { - $streamName = ContentStreamEventStreamName::fromContentStreamId($this->targetWorkspace->currentContentStreamId)->getEventStreamName(); + $streamName = ContentStreamEventStreamName::fromContentStreamId($this->contentStreamId)->getEventStreamName(); $eventStream = $this->eventStore->load($streamName); $eventFileResource = fopen('php://temp/maxmemory:5242880', 'rb+'); if ($eventFileResource === false) { - return ProcessorResult::error('Failed to create temporary event file resource'); + throw new \RuntimeException('Failed to create temporary event file resource', 1729506599); } - $numberOfExportedEvents = 0; foreach ($eventStream as $eventEnvelope) { if ($eventEnvelope->event->type->value === 'ContentStreamWasCreated') { // the content stream will be created in the import dynamically, so we prevent duplication here @@ -51,28 +45,12 @@ public function run(): ProcessorResult } $event = ExportedEvent::fromRawEvent($eventEnvelope->event); fwrite($eventFileResource, $event->toJson() . chr(10)); - $numberOfExportedEvents ++; } try { - $this->files->writeStream('events.jsonl', $eventFileResource); + $context->files->writeStream('events.jsonl', $eventFileResource); } catch (FilesystemException $e) { - return ProcessorResult::error(sprintf('Failed to write events.jsonl: %s', $e->getMessage())); + throw new \RuntimeException(sprintf('Failed to write events.jsonl: %s', $e->getMessage()), 1729506623, $e); } fclose($eventFileResource); - return ProcessorResult::success(sprintf('Exported %d event%s', $numberOfExportedEvents, $numberOfExportedEvents === 1 ? '' : 's')); - } - - /** --------------------------------------- */ - - - /** - * @phpstan-ignore-next-line currently this private method is unused ... but it does no harm keeping it - */ - private function dispatch(Severity $severity, string $message, mixed ...$args): void - { - $renderedMessage = sprintf($message, ...$args); - foreach ($this->callbacks as $callback) { - $callback($severity, $renderedMessage); - } } } diff --git a/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php b/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php index 50f4474f485..e35d882db95 100644 --- a/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php +++ b/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php @@ -1,9 +1,9 @@ */ - private array $callbacks = []; - private ?ContentStreamId $contentStreamId = null; public function __construct( private readonly bool $keepEventIds, - private readonly Filesystem $files, private readonly EventStoreInterface $eventStore, private readonly EventNormalizer $eventNormalizer, ?ContentStreamId $overrideContentStreamId @@ -49,16 +44,11 @@ public function __construct( } } - public function onMessage(\Closure $callback): void - { - $this->callbacks[] = $callback; - } - - public function run(): ProcessorResult + public function run(ProcessingContext $context): void { /** @var array $domainEvents */ $domainEvents = []; - $eventFileResource = $this->files->readStream('events.jsonl'); + $eventFileResource = $context->files->readStream('events.jsonl'); /** @var array $eventIdMap */ $eventIdMap = []; @@ -106,7 +96,7 @@ public function run(): ProcessorResult ) ); if (in_array($domainEvent::class, [ContentStreamWasCreated::class, ContentStreamWasForked::class, ContentStreamWasRemoved::class], true)) { - return ProcessorResult::error(sprintf('Failed to read events. %s is not expected in imported event stream.', $event->type)); + throw new \RuntimeException(sprintf('Failed to read events. %s is not expected in imported event stream.', $event->type), 1729506757); } $domainEvent = DecoratedEvent::create($domainEvent, eventId: EventId::fromString($event->identifier), metadata: $event->metadata); $domainEvents[] = $this->eventNormalizer->normalize($domainEvent); @@ -125,7 +115,7 @@ public function run(): ProcessorResult try { $contentStreamCreationCommitResult = $this->eventStore->commit($contentStreamStreamName, $events, ExpectedVersion::NO_STREAM()); } catch (ConcurrencyException $e) { - return ProcessorResult::error(sprintf('Failed to publish workspace events because the event stream "%s" already exists (1)', $this->contentStreamId->value)); + throw new \RuntimeException(sprintf('Failed to publish workspace events because the event stream "%s" already exists (1)', $this->contentStreamId->value), 1729506776, $e); } $workspaceName = WorkspaceName::forLive(); @@ -141,15 +131,14 @@ public function run(): ProcessorResult try { $this->eventStore->commit($workspaceStreamName, $events, ExpectedVersion::NO_STREAM()); } catch (ConcurrencyException $e) { - return ProcessorResult::error(sprintf('Failed to publish workspace events because the event stream "%s" already exists (2)', $workspaceStreamName->value)); + throw new \RuntimeException(sprintf('Failed to publish workspace events because the event stream "%s" already exists (2)', $workspaceStreamName->value), 1729506798, $e); } try { $this->eventStore->commit($contentStreamStreamName, Events::fromArray($domainEvents), ExpectedVersion::fromVersion($contentStreamCreationCommitResult->highestCommittedVersion)); } catch (ConcurrencyException $e) { - return ProcessorResult::error(sprintf('Failed to publish %d events because the event stream "%s" already exists (3)', count($domainEvents), $contentStreamStreamName->value)); + throw new \RuntimeException(sprintf('Failed to publish %d events because the event stream "%s" already exists (3)', count($domainEvents), $contentStreamStreamName->value), 1729506818, $e); } - return ProcessorResult::success(sprintf('Imported %d event%s into stream "%s"', count($domainEvents), count($domainEvents) === 1 ? '' : 's', $contentStreamStreamName->value)); } /** --------------------------- */ @@ -165,15 +154,4 @@ private static function extractContentStreamId(array $payload): ContentStreamId } return ContentStreamId::fromString($payload['contentStreamId']); } - - /** - * @phpstan-ignore-next-line currently this private method is unused ... but it does no harm keeping it - */ - private function dispatch(Severity $severity, string $message, mixed ...$args): void - { - $renderedMessage = sprintf($message, ...$args); - foreach ($this->callbacks as $callback) { - $callback($severity, $renderedMessage); - } - } } diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php index e95c5b27dc2..3c2ab1350d2 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php @@ -1,4 +1,5 @@ connection), new FileSystemResourceLoader($this->resourcesPath)); - /** @var ProcessorInterface[] $processors */ - $processors = [ + $processors = Processors::fromArray([ 'Exporting assets' => new NodeDataToAssetsProcessor($this->nodeTypeManager, $assetExporter, new NodeDataLoader($this->connection)), - 'Exporting node data' => new NodeDataToEventsProcessor($this->nodeTypeManager, $this->propertyMapper, $this->propertyConverter, $this->interDimensionalVariationGraph, $this->eventNormalizer, $filesystem, new NodeDataLoader($this->connection)), - 'Importing assets' => new AssetRepositoryImportProcessor($filesystem, $this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Importing events' => new EventStoreImportProcessor(true, $filesystem, $this->eventStore, $this->eventNormalizer, $this->contentStreamId), - ]; - + 'Exporting node data' => new NodeDataToEventsProcessor($this->nodeTypeManager, $this->propertyMapper, $this->propertyConverter, $this->interDimensionalVariationGraph, $this->eventNormalizer, new NodeDataLoader($this->connection)), + 'Importing assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), + 'Importing events' => new EventStoreImportProcessor(true, $this->eventStore, $this->eventNormalizer, $this->contentStreamId), + ]); + $processingContext = new ProcessingContext($filesystem, function (Severity $severity, string $message) use ($verbose, $outputLineFn) { + if ($severity !== Severity::NOTICE || $verbose) { + $outputLineFn('<%1$s>%2$s', [$severity === Severity::ERROR ? 'error' : 'comment', $message]); + } + }); foreach ($processors as $label => $processor) { $outputLineFn($label . '...'); - $processor->onMessage(function (Severity $severity, string $message) use ($verbose, $outputLineFn) { - if ($severity !== Severity::NOTICE || $verbose) { - $outputLineFn('<%1$s>%2$s', [$severity === Severity::ERROR ? 'error' : 'comment', $message]); - } - }); - $result = $processor->run(); - if ($result->severity === Severity::ERROR) { - throw new \RuntimeException($label . ': ' . $result->message); - } - $outputLineFn(' ' . $result->message); + $processor->run($processingContext); $outputLineFn(); } Files::unlink($temporaryFilePath); diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php index 86d38a6d8a5..6bde0f844e5 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php @@ -1,5 +1,5 @@ */ private array $processedAssetIds = []; - /** - * @var array<\Closure> - */ - private array $callbacks = []; /** * @param iterable> $nodeDataRows @@ -32,16 +28,11 @@ public function __construct( private readonly NodeTypeManager $nodeTypeManager, private readonly AssetExporter $assetExporter, private readonly iterable $nodeDataRows, - ) {} - - public function onMessage(\Closure $callback): void - { - $this->callbacks[] = $callback; + ) { } - public function run(): ProcessorResult + public function run(ProcessingContext $context): void { - $numberOfErrors = 0; foreach ($this->nodeDataRows as $nodeDataRow) { if ($nodeDataRow['path'] === '/sites') { // the sites node has no properties and is unstructured @@ -50,21 +41,20 @@ public function run(): ProcessorResult $nodeTypeName = NodeTypeName::fromString($nodeDataRow['nodetype']); $nodeType = $this->nodeTypeManager->getNodeType($nodeTypeName); if (!$nodeType) { - $this->dispatch(Severity::ERROR, 'The node type "%s" is not available. Node: "%s"', $nodeTypeName->value, $nodeDataRow['identifier']); + $context->dispatch(Severity::ERROR, "The node type \"{$nodeTypeName->value}\" is not available. Node: \"{$nodeDataRow['identifier']}\""); continue; } try { $properties = json_decode($nodeDataRow['properties'], true, 512, JSON_THROW_ON_ERROR); } catch (\JsonException $exception) { - $numberOfErrors ++; - $this->dispatch(Severity::ERROR, 'Failed to JSON-decode properties %s of node "%s" (type: "%s"): %s', $nodeDataRow['properties'], $nodeDataRow['identifier'], $nodeTypeName->value, $exception->getMessage()); + $context->dispatch(Severity::ERROR, "Failed to JSON-decode properties {$nodeDataRow['properties']} of node \"{$nodeDataRow['identifier']}\" (type: \"{$nodeTypeName->value}\"): {$exception->getMessage()}"); continue; } foreach ($properties as $propertyName => $propertyValue) { try { $propertyType = $nodeType->getPropertyType($propertyName); - } catch (\InvalidArgumentException $exception) { - $this->dispatch(Severity::WARNING, 'Skipped node data processing for the property "%s". The property name is not part of the NodeType schema for the NodeType "%s". (Node: %s)', $propertyName, $nodeType->name->value, $nodeDataRow['identifier']); + } catch (\InvalidArgumentException $e) { + $context->dispatch(Severity::WARNING, "Skipped node data processing for the property \"{$propertyName}\". The property name is not part of the NodeType schema for the NodeType \"{$nodeType->name->value}\". (Node: {$nodeDataRow['identifier']})"); continue; } foreach ($this->extractAssetIdentifiers($propertyType, $propertyValue) as $assetId) { @@ -75,15 +65,11 @@ public function run(): ProcessorResult try { $this->assetExporter->exportAsset($assetId); } catch (\Exception $exception) { - $numberOfErrors ++; - $this->dispatch(Severity::ERROR, 'Failed to extract assets of property "%s" of node "%s" (type: "%s"): %s', $propertyName, $nodeDataRow['identifier'], $nodeTypeName->value, $exception->getMessage()); + $context->dispatch(Severity::ERROR, "Failed to extract assets of property \"{$propertyName}\" of node \"{$nodeDataRow['identifier']}\" (type: \"{$nodeTypeName->value}\"): {$exception->getMessage()}"); } } } } - $numberOfExportedAssets = count($this->processedAssetIds); - $this->processedAssetIds = []; - return ProcessorResult::success(sprintf('Exported %d asset%s. Errors: %d', $numberOfExportedAssets, $numberOfExportedAssets === 1 ? '' : 's', $numberOfErrors)); } /** ----------------------------- */ @@ -110,8 +96,7 @@ private function extractAssetIdentifiers(string $type, mixed $value): array if ($parsedType['elementType'] === null) { return []; } - if (!is_subclass_of($parsedType['elementType'], ResourceBasedInterface::class, true) - && !is_subclass_of($parsedType['elementType'], \Stringable::class, true)) { + if (!is_subclass_of($parsedType['elementType'], ResourceBasedInterface::class) && !is_subclass_of($parsedType['elementType'], \Stringable::class)) { return []; } /** @var array> $assetIdentifiers */ @@ -122,13 +107,4 @@ private function extractAssetIdentifiers(string $type, mixed $value): array } return array_merge(...$assetIdentifiers); } - - private function dispatch(Severity $severity, string $message, mixed ...$args): void - { - $renderedMessage = sprintf($message, ...$args); - foreach ($this->callbacks as $callback) { - $callback($severity, $renderedMessage); - } - } - } diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php index 9379206d5ee..48ea7dd0bdb 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php @@ -1,12 +1,11 @@ - */ - private array $callbacks = []; private NodeTypeName $sitesNodeTypeName; private WorkspaceName $workspaceName; private ContentStreamId $contentStreamId; @@ -89,7 +84,6 @@ public function __construct( private readonly PropertyConverter $propertyConverter, private readonly InterDimensionalVariationGraph $interDimensionalVariationGraph, private readonly EventNormalizer $eventNormalizer, - private readonly Filesystem $files, private readonly iterable $nodeDataRows, ) { $this->sitesNodeTypeName = NodeTypeNameFactory::forSites(); @@ -115,12 +109,7 @@ public function setSitesNodeType(NodeTypeName $nodeTypeName): void $this->sitesNodeTypeName = $nodeTypeName; } - public function onMessage(\Closure $callback): void - { - $this->callbacks[] = $callback; - } - - public function run(): ProcessorResult + public function run(ProcessingContext $context): void { $this->resetRuntimeState(); @@ -132,13 +121,13 @@ public function run(): ProcessorResult continue; } if ($this->metaDataExported === false && $nodeDataRow['parentpath'] === '/sites') { - $this->exportMetaData($nodeDataRow); + $this->exportMetaData($context, $nodeDataRow); $this->metaDataExported = true; } try { - $this->processNodeData($nodeDataRow); - } catch (MigrationException $exception) { - return ProcessorResult::error($exception->getMessage()); + $this->processNodeData($context, $nodeDataRow); + } catch (MigrationException $e) { + throw new \RuntimeException($e->getMessage(), 1729506899, $e); } } // Set References, now when the full import is done. @@ -147,11 +136,10 @@ public function run(): ProcessorResult } try { - $this->files->writeStream('events.jsonl', $this->eventFileResource); - } catch (FilesystemException $exception) { - return ProcessorResult::error(sprintf('Failed to write events.jsonl: %s', $exception->getMessage())); + $context->files->writeStream('events.jsonl', $this->eventFileResource); + } catch (FilesystemException $e) { + throw new \RuntimeException(sprintf('Failed to write events.jsonl: %s', $e->getMessage()), 1729506930, $e); } - return ProcessorResult::success(sprintf('Exported %d event%s', $this->numberOfExportedEvents, $this->numberOfExportedEvents === 1 ? '' : 's')); } /** ----------------------------- */ @@ -188,10 +176,10 @@ private function exportEvent(EventInterface $event): void /** * @param array $nodeDataRow */ - private function exportMetaData(array $nodeDataRow): void + private function exportMetaData(ProcessingContext $context, array $nodeDataRow): void { - if ($this->files->fileExists('meta.json')) { - $data = json_decode($this->files->read('meta.json'), true, 512, JSON_THROW_ON_ERROR); + if ($context->files->fileExists('meta.json')) { + $data = json_decode($context->files->read('meta.json'), true, 512, JSON_THROW_ON_ERROR); } else { $data = []; } @@ -199,13 +187,13 @@ private function exportMetaData(array $nodeDataRow): void $data['sitePackageKey'] = strtok($nodeDataRow['nodetype'], ':'); $data['siteNodeName'] = substr($nodeDataRow['path'], 7); $data['siteNodeType'] = $nodeDataRow['nodetype']; - $this->files->write('meta.json', json_encode($data, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT)); + $context->files->write('meta.json', json_encode($data, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT)); } /** * @param array $nodeDataRow */ - private function processNodeData(array $nodeDataRow): void + private function processNodeData(ProcessingContext $context, array $nodeDataRow): void { $nodeAggregateId = NodeAggregateId::fromString($nodeDataRow['identifier']); @@ -235,11 +223,11 @@ private function processNodeData(array $nodeDataRow): void foreach ($this->interDimensionalVariationGraph->getDimensionSpacePoints() as $dimensionSpacePoint) { $originDimensionSpacePoint = OriginDimensionSpacePoint::fromDimensionSpacePoint($dimensionSpacePoint); if (!$this->visitedNodes->alreadyVisitedOriginDimensionSpacePoints($nodeAggregateId)->contains($originDimensionSpacePoint)) { - $this->processNodeDataWithoutFallbackToEmptyDimension($nodeAggregateId, $originDimensionSpacePoint, $nodeDataRow); + $this->processNodeDataWithoutFallbackToEmptyDimension($context, $nodeAggregateId, $originDimensionSpacePoint, $nodeDataRow); } } } else { - $this->processNodeDataWithoutFallbackToEmptyDimension($nodeAggregateId, $originDimensionSpacePoint, $nodeDataRow); + $this->processNodeDataWithoutFallbackToEmptyDimension($context, $nodeAggregateId, $originDimensionSpacePoint, $nodeDataRow); } } @@ -250,12 +238,12 @@ private function processNodeData(array $nodeDataRow): void * @param array $nodeDataRow * @return NodeName[]|void */ - public function processNodeDataWithoutFallbackToEmptyDimension(NodeAggregateId $nodeAggregateId, OriginDimensionSpacePoint $originDimensionSpacePoint, array $nodeDataRow) + public function processNodeDataWithoutFallbackToEmptyDimension(ProcessingContext $context, NodeAggregateId $nodeAggregateId, OriginDimensionSpacePoint $originDimensionSpacePoint, array $nodeDataRow) { $nodePath = NodePath::fromString(strtolower($nodeDataRow['path'])); $parentNodeAggregate = $this->visitedNodes->findMostSpecificParentNodeInDimensionGraph($nodePath, $originDimensionSpacePoint, $this->interDimensionalVariationGraph); if ($parentNodeAggregate === null) { - $this->dispatch(Severity::ERROR, 'Failed to find parent node for node with id "%s" and dimensions: %s. Please ensure that the new content repository has a valid content dimension configuration. Also note that the old CR can sometimes have orphaned nodes.', $nodeAggregateId->value, $originDimensionSpacePoint->toJson()); + $context->dispatch(Severity::ERROR, "Failed to find parent node for node with id \"{$nodeAggregateId->value}\" and dimensions: {$originDimensionSpacePoint->toJson()}. Please ensure that the new content repository has a valid content dimension configuration. Also note that the old CR can sometimes have orphaned nodes."); return; } $pathParts = $nodePath->getParts(); @@ -267,17 +255,15 @@ public function processNodeDataWithoutFallbackToEmptyDimension(NodeAggregateId $ $isSiteNode = $nodeDataRow['parentpath'] === '/sites'; if ($isSiteNode && !$nodeType?->isOfType(NodeTypeNameFactory::NAME_SITE)) { - throw new MigrationException(sprintf( - 'The site node "%s" (type: "%s") must be of type "%s"', $nodeDataRow['identifier'], $nodeTypeName->value, NodeTypeNameFactory::NAME_SITE - ), 1695801620); + throw new MigrationException(sprintf('The site node "%s" (type: "%s") must be of type "%s"', $nodeDataRow['identifier'], $nodeTypeName->value, NodeTypeNameFactory::NAME_SITE), 1695801620); } if (!$nodeType) { - $this->dispatch(Severity::ERROR, 'The node type "%s" is not available. Node: "%s"', $nodeTypeName->value, $nodeDataRow['identifier']); + $context->dispatch(Severity::ERROR, "The node type \"{$nodeTypeName->value}\" is not available. Node: \"{$nodeDataRow['identifier']}\""); return; } - $serializedPropertyValuesAndReferences = $this->extractPropertyValuesAndReferences($nodeDataRow, $nodeType); + $serializedPropertyValuesAndReferences = $this->extractPropertyValuesAndReferences($context, $nodeDataRow, $nodeType); if ($this->isAutoCreatedChildNode($parentNodeAggregate->nodeTypeName, $nodeName) && !$this->visitedNodes->containsNodeAggregate($nodeAggregateId)) { // Create tethered node if the node was not found before. @@ -335,7 +321,7 @@ public function processNodeDataWithoutFallbackToEmptyDimension(NodeAggregateId $ /** * @param array $nodeDataRow */ - public function extractPropertyValuesAndReferences(array $nodeDataRow, NodeType $nodeType): SerializedPropertyValuesAndReferences + public function extractPropertyValuesAndReferences(ProcessingContext $context, array $nodeDataRow, NodeType $nodeType): SerializedPropertyValuesAndReferences { $properties = []; $references = []; @@ -362,7 +348,7 @@ public function extractPropertyValuesAndReferences(array $nodeDataRow, NodeType } if (!$nodeType->hasProperty($propertyName)) { - $this->dispatch(Severity::WARNING, 'Skipped node data processing for the property "%s". The property name is not part of the NodeType schema for the NodeType "%s". (Node: %s)', $propertyName, $nodeType->name->value, $nodeDataRow['identifier']); + $context->dispatch(Severity::WARNING, "Skipped node data processing for the property \"{$propertyName}\". The property name is not part of the NodeType schema for the NodeType \"{$nodeType->name->value}\". (Node: {$nodeDataRow['identifier']})"); continue; } $type = $nodeType->getPropertyType($propertyName); @@ -375,7 +361,6 @@ public function extractPropertyValuesAndReferences(array $nodeDataRow, NodeType } else { $properties[$propertyName] = $this->propertyMapper->convert($propertyValue, $type); } - } catch (\Exception $e) { throw new MigrationException(sprintf('Failed to convert property "%s" of type "%s" (Node: %s): %s', $propertyName, $type, $nodeDataRow['identifier'], $e->getMessage()), 1655912878, $e); } @@ -397,7 +382,7 @@ public function extractPropertyValuesAndReferences(array $nodeDataRow, NodeType } } else { if ($nodeDataRow['hiddenbeforedatetime'] || $nodeDataRow['hiddenafterdatetime']) { - $this->dispatch(Severity::WARNING, 'Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them.'); + $context->dispatch(Severity::WARNING, 'Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them.'); } } @@ -505,14 +490,6 @@ private function isAutoCreatedChildNode(NodeTypeName $parentNodeTypeName, NodeNa return $nodeTypeOfParent->tetheredNodeTypeDefinitions->contain($nodeName); } - private function dispatch(Severity $severity, string $message, mixed ...$args): void - { - $renderedMessage = sprintf($message, ...$args); - foreach ($this->callbacks as $callback) { - $callback($severity, $renderedMessage); - } - } - /** * Determines actual hidden state based on "hidden", "hiddenafterdatetime" and "hiddenbeforedatetime" * @@ -530,19 +507,21 @@ private function isNodeHidden(array $nodeDataRow): bool $hiddenBeforeDateTime = $nodeDataRow['hiddenbeforedatetime'] ? new \DateTimeImmutable($nodeDataRow['hiddenbeforedatetime']) : null; // Hidden after a date time, without getting already re-enabled by hidden before date time - afterward - if ($hiddenAfterDateTime != null + if ( + $hiddenAfterDateTime != null && $hiddenAfterDateTime < $now && ( $hiddenBeforeDateTime == null || $hiddenBeforeDateTime > $now - || $hiddenBeforeDateTime<= $hiddenAfterDateTime + || $hiddenBeforeDateTime <= $hiddenAfterDateTime ) ) { return true; } // Hidden before a date time, without getting enabled by hidden after date time - before - if ($hiddenBeforeDateTime != null + if ( + $hiddenBeforeDateTime != null && $hiddenBeforeDateTime > $now && ( $hiddenAfterDateTime == null @@ -553,6 +532,5 @@ private function isNodeHidden(array $nodeDataRow): bool } return false; - } } diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 00421a6bea4..0e10464c0fc 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -12,7 +12,6 @@ use Neos\ContentRepository\BehavioralTests\TestSuite\Behavior\GherkinPyStringNodeBasedNodeTypeManagerFactory; use Neos\ContentRepository\BehavioralTests\TestSuite\Behavior\GherkinTableNodeBasedContentDimensionSourceFactory; use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\ContentRepositoryReadModel; use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; @@ -21,7 +20,6 @@ use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\Asset\AssetExporter; use Neos\ContentRepository\Export\Asset\AssetLoaderInterface; use Neos\ContentRepository\Export\Asset\ResourceLoaderInterface; @@ -29,7 +27,7 @@ use Neos\ContentRepository\Export\Asset\ValueObject\SerializedImageVariant; use Neos\ContentRepository\Export\Asset\ValueObject\SerializedResource; use Neos\ContentRepository\Export\Event\ValueObject\ExportedEvents; -use Neos\ContentRepository\Export\ProcessorResult; +use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepository\LegacyNodeMigration\NodeDataToAssetsProcessor; use Neos\ContentRepository\LegacyNodeMigration\NodeDataToEventsProcessor; @@ -59,7 +57,7 @@ class FeatureContext implements Context private InMemoryFilesystemAdapter $mockFilesystemAdapter; private Filesystem $mockFilesystem; - private ProcessorResult|null $lastMigrationResult = null; + private \Throwable|null $lastMigrationException = null; /** * @var array @@ -89,8 +87,8 @@ public function __construct() */ public function failIfLastMigrationHasErrors(): void { - if ($this->lastMigrationResult !== null && $this->lastMigrationResult->severity === Severity::ERROR) { - throw new \RuntimeException(sprintf('The last migration run led to an error: %s', $this->lastMigrationResult->message)); + if ($this->lastMigrationException !== null) { + throw new \RuntimeException(sprintf('The last migration run led to an exception: %s', $this->lastMigrationException->getMessage())); } if ($this->loggedErrors !== []) { throw new \RuntimeException(sprintf('The last migration run logged %d error%s', count($this->loggedErrors), count($this->loggedErrors) === 1 ? '' : 's')); @@ -146,20 +144,23 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor $propertyConverterAccess->propertyConverter, $this->currentContentRepository->getVariationGraph(), $this->getObject(EventNormalizer::class), - $this->mockFilesystem, $this->nodeDataRows ); if ($contentStream !== null) { $migration->setContentStreamId(ContentStreamId::fromString($contentStream)); } - $migration->onMessage(function (Severity $severity, string $message) { + $processingContext = new ProcessingContext($this->mockFilesystem, function (Severity $severity, string $message) { if ($severity === Severity::ERROR) { $this->loggedErrors[] = $message; } elseif ($severity === Severity::WARNING) { $this->loggedWarnings[] = $message; } }); - $this->lastMigrationResult = $migration->run(); + try { + $migration->run($processingContext); + } catch (\Throwable $e) { + $this->lastMigrationException = $e; + } } /** @@ -223,12 +224,11 @@ public function iExpectTheFollowingWarningsToBeLogged(TableNode $table): void */ public function iExpectAMigrationErrorWithTheMessage(PyStringNode $expectedMessage = null): void { - Assert::assertNotNull($this->lastMigrationResult, 'Expected the previous migration to contain errors, but no migration has been executed'); - Assert::assertSame(Severity::ERROR, $this->lastMigrationResult->severity, sprintf('Expected the previous migration to contain errors, but it ended with severity "%s"', $this->lastMigrationResult->severity->name)); + Assert::assertNotNull($this->lastMigrationException, 'Expected the previous migration to lead to an exception, but no exception was thrown'); if ($expectedMessage !== null) { - Assert::assertSame($expectedMessage->getRaw(), $this->lastMigrationResult->message); + Assert::assertSame($expectedMessage->getRaw(), $this->lastMigrationException->getMessage()); } - $this->lastMigrationResult = null; + $this->lastMigrationException = null; } /** @@ -293,8 +293,8 @@ public function theFollowingImageVariantsExist(TableNode $imageVariants): void public function iRunTheAssetMigration(): void { $nodeTypeManager = $this->currentContentRepository->getNodeTypeManager(); - $mockResourceLoader = new class ($this->mockResources) implements ResourceLoaderInterface { - + $mockResourceLoader = new class ($this->mockResources) implements ResourceLoaderInterface + { /** * @param array $mockResources */ @@ -329,14 +329,18 @@ public function findAssetById(string $assetId): SerializedAsset|SerializedImageV $this->mockFilesystemAdapter->deleteEverything(); $assetExporter = new AssetExporter($this->mockFilesystem, $mockAssetLoader, $mockResourceLoader); $migration = new NodeDataToAssetsProcessor($nodeTypeManager, $assetExporter, $this->nodeDataRows); - $migration->onMessage(function (Severity $severity, string $message) { + $processingContext = new ProcessingContext($this->mockFilesystem, function (Severity $severity, string $message) { if ($severity === Severity::ERROR) { $this->loggedErrors[] = $message; } elseif ($severity === Severity::WARNING) { $this->loggedWarnings[] = $message; } }); - $this->lastMigrationResult = $migration->run(); + try { + $migration->run($processingContext); + } catch (\Throwable $e) { + $this->lastMigrationException = $e; + } } /** diff --git a/Neos.Neos/Classes/Command/CrCommandController.php b/Neos.Neos/Classes/Command/CrCommandController.php deleted file mode 100644 index a6c2f0606ad..00000000000 --- a/Neos.Neos/Classes/Command/CrCommandController.php +++ /dev/null @@ -1,128 +0,0 @@ -contentRepositoryRegistry->get($contentRepositoryId); - - Files::createDirectoryRecursively($path); - $filesystem = new Filesystem(new LocalFilesystemAdapter($path)); - - $liveWorkspace = $contentRepositoryInstance->findWorkspaceByName(WorkspaceName::forLive()); - if ($liveWorkspace === null) { - throw new \RuntimeException('Failed to find live workspace', 1716652280); - } - - $exportService = $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new ExportServiceFactory( - $filesystem, - $liveWorkspace, - $this->assetRepository, - $this->assetUsageService, - ) - ); - assert($exportService instanceof ExportService); - $exportService->runAllProcessors($this->outputLine(...), $verbose); - $this->outputLine('Done'); - } - - /** - * Import the events from the path into the specified content repository - * - * @param string $path The path of the stored events like resource://Neos.Demo/Private/Content - * @param string $contentRepository The content repository identifier - * @param bool $verbose If set, all notices will be rendered - * @throws \Exception - */ - public function importCommand(string $path, string $contentRepository = 'default', bool $verbose = false): void - { - $filesystem = new Filesystem(new LocalFilesystemAdapter($path)); - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $contentStreamIdentifier = ContentStreamId::create(); - - $importService = $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new ImportServiceFactory( - $filesystem, - $contentStreamIdentifier, - $this->assetRepository, - $this->resourceRepository, - $this->resourceManager, - $this->persistenceManager, - ) - ); - assert($importService instanceof ImportService); - try { - $importService->runAllProcessors($this->outputLine(...), $verbose); - } catch (\RuntimeException $exception) { - $this->outputLine('Error: ' . $exception->getMessage() . ''); - $this->outputLine('Import stopped.'); - return; - } - - $this->outputLine('Replaying projections'); - - $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayServiceFactory); - $projectionService->replayAllProjections(CatchUpOptions::create()); - - $this->outputLine('Assigning live workspace role'); - // set the live-workspace title to (implicitly) create the metadata record for this workspace - $this->workspaceService->setWorkspaceTitle($contentRepositoryId, WorkspaceName::forLive(), WorkspaceTitle::fromString('Live workspace')); - $this->workspaceService->assignWorkspaceRole($contentRepositoryId, WorkspaceName::forLive(), WorkspaceRoleAssignment::createForGroup('Neos.Neos:LivePublisher', WorkspaceRole::COLLABORATOR)); - - $this->outputLine('Done'); - } -} diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index c3763cf7252..137a751488d 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -14,11 +14,16 @@ namespace Neos\Neos\Command; +use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Exception\NodeNameIsAlreadyCovered; use Neos\ContentRepository\Core\SharedModel\Exception\NodeTypeNotFound; +use Neos\ContentRepository\Export\ProcessorEventInterface; +use Neos\ContentRepository\Export\Severity; +use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Annotations as Flow; use Neos\Flow\Cli\CommandController; use Neos\Flow\Cli\Exception\StopCommandException; +use Neos\Flow\ObjectManagement\DependencyInjection\DependencyProxy; use Neos\Flow\Package\PackageManager; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Exception\SiteNodeNameIsAlreadyInUseByAnotherSite; @@ -26,6 +31,8 @@ use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\SiteRepository; use Neos\Neos\Domain\Service\NodeTypeNameFactory; +use Neos\Neos\Domain\Service\SiteImportService; +use Neos\Neos\Domain\Service\SiteImportServiceFactory; use Neos\Neos\Domain\Service\SiteService; /** @@ -53,12 +60,24 @@ class SiteCommandController extends CommandController */ protected $packageManager; + /** + * @Flow\Inject + * @var ContentRepositoryRegistry + */ + protected $contentRepositoryRegistry; + /** * @Flow\Inject * @var PersistenceManagerInterface */ protected $persistenceManager; + /** + * @Flow\Inject(lazy=false) + * @var SiteImportServiceFactory + */ + protected $siteImportServiceFactory; + /** * Create a new site * @@ -110,6 +129,28 @@ public function createCommand($name, $packageKey, $nodeType, $nodeName = null, $ ); } + public function importCommand(string $packageKey, string $contentRepository = 'default', bool $verbose = false): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $importService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->siteImportServiceFactory); + assert($importService instanceof SiteImportService); + + $onProcessor = function (string $processorLabel) { + $this->outputLine('%s...', [$processorLabel]); + }; + $onMessage = function (Severity $severity, string $message) use ($verbose) { + if (!$verbose && $severity === Severity::NOTICE) { + return; + } + $this->outputLine(match ($severity) { + Severity::NOTICE => $message, + Severity::WARNING => sprintf('Warning: %s', $message), + Severity::ERROR => sprintf('Error: %s', $message), + }); + }; + $importService->importFromPackage($packageKey, $onProcessor, $onMessage); + } + /** * Remove site with content and related data (with globbing) * diff --git a/Neos.Neos/Classes/Domain/Import/DoctrineMigrateProcessor.php b/Neos.Neos/Classes/Domain/Import/DoctrineMigrateProcessor.php new file mode 100644 index 00000000000..632cdc8a6b5 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Import/DoctrineMigrateProcessor.php @@ -0,0 +1,32 @@ +doctrineService->executeMigrations(); + } +} diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php new file mode 100644 index 00000000000..e4d6c0c6d92 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -0,0 +1,104 @@ +files->has('sites.json')) { + $sitesJson = $context->files->read('sites.json'); + try { + $sites = json_decode($sitesJson, true, 512, JSON_THROW_ON_ERROR); + } catch (JsonException $e) { + throw new \RuntimeException("Failed to decode sites.json: {$e->getMessage()}", 1729506117, $e); + } + } else { + $sites = self::extractSitesFromEventStream($context); + } + /** @var SiteShape $site */ + foreach ($sites as $site) { + $context->dispatch(Severity::NOTICE, "Creating site \"{$site['name']}\""); + + $siteNodeName = !empty($site['nodeName']) ? NodeName::fromString($site['nodeName']) : NodeName::transliterateFromString($site['name']); + if ($this->siteRepository->findOneByNodeName($siteNodeName->value)) { + $context->dispatch(Severity::NOTICE, "Site for node name \"{$siteNodeName->value}\" already exists, skipping"); + continue; + } + + // TODO use node aggregate identifier instead of node name + $siteInstance = new Site($siteNodeName->value); + $siteInstance->setSiteResourcesPackageKey($site['packageKey']); + $siteInstance->setState(($site['inactive'] ?? false) ? Site::STATE_OFFLINE : Site::STATE_ONLINE); + $siteInstance->setName($site['name']); + $this->siteRepository->add($siteInstance); + + // TODO add domains? + } + } + + /** + * @return array + */ + private static function extractSitesFromEventStream(ProcessingContext $context): array + { + $eventFileResource = $context->files->readStream('events.jsonl'); + $rootNodeAggregateIds = []; + $sites = []; + while (($line = fgets($eventFileResource)) !== false) { + $event = ExportedEvent::fromJson($line); + if ($event->type === 'RootNodeAggregateWithNodeWasCreated') { + $rootNodeAggregateIds[] = $event->payload['nodeAggregateId']; + continue; + } + if ($event->type === 'NodeAggregateWithNodeWasCreated' && in_array($event->payload['parentNodeAggregateId'], $rootNodeAggregateIds, true)) { + $sites[] = [ + 'packageKey' => self::extractPackageKeyFromNodeTypeName($event->payload['nodeTypeName']), + 'name' => $event->payload['initialPropertyValues']['title']['value'] ?? $event->payload['nodeTypeName'], + 'nodeTypeName' => $event->payload['nodeTypeName'], + 'nodeName' => $event->payload['nodeName'] ?? null, + ]; + } + }; + return $sites; + } + + private static function extractPackageKeyFromNodeTypeName(string $nodeTypeName): string + { + if (preg_match('/^([^:])+/', $nodeTypeName, $matches) !== 1) { + throw new \RuntimeException("Failed to extract package key from '$nodeTypeName'.", 1729505701); + } + return $matches[0]; + } +} diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php new file mode 100644 index 00000000000..39352751821 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -0,0 +1,53 @@ +packageManager->getPackage($packageKey); + $path = Files::concatenatePaths([$package->getPackagePath(), 'Resources/Private/Content']); + if (!is_dir($path)) { + throw new \InvalidArgumentException(sprintf('No contents for package "%s" at path "%s"', $packageKey, $path), 1728912269); + } + $filesystem = new Filesystem(new LocalFilesystemAdapter($path)); + $context = new ProcessingContext($filesystem, $onMessage); + foreach ($this->processors as $processorLabel => $processor) { + ($onProcessor)($processorLabel); + $processor->run($context); + } + } +} diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php b/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php new file mode 100644 index 00000000000..3ef71079a39 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php @@ -0,0 +1,67 @@ + + */ +#[Flow\Scope('singleton')] +final readonly class SiteImportServiceFactory implements ContentRepositoryServiceFactoryInterface +{ + public function __construct( + private PackageManager $packageManager, + private DoctrineService $doctrineService, + private SiteRepository $siteRepository, + private AssetRepository $assetRepository, + private ResourceRepository $resourceRepository, + private ResourceManager $resourceManager, + private PersistenceManagerInterface $persistenceManager, + ) { + } + + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): SiteImportService + { + // TODO: make configurable(?) + $processors = Processors::fromArray([ + 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), + 'Setup content repository' => new ContentRepositorySetupProcessor($serviceFactoryDependencies->contentRepository), + 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository), + // TODO create live workspace, etc + 'Import events' => new EventStoreImportProcessor(false, $serviceFactoryDependencies->eventStore, $serviceFactoryDependencies->eventNormalizer, null), + 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), + ]); + return new SiteImportService( + $processors, + $this->packageManager, + ); + } +} diff --git a/Neos.Neos/Classes/Domain/Service/SiteService.php b/Neos.Neos/Classes/Domain/Service/SiteService.php index 6041fceb202..75dc08b5aea 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteService.php @@ -167,7 +167,7 @@ public function createSite( ?string $nodeName = null, bool $inactive = false ): Site { - $siteNodeName = NodeName::fromString($nodeName ?: $siteName); + $siteNodeName = NodeName::transliterateFromString($nodeName ?: $siteName); if ($this->siteRepository->findOneByNodeName($siteNodeName->value)) { throw SiteNodeNameIsAlreadyInUseByAnotherSite::butWasAttemptedToBeClaimed($siteNodeName); From d192496a6673ec38d139c36cd9e8ae219701b7e5 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 21 Oct 2024 19:40:41 +0200 Subject: [PATCH 02/56] Extract workspace creation to separate processor --- .../Processors/EventStoreImportProcessor.php | 83 ++++--------------- .../Classes/LegacyMigrationService.php | 6 +- .../Classes/LegacyMigrationServiceFactory.php | 9 +- .../Import/LiveWorkspaceCreationProcessor.php | 50 +++++++++++ .../Service/SiteImportServiceFactory.php | 9 +- 5 files changed, 81 insertions(+), 76 deletions(-) create mode 100644 Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php diff --git a/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php b/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php index e35d882db95..7af1561c16f 100644 --- a/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php +++ b/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php @@ -4,6 +4,7 @@ namespace Neos\ContentRepository\Export\Processors; +use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\EventStore\DecoratedEvent; use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; @@ -29,19 +30,15 @@ /** * Processor that imports all events from an "events.jsonl" file to the event store */ -final class EventStoreImportProcessor implements ProcessorInterface, ContentRepositoryServiceInterface +final readonly class EventStoreImportProcessor implements ProcessorInterface, ContentRepositoryServiceInterface { - private ?ContentStreamId $contentStreamId = null; - public function __construct( - private readonly bool $keepEventIds, - private readonly EventStoreInterface $eventStore, - private readonly EventNormalizer $eventNormalizer, - ?ContentStreamId $overrideContentStreamId + private WorkspaceName $targetWorkspaceName, + private bool $keepEventIds, + private EventStoreInterface $eventStore, + private EventNormalizer $eventNormalizer, + private ContentRepository $contentRepository, ) { - if ($overrideContentStreamId) { - $this->contentStreamId = $overrideContentStreamId; - } } public function run(ProcessingContext $context): void @@ -53,16 +50,15 @@ public function run(ProcessingContext $context): void /** @var array $eventIdMap */ $eventIdMap = []; - $keepStreamName = false; + $workspace = $this->contentRepository->findWorkspaceByName($this->targetWorkspaceName); + if ($workspace === null) { + throw new \InvalidArgumentException("Workspace {$this->targetWorkspaceName} does not exist", 1729530978); + } + while (($line = fgets($eventFileResource)) !== false) { - $event = ExportedEvent::fromJson(trim($line)); - if ($this->contentStreamId === null) { - $this->contentStreamId = self::extractContentStreamId($event->payload); - $keepStreamName = true; - } - if (!$keepStreamName) { - $event = $event->processPayload(fn(array $payload) => isset($payload['contentStreamId']) ? [...$payload, 'contentStreamId' => $this->contentStreamId->value] : $payload); - } + $event = + ExportedEvent::fromJson(trim($line)) + ->processPayload(fn (array $payload) => [...$payload, 'contentStreamId' => $workspace->currentContentStreamId->value, 'workspaceName' => $this->targetWorkspaceName->value]); if (!$this->keepEventIds) { try { $newEventId = Algorithms::generateUUID(); @@ -102,56 +98,11 @@ public function run(ProcessingContext $context): void $domainEvents[] = $this->eventNormalizer->normalize($domainEvent); } - assert($this->contentStreamId !== null); - - $contentStreamStreamName = ContentStreamEventStreamName::fromContentStreamId($this->contentStreamId)->getEventStreamName(); - $events = Events::with( - $this->eventNormalizer->normalize( - new ContentStreamWasCreated( - $this->contentStreamId, - ) - ) - ); - try { - $contentStreamCreationCommitResult = $this->eventStore->commit($contentStreamStreamName, $events, ExpectedVersion::NO_STREAM()); - } catch (ConcurrencyException $e) { - throw new \RuntimeException(sprintf('Failed to publish workspace events because the event stream "%s" already exists (1)', $this->contentStreamId->value), 1729506776, $e); - } - - $workspaceName = WorkspaceName::forLive(); - $workspaceStreamName = WorkspaceEventStreamName::fromWorkspaceName($workspaceName)->getEventStreamName(); - $events = Events::with( - $this->eventNormalizer->normalize( - new RootWorkspaceWasCreated( - $workspaceName, - $this->contentStreamId - ) - ) - ); - try { - $this->eventStore->commit($workspaceStreamName, $events, ExpectedVersion::NO_STREAM()); - } catch (ConcurrencyException $e) { - throw new \RuntimeException(sprintf('Failed to publish workspace events because the event stream "%s" already exists (2)', $workspaceStreamName->value), 1729506798, $e); - } - + $contentStreamStreamName = ContentStreamEventStreamName::fromContentStreamId($workspace->currentContentStreamId)->getEventStreamName(); try { - $this->eventStore->commit($contentStreamStreamName, Events::fromArray($domainEvents), ExpectedVersion::fromVersion($contentStreamCreationCommitResult->highestCommittedVersion)); + $this->eventStore->commit($contentStreamStreamName, Events::fromArray($domainEvents), ExpectedVersion::ANY()); } catch (ConcurrencyException $e) { throw new \RuntimeException(sprintf('Failed to publish %d events because the event stream "%s" already exists (3)', count($domainEvents), $contentStreamStreamName->value), 1729506818, $e); } } - - /** --------------------------- */ - - /** - * @param array $payload - * @return ContentStreamId - */ - private static function extractContentStreamId(array $payload): ContentStreamId - { - if (!isset($payload['contentStreamId']) || !is_string($payload['contentStreamId'])) { - throw new \RuntimeException('Failed to extract "contentStreamId" from event', 1646404169); - } - return ContentStreamId::fromString($payload['contentStreamId']); - } } diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php index 3c2ab1350d2..d057db43fb9 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php @@ -18,12 +18,14 @@ use Doctrine\DBAL\Connection; use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; +use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; +use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\Asset\Adapters\DbalAssetLoader; use Neos\ContentRepository\Export\Asset\Adapters\FileSystemResourceLoader; use Neos\ContentRepository\Export\Asset\AssetExporter; @@ -58,7 +60,7 @@ public function __construct( private readonly EventNormalizer $eventNormalizer, private readonly PropertyConverter $propertyConverter, private readonly EventStoreInterface $eventStore, - private readonly ContentStreamId $contentStreamId, + private readonly ContentRepository $contentRepository, ) { } @@ -75,7 +77,7 @@ public function runAllProcessors(\Closure $outputLineFn, bool $verbose = false): 'Exporting assets' => new NodeDataToAssetsProcessor($this->nodeTypeManager, $assetExporter, new NodeDataLoader($this->connection)), 'Exporting node data' => new NodeDataToEventsProcessor($this->nodeTypeManager, $this->propertyMapper, $this->propertyConverter, $this->interDimensionalVariationGraph, $this->eventNormalizer, new NodeDataLoader($this->connection)), 'Importing assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Importing events' => new EventStoreImportProcessor(true, $this->eventStore, $this->eventNormalizer, $this->contentStreamId), + 'Importing events' => new EventStoreImportProcessor(WorkspaceName::forLive(), true, $this->eventStore, $this->eventNormalizer, $this->contentRepository), ]); $processingContext = new ProcessingContext($filesystem, function (Severity $severity, string $message) use ($verbose, $outputLineFn) { if ($severity !== Severity::NOTICE || $verbose) { diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationServiceFactory.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationServiceFactory.php index 67bd7df05c5..80ab8a5f84a 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationServiceFactory.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationServiceFactory.php @@ -1,4 +1,5 @@ connection, $this->resourcesPath, @@ -62,7 +59,7 @@ public function build( $serviceFactoryDependencies->eventNormalizer, $serviceFactoryDependencies->propertyConverter, $serviceFactoryDependencies->eventStore, - $this->contentStreamId, + $serviceFactoryDependencies->contentRepository, ); } } diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php new file mode 100644 index 00000000000..34a140eb7bd --- /dev/null +++ b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php @@ -0,0 +1,50 @@ +dispatch(Severity::NOTICE, 'Creating live workspace'); + $existingWorkspace = $this->contentRepository->findWorkspaceByName(WorkspaceName::forLive()); + if ($existingWorkspace !== null) { + $context->dispatch(Severity::NOTICE, 'Workspace already exists, skipping'); + return; + } + $this->workspaceService->createRootWorkspace($this->contentRepository->id, WorkspaceName::forLive(), WorkspaceTitle::fromString('Live workspace'), WorkspaceDescription::fromString('')); + $this->workspaceService->assignWorkspaceRole($this->contentRepository->id, WorkspaceName::forLive(), WorkspaceRoleAssignment::createForGroup('Neos.Neos:LivePublisher', WorkspaceRole::COLLABORATOR)); + } +} diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php b/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php index 3ef71079a39..10873e40d3b 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php @@ -16,6 +16,8 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; +use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; +use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\Processors; use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Processors\ContentRepositorySetupProcessor; @@ -29,6 +31,7 @@ use Neos\Media\Domain\Repository\AssetRepository; use Neos\Neos\Domain\Import\DoctrineMigrateProcessor; use Neos\Neos\Domain\Import\SiteCreationProcessor; +use Neos\Neos\Domain\Import\LiveWorkspaceCreationProcessor; use Neos\Neos\Domain\Repository\SiteRepository; /** @@ -45,6 +48,7 @@ public function __construct( private ResourceRepository $resourceRepository, private ResourceManager $resourceManager, private PersistenceManagerInterface $persistenceManager, + private WorkspaceService $workspaceService, ) { } @@ -54,9 +58,10 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor $processors = Processors::fromArray([ 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), 'Setup content repository' => new ContentRepositorySetupProcessor($serviceFactoryDependencies->contentRepository), + // TODO Check if target content stream is empty, otherwise => nice error "prune..." 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository), - // TODO create live workspace, etc - 'Import events' => new EventStoreImportProcessor(false, $serviceFactoryDependencies->eventStore, $serviceFactoryDependencies->eventNormalizer, null), + 'Create Live workspace' => new LiveWorkspaceCreationProcessor($serviceFactoryDependencies->contentRepository, $this->workspaceService), + 'Import events' => new EventStoreImportProcessor(WorkspaceName::forLive(), true, $serviceFactoryDependencies->eventStore, $serviceFactoryDependencies->eventNormalizer, $serviceFactoryDependencies->contentRepository), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), ]); return new SiteImportService( From 4d4e98d6a986dcdc0d0ed22828efbfde5c4e3e77 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Tue, 22 Oct 2024 13:10:14 +0200 Subject: [PATCH 03/56] Make `SiteImportService` a singleton And don't rely on the `ContentRepositoryServiceFactoryInterface` --- ...ContentRepositorySetupProcessorFactory.php | 22 ++++++ .../Factories/EventExportProcessorFactory.php | 29 ++++++++ .../EventStoreImportProcessorFactory.php | 33 +++++++++ .../Classes/Command/SiteCommandController.php | 55 +++++++++++--- .../Domain/Service/SiteImportService.php | 54 ++++++++++---- .../Service/SiteImportServiceFactory.php | 72 ------------------- 6 files changed, 173 insertions(+), 92 deletions(-) create mode 100644 Neos.ContentRepository.Export/src/Factories/ContentRepositorySetupProcessorFactory.php create mode 100644 Neos.ContentRepository.Export/src/Factories/EventExportProcessorFactory.php create mode 100644 Neos.ContentRepository.Export/src/Factories/EventStoreImportProcessorFactory.php delete mode 100644 Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php diff --git a/Neos.ContentRepository.Export/src/Factories/ContentRepositorySetupProcessorFactory.php b/Neos.ContentRepository.Export/src/Factories/ContentRepositorySetupProcessorFactory.php new file mode 100644 index 00000000000..076515afaaf --- /dev/null +++ b/Neos.ContentRepository.Export/src/Factories/ContentRepositorySetupProcessorFactory.php @@ -0,0 +1,22 @@ + + */ +final readonly class ContentRepositorySetupProcessorFactory implements ContentRepositoryServiceFactoryInterface +{ + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositorySetupProcessor + { + return new ContentRepositorySetupProcessor( + $serviceFactoryDependencies->contentRepository, + ); + } +} diff --git a/Neos.ContentRepository.Export/src/Factories/EventExportProcessorFactory.php b/Neos.ContentRepository.Export/src/Factories/EventExportProcessorFactory.php new file mode 100644 index 00000000000..6fe742cec21 --- /dev/null +++ b/Neos.ContentRepository.Export/src/Factories/EventExportProcessorFactory.php @@ -0,0 +1,29 @@ + + */ +final readonly class EventExportProcessorFactory implements ContentRepositoryServiceFactoryInterface +{ + public function __construct( + private ContentStreamId $contentStreamId, + ) { + } + + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): EventExportProcessor + { + return new EventExportProcessor( + $this->contentStreamId, + $serviceFactoryDependencies->eventStore, + ); + } +} diff --git a/Neos.ContentRepository.Export/src/Factories/EventStoreImportProcessorFactory.php b/Neos.ContentRepository.Export/src/Factories/EventStoreImportProcessorFactory.php new file mode 100644 index 00000000000..94b6aa2f8d4 --- /dev/null +++ b/Neos.ContentRepository.Export/src/Factories/EventStoreImportProcessorFactory.php @@ -0,0 +1,33 @@ + + */ +final readonly class EventStoreImportProcessorFactory implements ContentRepositoryServiceFactoryInterface +{ + public function __construct( + private WorkspaceName $targetWorkspaceName, + private bool $keepEventIds, + ) { + } + + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): EventStoreImportProcessor + { + return new EventStoreImportProcessor( + $this->targetWorkspaceName, + $this->keepEventIds, + $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->eventNormalizer, + $serviceFactoryDependencies->contentRepository, + ); + } +} diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index 137a751488d..713cc2f4da6 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -34,6 +34,7 @@ use Neos\Neos\Domain\Service\SiteImportService; use Neos\Neos\Domain\Service\SiteImportServiceFactory; use Neos\Neos\Domain\Service\SiteService; +use Neos\Utility\Files; /** * The Site Command Controller @@ -73,10 +74,10 @@ class SiteCommandController extends CommandController protected $persistenceManager; /** - * @Flow\Inject(lazy=false) - * @var SiteImportServiceFactory + * @Flow\Inject + * @var SiteImportService */ - protected $siteImportServiceFactory; + protected $siteImportService; /** * Create a new site @@ -129,12 +130,46 @@ public function createCommand($name, $packageKey, $nodeType, $nodeName = null, $ ); } - public function importCommand(string $packageKey, string $contentRepository = 'default', bool $verbose = false): void + /** + * Import sites content + * + * This command allows for importing one or more sites or partial content from the file system. The format must + * be identical to that produced by the export command. + * + * If a path is specified, this command expects the corresponding directory to contain the exported files + * + * If a package key is specified, this command expects the export files to be located in the private resources + * directory of the given package (Resources/Private/Content). + * + * @param string|null $packageKey Package key specifying the package containing the sites content + * @param string|null $path relative or absolute path and filename to the export files + * @return void + */ + public function importCommand(string $packageKey = null, string $path = null, string $contentRepository = 'default', bool $verbose = false): void { - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $importService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->siteImportServiceFactory); - assert($importService instanceof SiteImportService); + $exceedingArguments = $this->request->getExceedingArguments(); + if (isset($exceedingArguments[0]) && $packageKey === null && $path === null) { + if (file_exists($exceedingArguments[0])) { + $path = $exceedingArguments[0]; + } elseif ($this->packageManager->isPackageAvailable($exceedingArguments[0])) { + $packageKey = $exceedingArguments[0]; + } + } + if ($packageKey === null && $path === null) { + $this->outputLine('You have to specify either --package-key or --filename'); + $this->quit(1); + } + + // Since this command uses a lot of memory when large sites are imported, we warn the user to watch for + // the confirmation of a successful import. + $this->outputLine('This command can use a lot of memory when importing sites with many resources.'); + $this->outputLine('If the import is successful, you will see a message saying "Import of site ... finished".'); + $this->outputLine('If you do not see this message, the import failed, most likely due to insufficient memory.'); + $this->outputLine('Increase the memory_limit configuration parameter of your php CLI to attempt to fix this.'); + $this->outputLine('Starting import...'); + $this->outputLine('---'); + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $onProcessor = function (string $processorLabel) { $this->outputLine('%s...', [$processorLabel]); }; @@ -148,7 +183,11 @@ public function importCommand(string $packageKey, string $contentRepository = 'd Severity::ERROR => sprintf('Error: %s', $message), }); }; - $importService->importFromPackage($packageKey, $onProcessor, $onMessage); + if ($path === null) { + $package = $this->packageManager->getPackage($packageKey); + $path = Files::concatenatePaths([$package->getPackagePath(), 'Resources/Private/Content']); + } + $this->siteImportService->importFromPath($contentRepositoryId, $path, $onProcessor, $onMessage); } /** diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 39352751821..d04a48ca9dd 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -16,19 +16,38 @@ use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; -use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; +use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Export\Factories\ContentRepositorySetupProcessorFactory; +use Neos\ContentRepository\Export\Factories\EventStoreImportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepository\Export\Processors; +use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; -use Neos\Flow\Package\PackageManager; -use Neos\Utility\Files; +use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; +use Neos\Flow\Annotations as Flow; +use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; +use Neos\Flow\Persistence\PersistenceManagerInterface; +use Neos\Flow\ResourceManagement\ResourceManager; +use Neos\Flow\ResourceManagement\ResourceRepository; +use Neos\Media\Domain\Repository\AssetRepository; +use Neos\Neos\Domain\Import\DoctrineMigrateProcessor; +use Neos\Neos\Domain\Import\LiveWorkspaceCreationProcessor; +use Neos\Neos\Domain\Import\SiteCreationProcessor; +use Neos\Neos\Domain\Repository\SiteRepository; -final readonly class SiteImportService implements ContentRepositoryServiceInterface +#[Flow\Scope('singleton')] +final readonly class SiteImportService { public function __construct( - private Processors $processors, - private PackageManager $packageManager, + private ContentRepositoryRegistry $contentRepositoryRegistry, + private DoctrineService $doctrineService, + private SiteRepository $siteRepository, + private AssetRepository $assetRepository, + private ResourceRepository $resourceRepository, + private ResourceManager $resourceManager, + private PersistenceManagerInterface $persistenceManager, + private WorkspaceService $workspaceService, ) { } @@ -36,16 +55,27 @@ public function __construct( * @param \Closure(string): void $onProcessor Callback that is invoked for each {@see ProcessorInterface} that is processed * @param \Closure(Severity, string): void $onMessage Callback that is invoked whenever a {@see ProcessorInterface} dispatches a message */ - public function importFromPackage(string $packageKey, \Closure $onProcessor, \Closure $onMessage): void + public function importFromPath(ContentRepositoryId $contentRepositoryId, string $path, \Closure $onProcessor, \Closure $onMessage): void { - $package = $this->packageManager->getPackage($packageKey); - $path = Files::concatenatePaths([$package->getPackagePath(), 'Resources/Private/Content']); if (!is_dir($path)) { - throw new \InvalidArgumentException(sprintf('No contents for package "%s" at path "%s"', $packageKey, $path), 1728912269); + throw new \InvalidArgumentException(sprintf('Path "%s" is not a directory', $path), 1729593802); } $filesystem = new Filesystem(new LocalFilesystemAdapter($path)); $context = new ProcessingContext($filesystem, $onMessage); - foreach ($this->processors as $processorLabel => $processor) { + $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); + + // TODO make configurable (?) + /** @var array $processors */ + $processors = [ + 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), + 'Setup content repository' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositorySetupProcessorFactory()), + // TODO Check if target content stream is empty, otherwise => nice error "prune..." + 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository), + 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), + 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), + 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), + ]; + foreach ($processors as $processorLabel => $processor) { ($onProcessor)($processorLabel); $processor->run($context); } diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php b/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php deleted file mode 100644 index 10873e40d3b..00000000000 --- a/Neos.Neos/Classes/Domain/Service/SiteImportServiceFactory.php +++ /dev/null @@ -1,72 +0,0 @@ - - */ -#[Flow\Scope('singleton')] -final readonly class SiteImportServiceFactory implements ContentRepositoryServiceFactoryInterface -{ - public function __construct( - private PackageManager $packageManager, - private DoctrineService $doctrineService, - private SiteRepository $siteRepository, - private AssetRepository $assetRepository, - private ResourceRepository $resourceRepository, - private ResourceManager $resourceManager, - private PersistenceManagerInterface $persistenceManager, - private WorkspaceService $workspaceService, - ) { - } - - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): SiteImportService - { - // TODO: make configurable(?) - $processors = Processors::fromArray([ - 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), - 'Setup content repository' => new ContentRepositorySetupProcessor($serviceFactoryDependencies->contentRepository), - // TODO Check if target content stream is empty, otherwise => nice error "prune..." - 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository), - 'Create Live workspace' => new LiveWorkspaceCreationProcessor($serviceFactoryDependencies->contentRepository, $this->workspaceService), - 'Import events' => new EventStoreImportProcessor(WorkspaceName::forLive(), true, $serviceFactoryDependencies->eventStore, $serviceFactoryDependencies->eventNormalizer, $serviceFactoryDependencies->contentRepository), - 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - ]); - return new SiteImportService( - $processors, - $this->packageManager, - ); - } -} From 7b3ce80e9c56de85c18a558b68770d528123b506 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Tue, 22 Oct 2024 13:17:20 +0200 Subject: [PATCH 04/56] Fix `LegacyMigrationService` initialization --- .../Classes/Command/CrCommandController.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php index 337a3aa66ce..342bad83647 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php @@ -124,8 +124,6 @@ public function migrateLegacyDataCommand(bool $verbose = false, string $config = $projectionService->resetAllProjections(); $this->outputLine('Truncated events'); - $liveContentStreamId = ContentStreamId::create(); - $legacyMigrationService = $this->contentRepositoryRegistry->buildService( $contentRepositoryId, new LegacyMigrationServiceFactory( @@ -137,7 +135,6 @@ public function migrateLegacyDataCommand(bool $verbose = false, string $config = $this->resourceRepository, $this->resourceManager, $this->propertyMapper, - $liveContentStreamId ) ); assert($legacyMigrationService instanceof LegacyMigrationService); From c8b40bd1c60eeb9df70de2d8411e43e36592fdee Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Tue, 22 Oct 2024 14:13:44 +0200 Subject: [PATCH 05/56] Re-add `Neos.ContentRepository.Export` behat tests and centralize common behat features into Trait --- .../Bootstrap/CrImportExportTrait.php | 281 ++++++++++++++++++ .../Features/Bootstrap/FeatureContext.php | 78 +++++ .../Features/EventExportProcessor.feature | 43 +++ .../EventStoreImportProcessor.feature | 84 ++++++ .../Tests/Behavior/behat.yml.dist | 0 .../Behavior/Bootstrap/FeatureContext.php | 222 +------------- .../Tests/Behavior/Features/Errors.feature | 12 +- 7 files changed, 507 insertions(+), 213 deletions(-) create mode 100644 Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php create mode 100644 Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/FeatureContext.php create mode 100644 Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature create mode 100644 Neos.ContentRepository.Export/Tests/Behavior/Features/EventStoreImportProcessor.feature create mode 100644 Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php new file mode 100644 index 00000000000..8f06d03cf64 --- /dev/null +++ b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php @@ -0,0 +1,281 @@ + */ + private array $crImportExportTrait_loggedErrors = []; + + /** @var array */ + private array $crImportExportTrait_loggedWarnings = []; + + private function setupCrImportExportTrait(): void + { + $this->crImportExportTrait_filesystem = new Filesystem(new InMemoryFilesystemAdapter()); + } + + /** + * @AfterScenario + */ + public function failIfLastMigrationHasErrors(): void + { + if ($this->crImportExportTrait_lastMigrationException !== null) { + throw new \RuntimeException(sprintf('The last migration run led to an exception: %s', $this->crImportExportTrait_lastMigrationException->getMessage())); + } + if ($this->crImportExportTrait_loggedErrors !== []) { + throw new \RuntimeException(sprintf('The last migration run logged %d error%s', count($this->crImportExportTrait_loggedErrors), count($this->crImportExportTrait_loggedErrors) === 1 ? '' : 's')); + } + } + + private function runCrImportExportProcessors(ProcessorInterface ...$processors): void + { + $processingContext = new ProcessingContext($this->crImportExportTrait_filesystem, function (Severity $severity, string $message) { + if ($severity === Severity::ERROR) { + $this->crImportExportTrait_loggedErrors[] = $message; + } elseif ($severity === Severity::WARNING) { + $this->crImportExportTrait_loggedWarnings[] = $message; + } + }); + foreach ($processors as $processor) { + assert($processor instanceof ProcessorInterface); + try { + $processor->run($processingContext); + } catch (\Throwable $e) { + $this->crImportExportTrait_lastMigrationException = $e; + break; + } + } + } + + /** + * @When /^the events are exported$/ + */ + public function theEventsAreExported(): void + { + $eventExporter = $this->getContentRepositoryService(new EventExportProcessorFactory($this->currentContentRepository->findWorkspaceByName(WorkspaceName::forLive())->currentContentStreamId)); + assert($eventExporter instanceof EventExportProcessor); + $this->runCrImportExportProcessors($eventExporter); + } + + /** + * @When /^I import the events\.jsonl(?: into workspace "([^"]*)")?$/ + */ + public function iImportTheEventsJsonl(?string $workspace = null): void + { + $workspaceName = $workspace !== null ? WorkspaceName::fromString($workspace) : $this->currentWorkspaceName; + $eventImporter = $this->getContentRepositoryService(new EventStoreImportProcessorFactory($workspaceName, true)); + assert($eventImporter instanceof EventStoreImportProcessor); + $this->runCrImportExportProcessors($eventImporter); + } + + /** + * @Given /^using the following events\.jsonl:$/ + */ + public function usingTheFollowingEventsJsonl(PyStringNode $string): void + { + $this->crImportExportTrait_filesystem->write('events.jsonl', $string->getRaw()); + } + + /** + * @Then I expect the following jsonl: + */ + public function iExpectTheFollowingJsonL(PyStringNode $string): void + { + if (!$this->crImportExportTrait_filesystem->has('events.jsonl')) { + Assert::fail('No events were exported'); + } + + $jsonL = $this->crImportExportTrait_filesystem->read('events.jsonl'); + + $exportedEvents = ExportedEvents::fromJsonl($jsonL); + $eventsWithoutRandomIds = []; + + foreach ($exportedEvents as $exportedEvent) { + // we have to remove the event id in \Neos\ContentRepository\Core\Feature\Common\NodeAggregateEventPublisher::enrichWithCommand + // and the initiatingTimestamp to make the events diff able + $eventsWithoutRandomIds[] = $exportedEvent + ->withIdentifier('random-event-uuid') + ->processMetadata(function (array $metadata) { + $metadata['initiatingTimestamp'] = 'random-time'; + return $metadata; + }); + } + + Assert::assertSame($string->getRaw(), ExportedEvents::fromIterable($eventsWithoutRandomIds)->toJsonl()); + } + + /** + * @Then I expect the following events to be exported + */ + public function iExpectTheFollowingEventsToBeExported(TableNode $table): void + { + + if (!$this->crImportExportTrait_filesystem->has('events.jsonl')) { + Assert::fail('No events were exported'); + } + $eventsJson = $this->crImportExportTrait_filesystem->read('events.jsonl'); + $exportedEvents = iterator_to_array(ExportedEvents::fromJsonl($eventsJson)); + + $expectedEvents = $table->getHash(); + foreach ($exportedEvents as $exportedEvent) { + $expectedEventRow = array_shift($expectedEvents); + if ($expectedEventRow === null) { + Assert::assertCount(count($table->getHash()), $exportedEvents, 'Expected number of events does not match actual number'); + } + if (!empty($expectedEventRow['Type'])) { + Assert::assertSame($expectedEventRow['Type'], $exportedEvent->type, 'Event: ' . $exportedEvent->toJson()); + } + try { + $expectedEventPayload = json_decode($expectedEventRow['Payload'], true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new \RuntimeException(sprintf('Failed to decode expected JSON: %s', $expectedEventRow['Payload']), 1655811083); + } + $actualEventPayload = $exportedEvent->payload; + foreach (array_keys($actualEventPayload) as $key) { + if (!array_key_exists($key, $expectedEventPayload)) { + unset($actualEventPayload[$key]); + } + } + Assert::assertEquals($expectedEventPayload, $actualEventPayload, 'Actual event: ' . $exportedEvent->toJson()); + } + Assert::assertCount(count($table->getHash()), $exportedEvents, 'Expected number of events does not match actual number'); + } + + /** + * @Then I expect the following errors to be logged + */ + public function iExpectTheFollowingErrorsToBeLogged(TableNode $table): void + { + Assert::assertSame($table->getColumn(0), $this->crImportExportTrait_loggedErrors, 'Expected logged errors do not match'); + $this->crImportExportTrait_loggedErrors = []; + } + + /** + * @Then I expect the following warnings to be logged + */ + public function iExpectTheFollowingWarningsToBeLogged(TableNode $table): void + { + Assert::assertSame($table->getColumn(0), $this->crImportExportTrait_loggedWarnings, 'Expected logged warnings do not match'); + $this->crImportExportTrait_loggedWarnings = []; + } + + /** + * @Then I expect a migration exception + * @Then I expect a migration exception with the message + */ + public function iExpectAMigrationExceptionWithTheMessage(PyStringNode $expectedMessage = null): void + { + Assert::assertNotNull($this->crImportExportTrait_lastMigrationException, 'Expected the previous migration to lead to an exception, but no exception was thrown'); + if ($expectedMessage !== null) { + Assert::assertSame($expectedMessage->getRaw(), $this->crImportExportTrait_lastMigrationException->getMessage()); + } + $this->crImportExportTrait_lastMigrationException = null; + } + + /** + * @Given the following ImageVariants exist + */ + public function theFollowingImageVariantsExist(TableNode $imageVariants): void + { + foreach ($imageVariants->getHash() as $variantData) { + try { + $variantData['imageAdjustments'] = json_decode($variantData['imageAdjustments'], true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new \RuntimeException(sprintf('Failed to JSON decode imageAdjustments for variant "%s"', $variantData['identifier']), 1659530081, $e); + } + $variantData['width'] = (int)$variantData['width']; + $variantData['height'] = (int)$variantData['height']; + $mockImageVariant = SerializedImageVariant::fromArray($variantData); + $this->mockAssets[$mockImageVariant->identifier] = $mockImageVariant; + } + } + + /** + * @Then /^I expect the following (Assets|ImageVariants) to be exported:$/ + */ + public function iExpectTheFollowingAssetsOrImageVariantsToBeExported(string $type, PyStringNode $expectedAssets): void + { + $actualAssets = []; + if (!$this->crImportExportTrait_filesystem->directoryExists($type)) { + Assert::fail(sprintf('No %1$s have been exported (Directory "/%1$s" does not exist)', $type)); + } + /** @var FileAttributes $file */ + foreach ($this->crImportExportTrait_filesystem->listContents($type) as $file) { + $actualAssets[] = json_decode($this->crImportExportTrait_filesystem->read($file->path()), true, 512, JSON_THROW_ON_ERROR); + } + Assert::assertJsonStringEqualsJsonString($expectedAssets->getRaw(), json_encode($actualAssets, JSON_THROW_ON_ERROR)); + } + + + /** + * @Then /^I expect no (Assets|ImageVariants) to be exported$/ + */ + public function iExpectNoAssetsToBeExported(string $type): void + { + Assert::assertFalse($this->crImportExportTrait_filesystem->directoryExists($type)); + } + + /** + * @Then I expect the following PersistentResources to be exported: + */ + public function iExpectTheFollowingPersistentResourcesToBeExported(TableNode $expectedResources): void + { + $actualResources = []; + if (!$this->crImportExportTrait_filesystem->directoryExists('Resources')) { + Assert::fail('No PersistentResources have been exported (Directory "/Resources" does not exist)'); + } + /** @var FileAttributes $file */ + foreach ($this->crImportExportTrait_filesystem->listContents('Resources') as $file) { + $actualResources[] = ['Filename' => basename($file->path()), 'Contents' => $this->crImportExportTrait_filesystem->read($file->path())]; + } + Assert::assertSame($expectedResources->getHash(), $actualResources); + } + + /** + * @Then /^I expect no PersistentResources to be exported$/ + */ + public function iExpectNoPersistentResourcesToBeExported(): void + { + Assert::assertFalse($this->crImportExportTrait_filesystem->directoryExists('Resources')); + } +} diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/FeatureContext.php b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/FeatureContext.php new file mode 100644 index 00000000000..01310c7beff --- /dev/null +++ b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/FeatureContext.php @@ -0,0 +1,78 @@ +contentRepositoryRegistry = $this->getObject(ContentRepositoryRegistry::class); + + $this->setupCrImportExportTrait(); + } + + /** + * @BeforeScenario + */ + public function resetContentRepositoryComponents(BeforeScenarioScope $scope): void + { + GherkinTableNodeBasedContentDimensionSourceFactory::reset(); + GherkinPyStringNodeBasedNodeTypeManagerFactory::reset(); + } + + protected function getContentRepositoryService( + ContentRepositoryServiceFactoryInterface $factory + ): ContentRepositoryServiceInterface { + return $this->contentRepositoryRegistry->buildService( + $this->currentContentRepository->id, + $factory + ); + } + + protected function createContentRepository( + ContentRepositoryId $contentRepositoryId + ): ContentRepository { + $this->contentRepositoryRegistry->resetFactoryInstance($contentRepositoryId); + $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); + GherkinTableNodeBasedContentDimensionSourceFactory::reset(); + GherkinPyStringNodeBasedNodeTypeManagerFactory::reset(); + + return $contentRepository; + } +} diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature b/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature new file mode 100644 index 00000000000..9c3f28cad09 --- /dev/null +++ b/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature @@ -0,0 +1,43 @@ +@contentrepository +Feature: As a user of the CR I want to export the event stream using the EventExportProcessor + + Background: + Given using the following content dimensions: + | Identifier | Values | Generalizations | + | language | de, gsw, fr | gsw->de | + And using the following node types: + """yaml + 'Neos.ContentRepository.Testing:Document': [] + """ + And using identifier "default", I define a content repository + And I am in content repository "default" + And the command CreateRootWorkspace is executed with payload: + | Key | Value | + | workspaceName | "live" | + | newContentStreamId | "cs-identifier" | + And I am in workspace "live" + And the command CreateRootNodeAggregateWithNode is executed with payload: + | Key | Value | + | nodeAggregateId | "lady-eleonode-rootford" | + | nodeTypeName | "Neos.ContentRepository:Root" | + And the event NodeAggregateWithNodeWasCreated was published with payload: + | Key | Value | + | workspaceName | "live" | + | contentStreamId | "cs-identifier" | + | nodeAggregateId | "nody-mc-nodeface" | + | nodeTypeName | "Neos.ContentRepository.Testing:Document" | + | originDimensionSpacePoint | {"language":"de"} | + | coveredDimensionSpacePoints | [{"language":"de"},{"language":"gsw"},{"language":"fr"}] | + | parentNodeAggregateId | "lady-eleonode-rootford" | + | nodeName | "child-document" | + | nodeAggregateClassification | "regular" | + + Scenario: Export the event stream + Then I expect exactly 3 events to be published on stream with prefix "ContentStream:cs-identifier" + When the events are exported + Then I expect the following jsonl: + """ + {"identifier":"random-event-uuid","type":"RootNodeAggregateWithNodeWasCreated","payload":{"workspaceName":"live","contentStreamId":"cs-identifier","nodeAggregateId":"lady-eleonode-rootford","nodeTypeName":"Neos.ContentRepository:Root","coveredDimensionSpacePoints":[{"language":"de"},{"language":"gsw"},{"language":"fr"}],"nodeAggregateClassification":"root"},"metadata":{"commandClass":"Neos\\ContentRepository\\Core\\Feature\\RootNodeCreation\\Command\\CreateRootNodeAggregateWithNode","commandPayload":{"workspaceName":"live","nodeAggregateId":"lady-eleonode-rootford","nodeTypeName":"Neos.ContentRepository:Root","tetheredDescendantNodeAggregateIds":[]},"initiatingUserId":"system","initiatingTimestamp":"random-time"}} + {"identifier":"random-event-uuid","type":"NodeAggregateWithNodeWasCreated","payload":{"workspaceName":"live","contentStreamId":"cs-identifier","nodeAggregateId":"nody-mc-nodeface","nodeTypeName":"Neos.ContentRepository.Testing:Document","originDimensionSpacePoint":{"language":"de"},"succeedingSiblingsForCoverage":[{"dimensionSpacePoint":{"language":"de"},"nodeAggregateId":null},{"dimensionSpacePoint":{"language":"gsw"},"nodeAggregateId":null},{"dimensionSpacePoint":{"language":"fr"},"nodeAggregateId":null}],"parentNodeAggregateId":"lady-eleonode-rootford","nodeName":"child-document","initialPropertyValues":[],"nodeAggregateClassification":"regular"},"metadata":{"initiatingTimestamp":"random-time"}} + + """ diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/EventStoreImportProcessor.feature b/Neos.ContentRepository.Export/Tests/Behavior/Features/EventStoreImportProcessor.feature new file mode 100644 index 00000000000..443ae5ff474 --- /dev/null +++ b/Neos.ContentRepository.Export/Tests/Behavior/Features/EventStoreImportProcessor.feature @@ -0,0 +1,84 @@ +@contentrepository +Feature: As a user of the CR I want to import events using the EventStoreImportProcessor + + Background: + Given using no content dimensions + And using the following node types: + """yaml + Vendor.Site:HomePage': + superTypes: + Neos.Neos:Site: true + """ + And using identifier "default", I define a content repository + And I am in content repository "default" + And the command CreateRootWorkspace is executed with payload: + | Key | Value | + | workspaceName | "live" | + | newContentStreamId | "cs-identifier" | + And I am in workspace "live" + + Scenario: Import the event stream into a specific content stream + Given using the following events.jsonl: + """ + {"identifier":"9f64c281-e5b0-48d9-900b-288a8faf92a9","type":"RootNodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site-sites","nodeTypeName":"Neos.Neos:Sites","coveredDimensionSpacePoints":[[]],"nodeAggregateClassification":"root"},"metadata":[]} + {"identifier":"1640ebbf-7ffe-4526-b0f4-7575cefabfab","type":"NodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site","nodeTypeName":"Vendor.Site:HomePage","originDimensionSpacePoint":[],"succeedingSiblingsForCoverage":[{"dimensionSpacePoint":[],"nodeAggregateId":null}],"parentNodeAggregateId":"acme-site-sites","nodeName":"acme-site","initialPropertyValues":{"title":{"value":"My Site","type":"string"},"uriPathSegment":{"value":"my-site","type":"string"}},"nodeAggregateClassification":"regular"},"metadata":[]} + """ + And I import the events.jsonl into workspace "live" + Then I expect exactly 3 events to be published on stream with prefix "ContentStream:cs-identifier" + And event at index 0 is of type "ContentStreamWasCreated" with payload: + | Key | Expected | + | contentStreamId | "cs-identifier" | + And event at index 1 is of type "RootNodeAggregateWithNodeWasCreated" with payload: + | Key | Expected | + | workspaceName | "live" | + | contentStreamId | "cs-identifier" | + | nodeAggregateId | "acme-site-sites" | + | nodeTypeName | "Neos.Neos:Sites" | + And event at index 2 is of type "NodeAggregateWithNodeWasCreated" with payload: + | Key | Expected | + | workspaceName | "live" | + | contentStreamId | "cs-identifier" | + | nodeAggregateId | "acme-site" | + | nodeTypeName | "Vendor.Site:HomePage" | + + Scenario: Import the event stream + Given using the following events.jsonl: + """ + {"identifier":"9f64c281-e5b0-48d9-900b-288a8faf92a9","type":"RootNodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site-sites","nodeTypeName":"Neos.Neos:Sites","coveredDimensionSpacePoints":[[]],"nodeAggregateClassification":"root"},"metadata":[]} + {"identifier":"1640ebbf-7ffe-4526-b0f4-7575cefabfab","type":"NodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site","nodeTypeName":"Vendor.Site:HomePage","originDimensionSpacePoint":[],"succeedingSiblingsForCoverage":[{"dimensionSpacePoint":[],"nodeAggregateId":null}],"parentNodeAggregateId":"acme-site-sites","nodeName":"acme-site","initialPropertyValues":{"title":{"value":"My Site","type":"string"},"uriPathSegment":{"value":"my-site","type":"string"}},"nodeAggregateClassification":"regular"},"metadata":[]} + """ + And I import the events.jsonl + Then I expect exactly 3 events to be published on stream with prefix "ContentStream:cs-identifier" + And event at index 0 is of type "ContentStreamWasCreated" with payload: + | Key | Expected | + | contentStreamId | "cs-identifier" | + And event at index 1 is of type "RootNodeAggregateWithNodeWasCreated" with payload: + | Key | Expected | + | workspaceName | "live" | + | contentStreamId | "cs-identifier" | + | nodeAggregateId | "acme-site-sites" | + | nodeTypeName | "Neos.Neos:Sites" | + And event at index 2 is of type "NodeAggregateWithNodeWasCreated" with payload: + | Key | Expected | + | workspaceName | "live" | + | contentStreamId | "cs-identifier" | + | nodeAggregateId | "acme-site" | + | nodeTypeName | "Vendor.Site:HomePage" | + + Scenario: Import faulty event stream with explicit "ContentStreamWasCreated" does not duplicate content-stream + see issue https://github.com/neos/neos-development-collection/issues/4298 + + Given using the following events.jsonl: + """ + {"identifier":"5f2da12d-7037-4524-acb0-d52037342c77","type":"ContentStreamWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier"},"metadata":[]} + {"identifier":"9f64c281-e5b0-48d9-900b-288a8faf92a9","type":"RootNodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site-sites","nodeTypeName":"Neos.Neos:Sites","coveredDimensionSpacePoints":[[]],"nodeAggregateClassification":"root"},"metadata":[]} + {"identifier":"1640ebbf-7ffe-4526-b0f4-7575cefabfab","type":"NodeAggregateWithNodeWasCreated","payload":{"workspaceName":"workspace-name","contentStreamId":"cs-imported-identifier","nodeAggregateId":"acme-site","nodeTypeName":"Vendor.Site:HomePage","originDimensionSpacePoint":[],"coveredDimensionSpacePoints":[[]],"parentNodeAggregateId":"acme-site-sites","nodeName":"acme-site","initialPropertyValues":{"title":{"value":"My Site","type":"string"},"uriPathSegment":{"value":"my-site","type":"string"}},"nodeAggregateClassification":"regular","succeedingNodeAggregateId":null},"metadata":[]} + """ + And I import the events.jsonl + + And I expect a migration exception with the message + """ + Failed to read events. ContentStreamWasCreated is not expected in imported event stream. + """ + + Then I expect exactly 0 events to be published on stream with prefix "ContentStream:cs-imported-identifier" diff --git a/Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist b/Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 0e10464c0fc..7ec432e2230 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -1,6 +1,9 @@ */ private array $mockAssets = []; - private InMemoryFilesystemAdapter $mockFilesystemAdapter; - private Filesystem $mockFilesystem; - - private \Throwable|null $lastMigrationException = null; - - /** - * @var array - */ - private array $loggedErrors = []; - - /** - * @var array - */ - private array $loggedWarnings = []; - private ContentRepository $contentRepository; protected ContentRepositoryRegistry $contentRepositoryRegistry; @@ -78,21 +68,7 @@ public function __construct() self::bootstrapFlow(); $this->contentRepositoryRegistry = $this->getObject(ContentRepositoryRegistry::class); - $this->mockFilesystemAdapter = new InMemoryFilesystemAdapter(); - $this->mockFilesystem = new Filesystem($this->mockFilesystemAdapter); - } - - /** - * @AfterScenario - */ - public function failIfLastMigrationHasErrors(): void - { - if ($this->lastMigrationException !== null) { - throw new \RuntimeException(sprintf('The last migration run led to an exception: %s', $this->lastMigrationException->getMessage())); - } - if ($this->loggedErrors !== []) { - throw new \RuntimeException(sprintf('The last migration run logged %d error%s', count($this->loggedErrors), count($this->loggedErrors) === 1 ? '' : 's')); - } + $this->setupCrImportExportTrait(); } /** @@ -149,86 +125,7 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor if ($contentStream !== null) { $migration->setContentStreamId(ContentStreamId::fromString($contentStream)); } - $processingContext = new ProcessingContext($this->mockFilesystem, function (Severity $severity, string $message) { - if ($severity === Severity::ERROR) { - $this->loggedErrors[] = $message; - } elseif ($severity === Severity::WARNING) { - $this->loggedWarnings[] = $message; - } - }); - try { - $migration->run($processingContext); - } catch (\Throwable $e) { - $this->lastMigrationException = $e; - } - } - - /** - * @Then I expect the following events to be exported - */ - public function iExpectTheFollowingEventsToBeExported(TableNode $table): void - { - - if (!$this->mockFilesystem->has('events.jsonl')) { - Assert::fail('No events were exported'); - } - $eventsJson = $this->mockFilesystem->read('events.jsonl'); - $exportedEvents = iterator_to_array(ExportedEvents::fromJsonl($eventsJson)); - - $expectedEvents = $table->getHash(); - foreach ($exportedEvents as $exportedEvent) { - $expectedEventRow = array_shift($expectedEvents); - if ($expectedEventRow === null) { - Assert::assertCount(count($table->getHash()), $exportedEvents, 'Expected number of events does not match actual number'); - } - if (!empty($expectedEventRow['Type'])) { - Assert::assertSame($expectedEventRow['Type'], $exportedEvent->type, 'Event: ' . $exportedEvent->toJson()); - } - try { - $expectedEventPayload = json_decode($expectedEventRow['Payload'], true, 512, JSON_THROW_ON_ERROR); - } catch (JsonException $e) { - throw new \RuntimeException(sprintf('Failed to decode expected JSON: %s', $expectedEventRow['Payload']), 1655811083); - } - $actualEventPayload = $exportedEvent->payload; - foreach (array_keys($actualEventPayload) as $key) { - if (!array_key_exists($key, $expectedEventPayload)) { - unset($actualEventPayload[$key]); - } - } - Assert::assertEquals($expectedEventPayload, $actualEventPayload, 'Actual event: ' . $exportedEvent->toJson()); - } - Assert::assertCount(count($table->getHash()), $exportedEvents, 'Expected number of events does not match actual number'); - } - - /** - * @Then I expect the following errors to be logged - */ - public function iExpectTheFollowingErrorsToBeLogged(TableNode $table): void - { - Assert::assertSame($table->getColumn(0), $this->loggedErrors, 'Expected logged errors do not match'); - $this->loggedErrors = []; - } - - /** - * @Then I expect the following warnings to be logged - */ - public function iExpectTheFollowingWarningsToBeLogged(TableNode $table): void - { - Assert::assertSame($table->getColumn(0), $this->loggedWarnings, 'Expected logged warnings do not match'); - $this->loggedWarnings = []; - } - - /** - * @Then I expect a MigrationError - * @Then I expect a MigrationError with the message - */ - public function iExpectAMigrationErrorWithTheMessage(PyStringNode $expectedMessage = null): void - { - Assert::assertNotNull($this->lastMigrationException, 'Expected the previous migration to lead to an exception, but no exception was thrown'); - if ($expectedMessage !== null) { - Assert::assertSame($expectedMessage->getRaw(), $this->lastMigrationException->getMessage()); - } - $this->lastMigrationException = null; + $this->runCrImportExportProcessors($migration); } /** @@ -269,24 +166,6 @@ public function theFollowingAssetsExist(TableNode $images): void } } - /** - * @Given the following ImageVariants exist - */ - public function theFollowingImageVariantsExist(TableNode $imageVariants): void - { - foreach ($imageVariants->getHash() as $variantData) { - try { - $variantData['imageAdjustments'] = json_decode($variantData['imageAdjustments'], true, 512, JSON_THROW_ON_ERROR); - } catch (JsonException $e) { - throw new \RuntimeException(sprintf('Failed to JSON decode imageAdjustments for variant "%s"', $variantData['identifier']), 1659530081, $e); - } - $variantData['width'] = (int)$variantData['width']; - $variantData['height'] = (int)$variantData['height']; - $mockImageVariant = SerializedImageVariant::fromArray($variantData); - $this->mockAssets[$mockImageVariant->identifier] = $mockImageVariant; - } - } - /** * @When I run the asset migration */ @@ -298,7 +177,9 @@ public function iRunTheAssetMigration(): void /** * @param array $mockResources */ - public function __construct(private array $mockResources) {} + public function __construct(private array $mockResources) + { + } public function getStreamBySha1(string $sha1) { @@ -315,7 +196,9 @@ public function getStreamBySha1(string $sha1) /** * @param array $mockAssets */ - public function __construct(private array $mockAssets) {} + public function __construct(private array $mockAssets) + { + } public function findAssetById(string $assetId): SerializedAsset|SerializedImageVariant { @@ -326,88 +209,13 @@ public function findAssetById(string $assetId): SerializedAsset|SerializedImageV } }; - $this->mockFilesystemAdapter->deleteEverything(); - $assetExporter = new AssetExporter($this->mockFilesystem, $mockAssetLoader, $mockResourceLoader); + $assetExporter = new AssetExporter($this->crImportExportTrait_filesystem, $mockAssetLoader, $mockResourceLoader); $migration = new NodeDataToAssetsProcessor($nodeTypeManager, $assetExporter, $this->nodeDataRows); - $processingContext = new ProcessingContext($this->mockFilesystem, function (Severity $severity, string $message) { - if ($severity === Severity::ERROR) { - $this->loggedErrors[] = $message; - } elseif ($severity === Severity::WARNING) { - $this->loggedWarnings[] = $message; - } - }); - try { - $migration->run($processingContext); - } catch (\Throwable $e) { - $this->lastMigrationException = $e; - } + $this->runCrImportExportProcessors($migration); } - /** - * @Then /^I expect the following (Assets|ImageVariants) to be exported:$/ - */ - public function iExpectTheFollowingToBeExported(string $type, PyStringNode $expectedAssets): void - { - $actualAssets = []; - if (!$this->mockFilesystem->directoryExists($type)) { - Assert::fail(sprintf('No %1$s have been exported (Directory "/%1$s" does not exist)', $type)); - } - /** @var FileAttributes $file */ - foreach ($this->mockFilesystem->listContents($type) as $file) { - $actualAssets[] = json_decode($this->mockFilesystem->read($file->path()), true, 512, JSON_THROW_ON_ERROR); - } - Assert::assertJsonStringEqualsJsonString($expectedAssets->getRaw(), json_encode($actualAssets, JSON_THROW_ON_ERROR)); - } - - /** - * @Then /^I expect no (Assets|ImageVariants) to be exported$/ - */ - public function iExpectNoAssetsToBeExported(string $type): void - { - Assert::assertFalse($this->mockFilesystem->directoryExists($type)); - } - - /** - * @Then I expect the following PersistentResources to be exported: - */ - public function iExpectTheFollowingPersistentResourcesToBeExported(TableNode $expectedResources): void - { - $actualResources = []; - if (!$this->mockFilesystem->directoryExists('Resources')) { - Assert::fail('No PersistentResources have been exported (Directory "/Resources" does not exist)'); - } - /** @var FileAttributes $file */ - foreach ($this->mockFilesystem->listContents('Resources') as $file) { - $actualResources[] = ['Filename' => basename($file->path()), 'Contents' => $this->mockFilesystem->read($file->path())]; - } - Assert::assertSame($expectedResources->getHash(), $actualResources); - } - - /** - * @Then /^I expect no PersistentResources to be exported$/ - */ - public function iExpectNoPersistentResourcesToBeExported(): void - { - Assert::assertFalse($this->mockFilesystem->directoryExists('Resources')); - } - - /** ---------------------------------- */ - /** - * @param TableNode $table - * @return array - * @throws JsonException - */ - private function parseJsonTable(TableNode $table): array - { - return array_map(static function (array $row) { - return array_map(static function (string $jsonValue) { - return json_decode($jsonValue, true, 512, JSON_THROW_ON_ERROR); - }, $row); - }, $table->getHash()); - } - protected function getContentRepositoryService( ContentRepositoryServiceFactoryInterface $factory ): ContentRepositoryServiceInterface { diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature index 49b2b021c08..c41e915c647 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature @@ -34,7 +34,7 @@ Feature: Exceptional cases during migrations | site-node-id | /sites/test-site | Some.Package:Homepage | {"language": ["de"]} | | site-node-id | /sites/test-site | Some.Package:SomeOtherHomepage | {"language": ["en"]} | And I run the event migration - Then I expect a MigrationError with the message + Then I expect a migration exception with the message """ Node aggregate with id "site-node-id" has a type of "Some.Package:SomeOtherHomepage" in content dimension [{"language":"en"}]. I was visited previously for content dimension [{"language":"de"}] with the type "Some.Package:Homepage". Node variants must not have different types """ @@ -94,7 +94,7 @@ Feature: Exceptional cases during migrations | sites | /sites | | | a | /sites/a | not json | And I run the event migration - Then I expect a MigrationError + Then I expect a migration exception Scenario: Invalid node properties (no JSON) When I have the following node data rows: @@ -102,7 +102,7 @@ Feature: Exceptional cases during migrations | sites | /sites | | | | a | /sites/a | not json | Some.Package:Homepage | And I run the event migration - Then I expect a MigrationError with the message + Then I expect a migration exception with the message """ Failed to decode properties "not json" of node "a" (type: "Some.Package:Homepage"): Could not convert database value "not json" to Doctrine Type flow_json_array """ @@ -118,7 +118,7 @@ Feature: Exceptional cases during migrations | site-node-id | /sites/test-site | Some.Package:Homepage | {"language": ["ch"]} | | site-node-id | /sites/test-site | Some.Package:Homepage | {"language": ["ch"]} | And I run the event migration - Then I expect a MigrationError with the message + Then I expect a migration exception with the message """ Node "site-node-id" with dimension space point "{"language":"ch"}" was already visited before """ @@ -133,7 +133,7 @@ Feature: Exceptional cases during migrations | site-node-id | /sites/test-site | Some.Package:Homepage | {"language": ["de"]} | | site-node-id | /sites/test-site | Some.Package:Homepage | {"language": ["de"]} | And I run the event migration - Then I expect a MigrationError with the message + Then I expect a migration exception with the message """ Node "site-node-id" for dimension {"language":"de"} was already created previously """ @@ -144,7 +144,7 @@ Feature: Exceptional cases during migrations | sites-node-id | /sites | unstructured | | site-node-id | /sites/test-site | unstructured | And I run the event migration - Then I expect a MigrationError with the message + Then I expect a migration exception with the message """ The site node "site-node-id" (type: "unstructured") must be of type "Neos.Neos:Site" """ From 191ce88a0733b988cff2fd6e6bbcb941ed7945b0 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Tue, 22 Oct 2024 17:47:40 +0200 Subject: [PATCH 06/56] Rename `Factories` folder to `Factory --- .../Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php | 4 ++-- .../ContentRepositorySetupProcessorFactory.php | 2 +- .../{Factories => Factory}/EventExportProcessorFactory.php | 2 +- .../EventStoreImportProcessorFactory.php | 2 +- Neos.Neos/Classes/Domain/Service/SiteImportService.php | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) rename Neos.ContentRepository.Export/src/{Factories => Factory}/ContentRepositorySetupProcessorFactory.php (93%) rename Neos.ContentRepository.Export/src/{Factories => Factory}/EventExportProcessorFactory.php (94%) rename Neos.ContentRepository.Export/src/{Factories => Factory}/EventStoreImportProcessorFactory.php (95%) diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php index 8f06d03cf64..8fed952b2bd 100644 --- a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php +++ b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php @@ -22,8 +22,8 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\Asset\ValueObject\SerializedImageVariant; use Neos\ContentRepository\Export\Event\ValueObject\ExportedEvents; -use Neos\ContentRepository\Export\Factories\EventExportProcessorFactory; -use Neos\ContentRepository\Export\Factories\EventStoreImportProcessorFactory; +use Neos\ContentRepository\Export\Factory\EventExportProcessorFactory; +use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; use Neos\ContentRepository\Export\Processors\EventExportProcessor; diff --git a/Neos.ContentRepository.Export/src/Factories/ContentRepositorySetupProcessorFactory.php b/Neos.ContentRepository.Export/src/Factory/ContentRepositorySetupProcessorFactory.php similarity index 93% rename from Neos.ContentRepository.Export/src/Factories/ContentRepositorySetupProcessorFactory.php rename to Neos.ContentRepository.Export/src/Factory/ContentRepositorySetupProcessorFactory.php index 076515afaaf..945ef993da7 100644 --- a/Neos.ContentRepository.Export/src/Factories/ContentRepositorySetupProcessorFactory.php +++ b/Neos.ContentRepository.Export/src/Factory/ContentRepositorySetupProcessorFactory.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Export\Factories; +namespace Neos\ContentRepository\Export\Factory; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; diff --git a/Neos.ContentRepository.Export/src/Factories/EventExportProcessorFactory.php b/Neos.ContentRepository.Export/src/Factory/EventExportProcessorFactory.php similarity index 94% rename from Neos.ContentRepository.Export/src/Factories/EventExportProcessorFactory.php rename to Neos.ContentRepository.Export/src/Factory/EventExportProcessorFactory.php index 6fe742cec21..636a3a5a1be 100644 --- a/Neos.ContentRepository.Export/src/Factories/EventExportProcessorFactory.php +++ b/Neos.ContentRepository.Export/src/Factory/EventExportProcessorFactory.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Export\Factories; +namespace Neos\ContentRepository\Export\Factory; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; diff --git a/Neos.ContentRepository.Export/src/Factories/EventStoreImportProcessorFactory.php b/Neos.ContentRepository.Export/src/Factory/EventStoreImportProcessorFactory.php similarity index 95% rename from Neos.ContentRepository.Export/src/Factories/EventStoreImportProcessorFactory.php rename to Neos.ContentRepository.Export/src/Factory/EventStoreImportProcessorFactory.php index 94b6aa2f8d4..459a746c1e6 100644 --- a/Neos.ContentRepository.Export/src/Factories/EventStoreImportProcessorFactory.php +++ b/Neos.ContentRepository.Export/src/Factory/EventStoreImportProcessorFactory.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Export\Factories; +namespace Neos\ContentRepository\Export\Factory; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index d04a48ca9dd..07107821f2c 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -18,8 +18,8 @@ use League\Flysystem\Local\LocalFilesystemAdapter; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\ContentRepository\Export\Factories\ContentRepositorySetupProcessorFactory; -use Neos\ContentRepository\Export\Factories\EventStoreImportProcessorFactory; +use Neos\ContentRepository\Export\Factory\ContentRepositorySetupProcessorFactory; +use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; From aa8c6114080564396199ba2b1aa139972b9d8f2b Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Tue, 22 Oct 2024 18:14:27 +0200 Subject: [PATCH 07/56] Remove unused namespace imports --- .../Tests/Behavior/Bootstrap/FeatureContext.php | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 7ec432e2230..7a506466d0c 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -5,11 +5,7 @@ require_once(__DIR__ . '/../../../../Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php'); use Behat\Behat\Context\Context; -use Behat\Gherkin\Node\PyStringNode; use Behat\Gherkin\Node\TableNode; -use League\Flysystem\FileAttributes; -use League\Flysystem\Filesystem; -use League\Flysystem\InMemory\InMemoryFilesystemAdapter; use Neos\Behat\FlowBootstrapTrait; use Neos\ContentGraph\DoctrineDbalAdapter\Tests\Behavior\Features\Bootstrap\CrImportExportTrait; use Neos\ContentRepository\BehavioralTests\TestSuite\Behavior\CRBehavioralTestsSubjectProvider; @@ -30,16 +26,12 @@ use Neos\ContentRepository\Export\Asset\ValueObject\SerializedAsset; use Neos\ContentRepository\Export\Asset\ValueObject\SerializedImageVariant; use Neos\ContentRepository\Export\Asset\ValueObject\SerializedResource; -use Neos\ContentRepository\Export\Event\ValueObject\ExportedEvents; -use Neos\ContentRepository\Export\ProcessingContext; -use Neos\ContentRepository\Export\Severity; use Neos\ContentRepository\LegacyNodeMigration\NodeDataToAssetsProcessor; use Neos\ContentRepository\LegacyNodeMigration\NodeDataToEventsProcessor; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\CRTestSuiteTrait; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Property\PropertyMapper; use Neos\Flow\ResourceManagement\PersistentResource; -use PHPUnit\Framework\Assert; use PHPUnit\Framework\MockObject\Generator as MockGenerator; /** From 630bc09a2e0316c456e133119120cec10e1c35b2 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 10:50:58 +0200 Subject: [PATCH 08/56] Verify tha that Live workspace contains no events prior to importing --- .../Import/LiveWorkspaceIsEmptyProcessor.php | 53 +++++++++++++++++++ .../LiveWorkspaceIsEmptyProcessorFactory.php | 27 ++++++++++ .../Domain/Service/SiteImportService.php | 3 +- 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php create mode 100644 Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php new file mode 100644 index 00000000000..82f6a967c07 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php @@ -0,0 +1,53 @@ +dispatch(Severity::NOTICE, 'Ensures empty live workspace'); + + if ($this->workspaceHasEvents(WorkspaceName::forLive())) { + throw new \RuntimeException( 'Live workspace already contains events please run "cr:prune" before importing.'); + } + } + + private function workspaceHasEvents(WorkspaceName $workspaceName): bool + { + $workspaceStreamName = WorkspaceEventStreamName::fromWorkspaceName($workspaceName)->getEventStreamName(); + $eventStream = $this->eventStore->load($workspaceStreamName); + foreach ($eventStream as $event) { + return true; + } + return false; + } +} diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php new file mode 100644 index 00000000000..792b0495cac --- /dev/null +++ b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php @@ -0,0 +1,27 @@ +eventStore); + } +} diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 07107821f2c..50a1a044238 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -33,6 +33,7 @@ use Neos\Media\Domain\Repository\AssetRepository; use Neos\Neos\Domain\Import\DoctrineMigrateProcessor; use Neos\Neos\Domain\Import\LiveWorkspaceCreationProcessor; +use Neos\Neos\Domain\Import\LiveWorkspaceIsEmptyProcessorFactory; use Neos\Neos\Domain\Import\SiteCreationProcessor; use Neos\Neos\Domain\Repository\SiteRepository; @@ -69,7 +70,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string $processors = [ 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), 'Setup content repository' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositorySetupProcessorFactory()), - // TODO Check if target content stream is empty, otherwise => nice error "prune..." + 'Verify Live workspace does not exist yet' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new LiveWorkspaceIsEmptyProcessorFactory()), 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository), 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), From 221096b5afc90b221c266289ac71c22c61901b04 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 10:51:21 +0200 Subject: [PATCH 09/56] Ensure sites are persisted during import --- Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index e4d6c0c6d92..401a8fd4810 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -20,6 +20,7 @@ use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; use Neos\ContentRepository\Export\Severity; +use Neos\Flow\Persistence\Doctrine\PersistenceManager; use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\SiteRepository; @@ -32,6 +33,7 @@ { public function __construct( private SiteRepository $siteRepository, + private PersistenceManager $persistenceManager ) { } @@ -47,6 +49,7 @@ public function run(ProcessingContext $context): void } else { $sites = self::extractSitesFromEventStream($context); } + $persistAllIsRequired = false; /** @var SiteShape $site */ foreach ($sites as $site) { $context->dispatch(Severity::NOTICE, "Creating site \"{$site['name']}\""); @@ -56,16 +59,19 @@ public function run(ProcessingContext $context): void $context->dispatch(Severity::NOTICE, "Site for node name \"{$siteNodeName->value}\" already exists, skipping"); continue; } - // TODO use node aggregate identifier instead of node name $siteInstance = new Site($siteNodeName->value); $siteInstance->setSiteResourcesPackageKey($site['packageKey']); $siteInstance->setState(($site['inactive'] ?? false) ? Site::STATE_OFFLINE : Site::STATE_ONLINE); $siteInstance->setName($site['name']); $this->siteRepository->add($siteInstance); + $persistAllIsRequired = true; // TODO add domains? } + if ($persistAllIsRequired) { + $this->persistenceManager->persistAll(); + } } /** From 918a052176899ec8ea3e3f621f41c1e1b28c04c1 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 12:28:50 +0200 Subject: [PATCH 10/56] Add site:export command again --- .../Classes/Command/SiteCommandController.php | 65 ++++++++++++- .../Domain/Export/SiteExportProcessor.php | 67 +++++++++++++ .../Domain/Import/SiteCreationProcessor.php | 18 +++- .../Domain/Service/SiteExportService.php | 94 +++++++++++++++++++ 4 files changed, 239 insertions(+), 5 deletions(-) create mode 100644 Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php create mode 100644 Neos.Neos/Classes/Domain/Service/SiteExportService.php diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index 713cc2f4da6..ba2dc2602cc 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -31,6 +31,7 @@ use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\SiteRepository; use Neos\Neos\Domain\Service\NodeTypeNameFactory; +use Neos\Neos\Domain\Service\SiteExportService; use Neos\Neos\Domain\Service\SiteImportService; use Neos\Neos\Domain\Service\SiteImportServiceFactory; use Neos\Neos\Domain\Service\SiteService; @@ -79,6 +80,12 @@ class SiteCommandController extends CommandController */ protected $siteImportService; + /** + * @Flow\Inject + * @var SiteExportService + */ + protected $siteExportService; + /** * Create a new site * @@ -131,11 +138,13 @@ public function createCommand($name, $packageKey, $nodeType, $nodeName = null, $ } /** - * Import sites content + * Import sites * - * This command allows for importing one or more sites or partial content from the file system. The format must + * This command allows importing sites from the given path/packahe. The format must * be identical to that produced by the export command. * + * !!! At the moment the live workspace has to be empty prior to importing. This will be improved in future. !!! + * * If a path is specified, this command expects the corresponding directory to contain the exported files * * If a package key is specified, this command expects the export files to be located in the private resources @@ -190,6 +199,58 @@ public function importCommand(string $packageKey = null, string $path = null, st $this->siteImportService->importFromPath($contentRepositoryId, $path, $onProcessor, $onMessage); } + /** + * Export sites + * + * This command allows to export all current sites. + * + * !!! At the moment always all sites are exported. This will be improved in future!!! + * + * If a path is specified, this command expects the corresponding directory to contain the exported files + * + * If a package key is specified, this command expects the export files to be located in the private resources + * directory of the given package (Resources/Private/Content). + * + * @param string|null $packageKey Package key specifying the package containing the sites content + * @param string|null $path relative or absolute path and filename to the export files + * @return void + */ + public function exportCommand(string $packageKey = null, string $path = null, string $contentRepository = 'default', bool $verbose = false): void + { + $exceedingArguments = $this->request->getExceedingArguments(); + if (isset($exceedingArguments[0]) && $packageKey === null && $path === null) { + if (file_exists($exceedingArguments[0])) { + $path = $exceedingArguments[0]; + } elseif ($this->packageManager->isPackageAvailable($exceedingArguments[0])) { + $packageKey = $exceedingArguments[0]; + } + } + if ($packageKey === null && $path === null) { + $this->outputLine('You have to specify either --package-key or --filename'); + $this->quit(1); + } + + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $onProcessor = function (string $processorLabel) { + $this->outputLine('%s...', [$processorLabel]); + }; + $onMessage = function (Severity $severity, string $message) use ($verbose) { + if (!$verbose && $severity === Severity::NOTICE) { + return; + } + $this->outputLine(match ($severity) { + Severity::NOTICE => $message, + Severity::WARNING => sprintf('Warning: %s', $message), + Severity::ERROR => sprintf('Error: %s', $message), + }); + }; + if ($path === null) { + $package = $this->packageManager->getPackage($packageKey); + $path = Files::concatenatePaths([$package->getPackagePath(), 'Resources/Private/Content']); + } + $this->siteExportService->exportToPath($contentRepositoryId, $path, $onProcessor, $onMessage); + } + /** * Remove site with content and related data (with globbing) * diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php new file mode 100644 index 00000000000..c79bd416604 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php @@ -0,0 +1,67 @@ + [ + "name" => $site->getName(), + "nodeName" => $site->getNodeName()->value, + "siteResourcesPackageKey" => $site->getSiteResourcesPackageKey(), + "online" => $site->isOnline(), + "domains" => array_map( + fn(Domain $domain) => [ + 'hostname' => $domain->getHostname(), + 'scheme' => $domain->getScheme(), + 'port' => $domain->getPort(), + 'active' => $domain->getActive(), + 'primary' => $domain === $site->getPrimaryDomain(), + ], + $site->getDomains()->toArray() + ) + ], + $this->siteRepository->findAll()->toArray() + ); + + $context->files->write( + 'sites.json', + json_encode($sites, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE) + ); + } +} diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index 401a8fd4810..f783f3ab9e9 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -21,6 +21,7 @@ use Neos\ContentRepository\Export\ProcessorInterface; use Neos\ContentRepository\Export\Severity; use Neos\Flow\Persistence\Doctrine\PersistenceManager; +use Neos\Neos\Domain\Model\Domain; use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\SiteRepository; @@ -61,13 +62,24 @@ public function run(ProcessingContext $context): void } // TODO use node aggregate identifier instead of node name $siteInstance = new Site($siteNodeName->value); - $siteInstance->setSiteResourcesPackageKey($site['packageKey']); + $siteInstance->setSiteResourcesPackageKey($site['siteResourcesPackageKey']); $siteInstance->setState(($site['inactive'] ?? false) ? Site::STATE_OFFLINE : Site::STATE_ONLINE); $siteInstance->setName($site['name']); + + foreach ($site['domains'] ?? [] as $domain) { + $domainInstance = new Domain(); + $domainInstance->setSite($siteInstance); + $domainInstance->setHostname($domain['hostname']); + $domainInstance->setPort($domain['port']); + $domainInstance->setScheme($domain['scheme']); + $domainInstance->setActive($domain['active'] ?? false); + if ($domain['primary']) { + $siteInstance->setPrimaryDomain($domainInstance); + } + } + $this->siteRepository->add($siteInstance); $persistAllIsRequired = true; - - // TODO add domains? } if ($persistAllIsRequired) { $this->persistenceManager->persistAll(); diff --git a/Neos.Neos/Classes/Domain/Service/SiteExportService.php b/Neos.Neos/Classes/Domain/Service/SiteExportService.php new file mode 100644 index 00000000000..97b7a1bf064 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Service/SiteExportService.php @@ -0,0 +1,94 @@ +contentRepositoryRegistry->get($contentRepositoryId); + + $liveWorkspace = $contentRepository->findWorkspaceByName(WorkspaceName::forLive()); + if ($liveWorkspace === null) { + throw new \RuntimeException('Failed to find live workspace', 1716652280); + } + + // TODO make configurable (?) + /** @var array $processors */ + $processors = [ + 'Exporting events' => $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new EventExportProcessorFactory( + $liveWorkspace->currentContentStreamId + ) + ), + 'Exporting assets' => new AssetExportProcessor( + $contentRepositoryId, + $this->assetRepository, + $liveWorkspace, + $this->assetUsageService + ), + 'Export sites' => new SiteExportProcessor($this->siteRepository), + ]; + foreach ($processors as $processorLabel => $processor) { + ($onProcessor)($processorLabel); + $processor->run($context); + } + } +} From a37387f5bff85c7ed9ff1c6e576e3fad9c249880 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 12:29:12 +0200 Subject: [PATCH 11/56] Avoid exporting of contentStreamId --- .../src/Event/ValueObject/ExportedEvent.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Neos.ContentRepository.Export/src/Event/ValueObject/ExportedEvent.php b/Neos.ContentRepository.Export/src/Event/ValueObject/ExportedEvent.php index 8c50097a96e..5392aa14ac6 100644 --- a/Neos.ContentRepository.Export/src/Event/ValueObject/ExportedEvent.php +++ b/Neos.ContentRepository.Export/src/Event/ValueObject/ExportedEvent.php @@ -24,10 +24,13 @@ public function __construct( public static function fromRawEvent(Event $event): self { + $payload = \json_decode($event->data->value, true, 512, JSON_THROW_ON_ERROR); + // unset content stream id as this is overwritten during import + unset($payload['contentStreamId']); return new self( $event->id->value, $event->type->value, - \json_decode($event->data->value, true, 512, JSON_THROW_ON_ERROR), + $payload, $event->metadata?->value ?? [], ); } From 4478619be5e3874306ab7ad7e7962104ceb19564 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 13:27:58 +0200 Subject: [PATCH 12/56] TASK: Make linter happy --- .../Domain/Export/SiteExportProcessor.php | 23 +++++++++++++------ .../Import/LiveWorkspaceIsEmptyProcessor.php | 3 ++- .../LiveWorkspaceIsEmptyProcessorFactory.php | 4 ++++ .../Domain/Import/SiteCreationProcessor.php | 17 +++++++------- .../Domain/Service/SiteExportService.php | 5 ---- .../Domain/Service/SiteImportService.php | 2 +- 6 files changed, 32 insertions(+), 22 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php index c79bd416604..6fb159c1914 100644 --- a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php +++ b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php @@ -28,7 +28,9 @@ /** * Export processor exports Neos {@see Site} instances as json * - * @phpstan-type SiteShape array{name:string, packageKey:string, nodeName?: string, inactive?:bool} + * @phpstan-type DomainShape array{hostname: string, scheme?: ?string, port?: ?int, active?: ?bool, primary?: ?bool } + * @phpstan-type SiteShape array{name:string, siteResourcesPackageKey:string, nodeName?: string, online?:bool, domains?: ?DomainShape[] } + * */ final readonly class SiteExportProcessor implements ProcessorInterface { @@ -39,7 +41,19 @@ public function __construct( public function run(ProcessingContext $context): void { - $sites = array_map( + $sites = $this->getSiteData(); + $context->files->write( + 'sites.json', + json_encode($sites, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE) + ); + } + + /** + * @return SiteShape[] + */ + private function getSiteData(): array + { + return array_map( fn(Site $site) => [ "name" => $site->getName(), "nodeName" => $site->getNodeName()->value, @@ -58,10 +72,5 @@ public function run(ProcessingContext $context): void ], $this->siteRepository->findAll()->toArray() ); - - $context->files->write( - 'sites.json', - json_encode($sites, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE) - ); } } diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php index 82f6a967c07..90c942279ff 100644 --- a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php @@ -37,12 +37,13 @@ public function run(ProcessingContext $context): void $context->dispatch(Severity::NOTICE, 'Ensures empty live workspace'); if ($this->workspaceHasEvents(WorkspaceName::forLive())) { - throw new \RuntimeException( 'Live workspace already contains events please run "cr:prune" before importing.'); + throw new \RuntimeException('Live workspace already contains events please run "cr:prune" before importing.'); } } private function workspaceHasEvents(WorkspaceName $workspaceName): bool { + /** @phpstan-ignore-next-line internal method of the cr is called */ $workspaceStreamName = WorkspaceEventStreamName::fromWorkspaceName($workspaceName)->getEventStreamName(); $eventStream = $this->eventStore->load($workspaceStreamName); foreach ($eventStream as $event) { diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php index 792b0495cac..d74eaac3215 100644 --- a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php +++ b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php @@ -17,7 +17,11 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; +use Neos\ContentRepository\Export\Processors\EventExportProcessor; +/** + * @implements ContentRepositoryServiceFactoryInterface + */ final readonly class LiveWorkspaceIsEmptyProcessorFactory implements ContentRepositoryServiceFactoryInterface { public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index f783f3ab9e9..5095e2173d8 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -20,7 +20,7 @@ use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; use Neos\ContentRepository\Export\Severity; -use Neos\Flow\Persistence\Doctrine\PersistenceManager; +use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Model\Domain; use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\SiteRepository; @@ -28,13 +28,14 @@ /** * Import processor that creates and persists a Neos {@see Site} instance * - * @phpstan-type SiteShape array{name:string, packageKey:string, nodeName?: string, inactive?:bool} + * @phpstan-type DomainShape array{hostname: string, scheme?: ?string, port?: ?int, active?: ?bool, primary?: ?bool } + * @phpstan-type SiteShape array{name:string, siteResourcesPackageKey:string, nodeName?: string, online?:bool, domains?: ?DomainShape[] } */ final readonly class SiteCreationProcessor implements ProcessorInterface { public function __construct( private SiteRepository $siteRepository, - private PersistenceManager $persistenceManager + private PersistenceManagerInterface $persistenceManager ) { } @@ -63,17 +64,17 @@ public function run(ProcessingContext $context): void // TODO use node aggregate identifier instead of node name $siteInstance = new Site($siteNodeName->value); $siteInstance->setSiteResourcesPackageKey($site['siteResourcesPackageKey']); - $siteInstance->setState(($site['inactive'] ?? false) ? Site::STATE_OFFLINE : Site::STATE_ONLINE); + $siteInstance->setState(($site['online'] ?? false) ? Site::STATE_OFFLINE : Site::STATE_ONLINE); $siteInstance->setName($site['name']); foreach ($site['domains'] ?? [] as $domain) { $domainInstance = new Domain(); $domainInstance->setSite($siteInstance); $domainInstance->setHostname($domain['hostname']); - $domainInstance->setPort($domain['port']); - $domainInstance->setScheme($domain['scheme']); + $domainInstance->setPort($domain['port'] ?? null); + $domainInstance->setScheme($domain['scheme'] ?? null); $domainInstance->setActive($domain['active'] ?? false); - if ($domain['primary']) { + if ($domain['primary'] ?? false) { $siteInstance->setPrimaryDomain($domainInstance); } } @@ -102,7 +103,7 @@ private static function extractSitesFromEventStream(ProcessingContext $context): } if ($event->type === 'NodeAggregateWithNodeWasCreated' && in_array($event->payload['parentNodeAggregateId'], $rootNodeAggregateIds, true)) { $sites[] = [ - 'packageKey' => self::extractPackageKeyFromNodeTypeName($event->payload['nodeTypeName']), + 'siteResourcesPackageKey' => self::extractPackageKeyFromNodeTypeName($event->payload['nodeTypeName']), 'name' => $event->payload['initialPropertyValues']['title']['value'] ?? $event->payload['nodeTypeName'], 'nodeTypeName' => $event->payload['nodeTypeName'], 'nodeName' => $event->payload['nodeName'] ?? null, diff --git a/Neos.Neos/Classes/Domain/Service/SiteExportService.php b/Neos.Neos/Classes/Domain/Service/SiteExportService.php index 97b7a1bf064..7735a131e94 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteExportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteExportService.php @@ -40,14 +40,9 @@ { public function __construct( private ContentRepositoryRegistry $contentRepositoryRegistry, - private DoctrineService $doctrineService, private SiteRepository $siteRepository, private AssetRepository $assetRepository, private AssetUsageService $assetUsageService, - private ResourceRepository $resourceRepository, - private ResourceManager $resourceManager, - private PersistenceManagerInterface $persistenceManager, - private WorkspaceService $workspaceService, ) { } diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 50a1a044238..d3a35ac5e05 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -71,7 +71,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), 'Setup content repository' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositorySetupProcessorFactory()), 'Verify Live workspace does not exist yet' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new LiveWorkspaceIsEmptyProcessorFactory()), - 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository), + 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->persistenceManager), 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), From 4a54aced2c19c0a08f192bab53eeb899257b909b Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 14:14:53 +0200 Subject: [PATCH 13/56] Add replayall to site:import process --- .../Classes/Service/ProjectionReplayService.php | 9 ++++++++- Neos.Neos/Classes/Domain/Service/SiteImportService.php | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php index 00a946be01a..87753b37e58 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php @@ -9,6 +9,8 @@ use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\Projections; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; +use Neos\ContentRepository\Export\ProcessingContext; +use Neos\ContentRepository\Export\ProcessorInterface; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStream\VirtualStreamName; @@ -18,7 +20,7 @@ * * @internal this is currently only used by the {@see CrCommandController} */ -final class ProjectionReplayService implements ContentRepositoryServiceInterface +final class ProjectionReplayService implements ProcessorInterface, ContentRepositoryServiceInterface { public function __construct( @@ -28,6 +30,11 @@ public function __construct( ) { } + public function run(ProcessingContext $context): void + { + $this->replayAllProjections(CatchUpOptions::create()); + } + public function replayProjection(string $projectionAliasOrClassName, CatchUpOptions $options): void { $projectionClassName = $this->resolveProjectionClassName($projectionAliasOrClassName); diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index d3a35ac5e05..9eeddb4d148 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -25,6 +25,7 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; +use Neos\ContentRepositoryRegistry\Service\ProjectionReplayServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -49,6 +50,7 @@ public function __construct( private ResourceManager $resourceManager, private PersistenceManagerInterface $persistenceManager, private WorkspaceService $workspaceService, + private ProjectionReplayServiceFactory $projectionReplayServiceFactory, ) { } @@ -75,7 +77,9 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), + 'Replay all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayServiceFactory), ]; + foreach ($processors as $processorLabel => $processor) { ($onProcessor)($processorLabel); $processor->run($context); From 9f68937cf3fca6c0df52bc7aadbbe0ef859ac05a Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 16:39:32 +0200 Subject: [PATCH 14/56] Rename command `site:import` and `site:export` to `site:importAll` and `site:exportAll` to be in line with current behavior --- .../Classes/Command/SiteCommandController.php | 23 +++++---- .../Domain/Export/SiteExportProcessor.php | 51 +++++++++++++++---- .../Export/SiteExportProcessorFactory.php | 43 ++++++++++++++++ .../Domain/Service/SiteExportService.php | 16 +++--- 4 files changed, 104 insertions(+), 29 deletions(-) create mode 100644 Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index ba2dc2602cc..7ed76cf0103 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -140,10 +140,10 @@ public function createCommand($name, $packageKey, $nodeType, $nodeName = null, $ /** * Import sites * - * This command allows importing sites from the given path/packahe. The format must - * be identical to that produced by the export command. + * This command allows importing sites from the given path/package. The format must + * be identical to that produced by the exportAll command. * - * !!! At the moment the live workspace has to be empty prior to importing. This will be improved in future. !!! + * !!! The live workspace has to be empty prior to importing. !!! * * If a path is specified, this command expects the corresponding directory to contain the exported files * @@ -154,7 +154,7 @@ public function createCommand($name, $packageKey, $nodeType, $nodeName = null, $ * @param string|null $path relative or absolute path and filename to the export files * @return void */ - public function importCommand(string $packageKey = null, string $path = null, string $contentRepository = 'default', bool $verbose = false): void + public function importAllCommand(string $packageKey = null, string $path = null, string $contentRepository = 'default', bool $verbose = false): void { $exceedingArguments = $this->request->getExceedingArguments(); if (isset($exceedingArguments[0]) && $packageKey === null && $path === null) { @@ -202,20 +202,18 @@ public function importCommand(string $packageKey = null, string $path = null, st /** * Export sites * - * This command allows to export all current sites. + * This command exports all sites of the content repository. + ** + * If a path is specified, this command creates the directory if needed and exports into that. * - * !!! At the moment always all sites are exported. This will be improved in future!!! - * - * If a path is specified, this command expects the corresponding directory to contain the exported files - * - * If a package key is specified, this command expects the export files to be located in the private resources + * If a package key is specified, this command exports to the private resources * directory of the given package (Resources/Private/Content). * * @param string|null $packageKey Package key specifying the package containing the sites content * @param string|null $path relative or absolute path and filename to the export files * @return void */ - public function exportCommand(string $packageKey = null, string $path = null, string $contentRepository = 'default', bool $verbose = false): void + public function exportAllCommand(string $packageKey = null, string $path = null, string $contentRepository = 'default', bool $verbose = false): void { $exceedingArguments = $this->request->getExceedingArguments(); if (isset($exceedingArguments[0]) && $packageKey === null && $path === null) { @@ -248,6 +246,9 @@ public function exportCommand(string $packageKey = null, string $path = null, st $package = $this->packageManager->getPackage($packageKey); $path = Files::concatenatePaths([$package->getPackagePath(), 'Resources/Private/Content']); } + if (file_exists($path) === false) { + Files::createDirectoryRecursively($path); + } $this->siteExportService->exportToPath($contentRepositoryId, $path, $onProcessor, $onMessage); } diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php index 6fb159c1914..6f7ac14f0a4 100644 --- a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php +++ b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php @@ -14,16 +14,15 @@ namespace Neos\Neos\Domain\Export; -use JsonException; -use Neos\ContentRepository\Core\SharedModel\Node\NodeName; -use Neos\ContentRepository\Export\Event\ValueObject\ExportedEvent; +use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; +use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepository\Export\Severity; -use Neos\Flow\Persistence\Doctrine\PersistenceManager; use Neos\Neos\Domain\Model\Domain; use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\SiteRepository; +use Neos\Neos\Domain\Service\NodeTypeNameFactory; /** * Export processor exports Neos {@see Site} instances as json @@ -32,9 +31,11 @@ * @phpstan-type SiteShape array{name:string, siteResourcesPackageKey:string, nodeName?: string, online?:bool, domains?: ?DomainShape[] } * */ -final readonly class SiteExportProcessor implements ProcessorInterface +final readonly class SiteExportProcessor implements ProcessorInterface, ContentRepositoryServiceInterface { public function __construct( + private ContentRepository $contentRepository, + private WorkspaceName $workspaceName, private SiteRepository $siteRepository, ) { } @@ -53,8 +54,9 @@ public function run(ProcessingContext $context): void */ private function getSiteData(): array { - return array_map( - fn(Site $site) => [ + $siteData = []; + foreach ($this->findSites($this->workspaceName) as $site) { + $siteData[] = [ "name" => $site->getName(), "nodeName" => $site->getNodeName()->value, "siteResourcesPackageKey" => $site->getSiteResourcesPackageKey(), @@ -69,8 +71,35 @@ private function getSiteData(): array ], $site->getDomains()->toArray() ) - ], - $this->siteRepository->findAll()->toArray() - ); + ]; + } + + return $siteData; + } + + /** + * @param WorkspaceName $workspaceName + * @return \Traversable + */ + private function findSites(WorkspaceName $workspaceName): \Traversable + { + $contentGraph = $this->contentRepository->getContentGraph($workspaceName); + $sitesNodeAggregate = $contentGraph->findRootNodeAggregateByType(NodeTypeNameFactory::forSites()); + if ($sitesNodeAggregate === null) { + return; + } + + $siteNodeAggregates = $contentGraph->findChildNodeAggregates($sitesNodeAggregate->nodeAggregateId); + foreach ($siteNodeAggregates as $siteNodeAggregate) { + $siteNodeName = $siteNodeAggregate->nodeName?->value; + if ($siteNodeName === null) { + continue; + } + $site = $this->siteRepository->findOneByNodeName($siteNodeName); + if ($site === null) { + continue; + } + yield $site; + } } } diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php new file mode 100644 index 00000000000..07832016e15 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php @@ -0,0 +1,43 @@ + + */ +final readonly class SiteExportProcessorFactory implements ContentRepositoryServiceFactoryInterface +{ + public function __construct( + private WorkspaceName $workspaceName, + private SiteRepository $siteRepository, + ) { + } + + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + return new SiteExportProcessor( + $serviceFactoryDependencies->contentRepository, + $this->workspaceName, + $this->siteRepository, + ); + } +} diff --git a/Neos.Neos/Classes/Domain/Service/SiteExportService.php b/Neos.Neos/Classes/Domain/Service/SiteExportService.php index 7735a131e94..f5b2e21bf21 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteExportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteExportService.php @@ -25,14 +25,10 @@ use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Annotations as Flow; -use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; -use Neos\Flow\Persistence\PersistenceManagerInterface; -use Neos\Flow\ResourceManagement\ResourceManager; -use Neos\Flow\ResourceManagement\ResourceRepository; use Neos\Media\Domain\Repository\AssetRepository; use Neos\Neos\AssetUsage\AssetUsageService; -use Neos\Neos\Domain\Import\SiteCreationProcessor; use Neos\Neos\Domain\Export\SiteExportProcessor; +use Neos\Neos\Domain\Export\SiteExportProcessorFactory; use Neos\Neos\Domain\Repository\SiteRepository; #[Flow\Scope('singleton')] @@ -40,9 +36,9 @@ { public function __construct( private ContentRepositoryRegistry $contentRepositoryRegistry, - private SiteRepository $siteRepository, private AssetRepository $assetRepository, private AssetUsageService $assetUsageService, + private SiteRepository $siteRepository, ) { } @@ -79,7 +75,13 @@ public function exportToPath(ContentRepositoryId $contentRepositoryId, string $p $liveWorkspace, $this->assetUsageService ), - 'Export sites' => new SiteExportProcessor($this->siteRepository), + 'Export sites' => $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new SiteExportProcessorFactory( + $liveWorkspace->workspaceName, + $this->siteRepository, + ) + ), ]; foreach ($processors as $processorLabel => $processor) { ($onProcessor)($processorLabel); From cd550d74bc539343ff33b5381a0e166290367eba Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 17:46:19 +0200 Subject: [PATCH 15/56] Add `site:pruneAll` command that empties the current cr and removes all referenced site records --- .../Classes/Command/CrCommandController.php | 69 ---------- .../Classes/Command/SiteCommandController.php | 118 +++++++++++++++--- .../Domain/Export/SiteExportProcessor.php | 13 +- .../Domain/Import/SiteCreationProcessor.php | 16 +-- .../Domain/Service/SiteExportService.php | 1 - .../Domain/Service/SiteImportService.php | 4 +- 6 files changed, 117 insertions(+), 104 deletions(-) delete mode 100644 Neos.Neos/Classes/Command/CrCommandController.php diff --git a/Neos.Neos/Classes/Command/CrCommandController.php b/Neos.Neos/Classes/Command/CrCommandController.php deleted file mode 100644 index 05bf03e9179..00000000000 --- a/Neos.Neos/Classes/Command/CrCommandController.php +++ /dev/null @@ -1,69 +0,0 @@ -output->askConfirmation(sprintf('> This will prune your content repository "%s". Are you sure to proceed? (y/n) ', $contentRepository), false)) { - $this->outputLine('Abort.'); - return; - } - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - - $contentStreamPruner = $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new ContentStreamPrunerFactory() - ); - - $workspaceMaintenanceService = $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new WorkspaceMaintenanceServiceFactory() - ); - - $projectionService = $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - $this->projectionServiceFactory - ); - - // remove the workspace metadata and roles for this cr - $this->workspaceService->pruneRoleAsssignments($contentRepositoryId); - $this->workspaceService->pruneWorkspaceMetadata($contentRepositoryId); - - // reset the events table - $contentStreamPruner->pruneAll(); - $workspaceMaintenanceService->pruneAll(); - - // reset the projections state - $projectionService->resetAllProjections(); - - $this->outputLine('Done.'); - } -} diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index 7ed76cf0103..7450e3fcd84 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -14,27 +14,31 @@ namespace Neos\Neos\Command; +use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; +use Neos\ContentRepository\Core\Service\WorkspaceMaintenanceServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Exception\NodeNameIsAlreadyCovered; use Neos\ContentRepository\Core\SharedModel\Exception\NodeTypeNotFound; -use Neos\ContentRepository\Export\ProcessorEventInterface; +use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; +use Neos\ContentRepositoryRegistry\Service\ProjectionReplayServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Cli\CommandController; use Neos\Flow\Cli\Exception\StopCommandException; -use Neos\Flow\ObjectManagement\DependencyInjection\DependencyProxy; use Neos\Flow\Package\PackageManager; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Exception\SiteNodeNameIsAlreadyInUseByAnotherSite; use Neos\Neos\Domain\Exception\SiteNodeTypeIsInvalid; use Neos\Neos\Domain\Model\Site; +use Neos\Neos\Domain\Repository\DomainRepository; use Neos\Neos\Domain\Repository\SiteRepository; use Neos\Neos\Domain\Service\NodeTypeNameFactory; use Neos\Neos\Domain\Service\SiteExportService; use Neos\Neos\Domain\Service\SiteImportService; -use Neos\Neos\Domain\Service\SiteImportServiceFactory; use Neos\Neos\Domain\Service\SiteService; +use Neos\Neos\Domain\Service\WorkspaceService; use Neos\Utility\Files; /** @@ -50,6 +54,12 @@ class SiteCommandController extends CommandController */ protected $siteRepository; + /** + * @Flow\Inject + * @var DomainRepository + */ + protected $domainRepository; + /** * @Flow\Inject * @var SiteService @@ -86,6 +96,18 @@ class SiteCommandController extends CommandController */ protected $siteExportService; + /** + * @Flow\Inject + * @var WorkspaceService + */ + protected $workspaceService; + + /** + * @Flow\Inject(lazy=false) + * @var ProjectionReplayServiceFactory + */ + protected $projectionServiceFactory; + /** * Create a new site * @@ -253,31 +275,56 @@ public function exportAllCommand(string $packageKey = null, string $path = null, } /** - * Remove site with content and related data (with globbing) + * This will completely prune the data of the specified content repository and remove all site-records. * - * In the future we need some more sophisticated cleanup. - * - * @param string $siteNode Name for site root nodes to clear only content of this sites (globbing is supported) + * @param string $contentRepository Name of the content repository where the data should be pruned from. + * @param bool $force Prune the cr without confirmation. This cannot be reverted! * @return void */ - public function pruneCommand($siteNode) + public function pruneAllCommand(string $contentRepository = 'default', bool $force = false): void { - $sites = $this->findSitesByNodeNamePattern($siteNode); - if (empty($sites)) { - $this->outputLine('No Site found for pattern "%s".', [$siteNode]); - // Help the user a little about what he needs to provide as a parameter here - $this->outputLine('To find out which sites you have, use the site:list command.'); - $this->outputLine('The site:prune command expects the "Node name" from the site list as a parameter.'); - $this->outputLine('If you want to delete all sites, you can run site:prune \'*\'.'); - $this->quit(1); + if (!$force && !$this->output->askConfirmation(sprintf('> This will prune your content repository "%s". Are you sure to proceed? (y/n) ', $contentRepository), false)) { + $this->outputLine('Abort.'); + return; } + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + + // find and remove all sites + $sites = $this->findAllSites( + $this->contentRepositoryRegistry->get($contentRepositoryId), + WorkspaceName::forLive() + ); foreach ($sites as $site) { $this->siteService->pruneSite($site); - $this->outputLine( - 'Site with root "%s" matched pattern "%s" and has been removed.', - [$site->getNodeName(), $siteNode] - ); } + + // remove cr data + $contentStreamPruner = $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new ContentStreamPrunerFactory() + ); + $workspaceMaintenanceService = $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new WorkspaceMaintenanceServiceFactory() + ); + + $projectionService = $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + $this->projectionServiceFactory + ); + + // remove the workspace metadata and roles for this cr + $this->workspaceService->pruneRoleAsssignments($contentRepositoryId); + $this->workspaceService->pruneWorkspaceMetadata($contentRepositoryId); + + // reset the events table + $contentStreamPruner->pruneAll(); + $workspaceMaintenanceService->pruneAll(); + + // reset the projections state + $projectionService->resetAllProjections(); + + $this->outputLine('Done.'); } /** @@ -362,4 +409,35 @@ function (Site $site) use ($siteNodePattern) { } ); } + + /** + * Find all sites in a cr by finding the children of the sites node + * + * @param ContentRepository $contentRepository + * @param WorkspaceName $workspaceName + * @return Site[] + */ + protected function findAllSites(ContentRepository $contentRepository, WorkspaceName $workspaceName): array + { + $contentGraph = $contentRepository->getContentGraph($workspaceName); + $sitesNodeAggregate = $contentGraph->findRootNodeAggregateByType(NodeTypeNameFactory::forSites()); + if ($sitesNodeAggregate === null) { + return []; + } + + $siteNodeAggregates = $contentGraph->findChildNodeAggregates($sitesNodeAggregate->nodeAggregateId); + $sites = []; + foreach ($siteNodeAggregates as $siteNodeAggregate) { + $siteNodeName = $siteNodeAggregate->nodeName?->value; + if ($siteNodeName === null) { + continue; + } + $site = $this->siteRepository->findOneByNodeName($siteNodeName); + if ($site === null) { + continue; + } + $sites[] = $site; + } + return $sites; + } } diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php index 6f7ac14f0a4..643845d8b49 100644 --- a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php +++ b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php @@ -54,8 +54,9 @@ public function run(ProcessingContext $context): void */ private function getSiteData(): array { + $sites = $this->findSites($this->workspaceName); $siteData = []; - foreach ($this->findSites($this->workspaceName) as $site) { + foreach ($sites as $site) { $siteData[] = [ "name" => $site->getName(), "nodeName" => $site->getNodeName()->value, @@ -79,17 +80,18 @@ private function getSiteData(): array /** * @param WorkspaceName $workspaceName - * @return \Traversable + * @return Site[] */ - private function findSites(WorkspaceName $workspaceName): \Traversable + private function findSites(WorkspaceName $workspaceName): array { $contentGraph = $this->contentRepository->getContentGraph($workspaceName); $sitesNodeAggregate = $contentGraph->findRootNodeAggregateByType(NodeTypeNameFactory::forSites()); if ($sitesNodeAggregate === null) { - return; + return []; } $siteNodeAggregates = $contentGraph->findChildNodeAggregates($sitesNodeAggregate->nodeAggregateId); + $sites = []; foreach ($siteNodeAggregates as $siteNodeAggregate) { $siteNodeName = $siteNodeAggregate->nodeName?->value; if ($siteNodeName === null) { @@ -99,7 +101,8 @@ private function findSites(WorkspaceName $workspaceName): \Traversable if ($site === null) { continue; } - yield $site; + $sites[] = $site; } + return $sites; } } diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index 5095e2173d8..947011aea80 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -23,6 +23,7 @@ use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Model\Domain; use Neos\Neos\Domain\Model\Site; +use Neos\Neos\Domain\Repository\DomainRepository; use Neos\Neos\Domain\Repository\SiteRepository; /** @@ -35,6 +36,7 @@ { public function __construct( private SiteRepository $siteRepository, + private DomainRepository $domainRepository, private PersistenceManagerInterface $persistenceManager ) { } @@ -51,7 +53,7 @@ public function run(ProcessingContext $context): void } else { $sites = self::extractSitesFromEventStream($context); } - $persistAllIsRequired = false; + /** @var SiteShape $site */ foreach ($sites as $site) { $context->dispatch(Severity::NOTICE, "Creating site \"{$site['name']}\""); @@ -66,7 +68,8 @@ public function run(ProcessingContext $context): void $siteInstance->setSiteResourcesPackageKey($site['siteResourcesPackageKey']); $siteInstance->setState(($site['online'] ?? false) ? Site::STATE_OFFLINE : Site::STATE_ONLINE); $siteInstance->setName($site['name']); - + $this->siteRepository->add($siteInstance); + $this->persistenceManager->persistAll(); foreach ($site['domains'] ?? [] as $domain) { $domainInstance = new Domain(); $domainInstance->setSite($siteInstance); @@ -74,16 +77,13 @@ public function run(ProcessingContext $context): void $domainInstance->setPort($domain['port'] ?? null); $domainInstance->setScheme($domain['scheme'] ?? null); $domainInstance->setActive($domain['active'] ?? false); + $this->domainRepository->add($domainInstance); if ($domain['primary'] ?? false) { $siteInstance->setPrimaryDomain($domainInstance); + $this->siteRepository->update($siteInstance); } + $this->persistenceManager->persistAll(); } - - $this->siteRepository->add($siteInstance); - $persistAllIsRequired = true; - } - if ($persistAllIsRequired) { - $this->persistenceManager->persistAll(); } } diff --git a/Neos.Neos/Classes/Domain/Service/SiteExportService.php b/Neos.Neos/Classes/Domain/Service/SiteExportService.php index f5b2e21bf21..87c581edea9 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteExportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteExportService.php @@ -27,7 +27,6 @@ use Neos\Flow\Annotations as Flow; use Neos\Media\Domain\Repository\AssetRepository; use Neos\Neos\AssetUsage\AssetUsageService; -use Neos\Neos\Domain\Export\SiteExportProcessor; use Neos\Neos\Domain\Export\SiteExportProcessorFactory; use Neos\Neos\Domain\Repository\SiteRepository; diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 9eeddb4d148..addb86244b1 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -36,6 +36,7 @@ use Neos\Neos\Domain\Import\LiveWorkspaceCreationProcessor; use Neos\Neos\Domain\Import\LiveWorkspaceIsEmptyProcessorFactory; use Neos\Neos\Domain\Import\SiteCreationProcessor; +use Neos\Neos\Domain\Repository\DomainRepository; use Neos\Neos\Domain\Repository\SiteRepository; #[Flow\Scope('singleton')] @@ -45,6 +46,7 @@ public function __construct( private ContentRepositoryRegistry $contentRepositoryRegistry, private DoctrineService $doctrineService, private SiteRepository $siteRepository, + private DomainRepository $domainRepository, private AssetRepository $assetRepository, private ResourceRepository $resourceRepository, private ResourceManager $resourceManager, @@ -73,7 +75,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), 'Setup content repository' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositorySetupProcessorFactory()), 'Verify Live workspace does not exist yet' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new LiveWorkspaceIsEmptyProcessorFactory()), - 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->persistenceManager), + 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), From e260954dfebe85d675bba448e9178c2fc7563e04 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 18:25:09 +0200 Subject: [PATCH 16/56] Improve after review - centralize pathDetermination and closure generationIn site command controller - use catch-up instead of replayall after import - remove workspace name from exported events --- .../src/Event/ValueObject/ExportedEvent.php | 2 +- .../Service/ProjectionCatchupService.php | 95 ++++++++++++++ .../ProjectionCatchupServiceFactory.php | 30 +++++ .../Classes/Command/SiteCommandController.php | 121 +++++++++--------- .../Import/LiveWorkspaceIsEmptyProcessor.php | 2 +- .../Domain/Service/SiteImportService.php | 6 +- 6 files changed, 190 insertions(+), 66 deletions(-) create mode 100644 Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupService.php create mode 100644 Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupServiceFactory.php diff --git a/Neos.ContentRepository.Export/src/Event/ValueObject/ExportedEvent.php b/Neos.ContentRepository.Export/src/Event/ValueObject/ExportedEvent.php index 5392aa14ac6..a9ca8474d3a 100644 --- a/Neos.ContentRepository.Export/src/Event/ValueObject/ExportedEvent.php +++ b/Neos.ContentRepository.Export/src/Event/ValueObject/ExportedEvent.php @@ -26,7 +26,7 @@ public static function fromRawEvent(Event $event): self { $payload = \json_decode($event->data->value, true, 512, JSON_THROW_ON_ERROR); // unset content stream id as this is overwritten during import - unset($payload['contentStreamId']); + unset($payload['contentStreamId'], $payload['workspaceName']); return new self( $event->id->value, $event->type->value, diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupService.php new file mode 100644 index 00000000000..7112ad85486 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupService.php @@ -0,0 +1,95 @@ +catchupAllProjections(CatchUpOptions::create()); + } + + public function catchupProjection(string $projectionAliasOrClassName, CatchUpOptions $options): void + { + $projectionClassName = $this->resolveProjectionClassName($projectionAliasOrClassName); + $this->contentRepository->catchUpProjection($projectionClassName, $options); + } + + public function catchupAllProjections(CatchUpOptions $options, ?\Closure $progressCallback = null): void + { + foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { + if ($progressCallback) { + $progressCallback($classNamesAndAlias['alias']); + } + $this->contentRepository->catchUpProjection($classNamesAndAlias['className'], $options); + } + } + + /** + * @return class-string> + */ + private function resolveProjectionClassName(string $projectionAliasOrClassName): string + { + $lowerCaseProjectionName = strtolower($projectionAliasOrClassName); + $projectionClassNamesAndAliases = $this->projectionClassNamesAndAliases(); + foreach ($projectionClassNamesAndAliases as $classNamesAndAlias) { + if (strtolower($classNamesAndAlias['className']) === $lowerCaseProjectionName || strtolower($classNamesAndAlias['alias']) === $lowerCaseProjectionName) { + return $classNamesAndAlias['className']; + } + } + throw new \InvalidArgumentException(sprintf( + 'The projection "%s" is not registered for this Content Repository. The following projection aliases (or fully qualified class names) can be used: %s', + $projectionAliasOrClassName, + implode('', array_map(static fn (array $classNamesAndAlias) => sprintf(chr(10) . ' * %s (%s)', $classNamesAndAlias['alias'], $classNamesAndAlias['className']), $projectionClassNamesAndAliases)) + ), 1680519624); + } + + /** + * @return array>, alias: string}> + */ + private function projectionClassNamesAndAliases(): array + { + return array_map( + static fn (string $projectionClassName) => [ + 'className' => $projectionClassName, + 'alias' => self::projectionAlias($projectionClassName), + ], + $this->projections->getClassNames() + ); + } + + private static function projectionAlias(string $className): string + { + $alias = lcfirst(substr(strrchr($className, '\\') ?: '\\' . $className, 1)); + if (str_ends_with($alias, 'Projection')) { + $alias = substr($alias, 0, -10); + } + return $alias; + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupServiceFactory.php new file mode 100644 index 00000000000..126de8eec64 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupServiceFactory.php @@ -0,0 +1,30 @@ + + * @internal this is currently only used by the {@see CrCommandController} + */ +#[Flow\Scope("singleton")] +final class ProjectionCatchupServiceFactory implements ContentRepositoryServiceFactoryInterface +{ + + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + return new ProjectionCatchupService( + $serviceFactoryDependencies->projections, + $serviceFactoryDependencies->contentRepository, + $serviceFactoryDependencies->eventStore, + ); + } +} diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index 7450e3fcd84..6a98c12a8b4 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -178,19 +178,6 @@ public function createCommand($name, $packageKey, $nodeType, $nodeName = null, $ */ public function importAllCommand(string $packageKey = null, string $path = null, string $contentRepository = 'default', bool $verbose = false): void { - $exceedingArguments = $this->request->getExceedingArguments(); - if (isset($exceedingArguments[0]) && $packageKey === null && $path === null) { - if (file_exists($exceedingArguments[0])) { - $path = $exceedingArguments[0]; - } elseif ($this->packageManager->isPackageAvailable($exceedingArguments[0])) { - $packageKey = $exceedingArguments[0]; - } - } - if ($packageKey === null && $path === null) { - $this->outputLine('You have to specify either --package-key or --filename'); - $this->quit(1); - } - // Since this command uses a lot of memory when large sites are imported, we warn the user to watch for // the confirmation of a successful import. $this->outputLine('This command can use a lot of memory when importing sites with many resources.'); @@ -200,25 +187,16 @@ public function importAllCommand(string $packageKey = null, string $path = null, $this->outputLine('Starting import...'); $this->outputLine('---'); + $path = $this->determineTargetPath($packageKey, $path); + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $onProcessor = function (string $processorLabel) { - $this->outputLine('%s...', [$processorLabel]); - }; - $onMessage = function (Severity $severity, string $message) use ($verbose) { - if (!$verbose && $severity === Severity::NOTICE) { - return; - } - $this->outputLine(match ($severity) { - Severity::NOTICE => $message, - Severity::WARNING => sprintf('Warning: %s', $message), - Severity::ERROR => sprintf('Error: %s', $message), - }); - }; - if ($path === null) { - $package = $this->packageManager->getPackage($packageKey); - $path = Files::concatenatePaths([$package->getPackagePath(), 'Resources/Private/Content']); - } - $this->siteImportService->importFromPath($contentRepositoryId, $path, $onProcessor, $onMessage); + + $this->siteImportService->importFromPath( + $contentRepositoryId, + $path, + $this->createOnProcessorClosure(), + $this->createOnMessageClosure($verbose) + ); } /** @@ -237,41 +215,17 @@ public function importAllCommand(string $packageKey = null, string $path = null, */ public function exportAllCommand(string $packageKey = null, string $path = null, string $contentRepository = 'default', bool $verbose = false): void { - $exceedingArguments = $this->request->getExceedingArguments(); - if (isset($exceedingArguments[0]) && $packageKey === null && $path === null) { - if (file_exists($exceedingArguments[0])) { - $path = $exceedingArguments[0]; - } elseif ($this->packageManager->isPackageAvailable($exceedingArguments[0])) { - $packageKey = $exceedingArguments[0]; - } - } - if ($packageKey === null && $path === null) { - $this->outputLine('You have to specify either --package-key or --filename'); - $this->quit(1); - } - + $path = $this->determineTargetPath($packageKey, $path); $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $onProcessor = function (string $processorLabel) { - $this->outputLine('%s...', [$processorLabel]); - }; - $onMessage = function (Severity $severity, string $message) use ($verbose) { - if (!$verbose && $severity === Severity::NOTICE) { - return; - } - $this->outputLine(match ($severity) { - Severity::NOTICE => $message, - Severity::WARNING => sprintf('Warning: %s', $message), - Severity::ERROR => sprintf('Error: %s', $message), - }); - }; - if ($path === null) { - $package = $this->packageManager->getPackage($packageKey); - $path = Files::concatenatePaths([$package->getPackagePath(), 'Resources/Private/Content']); - } if (file_exists($path) === false) { Files::createDirectoryRecursively($path); } - $this->siteExportService->exportToPath($contentRepositoryId, $path, $onProcessor, $onMessage); + $this->siteExportService->exportToPath( + $contentRepositoryId, + $path, + $this->createOnProcessorClosure(), + $this->createOnMessageClosure($verbose) + ); } /** @@ -440,4 +394,47 @@ protected function findAllSites(ContentRepository $contentRepository, WorkspaceN } return $sites; } + + protected function determineTargetPath(?string $packageKey, ?string $path): string + { + $exceedingArguments = $this->request->getExceedingArguments(); + if (isset($exceedingArguments[0]) && $packageKey === null && $path === null) { + if (file_exists($exceedingArguments[0])) { + $path = $exceedingArguments[0]; + } elseif ($this->packageManager->isPackageAvailable($exceedingArguments[0])) { + $packageKey = $exceedingArguments[0]; + } + } + if ($packageKey === null && $path === null) { + $this->outputLine('You have to specify either --package-key or --filename'); + $this->quit(1); + } + if ($path === null) { + $package = $this->packageManager->getPackage($packageKey); + $path = Files::concatenatePaths([$package->getPackagePath(), 'Resources/Private/Content']); + } + return $path; + } + + protected function createOnProcessorClosure(): \Closure + { + $onProcessor = function (string $processorLabel) { + $this->outputLine('%s...', [$processorLabel]); + }; + return $onProcessor; + } + + protected function createOnMessageClosure(bool $verbose): \Closure + { + return function (Severity $severity, string $message) use ($verbose) { + if (!$verbose && $severity === Severity::NOTICE) { + return; + } + $this->outputLine(match ($severity) { + Severity::NOTICE => $message, + Severity::WARNING => sprintf('Warning: %s', $message), + Severity::ERROR => sprintf('Error: %s', $message), + }); + }; + } } diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php index 90c942279ff..3aee0df7594 100644 --- a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php @@ -37,7 +37,7 @@ public function run(ProcessingContext $context): void $context->dispatch(Severity::NOTICE, 'Ensures empty live workspace'); if ($this->workspaceHasEvents(WorkspaceName::forLive())) { - throw new \RuntimeException('Live workspace already contains events please run "cr:prune" before importing.'); + throw new \RuntimeException('Live workspace already contains events please run "site:pruneAll" before importing.'); } } diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index addb86244b1..57d9e8b7c3d 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -25,6 +25,8 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; +use Neos\ContentRepositoryRegistry\Service\ProjectionCatchupService; +use Neos\ContentRepositoryRegistry\Service\ProjectionCatchupServiceFactory; use Neos\ContentRepositoryRegistry\Service\ProjectionReplayServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; @@ -52,7 +54,7 @@ public function __construct( private ResourceManager $resourceManager, private PersistenceManagerInterface $persistenceManager, private WorkspaceService $workspaceService, - private ProjectionReplayServiceFactory $projectionReplayServiceFactory, + private ProjectionCatchupServiceFactory $projectionCatchupServiceFactory, ) { } @@ -79,7 +81,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Replay all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayServiceFactory), + 'Catchup all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionCatchupServiceFactory), ]; foreach ($processors as $processorLabel => $processor) { From 361c71e4c71d9400dfadaeeedbe580991b5f88cc Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 23 Oct 2024 20:02:24 +0200 Subject: [PATCH 17/56] Extract SitePruningService --- .../Classes/Command/SiteCommandController.php | 87 ++-------------- .../ContentRepositoryPruningProcessor.php | 58 +++++++++++ ...ntentRepositoryPruningProcessorFactory.php | 45 +++++++++ .../RoleAndMetadataPruningProcessor.php | 54 ++++++++++ ...RoleAndMetadataPruningProcessorFactory.php | 47 +++++++++ .../Domain/Pruning/SitePruningProcessor.php | 91 +++++++++++++++++ .../Pruning/SitePruningProcessorFactory.php | 52 ++++++++++ .../Domain/Service/SitePruningService.php | 99 +++++++++++++++++++ 8 files changed, 456 insertions(+), 77 deletions(-) create mode 100644 Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php create mode 100644 Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php create mode 100644 Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php create mode 100644 Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php create mode 100644 Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php create mode 100644 Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php create mode 100644 Neos.Neos/Classes/Domain/Service/SitePruningService.php diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index 6a98c12a8b4..7a1de11bc18 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -14,16 +14,11 @@ namespace Neos\Neos\Command; -use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; -use Neos\ContentRepository\Core\Service\WorkspaceMaintenanceServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Exception\NodeNameIsAlreadyCovered; use Neos\ContentRepository\Core\SharedModel\Exception\NodeTypeNotFound; -use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Service\ProjectionReplayServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Cli\CommandController; use Neos\Flow\Cli\Exception\StopCommandException; @@ -37,6 +32,7 @@ use Neos\Neos\Domain\Service\NodeTypeNameFactory; use Neos\Neos\Domain\Service\SiteExportService; use Neos\Neos\Domain\Service\SiteImportService; +use Neos\Neos\Domain\Service\SitePruningService; use Neos\Neos\Domain\Service\SiteService; use Neos\Neos\Domain\Service\WorkspaceService; use Neos\Utility\Files; @@ -98,15 +94,15 @@ class SiteCommandController extends CommandController /** * @Flow\Inject - * @var WorkspaceService + * @var SitePruningService */ - protected $workspaceService; + protected $sitePruningService; /** - * @Flow\Inject(lazy=false) - * @var ProjectionReplayServiceFactory + * @Flow\Inject + * @var WorkspaceService */ - protected $projectionServiceFactory; + protected $workspaceService; /** * Create a new site @@ -231,11 +227,10 @@ public function exportAllCommand(string $packageKey = null, string $path = null, /** * This will completely prune the data of the specified content repository and remove all site-records. * - * @param string $contentRepository Name of the content repository where the data should be pruned from. * @param bool $force Prune the cr without confirmation. This cannot be reverted! * @return void */ - public function pruneAllCommand(string $contentRepository = 'default', bool $force = false): void + public function pruneAllCommand(string $contentRepository = 'default', bool $force = false, bool $verbose = false): void { if (!$force && !$this->output->askConfirmation(sprintf('> This will prune your content repository "%s". Are you sure to proceed? (y/n) ', $contentRepository), false)) { $this->outputLine('Abort.'); @@ -243,42 +238,11 @@ public function pruneAllCommand(string $contentRepository = 'default', bool $for } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - // find and remove all sites - $sites = $this->findAllSites( - $this->contentRepositoryRegistry->get($contentRepositoryId), - WorkspaceName::forLive() - ); - foreach ($sites as $site) { - $this->siteService->pruneSite($site); - } - - // remove cr data - $contentStreamPruner = $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new ContentStreamPrunerFactory() - ); - $workspaceMaintenanceService = $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new WorkspaceMaintenanceServiceFactory() - ); - - $projectionService = $this->contentRepositoryRegistry->buildService( + $this->sitePruningService->pruneAll( $contentRepositoryId, - $this->projectionServiceFactory + $this->createOnProcessorClosure(), + $this->createOnMessageClosure($verbose) ); - - // remove the workspace metadata and roles for this cr - $this->workspaceService->pruneRoleAsssignments($contentRepositoryId); - $this->workspaceService->pruneWorkspaceMetadata($contentRepositoryId); - - // reset the events table - $contentStreamPruner->pruneAll(); - $workspaceMaintenanceService->pruneAll(); - - // reset the projections state - $projectionService->resetAllProjections(); - - $this->outputLine('Done.'); } /** @@ -364,37 +328,6 @@ function (Site $site) use ($siteNodePattern) { ); } - /** - * Find all sites in a cr by finding the children of the sites node - * - * @param ContentRepository $contentRepository - * @param WorkspaceName $workspaceName - * @return Site[] - */ - protected function findAllSites(ContentRepository $contentRepository, WorkspaceName $workspaceName): array - { - $contentGraph = $contentRepository->getContentGraph($workspaceName); - $sitesNodeAggregate = $contentGraph->findRootNodeAggregateByType(NodeTypeNameFactory::forSites()); - if ($sitesNodeAggregate === null) { - return []; - } - - $siteNodeAggregates = $contentGraph->findChildNodeAggregates($sitesNodeAggregate->nodeAggregateId); - $sites = []; - foreach ($siteNodeAggregates as $siteNodeAggregate) { - $siteNodeName = $siteNodeAggregate->nodeName?->value; - if ($siteNodeName === null) { - continue; - } - $site = $this->siteRepository->findOneByNodeName($siteNodeName); - if ($site === null) { - continue; - } - $sites[] = $site; - } - return $sites; - } - protected function determineTargetPath(?string $packageKey, ?string $path): string { $exceedingArguments = $this->request->getExceedingArguments(); diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php new file mode 100644 index 00000000000..be73e71a1d1 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php @@ -0,0 +1,58 @@ +contentRepository->findContentStreams() as $contentStream) { + $streamName = ContentStreamEventStreamName::fromContentStreamId($contentStream->id)->getEventStreamName(); + $this->eventStore->deleteStream($streamName); + } + foreach ($this->contentRepository->findWorkspaces() as $workspace) { + $streamName = WorkspaceEventStreamName::fromWorkspaceName($workspace->workspaceName)->getEventStreamName(); + $this->eventStore->deleteStream($streamName); + } + } +} diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php new file mode 100644 index 00000000000..84b645624f4 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php @@ -0,0 +1,45 @@ + + */ +final readonly class ContentRepositoryPruningProcessorFactory implements ContentRepositoryServiceFactoryInterface +{ + + public function __construct( + ) { + } + + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + return new ContentRepositoryPruningProcessor( + $serviceFactoryDependencies->contentRepository, + $serviceFactoryDependencies->eventStore, + ); + } +} diff --git a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php new file mode 100644 index 00000000000..c4aa7592586 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php @@ -0,0 +1,54 @@ +workspaceService->pruneRoleAsssignments($this->contentRepositoryId); + $this->workspaceService->pruneWorkspaceMetadata($this->contentRepositoryId); + } +} diff --git a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php new file mode 100644 index 00000000000..116821465f6 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php @@ -0,0 +1,47 @@ + + */ +final readonly class RoleAndMetadataPruningProcessorFactory implements ContentRepositoryServiceFactoryInterface +{ + + public function __construct( + private WorkspaceService $workspaceService, + ) { + } + + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + return new RoleAndMetadataPruningProcessor( + $serviceFactoryDependencies->contentRepositoryId, + $this->workspaceService, + ); + } +} diff --git a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php new file mode 100644 index 00000000000..7c272774d91 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php @@ -0,0 +1,91 @@ +findAllSites(); + foreach ($sites as $site) { + $domains = $site->getDomains(); + if ($site->getPrimaryDomain() !== null) { + $site->setPrimaryDomain(null); + $this->siteRepository->update($site); + } + foreach ($domains as $domain) { + $this->domainRepository->remove($domain); + } + $this->persistenceManager->persistAll(); + $this->siteRepository->remove($site); + $this->persistenceManager->persistAll(); + } + } + + /** + * @return Site[] + */ + protected function findAllSites(): array + { + $contentGraph = $this->contentRepository->getContentGraph($this->workspaceName); + $sitesNodeAggregate = $contentGraph->findRootNodeAggregateByType(NodeTypeNameFactory::forSites()); + if ($sitesNodeAggregate === null) { + return []; + } + + $siteNodeAggregates = $contentGraph->findChildNodeAggregates($sitesNodeAggregate->nodeAggregateId); + $sites = []; + foreach ($siteNodeAggregates as $siteNodeAggregate) { + $siteNodeName = $siteNodeAggregate->nodeName?->value; + if ($siteNodeName === null) { + continue; + } + $site = $this->siteRepository->findOneByNodeName($siteNodeName); + if ($site === null) { + continue; + } + $sites[] = $site; + } + return $sites; + } +} diff --git a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php new file mode 100644 index 00000000000..76f0e0b55d0 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php @@ -0,0 +1,52 @@ + + */ +final readonly class SitePruningProcessorFactory implements ContentRepositoryServiceFactoryInterface +{ + + public function __construct( + private WorkspaceName $workspaceName, + private SiteRepository $siteRepository, + private DomainRepository $domainRepository, + private PersistenceManagerInterface $persistenceManager, + ) { + } + + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + return new SitePruningProcessor( + $serviceFactoryDependencies->contentRepository, + $this->workspaceName, + $this->siteRepository, + $this->domainRepository, + $this->persistenceManager + ); + } +} diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php new file mode 100644 index 00000000000..fc5069768d1 --- /dev/null +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -0,0 +1,99 @@ + $processors */ + $processors = [ + 'Remove site nodes' => $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new SitePruningProcessorFactory( + WorkspaceName::forLive(), + $this->siteRepository, + $this->domainRepository, + $this->persistenceManager + ) + ), + 'Prune content repository' => $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new ContentRepositoryPruningProcessorFactory() + ), + 'Prune roles and metadata' => $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new RoleAndMetadataPruningProcessorFactory( + $this->workspaceService + ) + ), + 'Replay all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayServiceFactory), + ]; + + foreach ($processors as $processorLabel => $processor) { + ($onProcessor)($processorLabel); + $processor->run($context); + } + } +} From a05ad66f6e4c4ae6a04e6353a55f4e9ed5461fa8 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Thu, 24 Oct 2024 11:34:57 +0200 Subject: [PATCH 18/56] Remove ProcessorInterface from ProjectionService and introduce separate ProjectionCatchUpProcessor and ProjectionReplayProcessor. --- .../Classes/Command/CrCommandController.php | 22 ++--- .../composer.json | 1 + .../Classes/Command/CrCommandController.php | 4 +- .../Processors/ProjectionCatchupProcessor.php | 29 ++++++ .../ProjectionCatchupProcessorFactory.php | 36 +++++++ .../Processors/ProjectionReplayProcessor.php | 30 ++++++ .../ProjectionReplayProcessorFactory.php | 34 +++++++ .../Service/ProjectionCatchupService.php | 95 ------------------- .../ProjectionCatchupServiceFactory.php | 30 ------ ...eplayService.php => ProjectionService.php} | 28 ++++-- ...ctory.php => ProjectionServiceFactory.php} | 9 +- .../Classes/Command/SiteCommandController.php | 2 +- .../ContentRepositoryPruningProcessor.php | 13 +-- ...ntentRepositoryPruningProcessorFactory.php | 12 +-- .../RoleAndMetadataPruningProcessor.php | 15 --- ...RoleAndMetadataPruningProcessorFactory.php | 8 -- .../Pruning/SitePruningProcessorFactory.php | 3 +- .../Domain/Service/SiteImportService.php | 26 ++--- .../Domain/Service/SitePruningService.php | 21 ++-- phpstan-baseline.neon | 2 +- 20 files changed, 196 insertions(+), 224 deletions(-) create mode 100644 Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php create mode 100644 Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php create mode 100644 Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php create mode 100644 Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php delete mode 100644 Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupService.php delete mode 100644 Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupServiceFactory.php rename Neos.ContentRepositoryRegistry/Classes/Service/{ProjectionReplayService.php => ProjectionService.php} (83%) rename Neos.ContentRepositoryRegistry/Classes/Service/{ProjectionReplayServiceFactory.php => ProjectionServiceFactory.php} (81%) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php index 342bad83647..90b0fa78049 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php @@ -24,7 +24,7 @@ use Neos\ContentRepository\LegacyNodeMigration\LegacyMigrationServiceFactory; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Factory\EventStore\DoctrineEventStoreFactory; -use Neos\ContentRepositoryRegistry\Service\ProjectionReplayServiceFactory; +use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Cli\CommandController; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Flow\Property\PropertyMapper; @@ -38,16 +38,16 @@ class CrCommandController extends CommandController { public function __construct( - private readonly Connection $connection, - private readonly Environment $environment, + private readonly Connection $connection, + private readonly Environment $environment, private readonly PersistenceManagerInterface $persistenceManager, - private readonly AssetRepository $assetRepository, - private readonly ResourceRepository $resourceRepository, - private readonly ResourceManager $resourceManager, - private readonly PropertyMapper $propertyMapper, - private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - private readonly SiteRepository $siteRepository, - private readonly ProjectionReplayServiceFactory $projectionReplayServiceFactory, + private readonly AssetRepository $assetRepository, + private readonly ResourceRepository $resourceRepository, + private readonly ResourceManager $resourceManager, + private readonly PropertyMapper $propertyMapper, + private readonly ContentRepositoryRegistry $contentRepositoryRegistry, + private readonly SiteRepository $siteRepository, + private readonly ProjectionServiceFactory $projectionServiceFactory, ) { parent::__construct(); } @@ -120,7 +120,7 @@ public function migrateLegacyDataCommand(bool $verbose = false, string $config = } $this->connection->executeStatement('TRUNCATE ' . $connection->quoteIdentifier($eventTableName)); // we also need to reset the projections; in order to ensure the system runs deterministically - $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayServiceFactory); + $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionServiceFactory); $projectionService->resetAllProjections(); $this->outputLine('Truncated events'); diff --git a/Neos.ContentRepository.LegacyNodeMigration/composer.json b/Neos.ContentRepository.LegacyNodeMigration/composer.json index acb3ddbb2d1..f88ba50c268 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/composer.json +++ b/Neos.ContentRepository.LegacyNodeMigration/composer.json @@ -12,6 +12,7 @@ ], "require": { "php": ">=8.2", + "neoe/neos": "self.version", "neos/contentrepository-core": "self.version", "neos/contentrepository-export": "self.version", "league/flysystem": "^3" diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 459398dfeb0..38149fcbe74 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -9,7 +9,7 @@ use Neos\ContentRepository\Core\Service\WorkspaceMaintenanceServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Service\ProjectionReplayServiceFactory; +use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStore\StatusType; use Neos\Flow\Cli\CommandController; @@ -23,7 +23,7 @@ final class CrCommandController extends CommandController public function __construct( private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - private readonly ProjectionReplayServiceFactory $projectionServiceFactory, + private readonly ProjectionServiceFactory $projectionServiceFactory, ) { parent::__construct(); } diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php new file mode 100644 index 00000000000..3f837b91571 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php @@ -0,0 +1,29 @@ +projectionservice->catchupAllProjections(CatchUpOptions::create()); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php new file mode 100644 index 00000000000..14436d9dc52 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php @@ -0,0 +1,36 @@ + + * @internal this is currently only used by the {@see SiteImportService} {@see SitePruningService} + */ +#[Flow\Scope("singleton")] +final class ProjectionCatchupProcessorFactory implements ContentRepositoryServiceFactoryInterface +{ + + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + return new ProjectionCatchupProcessor( + new ProjectionService( + $serviceFactoryDependencies->projections, + $serviceFactoryDependencies->contentRepository, + $serviceFactoryDependencies->eventStore, + ) + ); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php new file mode 100644 index 00000000000..91a764eef56 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php @@ -0,0 +1,30 @@ +projectionService->replayAllProjections(CatchUpOptions::create()); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php new file mode 100644 index 00000000000..4b59f652c5d --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php @@ -0,0 +1,34 @@ + + * @internal this is currently only used by the {@see SitePruningService} + */ +#[Flow\Scope("singleton")] +final class ProjectionReplayProcessorFactory implements ContentRepositoryServiceFactoryInterface +{ + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + return new ProjectionReplayProcessor( + new ProjectionService( + $serviceFactoryDependencies->projections, + $serviceFactoryDependencies->contentRepository, + $serviceFactoryDependencies->eventStore, + ) + ); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupService.php deleted file mode 100644 index 7112ad85486..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupService.php +++ /dev/null @@ -1,95 +0,0 @@ -catchupAllProjections(CatchUpOptions::create()); - } - - public function catchupProjection(string $projectionAliasOrClassName, CatchUpOptions $options): void - { - $projectionClassName = $this->resolveProjectionClassName($projectionAliasOrClassName); - $this->contentRepository->catchUpProjection($projectionClassName, $options); - } - - public function catchupAllProjections(CatchUpOptions $options, ?\Closure $progressCallback = null): void - { - foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { - if ($progressCallback) { - $progressCallback($classNamesAndAlias['alias']); - } - $this->contentRepository->catchUpProjection($classNamesAndAlias['className'], $options); - } - } - - /** - * @return class-string> - */ - private function resolveProjectionClassName(string $projectionAliasOrClassName): string - { - $lowerCaseProjectionName = strtolower($projectionAliasOrClassName); - $projectionClassNamesAndAliases = $this->projectionClassNamesAndAliases(); - foreach ($projectionClassNamesAndAliases as $classNamesAndAlias) { - if (strtolower($classNamesAndAlias['className']) === $lowerCaseProjectionName || strtolower($classNamesAndAlias['alias']) === $lowerCaseProjectionName) { - return $classNamesAndAlias['className']; - } - } - throw new \InvalidArgumentException(sprintf( - 'The projection "%s" is not registered for this Content Repository. The following projection aliases (or fully qualified class names) can be used: %s', - $projectionAliasOrClassName, - implode('', array_map(static fn (array $classNamesAndAlias) => sprintf(chr(10) . ' * %s (%s)', $classNamesAndAlias['alias'], $classNamesAndAlias['className']), $projectionClassNamesAndAliases)) - ), 1680519624); - } - - /** - * @return array>, alias: string}> - */ - private function projectionClassNamesAndAliases(): array - { - return array_map( - static fn (string $projectionClassName) => [ - 'className' => $projectionClassName, - 'alias' => self::projectionAlias($projectionClassName), - ], - $this->projections->getClassNames() - ); - } - - private static function projectionAlias(string $className): string - { - $alias = lcfirst(substr(strrchr($className, '\\') ?: '\\' . $className, 1)); - if (str_ends_with($alias, 'Projection')) { - $alias = substr($alias, 0, -10); - } - return $alias; - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupServiceFactory.php deleted file mode 100644 index 126de8eec64..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionCatchupServiceFactory.php +++ /dev/null @@ -1,30 +0,0 @@ - - * @internal this is currently only used by the {@see CrCommandController} - */ -#[Flow\Scope("singleton")] -final class ProjectionCatchupServiceFactory implements ContentRepositoryServiceFactoryInterface -{ - - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new ProjectionCatchupService( - $serviceFactoryDependencies->projections, - $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventStore, - ); - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php similarity index 83% rename from Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php rename to Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php index 87753b37e58..3a6cc5e242f 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php @@ -9,20 +9,17 @@ use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\Projections; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Export\ProcessingContext; -use Neos\ContentRepository\Export\ProcessorInterface; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStream\VirtualStreamName; /** - * Content Repository service to perform Projection replays + * Content Repository service to perform Projection operations * * @internal this is currently only used by the {@see CrCommandController} */ -final class ProjectionReplayService implements ProcessorInterface, ContentRepositoryServiceInterface +final class ProjectionService implements ContentRepositoryServiceInterface { - public function __construct( private readonly Projections $projections, private readonly ContentRepository $contentRepository, @@ -30,11 +27,6 @@ public function __construct( ) { } - public function run(ProcessingContext $context): void - { - $this->replayAllProjections(CatchUpOptions::create()); - } - public function replayProjection(string $projectionAliasOrClassName, CatchUpOptions $options): void { $projectionClassName = $this->resolveProjectionClassName($projectionAliasOrClassName); @@ -60,6 +52,22 @@ public function resetAllProjections(): void } } + public function catchupProjection(string $projectionAliasOrClassName, CatchUpOptions $options): void + { + $projectionClassName = $this->resolveProjectionClassName($projectionAliasOrClassName); + $this->contentRepository->catchUpProjection($projectionClassName, $options); + } + + public function catchupAllProjections(CatchUpOptions $options, ?\Closure $progressCallback = null): void + { + foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { + if ($progressCallback) { + $progressCallback($classNamesAndAlias['alias']); + } + $this->contentRepository->catchUpProjection($classNamesAndAlias['className'], $options); + } + } + public function highestSequenceNumber(): SequenceNumber { foreach ($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1) as $eventEnvelope) { diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php similarity index 81% rename from Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayServiceFactory.php rename to Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php index 337297d9bb6..43b2c53fdf5 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayServiceFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php @@ -10,18 +10,17 @@ use Neos\Flow\Annotations as Flow; /** - * Factory for the {@see ProjectionReplayService} + * Factory for the {@see ProjectionService} * - * @implements ContentRepositoryServiceFactoryInterface + * @implements ContentRepositoryServiceFactoryInterface * @internal this is currently only used by the {@see CrCommandController} */ #[Flow\Scope("singleton")] -final class ProjectionReplayServiceFactory implements ContentRepositoryServiceFactoryInterface +final class ProjectionServiceFactory implements ContentRepositoryServiceFactoryInterface { - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { - return new ProjectionReplayService( + return new ProjectionService( $serviceFactoryDependencies->projections, $serviceFactoryDependencies->contentRepository, $serviceFactoryDependencies->eventStore, diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index 7a1de11bc18..f0f93259a18 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -339,7 +339,7 @@ protected function determineTargetPath(?string $packageKey, ?string $path): stri } } if ($packageKey === null && $path === null) { - $this->outputLine('You have to specify either --package-key or --filename'); + $this->outputLine('You have to specify either --package-key or --path'); $this->quit(1); } if ($path === null) { diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php index be73e71a1d1..d6f6330d373 100644 --- a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php @@ -14,24 +14,13 @@ namespace Neos\Neos\Domain\Pruning; -use JsonException; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; -use Neos\ContentRepository\Core\SharedModel\Node\NodeName; -use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\ContentRepository\Export\Event\ValueObject\ExportedEvent; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepository\Export\Severity; use Neos\EventStore\EventStoreInterface; -use Neos\Flow\Persistence\PersistenceManagerInterface; -use Neos\Neos\Domain\Model\Domain; -use Neos\Neos\Domain\Model\Site; -use Neos\Neos\Domain\Repository\DomainRepository; -use Neos\Neos\Domain\Repository\SiteRepository; -use Neos\Neos\Domain\Service\NodeTypeNameFactory; /** * Pruning processor that removes all events from the given cr @@ -47,10 +36,12 @@ public function __construct( public function run(ProcessingContext $context): void { foreach ($this->contentRepository->findContentStreams() as $contentStream) { + /** @phpstan-ignore-next-line calling internal method */ $streamName = ContentStreamEventStreamName::fromContentStreamId($contentStream->id)->getEventStreamName(); $this->eventStore->deleteStream($streamName); } foreach ($this->contentRepository->findWorkspaces() as $workspace) { + /** @phpstan-ignore-next-line calling internal method */ $streamName = WorkspaceEventStreamName::fromWorkspaceName($workspace->workspaceName)->getEventStreamName(); $this->eventStore->deleteStream($streamName); } diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php index 84b645624f4..db2299fa348 100644 --- a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php +++ b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php @@ -17,22 +17,14 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\ContentRepository\Export\Processors\EventExportProcessor; -use Neos\Flow\Persistence\Doctrine\PersistenceManager; -use Neos\Flow\Persistence\PersistenceManagerInterface; -use Neos\Neos\Domain\Import\LiveWorkspaceIsEmptyProcessor; -use Neos\Neos\Domain\Repository\DomainRepository; -use Neos\Neos\Domain\Repository\SiteRepository; /** * @implements ContentRepositoryServiceFactoryInterface */ final readonly class ContentRepositoryPruningProcessorFactory implements ContentRepositoryServiceFactoryInterface { - - public function __construct( - ) { + public function __construct() + { } public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface diff --git a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php index c4aa7592586..81369e66307 100644 --- a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php @@ -14,25 +14,10 @@ namespace Neos\Neos\Domain\Pruning; -use JsonException; -use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; -use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; -use Neos\ContentRepository\Core\SharedModel\Node\NodeName; -use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\ContentRepository\Export\Event\ValueObject\ExportedEvent; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepository\Export\Severity; -use Neos\EventStore\EventStoreInterface; -use Neos\Flow\Persistence\PersistenceManagerInterface; -use Neos\Neos\Domain\Model\Domain; -use Neos\Neos\Domain\Model\Site; -use Neos\Neos\Domain\Repository\DomainRepository; -use Neos\Neos\Domain\Repository\SiteRepository; -use Neos\Neos\Domain\Service\NodeTypeNameFactory; use Neos\Neos\Domain\Service\WorkspaceService; /** diff --git a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php index 116821465f6..456e0c10623 100644 --- a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php +++ b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php @@ -17,13 +17,6 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\ContentRepository\Export\Processors\EventExportProcessor; -use Neos\Flow\Persistence\Doctrine\PersistenceManager; -use Neos\Flow\Persistence\PersistenceManagerInterface; -use Neos\Neos\Domain\Import\LiveWorkspaceIsEmptyProcessor; -use Neos\Neos\Domain\Repository\DomainRepository; -use Neos\Neos\Domain\Repository\SiteRepository; use Neos\Neos\Domain\Service\WorkspaceService; /** @@ -31,7 +24,6 @@ */ final readonly class RoleAndMetadataPruningProcessorFactory implements ContentRepositoryServiceFactoryInterface { - public function __construct( private WorkspaceService $workspaceService, ) { diff --git a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php index 76f0e0b55d0..5c23d8cc895 100644 --- a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php +++ b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php @@ -26,11 +26,10 @@ use Neos\Neos\Domain\Repository\SiteRepository; /** - * @implements ContentRepositoryServiceFactoryInterface + * @implements ContentRepositoryServiceFactoryInterface */ final readonly class SitePruningProcessorFactory implements ContentRepositoryServiceFactoryInterface { - public function __construct( private WorkspaceName $workspaceName, private SiteRepository $siteRepository, diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 57d9e8b7c3d..7b8ba4503ba 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -25,9 +25,9 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Service\ProjectionCatchupService; -use Neos\ContentRepositoryRegistry\Service\ProjectionCatchupServiceFactory; -use Neos\ContentRepositoryRegistry\Service\ProjectionReplayServiceFactory; +use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; +use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessorFactory; +use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -45,16 +45,16 @@ final readonly class SiteImportService { public function __construct( - private ContentRepositoryRegistry $contentRepositoryRegistry, - private DoctrineService $doctrineService, - private SiteRepository $siteRepository, - private DomainRepository $domainRepository, - private AssetRepository $assetRepository, - private ResourceRepository $resourceRepository, - private ResourceManager $resourceManager, - private PersistenceManagerInterface $persistenceManager, - private WorkspaceService $workspaceService, - private ProjectionCatchupServiceFactory $projectionCatchupServiceFactory, + private ContentRepositoryRegistry $contentRepositoryRegistry, + private DoctrineService $doctrineService, + private SiteRepository $siteRepository, + private DomainRepository $domainRepository, + private AssetRepository $assetRepository, + private ResourceRepository $resourceRepository, + private ResourceManager $resourceManager, + private PersistenceManagerInterface $persistenceManager, + private WorkspaceService $workspaceService, + private ProjectionCatchupProcessorFactory $projectionCatchupServiceFactory, ) { } diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index fc5069768d1..0c38ff399b6 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -25,9 +25,10 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Service\ProjectionCatchupService; -use Neos\ContentRepositoryRegistry\Service\ProjectionCatchupServiceFactory; -use Neos\ContentRepositoryRegistry\Service\ProjectionReplayServiceFactory; +use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; +use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessorFactory; +use Neos\ContentRepositoryRegistry\Processors\ProjectionReplayProcessorFactory; +use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -48,12 +49,12 @@ final readonly class SitePruningService { public function __construct( - private ContentRepositoryRegistry $contentRepositoryRegistry, - private SiteRepository $siteRepository, - private DomainRepository $domainRepository, - private PersistenceManagerInterface $persistenceManager, - private ProjectionReplayServiceFactory $projectionReplayServiceFactory, - private WorkspaceService $workspaceService, + private ContentRepositoryRegistry $contentRepositoryRegistry, + private SiteRepository $siteRepository, + private DomainRepository $domainRepository, + private PersistenceManagerInterface $persistenceManager, + private ProjectionReplayProcessorFactory $projectionReplayServiceFactory, + private WorkspaceService $workspaceService, ) { } @@ -63,7 +64,7 @@ public function __construct( */ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onProcessor, \Closure $onMessage): void { - $filesystem = new Filesystem( new LocalFilesystemAdapter('.')); + $filesystem = new Filesystem(new LocalFilesystemAdapter('.')); $context = new ProcessingContext($filesystem, $onMessage); // TODO make configurable (?) diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index ebbd2ba29a9..12d96178fbb 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -3,7 +3,7 @@ parameters: - message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Projection\\\\Projections\\:\\:getClassNames\" is called\\.$#" count: 1 - path: Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php + path: Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php - message: "#^Method Neos\\\\Neos\\\\Controller\\\\Backend\\\\MenuHelper\\:\\:buildModuleList\\(\\) return type has no value type specified in iterable type array\\.$#" From bd1ff93b0fdf4f8a8c13f93856b66f31ac1d8b1d Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Thu, 24 Oct 2024 14:46:43 +0200 Subject: [PATCH 19/56] Refactor migration of legacyData and add additional command `cr:exportLegacyData` alongside `cr:migrateLegacyData` --- .../Classes/Command/CrCommandController.php | 159 ++++++++++-------- .../Classes/Helpers/DomainDataLoader.php | 33 ++++ .../Classes/Helpers/SiteDataLoader.php | 33 ++++ .../Classes/LegacyExportService.php | 74 ++++++++ ...ory.php => LegacyExportServiceFactory.php} | 25 +-- .../Classes/LegacyMigrationService.php | 94 ----------- .../AssetExportProcessor.php} | 4 +- .../EventExportProcessor.php} | 42 +---- .../Processors/SitesExportProcessor.php | 66 ++++++++ .../Behavior/Bootstrap/FeatureContext.php | 8 +- .../composer.json | 2 +- .../Domain/Service/SiteImportService.php | 22 +-- .../Domain/Service/SitePruningService.php | 14 +- 13 files changed, 332 insertions(+), 244 deletions(-) create mode 100644 Neos.ContentRepository.LegacyNodeMigration/Classes/Helpers/DomainDataLoader.php create mode 100644 Neos.ContentRepository.LegacyNodeMigration/Classes/Helpers/SiteDataLoader.php create mode 100644 Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportService.php rename Neos.ContentRepository.LegacyNodeMigration/Classes/{LegacyMigrationServiceFactory.php => LegacyExportServiceFactory.php} (56%) delete mode 100644 Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php rename Neos.ContentRepository.LegacyNodeMigration/Classes/{NodeDataToAssetsProcessor.php => Processors/AssetExportProcessor.php} (97%) rename Neos.ContentRepository.LegacyNodeMigration/Classes/{NodeDataToEventsProcessor.php => Processors/EventExportProcessor.php} (94%) create mode 100644 Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php index 90b0fa78049..3c35ce735d5 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php @@ -18,36 +18,26 @@ use Doctrine\DBAL\DriverManager; use Doctrine\DBAL\Exception as DBALException; use Doctrine\DBAL\Exception\ConnectionException; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; +use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Export\Severity; +use Neos\ContentRepository\LegacyNodeMigration\LegacyExportServiceFactory; use Neos\ContentRepository\LegacyNodeMigration\LegacyMigrationService; use Neos\ContentRepository\LegacyNodeMigration\LegacyMigrationServiceFactory; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Factory\EventStore\DoctrineEventStoreFactory; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Cli\CommandController; -use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Flow\Property\PropertyMapper; -use Neos\Flow\ResourceManagement\ResourceManager; -use Neos\Flow\ResourceManagement\ResourceRepository; use Neos\Flow\Utility\Environment; -use Neos\Media\Domain\Repository\AssetRepository; -use Neos\Neos\Domain\Model\Site; -use Neos\Neos\Domain\Repository\SiteRepository; +use Neos\Neos\Domain\Service\SiteImportService; +use Neos\Utility\Files; class CrCommandController extends CommandController { public function __construct( private readonly Connection $connection, private readonly Environment $environment, - private readonly PersistenceManagerInterface $persistenceManager, - private readonly AssetRepository $assetRepository, - private readonly ResourceRepository $resourceRepository, - private readonly ResourceManager $resourceManager, private readonly PropertyMapper $propertyMapper, private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - private readonly SiteRepository $siteRepository, - private readonly ProjectionServiceFactory $projectionServiceFactory, + private readonly SiteImportService $siteImportService, ) { parent::__construct(); } @@ -55,11 +45,10 @@ public function __construct( /** * Migrate from the Legacy CR * - * @param bool $verbose If set, all notices will be rendered * @param string|null $config JSON encoded configuration, for example '{"dbal": {"dbname": "some-other-db"}, "resourcesPath": "/some/absolute/path"}' * @throws \Exception */ - public function migrateLegacyDataCommand(bool $verbose = false, string $config = null): void + public function migrateLegacyDataCommand(string $contentRepository = 'default', bool $verbose = false, string $config = null): void { if ($config !== null) { try { @@ -83,68 +72,84 @@ public function migrateLegacyDataCommand(bool $verbose = false, string $config = } $this->verifyDatabaseConnection($connection); + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $temporaryFilePath = $this->environment->getPathToTemporaryDirectory() . uniqid('Export', true); + Files::createDirectoryRecursively($temporaryFilePath); - $siteRows = $connection->fetchAllAssociativeIndexed('SELECT nodename, name, siteresourcespackagekey FROM neos_neos_domain_model_site'); - $siteNodeName = $this->output->select('Which site to migrate?', array_map(static fn (array $siteRow) => $siteRow['name'] . ' (' . $siteRow['siteresourcespackagekey'] . ')', $siteRows)); - assert(is_string($siteNodeName)); - $siteRow = $siteRows[$siteNodeName]; + $legacyExportService = $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new LegacyExportServiceFactory( + $connection, + $resourcesPath, + $this->propertyMapper, + ) + ); - $site = $this->siteRepository->findOneByNodeName($siteNodeName); - if ($site !== null) { - if (!$this->output->askConfirmation(sprintf('Site "%s" already exists, update it? [n] ', $siteNodeName), false)) { - $this->outputLine('Cancelled...'); - $this->quit(); - } + $legacyExportService->exportToPath( + $temporaryFilePath, + $this->createOnProcessorClosure(), + $this->createOnMessageClosure($verbose) + ); - $site->setSiteResourcesPackageKey($siteRow['siteresourcespackagekey']); - $site->setState(Site::STATE_ONLINE); - $site->setName($siteRow['name']); - $this->siteRepository->update($site); - $this->persistenceManager->persistAll(); - } else { - $site = new Site($siteNodeName); - $site->setSiteResourcesPackageKey($siteRow['siteresourcespackagekey']); - $site->setState(Site::STATE_ONLINE); - $site->setName($siteRow['name']); - $this->siteRepository->add($site); - $this->persistenceManager->persistAll(); - } + $this->siteImportService->importFromPath( + $contentRepositoryId, + $temporaryFilePath, + $this->createOnProcessorClosure(), + $this->createOnMessageClosure($verbose) + ); + + Files::unlink($temporaryFilePath); - $contentRepositoryId = $site->getConfiguration()->contentRepositoryId; + $this->outputLine('Done'); + } - $eventTableName = DoctrineEventStoreFactory::databaseTableName($contentRepositoryId); - $confirmed = $this->output->askConfirmation(sprintf('We will clear the events from "%s". ARE YOU SURE [n]? ', $eventTableName), false); - if (!$confirmed) { - $this->outputLine('Cancelled...'); - $this->quit(); + /** + * Export from the Legacy CR into a specified directory path + * + * @param string $path The path to the directory, will be created if missing + * @param string|null $config JSON encoded configuration, for example '{"dbal": {"dbname": "some-other-db"}, "resourcesPath": "/some/absolute/path"}' + * @throws \Exception + */ + public function exportLegacyDataCommand(string $path, bool $verbose = false, string $config = null): void + { + if ($config !== null) { + try { + $parsedConfig = json_decode($config, true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new \InvalidArgumentException(sprintf('Failed to parse --config parameter: %s', $e->getMessage()), 1659526855, $e); + } + $resourcesPath = $parsedConfig['resourcesPath'] ?? self::defaultResourcesPath(); + try { + $connection = isset($parsedConfig['dbal']) ? DriverManager::getConnection(array_merge($this->connection->getParams(), $parsedConfig['dbal']), new Configuration()) : $this->connection; + } catch (DBALException $e) { + throw new \InvalidArgumentException(sprintf('Failed to get database connection, check the --config parameter: %s', $e->getMessage()), 1659527201, $e); + } + } else { + $resourcesPath = $this->determineResourcesPath(); + if (!$this->output->askConfirmation(sprintf('Do you want to migrate nodes from the current database "%s@%s" (y/n)? ', $this->connection->getParams()['dbname'] ?? '?', $this->connection->getParams()['host'] ?? '?'))) { + $connection = $this->adjustDataBaseConnection($this->connection); + } else { + $connection = $this->connection; + } } - $this->connection->executeStatement('TRUNCATE ' . $connection->quoteIdentifier($eventTableName)); - // we also need to reset the projections; in order to ensure the system runs deterministically - $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionServiceFactory); - $projectionService->resetAllProjections(); - $this->outputLine('Truncated events'); + $this->verifyDatabaseConnection($connection); - $legacyMigrationService = $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new LegacyMigrationServiceFactory( + Files::createDirectoryRecursively($path); + $legacyExportService = $this->contentRepositoryRegistry->buildService( + ContentRepositoryId::fromString('default'), + new LegacyExportServiceFactory( $connection, $resourcesPath, - $this->environment, - $this->persistenceManager, - $this->assetRepository, - $this->resourceRepository, - $this->resourceManager, $this->propertyMapper, ) ); - assert($legacyMigrationService instanceof LegacyMigrationService); - $legacyMigrationService->runAllProcessors($this->outputLine(...), $verbose); + $legacyExportService->exportToPath( + $path, + $this->createOnProcessorClosure(), + $this->createOnMessageClosure($verbose) + ); - $this->outputLine(); - - $this->outputLine('Replaying projections'); - $projectionService->replayAllProjections(CatchUpOptions::create()); $this->outputLine('Done'); } @@ -199,4 +204,26 @@ private static function defaultResourcesPath(): string { return FLOW_PATH_DATA . 'Persistent/Resources'; } + + protected function createOnProcessorClosure(): \Closure + { + $onProcessor = function (string $processorLabel) { + $this->outputLine('%s...', [$processorLabel]); + }; + return $onProcessor; + } + + protected function createOnMessageClosure(bool $verbose): \Closure + { + return function (Severity $severity, string $message) use ($verbose) { + if (!$verbose && $severity === Severity::NOTICE) { + return; + } + $this->outputLine(match ($severity) { + Severity::NOTICE => $message, + Severity::WARNING => sprintf('Warning: %s', $message), + Severity::ERROR => sprintf('Error: %s', $message), + }); + }; + } } diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Helpers/DomainDataLoader.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Helpers/DomainDataLoader.php new file mode 100644 index 00000000000..beb74a87fee --- /dev/null +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Helpers/DomainDataLoader.php @@ -0,0 +1,33 @@ +> + */ +final class DomainDataLoader implements \IteratorAggregate +{ + public function __construct( + private readonly Connection $connection, + ) { + } + + /** + * @return \Traversable> + */ + public function getIterator(): \Traversable + { + $query = $this->connection->executeQuery(' + SELECT + * + FROM + neos_neos_domain_model_domain + '); + return $query->iterateAssociative(); + } +} + + diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Helpers/SiteDataLoader.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Helpers/SiteDataLoader.php new file mode 100644 index 00000000000..2d878f9302f --- /dev/null +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Helpers/SiteDataLoader.php @@ -0,0 +1,33 @@ +> + */ +final class SiteDataLoader implements \IteratorAggregate +{ + public function __construct( + private readonly Connection $connection, + ) { + } + + /** + * @return \Traversable> + */ + public function getIterator(): \Traversable + { + $query = $this->connection->executeQuery(' + SELECT + * + FROM + neos_neos_domain_model_site + '); + return $query->iterateAssociative(); + } +} + + diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportService.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportService.php new file mode 100644 index 00000000000..451cb162d86 --- /dev/null +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportService.php @@ -0,0 +1,74 @@ +connection), new FileSystemResourceLoader($this->resourcesPath)); + + $processors = Processors::fromArray([ + 'Exporting assets' => new AssetExportProcessor($this->nodeTypeManager, $assetExporter, new NodeDataLoader($this->connection)), + 'Exporting node data' => new EventExportProcessor($this->nodeTypeManager, $this->propertyMapper, $this->propertyConverter, $this->interDimensionalVariationGraph, $this->eventNormalizer, new NodeDataLoader($this->connection)), + 'Exporting sites data' => new SitesExportProcessor(new SiteDataLoader($this->connection), new DomainDataLoader($this->connection)), + ]); + + $processingContext = new ProcessingContext($filesystem, $onMessage); + foreach ($processors as $processorLabel => $processor) { + ($onProcessor)($processorLabel); + $processor->run($processingContext); + } + } +} diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationServiceFactory.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportServiceFactory.php similarity index 56% rename from Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationServiceFactory.php rename to Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportServiceFactory.php index 80ab8a5f84a..8497f3ff376 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationServiceFactory.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportServiceFactory.php @@ -14,52 +14,35 @@ * source code. */ - use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; -use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Flow\Property\PropertyMapper; -use Neos\Flow\ResourceManagement\ResourceManager; -use Neos\Flow\ResourceManagement\ResourceRepository; use Neos\Flow\Utility\Environment; -use Neos\Media\Domain\Repository\AssetRepository; /** - * @implements ContentRepositoryServiceFactoryInterface + * @implements ContentRepositoryServiceFactoryInterface */ -class LegacyMigrationServiceFactory implements ContentRepositoryServiceFactoryInterface +class LegacyExportServiceFactory implements ContentRepositoryServiceFactoryInterface { public function __construct( private readonly Connection $connection, private readonly string $resourcesPath, - private readonly Environment $environment, - private readonly PersistenceManagerInterface $persistenceManager, - private readonly AssetRepository $assetRepository, - private readonly ResourceRepository $resourceRepository, - private readonly ResourceManager $resourceManager, private readonly PropertyMapper $propertyMapper, ) { } public function build( ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies - ): LegacyMigrationService { - return new LegacyMigrationService( + ): LegacyExportService { + return new LegacyExportService( $this->connection, $this->resourcesPath, - $this->environment, - $this->persistenceManager, - $this->assetRepository, - $this->resourceRepository, - $this->resourceManager, $serviceFactoryDependencies->interDimensionalVariationGraph, $serviceFactoryDependencies->nodeTypeManager, $this->propertyMapper, $serviceFactoryDependencies->eventNormalizer, $serviceFactoryDependencies->propertyConverter, - $serviceFactoryDependencies->eventStore, - $serviceFactoryDependencies->contentRepository, ); } } diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php deleted file mode 100644 index d057db43fb9..00000000000 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyMigrationService.php +++ /dev/null @@ -1,94 +0,0 @@ -environment->getPathToTemporaryDirectory() . uniqid('Export', true); - Files::createDirectoryRecursively($temporaryFilePath); - $filesystem = new Filesystem(new LocalFilesystemAdapter($temporaryFilePath)); - - $assetExporter = new AssetExporter($filesystem, new DbalAssetLoader($this->connection), new FileSystemResourceLoader($this->resourcesPath)); - - $processors = Processors::fromArray([ - 'Exporting assets' => new NodeDataToAssetsProcessor($this->nodeTypeManager, $assetExporter, new NodeDataLoader($this->connection)), - 'Exporting node data' => new NodeDataToEventsProcessor($this->nodeTypeManager, $this->propertyMapper, $this->propertyConverter, $this->interDimensionalVariationGraph, $this->eventNormalizer, new NodeDataLoader($this->connection)), - 'Importing assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Importing events' => new EventStoreImportProcessor(WorkspaceName::forLive(), true, $this->eventStore, $this->eventNormalizer, $this->contentRepository), - ]); - $processingContext = new ProcessingContext($filesystem, function (Severity $severity, string $message) use ($verbose, $outputLineFn) { - if ($severity !== Severity::NOTICE || $verbose) { - $outputLineFn('<%1$s>%2$s', [$severity === Severity::ERROR ? 'error' : 'comment', $message]); - } - }); - foreach ($processors as $label => $processor) { - $outputLineFn($label . '...'); - $processor->run($processingContext); - $outputLineFn(); - } - Files::unlink($temporaryFilePath); - } -} diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/AssetExportProcessor.php similarity index 97% rename from Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php rename to Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/AssetExportProcessor.php index 6bde0f844e5..57794d129eb 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/AssetExportProcessor.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Neos\ContentRepository\LegacyNodeMigration; +namespace Neos\ContentRepository\LegacyNodeMigration\Processors; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; @@ -14,7 +14,7 @@ use Neos\Utility\Exception\InvalidTypeException; use Neos\Utility\TypeHandling; -final class NodeDataToAssetsProcessor implements ProcessorInterface +final class AssetExportProcessor implements ProcessorInterface { /** * @var array diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php similarity index 94% rename from Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php rename to Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php index 48ea7dd0bdb..10e083b3822 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Neos\ContentRepository\LegacyNodeMigration; +namespace Neos\ContentRepository\LegacyNodeMigration\Processors; use Doctrine\DBAL\Platforms\PostgreSQLPlatform; use Doctrine\DBAL\Types\ConversionException; @@ -54,7 +54,7 @@ use Neos\Neos\Domain\Service\NodeTypeNameFactory; use Webmozart\Assert\Assert; -final class NodeDataToEventsProcessor implements ProcessorInterface +final class EventExportProcessor implements ProcessorInterface { private NodeTypeName $sitesNodeTypeName; private WorkspaceName $workspaceName; @@ -68,8 +68,6 @@ final class NodeDataToEventsProcessor implements ProcessorInterface private int $numberOfExportedEvents = 0; - private bool $metaDataExported = false; - /** * @var resource|null */ @@ -97,18 +95,6 @@ public function setContentStreamId(ContentStreamId $contentStreamId): void $this->contentStreamId = $contentStreamId; } - public function setSitesNodeType(NodeTypeName $nodeTypeName): void - { - $nodeType = $this->nodeTypeManager->getNodeType($nodeTypeName); - if (!$nodeType?->isOfType(NodeTypeNameFactory::NAME_SITES)) { - throw new \InvalidArgumentException( - sprintf('Sites NodeType "%s" must be of type "%s"', $nodeTypeName->value, NodeTypeNameFactory::NAME_SITES), - 1695802415 - ); - } - $this->sitesNodeTypeName = $nodeTypeName; - } - public function run(ProcessingContext $context): void { $this->resetRuntimeState(); @@ -120,10 +106,6 @@ public function run(ProcessingContext $context): void $this->exportEvent(new RootNodeAggregateWithNodeWasCreated($this->workspaceName, $this->contentStreamId, $sitesNodeAggregateId, $this->sitesNodeTypeName, $this->interDimensionalVariationGraph->getDimensionSpacePoints(), NodeAggregateClassification::CLASSIFICATION_ROOT)); continue; } - if ($this->metaDataExported === false && $nodeDataRow['parentpath'] === '/sites') { - $this->exportMetaData($context, $nodeDataRow); - $this->metaDataExported = true; - } try { $this->processNodeData($context, $nodeDataRow); } catch (MigrationException $e) { @@ -149,7 +131,6 @@ private function resetRuntimeState(): void $this->visitedNodes = new VisitedNodeAggregates(); $this->nodeReferencesWereSetEvents = []; $this->numberOfExportedEvents = 0; - $this->metaDataExported = false; $this->eventFileResource = fopen('php://temp/maxmemory:5242880', 'rb+') ?: null; Assert::resource($this->eventFileResource, null, 'Failed to create temporary event file resource'); } @@ -162,6 +143,8 @@ private function exportEvent(EventInterface $event): void } catch (\JsonException $e) { throw new \RuntimeException(sprintf('Failed to JSON-decode "%s": %s', $normalizedEvent->data->value, $e->getMessage()), 1723032243, $e); } + // do not export crid and workspace as they are always imported into a single workspace + unset($exportedEventPayload['contentStreamId'], $exportedEventPayload['workspaceName']); $exportedEvent = new ExportedEvent( $normalizedEvent->id->value, $normalizedEvent->type->value, @@ -173,23 +156,6 @@ private function exportEvent(EventInterface $event): void $this->numberOfExportedEvents++; } - /** - * @param array $nodeDataRow - */ - private function exportMetaData(ProcessingContext $context, array $nodeDataRow): void - { - if ($context->files->fileExists('meta.json')) { - $data = json_decode($context->files->read('meta.json'), true, 512, JSON_THROW_ON_ERROR); - } else { - $data = []; - } - $data['version'] = 1; - $data['sitePackageKey'] = strtok($nodeDataRow['nodetype'], ':'); - $data['siteNodeName'] = substr($nodeDataRow['path'], 7); - $data['siteNodeType'] = $nodeDataRow['nodetype']; - $context->files->write('meta.json', json_encode($data, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT)); - } - /** * @param array $nodeDataRow */ diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php new file mode 100644 index 00000000000..d8b7fe6b3f7 --- /dev/null +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php @@ -0,0 +1,66 @@ +> $siteRows + * @param iterable> $domainRows + */ + public function __construct( + private readonly iterable $siteRows, + private readonly iterable $domainRows + ) { + } + + public function run(ProcessingContext $context): void + { + $sitesData = $this->getSiteData(); + $context->files->write('sites.json', json_encode($sitesData, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT)); + } + + /** + * @return SiteShape[] + */ + private function getSiteData(): array + { + $siteData = []; + foreach ($this->siteRows as $siteRow) { + $siteData[] = [ + "name" => $siteRow['name'], + "nodeName" => $siteRow['nodename'], + "siteResourcesPackageKey" => $siteRow['siteresourcespackagekey'], + "online" => $siteRow['state'] === 1, + "domains" => array_filter( + array_map( + function(array $domainRow) use ($siteRow) { + if ($siteRow['persistence_object_identifier'] !== $domainRow['site']) { + return null; + } + return [ + 'hostname' => $domainRow['hostname'], + 'scheme' => $domainRow['scheme'], + 'port' => $domainRow['port'], + 'active' => $domainRow['active'], + 'primary' => $domainRow === $siteRow['primarydomain'], + ]; + }, + iterator_to_array($this->domainRows) + ) + ) + ]; + } + + return $siteData; + } +} diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 7a506466d0c..188d6b53731 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -26,8 +26,8 @@ use Neos\ContentRepository\Export\Asset\ValueObject\SerializedAsset; use Neos\ContentRepository\Export\Asset\ValueObject\SerializedImageVariant; use Neos\ContentRepository\Export\Asset\ValueObject\SerializedResource; -use Neos\ContentRepository\LegacyNodeMigration\NodeDataToAssetsProcessor; -use Neos\ContentRepository\LegacyNodeMigration\NodeDataToEventsProcessor; +use Neos\ContentRepository\LegacyNodeMigration\Processors\AssetExportProcessor; +use Neos\ContentRepository\LegacyNodeMigration\SitesToSitesProcessor; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\CRTestSuiteTrait; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Property\PropertyMapper; @@ -106,7 +106,7 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor }; $this->getContentRepositoryService($propertyConverterAccess); - $migration = new NodeDataToEventsProcessor( + $migration = new SitesToSitesProcessor( $nodeTypeManager, $propertyMapper, $propertyConverterAccess->propertyConverter, @@ -202,7 +202,7 @@ public function findAssetById(string $assetId): SerializedAsset|SerializedImageV }; $assetExporter = new AssetExporter($this->crImportExportTrait_filesystem, $mockAssetLoader, $mockResourceLoader); - $migration = new NodeDataToAssetsProcessor($nodeTypeManager, $assetExporter, $this->nodeDataRows); + $migration = new AssetExportProcessor($nodeTypeManager, $assetExporter, $this->nodeDataRows); $this->runCrImportExportProcessors($migration); } diff --git a/Neos.ContentRepository.LegacyNodeMigration/composer.json b/Neos.ContentRepository.LegacyNodeMigration/composer.json index f88ba50c268..e6a0f1c067f 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/composer.json +++ b/Neos.ContentRepository.LegacyNodeMigration/composer.json @@ -12,7 +12,7 @@ ], "require": { "php": ">=8.2", - "neoe/neos": "self.version", + "neos/neos": "self.version", "neos/contentrepository-core": "self.version", "neos/contentrepository-export": "self.version", "league/flysystem": "^3" diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 7b8ba4503ba..67c2e457d19 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -45,16 +45,16 @@ final readonly class SiteImportService { public function __construct( - private ContentRepositoryRegistry $contentRepositoryRegistry, - private DoctrineService $doctrineService, - private SiteRepository $siteRepository, - private DomainRepository $domainRepository, - private AssetRepository $assetRepository, - private ResourceRepository $resourceRepository, - private ResourceManager $resourceManager, - private PersistenceManagerInterface $persistenceManager, - private WorkspaceService $workspaceService, - private ProjectionCatchupProcessorFactory $projectionCatchupServiceFactory, + private ContentRepositoryRegistry $contentRepositoryRegistry, + private DoctrineService $doctrineService, + private SiteRepository $siteRepository, + private DomainRepository $domainRepository, + private AssetRepository $assetRepository, + private ResourceRepository $resourceRepository, + private ResourceManager $resourceManager, + private PersistenceManagerInterface $persistenceManager, + private WorkspaceService $workspaceService, + private ProjectionCatchupProcessorFactory $projectionCatchupProcessorFactory, ) { } @@ -81,7 +81,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Catchup all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionCatchupServiceFactory), + 'Catchup all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionCatchupProcessorFactory), ]; foreach ($processors as $processorLabel => $processor) { diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index 0c38ff399b6..bb082eff395 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -49,12 +49,12 @@ final readonly class SitePruningService { public function __construct( - private ContentRepositoryRegistry $contentRepositoryRegistry, - private SiteRepository $siteRepository, - private DomainRepository $domainRepository, - private PersistenceManagerInterface $persistenceManager, - private ProjectionReplayProcessorFactory $projectionReplayServiceFactory, - private WorkspaceService $workspaceService, + private ContentRepositoryRegistry $contentRepositoryRegistry, + private SiteRepository $siteRepository, + private DomainRepository $domainRepository, + private PersistenceManagerInterface $persistenceManager, + private ProjectionReplayProcessorFactory $projectionReplayProcessorFactory, + private WorkspaceService $workspaceService, ) { } @@ -89,7 +89,7 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $this->workspaceService ) ), - 'Replay all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayServiceFactory), + 'Replay all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayProcessorFactory), ]; foreach ($processors as $processorLabel => $processor) { From 14523b2086b1d13c38ce03484405cc2f1a45430e Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Thu, 24 Oct 2024 17:03:48 +0200 Subject: [PATCH 20/56] Rename `cr:migratelegacydata`, `cr:exportlegacydata`, to `site:migratelegacydata`, `site:exportlegacydata` and adjust the readmes --- .../{CrCommandController.php => SiteCommandController.php} | 2 +- .../Classes/Service/ProjectionService.php | 2 +- README.md | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) rename Neos.ContentRepository.LegacyNodeMigration/Classes/Command/{CrCommandController.php => SiteCommandController.php} (99%) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php similarity index 99% rename from Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php rename to Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php index 3c35ce735d5..afa3dafc704 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php @@ -30,7 +30,7 @@ use Neos\Neos\Domain\Service\SiteImportService; use Neos\Utility\Files; -class CrCommandController extends CommandController +class SiteCommandController extends CommandController { public function __construct( private readonly Connection $connection, diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php index 3a6cc5e242f..02ae8cd9bc9 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php @@ -16,7 +16,7 @@ /** * Content Repository service to perform Projection operations * - * @internal this is currently only used by the {@see CrCommandController} + * @internal this is currently only used by the {@see SiteCommandController} */ final class ProjectionService implements ContentRepositoryServiceInterface { diff --git a/README.md b/README.md index b7cb5af08a2..ea185d368e3 100644 --- a/README.md +++ b/README.md @@ -76,14 +76,16 @@ You can chose from one of the following options: # the following config points to a Neos 8.0 database (adjust to your needs), created by # the legacy "./flow site:import Neos.Demo" command. -./flow cr:migrateLegacyData --config '{"dbal": {"dbname": "neos80"}, "resourcesPath": "/path/to/neos-8.0/Data/Persistent/Resources"}' +./flow site:migrateLegacyData --config '{"dbal": {"dbname": "neos80"}, "resourcesPath": "/path/to/neos-8.0/Data/Persistent/Resources"}' ``` #### Importing an existing (Neos >= 9.0) Site from an Export ``` bash +# make sure this cr is empty +./flow site:pruneAll # import the event stream from the Neos.Demo package -./flow cr:import Packages/Sites/Neos.Demo/Resources/Private/Content +./flow site:importAll Packages/Sites/Neos.Demo/Resources/Private/Content ``` ### Running Neos From 1c2afb74b62dbd52fc446a5a930eeb15c1794641 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Thu, 24 Oct 2024 20:15:45 +0200 Subject: [PATCH 21/56] Adjust behat tests for legacy export and add sites.feature --- .../Bootstrap/CrImportExportTrait.php | 23 +++++ .../Features/EventExportProcessor.feature | 4 +- .../Tests/Behavior/behat.yml.dist | 11 +++ .../Processors/EventExportProcessor.php | 5 -- .../Processors/SitesExportProcessor.php | 32 +++---- .../Behavior/Bootstrap/FeatureContext.php | 53 ++++++++++-- .../Tests/Behavior/Features/Basic.feature | 6 +- .../Tests/Behavior/Features/Hidden.feature | 84 +++++++++---------- ...iddenWithoutTimeableNodeVisibility.feature | 84 +++++++++---------- .../Tests/Behavior/Features/Sites.feature | 46 ++++++++++ 10 files changed, 231 insertions(+), 117 deletions(-) create mode 100644 Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Sites.feature diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php index 8fed952b2bd..38f5024e6c0 100644 --- a/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php +++ b/Neos.ContentRepository.Export/Tests/Behavior/Features/Bootstrap/CrImportExportTrait.php @@ -181,6 +181,29 @@ public function iExpectTheFollowingEventsToBeExported(TableNode $table): void Assert::assertCount(count($table->getHash()), $exportedEvents, 'Expected number of events does not match actual number'); } + /** + * @Then I expect the following sites to be exported + */ + public function iExpectTheFollowingSitesToBeExported(TableNode $table): void + { + if (!$this->crImportExportTrait_filesystem->has('sites.json')) { + Assert::fail('No events were exported'); + } + $actualSitesJson = $this->crImportExportTrait_filesystem->read('sites.json'); + $actualSiteRows = json_decode($actualSitesJson, true, 512, JSON_THROW_ON_ERROR); + + $expectedSites = $table->getHash(); + foreach ($expectedSites as $key => $expectedSiteData) { + $actualSiteData = $actualSiteRows[$key] ?? []; + $expectedSiteData = array_map( + fn(string $value) => json_decode($value, true, 512, JSON_THROW_ON_ERROR), + $expectedSiteData + ); + Assert::assertEquals($expectedSiteData, $actualSiteData, 'Actual site: ' . json_encode($actualSiteData, JSON_THROW_ON_ERROR)); + } + Assert::assertCount(count($table->getHash()), $actualSiteRows, 'Expected number of sites does not match actual number'); + } + /** * @Then I expect the following errors to be logged */ diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature b/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature index 9c3f28cad09..a8a67030e9e 100644 --- a/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature +++ b/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature @@ -37,7 +37,7 @@ Feature: As a user of the CR I want to export the event stream using the EventEx When the events are exported Then I expect the following jsonl: """ - {"identifier":"random-event-uuid","type":"RootNodeAggregateWithNodeWasCreated","payload":{"workspaceName":"live","contentStreamId":"cs-identifier","nodeAggregateId":"lady-eleonode-rootford","nodeTypeName":"Neos.ContentRepository:Root","coveredDimensionSpacePoints":[{"language":"de"},{"language":"gsw"},{"language":"fr"}],"nodeAggregateClassification":"root"},"metadata":{"commandClass":"Neos\\ContentRepository\\Core\\Feature\\RootNodeCreation\\Command\\CreateRootNodeAggregateWithNode","commandPayload":{"workspaceName":"live","nodeAggregateId":"lady-eleonode-rootford","nodeTypeName":"Neos.ContentRepository:Root","tetheredDescendantNodeAggregateIds":[]},"initiatingUserId":"system","initiatingTimestamp":"random-time"}} - {"identifier":"random-event-uuid","type":"NodeAggregateWithNodeWasCreated","payload":{"workspaceName":"live","contentStreamId":"cs-identifier","nodeAggregateId":"nody-mc-nodeface","nodeTypeName":"Neos.ContentRepository.Testing:Document","originDimensionSpacePoint":{"language":"de"},"succeedingSiblingsForCoverage":[{"dimensionSpacePoint":{"language":"de"},"nodeAggregateId":null},{"dimensionSpacePoint":{"language":"gsw"},"nodeAggregateId":null},{"dimensionSpacePoint":{"language":"fr"},"nodeAggregateId":null}],"parentNodeAggregateId":"lady-eleonode-rootford","nodeName":"child-document","initialPropertyValues":[],"nodeAggregateClassification":"regular"},"metadata":{"initiatingTimestamp":"random-time"}} + {"identifier":"random-event-uuid","type":"RootNodeAggregateWithNodeWasCreated","payload":{"nodeAggregateId":"lady-eleonode-rootford","nodeTypeName":"Neos.ContentRepository:Root","coveredDimensionSpacePoints":[{"language":"de"},{"language":"gsw"},{"language":"fr"}],"nodeAggregateClassification":"root"},"metadata":{"commandClass":"Neos\\ContentRepository\\Core\\Feature\\RootNodeCreation\\Command\\CreateRootNodeAggregateWithNode","commandPayload":{"workspaceName":"live","nodeAggregateId":"lady-eleonode-rootford","nodeTypeName":"Neos.ContentRepository:Root","tetheredDescendantNodeAggregateIds":[]},"initiatingUserId":"system","initiatingTimestamp":"random-time"}} + {"identifier":"random-event-uuid","type":"NodeAggregateWithNodeWasCreated","payload":{"nodeAggregateId":"nody-mc-nodeface","nodeTypeName":"Neos.ContentRepository.Testing:Document","originDimensionSpacePoint":{"language":"de"},"succeedingSiblingsForCoverage":[{"dimensionSpacePoint":{"language":"de"},"nodeAggregateId":null},{"dimensionSpacePoint":{"language":"gsw"},"nodeAggregateId":null},{"dimensionSpacePoint":{"language":"fr"},"nodeAggregateId":null}],"parentNodeAggregateId":"lady-eleonode-rootford","nodeName":"child-document","initialPropertyValues":[],"nodeAggregateClassification":"regular"},"metadata":{"initiatingTimestamp":"random-time"}} """ diff --git a/Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist b/Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist index e69de29bb2d..11e2845b599 100644 --- a/Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist +++ b/Neos.ContentRepository.Export/Tests/Behavior/behat.yml.dist @@ -0,0 +1,11 @@ + +default: + autoload: + '': "%paths.base%/Features/Bootstrap" + suites: + cr: + paths: + - "%paths.base%/Features" + + contexts: + - FeatureContext diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php index 10e083b3822..747178f37a7 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php @@ -90,11 +90,6 @@ public function __construct( $this->visitedNodes = new VisitedNodeAggregates(); } - public function setContentStreamId(ContentStreamId $contentStreamId): void - { - $this->contentStreamId = $contentStreamId; - } - public function run(ProcessingContext $context): void { $this->resetRuntimeState(); diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php index d8b7fe6b3f7..9b66a0f0f94 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php @@ -41,21 +41,23 @@ private function getSiteData(): array "nodeName" => $siteRow['nodename'], "siteResourcesPackageKey" => $siteRow['siteresourcespackagekey'], "online" => $siteRow['state'] === 1, - "domains" => array_filter( - array_map( - function(array $domainRow) use ($siteRow) { - if ($siteRow['persistence_object_identifier'] !== $domainRow['site']) { - return null; - } - return [ - 'hostname' => $domainRow['hostname'], - 'scheme' => $domainRow['scheme'], - 'port' => $domainRow['port'], - 'active' => $domainRow['active'], - 'primary' => $domainRow === $siteRow['primarydomain'], - ]; - }, - iterator_to_array($this->domainRows) + "domains" => array_values( + array_filter( + array_map( + function(array $domainRow) use ($siteRow) { + if ($siteRow['persistence_object_identifier'] !== $domainRow['site']) { + return null; + } + return [ + 'hostname' => $domainRow['hostname'], + 'scheme' => $domainRow['scheme'], + 'port' => $domainRow['port'], + 'active' => $domainRow['active'], + 'primary' => $domainRow['persistence_object_identifier'] === $siteRow['primarydomain'], + ]; + }, + iterator_to_array($this->domainRows) + ) ) ) ]; diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 188d6b53731..a1aa3031b1e 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -27,7 +27,8 @@ use Neos\ContentRepository\Export\Asset\ValueObject\SerializedImageVariant; use Neos\ContentRepository\Export\Asset\ValueObject\SerializedResource; use Neos\ContentRepository\LegacyNodeMigration\Processors\AssetExportProcessor; -use Neos\ContentRepository\LegacyNodeMigration\SitesToSitesProcessor; +use Neos\ContentRepository\LegacyNodeMigration\Processors\EventExportProcessor; +use Neos\ContentRepository\LegacyNodeMigration\Processors\SitesExportProcessor; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\CRTestSuiteTrait; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Property\PropertyMapper; @@ -47,6 +48,8 @@ class FeatureContext implements Context protected $isolated = false; private array $nodeDataRows = []; + private array $siteDataRows = []; + private array $domainDataRows = []; /** @var array */ private array $mockResources = []; /** @var array */ @@ -86,9 +89,9 @@ public function iHaveTheFollowingNodeDataRows(TableNode $nodeDataRows): void /** * @When I run the event migration - * @When I run the event migration for content stream :contentStream + * @When I run the event migration for workspace :workspace */ - public function iRunTheEventMigration(string $contentStream = null): void + public function iRunTheEventMigration(string $workspace = null): void { $nodeTypeManager = $this->currentContentRepository->getNodeTypeManager(); $propertyMapper = $this->getObject(PropertyMapper::class); @@ -106,7 +109,7 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor }; $this->getContentRepositoryService($propertyConverterAccess); - $migration = new SitesToSitesProcessor( + $eventExportProcessor = new EventExportProcessor( $nodeTypeManager, $propertyMapper, $propertyConverterAccess->propertyConverter, @@ -114,10 +117,8 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor $this->getObject(EventNormalizer::class), $this->nodeDataRows ); - if ($contentStream !== null) { - $migration->setContentStreamId(ContentStreamId::fromString($contentStream)); - } - $this->runCrImportExportProcessors($migration); + + $this->runCrImportExportProcessors($eventExportProcessor); } /** @@ -206,6 +207,42 @@ public function findAssetById(string $assetId): SerializedAsset|SerializedImageV $this->runCrImportExportProcessors($migration); } + /** + * @When I have the following site data rows: + */ + public function iHaveTheFollowingSiteDataRows(TableNode $siteDataRows): void + { + $this->siteDataRows = array_map( + fn (array $row) => array_map( + fn(string $value) => json_decode($value, true), + $row + ), + $siteDataRows->getHash() + ); + } + + /** + * @When I have the following domain data rows: + */ + public function iHaveTheFollowingDomainDataRows(TableNode $domainDataRows): void + { + $this->domainDataRows = array_map(static function (array $row) { + return array_map( + fn(string $value) => json_decode($value, true), + $row + ); + }, $domainDataRows->getHash()); + } + + /** + * @When I run the site migration + */ + public function iRunTheSiteMigration(): void + { + $migration = new SitesExportProcessor($this->siteDataRows, $this->domainDataRows); + $this->runCrImportExportProcessors($migration); + } + /** ---------------------------------- */ protected function getContentRepositoryService( diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Basic.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Basic.feature index 6dd05a8d31c..f99d5d3c399 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Basic.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Basic.feature @@ -22,8 +22,8 @@ Feature: Simple migrations without content dimensions | Identifier | Path | Node Type | Properties | | sites-node-id | /sites | unstructured | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Hidden.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Hidden.feature index ef3ec403e0e..bb212a59393 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Hidden.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Hidden.feature @@ -30,46 +30,46 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 1 | | | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | Scenario: A node with a "hidden" property false must not get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | | | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | Scenario: A node with active "hidden after" property, after a "hidden before" property must get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 1990-01-01 10:10:10 | 1989-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "1989-01-01 10:10:10"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "1989-01-01 10:10:10"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | Scenario: A node with active "hidden before" property, after a "hidden after" property must not get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 1989-01-01 10:10:10 | 1990-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "1989-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "1989-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}}} | Scenario: A node with a active "hidden before" property and a "hidden after" property in future must not get disabled @@ -77,90 +77,90 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 2099-01-01 10:10:10 | 1990-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}}} | Scenario: A node with a active "hidden after" property and a "hidden before" property in future must get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 1990-01-01 10:10:10 | 2099-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | Scenario: A node with a "hidden after" property in future and a "hidden before" property later in future must not get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 2098-01-01 10:10:10 | 2099-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "2098-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "2098-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}}} | Scenario: A node with a "hidden before" property in future and a "hidden after" property later in future must get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 2099-01-01 10:10:10 | 2098-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "2098-01-01 10:10:10"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "2098-01-01 10:10:10"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | Scenario: A node with a active "hidden before" property must not get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | | 1990-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}}} | Scenario: A node with a active "hidden after" property must get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 1990-01-01 10:10:10 | | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "1990-01-01 10:10:10"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | Scenario: A node with a "hidden after" property in future must not get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 2099-01-01 10:10:10 | | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "disableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}}} | Scenario: A node with a "hidden before" property in future must get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | | 2099-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}, "enableAfterDateTime": {"type": "DateTimeImmutable", "value": "2099-01-01 10:10:10"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/HiddenWithoutTimeableNodeVisibility.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/HiddenWithoutTimeableNodeVisibility.feature index bb701dab822..59d82c5b8e8 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/HiddenWithoutTimeableNodeVisibility.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/HiddenWithoutTimeableNodeVisibility.feature @@ -23,35 +23,35 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 1 | | | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | Scenario: A node with a "hidden" property false must not get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | | | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | Scenario: A node with active "hidden after" property, after a "hidden before" property must get disabled When I have the following node data rows: | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 1990-01-01 10:10:10 | 1989-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | @@ -60,11 +60,11 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 1989-01-01 10:10:10 | 1990-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | @@ -73,11 +73,11 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 2099-01-01 10:10:10 | 1990-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | @@ -86,12 +86,12 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 1990-01-01 10:10:10 | 2099-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | @@ -100,11 +100,11 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 2098-01-01 10:10:10 | 2099-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | @@ -113,12 +113,12 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 2099-01-01 10:10:10 | 2098-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | @@ -127,11 +127,11 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | | 1990-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | @@ -140,12 +140,12 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 1990-01-01 10:10:10 | | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | @@ -154,11 +154,11 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | 2099-01-01 10:10:10 | | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | @@ -167,11 +167,11 @@ Feature: Simple migrations without content dimensions for hidden state migration | Identifier | Path | Node Type | Properties | Hidden | Hidden after DateTime | Hidden before DateTime | | sites-node-id | /sites | unstructured | | 0 | | | | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | 0 | | 2099-01-01 10:10:10 | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | - | SubtreeWasTagged | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "tag": "disabled"} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | SubtreeWasTagged | {"nodeAggregateId": "site-node-id", "tag": "disabled"} | And I expect the following warnings to be logged | Skipped the migration of your "hiddenBeforeDateTime" and "hiddenAfterDateTime" properties as your target NodeTypes do not inherit "Neos.TimeableNodeVisibility:Timeable". Please install neos/timeable-node-visibility, if you want to migrate them. | diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Sites.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Sites.feature new file mode 100644 index 00000000000..12a1e808c17 --- /dev/null +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Sites.feature @@ -0,0 +1,46 @@ +@contentrepository +Feature: Simple migrations without content dimensions + + Background: + Given using no content dimensions + And using the following node types: + """yaml + 'Neos.Neos:Site': {} + 'Some.Package:Homepage': + superTypes: + 'Neos.Neos:Site': true + properties: + 'text': + type: string + defaultValue: 'My default text' + """ + And using identifier "default", I define a content repository + And I am in content repository "default" + + Scenario: Site records without domains + When I have the following site data rows: + | persistence_object_identifier | name | nodename | siteresourcespackagekey | state | domains | primarydomain | + | "site1" | "Site 1" | "site_1_node" | "Site1.Package" | 1 | null | null | + | "site2" | "Site 2" | "site_2_node" | "Site2.Package" | 2 | null | null | + And I run the site migration + Then I expect the following sites to be exported + | name | nodeName | siteResourcesPackageKey | online | domains | + | "Site 1" | "site_1_node" | "Site1.Package" | true | [] | + | "Site 2" | "site_2_node" | "Site2.Package" | false | [] | + + Scenario: Site records with domains + When I have the following site data rows: + | persistence_object_identifier | name | nodename | siteresourcespackagekey | state | domains | primarydomain | + | "site1" | "Site 1" | "site_1_node" | "Site1.Package" | 1 | null | "domain2" | + | "site2" | "Site 2" | "site_2_node" | "Site2.Package" | 1 | null | null | + When I have the following domain data rows: + | persistence_object_identifier | hostname | scheme | port | active | site | + | "domain1" | "domain_1.tld" | "https" | 123 | true | "site1" | + | "domain2" | "domain_2.tld" | "http" | null | true | "site1" | + | "domain3" | "domain_3.tld" | null | null | true | "site2" | + | "domain4" | "domain_4.tld" | null | null | false | "site2" | + And I run the site migration + Then I expect the following sites to be exported + | name | nodeName | siteResourcesPackageKey | online | domains | + | "Site 1" | "site_1_node" | "Site1.Package" | true | [{"hostname": "domain_1.tld", "scheme": "https", "port": 123, "active": true, "primary": false},{"hostname": "domain_2.tld", "scheme": "http", "port": null, "active": true, "primary": true}] | + | "Site 2" | "site_2_node" | "Site2.Package" | true | [{"hostname": "domain_3.tld", "scheme": null, "port": null, "active": true, "primary": false},{"hostname": "domain_4.tld", "scheme": null, "port": null, "active": false, "primary": false}] | From 4423476ce6e39c5504f09c683a2c9953b6e9dce2 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Thu, 24 Oct 2024 21:02:07 +0200 Subject: [PATCH 22/56] Format arguments of site command controller properly --- .../Classes/Command/SiteCommandController.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php index afa3dafc704..5e81b340027 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php @@ -33,11 +33,11 @@ class SiteCommandController extends CommandController { public function __construct( - private readonly Connection $connection, - private readonly Environment $environment, - private readonly PropertyMapper $propertyMapper, - private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - private readonly SiteImportService $siteImportService, + private readonly Connection $connection, + private readonly Environment $environment, + private readonly PropertyMapper $propertyMapper, + private readonly ContentRepositoryRegistry $contentRepositoryRegistry, + private readonly SiteImportService $siteImportService, ) { parent::__construct(); } From a9fe6ceb980eaddaddefe3e0ae088f7e2a965965 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Mon, 28 Oct 2024 19:04:04 +0100 Subject: [PATCH 23/56] Adjust after QS Feedback: - Add identifier "neos.cr.internal" for ignored internal calls - Reuse shapes from Export in Import - Remove comment about extensibility via configuration --- .../Classes/PhpstanRules/ApiOrInternalAnnotationRule.php | 2 +- ...InternalMethodsNotAllowedOutsideContentRepositoryRule.php | 2 +- .../Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php | 2 +- Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php | 5 +++-- Neos.Neos/Classes/Domain/Service/SiteExportService.php | 1 - Neos.Neos/Classes/Domain/Service/SiteImportService.php | 1 - Neos.Neos/Classes/Domain/Service/SitePruningService.php | 1 - composer.json | 4 ++-- 8 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Classes/PhpstanRules/ApiOrInternalAnnotationRule.php b/Neos.ContentRepository.BehavioralTests/Classes/PhpstanRules/ApiOrInternalAnnotationRule.php index 1f76b0e16bd..3821a0f9f2e 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/PhpstanRules/ApiOrInternalAnnotationRule.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/PhpstanRules/ApiOrInternalAnnotationRule.php @@ -54,7 +54,7 @@ public function processNode(Node $node, Scope $scope): array return [ RuleErrorBuilder::message( 'Class needs @api or @internal annotation.' - )->build(), + )->identifier('neos.cr.internal')->build(), ]; } return []; diff --git a/Neos.ContentRepository.BehavioralTests/Classes/PhpstanRules/InternalMethodsNotAllowedOutsideContentRepositoryRule.php b/Neos.ContentRepository.BehavioralTests/Classes/PhpstanRules/InternalMethodsNotAllowedOutsideContentRepositoryRule.php index 5dae4e82dfc..7334cfa78db 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/PhpstanRules/InternalMethodsNotAllowedOutsideContentRepositoryRule.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/PhpstanRules/InternalMethodsNotAllowedOutsideContentRepositoryRule.php @@ -74,7 +74,7 @@ public function processNode(Node $node, Scope $scope): array $targetClassName, $node->name->toString() ) - )->build(), + )->identifier('neos.cr.internal')->build(), ]; } } diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php index 3aee0df7594..6d69c4f771e 100644 --- a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php @@ -43,7 +43,7 @@ public function run(ProcessingContext $context): void private function workspaceHasEvents(WorkspaceName $workspaceName): bool { - /** @phpstan-ignore-next-line internal method of the cr is called */ + /** @phpstan-ignore neos.cr.internal */ $workspaceStreamName = WorkspaceEventStreamName::fromWorkspaceName($workspaceName)->getEventStreamName(); $eventStream = $this->eventStore->load($workspaceStreamName); foreach ($eventStream as $event) { diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index 947011aea80..d1a1a06d3ba 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -21,6 +21,7 @@ use Neos\ContentRepository\Export\ProcessorInterface; use Neos\ContentRepository\Export\Severity; use Neos\Flow\Persistence\PersistenceManagerInterface; +use Neos\Neos\Domain\Export\SiteExportProcessor; use Neos\Neos\Domain\Model\Domain; use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\DomainRepository; @@ -29,8 +30,8 @@ /** * Import processor that creates and persists a Neos {@see Site} instance * - * @phpstan-type DomainShape array{hostname: string, scheme?: ?string, port?: ?int, active?: ?bool, primary?: ?bool } - * @phpstan-type SiteShape array{name:string, siteResourcesPackageKey:string, nodeName?: string, online?:bool, domains?: ?DomainShape[] } + * @phpstan-import-type DomainShape from SiteExportProcessor + * @phpstan-import-type SiteShape from SiteExportProcessor */ final readonly class SiteCreationProcessor implements ProcessorInterface { diff --git a/Neos.Neos/Classes/Domain/Service/SiteExportService.php b/Neos.Neos/Classes/Domain/Service/SiteExportService.php index 87c581edea9..913351c5112 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteExportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteExportService.php @@ -59,7 +59,6 @@ public function exportToPath(ContentRepositoryId $contentRepositoryId, string $p throw new \RuntimeException('Failed to find live workspace', 1716652280); } - // TODO make configurable (?) /** @var array $processors */ $processors = [ 'Exporting events' => $this->contentRepositoryRegistry->buildService( diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 67c2e457d19..483dd2056df 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -71,7 +71,6 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string $context = new ProcessingContext($filesystem, $onMessage); $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); - // TODO make configurable (?) /** @var array $processors */ $processors = [ 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index bb082eff395..15eabcf7911 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -67,7 +67,6 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $filesystem = new Filesystem(new LocalFilesystemAdapter('.')); $context = new ProcessingContext($filesystem, $onMessage); - // TODO make configurable (?) /** @var array $processors */ $processors = [ 'Remove site nodes' => $this->contentRepositoryRegistry->buildService( diff --git a/composer.json b/composer.json index 4425532f575..5c059f778c1 100644 --- a/composer.json +++ b/composer.json @@ -96,7 +96,7 @@ "scripts": { "lint:phpcs": "../../bin/phpcs --colors", "lint:phpcs:fix": "../../bin/phpcbf --colors", - "lint:phpstan": "../../bin/phpstan analyse", + "lint:phpstan": "../../bin/phpstan analyse -v", "lint:phpstan-generate-baseline": "../../bin/phpstan analyse --generate-baseline", "lint:distributionintegrity": "[ -d 'Neos.ContentRepository' ] && { echo 'Package Neos.ContentRepository should not exist.' 1>&2; exit 1; } || exit 0;", "lint": [ @@ -293,7 +293,7 @@ }, "require-dev": { "roave/security-advisories": "dev-latest", - "phpstan/phpstan": "^1.8", + "phpstan/phpstan": "^1.11", "squizlabs/php_codesniffer": "^3.6", "phpunit/phpunit": "^9.0", "neos/behat": "*", From f526e739d770a4707150110e72f96c961af41f92 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 17:09:12 +0100 Subject: [PATCH 24/56] TASK: Post merge adjustments to make phpstan happy --- .../Classes/Processors/ProjectionCatchupProcessorFactory.php | 2 +- .../Classes/Processors/ProjectionReplayProcessorFactory.php | 2 +- .../Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php index 14436d9dc52..305e8c30172 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php @@ -27,7 +27,7 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor { return new ProjectionCatchupProcessor( new ProjectionService( - $serviceFactoryDependencies->projections, + $serviceFactoryDependencies->projectionsAndCatchUpHooks->projections, $serviceFactoryDependencies->contentRepository, $serviceFactoryDependencies->eventStore, ) diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php index 4b59f652c5d..6c9c32b6200 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php @@ -25,7 +25,7 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor { return new ProjectionReplayProcessor( new ProjectionService( - $serviceFactoryDependencies->projections, + $serviceFactoryDependencies->projectionsAndCatchUpHooks->projections, $serviceFactoryDependencies->contentRepository, $serviceFactoryDependencies->eventStore, ) diff --git a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php index 81369e66307..f341f825b24 100644 --- a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php @@ -33,7 +33,7 @@ public function __construct( public function run(ProcessingContext $context): void { - $this->workspaceService->pruneRoleAsssignments($this->contentRepositoryId); + $this->workspaceService->pruneRoleAssignments($this->contentRepositoryId); $this->workspaceService->pruneWorkspaceMetadata($this->contentRepositoryId); } } From 0772b6a8fe724587e4b6820f8b7ea14495d72804 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 17:25:34 +0100 Subject: [PATCH 25/56] TASK: Replace `LiveWorkspaceIsEmptyProcessor` with read model checks --- .../Export/SiteExportProcessorFactory.php | 1 - .../Import/LiveWorkspaceCreationProcessor.php | 9 +++- .../Import/LiveWorkspaceIsEmptyProcessor.php | 54 ------------------- .../LiveWorkspaceIsEmptyProcessorFactory.php | 31 ----------- .../Pruning/SitePruningProcessorFactory.php | 3 -- .../Domain/Service/SiteImportService.php | 6 +-- .../Domain/Service/SitePruningService.php | 14 ----- 7 files changed, 8 insertions(+), 110 deletions(-) delete mode 100644 Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php delete mode 100644 Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php index 07832016e15..ed313aba1da 100644 --- a/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php +++ b/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php @@ -18,7 +18,6 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\Neos\Domain\Import\LiveWorkspaceIsEmptyProcessor; use Neos\Neos\Domain\Repository\SiteRepository; /** diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php index 34a140eb7bd..97824318b99 100644 --- a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php @@ -39,8 +39,13 @@ public function __construct( public function run(ProcessingContext $context): void { $context->dispatch(Severity::NOTICE, 'Creating live workspace'); - $existingWorkspace = $this->contentRepository->findWorkspaceByName(WorkspaceName::forLive()); - if ($existingWorkspace !== null) { + $liveWorkspace = $this->contentRepository->findWorkspaceByName(WorkspaceName::forLive()); + $liveContentStreamVersion = $liveWorkspace ? $this->contentRepository->findContentStreamById($liveWorkspace->currentContentStreamId)?->version : null; + if ($liveWorkspace && $liveContentStreamVersion !== 0) { + // todo we cannot use `hasPublishableChanges` here... maybe introduce `hasChanges`? + throw new \RuntimeException('Live workspace already contains content please run "site:pruneAll" before importing.'); + } + if ($liveWorkspace !== null) { $context->dispatch(Severity::NOTICE, 'Workspace already exists, skipping'); return; } diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php deleted file mode 100644 index 6d69c4f771e..00000000000 --- a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessor.php +++ /dev/null @@ -1,54 +0,0 @@ -dispatch(Severity::NOTICE, 'Ensures empty live workspace'); - - if ($this->workspaceHasEvents(WorkspaceName::forLive())) { - throw new \RuntimeException('Live workspace already contains events please run "site:pruneAll" before importing.'); - } - } - - private function workspaceHasEvents(WorkspaceName $workspaceName): bool - { - /** @phpstan-ignore neos.cr.internal */ - $workspaceStreamName = WorkspaceEventStreamName::fromWorkspaceName($workspaceName)->getEventStreamName(); - $eventStream = $this->eventStore->load($workspaceStreamName); - foreach ($eventStream as $event) { - return true; - } - return false; - } -} diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php deleted file mode 100644 index d74eaac3215..00000000000 --- a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceIsEmptyProcessorFactory.php +++ /dev/null @@ -1,31 +0,0 @@ - - */ -final readonly class LiveWorkspaceIsEmptyProcessorFactory implements ContentRepositoryServiceFactoryInterface -{ - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new LiveWorkspaceIsEmptyProcessor($serviceFactoryDependencies->eventStore); - } -} diff --git a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php index 5c23d8cc895..40c52fa07bd 100644 --- a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php +++ b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php @@ -18,10 +18,7 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\ContentRepository\Export\Processors\EventExportProcessor; -use Neos\Flow\Persistence\Doctrine\PersistenceManager; use Neos\Flow\Persistence\PersistenceManagerInterface; -use Neos\Neos\Domain\Import\LiveWorkspaceIsEmptyProcessor; use Neos\Neos\Domain\Repository\DomainRepository; use Neos\Neos\Domain\Repository\SiteRepository; diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 483dd2056df..7cfee7c08a1 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -25,9 +25,7 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessorFactory; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -36,7 +34,6 @@ use Neos\Media\Domain\Repository\AssetRepository; use Neos\Neos\Domain\Import\DoctrineMigrateProcessor; use Neos\Neos\Domain\Import\LiveWorkspaceCreationProcessor; -use Neos\Neos\Domain\Import\LiveWorkspaceIsEmptyProcessorFactory; use Neos\Neos\Domain\Import\SiteCreationProcessor; use Neos\Neos\Domain\Repository\DomainRepository; use Neos\Neos\Domain\Repository\SiteRepository; @@ -75,9 +72,8 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string $processors = [ 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), 'Setup content repository' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositorySetupProcessorFactory()), - 'Verify Live workspace does not exist yet' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new LiveWorkspaceIsEmptyProcessorFactory()), - 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), + 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), 'Catchup all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionCatchupProcessorFactory), diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index 15eabcf7911..ed6ef56c3ff 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -18,27 +18,13 @@ use League\Flysystem\Local\LocalFilesystemAdapter; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\ContentRepository\Export\Factory\ContentRepositorySetupProcessorFactory; -use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; -use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessorFactory; use Neos\ContentRepositoryRegistry\Processors\ProjectionReplayProcessorFactory; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; -use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; -use Neos\Flow\ResourceManagement\ResourceManager; -use Neos\Flow\ResourceManagement\ResourceRepository; -use Neos\Media\Domain\Repository\AssetRepository; -use Neos\Neos\Domain\Import\DoctrineMigrateProcessor; -use Neos\Neos\Domain\Import\LiveWorkspaceCreationProcessor; -use Neos\Neos\Domain\Import\LiveWorkspaceIsEmptyProcessorFactory; -use Neos\Neos\Domain\Import\SiteCreationProcessor; use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessorFactory; use Neos\Neos\Domain\Pruning\RoleAndMetadataPruningProcessorFactory; use Neos\Neos\Domain\Pruning\SitePruningProcessorFactory; From e57ce53ea98ad8b3e72d8452a21c1208516a2ef3 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 17:29:36 +0100 Subject: [PATCH 26/56] TASK: Make `RoleAndMetadataPruningProcessor` a simple processor we dont want to use the internal cr service factory pattern here --- .../RoleAndMetadataPruningProcessor.php | 3 +- ...RoleAndMetadataPruningProcessorFactory.php | 39 ------------------- .../Domain/Service/SitePruningService.php | 9 +---- 3 files changed, 3 insertions(+), 48 deletions(-) delete mode 100644 Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php diff --git a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php index f341f825b24..063f7e6f4de 100644 --- a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessor.php @@ -14,7 +14,6 @@ namespace Neos\Neos\Domain\Pruning; -use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; @@ -23,7 +22,7 @@ /** * Pruning processor that removes role and metadata for a specified content repository */ -final readonly class RoleAndMetadataPruningProcessor implements ProcessorInterface, ContentRepositoryServiceInterface +final readonly class RoleAndMetadataPruningProcessor implements ProcessorInterface { public function __construct( private ContentRepositoryId $contentRepositoryId, diff --git a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php deleted file mode 100644 index 456e0c10623..00000000000 --- a/Neos.Neos/Classes/Domain/Pruning/RoleAndMetadataPruningProcessorFactory.php +++ /dev/null @@ -1,39 +0,0 @@ - - */ -final readonly class RoleAndMetadataPruningProcessorFactory implements ContentRepositoryServiceFactoryInterface -{ - public function __construct( - private WorkspaceService $workspaceService, - ) { - } - - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new RoleAndMetadataPruningProcessor( - $serviceFactoryDependencies->contentRepositoryId, - $this->workspaceService, - ); - } -} diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index ed6ef56c3ff..a77590d0bb4 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -26,7 +26,7 @@ use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessorFactory; -use Neos\Neos\Domain\Pruning\RoleAndMetadataPruningProcessorFactory; +use Neos\Neos\Domain\Pruning\RoleAndMetadataPruningProcessor; use Neos\Neos\Domain\Pruning\SitePruningProcessorFactory; use Neos\Neos\Domain\Repository\DomainRepository; use Neos\Neos\Domain\Repository\SiteRepository; @@ -68,12 +68,7 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $contentRepositoryId, new ContentRepositoryPruningProcessorFactory() ), - 'Prune roles and metadata' => $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new RoleAndMetadataPruningProcessorFactory( - $this->workspaceService - ) - ), + 'Prune roles and metadata' => new RoleAndMetadataPruningProcessor($contentRepositoryId, $this->workspaceService), 'Replay all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayProcessorFactory), ]; From ae03530c77ff5021f72e156ce929cfe184b17627 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 17:34:29 +0100 Subject: [PATCH 27/56] TASK: Make `ContentRepositoryPruningProcessor` use the `ContentStreamPruner` --- .../ContentRepositoryPruningProcessor.php | 19 ++-------- ...ntentRepositoryPruningProcessorFactory.php | 37 ------------------- .../Domain/Service/SitePruningService.php | 11 ++++-- 3 files changed, 10 insertions(+), 57 deletions(-) delete mode 100644 Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php index d6f6330d373..4660e6aea7e 100644 --- a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php @@ -14,13 +14,10 @@ namespace Neos\Neos\Domain\Pruning; -use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; -use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; +use Neos\ContentRepository\Core\Service\ContentStreamPruner; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\EventStore\EventStoreInterface; /** * Pruning processor that removes all events from the given cr @@ -28,22 +25,12 @@ final readonly class ContentRepositoryPruningProcessor implements ProcessorInterface, ContentRepositoryServiceInterface { public function __construct( - private ContentRepository $contentRepository, - private EventStoreInterface $eventStore, + private ContentStreamPruner $contentStreamPruner, ) { } public function run(ProcessingContext $context): void { - foreach ($this->contentRepository->findContentStreams() as $contentStream) { - /** @phpstan-ignore-next-line calling internal method */ - $streamName = ContentStreamEventStreamName::fromContentStreamId($contentStream->id)->getEventStreamName(); - $this->eventStore->deleteStream($streamName); - } - foreach ($this->contentRepository->findWorkspaces() as $workspace) { - /** @phpstan-ignore-next-line calling internal method */ - $streamName = WorkspaceEventStreamName::fromWorkspaceName($workspace->workspaceName)->getEventStreamName(); - $this->eventStore->deleteStream($streamName); - } + $this->contentStreamPruner->pruneAllWorkspacesAndContentStreamsFromEventStream(); } } diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php deleted file mode 100644 index db2299fa348..00000000000 --- a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessorFactory.php +++ /dev/null @@ -1,37 +0,0 @@ - - */ -final readonly class ContentRepositoryPruningProcessorFactory implements ContentRepositoryServiceFactoryInterface -{ - public function __construct() - { - } - - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new ContentRepositoryPruningProcessor( - $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventStore, - ); - } -} diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index a77590d0bb4..25fd6e8c2d3 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -16,6 +16,7 @@ use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; +use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\ProcessingContext; @@ -25,7 +26,7 @@ use Neos\ContentRepositoryRegistry\Processors\ProjectionReplayProcessorFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\PersistenceManagerInterface; -use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessorFactory; +use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessor; use Neos\Neos\Domain\Pruning\RoleAndMetadataPruningProcessor; use Neos\Neos\Domain\Pruning\SitePruningProcessorFactory; use Neos\Neos\Domain\Repository\DomainRepository; @@ -64,9 +65,11 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $this->persistenceManager ) ), - 'Prune content repository' => $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new ContentRepositoryPruningProcessorFactory() + 'Prune content repository' => new ContentRepositoryPruningProcessor( + $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new ContentStreamPrunerFactory() + ) ), 'Prune roles and metadata' => new RoleAndMetadataPruningProcessor($contentRepositoryId, $this->workspaceService), 'Replay all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayProcessorFactory), From cb82c2d71e5d000a0781a3a84a8b95a49ea4f845 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 17:43:00 +0100 Subject: [PATCH 28/56] TASK: Turn `SitePruningProcessor` into simple processor without factory --- .../Domain/Pruning/SitePruningProcessor.php | 8 +--- .../Pruning/SitePruningProcessorFactory.php | 48 ------------------- .../Domain/Service/SitePruningService.php | 20 ++++---- 3 files changed, 11 insertions(+), 65 deletions(-) delete mode 100644 Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php diff --git a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php index 7c272774d91..fcb6b9c7a89 100644 --- a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php @@ -14,17 +14,11 @@ namespace Neos\Neos\Domain\Pruning; -use JsonException; use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\SharedModel\Node\NodeName; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\ContentRepository\Export\Event\ValueObject\ExportedEvent; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepository\Export\Severity; use Neos\Flow\Persistence\PersistenceManagerInterface; -use Neos\Neos\Domain\Model\Domain; use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\DomainRepository; use Neos\Neos\Domain\Repository\SiteRepository; @@ -33,7 +27,7 @@ /** * Pruning processor that removes all Neos {@see Site} instances referenced by the current content repository */ -final readonly class SitePruningProcessor implements ProcessorInterface, ContentRepositoryServiceInterface +final readonly class SitePruningProcessor implements ProcessorInterface { public function __construct( private ContentRepository $contentRepository, diff --git a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php deleted file mode 100644 index 40c52fa07bd..00000000000 --- a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessorFactory.php +++ /dev/null @@ -1,48 +0,0 @@ - - */ -final readonly class SitePruningProcessorFactory implements ContentRepositoryServiceFactoryInterface -{ - public function __construct( - private WorkspaceName $workspaceName, - private SiteRepository $siteRepository, - private DomainRepository $domainRepository, - private PersistenceManagerInterface $persistenceManager, - ) { - } - - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new SitePruningProcessor( - $serviceFactoryDependencies->contentRepository, - $this->workspaceName, - $this->siteRepository, - $this->domainRepository, - $this->persistenceManager - ); - } -} diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index 25fd6e8c2d3..04dd6875923 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -28,7 +28,7 @@ use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessor; use Neos\Neos\Domain\Pruning\RoleAndMetadataPruningProcessor; -use Neos\Neos\Domain\Pruning\SitePruningProcessorFactory; +use Neos\Neos\Domain\Pruning\SitePruningProcessor; use Neos\Neos\Domain\Repository\DomainRepository; use Neos\Neos\Domain\Repository\SiteRepository; @@ -54,16 +54,16 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $filesystem = new Filesystem(new LocalFilesystemAdapter('.')); $context = new ProcessingContext($filesystem, $onMessage); - /** @var array $processors */ + $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); + + /** @var array $processors */ $processors = [ - 'Remove site nodes' => $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new SitePruningProcessorFactory( - WorkspaceName::forLive(), - $this->siteRepository, - $this->domainRepository, - $this->persistenceManager - ) + 'Remove site nodes' => new SitePruningProcessor( + $contentRepository, + WorkspaceName::forLive(), + $this->siteRepository, + $this->domainRepository, + $this->persistenceManager ), 'Prune content repository' => new ContentRepositoryPruningProcessor( $this->contentRepositoryRegistry->buildService( From dff6885bd6f92331772a2dbb8148756e243167b4 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 17:45:41 +0100 Subject: [PATCH 29/56] TASK: Turn `SiteExportProcessor` into simple processor without factory --- .../Domain/Export/SiteExportProcessor.php | 3 +- .../Export/SiteExportProcessorFactory.php | 42 ------------------- .../Domain/Service/SiteExportService.php | 12 +++--- 3 files changed, 6 insertions(+), 51 deletions(-) delete mode 100644 Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php index 643845d8b49..f71a91f4c65 100644 --- a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php +++ b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php @@ -15,7 +15,6 @@ namespace Neos\Neos\Domain\Export; use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; @@ -31,7 +30,7 @@ * @phpstan-type SiteShape array{name:string, siteResourcesPackageKey:string, nodeName?: string, online?:bool, domains?: ?DomainShape[] } * */ -final readonly class SiteExportProcessor implements ProcessorInterface, ContentRepositoryServiceInterface +final readonly class SiteExportProcessor implements ProcessorInterface { public function __construct( private ContentRepository $contentRepository, diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php deleted file mode 100644 index ed313aba1da..00000000000 --- a/Neos.Neos/Classes/Domain/Export/SiteExportProcessorFactory.php +++ /dev/null @@ -1,42 +0,0 @@ - - */ -final readonly class SiteExportProcessorFactory implements ContentRepositoryServiceFactoryInterface -{ - public function __construct( - private WorkspaceName $workspaceName, - private SiteRepository $siteRepository, - ) { - } - - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new SiteExportProcessor( - $serviceFactoryDependencies->contentRepository, - $this->workspaceName, - $this->siteRepository, - ); - } -} diff --git a/Neos.Neos/Classes/Domain/Service/SiteExportService.php b/Neos.Neos/Classes/Domain/Service/SiteExportService.php index 913351c5112..6deee3f87ba 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteExportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteExportService.php @@ -27,7 +27,7 @@ use Neos\Flow\Annotations as Flow; use Neos\Media\Domain\Repository\AssetRepository; use Neos\Neos\AssetUsage\AssetUsageService; -use Neos\Neos\Domain\Export\SiteExportProcessorFactory; +use Neos\Neos\Domain\Export\SiteExportProcessor; use Neos\Neos\Domain\Repository\SiteRepository; #[Flow\Scope('singleton')] @@ -73,12 +73,10 @@ public function exportToPath(ContentRepositoryId $contentRepositoryId, string $p $liveWorkspace, $this->assetUsageService ), - 'Export sites' => $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new SiteExportProcessorFactory( - $liveWorkspace->workspaceName, - $this->siteRepository, - ) + 'Export sites' => new SiteExportProcessor( + $contentRepository, + $liveWorkspace->workspaceName, + $this->siteRepository ), ]; foreach ($processors as $processorLabel => $processor) { From 71c65a2ecebdc335ccd76b31aef8acca036f838d Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 17:53:00 +0100 Subject: [PATCH 30/56] TASK: Turn `ProjectionCatchupProcessor` and `ProjectionReplayProcessor` into simple processor without factory --- .../Processors/ProjectionCatchupProcessor.php | 4 +-- .../ProjectionCatchupProcessorFactory.php | 36 ------------------- .../Processors/ProjectionReplayProcessor.php | 4 +-- .../ProjectionReplayProcessorFactory.php | 34 ------------------ .../ContentRepositoryPruningProcessor.php | 3 +- .../Domain/Service/SiteImportService.php | 6 ++-- .../Domain/Service/SitePruningService.php | 6 ++-- 7 files changed, 9 insertions(+), 84 deletions(-) delete mode 100644 Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php delete mode 100644 Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php index 3f837b91571..c6e66fc2e9e 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php @@ -3,7 +3,6 @@ namespace Neos\ContentRepositoryRegistry\Processors; -use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; @@ -14,9 +13,8 @@ * * @internal this is currently only used by the {@see SiteImportService} {@see SitePruningService} */ -final class ProjectionCatchupProcessor implements ProcessorInterface, ContentRepositoryServiceInterface +final class ProjectionCatchupProcessor implements ProcessorInterface { - public function __construct( private readonly ProjectionService $projectionservice, ) { diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php deleted file mode 100644 index 305e8c30172..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessorFactory.php +++ /dev/null @@ -1,36 +0,0 @@ - - * @internal this is currently only used by the {@see SiteImportService} {@see SitePruningService} - */ -#[Flow\Scope("singleton")] -final class ProjectionCatchupProcessorFactory implements ContentRepositoryServiceFactoryInterface -{ - - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new ProjectionCatchupProcessor( - new ProjectionService( - $serviceFactoryDependencies->projectionsAndCatchUpHooks->projections, - $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventStore, - ) - ); - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php index 91a764eef56..76c6c5b49d4 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php @@ -3,7 +3,6 @@ namespace Neos\ContentRepositoryRegistry\Processors; -use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; @@ -15,9 +14,8 @@ * * @internal this is currently only used by the {@see SitePruningService} */ -final class ProjectionReplayProcessor implements ProcessorInterface, ContentRepositoryServiceInterface +final class ProjectionReplayProcessor implements ProcessorInterface { - public function __construct( private readonly ProjectionService $projectionService, ) { diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php deleted file mode 100644 index 6c9c32b6200..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessorFactory.php +++ /dev/null @@ -1,34 +0,0 @@ - - * @internal this is currently only used by the {@see SitePruningService} - */ -#[Flow\Scope("singleton")] -final class ProjectionReplayProcessorFactory implements ContentRepositoryServiceFactoryInterface -{ - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new ProjectionReplayProcessor( - new ProjectionService( - $serviceFactoryDependencies->projectionsAndCatchUpHooks->projections, - $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventStore, - ) - ); - } -} diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php index 4660e6aea7e..0b94195c9d1 100644 --- a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php @@ -14,7 +14,6 @@ namespace Neos\Neos\Domain\Pruning; -use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Service\ContentStreamPruner; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; @@ -22,7 +21,7 @@ /** * Pruning processor that removes all events from the given cr */ -final readonly class ContentRepositoryPruningProcessor implements ProcessorInterface, ContentRepositoryServiceInterface +final readonly class ContentRepositoryPruningProcessor implements ProcessorInterface { public function __construct( private ContentStreamPruner $contentStreamPruner, diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 7cfee7c08a1..b7276a6f8c4 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -25,7 +25,8 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessorFactory; +use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; +use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -51,7 +52,6 @@ public function __construct( private ResourceManager $resourceManager, private PersistenceManagerInterface $persistenceManager, private WorkspaceService $workspaceService, - private ProjectionCatchupProcessorFactory $projectionCatchupProcessorFactory, ) { } @@ -76,7 +76,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Catchup all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionCatchupProcessorFactory), + 'Catchup all projections' => new ProjectionCatchupProcessor($this->contentRepositoryRegistry->buildService($contentRepositoryId, new ProjectionServiceFactory())), ]; foreach ($processors as $processorLabel => $processor) { diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index 04dd6875923..f427c9fac81 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -23,7 +23,8 @@ use Neos\ContentRepository\Export\ProcessorInterface; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionReplayProcessorFactory; +use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; +use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessor; @@ -40,7 +41,6 @@ public function __construct( private SiteRepository $siteRepository, private DomainRepository $domainRepository, private PersistenceManagerInterface $persistenceManager, - private ProjectionReplayProcessorFactory $projectionReplayProcessorFactory, private WorkspaceService $workspaceService, ) { } @@ -72,7 +72,7 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP ) ), 'Prune roles and metadata' => new RoleAndMetadataPruningProcessor($contentRepositoryId, $this->workspaceService), - 'Replay all projections' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionReplayProcessorFactory), + 'Replay all projections' => new ProjectionCatchupProcessor($this->contentRepositoryRegistry->buildService($contentRepositoryId, new ProjectionServiceFactory())) ]; foreach ($processors as $processorLabel => $processor) { From 9ba32fd2f3ae14c9a41252fc0bb9baf78499c8ae Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 19:06:23 +0100 Subject: [PATCH 31/56] TASK: Document and improve `extractSitesFromEventStream` --- .../Classes/Domain/Import/SiteCreationProcessor.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index d1a1a06d3ba..409809374d3 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -26,6 +26,7 @@ use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\DomainRepository; use Neos\Neos\Domain\Repository\SiteRepository; +use Neos\Neos\Domain\Service\NodeTypeNameFactory; /** * Import processor that creates and persists a Neos {@see Site} instance @@ -52,6 +53,7 @@ public function run(ProcessingContext $context): void throw new \RuntimeException("Failed to decode sites.json: {$e->getMessage()}", 1729506117, $e); } } else { + $context->dispatch(Severity::WARNING, 'Deprecated legacy handling: No "sites.json" in export found, attempting to extract neos sites from the events. Please update the export soonish.'); $sites = self::extractSitesFromEventStream($context); } @@ -89,20 +91,21 @@ public function run(ProcessingContext $context): void } /** + * @deprecated with Neos 9 Beta 15 please make sure that exports contain `sites.json` * @return array */ private static function extractSitesFromEventStream(ProcessingContext $context): array { $eventFileResource = $context->files->readStream('events.jsonl'); - $rootNodeAggregateIds = []; + $siteRooNodeAggregateId = null; $sites = []; while (($line = fgets($eventFileResource)) !== false) { $event = ExportedEvent::fromJson($line); - if ($event->type === 'RootNodeAggregateWithNodeWasCreated') { - $rootNodeAggregateIds[] = $event->payload['nodeAggregateId']; + if ($event->type === 'RootNodeAggregateWithNodeWasCreated' && $event->payload['nodeTypeName'] === NodeTypeNameFactory::NAME_SITES) { + $siteRooNodeAggregateId = $event->payload['nodeAggregateId']; continue; } - if ($event->type === 'NodeAggregateWithNodeWasCreated' && in_array($event->payload['parentNodeAggregateId'], $rootNodeAggregateIds, true)) { + if ($event->type === 'NodeAggregateWithNodeWasCreated' && $event->payload['parentNodeAggregateId'] === $siteRooNodeAggregateId) { $sites[] = [ 'siteResourcesPackageKey' => self::extractPackageKeyFromNodeTypeName($event->payload['nodeTypeName']), 'name' => $event->payload['initialPropertyValues']['title']['value'] ?? $event->payload['nodeTypeName'], From 9e7c5c19472ac63e7b1e8b33d3e9a718d366aa2e Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 19:10:41 +0100 Subject: [PATCH 32/56] TASK: Move todo around :) --- Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php | 1 - Neos.Neos/Classes/Domain/Model/Site.php | 5 ++++- Neos.Neos/Classes/Domain/Service/SiteService.php | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index 409809374d3..fb6d4222e87 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -66,7 +66,6 @@ public function run(ProcessingContext $context): void $context->dispatch(Severity::NOTICE, "Site for node name \"{$siteNodeName->value}\" already exists, skipping"); continue; } - // TODO use node aggregate identifier instead of node name $siteInstance = new Site($siteNodeName->value); $siteInstance->setSiteResourcesPackageKey($site['siteResourcesPackageKey']); $siteInstance->setState(($site['online'] ?? false) ? Site::STATE_OFFLINE : Site::STATE_ONLINE); diff --git a/Neos.Neos/Classes/Domain/Model/Site.php b/Neos.Neos/Classes/Domain/Model/Site.php index d6937a6655d..eb6cedae3c0 100644 --- a/Neos.Neos/Classes/Domain/Model/Site.php +++ b/Neos.Neos/Classes/Domain/Model/Site.php @@ -63,7 +63,10 @@ class Site * Node name of this site in the content repository. * * The first level of nodes of a site can be reached via a path like - * "/Sites/MySite/" where "MySite" is the nodeName. + * "//my-site" where "my-site" is the nodeName. + * + * TODO use node aggregate identifier instead of node name + * see https://github.com/neos/neos-development-collection/issues/4470 * * @var string * @Flow\Identity diff --git a/Neos.Neos/Classes/Domain/Service/SiteService.php b/Neos.Neos/Classes/Domain/Service/SiteService.php index 75dc08b5aea..42f7bea37e6 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteService.php @@ -173,7 +173,6 @@ public function createSite( throw SiteNodeNameIsAlreadyInUseByAnotherSite::butWasAttemptedToBeClaimed($siteNodeName); } - // @todo use node aggregate identifier instead of node name $site = new Site($siteNodeName->value); $site->setSiteResourcesPackageKey($packageKey); $site->setState($inactive ? Site::STATE_OFFLINE : Site::STATE_ONLINE); From 9c9ce5b40bc9dc825f7812ee1063e4b6de80229c Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 19:34:46 +0100 Subject: [PATCH 33/56] TASK: Remove redundant documentation --- .../Classes/Processors/ProjectionCatchupProcessor.php | 4 +--- .../Classes/Processors/ProjectionReplayProcessor.php | 5 +---- .../Classes/Service/ProjectionService.php | 2 +- .../Classes/Service/ProjectionServiceFactory.php | 3 +-- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php index c6e66fc2e9e..69587bb4806 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php @@ -9,9 +9,7 @@ use Neos\ContentRepositoryRegistry\Service\ProjectionService; /** - * Content Repository service to perform Projection replays - * - * @internal this is currently only used by the {@see SiteImportService} {@see SitePruningService} + * @internal */ final class ProjectionCatchupProcessor implements ProcessorInterface { diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php index 76c6c5b49d4..b77110a42a4 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php @@ -7,12 +7,9 @@ use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; use Neos\ContentRepositoryRegistry\Service\ProjectionService; -use Neos\Neos\Domain\Service\SitePruningService; /** - * Content Repository service to perform Projection replays - * - * @internal this is currently only used by the {@see SitePruningService} + * @internal */ final class ProjectionReplayProcessor implements ProcessorInterface { diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php index 02ae8cd9bc9..06f11984d74 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php @@ -16,7 +16,7 @@ /** * Content Repository service to perform Projection operations * - * @internal this is currently only used by the {@see SiteCommandController} + * @internal */ final class ProjectionService implements ContentRepositoryServiceInterface { diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php index 8d167b1c2d8..92114d47f1a 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php @@ -6,14 +6,13 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepositoryRegistry\Command\CrCommandController; use Neos\Flow\Annotations as Flow; /** * Factory for the {@see ProjectionService} * * @implements ContentRepositoryServiceFactoryInterface - * @internal this is currently only used by the {@see CrCommandController} + * @internal */ #[Flow\Scope("singleton")] final class ProjectionServiceFactory implements ContentRepositoryServiceFactoryInterface From f674d5ed13d817e0a62d3bce09153fd2eab2b42a Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 19:39:57 +0100 Subject: [PATCH 34/56] TASK: Reintroduce resetting of projection state via `ProjectionResetProcessor` as in `cr:prune` --- .../Processors/ProjectionResetProcessor.php | 24 +++++++++++++++++++ .../Domain/Service/SitePruningService.php | 8 +++++-- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php new file mode 100644 index 00000000000..7a5a1f9013f --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php @@ -0,0 +1,24 @@ +projectionService->resetAllProjections(); + } +} diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index f427c9fac81..653f6a45300 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -23,7 +23,7 @@ use Neos\ContentRepository\Export\ProcessorInterface; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; +use Neos\ContentRepositoryRegistry\Processors\ProjectionResetProcessor; use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -72,7 +72,11 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP ) ), 'Prune roles and metadata' => new RoleAndMetadataPruningProcessor($contentRepositoryId, $this->workspaceService), - 'Replay all projections' => new ProjectionCatchupProcessor($this->contentRepositoryRegistry->buildService($contentRepositoryId, new ProjectionServiceFactory())) + 'Reset all projections' => new ProjectionResetProcessor( + $this->contentRepositoryRegistry->buildService( + $contentRepositoryId, + new ProjectionServiceFactory() + )) ]; foreach ($processors as $processorLabel => $processor) { From 7e0021dd15fa8ecfc68a046fcee667aa08ae7865 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 19:44:13 +0100 Subject: [PATCH 35/56] TASK: Use `Processors::fromArray` --- Neos.Neos/Classes/Domain/Service/SiteExportService.php | 7 ++++--- Neos.Neos/Classes/Domain/Service/SiteImportService.php | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Service/SiteExportService.php b/Neos.Neos/Classes/Domain/Service/SiteExportService.php index 6deee3f87ba..ba305374607 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteExportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteExportService.php @@ -21,6 +21,7 @@ use Neos\ContentRepository\Export\Factory\EventExportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; +use Neos\ContentRepository\Export\Processors; use Neos\ContentRepository\Export\Processors\AssetExportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; @@ -59,8 +60,7 @@ public function exportToPath(ContentRepositoryId $contentRepositoryId, string $p throw new \RuntimeException('Failed to find live workspace', 1716652280); } - /** @var array $processors */ - $processors = [ + $processors = Processors::fromArray([ 'Exporting events' => $this->contentRepositoryRegistry->buildService( $contentRepositoryId, new EventExportProcessorFactory( @@ -78,7 +78,8 @@ public function exportToPath(ContentRepositoryId $contentRepositoryId, string $p $liveWorkspace->workspaceName, $this->siteRepository ), - ]; + ]); + foreach ($processors as $processorLabel => $processor) { ($onProcessor)($processorLabel); $processor->run($context); diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index b7276a6f8c4..1a65686f912 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -22,6 +22,7 @@ use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; +use Neos\ContentRepository\Export\Processors; use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; @@ -68,8 +69,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string $context = new ProcessingContext($filesystem, $onMessage); $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); - /** @var array $processors */ - $processors = [ + $processors = Processors::fromArray([ 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), 'Setup content repository' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositorySetupProcessorFactory()), 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), @@ -77,7 +77,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), 'Catchup all projections' => new ProjectionCatchupProcessor($this->contentRepositoryRegistry->buildService($contentRepositoryId, new ProjectionServiceFactory())), - ]; + ]); foreach ($processors as $processorLabel => $processor) { ($onProcessor)($processorLabel); From 7fea08b13b913d080b1ed9bf18f9f076fb0bd849 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 20:01:59 +0100 Subject: [PATCH 36/56] TASK: Fail early if `resource://` paths are used in export see https://github.com/neos/neos-development-collection/issues/4912 --- Neos.Neos/Classes/Command/SiteCommandController.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index f0f93259a18..08199679ae6 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -346,6 +346,10 @@ protected function determineTargetPath(?string $packageKey, ?string $path): stri $package = $this->packageManager->getPackage($packageKey); $path = Files::concatenatePaths([$package->getPackagePath(), 'Resources/Private/Content']); } + if (str_starts_with($path, 'resource://')) { + $this->outputLine('Resource paths are not allowed, please use --package-key instead or a real path.'); + $this->quit(1); + } return $path; } From c98384d434013e14d8ba56dab32e15b82437668b Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 2 Nov 2024 20:50:56 +0100 Subject: [PATCH 37/56] TASK: Use `Processors::fromArray` --- Neos.Neos/Classes/Domain/Service/SitePruningService.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index 653f6a45300..16414ad76bc 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -21,6 +21,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; +use Neos\ContentRepository\Export\Processors; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Processors\ProjectionResetProcessor; @@ -56,8 +57,7 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); - /** @var array $processors */ - $processors = [ + $processors = Processors::fromArray([ 'Remove site nodes' => new SitePruningProcessor( $contentRepository, WorkspaceName::forLive(), @@ -77,7 +77,7 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $contentRepositoryId, new ProjectionServiceFactory() )) - ]; + ]); foreach ($processors as $processorLabel => $processor) { ($onProcessor)($processorLabel); From adf7b38f26966ae5440578102e2700d0d3b7be33 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:46:57 +0100 Subject: [PATCH 38/56] TASK: Remove automatic schema creation from import and redirect user to `./flow cr:setup` `./flow doctrine:migrate` does not work without hacks in the same process and its also an expensive task and should not delay the import and obscure errors and swallow output if done as part of the import. --- ...ContentRepositorySetupProcessorFactory.php | 22 ----------- .../ContentRepositorySetupProcessor.php | 28 -------------- .../Import/DoctrineMigrateProcessor.php | 32 ---------------- .../Domain/Service/SiteImportService.php | 37 ++++++++++++++++--- 4 files changed, 32 insertions(+), 87 deletions(-) delete mode 100644 Neos.ContentRepository.Export/src/Factory/ContentRepositorySetupProcessorFactory.php delete mode 100644 Neos.ContentRepository.Export/src/Processors/ContentRepositorySetupProcessor.php delete mode 100644 Neos.Neos/Classes/Domain/Import/DoctrineMigrateProcessor.php diff --git a/Neos.ContentRepository.Export/src/Factory/ContentRepositorySetupProcessorFactory.php b/Neos.ContentRepository.Export/src/Factory/ContentRepositorySetupProcessorFactory.php deleted file mode 100644 index 945ef993da7..00000000000 --- a/Neos.ContentRepository.Export/src/Factory/ContentRepositorySetupProcessorFactory.php +++ /dev/null @@ -1,22 +0,0 @@ - - */ -final readonly class ContentRepositorySetupProcessorFactory implements ContentRepositoryServiceFactoryInterface -{ - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositorySetupProcessor - { - return new ContentRepositorySetupProcessor( - $serviceFactoryDependencies->contentRepository, - ); - } -} diff --git a/Neos.ContentRepository.Export/src/Processors/ContentRepositorySetupProcessor.php b/Neos.ContentRepository.Export/src/Processors/ContentRepositorySetupProcessor.php deleted file mode 100644 index 57d322921da..00000000000 --- a/Neos.ContentRepository.Export/src/Processors/ContentRepositorySetupProcessor.php +++ /dev/null @@ -1,28 +0,0 @@ -dispatch(Severity::NOTICE, "Setting up content repository \"{$this->contentRepository->id->value}\""); - $this->contentRepository->setUp(); - } -} diff --git a/Neos.Neos/Classes/Domain/Import/DoctrineMigrateProcessor.php b/Neos.Neos/Classes/Domain/Import/DoctrineMigrateProcessor.php deleted file mode 100644 index 632cdc8a6b5..00000000000 --- a/Neos.Neos/Classes/Domain/Import/DoctrineMigrateProcessor.php +++ /dev/null @@ -1,32 +0,0 @@ -doctrineService->executeMigrations(); - } -} diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 1a65686f912..741424a02e2 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -14,11 +14,12 @@ namespace Neos\Neos\Domain\Service; +use Doctrine\DBAL\Exception as DBALException; use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; +use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; -use Neos\ContentRepository\Export\Factory\ContentRepositorySetupProcessorFactory; use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; @@ -34,7 +35,6 @@ use Neos\Flow\ResourceManagement\ResourceManager; use Neos\Flow\ResourceManagement\ResourceRepository; use Neos\Media\Domain\Repository\AssetRepository; -use Neos\Neos\Domain\Import\DoctrineMigrateProcessor; use Neos\Neos\Domain\Import\LiveWorkspaceCreationProcessor; use Neos\Neos\Domain\Import\SiteCreationProcessor; use Neos\Neos\Domain\Repository\DomainRepository; @@ -65,13 +65,15 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string if (!is_dir($path)) { throw new \InvalidArgumentException(sprintf('Path "%s" is not a directory', $path), 1729593802); } + $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); + + $this->requireDataBaseSchemaToBeSetup(); + $this->requireContentRepositoryToBeSetup($contentRepository); + $filesystem = new Filesystem(new LocalFilesystemAdapter($path)); $context = new ProcessingContext($filesystem, $onMessage); - $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); $processors = Processors::fromArray([ - 'Run doctrine migrations' => new DoctrineMigrateProcessor($this->doctrineService), - 'Setup content repository' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositorySetupProcessorFactory()), 'Create Live workspace' => new LiveWorkspaceCreationProcessor($contentRepository, $this->workspaceService), 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), @@ -84,4 +86,29 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string $processor->run($context); } } + + private function requireContentRepositoryToBeSetup(ContentRepository $contentRepository): void + { + $status = $contentRepository->status(); + if (!$status->isOk()) { + throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepository->id->value)); + } + } + + private function requireDataBaseSchemaToBeSetup(): void + { + try { + [ + 'new' => $_newMigrationCount, + 'executed' => $executedMigrationCount, + 'available' => $availableMigrationCount + ] = $this->doctrineService->getMigrationStatus(); + } catch (DBALException | \PDOException) { + throw new \RuntimeException('Not database connected. Please check your database connection settings or run `./flow setup` for further information.', 1684075689386); + } + + if ($executedMigrationCount === 0 && $availableMigrationCount > 0) { + throw new \RuntimeException('No doctrine migrations have been executed. Please run `./flow doctrine:migrate`'); + } + } } From b701b17d1b7a7584bc7a708a4ab3c3f0b32436fe Mon Sep 17 00:00:00 2001 From: Denny Lubitz Date: Fri, 8 Nov 2024 14:16:47 +0100 Subject: [PATCH 39/56] TASK: Post-merge fix of tests --- .../Tests/Behavior/Bootstrap/FeatureContext.php | 7 +++---- .../Behavior/Features/RootNodeTypeMapping.feature | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 18a49dbdee0..1812c6cbeb7 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -86,13 +86,12 @@ public function iHaveTheFollowingNodeDataRows(TableNode $nodeDataRows): void } /** - * @When /^I run the event migration for content stream (.*) with rootNode mapping (.*)$/ + * @When /^I run the event migration with rootNode mapping (.*)$/ */ - public function iRunTheEventMigrationForContentStreamWithRootnodeMapping(string $contentStream = null, string $rootNodeMapping): void + public function iRunTheEventMigrationWithRootnodeMapping(string $rootNodeMapping): void { - $contentStream = trim($contentStream, '"'); $rootNodeTypeMapping = RootNodeTypeMapping::fromArray(json_decode($rootNodeMapping, true)); - $this->iRunTheEventMigration($contentStream, $rootNodeTypeMapping); + $this->iRunTheEventMigration(null, $rootNodeTypeMapping); } /** diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/RootNodeTypeMapping.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/RootNodeTypeMapping.feature index 2834b27a237..6fd919dbf24 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/RootNodeTypeMapping.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/RootNodeTypeMapping.feature @@ -24,7 +24,7 @@ Feature: Simple migrations without content dimensions but other root nodetype na | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | | test-root-node-id | /test | unstructured | | | test-node-id | /test/test-site | Some.Package:Homepage | {"text": "foo"} | - And I run the event migration for content stream "cs-id" + And I run the event migration Then I expect the following errors to be logged | Failed to find parent node for node with id "test-root-node-id" and dimensions: []. Please ensure that the new content repository has a valid content dimension configuration. Also note that the old CR can sometimes have orphaned nodes. | | Failed to find parent node for node with id "test-node-id" and dimensions: []. Please ensure that the new content repository has a valid content dimension configuration. Also note that the old CR can sometimes have orphaned nodes. | @@ -37,10 +37,10 @@ Feature: Simple migrations without content dimensions but other root nodetype na | site-node-id | /sites/test-site | Some.Package:Homepage | {"text": "foo"} | | test-root-node-id | /test | unstructured | | | test-node-id | /test/test-site | Some.Package:Homepage | {"text": "foo"} | - And I run the event migration for content stream "cs-id" with rootNode mapping {"/sites": "Neos.Neos:Sites", "/test": "Neos.ContentRepository.LegacyNodeMigration:TestRoot"} + And I run the event migration with rootNode mapping {"/sites": "Neos.Neos:Sites", "/test": "Neos.ContentRepository.LegacyNodeMigration:TestRoot"} Then I expect the following events to be exported | Type | Payload | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | - | RootNodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "test-root-node-id", "nodeTypeName": "Neos.ContentRepository.LegacyNodeMigration:TestRoot", "nodeAggregateClassification": "root"} | - | NodeAggregateWithNodeWasCreated | {"contentStreamId": "cs-id", "nodeAggregateId": "test-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "test-root-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "sites-node-id", "nodeTypeName": "Neos.Neos:Sites", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "site-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "sites-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | + | RootNodeAggregateWithNodeWasCreated | {"nodeAggregateId": "test-root-node-id", "nodeTypeName": "Neos.ContentRepository.LegacyNodeMigration:TestRoot", "nodeAggregateClassification": "root"} | + | NodeAggregateWithNodeWasCreated | {"nodeAggregateId": "test-node-id", "nodeTypeName": "Some.Package:Homepage", "nodeName": "test-site", "parentNodeAggregateId": "test-root-node-id", "nodeAggregateClassification": "regular", "initialPropertyValues": {"text": {"type": "string", "value": "foo"}}} | From 717688daf853245dddf9d79381080baad378f420 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:16:53 +0100 Subject: [PATCH 40/56] TASK: Assert that no events exist on the live content stream ... but later --- .../src/Processors/EventStoreImportProcessor.php | 8 +++----- .../Domain/Import/LiveWorkspaceCreationProcessor.php | 5 ----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php b/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php index 7af1561c16f..15e9c718427 100644 --- a/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php +++ b/Neos.ContentRepository.Export/src/Processors/EventStoreImportProcessor.php @@ -12,9 +12,6 @@ use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; use Neos\ContentRepository\Core\Feature\ContentStreamForking\Event\ContentStreamWasForked; use Neos\ContentRepository\Core\Feature\ContentStreamRemoval\Event\ContentStreamWasRemoved; -use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Event\RootWorkspaceWasCreated; -use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; -use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\Event\ValueObject\ExportedEvent; use Neos\ContentRepository\Export\ProcessingContext; @@ -23,6 +20,7 @@ use Neos\EventStore\Exception\ConcurrencyException; use Neos\EventStore\Model\Event; use Neos\EventStore\Model\Event\EventId; +use Neos\EventStore\Model\Event\Version; use Neos\EventStore\Model\Events; use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\Flow\Utility\Algorithms; @@ -100,9 +98,9 @@ public function run(ProcessingContext $context): void $contentStreamStreamName = ContentStreamEventStreamName::fromContentStreamId($workspace->currentContentStreamId)->getEventStreamName(); try { - $this->eventStore->commit($contentStreamStreamName, Events::fromArray($domainEvents), ExpectedVersion::ANY()); + $this->eventStore->commit($contentStreamStreamName, Events::fromArray($domainEvents), ExpectedVersion::fromVersion(Version::first())); } catch (ConcurrencyException $e) { - throw new \RuntimeException(sprintf('Failed to publish %d events because the event stream "%s" already exists (3)', count($domainEvents), $contentStreamStreamName->value), 1729506818, $e); + throw new \RuntimeException(sprintf('Failed to publish %d events because the event stream "%s" for workspace "%s" already contains events.', count($domainEvents), $contentStreamStreamName->value, $workspace->workspaceName->value), 1729506818, $e); } } } diff --git a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php index 97824318b99..788dad93734 100644 --- a/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/LiveWorkspaceCreationProcessor.php @@ -40,11 +40,6 @@ public function run(ProcessingContext $context): void { $context->dispatch(Severity::NOTICE, 'Creating live workspace'); $liveWorkspace = $this->contentRepository->findWorkspaceByName(WorkspaceName::forLive()); - $liveContentStreamVersion = $liveWorkspace ? $this->contentRepository->findContentStreamById($liveWorkspace->currentContentStreamId)?->version : null; - if ($liveWorkspace && $liveContentStreamVersion !== 0) { - // todo we cannot use `hasPublishableChanges` here... maybe introduce `hasChanges`? - throw new \RuntimeException('Live workspace already contains content please run "site:pruneAll" before importing.'); - } if ($liveWorkspace !== null) { $context->dispatch(Severity::NOTICE, 'Workspace already exists, skipping'); return; From efb7fbd979cadee2747fce2dd38abf8c15248938 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:18:41 +0100 Subject: [PATCH 41/56] TASK: Remove obsolete `I run the event migration for workspace :workspace` --- .../Tests/Behavior/Bootstrap/FeatureContext.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 1812c6cbeb7..415bf6d8b04 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -91,14 +91,13 @@ public function iHaveTheFollowingNodeDataRows(TableNode $nodeDataRows): void public function iRunTheEventMigrationWithRootnodeMapping(string $rootNodeMapping): void { $rootNodeTypeMapping = RootNodeTypeMapping::fromArray(json_decode($rootNodeMapping, true)); - $this->iRunTheEventMigration(null, $rootNodeTypeMapping); + $this->iRunTheEventMigration($rootNodeTypeMapping); } /** * @When I run the event migration - * @When I run the event migration for workspace :workspace */ - public function iRunTheEventMigration(string $workspace = null, RootNodeTypeMapping $rootNodeTypeMapping = null): void + public function iRunTheEventMigration(RootNodeTypeMapping $rootNodeTypeMapping = null): void { $nodeTypeManager = $this->currentContentRepository->getNodeTypeManager(); $propertyMapper = $this->getObject(PropertyMapper::class); From ea6719504b46330fde2d28741eb90acf358c6050 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 8 Nov 2024 20:30:26 +0100 Subject: [PATCH 42/56] TASK: Adjust documentation and todos in legacy migration --- .../Classes/Command/SiteCommandController.php | 25 ++++++++++++------- .../Classes/LegacyExportService.php | 3 +-- .../Processors/EventExportProcessor.php | 6 +---- .../Classes/Command/SiteCommandController.php | 6 ++--- README.md | 12 ++------- 5 files changed, 23 insertions(+), 29 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php index ba608344a5e..8f57f0dc526 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php @@ -21,8 +21,6 @@ use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepository\LegacyNodeMigration\LegacyExportServiceFactory; -use Neos\ContentRepository\LegacyNodeMigration\LegacyMigrationService; -use Neos\ContentRepository\LegacyNodeMigration\LegacyMigrationServiceFactory; use Neos\ContentRepository\LegacyNodeMigration\RootNodeTypeMapping; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Cli\CommandController; @@ -47,10 +45,13 @@ public function __construct( /** * Migrate from the Legacy CR * + * Note that the dimension configuration and the node type schema must be migrated of the content repository to import to and it must be setup. + * + * @param string $contentRepository The target content repository that will be used for importing into * @param string|null $config JSON encoded configuration, for example '{"dbal": {"dbname": "some-other-db"}, "resourcesPath": "/some/absolute/path", "rootNodes": {"/sites": "Neos.Neos:Sites", "/other": "My.Package:SomeOtherRoot"}}' * @throws \Exception */ - public function migrateLegacyDataCommand(string $contentRepository = 'default', bool $verbose = false, string $config = null): void + public function migrateLegacyDataCommand(string $contentRepository = 'default', string $config = null, bool $verbose = false, ): void { if ($config !== null) { try { @@ -69,7 +70,7 @@ public function migrateLegacyDataCommand(string $contentRepository = 'default', $resourcesPath = $this->determineResourcesPath(); $rootNodes = $this->getDefaultRootNodes(); if (!$this->output->askConfirmation(sprintf('Do you want to migrate nodes from the current database "%s@%s" (y/n)? ', $this->connection->getParams()['dbname'] ?? '?', $this->connection->getParams()['host'] ?? '?'))) { - $connection = $this->adjustDataBaseConnection($this->connection); + $connection = $this->adjustDatabaseConnection($this->connection); } else { $connection = $this->connection; } @@ -96,6 +97,9 @@ public function migrateLegacyDataCommand(string $contentRepository = 'default', $this->createOnMessageClosure($verbose) ); + $this->outputLine('Migrated data. Importing into new content repository ...'); + + // todo check if cr is setup before!!! do not fail here!!! $this->siteImportService->importFromPath( $contentRepositoryId, $temporaryFilePath, @@ -111,11 +115,14 @@ public function migrateLegacyDataCommand(string $contentRepository = 'default', /** * Export from the Legacy CR into a specified directory path * + * Note that the dimension configuration and the node type schema must be migrated of the reference content repository + * + * @param string $contentRepository The reference content repository that can later be used for importing into * @param string $path The path to the directory, will be created if missing * @param string|null $config JSON encoded configuration, for example '{"dbal": {"dbname": "some-other-db"}, "resourcesPath": "/some/absolute/path", "rootNodes": {"/sites": "Neos.Neos:Sites", "/other": "My.Package:SomeOtherRoot"}}' * @throws \Exception */ - public function exportLegacyDataCommand(string $path, bool $verbose = false, string $config = null): void + public function exportLegacyDataCommand(string $path, string $contentRepository = 'default', string $config = null, bool $verbose = false): void { if ($config !== null) { try { @@ -134,7 +141,7 @@ public function exportLegacyDataCommand(string $path, bool $verbose = false, str $resourcesPath = $this->determineResourcesPath(); $rootNodes = $this->getDefaultRootNodes(); if (!$this->output->askConfirmation(sprintf('Do you want to migrate nodes from the current database "%s@%s" (y/n)? ', $this->connection->getParams()['dbname'] ?? '?', $this->connection->getParams()['host'] ?? '?'))) { - $connection = $this->adjustDataBaseConnection($this->connection); + $connection = $this->adjustDatabaseConnection($this->connection); } else { $connection = $this->connection; } @@ -143,7 +150,7 @@ public function exportLegacyDataCommand(string $path, bool $verbose = false, str Files::createDirectoryRecursively($path); $legacyExportService = $this->contentRepositoryRegistry->buildService( - ContentRepositoryId::fromString('default'), + ContentRepositoryId::fromString($contentRepository), new LegacyExportServiceFactory( $connection, $resourcesPath, @@ -164,7 +171,7 @@ public function exportLegacyDataCommand(string $path, bool $verbose = false, str /** * @throws DBALException */ - private function adjustDataBaseConnection(Connection $connection): Connection + private function adjustDatabaseConnection(Connection $connection): Connection { $connectionParams = $connection->getParams(); $connectionParams['driver'] = $this->output->select(sprintf('Driver? [%s] ', $connectionParams['driver'] ?? ''), ['pdo_mysql', 'pdo_sqlite', 'pdo_pgsql'], $connectionParams['driver'] ?? null); @@ -189,7 +196,7 @@ private function verifyDatabaseConnection(Connection $connection): void } catch (ConnectionException $exception) { $this->outputLine('Failed to connect to database "%s": %s', [$connection->getDatabase(), $exception->getMessage()]); $this->outputLine('Please verify connection parameters...'); - $this->adjustDataBaseConnection($connection); + $this->adjustDatabaseConnection($connection); } } while (true); } diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportService.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportService.php index 49399f4eb3e..dc59b6a79ce 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportService.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/LegacyExportService.php @@ -22,7 +22,6 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Export\Asset\Adapters\DbalAssetLoader; use Neos\ContentRepository\Export\Asset\Adapters\FileSystemResourceLoader; use Neos\ContentRepository\Export\Asset\AssetExporter; @@ -62,7 +61,7 @@ public function exportToPath(string $path, \Closure $onProcessor, \Closure $onMe $processors = Processors::fromArray([ 'Exporting assets' => new AssetExportProcessor($this->nodeTypeManager, $assetExporter, new NodeDataLoader($this->connection)), - 'Exporting node data' => new EventExportProcessor( $this->nodeTypeManager, $this->propertyMapper, $this->propertyConverter, $this->interDimensionalVariationGraph, $this->eventNormalizer, $this->rootNodeTypeMapping,new NodeDataLoader($this->connection)), + 'Exporting node data' => new EventExportProcessor($this->nodeTypeManager, $this->propertyMapper, $this->propertyConverter, $this->interDimensionalVariationGraph, $this->eventNormalizer, $this->rootNodeTypeMapping, new NodeDataLoader($this->connection)), 'Exporting sites data' => new SitesExportProcessor(new SiteDataLoader($this->connection), new DomainDataLoader($this->connection)), ]); diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php index 7b9447904a6..3020baf2b2a 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php @@ -105,11 +105,7 @@ public function run(ProcessingContext $context): void continue; } } - try { - $this->processNodeData($context, $nodeDataRow); - } catch (MigrationException $e) { - throw new \RuntimeException($e->getMessage(), 1729506899, $e); - } + $this->processNodeData($context, $nodeDataRow); } // Set References, now when the full import is done. foreach ($this->nodeReferencesWereSetEvents as $nodeReferencesWereSetEvent) { diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index 08199679ae6..59601e38d6f 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -161,13 +161,13 @@ public function createCommand($name, $packageKey, $nodeType, $nodeName = null, $ * This command allows importing sites from the given path/package. The format must * be identical to that produced by the exportAll command. * - * !!! The live workspace has to be empty prior to importing. !!! - * * If a path is specified, this command expects the corresponding directory to contain the exported files * * If a package key is specified, this command expects the export files to be located in the private resources * directory of the given package (Resources/Private/Content). * + * **Note that the live workspace has to be empty prior to importing.** + * * @param string|null $packageKey Package key specifying the package containing the sites content * @param string|null $path relative or absolute path and filename to the export files * @return void @@ -199,7 +199,7 @@ public function importAllCommand(string $packageKey = null, string $path = null, * Export sites * * This command exports all sites of the content repository. - ** + * * If a path is specified, this command creates the directory if needed and exports into that. * * If a package key is specified, this command exports to the private resources diff --git a/README.md b/README.md index ea185d368e3..5df252f9b2e 100644 --- a/README.md +++ b/README.md @@ -70,22 +70,14 @@ You can chose from one of the following options: #### Migrating an existing (Neos < 9.0) Site ``` bash -# WORKAROUND: for now, you still need to create a site (which must match the root node name) -# !! in the future, you would want to import *INTO* a given site (and replace its root node) -./flow site:create neosdemo Neos.Demo Neos.Demo:Document.Homepage - -# the following config points to a Neos 8.0 database (adjust to your needs), created by -# the legacy "./flow site:import Neos.Demo" command. +# the following config points to a Neos 8.0 database (adjust to your needs) ./flow site:migrateLegacyData --config '{"dbal": {"dbname": "neos80"}, "resourcesPath": "/path/to/neos-8.0/Data/Persistent/Resources"}' ``` #### Importing an existing (Neos >= 9.0) Site from an Export ``` bash -# make sure this cr is empty -./flow site:pruneAll -# import the event stream from the Neos.Demo package -./flow site:importAll Packages/Sites/Neos.Demo/Resources/Private/Content +./flow site:importAll --package-key Neos.Demo ``` ### Running Neos From 210a087cd6a580ca45671e85175f0753a4edd9b7 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 8 Nov 2024 20:49:41 +0100 Subject: [PATCH 43/56] TASK: Remove obsolete `ProjectionReplayProcessor` --- .../Processors/ProjectionReplayProcessor.php | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php deleted file mode 100644 index b77110a42a4..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php +++ /dev/null @@ -1,25 +0,0 @@ -projectionService->replayAllProjections(CatchUpOptions::create()); - } -} From c1d64ba3f4ccf01f60fd4f3f20f93a1642ba3ff2 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 10:46:57 +0100 Subject: [PATCH 44/56] BUGFIX: Import `Duplicate entry 'onedimension.localhost' for key 'flow_identity_neos_neos_domain_model_domain` Instead, we will now gracefully handle this case: Domain "onedimension.localhost" already exists. Adding it to site "Neos Test Site". --- .../Domain/Import/SiteCreationProcessor.php | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index fb6d4222e87..3ea256fe587 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -14,7 +14,6 @@ namespace Neos\Neos\Domain\Import; -use JsonException; use Neos\ContentRepository\Core\SharedModel\Node\NodeName; use Neos\ContentRepository\Export\Event\ValueObject\ExportedEvent; use Neos\ContentRepository\Export\ProcessingContext; @@ -49,7 +48,7 @@ public function run(ProcessingContext $context): void $sitesJson = $context->files->read('sites.json'); try { $sites = json_decode($sitesJson, true, 512, JSON_THROW_ON_ERROR); - } catch (JsonException $e) { + } catch (\JsonException $e) { throw new \RuntimeException("Failed to decode sites.json: {$e->getMessage()}", 1729506117, $e); } } else { @@ -59,11 +58,11 @@ public function run(ProcessingContext $context): void /** @var SiteShape $site */ foreach ($sites as $site) { - $context->dispatch(Severity::NOTICE, "Creating site \"{$site['name']}\""); + $context->dispatch(Severity::NOTICE, sprintf('Creating site "%s"', $site['name'])); $siteNodeName = !empty($site['nodeName']) ? NodeName::fromString($site['nodeName']) : NodeName::transliterateFromString($site['name']); if ($this->siteRepository->findOneByNodeName($siteNodeName->value)) { - $context->dispatch(Severity::NOTICE, "Site for node name \"{$siteNodeName->value}\" already exists, skipping"); + $context->dispatch(Severity::NOTICE, sprintf('Site for node name "%s" already exists, skipping', $siteNodeName->value)); continue; } $siteInstance = new Site($siteNodeName->value); @@ -73,13 +72,18 @@ public function run(ProcessingContext $context): void $this->siteRepository->add($siteInstance); $this->persistenceManager->persistAll(); foreach ($site['domains'] ?? [] as $domain) { - $domainInstance = new Domain(); - $domainInstance->setSite($siteInstance); - $domainInstance->setHostname($domain['hostname']); - $domainInstance->setPort($domain['port'] ?? null); - $domainInstance->setScheme($domain['scheme'] ?? null); - $domainInstance->setActive($domain['active'] ?? false); - $this->domainRepository->add($domainInstance); + $domainInstance = $this->domainRepository->findOneByHost($domain['hostname']); + if ($domainInstance) { + $context->dispatch(Severity::NOTICE, sprintf('Domain "%s" already exists. Adding it to site "%s".', $domain['hostname'], $site['name'])); + } else { + $domainInstance = new Domain(); + $domainInstance->setSite($siteInstance); + $domainInstance->setHostname($domain['hostname']); + $domainInstance->setPort($domain['port'] ?? null); + $domainInstance->setScheme($domain['scheme'] ?? null); + $domainInstance->setActive($domain['active'] ?? false); + $this->domainRepository->add($domainInstance); + } if ($domain['primary'] ?? false) { $siteInstance->setPrimaryDomain($domainInstance); $this->siteRepository->update($siteInstance); From 49234ffb1bd3a5f3391120f8eb0be72c6d831952 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 10:49:11 +0100 Subject: [PATCH 45/56] TASK: Migration, make sure that NodeType does not exist error is thrown first --- .../Classes/Processors/EventExportProcessor.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php index 3020baf2b2a..6d547c385bc 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/EventExportProcessor.php @@ -215,15 +215,17 @@ public function processNodeDataWithoutFallbackToEmptyDimension(ProcessingContext $nodeType = $this->nodeTypeManager->getNodeType($nodeTypeName); $isSiteNode = $nodeDataRow['parentpath'] === '/sites'; - if ($isSiteNode && !$nodeType?->isOfType(NodeTypeNameFactory::NAME_SITE)) { - throw new MigrationException(sprintf('The site node "%s" (type: "%s") must be of type "%s"', $nodeDataRow['identifier'], $nodeTypeName->value, NodeTypeNameFactory::NAME_SITE), 1695801620); - } if (!$nodeType) { $context->dispatch(Severity::ERROR, "The node type \"{$nodeTypeName->value}\" is not available. Node: \"{$nodeDataRow['identifier']}\""); return; } + if ($isSiteNode && !$nodeType->isOfType(NodeTypeNameFactory::NAME_SITE)) { + $declaredSuperTypes = array_keys($nodeType->getDeclaredSuperTypes()); + throw new MigrationException(sprintf('The site node "%s" (type: "%s") must be of type "%s". Currently declared super types: "%s"', $nodeDataRow['identifier'], $nodeTypeName->value, NodeTypeNameFactory::NAME_SITE, join(',', $declaredSuperTypes)), 1695801620); + } + $serializedPropertyValuesAndReferences = $this->extractPropertyValuesAndReferences($context, $nodeDataRow, $nodeType); if ($this->isAutoCreatedChildNode($parentNodeAggregate->nodeTypeName, $nodeName) && !$this->visitedNodes->containsNodeAggregate($nodeAggregateId)) { From 5f3d350f0e9caf41ff5bdf96af0f3b49eee4cc56 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 11:53:26 +0100 Subject: [PATCH 46/56] TASK: Enforce file creation first (to throw possibly errors) --- .../Classes/Command/SiteCommandController.php | 2 +- Neos.Neos/Classes/Command/SiteCommandController.php | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php index 8f57f0dc526..11b40881f03 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php @@ -124,6 +124,7 @@ public function migrateLegacyDataCommand(string $contentRepository = 'default', */ public function exportLegacyDataCommand(string $path, string $contentRepository = 'default', string $config = null, bool $verbose = false): void { + Files::createDirectoryRecursively($path); if ($config !== null) { try { $parsedConfig = json_decode($config, true, 512, JSON_THROW_ON_ERROR); @@ -148,7 +149,6 @@ public function exportLegacyDataCommand(string $path, string $contentRepository } $this->verifyDatabaseConnection($connection); - Files::createDirectoryRecursively($path); $legacyExportService = $this->contentRepositoryRegistry->buildService( ContentRepositoryId::fromString($contentRepository), new LegacyExportServiceFactory( diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index 59601e38d6f..c281add9ce3 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -213,9 +213,7 @@ public function exportAllCommand(string $packageKey = null, string $path = null, { $path = $this->determineTargetPath($packageKey, $path); $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - if (file_exists($path) === false) { - Files::createDirectoryRecursively($path); - } + Files::createDirectoryRecursively($path); $this->siteExportService->exportToPath( $contentRepositoryId, $path, From b6c4cd48a350d8cff827cc0ca079cda77de0a0cb Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 11:54:03 +0100 Subject: [PATCH 47/56] TASK: Dont use `findOneByHost` because it returns not the exact result In our case we want to skip the creation of a domain if it exists based on the host name (see c1d64ba3f4ccf01f60fd4f3f20f93a1642ba3ff2) but now we always skip sub domains, if the main domain is created first. --- Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php | 4 ++-- Neos.Neos/Classes/Domain/Repository/DomainRepository.php | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index 3ea256fe587..9f0897ab612 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -72,8 +72,8 @@ public function run(ProcessingContext $context): void $this->siteRepository->add($siteInstance); $this->persistenceManager->persistAll(); foreach ($site['domains'] ?? [] as $domain) { - $domainInstance = $this->domainRepository->findOneByHost($domain['hostname']); - if ($domainInstance) { + $domainInstance = $this->domainRepository->findByHostname($domain['hostname'])->getFirst(); + if ($domainInstance instanceof Domain) { $context->dispatch(Severity::NOTICE, sprintf('Domain "%s" already exists. Adding it to site "%s".', $domain['hostname'], $site['name'])); } else { $domainInstance = new Domain(); diff --git a/Neos.Neos/Classes/Domain/Repository/DomainRepository.php b/Neos.Neos/Classes/Domain/Repository/DomainRepository.php index a86fe3eac93..6b1cb089ef2 100644 --- a/Neos.Neos/Classes/Domain/Repository/DomainRepository.php +++ b/Neos.Neos/Classes/Domain/Repository/DomainRepository.php @@ -84,6 +84,7 @@ public function findByHost($hostname, $onlyActive = false) public function findOneByHost($hostname, $onlyActive = false): ?Domain { $allMatchingDomains = $this->findByHost($hostname, $onlyActive); + // Fixme, requesting `onedimension.localhost` if domain `localhost` exists in the set would return the latter because of `getSortedMatches` return count($allMatchingDomains) > 0 ? $allMatchingDomains[0] : null; } From 6fc88e0b7699f4403d18e12b00035fc67cf149ca Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 11:59:50 +0100 Subject: [PATCH 48/56] BUGFIX: Create site with correct online state (inverse condition) --- Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index 9f0897ab612..a06818c5fb9 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -67,7 +67,7 @@ public function run(ProcessingContext $context): void } $siteInstance = new Site($siteNodeName->value); $siteInstance->setSiteResourcesPackageKey($site['siteResourcesPackageKey']); - $siteInstance->setState(($site['online'] ?? false) ? Site::STATE_OFFLINE : Site::STATE_ONLINE); + $siteInstance->setState($site['online'] ? Site::STATE_ONLINE : Site::STATE_OFFLINE); $siteInstance->setName($site['name']); $this->siteRepository->add($siteInstance); $this->persistenceManager->persistAll(); From 02e9cdea260c3fbaea0baa3b10c7d92c2868d957 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 12:00:11 +0100 Subject: [PATCH 49/56] TASK: Simplify `SiteShape` --- .../Classes/Domain/Export/SiteExportProcessor.php | 2 +- .../Classes/Domain/Import/SiteCreationProcessor.php | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php index f71a91f4c65..64f765847fa 100644 --- a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php +++ b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php @@ -27,7 +27,7 @@ * Export processor exports Neos {@see Site} instances as json * * @phpstan-type DomainShape array{hostname: string, scheme?: ?string, port?: ?int, active?: ?bool, primary?: ?bool } - * @phpstan-type SiteShape array{name:string, siteResourcesPackageKey:string, nodeName?: string, online?:bool, domains?: ?DomainShape[] } + * @phpstan-type SiteShape array{name:string, siteResourcesPackageKey:string, nodeName: string, online:bool, domains: DomainShape[] } * */ final readonly class SiteExportProcessor implements ProcessorInterface diff --git a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php index a06818c5fb9..9c785a8a6d1 100644 --- a/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php +++ b/Neos.Neos/Classes/Domain/Import/SiteCreationProcessor.php @@ -60,7 +60,7 @@ public function run(ProcessingContext $context): void foreach ($sites as $site) { $context->dispatch(Severity::NOTICE, sprintf('Creating site "%s"', $site['name'])); - $siteNodeName = !empty($site['nodeName']) ? NodeName::fromString($site['nodeName']) : NodeName::transliterateFromString($site['name']); + $siteNodeName = NodeName::fromString($site['nodeName']); if ($this->siteRepository->findOneByNodeName($siteNodeName->value)) { $context->dispatch(Severity::NOTICE, sprintf('Site for node name "%s" already exists, skipping', $siteNodeName->value)); continue; @@ -71,7 +71,7 @@ public function run(ProcessingContext $context): void $siteInstance->setName($site['name']); $this->siteRepository->add($siteInstance); $this->persistenceManager->persistAll(); - foreach ($site['domains'] ?? [] as $domain) { + foreach ($site['domains'] as $domain) { $domainInstance = $this->domainRepository->findByHostname($domain['hostname'])->getFirst(); if ($domainInstance instanceof Domain) { $context->dispatch(Severity::NOTICE, sprintf('Domain "%s" already exists. Adding it to site "%s".', $domain['hostname'], $site['name'])); @@ -109,11 +109,16 @@ private static function extractSitesFromEventStream(ProcessingContext $context): continue; } if ($event->type === 'NodeAggregateWithNodeWasCreated' && $event->payload['parentNodeAggregateId'] === $siteRooNodeAggregateId) { + if (!isset($event->payload['nodeName'])) { + throw new \RuntimeException(sprintf('The nodeName of the site node "%s" must not be empty', $event->payload['nodeAggregateId']), 1731236316); + } $sites[] = [ 'siteResourcesPackageKey' => self::extractPackageKeyFromNodeTypeName($event->payload['nodeTypeName']), 'name' => $event->payload['initialPropertyValues']['title']['value'] ?? $event->payload['nodeTypeName'], 'nodeTypeName' => $event->payload['nodeTypeName'], - 'nodeName' => $event->payload['nodeName'] ?? null, + 'nodeName' => $event->payload['nodeName'], + 'domains' => [], + 'online' => true ]; } }; From b11bf2e4a5418e76a28836a397c4db4ce3d83e10 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 12:02:02 +0100 Subject: [PATCH 50/56] TASK: Remove `site:migrateLegacyData` (previously `cr:migrateLegacyData`) replaced with ``` ./flow site:exportLegacyDataCommand --path ./migratedContent ./flow site:importAll --path ./migratedContent ``` --- .../Classes/Command/SiteCommandController.php | 82 ++----------------- README.md | 4 +- 2 files changed, 10 insertions(+), 76 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php index 11b40881f03..6b887ce0a28 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php @@ -25,19 +25,15 @@ use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Cli\CommandController; use Neos\Flow\Property\PropertyMapper; -use Neos\Flow\Utility\Environment; -use Neos\Neos\Domain\Service\SiteImportService; -use Neos\Utility\Files; use Neos\Neos\Domain\Service\NodeTypeNameFactory; +use Neos\Utility\Files; class SiteCommandController extends CommandController { public function __construct( private readonly Connection $connection, - private readonly Environment $environment, private readonly PropertyMapper $propertyMapper, private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - private readonly SiteImportService $siteImportService, ) { parent::__construct(); } @@ -45,81 +41,17 @@ public function __construct( /** * Migrate from the Legacy CR * - * Note that the dimension configuration and the node type schema must be migrated of the content repository to import to and it must be setup. + * This command creates a Neos 9 export format based on the data from the specified legacy content repository database connection + * The export will be placed in the specified directory path, and can be imported via "site:importAll": * - * @param string $contentRepository The target content repository that will be used for importing into - * @param string|null $config JSON encoded configuration, for example '{"dbal": {"dbname": "some-other-db"}, "resourcesPath": "/some/absolute/path", "rootNodes": {"/sites": "Neos.Neos:Sites", "/other": "My.Package:SomeOtherRoot"}}' - * @throws \Exception - */ - public function migrateLegacyDataCommand(string $contentRepository = 'default', string $config = null, bool $verbose = false, ): void - { - if ($config !== null) { - try { - $parsedConfig = json_decode($config, true, 512, JSON_THROW_ON_ERROR); - } catch (\JsonException $e) { - throw new \InvalidArgumentException(sprintf('Failed to parse --config parameter: %s', $e->getMessage()), 1659526855, $e); - } - $resourcesPath = $parsedConfig['resourcesPath'] ?? self::defaultResourcesPath(); - $rootNodes = isset($parsedConfig['rootNodes']) ? RootNodeTypeMapping::fromArray($parsedConfig['rootNodes']) : $this->getDefaultRootNodes(); - try { - $connection = isset($parsedConfig['dbal']) ? DriverManager::getConnection(array_merge($this->connection->getParams(), $parsedConfig['dbal']), new Configuration()) : $this->connection; - } catch (DBALException $e) { - throw new \InvalidArgumentException(sprintf('Failed to get database connection, check the --config parameter: %s', $e->getMessage()), 1659527201, $e); - } - } else { - $resourcesPath = $this->determineResourcesPath(); - $rootNodes = $this->getDefaultRootNodes(); - if (!$this->output->askConfirmation(sprintf('Do you want to migrate nodes from the current database "%s@%s" (y/n)? ', $this->connection->getParams()['dbname'] ?? '?', $this->connection->getParams()['host'] ?? '?'))) { - $connection = $this->adjustDatabaseConnection($this->connection); - } else { - $connection = $this->connection; - } - } - $this->verifyDatabaseConnection($connection); - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $temporaryFilePath = $this->environment->getPathToTemporaryDirectory() . uniqid('Export', true); - Files::createDirectoryRecursively($temporaryFilePath); - - $legacyExportService = $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new LegacyExportServiceFactory( - $connection, - $resourcesPath, - $this->propertyMapper, - $rootNodes - ) - ); - - $legacyExportService->exportToPath( - $temporaryFilePath, - $this->createOnProcessorClosure(), - $this->createOnMessageClosure($verbose) - ); - - $this->outputLine('Migrated data. Importing into new content repository ...'); - - // todo check if cr is setup before!!! do not fail here!!! - $this->siteImportService->importFromPath( - $contentRepositoryId, - $temporaryFilePath, - $this->createOnProcessorClosure(), - $this->createOnMessageClosure($verbose) - ); - - Files::unlink($temporaryFilePath); - - $this->outputLine('Done'); - } - - /** - * Export from the Legacy CR into a specified directory path + * ./flow site:exportLegacyDataCommand --path ./migratedContent + * ./flow site:importAll --path ./migratedContent * * Note that the dimension configuration and the node type schema must be migrated of the reference content repository * * @param string $contentRepository The reference content repository that can later be used for importing into - * @param string $path The path to the directory, will be created if missing - * @param string|null $config JSON encoded configuration, for example '{"dbal": {"dbname": "some-other-db"}, "resourcesPath": "/some/absolute/path", "rootNodes": {"/sites": "Neos.Neos:Sites", "/other": "My.Package:SomeOtherRoot"}}' + * @param string $path The path to the directory to export to, will be created if missing + * @param string|null $config JSON encoded configuration, for example --config '{"dbal": {"dbname": "some-other-db"}, "resourcesPath": "/absolute-path/Data/Persistent/Resources", "rootNodes": {"/sites": "Neos.Neos:Sites", "/other": "My.Package:SomeOtherRoot"}}' * @throws \Exception */ public function exportLegacyDataCommand(string $path, string $contentRepository = 'default', string $config = null, bool $verbose = false): void diff --git a/README.md b/README.md index 5df252f9b2e..6b4d49d0d3c 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,9 @@ You can chose from one of the following options: ``` bash # the following config points to a Neos 8.0 database (adjust to your needs) -./flow site:migrateLegacyData --config '{"dbal": {"dbname": "neos80"}, "resourcesPath": "/path/to/neos-8.0/Data/Persistent/Resources"}' +./flow site:exportLegacyDataCommand --path ./migratedContent --config '{"dbal": {"dbname": "neos80"}, "resourcesPath": "/path/to/neos-8.0/Data/Persistent/Resources"}' +# import the migrated data +./flow site:importAll --path ./migratedContent ``` #### Importing an existing (Neos >= 9.0) Site from an Export From 41e472af735a37a34d766ad1d5d3736a336542b1 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 13:04:52 +0100 Subject: [PATCH 51/56] BUGFIX: Export correctly export `primary` domain previously `primary` was always true if there is only one domain --- .../Classes/Processors/SitesExportProcessor.php | 2 +- Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php | 2 +- Neos.Neos/Classes/Domain/Model/Site.php | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php index 9b66a0f0f94..0f2510586a3 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Processors/SitesExportProcessor.php @@ -52,7 +52,7 @@ function(array $domainRow) use ($siteRow) { 'hostname' => $domainRow['hostname'], 'scheme' => $domainRow['scheme'], 'port' => $domainRow['port'], - 'active' => $domainRow['active'], + 'active' => (bool)$domainRow['active'], 'primary' => $domainRow['persistence_object_identifier'] === $siteRow['primarydomain'], ]; }, diff --git a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php index 64f765847fa..a0c4beb5b53 100644 --- a/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php +++ b/Neos.Neos/Classes/Domain/Export/SiteExportProcessor.php @@ -67,7 +67,7 @@ private function getSiteData(): array 'scheme' => $domain->getScheme(), 'port' => $domain->getPort(), 'active' => $domain->getActive(), - 'primary' => $domain === $site->getPrimaryDomain(), + 'primary' => $domain === $site->getPrimaryDomain(fallbackToActive: false), ], $site->getDomains()->toArray() ) diff --git a/Neos.Neos/Classes/Domain/Model/Site.php b/Neos.Neos/Classes/Domain/Model/Site.php index eb6cedae3c0..98d3cf8f7c5 100644 --- a/Neos.Neos/Classes/Domain/Model/Site.php +++ b/Neos.Neos/Classes/Domain/Model/Site.php @@ -331,11 +331,15 @@ public function setPrimaryDomain(Domain $domain = null) /** * Returns the primary domain, if one has been defined. * + * @param boolean $fallbackToActive if true falls back to the first active domain instead returning null if no primary domain was explicitly set * @return ?Domain The primary domain or NULL * @api */ - public function getPrimaryDomain(): ?Domain + public function getPrimaryDomain(bool $fallbackToActive = true): ?Domain { + if (!$fallbackToActive) { + return $this->primaryDomain; + } return $this->primaryDomain instanceof Domain && $this->primaryDomain->getActive() ? $this->primaryDomain : $this->getFirstActiveDomain(); From aa713471dec285d4123a3befc4a9371efde78b49 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 13:05:40 +0100 Subject: [PATCH 52/56] BUGFIX: `site:importAll` import `copyrightNotice` of assets previously we didnt use the value that was in the export format. --- .../src/Processors/AssetRepositoryImportProcessor.php | 1 + 1 file changed, 1 insertion(+) diff --git a/Neos.ContentRepository.Export/src/Processors/AssetRepositoryImportProcessor.php b/Neos.ContentRepository.Export/src/Processors/AssetRepositoryImportProcessor.php index 2021098fb19..c7f7535069f 100644 --- a/Neos.ContentRepository.Export/src/Processors/AssetRepositoryImportProcessor.php +++ b/Neos.ContentRepository.Export/src/Processors/AssetRepositoryImportProcessor.php @@ -101,6 +101,7 @@ private function importAsset(ProcessingContext $context, StorageAttributes $file ObjectAccess::setProperty($asset, 'Persistence_Object_Identifier', $serializedAsset->identifier, true); $asset->setTitle($serializedAsset->title); $asset->setCaption($serializedAsset->caption); + $asset->setCopyrightNotice($serializedAsset->copyrightNotice); $this->assetRepository->add($asset); $this->persistenceManager->persistAll(); } From c4541173cd3587691b20196ef2f4dab3f3289261 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 13:30:43 +0100 Subject: [PATCH 53/56] TASK: Trivial cosmetic changes --- .../Classes/Command/SiteCommandController.php | 2 +- Neos.Neos/Classes/Command/SiteCommandController.php | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php index 6b887ce0a28..c29da54c1d7 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php @@ -168,7 +168,7 @@ protected function createOnMessageClosure(bool $verbose): \Closure } $this->outputLine(match ($severity) { Severity::NOTICE => $message, - Severity::WARNING => sprintf('Warning: %s', $message), + Severity::WARNING => sprintf('Warning: %s', $message), Severity::ERROR => sprintf('Error: %s', $message), }); }; diff --git a/Neos.Neos/Classes/Command/SiteCommandController.php b/Neos.Neos/Classes/Command/SiteCommandController.php index c281add9ce3..24037d9b95e 100644 --- a/Neos.Neos/Classes/Command/SiteCommandController.php +++ b/Neos.Neos/Classes/Command/SiteCommandController.php @@ -174,10 +174,11 @@ public function createCommand($name, $packageKey, $nodeType, $nodeName = null, $ */ public function importAllCommand(string $packageKey = null, string $path = null, string $contentRepository = 'default', bool $verbose = false): void { + // TODO check if this warning is still necessary with Neos 9 // Since this command uses a lot of memory when large sites are imported, we warn the user to watch for // the confirmation of a successful import. $this->outputLine('This command can use a lot of memory when importing sites with many resources.'); - $this->outputLine('If the import is successful, you will see a message saying "Import of site ... finished".'); + $this->outputLine('If the import is successful, you will see a message saying "Import finished".'); $this->outputLine('If you do not see this message, the import failed, most likely due to insufficient memory.'); $this->outputLine('Increase the memory_limit configuration parameter of your php CLI to attempt to fix this.'); $this->outputLine('Starting import...'); @@ -193,6 +194,8 @@ public function importAllCommand(string $packageKey = null, string $path = null, $this->createOnProcessorClosure(), $this->createOnMessageClosure($verbose) ); + + $this->outputLine('Import finished.'); } /** @@ -230,7 +233,7 @@ public function exportAllCommand(string $packageKey = null, string $path = null, */ public function pruneAllCommand(string $contentRepository = 'default', bool $force = false, bool $verbose = false): void { - if (!$force && !$this->output->askConfirmation(sprintf('> This will prune your content repository "%s". Are you sure to proceed? (y/n) ', $contentRepository), false)) { + if (!$force && !$this->output->askConfirmation(sprintf('> This will prune your content repository "%s" and all its attached sites. Are you sure to proceed? (y/n) ', $contentRepository), false)) { $this->outputLine('Abort.'); return; } @@ -367,7 +370,7 @@ protected function createOnMessageClosure(bool $verbose): \Closure } $this->outputLine(match ($severity) { Severity::NOTICE => $message, - Severity::WARNING => sprintf('Warning: %s', $message), + Severity::WARNING => sprintf('Warning: %s', $message), Severity::ERROR => sprintf('Error: %s', $message), }); }; From b8670cf6fbbf69a94339a239bd8b017d7f541a91 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 13:31:44 +0100 Subject: [PATCH 54/56] TASK: Make pruning a noop if the workspace does not exist --- .../Domain/Pruning/SitePruningProcessor.php | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php index fcb6b9c7a89..d393d13f828 100644 --- a/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/SitePruningProcessor.php @@ -15,9 +15,12 @@ namespace Neos\Neos\Domain\Pruning; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; +use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; +use Neos\ContentRepository\Export\Severity; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Model\Site; use Neos\Neos\Domain\Repository\DomainRepository; @@ -40,7 +43,13 @@ public function __construct( public function run(ProcessingContext $context): void { - $sites = $this->findAllSites(); + try { + $contentGraph = $this->contentRepository->getContentGraph($this->workspaceName); + } catch (WorkspaceDoesNotExist) { + $context->dispatch(Severity::NOTICE, sprintf('Could not find any matching sites, because the workspace "%s" does not exist.', $this->workspaceName->value)); + return; + } + $sites = $this->findAllSites($contentGraph); foreach ($sites as $site) { $domains = $site->getDomains(); if ($site->getPrimaryDomain() !== null) { @@ -59,9 +68,8 @@ public function run(ProcessingContext $context): void /** * @return Site[] */ - protected function findAllSites(): array + protected function findAllSites(ContentGraphInterface $contentGraph): array { - $contentGraph = $this->contentRepository->getContentGraph($this->workspaceName); $sitesNodeAggregate = $contentGraph->findRootNodeAggregateByType(NodeTypeNameFactory::forSites()); if ($sitesNodeAggregate === null) { return []; From 17db2eb1f134edb551bc837b292d1509a7a60f95 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 13:41:33 +0100 Subject: [PATCH 55/56] TASK: Adjust documentation to reference correct command --- .../Classes/Command/SiteCommandController.php | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php index c29da54c1d7..9f2cc2d4a93 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/Command/SiteCommandController.php @@ -44,7 +44,7 @@ public function __construct( * This command creates a Neos 9 export format based on the data from the specified legacy content repository database connection * The export will be placed in the specified directory path, and can be imported via "site:importAll": * - * ./flow site:exportLegacyDataCommand --path ./migratedContent + * ./flow site:exportLegacyData --path ./migratedContent * ./flow site:importAll --path ./migratedContent * * Note that the dimension configuration and the node type schema must be migrated of the reference content repository diff --git a/README.md b/README.md index 6b4d49d0d3c..4183dd4275c 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ You can chose from one of the following options: ``` bash # the following config points to a Neos 8.0 database (adjust to your needs) -./flow site:exportLegacyDataCommand --path ./migratedContent --config '{"dbal": {"dbname": "neos80"}, "resourcesPath": "/path/to/neos-8.0/Data/Persistent/Resources"}' +./flow site:exportLegacyData --path ./migratedContent --config '{"dbal": {"dbname": "neos80"}, "resourcesPath": "/path/to/neos-8.0/Data/Persistent/Resources"}' # import the migrated data ./flow site:importAll --path ./migratedContent ``` From 5696f369130fa20853011945f9f1f3725645cf2d Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 10 Nov 2024 13:43:46 +0100 Subject: [PATCH 56/56] TASK: Adjust test to 49234ffb1bd3a5f3391120f8eb0be72c6d831952 --- .../Tests/Behavior/Features/Errors.feature | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature index c41e915c647..3dc37a50acc 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature @@ -5,6 +5,7 @@ Feature: Exceptional cases during migrations Given using no content dimensions And using the following node types: """yaml + 'unstructured': {} 'Neos.Neos:Site': {} 'Some.Package:Homepage': superTypes: @@ -146,5 +147,5 @@ Feature: Exceptional cases during migrations And I run the event migration Then I expect a migration exception with the message """ - The site node "site-node-id" (type: "unstructured") must be of type "Neos.Neos:Site" + The site node "site-node-id" (type: "unstructured") must be of type "Neos.Neos:Site". Currently declared super types: "" """