From c7637e8055c1dff22bcf845d8a1e5ec426df0faa Mon Sep 17 00:00:00 2001 From: Sebastian Helzle Date: Wed, 26 Jun 2024 10:58:16 +0200 Subject: [PATCH] TASK: Speedup nodedata requests by prefetching all childnodes This way the followup methods that add the childnodes to the resulting noderesults can directly read the children from the 1st level cache. The speedup is ~50% (1s -> 500ms) in my tests for 175 nodes. --- .../ContentRepository/Service/NodeService.php | 233 ++++++++++++++++++ .../Controller/BackendServiceController.php | 9 +- 2 files changed, 239 insertions(+), 3 deletions(-) diff --git a/Classes/ContentRepository/Service/NodeService.php b/Classes/ContentRepository/Service/NodeService.php index 49be79d31e..f9cbd2a4c4 100644 --- a/Classes/ContentRepository/Service/NodeService.php +++ b/Classes/ContentRepository/Service/NodeService.php @@ -52,6 +52,12 @@ class NodeService */ protected array $contextCache = []; + /** + * @Flow\InjectConfiguration(path="nodeTypeRoles.ignored", package="Neos.Neos.Ui") + * @var string + */ + protected $ignoredNodeTypeRole; + /** * Helper method to retrieve the closest document for a node * @@ -134,6 +140,233 @@ public function getNodeFromContextPath($contextPath, Site $site = null, Domain $ return $context->getNode($nodePath); } + /** + * Converts given context paths to a node objects + * + * @param string[] $nodeContextPaths + * @return NodeInterface[]|Error + */ + public function getNodesFromContextPaths(array $nodeContextPaths, Site $site = null, Domain $domain = null, $includeAll = false): array|Error + { + if (!$nodeContextPaths) { + return []; + } + + $nodePaths = array_map(static function($nodeContextPath) { + return NodePaths::explodeContextPath($nodeContextPath)['nodePath']; + }, $nodeContextPaths); + + $nodePathAndContext = NodePaths::explodeContextPath($nodeContextPaths[0]); + $nodePath = $nodePathAndContext['nodePath']; + $workspaceName = $nodePathAndContext['workspaceName']; + $dimensions = $nodePathAndContext['dimensions']; + $siteNodeName = explode('/', $nodePath)[2]; + $contextProperties = $this->prepareContextProperties($workspaceName, $dimensions); + + if ($site === null) { + $site = $this->siteRepository->findOneByNodeName($siteNodeName); + } + + if ($domain === null) { + $domain = $this->domainRepository->findOneBySite($site); + } + + $contextProperties['currentSite'] = $site; + $contextProperties['currentDomain'] = $domain; + if ($includeAll === true) { + $contextProperties['invisibleContentShown'] = true; + $contextProperties['removedContentShown'] = true; + $contextProperties['inaccessibleContentShown'] = true; + } + $context = $this->contextFactory->create($contextProperties); + + $workspace = $context->getWorkspace(false); + if (!$workspace) { + return new Error( + sprintf('Could not convert the given source to Node object because the workspace "%s" as specified in the context node path does not exist.', $workspaceName), + 1451392329 + ); + } + + // Query nodes and their variants from the database + $queryBuilder = $this->entityManager->createQueryBuilder(); + $workspaces = $this->collectWorkspaceAndAllBaseWorkspaces($workspace); + $workspacesNames = array_map(static function(Workspace $workspace) { return $workspace->getName(); }, $workspaces); + + // Filter by workspace and its parents + $queryBuilder->select('n') + ->from(NodeData::class, 'n') + ->where('n.workspace IN (:workspaces)') + ->andWhere('n.movedTo IS NULL') + ->andWhere('n.path IN (:nodePaths)') + ->setParameter('workspaces', $workspacesNames) + ->setParameter('nodePaths', $nodePaths); + $query = $queryBuilder->getQuery(); + $nodeDataWithVariants = $query->getResult(); + + // Remove node duplicates + $reducedNodeData = $this->reduceNodeVariantsByWorkspacesAndDimensions($nodeDataWithVariants, $workspaces, $dimensions); + + // Convert nodedata objects to nodes + return array_reduce($reducedNodeData, function (array $carry, NodeData $nodeData) use ($context) { + $node = $this->nodeFactory->createFromNodeData($nodeData, $context); + if ($node !== null) { + $carry[] = $node; + } + $context->getFirstLevelNodeCache()->setByPath($node->getPath(), $node); + return $carry; + }, []); + } + + /** + * @param NodeInterface[] $parentNodes + */ + public function preloadChildNodesForNodes(array $parentNodes): void + { + if (empty($parentNodes)) { + return; + } + + $workspace = $parentNodes[0]->getWorkspace(); + $context = $parentNodes[0]->getContext(); + $dimensions = $context->getDimensions(); + + $parentPaths = array_map(static function(NodeInterface $parentNode) { + return $parentNode->getPath(); + }, $parentNodes); + + // Query nodes and their variants from the database + $queryBuilder = $this->entityManager->createQueryBuilder(); + $workspaces = $this->collectWorkspaceAndAllBaseWorkspaces($workspace); + $workspacesNames = array_map(static function(Workspace $workspace) { return $workspace->getName(); }, $workspaces); + + // Filter by workspace and its parents + $queryBuilder->select('n') + ->from(NodeData::class, 'n') + ->where('n.workspace IN (:workspaces)') + ->andWhere('n.movedTo IS NULL') + ->andWhere('n.parentPath IN (:parentPaths)') + ->setParameter('workspaces', $workspacesNames) + ->setParameter('parentPaths', $parentPaths); + $query = $queryBuilder->getQuery(); + $nodeDataWithVariants = $query->getResult(); + + // Remove node duplicates + $reducedNodeData = $this->reduceNodeVariantsByWorkspacesAndDimensions( + $nodeDataWithVariants, + $workspaces, + $dimensions + ); + + // Convert nodedata objects to nodes and group them by parent path + $childNodesByParentPath = array_reduce($reducedNodeData, function (array $carry, NodeData $nodeData) use ($context) { + $node = $this->nodeFactory->createFromNodeData($nodeData, $context); + if ($node !== null) { + if (!isset($carry[$node->getParentPath()])) { + $carry[$node->getParentPath()] = [$node]; + } else { + $carry[$node->getParentPath()][] = $node; + } + } + return $carry; + }, []); + + foreach ($childNodesByParentPath as $parentPath => $childNodes) { + usort($childNodes, static function(NodeInterface $a, NodeInterface $b) { + return $a->getIndex() <=> $b->getIndex(); + }); + $context->getFirstLevelNodeCache()->setChildNodesByPathAndNodeTypeFilter( + $parentPath, '!' . + $this->ignoredNodeTypeRole, + $childNodes + ); + } + } + + /** + * Given an array with duplicate nodes (from different workspaces and dimensions) those are reduced to uniqueness (by node identifier) + * Copied from Neos\ContentRepository\Domain\Repository\NodeDataRepository + * + * @param NodeData[] $nodes NodeData result with multiple and duplicate identifiers (different nodes and redundant results for node variants with different dimensions) + * @param Workspace[] $workspaces + * @param array $dimensions + * @return NodeData[] Array of unique node results indexed by identifier + */ + protected function reduceNodeVariantsByWorkspacesAndDimensions(array $nodes, array $workspaces, array $dimensions): array + { + $reducedNodes = []; + + $minimalDimensionPositionsByIdentifier = []; + + $workspaceNames = array_map(static fn (Workspace $workspace) => $workspace->getName(), $workspaces); + + foreach ($nodes as $node) { + $nodeDimensions = $node->getDimensionValues(); + + // Find the position of the workspace, a smaller value means more priority + $workspacePosition = array_search($node->getWorkspace()->getName(), $workspaceNames); + if ($workspacePosition === false) { + throw new \Exception(sprintf( + 'Node workspace "%s" not found in allowed workspaces (%s), this could result from a detached workspace entity in the context.', + $node->getWorkspace()->getName(), + implode(', ', $workspaceNames) + ), 1718740117); + } + + // Find positions in dimensions, add workspace in front for highest priority + $dimensionPositions = []; + + // Special case for no dimensions + if ($dimensions === []) { + // We can just decide if the given node has no dimensions. + $dimensionPositions[] = ($nodeDimensions === []) ? 0 : 1; + } + + foreach ($dimensions as $dimensionName => $dimensionValues) { + if (isset($nodeDimensions[$dimensionName])) { + foreach ($nodeDimensions[$dimensionName] as $nodeDimensionValue) { + $position = array_search($nodeDimensionValue, $dimensionValues); + if ($position === false) { + $position = PHP_INT_MAX; + } + $dimensionPositions[$dimensionName] = isset($dimensionPositions[$dimensionName]) ? min( + $dimensionPositions[$dimensionName], + $position + ) : $position; + } + } else { + $dimensionPositions[$dimensionName] = isset($dimensionPositions[$dimensionName]) ? min( + $dimensionPositions[$dimensionName], + PHP_INT_MAX + ) : PHP_INT_MAX; + } + } + $dimensionPositions[] = $workspacePosition; + + $identifier = $node->getIdentifier(); + // Yes, it seems to work comparing arrays that way! + if (!isset($minimalDimensionPositionsByIdentifier[$identifier]) || $dimensionPositions < $minimalDimensionPositionsByIdentifier[$identifier]) { + $reducedNodes[$identifier] = $node; + $minimalDimensionPositionsByIdentifier[$identifier] = $dimensionPositions; + } + } + + return $reducedNodes; + } + + /** + * @return Workspace[] + */ + protected function collectWorkspaceAndAllBaseWorkspaces(Workspace $workspace): array + { + $workspaces = []; + while ($workspace !== null) { + $workspaces[] = $workspace; + $workspace = $workspace->getBaseWorkspace(); + } + return $workspaces; + } + /** * Checks if the given node exists in the given workspace * diff --git a/Classes/Controller/BackendServiceController.php b/Classes/Controller/BackendServiceController.php index 498a444d8a..9d9aff3f04 100644 --- a/Classes/Controller/BackendServiceController.php +++ b/Classes/Controller/BackendServiceController.php @@ -545,25 +545,28 @@ function ($contextPath) { $flowQuery = call_user_func_array([$flowQuery, $operation['type']], $operation['payload']); } + $nodes = array_filter($flowQuery->get()); + $this->nodeService->preloadChildNodesForNodes(array_values($nodes)); + $nodeInfoHelper = new NodeInfoHelper(); $result = []; switch ($finisher['type']) { case 'get': $result = $nodeInfoHelper->renderNodes( - array_filter($flowQuery->get()), + $nodes, $this->getControllerContext() ); break; case 'getForTree': $result = $nodeInfoHelper->renderNodes( - array_filter($flowQuery->get()), + $nodes, $this->getControllerContext(), true ); break; case 'getForTreeWithParents': $result = $nodeInfoHelper->renderNodesWithParents( - array_filter($flowQuery->get()), + $nodes, $this->getControllerContext() ); break;