Merge 2.17.x into 3.0.x

UPGRADE.md

@@ -666,6 +666,15 @@ Use `toIterable()` instead.
 
 # Upgrade to 2.17
 
+## Deprecate annotations classes for named queries
+
+The following classes have been deprecated:
+
+* `Doctrine\ORM\Mapping\NamedNativeQueries`
+* `Doctrine\ORM\Mapping\NamedNativeQuery`
+* `Doctrine\ORM\Mapping\NamedQueries`
+* `Doctrine\ORM\Mapping\NamedQuery`
+
 ## Deprecate `Doctrine\ORM\Query\Exec\AbstractSqlExecutor::_sqlStatements`
 
 Use `Doctrine\ORM\Query\Exec\AbstractSqlExecutor::sqlStatements` instead.

docs/en/reference/working-with-objects.rst

@@ -711,6 +711,23 @@ and these associations are mapped as EAGER, they will automatically
 be loaded together with the entity being queried and is thus
 immediately available to your application.
 
+Eager Loading can also be configured at runtime through
+``AbstractQuery::setFetchMode`` in DQL or Native Queries.
+
+Eager loading for many-to-one and one-to-one associations is using either a
+LEFT JOIN or a second query for fetching the related entity eagerly.
+
+Eager loading for many-to-one associations uses a second query to load
+the collections for several entities at the same time.
+
+When many-to-many, one-to-one or one-to-many associations are eagerly loaded,
+then the global batch size configuration is used to avoid IN(?) queries with
+too many arguments. The default batch size is 100 and can be changed with
+``Configuration::setEagerFetchBatchSize()``.
+
+For eagerly loaded Many-To-Many associations one query has to be made for each
+collection.
+
 By Lazy Loading
 ~~~~~~~~~~~~~~~
 

lib/Doctrine/ORM/Configuration.php

@@ -636,4 +636,14 @@ public function isRejectIdCollisionInIdentityMapEnabled(): bool
     {
         return true;
     }
+
+    public function setEagerFetchBatchSize(int $batchSize = 100): void
+    {
+        $this->attributes['fetchModeSubselectBatchSize'] = $batchSize;
+    }
+
+    public function getEagerFetchBatchSize(): int
+    {
+        return $this->attributes['fetchModeSubselectBatchSize'] ?? 100;
+    }
 }

lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php

@@ -37,6 +37,7 @@
 use Doctrine\ORM\Repository\Exception\InvalidFindByCall;
 use Doctrine\ORM\UnitOfWork;
 use Doctrine\ORM\Utility\IdentifierFlattener;
+use Doctrine\ORM\Utility\LockSqlHelper;
 use Doctrine\ORM\Utility\PersisterHelper;
 use LengthException;
 
@@ -96,6 +97,8 @@
  */
 class BasicEntityPersister implements EntityPersister
 {
+    use LockSqlHelper;
+
     /** @var array<string,string> */
     private static array $comparisonMap = [
         Comparison::EQ          => '= %s',
@@ -1074,8 +1077,8 @@ public function getSelectSQL(
             : $this->getSelectConditionSQL($criteria, $assoc);
 
         $lockSql = match ($lockMode) {
-            LockMode::PESSIMISTIC_READ => ' ' . $this->platform->getReadLockSQL(),
-            LockMode::PESSIMISTIC_WRITE => ' ' . $this->platform->getWriteLockSQL(),
+            LockMode::PESSIMISTIC_READ => ' ' . $this->getReadLockSQL($this->platform),
+            LockMode::PESSIMISTIC_WRITE => ' ' . $this->getWriteLockSQL($this->platform),
             default => '',
         };
 
@@ -1217,7 +1220,7 @@ protected function getSelectColumnsSQL(): string
             }
 
             $isAssocToOneInverseSide = $assoc->isToOne() && ! $assoc->isOwningSide();
-            $isAssocFromOneEager     = ! $assoc->isManyToMany() && $assoc->fetch === ClassMetadata::FETCH_EAGER;
+            $isAssocFromOneEager     = $assoc->isToOne() && $assoc->fetch === ClassMetadata::FETCH_EAGER;
 
             if (! ($isAssocFromOneEager || $isAssocToOneInverseSide)) {
                 continue;
@@ -1505,8 +1508,8 @@ public function lock(array $criteria, LockMode|int $lockMode): void
         $conditionSql = $this->getSelectConditionSQL($criteria);
 
         $lockSql = match ($lockMode) {
-            LockMode::PESSIMISTIC_READ => $this->platform->getReadLockSQL(),
-            LockMode::PESSIMISTIC_WRITE => $this->platform->getWriteLockSQL(),
+            LockMode::PESSIMISTIC_READ => $this->getReadLockSQL($this->platform),
+            LockMode::PESSIMISTIC_WRITE => $this->getWriteLockSQL($this->platform),
             default => '',
         };
 

lib/Doctrine/ORM/Persisters/Entity/JoinedSubclassPersister.php

@@ -11,6 +11,7 @@
 use Doctrine\ORM\Internal\SQLResultCasing;
 use Doctrine\ORM\Mapping\AssociationMapping;
 use Doctrine\ORM\Mapping\ClassMetadata;
+use Doctrine\ORM\Utility\LockSqlHelper;
 use Doctrine\ORM\Utility\PersisterHelper;
 use LengthException;
 
@@ -27,6 +28,7 @@
  */
 class JoinedSubclassPersister extends AbstractEntityInheritancePersister
 {
+    use LockSqlHelper;
     use SQLResultCasing;
 
     /**
@@ -279,12 +281,12 @@ public function getSelectSQL(
 
         switch ($lockMode) {
             case LockMode::PESSIMISTIC_READ:
-                $lockSql = ' ' . $this->platform->getReadLockSQL();
+                $lockSql = ' ' . $this->getReadLockSQL($this->platform);
 
                 break;
 
             case LockMode::PESSIMISTIC_WRITE:
-                $lockSql = ' ' . $this->platform->getWriteLockSQL();
+                $lockSql = ' ' . $this->getWriteLockSQL($this->platform);
 
                 break;
         }

lib/Doctrine/ORM/Query/QueryException.php

@@ -117,6 +117,14 @@ public static function iterateWithFetchJoinNotAllowed(AssociationMapping $assoc)
         );
     }
 
+    public static function eagerFetchJoinWithNotAllowed(string $sourceEntity, string $fieldName): self
+    {
+        return new self(
+            'Associations with fetch-mode=EAGER may not be using WITH conditions in
+             "' . $sourceEntity . '#' . $fieldName . '".',
+        );
+    }
+
     public static function iterateWithMixedResultNotAllowed(): self
     {
         return new self('Iterating a query with mixed results (using scalars) is not supported.');

lib/Doctrine/ORM/Query/SqlWalker.php

@@ -15,6 +15,7 @@
 use Doctrine\ORM\OptimisticLockException;
 use Doctrine\ORM\Query;
 use Doctrine\ORM\Utility\HierarchyDiscriminatorResolver;
+use Doctrine\ORM\Utility\LockSqlHelper;
 use Doctrine\ORM\Utility\PersisterHelper;
 use InvalidArgumentException;
 use LogicException;
@@ -46,6 +47,8 @@
  */
 class SqlWalker
 {
+    use LockSqlHelper;
+
     public const HINT_DISTINCT = 'doctrine.distinct';
 
     private readonly ResultSetMapping $rsm;
@@ -475,11 +478,11 @@ public function walkSelectStatement(AST\SelectStatement $selectStatement): strin
         }
 
         if ($lockMode === LockMode::PESSIMISTIC_READ) {
-            return $sql . ' ' . $this->platform->getReadLockSQL();
+            return $sql . ' ' . $this->getReadLockSQL($this->platform);
         }
 
         if ($lockMode === LockMode::PESSIMISTIC_WRITE) {
-            return $sql . ' ' . $this->platform->getWriteLockSQL();
+            return $sql . ' ' . $this->getWriteLockSQL($this->platform);
         }
 
         if ($lockMode !== LockMode::OPTIMISTIC) {
@@ -891,7 +894,9 @@ public function walkJoinAssociationDeclaration(
             }
         }
 
-        $targetTableJoin = null;
+        if ($relation->fetch === ClassMetadata::FETCH_EAGER && $condExpr !== null) {
+            throw QueryException::eagerFetchJoinWithNotAllowed($assoc->sourceEntity, $assoc->fieldName);
+        }
 
         // This condition is not checking ClassMetadata::MANY_TO_ONE, because by definition it cannot
         // be the owning side and previously we ensured that $assoc is always the owning side of the associations.

lib/Doctrine/ORM/UnitOfWork.php

@@ -33,6 +33,7 @@
 use Doctrine\ORM\Mapping\AssociationMapping;
 use Doctrine\ORM\Mapping\ClassMetadata;
 use Doctrine\ORM\Mapping\MappingException;
+use Doctrine\ORM\Mapping\ToManyInverseSideMapping;
 use Doctrine\ORM\Persisters\Collection\CollectionPersister;
 use Doctrine\ORM\Persisters\Collection\ManyToManyPersister;
 use Doctrine\ORM\Persisters\Collection\OneToManyPersister;
@@ -50,6 +51,7 @@
 use Throwable;
 use UnexpectedValueException;
 
+use function array_chunk;
 use function array_combine;
 use function array_diff_key;
 use function array_filter;
@@ -292,6 +294,9 @@ class UnitOfWork implements PropertyChangedListener
      */
     private array $eagerLoadingEntities = [];
 
+    /** @var array<string, array<string, mixed>> */
+    private array $eagerLoadingCollections = [];
+
     protected bool $hasCache = false;
 
     /**
@@ -2218,6 +2223,7 @@ public function clear(): void
         $this->pendingCollectionElementRemovals =
         $this->visitedCollections               =
         $this->eagerLoadingEntities             =
+        $this->eagerLoadingCollections          =
         $this->orphanRemovals                   = [];
 
         if ($this->evm->hasListeners(Events::onClear)) {
@@ -2352,6 +2358,10 @@ public function createEntity(string $className, array $data, array &$hints = [])
                 continue;
             }
 
+            if (! isset($hints['fetchMode'][$class->name][$field])) {
+                $hints['fetchMode'][$class->name][$field] = $assoc->fetch;
+            }
+
             $targetClass = $this->em->getClassMetadata($assoc->targetEntity);
 
             switch (true) {
@@ -2416,10 +2426,6 @@ public function createEntity(string $className, array $data, array &$hints = [])
                         break;
                     }
 
-                    if (! isset($hints['fetchMode'][$class->name][$field])) {
-                        $hints['fetchMode'][$class->name][$field] = $assoc->fetch;
-                    }
-
                     // Foreign key is set
                     // Check identity map first
                     // FIXME: Can break easily with composite keys if join column values are in
@@ -2514,9 +2520,13 @@ public function createEntity(string $className, array $data, array &$hints = [])
                     $reflField = $class->reflFields[$field];
                     $reflField->setValue($entity, $pColl);
 
-                    if ($assoc->fetch === ClassMetadata::FETCH_EAGER) {
-                        $this->loadCollection($pColl);
-                        $pColl->takeSnapshot();
+                    if ($hints['fetchMode'][$class->name][$field] === ClassMetadata::FETCH_EAGER) {
+                        if ($assoc->isOneToMany()) {
+                            $this->scheduleCollectionForBatchLoading($pColl, $class);
+                        } elseif ($assoc->isManyToMany()) {
+                            $this->loadCollection($pColl);
+                            $pColl->takeSnapshot();
+                        }
                     }
 
                     $this->originalEntityData[$oid][$field] = $pColl;
@@ -2532,7 +2542,7 @@ public function createEntity(string $className, array $data, array &$hints = [])
 
     public function triggerEagerLoads(): void
     {
-        if (! $this->eagerLoadingEntities) {
+        if (! $this->eagerLoadingEntities && ! $this->eagerLoadingCollections) {
             return;
         }
 
@@ -2545,11 +2555,69 @@ public function triggerEagerLoads(): void
                 continue;
             }
 
-            $class = $this->em->getClassMetadata($entityName);
+            $class   = $this->em->getClassMetadata($entityName);
+            $batches = array_chunk($ids, $this->em->getConfiguration()->getEagerFetchBatchSize());
 
-            $this->getEntityPersister($entityName)->loadAll(
-                array_combine($class->identifier, [array_values($ids)]),
-            );
+            foreach ($batches as $batchedIds) {
+                $this->getEntityPersister($entityName)->loadAll(
+                    array_combine($class->identifier, [$batchedIds]),
+                );
+            }
+        }
+
+        $eagerLoadingCollections       = $this->eagerLoadingCollections; // avoid recursion
+        $this->eagerLoadingCollections = [];
+
+        foreach ($eagerLoadingCollections as $group) {
+            $this->eagerLoadCollections($group['items'], $group['mapping']);
+        }
+    }
+
+    /**
+     * Load all data into the given collections, according to the specified mapping
+     *
+     * @param PersistentCollection[] $collections
+     */
+    private function eagerLoadCollections(array $collections, ToManyInverseSideMapping $mapping): void
+    {
+        $targetEntity = $mapping->targetEntity;
+        $class        = $this->em->getClassMetadata($mapping->sourceEntity);
+        $mappedBy     = $mapping->mappedBy;
+
+        $batches = array_chunk($collections, $this->em->getConfiguration()->getEagerFetchBatchSize(), true);
+
+        foreach ($batches as $collectionBatch) {
+            $entities = [];
+
+            foreach ($collectionBatch as $collection) {
+                $entities[] = $collection->getOwner();
+            }
+
+            $found = $this->getEntityPersister($targetEntity)->loadAll([$mappedBy => $entities]);
+
+            $targetClass    = $this->em->getClassMetadata($targetEntity);
+            $targetProperty = $targetClass->getReflectionProperty($mappedBy);
+            assert($targetProperty !== null);
+
+            foreach ($found as $targetValue) {
+                $sourceEntity = $targetProperty->getValue($targetValue);
+
+                $id     = $this->identifierFlattener->flattenIdentifier($class, $class->getIdentifierValues($sourceEntity));
+                $idHash = implode(' ', $id);
+
+                if ($mapping->indexBy !== null) {
+                    $indexByProperty = $targetClass->getReflectionProperty($mapping->indexBy);
+                    assert($indexByProperty !== null);
+                    $collectionBatch[$idHash]->hydrateSet($indexByProperty->getValue($targetValue), $targetValue);
+                } else {
+                    $collectionBatch[$idHash]->add($targetValue);
+                }
+            }
+        }
+
+        foreach ($collections as $association) {
+            $association->setInitialized(true);
+            $association->takeSnapshot();
         }
     }
 
@@ -2576,6 +2644,33 @@ public function loadCollection(PersistentCollection $collection): void
         $collection->setInitialized(true);
     }
 
+    /**
+     * Schedule this collection for batch loading at the end of the UnitOfWork
+     */
+    private function scheduleCollectionForBatchLoading(PersistentCollection $collection, ClassMetadata $sourceClass): void
+    {
+        $mapping = $collection->getMapping();
+        $name    = $mapping['sourceEntity'] . '#' . $mapping['fieldName'];
+
+        if (! isset($this->eagerLoadingCollections[$name])) {
+            $this->eagerLoadingCollections[$name] = [
+                'items'   => [],
+                'mapping' => $mapping,
+            ];
+        }
+
+        $owner = $collection->getOwner();
+        assert($owner !== null);
+
+        $id     = $this->identifierFlattener->flattenIdentifier(
+            $sourceClass,
+            $sourceClass->getIdentifierValues($owner),
+        );
+        $idHash = implode(' ', $id);
+
+        $this->eagerLoadingCollections[$name]['items'][$idHash] = $collection;
+    }
+
     /**
      * Gets the identity map of the UnitOfWork.
      *