From c87b0ff3079117f9abbe8f6d10b3cef96213b139 Mon Sep 17 00:00:00 2001 From: Tom Richards Date: Tue, 19 Sep 2023 12:42:10 +0100 Subject: [PATCH] move 'reapable' eligibility logic into `common-lib` in preparation for use in `thrall` (along with supporting classes) --- .../lib/config/CommonConfigWithElastic.scala | 8 ++++ .../lib/elasticsearch/PersistedQueries.scala | 42 +++++++++++++++++++ .../elasticsearch/ReapableEligibility.scala | 34 +++++++++++++++ .../lib/elasticsearch/filters.scala | 4 +- .../app/lib/ImagePersistenceReasons.scala | 2 +- media-api/app/lib/MediaApiConfig.scala | 10 ----- .../app/lib/elasticsearch/ElasticSearch.scala | 1 + .../app/lib/elasticsearch/IsQueryFilter.scala | 24 ++--------- .../app/lib/elasticsearch/SearchFilters.scala | 34 +-------------- .../lib/elasticsearch/SyndicationFilter.scala | 1 + 10 files changed, 93 insertions(+), 67 deletions(-) create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/PersistedQueries.scala create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala rename {media-api/app => common-lib/src/main/scala/com/gu/mediaservice}/lib/elasticsearch/filters.scala (98%) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala index 1c0f3b2980..1d5041b7b1 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala @@ -14,4 +14,12 @@ class CommonConfigWithElastic(resources: GridConfigResources) extends CommonConf shards = string("es6.shards").toInt, replicas = string("es6.replicas").toInt ) + + val persistenceIdentifier = string("persistence.identifier") + val queriableIdentifiers = Seq(persistenceIdentifier) + + val persistedRootCollections: List[String] = stringOpt("persistence.collections") match { + case Some(collections) => collections.split(',').toList + case None => List(s"${staffPhotographerOrganisation} Archive") + } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/PersistedQueries.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/PersistedQueries.scala new file mode 100644 index 0000000000..b281c13e21 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/PersistedQueries.scala @@ -0,0 +1,42 @@ +package com.gu.mediaservice.lib.elasticsearch + +import com.gu.mediaservice.lib.ImageFields +import com.gu.mediaservice.model._ +import scalaz.NonEmptyList +import scalaz.syntax.std.list._ + +object PersistedQueries extends ImageFields { + val photographerCategories = NonEmptyList( + StaffPhotographer.category, + ContractPhotographer.category, + CommissionedPhotographer.category + ) + + val illustratorCategories = NonEmptyList( + ContractIllustrator.category, + StaffIllustrator.category, + CommissionedIllustrator.category + ) + + val agencyCommissionedCategories = NonEmptyList( + CommissionedAgency.category + ) + + val hasCrops = filters.bool.must(filters.existsOrMissing("exports", exists = true)) + val usedInContent = filters.nested("usages", filters.exists(NonEmptyList("usages"))) + + def existedPreGrid(persistenceIdentifier: String) = filters.exists(NonEmptyList(identifierField(persistenceIdentifier))) + + val addedToLibrary = filters.bool.must(filters.boolTerm(editsField("archived"), value = true)) + val hasUserEditsToImageMetadata = filters.exists(NonEmptyList(editsField("metadata"))) + val hasPhotographerUsageRights = filters.bool.must(filters.terms(usageRightsField("category"), photographerCategories)) + val hasIllustratorUsageRights = filters.bool.must(filters.terms(usageRightsField("category"), illustratorCategories)) + val hasAgencyCommissionedUsageRights = filters.bool.must(filters.terms(usageRightsField("category"), agencyCommissionedCategories)) + + def addedGNMArchiveOrPersistedCollections(persistedRootCollections: List[String]) = filters.bool.must(filters.terms(collectionsField("path"), persistedRootCollections.toNel.get)) + + val addedToPhotoshoot = filters.exists(NonEmptyList(editsField("photoshoot"))) + val hasLabels = filters.exists(NonEmptyList(editsField("labels"))) + val hasLeases = filters.exists(NonEmptyList(leasesField("leases"))) + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala new file mode 100644 index 0000000000..9316d06ab5 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala @@ -0,0 +1,34 @@ +package com.gu.mediaservice.lib.elasticsearch + +import com.sksamuel.elastic4s.ElasticDsl.matchAllQuery +import com.sksamuel.elastic4s.requests.searches.queries.Query +import org.joda.time.DateTime + +trait ReapableEligibility { + + val persistedRootCollections: List[String] // typically from config + val persistenceIdentifier: String // typically from config + + private def moreThanTwentyDaysOld = + filters.date("uploadTime", None, Some(DateTime.now().minusDays(20))).getOrElse(matchAllQuery()) + + private lazy val persistedQueries = filters.or( + PersistedQueries.hasCrops, + PersistedQueries.usedInContent, + PersistedQueries.addedToLibrary, + PersistedQueries.hasUserEditsToImageMetadata, + PersistedQueries.hasPhotographerUsageRights, + PersistedQueries.hasIllustratorUsageRights, + PersistedQueries.hasAgencyCommissionedUsageRights, + PersistedQueries.addedToPhotoshoot, + PersistedQueries.hasLabels, + PersistedQueries.hasLeases, + PersistedQueries.existedPreGrid(persistenceIdentifier), + PersistedQueries.addedGNMArchiveOrPersistedCollections(persistedRootCollections) + ) + + def query: Query = filters.and( + moreThanTwentyDaysOld, + filters.not(persistedQueries) + ) +} diff --git a/media-api/app/lib/elasticsearch/filters.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/filters.scala similarity index 98% rename from media-api/app/lib/elasticsearch/filters.scala rename to common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/filters.scala index 990662746b..c87b43b3be 100644 --- a/media-api/app/lib/elasticsearch/filters.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/filters.scala @@ -1,10 +1,10 @@ -package lib.elasticsearch +package com.gu.mediaservice.lib.elasticsearch import com.gu.mediaservice.lib.formatting.printDateTime import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s.requests.searches.queries.{NestedQuery, Query} import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery +import com.sksamuel.elastic4s.requests.searches.queries.{NestedQuery, Query} import com.sksamuel.elastic4s.requests.searches.term.TermQuery import org.joda.time.DateTime import scalaz.NonEmptyList diff --git a/media-api/app/lib/ImagePersistenceReasons.scala b/media-api/app/lib/ImagePersistenceReasons.scala index 1e7b221934..9c3309161d 100644 --- a/media-api/app/lib/ImagePersistenceReasons.scala +++ b/media-api/app/lib/ImagePersistenceReasons.scala @@ -1,8 +1,8 @@ package lib +import com.gu.mediaservice.lib.elasticsearch.PersistedQueries import com.gu.mediaservice.model.{CommissionedAgency, Illustrator, Image, ImageMetadata, Photographer} import com.sksamuel.elastic4s.requests.searches.queries.Query -import lib.elasticsearch.PersistedQueries case class ImagePersistenceReasons(persistedRootCollections: List[String], persistenceIdentifier: String) { diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala index a91c0879ef..9ec37f74b5 100644 --- a/media-api/app/lib/MediaApiConfig.scala +++ b/media-api/app/lib/MediaApiConfig.scala @@ -47,16 +47,6 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla val requiredMetadata = List("credit", "description", "usageRights") - val persistenceIdentifier = string("persistence.identifier") - val queriableIdentifiers = Seq(persistenceIdentifier) - - val persistedRootCollections: List[String] = stringOpt("persistence.collections") match { - case Some(collections) => collections.split(',').toList - case None => List(s"${staffPhotographerOrganisation} Archive") - } - - def convertToInt(s: String): Option[Int] = Try { s.toInt }.toOption - val syndicationStartDate: Option[DateTime] = Try { stringOpt("syndication.start").map(d => DateTime.parse(d).withTimeAtStartOfDay()) }.toOption.flatten diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index d2f8e184b0..b1c69c70c4 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -2,6 +2,7 @@ package lib.elasticsearch import akka.actor.Scheduler import com.gu.mediaservice.lib.ImageFields +import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.elasticsearch.{CompletionPreview, ElasticSearchClient, ElasticSearchConfig, MigrationStatusProvider, Running} import com.gu.mediaservice.lib.logging.{GridLogging, MarkerMap} diff --git a/media-api/app/lib/elasticsearch/IsQueryFilter.scala b/media-api/app/lib/elasticsearch/IsQueryFilter.scala index ce4e45b7e5..3a2b8923eb 100644 --- a/media-api/app/lib/elasticsearch/IsQueryFilter.scala +++ b/media-api/app/lib/elasticsearch/IsQueryFilter.scala @@ -1,6 +1,7 @@ package lib.elasticsearch import com.gu.mediaservice.lib.ImageFields +import com.gu.mediaservice.lib.elasticsearch.{ReapableEligibility, filters} import com.gu.mediaservice.model._ import com.sksamuel.elastic4s.ElasticDsl.matchAllQuery import com.sksamuel.elastic4s.requests.searches.queries.Query @@ -67,25 +68,6 @@ case class IsDeleted(isDeleted: Boolean) extends IsQueryFilter { ) } -case class IsReapable(persistedRootCollections: List[String], persistenceIdentifier: String) extends IsQueryFilter { - val moreThanTwentyDaysOld = filters.date("uploadTime", None, Some(DateTime.now().minusDays(20))).getOrElse(matchAllQuery()) - - val persistedQueries = filters.or( - PersistedQueries.hasCrops, - PersistedQueries.usedInContent, - PersistedQueries.addedToLibrary, - PersistedQueries.hasUserEditsToImageMetadata, - PersistedQueries.hasPhotographerUsageRights, - PersistedQueries.hasIllustratorUsageRights, - PersistedQueries.hasAgencyCommissionedUsageRights, - PersistedQueries.addedToPhotoshoot, - PersistedQueries.hasLabels, - PersistedQueries.hasLeases, - PersistedQueries.existedPreGrid(persistenceIdentifier), - PersistedQueries.addedGNMArchiveOrPersistedCollections(persistedRootCollections) - ) - override def query: Query = filters.and( - moreThanTwentyDaysOld, - filters.not(persistedQueries) - ) +case class IsReapable(persistedRootCollections: List[String], persistenceIdentifier: String) + extends IsQueryFilter with ReapableEligibility { } diff --git a/media-api/app/lib/elasticsearch/SearchFilters.scala b/media-api/app/lib/elasticsearch/SearchFilters.scala index 57237346cb..d9ddd1dfd6 100644 --- a/media-api/app/lib/elasticsearch/SearchFilters.scala +++ b/media-api/app/lib/elasticsearch/SearchFilters.scala @@ -2,44 +2,12 @@ package lib.elasticsearch import com.gu.mediaservice.lib.ImageFields import com.gu.mediaservice.lib.auth.{Syndication, Tier} -import com.gu.mediaservice.lib.config.RuntimeUsageRightsConfig +import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.model._ import com.sksamuel.elastic4s.requests.searches.queries.Query import lib.{ImagePersistenceReasons, MediaApiConfig, PersistenceReason} -import scalaz.NonEmptyList import scalaz.syntax.std.list._ -object PersistedQueries extends ImageFields { - val photographerCategories = NonEmptyList( - StaffPhotographer.category, - ContractPhotographer.category, - CommissionedPhotographer.category - ) - - val illustratorCategories = NonEmptyList( - ContractIllustrator.category, - StaffIllustrator.category, - CommissionedIllustrator.category - ) - - val agencyCommissionedCategories = NonEmptyList( - CommissionedAgency.category - ) - - val hasCrops = filters.bool.must(filters.existsOrMissing("exports", exists = true)) - val usedInContent = filters.nested("usages", filters.exists(NonEmptyList("usages"))) - def existedPreGrid(persistenceIdentifier: String) = filters.exists(NonEmptyList(identifierField(persistenceIdentifier))) - val addedToLibrary = filters.bool.must(filters.boolTerm(editsField("archived"), value = true)) - val hasUserEditsToImageMetadata = filters.exists(NonEmptyList(editsField("metadata"))) - val hasPhotographerUsageRights = filters.bool.must(filters.terms(usageRightsField("category"), photographerCategories)) - val hasIllustratorUsageRights = filters.bool.must(filters.terms(usageRightsField("category"), illustratorCategories)) - val hasAgencyCommissionedUsageRights = filters.bool.must(filters.terms(usageRightsField("category"), agencyCommissionedCategories)) - def addedGNMArchiveOrPersistedCollections(persistedRootCollections: List[String]) = filters.bool.must(filters.terms(collectionsField("path"), persistedRootCollections.toNel.get)) - val addedToPhotoshoot = filters.exists(NonEmptyList(editsField("photoshoot"))) - val hasLabels = filters.exists(NonEmptyList(editsField("labels"))) - val hasLeases = filters.exists(NonEmptyList(leasesField("leases"))) - -} class SearchFilters(config: MediaApiConfig) extends ImageFields { diff --git a/media-api/app/lib/elasticsearch/SyndicationFilter.scala b/media-api/app/lib/elasticsearch/SyndicationFilter.scala index 75d558404b..1b7003806c 100644 --- a/media-api/app/lib/elasticsearch/SyndicationFilter.scala +++ b/media-api/app/lib/elasticsearch/SyndicationFilter.scala @@ -1,6 +1,7 @@ package lib.elasticsearch import com.gu.mediaservice.lib.ImageFields +import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.model._ import com.gu.mediaservice.model.leases.{AllowSyndicationLease, DenySyndicationLease} import com.gu.mediaservice.model.usage.SyndicationUsage