ensure images with the new derivative-of-media-ids and replaces-media-id identifiers are persisted (i.e. not 'reaped')

Author: twrichards

common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala

@@ -2,6 +2,7 @@ package com.gu.mediaservice.lib.config
 
 import com.gu.mediaservice.lib.ImageStorageProps
 import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearchConfig}
+import scalaz.NonEmptyList
 
 class CommonConfigWithElastic(resources: GridConfigResources) extends CommonConfig(resources) {
 
@@ -15,7 +16,12 @@ class CommonConfigWithElastic(resources: GridConfigResources) extends CommonConf
     replicas = string("es6.replicas").toInt
   )
 
-  val persistenceIdentifier = string("persistence.identifier")
+  private val persistenceIdentifier = string("persistence.identifier")
+  val persistenceIdentifiers = NonEmptyList(
+    persistenceIdentifier,
+    ImageStorageProps.derivativeOfMediaIdsIdentifierKey,
+    ImageStorageProps.replacesMediaIdIdentifierKey
+  )
   val queriableIdentifiers = Seq(
     persistenceIdentifier,
     ImageStorageProps.derivativeOfMediaIdsIdentifierKey,

common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/PersistedQueries.scala

@@ -25,7 +25,7 @@ object PersistedQueries extends ImageFields {
   val hasCrops = filters.bool().must(filters.existsOrMissing("exports", exists = true))
   val usedInContent = filters.nested("usages", filters.exists(NonEmptyList("usages")))
 
-  def existedPreGrid(persistenceIdentifier: String) = filters.exists(NonEmptyList(identifierField(persistenceIdentifier)))
+  def hasPersistedIdentifier(persistenceIdentifiers: NonEmptyList[String]) = filters.exists(persistenceIdentifiers.map(identifierField))
 
   val addedToLibrary = filters.bool().must(filters.boolTerm(editsField("archived"), value = true))
   val hasUserEditsToImageMetadata = filters.exists(NonEmptyList(editsField("metadata")))

common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala

@@ -3,7 +3,9 @@ package com.gu.mediaservice.lib.elasticsearch
 import com.sksamuel.elastic4s.ElasticDsl.matchAllQuery
 import com.sksamuel.elastic4s.requests.searches.queries.Query
 import org.joda.time.DateTime
-import com.gu.mediaservice.lib.config. Provider
+import com.gu.mediaservice.lib.config.Provider
+import scalaz.NonEmptyList
+
 import scala.concurrent.Future
 
 trait ReapableEligibility extends Provider{
@@ -13,7 +15,7 @@ trait ReapableEligibility extends Provider{
 
 
   val maybePersistOnlyTheseCollections: Option[Set[String]] // typically from config
-  val persistenceIdentifier: String // typically from config
+  val persistenceIdentifiers: NonEmptyList[String] // typically from config
 
   private def moreThanTwentyDaysOld =
     filters.date("uploadTime", None, Some(DateTime.now().minusDays(20))).getOrElse(matchAllQuery())
@@ -29,7 +31,7 @@ trait ReapableEligibility extends Provider{
     PersistedQueries.addedToPhotoshoot,
     PersistedQueries.hasLabels,
     PersistedQueries.hasLeases,
-    PersistedQueries.existedPreGrid(persistenceIdentifier),
+    PersistedQueries.hasPersistedIdentifier(persistenceIdentifiers),
     PersistedQueries.isInPersistedCollection(maybePersistOnlyTheseCollections)
   )
 

media-api/app/lib/ImagePersistenceReasons.scala

@@ -3,12 +3,13 @@ package lib
 import com.gu.mediaservice.lib.elasticsearch.PersistedQueries
 import com.gu.mediaservice.model.{CommissionedAgency, Illustrator, Image, ImageMetadata, Photographer}
 import com.sksamuel.elastic4s.requests.searches.queries.Query
+import scalaz.NonEmptyList
 
 
-case class ImagePersistenceReasons(maybePersistOnlyTheseCollections: Option[Set[String]], persistenceIdentifier: String) {
+case class ImagePersistenceReasons(maybePersistOnlyTheseCollections: Option[Set[String]], persistenceIdentifiers: NonEmptyList[String]) {
   val allReasons: List[PersistenceReason] =
     List(
-      HasPersistenceIdentifier(persistenceIdentifier),
+      HasPersistenceIdentifier(persistenceIdentifiers),
       HasExports,
       HasUsages,
       IsArchived,
@@ -31,12 +32,12 @@ sealed trait PersistenceReason {
   val reason: String
 }
 
-case class HasPersistenceIdentifier(persistenceIdentifier: String) extends PersistenceReason {
-  override def shouldPersist(image: Image): Boolean = image.identifiers.contains(persistenceIdentifier)
+case class HasPersistenceIdentifier(persistenceIdentifiers: NonEmptyList[String]) extends PersistenceReason {
+  override def shouldPersist(image: Image): Boolean = persistenceIdentifiers.list.toList.exists(image.identifiers.contains)
 
   override val reason: String = "persistence-identifier"
 
-  override val query: Query = PersistedQueries.existedPreGrid(persistenceIdentifier)
+  override val query: Query = PersistedQueries.hasPersistedIdentifier(persistenceIdentifiers)
 }
 
 object HasExports extends PersistenceReason {

media-api/app/lib/ImageResponse.scala

@@ -47,7 +47,7 @@ class ImageResponse(config: MediaApiConfig, s3Client: S3Client, usageQuota: Usag
 
   private val imgPersistenceReasons = ImagePersistenceReasons(
     config.maybePersistOnlyTheseCollections,
-    config.persistenceIdentifier
+    config.persistenceIdentifiers
   )
 
   def imagePersistenceReasons(image: Image): List[String] = imgPersistenceReasons.reasons(image)

media-api/app/lib/elasticsearch/IsQueryFilter.scala

@@ -7,6 +7,7 @@ import com.sksamuel.elastic4s.ElasticDsl.matchAllQuery
 import com.sksamuel.elastic4s.requests.searches.queries.Query
 import lib.MediaApiConfig
 import org.joda.time.DateTime
+import scalaz.NonEmptyList
 import scalaz.syntax.std.list._
 
 sealed trait IsQueryFilter extends Query with ImageFields {
@@ -32,7 +33,7 @@ object IsQueryFilter {
       case s if s == s"$organisation-owned" => Some(IsOwnedImage(organisation))
       case "under-quota" => Some(IsUnderQuota(overQuotaAgencies()))
       case "deleted" => Some(IsDeleted(true))
-      case "reapable" => Some(IsReapable(config.maybePersistOnlyTheseCollections, config.persistenceIdentifier))
+      case "reapable" => Some(IsReapable(config.maybePersistOnlyTheseCollections, config.persistenceIdentifiers))
       case _ => None
     }
   }
@@ -68,6 +69,6 @@ case class IsDeleted(isDeleted: Boolean) extends IsQueryFilter {
   )
 }
 
-case class IsReapable(maybePersistOnlyTheseCollections: Option[Set[String]], persistenceIdentifier: String)
+case class IsReapable(maybePersistOnlyTheseCollections: Option[Set[String]], persistenceIdentifiers: NonEmptyList[String])
   extends IsQueryFilter with ReapableEligibility {
 }

media-api/app/lib/elasticsearch/SearchFilters.scala

@@ -52,7 +52,7 @@ class SearchFilters(config: MediaApiConfig) extends ImageFields {
   lazy val freeToUseCategories: List[String] =
     usageRights.filter(ur => ur.defaultCost.exists(cost => cost == Free || cost == Conditional)).map(ur => ur.category)
 
-  val persistedReasons: List[PersistenceReason] = ImagePersistenceReasons(config.maybePersistOnlyTheseCollections, config.persistenceIdentifier).allReasons
+  val persistedReasons: List[PersistenceReason] = ImagePersistenceReasons(config.maybePersistOnlyTheseCollections, config.persistenceIdentifiers).allReasons
 
   val persistedFilter: Query = filters.or(persistedReasons.map(_.query): _*)
 

media-api/test/lib/ImagePersistenceReasonsTest.scala

@@ -6,6 +6,7 @@ import com.gu.mediaservice.model.usage.Usage
 import org.joda.time.DateTime.now
 import org.scalatest.funspec.AnyFunSpec
 import org.scalatest.matchers.should.Matchers
+import scalaz.NonEmptyList
 
 class ImagePersistenceReasonsTest extends AnyFunSpec with Matchers {
 
@@ -15,8 +16,8 @@ class ImagePersistenceReasonsTest extends AnyFunSpec with Matchers {
 
     val persistedIdentifier = "test-p-id"
     val persistedCollections = Set("coll1", "coll2", "coll3")
-    val imgPersistenceReasons = ImagePersistenceReasons(Some(persistedCollections), persistedIdentifier)
-    val imagePersistenceReasonsWithEmptyListOfPersistedCollections = ImagePersistenceReasons(Some(Set.empty), persistedIdentifier)
+    val imgPersistenceReasons = ImagePersistenceReasons(Some(persistedCollections), NonEmptyList(persistedIdentifier))
+    val imagePersistenceReasonsWithEmptyListOfPersistedCollections = ImagePersistenceReasons(Some(Set.empty), NonEmptyList(persistedIdentifier))
 
     imgPersistenceReasons.reasons(img) shouldBe Nil
     val imgWithPersistenceIdentifier = img.copy(identifiers = Map(persistedIdentifier -> "test-id"))

thrall/app/controllers/ReaperController.scala

@@ -14,6 +14,7 @@ import lib.elasticsearch.ElasticSearch
 import org.joda.time.{DateTime, DateTimeZone}
 import play.api.libs.json.{JsValue, Json}
 import play.api.mvc.{Action, AnyContent, ControllerComponents}
+import scalaz.NonEmptyList
 
 import scala.concurrent.{ExecutionContext, Future}
 import scala.jdk.CollectionConverters._
@@ -44,7 +45,7 @@ class ReaperController(
   private val isReapable = maybeCustomReapableEligibility getOrElse {
     new ReapableEligibility {
       override val maybePersistOnlyTheseCollections: Option[Set[String]] = config.maybePersistOnlyTheseCollections
-      override val persistenceIdentifier: String = config.persistenceIdentifier
+      override val persistenceIdentifiers: NonEmptyList[String] = config.persistenceIdentifiers
     }
   }