T2407 original identifier

common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala

@@ -253,6 +253,13 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo
     }
   }
 
+  def postUsage(usageType: String, data: JsObject, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext) = {
+    val url = new URL(s"${services.usageBaseUri}/usages/$usageType")
+    val request: WSRequest = wsClient.url(url.toString)
+    val authorisedRequest = authFn(request)
+    authorisedRequest.post(Json.obj("data" -> data)).map { response => validateResponse(response, url)}
+  }
+
 }
 
 class DownstreamApiInBadStateException(message: String, downstreamMessage: String) extends IllegalStateException(message) {

common-lib/src/main/scala/com/gu/mediaservice/lib/ImageStorage.scala

@@ -17,6 +17,8 @@ object ImageStorageProps {
   val uploadTimeMetadataKey: String = "upload-time"
   val uploadedByMetadataKey: String = "uploaded-by"
   val identifierMetadataKeyPrefix: String = "identifier!"
+  val derivativeOfMediaIdsIdentifierKey: String = "derivative-of-media-ids"
+  val replacesMediaIdIdentifierKey: String = "replaces-media-id"
 }
 
 trait ImageStorage {

common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfigWithElastic.scala

@@ -1,6 +1,8 @@
 package com.gu.mediaservice.lib.config
 
+import com.gu.mediaservice.lib.ImageStorageProps
 import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearchConfig}
+import scalaz.NonEmptyList
 
 class CommonConfigWithElastic(resources: GridConfigResources) extends CommonConfig(resources) {
 
@@ -14,8 +16,16 @@ class CommonConfigWithElastic(resources: GridConfigResources) extends CommonConf
     replicas = string("es6.replicas").toInt
   )
 
-  val persistenceIdentifier = string("persistence.identifier")
-  val queriableIdentifiers = Seq(persistenceIdentifier)
+  private val persistenceIdentifier = string("persistence.identifier")
+  val persistenceIdentifiers = NonEmptyList(
+    persistenceIdentifier,
+    ImageStorageProps.derivativeOfMediaIdsIdentifierKey,
+    ImageStorageProps.replacesMediaIdIdentifierKey
+  )
+  val queriableIdentifiers = Seq(
+    persistenceIdentifier,
+    ImageStorageProps.derivativeOfMediaIdsIdentifierKey,
+  )
 
   // note this will match any part of the collection path, e.g. "bar" will match "bar", "foo/bar", "bar/baz"
   val maybePersistOnlyTheseCollections: Option[Set[String]] = getOptionalStringSet("persistence.onlyTheseCollections")

common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala

@@ -196,6 +196,10 @@ object MappingTest {
       )),
       downloadUsageMetadata = Some(DownloadUsageMetadata(
         downloadedBy = "me"
+      )),
+      childUsageMetadata = Some(ChildUsageMetadata(
+        addedBy = "me",
+        childMediaId = "child-id"
       ))
     )),
     leases = LeasesByMedia.build(List(

common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/Mappings.scala

@@ -299,6 +299,11 @@ object Mappings {
     keywordField("downloadedBy")
   ))
 
+  def childUsageMetadata(name: String): ObjectField = nonDynamicObjectField(name).copy(properties = Seq(
+    keywordField("addedBy"),
+    keywordField("childMediaId")
+  ))
+
   def usagesMapping(name: String): NestedField = nestedField(name).copy(properties = Seq(
     keywordField("id"),
     sStemmerAnalysed("title"),
@@ -313,7 +318,8 @@ object Mappings {
     digitalUsageMetadata("digitalUsageMetadata"),
     syndicationUsageMetadata("syndicationUsageMetadata"),
     frontUsageMetadata("frontUsageMetadata"),
-    downloadUsageMetadata("downloadUsageMetadata")
+    downloadUsageMetadata("downloadUsageMetadata"),
+    childUsageMetadata("childUsageMetadata")
   ))
 
   def leaseMapping(name: String): ObjectField = nonDynamicObjectField(name).copy(properties = Seq(

common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/PersistedQueries.scala

@@ -25,7 +25,7 @@ object PersistedQueries extends ImageFields {
   val hasCrops = filters.bool().must(filters.existsOrMissing("exports", exists = true))
   val usedInContent = filters.nested("usages", filters.exists(NonEmptyList("usages")))
 
-  def existedPreGrid(persistenceIdentifier: String) = filters.exists(NonEmptyList(identifierField(persistenceIdentifier)))
+  def hasPersistedIdentifier(persistenceIdentifiers: NonEmptyList[String]) = filters.exists(persistenceIdentifiers.map(identifierField))
 
   val addedToLibrary = filters.bool().must(filters.boolTerm(editsField("archived"), value = true))
   val hasUserEditsToImageMetadata = filters.exists(NonEmptyList(editsField("metadata")))

common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ReapableEligibility.scala

@@ -3,7 +3,9 @@ package com.gu.mediaservice.lib.elasticsearch
 import com.sksamuel.elastic4s.ElasticDsl.matchAllQuery
 import com.sksamuel.elastic4s.requests.searches.queries.Query
 import org.joda.time.DateTime
-import com.gu.mediaservice.lib.config. Provider
+import com.gu.mediaservice.lib.config.Provider
+import scalaz.NonEmptyList
+
 import scala.concurrent.Future
 
 trait ReapableEligibility extends Provider{
@@ -13,7 +15,7 @@ trait ReapableEligibility extends Provider{
 
 
   val maybePersistOnlyTheseCollections: Option[Set[String]] // typically from config
-  val persistenceIdentifier: String // typically from config
+  val persistenceIdentifiers: NonEmptyList[String] // typically from config
 
   private def moreThanTwentyDaysOld =
     filters.date("uploadTime", None, Some(DateTime.now().minusDays(20))).getOrElse(matchAllQuery())
@@ -29,7 +31,7 @@ trait ReapableEligibility extends Provider{
     PersistedQueries.addedToPhotoshoot,
     PersistedQueries.hasLabels,
     PersistedQueries.hasLeases,
-    PersistedQueries.existedPreGrid(persistenceIdentifier),
+    PersistedQueries.hasPersistedIdentifier(persistenceIdentifiers),
     PersistedQueries.isInPersistedCollection(maybePersistOnlyTheseCollections)
   )
 

common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala

@@ -29,6 +29,8 @@ object ItemToMediaUsage {
         .map(_.asScala.toMap).flatMap(buildFront),
       Option(item.getMap[Any]("download_metadata"))
         .map(_.asScala.toMap).flatMap(buildDownload),
+      Option(item.getMap[Any]("child_metadata"))
+        .map(_.asScala.toMap).flatMap(buildChild),
       new DateTime(item.getLong("last_modified")),
       Try {
         item.getLong("date_added")
@@ -99,4 +101,13 @@ object ItemToMediaUsage {
       )
     }.toOption
   }
+
+  private def buildChild(metadataMap: Map[String, Any]): Option[ChildUsageMetadata] = {
+    Try {
+      ChildUsageMetadata(
+        metadataMap("addedBy").asInstanceOf[String],
+        metadataMap("childMediaId").asInstanceOf[String],
+      )
+    }.toOption
+  }
 }

common-lib/src/main/scala/com/gu/mediaservice/lib/usage/UsageBuilder.scala

@@ -3,22 +3,25 @@ package com.gu.mediaservice.lib.usage
 import com.gu.mediaservice.model.usage._
 import org.joda.time.DateTime
 
+import java.net.URI
+
 object UsageBuilder {
 
   def build(usage: MediaUsage) = Usage(
-    buildId(usage),
-    buildUsageReference(usage),
-    usage.usageType,
-    usage.mediaType,
-    buildStatusString(usage),
-    usage.dateAdded,
-    usage.dateRemoved,
-    usage.lastModified,
-    usage.printUsageMetadata,
-    usage.digitalUsageMetadata,
-    usage.syndicationUsageMetadata,
-    usage.frontUsageMetadata,
-    usage.downloadUsageMetadata
+    id = buildId(usage),
+    references = buildUsageReference(usage),
+    platform = usage.usageType,
+    media = usage.mediaType,
+    status = buildStatusString(usage),
+    dateAdded = usage.dateAdded,
+    dateRemoved = usage.dateRemoved,
+    lastModified = usage.lastModified,
+    printUsageMetadata = usage.printUsageMetadata,
+    digitalUsageMetadata = usage.digitalUsageMetadata,
+    syndicationUsageMetadata = usage.syndicationUsageMetadata,
+    frontUsageMetadata = usage.frontUsageMetadata,
+    downloadUsageMetadata = usage.downloadUsageMetadata,
+    childUsageMetadata = usage.childUsageMetadata
   )
 
   private def buildStatusString(usage: MediaUsage): UsageStatus = if (usage.isRemoved) RemovedUsageStatus else usage.status
@@ -33,6 +36,8 @@ object UsageBuilder {
       case PrintUsage => buildPrintUsageReference(usage)
       case SyndicationUsage => buildSyndicationUsageReference(usage)
       case DownloadUsage => buildDownloadUsageReference(usage)
+      case DerivativeUsage => buildChildUsageReference(usage, GridUsageReference)
+      case ReplacedUsage => buildChildUsageReference(usage, GridUsageReference)
     }
   }
 
@@ -80,4 +85,16 @@ object UsageBuilder {
   }).getOrElse(
     List[UsageReference]()
   )
+
+  private def buildChildUsageReference(usage: MediaUsage, usageReferenceType: UsageReferenceType): List[UsageReference] = usage.childUsageMetadata.map (metadata => {
+    List(
+      UsageReference(
+        `type` = usageReferenceType,
+        uri = Some(new URI(metadata.childMediaId)), // should manifest as a relative link
+        name = Some(metadata.childMediaId)
+      )
+    )
+  }).getOrElse(
+    List[UsageReference]()
+  )
 }

common-lib/src/main/scala/com/gu/mediaservice/model/usage/ChildUsageMetadata.scala

@@ -0,0 +1,19 @@
+package com.gu.mediaservice.model.usage
+
+import play.api.libs.json.{Json, Reads, Writes}
+
+case class ChildUsageMetadata(
+  addedBy: String,
+  childMediaId: String
+
+) extends UsageMetadata {
+  override def toMap: Map[String, Any] = Map(
+    "addedBy" -> addedBy,
+    "childMediaId" -> childMediaId
+  )
+}
+
+object ChildUsageMetadata {
+  implicit val reader: Reads[ChildUsageMetadata] = Json.reads[ChildUsageMetadata]
+  implicit val writer: Writes[ChildUsageMetadata] = Json.writes[ChildUsageMetadata]
+}

common-lib/src/main/scala/com/gu/mediaservice/model/usage/MediaUsage.scala

@@ -22,6 +22,7 @@ case class MediaUsage(
   syndicationUsageMetadata: Option[SyndicationUsageMetadata],
   frontUsageMetadata: Option[FrontUsageMetadata],
   downloadUsageMetadata: Option[DownloadUsageMetadata],
+  childUsageMetadata: Option[ChildUsageMetadata],
   lastModified: DateTime,
   dateAdded: Option[DateTime] = None,
   dateRemoved: Option[DateTime] = None

common-lib/src/main/scala/com/gu/mediaservice/model/usage/Usage.scala

@@ -19,7 +19,8 @@ case class Usage(
   digitalUsageMetadata: Option[DigitalUsageMetadata] = None,
   syndicationUsageMetadata: Option[SyndicationUsageMetadata] = None,
   frontUsageMetadata: Option[FrontUsageMetadata] = None,
-  downloadUsageMetadata: Option[DownloadUsageMetadata] = None
+  downloadUsageMetadata: Option[DownloadUsageMetadata] = None,
+  childUsageMetadata: Option[ChildUsageMetadata] = None
 )
 object Usage {
   import com.gu.mediaservice.lib.formatting._
@@ -37,7 +38,8 @@ object Usage {
       (__ \ "digitalUsageMetadata").writeNullable[DigitalUsageMetadata] ~
       (__ \ "syndicationUsageMetadata").writeNullable[SyndicationUsageMetadata] ~
       (__ \ "frontUsageMetadata").writeNullable[FrontUsageMetadata] ~
-      (__ \ "downloadUsageMetadata").writeNullable[DownloadUsageMetadata]
+      (__ \ "downloadUsageMetadata").writeNullable[DownloadUsageMetadata] ~
+      (__ \ "childUsageMetadata").writeNullable[ChildUsageMetadata]
     )(unlift(Usage.unapply))
 
   implicit val reads: Reads[Usage] = (
@@ -53,6 +55,7 @@ object Usage {
       (__ \ "digitalUsageMetadata").readNullable[DigitalUsageMetadata] ~
       (__ \ "syndicationUsageMetadata").readNullable[SyndicationUsageMetadata] ~
       (__ \ "frontUsageMetadata").readNullable[FrontUsageMetadata] ~
-      (__ \ "downloadUsageMetadata").readNullable[DownloadUsageMetadata]
+      (__ \ "downloadUsageMetadata").readNullable[DownloadUsageMetadata] ~
+      (__ \ "childUsageMetadata").readNullable[ChildUsageMetadata]
     )(Usage.apply _)
 }

common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageReferenceType.scala

@@ -10,6 +10,7 @@ trait UsageReferenceType {
     case SyndicationUsageReference => "syndication"
     case FrontUsageReference => "front"
     case DownloadUsageReference => "download"
+    case GridUsageReference => "grid"
   }
 }
 
@@ -24,6 +25,7 @@ object UsageReferenceType {
     case "syndication" => SyndicationUsageReference
     case "front" => FrontUsageReference
     case "download" => DownloadUsageReference
+    case "grid" => GridUsageReference
   }
 }
 
@@ -33,3 +35,4 @@ object ComposerUsageReference extends UsageReferenceType
 object SyndicationUsageReference extends UsageReferenceType
 object FrontUsageReference extends UsageReferenceType
 object DownloadUsageReference extends UsageReferenceType
+object GridUsageReference extends UsageReferenceType

common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageStatus.scala

@@ -9,6 +9,8 @@ sealed trait UsageStatus {
     case RemovedUsageStatus => "removed"
     case SyndicatedUsageStatus => "syndicated"
     case DownloadedUsageStatus => "downloaded"
+    case DerivativeUsageStatus => "derivative"
+    case ReplacedUsageStatus => "replaced"
     case FailedUsageStatus => "failed"
     case UnknownUsageStatus => "unknown"
   }
@@ -21,6 +23,8 @@ object UsageStatus {
     case "removed" => RemovedUsageStatus
     case "syndicated" => SyndicatedUsageStatus
     case "downloaded" => DownloadedUsageStatus
+    case "derivative" => DerivativeUsageStatus
+    case "replaced" => ReplacedUsageStatus
     case "failed" => FailedUsageStatus
     case "unknown" => UnknownUsageStatus
     case _ => throw new IllegalArgumentException("Invalid usage status")
@@ -38,6 +42,8 @@ object PublishedUsageStatus extends UsageStatus
 object RemovedUsageStatus extends UsageStatus
 object SyndicatedUsageStatus extends UsageStatus
 object DownloadedUsageStatus extends UsageStatus
+object DerivativeUsageStatus extends UsageStatus
+object ReplacedUsageStatus extends UsageStatus
 object FailedUsageStatus extends UsageStatus
 
 // For Fronts usages as we don't know if a front is in draft or is live

common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageType.scala

@@ -8,6 +8,8 @@ trait UsageType {
     case DigitalUsage => "digital"
     case SyndicationUsage => "syndication"
     case DownloadUsage => "download"
+    case DerivativeUsage => "derivative"
+    case ReplacedUsage => "replaced"
   }
 }
 
@@ -20,10 +22,14 @@ object UsageType {
     case "digital" => DigitalUsage
     case "syndication" => SyndicationUsage
     case "download" => DownloadUsage
+    case "derivative" => DerivativeUsage
+    case "replaced" => ReplacedUsage
   }
 }
 
 object PrintUsage extends UsageType
 object DigitalUsage extends UsageType
 object SyndicationUsage extends UsageType
 object DownloadUsage extends UsageType
+object DerivativeUsage extends UsageType
+object ReplacedUsage extends UsageType

image-loader/app/ImageLoaderComponents.scala

@@ -21,13 +21,16 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne
     logger.info(s" $index -> ${processor.description}")
   }
 
+  val services = new Services(config.domainRoot, config.serviceHosts, Set.empty)
+  private val gridClient = GridClient(services)(wsClient)
+
   val store = new ImageLoaderStore(config)
   val maybeIngestQueue = config.maybeIngestSqsQueueUrl.map(queueUrl => new SimpleSqsMessageConsumer(queueUrl, config))
   val uploadStatusTable = new UploadStatusTable(config)
   val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath)
   val notifications = new Notifications(config)
   val downloader = new Downloader()(ec,wsClient)
-  val uploader = new Uploader(store, config, imageOperations, notifications, imageProcessor)
+  val uploader = new Uploader(store, config, imageOperations, notifications, imageProcessor, gridClient, auth)
   val projector = Projector(config, imageOperations, imageProcessor, auth)
   val quarantineUploader: Option[QuarantineUploader] = (config.uploadToQuarantineEnabled, config.quarantineBucket) match {
     case (true, Some(bucketName)) =>{
@@ -38,9 +41,6 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne
     case (false, _) => None
   }
 
-  val services = new Services(config.domainRoot, config.serviceHosts, Set.empty)
-  private val gridClient = GridClient(services)(wsClient)
-
   val metrics = new ImageLoaderMetrics(config, actorSystem, applicationLifecycle)
 
   val controller = new ImageLoaderController(