add ability to optionally sample a percentage of images to a lower environment's queue bucket, after they've successfully uploaded via S3 copy operation (quick/cheap) - triggered by presence of new s3.sampling.lowerEnvironmentQueueBucket config option. Sampling is 1% by default, but can be configured via s3.sampling.percentage

Author: twrichards

image-loader/app/lib/ImageLoaderConfig.scala

@@ -18,6 +18,9 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res
   val quarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket")
   val uploadToQuarantineEnabled: Boolean = boolean("upload.quarantine.enabled")
 
+  val lowerEnvironmentSamplingPercentageAsDecimal = intOpt("s3.sampling.percentage").getOrElse(1) / 100.0
+  val maybeLowerEnvironmentQueueBucketToSampleInto = stringOpt("s3.sampling.lowerEnvironmentQueueBucket")
+
   val tempDir: File = new File(stringDefault("upload.tmp.dir", "/tmp"))
 
   val thumbWidth: Int = 256

image-loader/app/model/Uploader.scala

@@ -2,6 +2,7 @@ package model
 
 import com.gu.mediaservice.{GridClient, ImageDataMerger}
 import com.gu.mediaservice.lib.Files.createTempFile
+import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId
 
 import java.io.File
 import java.net.URLEncoder
@@ -395,7 +396,25 @@ class Uploader(val store: ImageLoaderStore,
       updateMessage = UpdateMessage(subject = Image, image = Some(imageUpload.image))
       _ <- Future { notifications.publish(updateMessage) }
       // TODO: centralise where all these URLs are constructed
-    } yield UploadStatusUri(s"${config.rootUri}/uploadStatus/${uploadRequest.imageId}")
+    } yield {
+      config.maybeLowerEnvironmentQueueBucketToSampleInto.foreach { lowerEnvironmentQueueBucket =>
+        if (math.random() < config.lowerEnvironmentSamplingPercentageAsDecimal) {
+          val mediaId = imageUpload.image.id
+          logger.info(logMarker, s"Copying $mediaId to lower environment queue bucket $lowerEnvironmentQueueBucket")
+          try {
+            store.client.copyObject(
+              config.imageBucket, fileKeyFromId(mediaId),
+              lowerEnvironmentQueueBucket, s"${uploadRequest.uploadedBy}/${uploadRequest.uploadInfo.filename.getOrElse(mediaId)}"
+            )
+          } catch {
+            case e: Throwable =>
+              logger.error(logMarker, s"Failed to copy $mediaId to lower environment queue bucket $lowerEnvironmentQueueBucket", e)
+          }
+        }
+      }
+
+      UploadStatusUri(s"${config.rootUri}/uploadStatus/${uploadRequest.imageId}")
+    }
 
   }