Merge branch 't2062-special-chars' of github.com:bbc/grid into t2062-…

…special-chars

auth/app/auth/AuthConfig.scala

@@ -5,5 +5,4 @@ import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
 class AuthConfig(resources: GridConfigResources) extends CommonConfig(resources) {
   val rootUri: String = services.authBaseUri
   val mediaApiUri: String = services.apiBaseUri
-  val kahunaUri = services.kahunaBaseUri
 }

build.sbt

@@ -129,8 +129,8 @@ lazy val cropper = playProject("cropper", 9006)
 
 lazy val imageLoader = playProject("image-loader", 9003).settings {
   libraryDependencies ++= Seq(
-    "org.apache.tika" % "tika-core" % "1.20",
-    "com.drewnoakes" % "metadata-extractor" % "2.17.0"
+    "org.apache.tika" % "tika-core" % "1.28.5",
+    "com.drewnoakes" % "metadata-extractor" % "2.19.0"
   )
 }
 
@@ -158,14 +158,14 @@ lazy val thrall = playProject("thrall", 9002)
     pipelineStages := Seq(digest, gzip),
     libraryDependencies ++= Seq(
       "org.codehaus.groovy" % "groovy-json" % "3.0.7",
-      "software.amazon.kinesis" % "amazon-kinesis-client" % "2.4.2",
+      // TODO upgrading kcl to v3? check if you can remove avro override below
+      "software.amazon.kinesis" % "amazon-kinesis-client" % "2.6.0",
       "io.github.streetcontxt" %% "kcl-akka-stream" % "4.1.1",
       "org.testcontainers" % "elasticsearch" % "1.19.2" % Test,
       "com.google.protobuf" % "protobuf-java" % "3.19.6"
     ),
-    // amazon-kinesis-client 2.4.2 brings in a critically vulnerable version of apache avro,
-    // but we cannot upgrade amazon-kinesis-client further until we move into slf4j v2.
-    // TODO when upgrading kinesis-client - can we remove this override?
+    // amazon-kinesis-client 2.6.0 brings in a critically vulnerable version of apache avro,
+    // but we cannot upgrade amazon-kinesis-client further without performing the v2->v3 upgrade https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html
     dependencyOverrides += "org.apache.avro" % "avro" % "1.11.4"
   )
 

collections/app/lib/CollectionsConfig.scala

@@ -8,6 +8,4 @@ class CollectionsConfig(resources: GridConfigResources) extends CommonConfig(res
   val imageCollectionsTable = string("dynamo.table.imageCollections")
 
   val rootUri = services.collectionsBaseUri
-  val kahunaUri = services.kahunaBaseUri
-  val loginUriTemplate = services.loginUriTemplate
 }

common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala

@@ -54,6 +54,8 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
   val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket")
   val maybeFailBucket: Option[String] = stringOpt("s3.fail.bucket")
 
+  val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1_000_000)
+
   // Note: had to make these lazy to avoid init order problems ;_;
   val domainRoot: String = string("domain.root")
   val domainRootOverride: Option[String] = stringOpt("domain.root-override")

cropper/app/lib/CropStore.scala

@@ -31,7 +31,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS
       crops.foldLeft(Map[String, Crop]()) {
         case (map, (s3Object)) => {
           val filename::containingFolder::_ = s3Object.uri.getPath.split("/").reverse.toList
-          var isMaster       = containingFolder == "master"
+          val isMaster = containingFolder == "master"
           val userMetadata   = s3Object.metadata.userMetadata
           val objectMetadata = s3Object.metadata.objectMetadata
 

cropper/app/lib/CropperConfig.scala

@@ -16,8 +16,6 @@ class CropperConfig(resources: GridConfigResources) extends CommonConfig(resourc
 
   val rootUri = services.cropperBaseUri
   val apiUri = services.apiBaseUri
-  val kahunaUri = services.kahunaBaseUri
-  val loginUriTemplate = services.loginUriTemplate
 
   val tempDir: File = new File(stringDefault("crop.output.tmp.dir", "/tmp"))
 

image-loader/app/controllers/ImageLoaderController.scala

@@ -141,7 +141,19 @@ class ImageLoaderController(auth: Authentication,
 
         val approximateReceiveCount = getApproximateReceiveCount(sqsMessage)
 
-        if (approximateReceiveCount > 2) {
+        if(config.maybeUploadLimitInBytes.exists(_ < s3IngestObject.contentLength)){
+          val errorMessage = s"File size exceeds the maximum allowed size (${config.maybeUploadLimitInBytes.get / 1_000_000}MB). Moving to fail bucket."
+          logger.warn(logMarker, errorMessage)
+          store.moveObjectToFailedBucket(s3IngestObject.key)
+          s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
+            uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
+              imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
+            )
+          }
+          metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+          Future.unit
+        }
+        else if (approximateReceiveCount > 2) {
           metrics.abandonedMessagesFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
           val errorMessage = s"File processing has been attempted $approximateReceiveCount times. Moving to fail bucket."
           logger.warn(logMarker, errorMessage)

kahuna/app/views/main.scala.html

@@ -85,6 +85,7 @@
           permissionsDefault: "@kahunaConfig.permissionsDefault",
           defaultShouldBlurGraphicImages: @kahunaConfig.defaultShouldBlurGraphicImages,
           shouldUploadStraightToBucket: @kahunaConfig.shouldUploadStraightToBucket,
+          maybeUploadLimitInBytes: @kahunaConfig.maybeUploadLimitInBytes,
           announcements: @Html(announcements),
           imageTypes: @Html(imageTypes),
         }

kahuna/public/js/upload/manager.js

@@ -31,7 +31,21 @@ upload.factory('uploadManager',
         };
     }
 
-    async function createJobItems(files){
+    async function createJobItems(_files){
+
+      const maybeUploadLimitInBytes = window._clientConfig.maybeUploadLimitInBytes;
+      const maybeFilesAboveSizeLimit = !!maybeUploadLimitInBytes && _files.filter(file => file.size > maybeUploadLimitInBytes);
+
+      if (maybeFilesAboveSizeLimit && maybeFilesAboveSizeLimit.length > 0){
+        alert(`The following files will be skipped as they are above the size limit of ${maybeUploadLimitInBytes / 1_000_000}MB:\n${
+          maybeFilesAboveSizeLimit.map(file => file.name).join("\n")
+        }`);
+      }
+
+      const files = maybeFilesAboveSizeLimit && maybeFilesAboveSizeLimit.length > 0
+        ? _files.filter(file => !maybeFilesAboveSizeLimit.includes(file))
+        : _files;
+
       if (window._clientConfig.shouldUploadStraightToBucket) {
         const mediaIdToFileMap = Object.fromEntries(
           await Promise.all(

leases/app/lib/LeasesConfig.scala

@@ -10,8 +10,6 @@ class LeasesConfig(resources: GridConfigResources) extends CommonConfig(resource
   val leasesTable = string("dynamo.tablename.leasesTable")
 
   val rootUri: String = services.leasesBaseUri
-  val kahunaUri: String = services.kahunaBaseUri
-  val loginUriTemplate: String = services.loginUriTemplate
 
   private def uri(u: String) = URI.create(u)
 

media-api/app/MediaApiComponents.scala

@@ -1,5 +1,4 @@
 import com.gu.mediaservice.lib.aws.ThrallMessageSender
-import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.management.{ElasticSearchHealthCheck, InnerServiceStatusCheckController, Management}
 import com.gu.mediaservice.lib.metadata.SoftDeletedMetadataTable
 import com.gu.mediaservice.lib.play.GridComponents
@@ -14,8 +13,6 @@ import scala.concurrent.Future
 class MediaApiComponents(context: Context) extends GridComponents(context, new MediaApiConfig(_)) {
   final override val buildInfo = utils.buildinfo.BuildInfo
 
-  val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath)
-
   val messageSender = new ThrallMessageSender(config.thrallKinesisStreamConfig)
   val mediaApiMetrics = new MediaApiMetrics(config, actorSystem, applicationLifecycle)
 

media-api/app/lib/ImageResponse.scala

@@ -256,10 +256,6 @@ class ImageResponse(config: MediaApiConfig, s3Client: S3Client, usageQuota: Usag
   def makeImgopsUri(uri: URI): String =
     config.imgopsUri + List(uri.getPath, uri.getRawQuery).mkString("?") + "{&w,h,q}"
 
-  def makeOptimisedPngImageopsUri(uri: URI): String = {
-    config.imgopsUri + List(uri.getPath, uri.getRawQuery).mkString("?") + "{&w, h, q}"
-  }
-
   private def updateCustomSpecialInstructions(source: JsValue): Reads[JsObject] = {
      (source \ "usageRights" \ "category") match {
         case JsDefined(category) =>

media-api/app/lib/MediaApiConfig.scala

@@ -35,13 +35,11 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla
   val kahunaUri: String = services.kahunaBaseUri
   val cropperUri: String = services.cropperBaseUri
   val loaderUri: String = services.loaderBaseUri
-  val projectionUri: String = services.projectionBaseUri
   val metadataUri: String = services.metadataBaseUri
   val imgopsUri: String = services.imgopsBaseUri
   val usageUri: String = services.usageBaseUri
   val leasesUri: String = services.leasesBaseUri
   val authUri: String = services.authBaseUri
-  val loginUriTemplate: String = services.loginUriTemplate
   val collectionsUri: String = services.collectionsBaseUri
 
   val requiredMetadata = NonEmptyList("credit", "description", "usageRights")

metadata-editor/app/MetadataEditorComponents.scala

@@ -1,4 +1,3 @@
-import com.gu.mediaservice.lib.imaging.ImageOperations
 import com.gu.mediaservice.lib.management.InnerServiceStatusCheckController
 import com.gu.mediaservice.lib.play.GridComponents
 import controllers.{EditsApi, EditsController, SyndicationController}
@@ -12,7 +11,6 @@ class MetadataEditorComponents(context: Context) extends GridComponents(context,
   val editsStore = new EditsStore(config)
   val syndicationStore = new SyndicationStore(config)
   val notifications = new Notifications(config)
-  val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath)
 
   val metrics = new MetadataEditorMetrics(config, actorSystem, applicationLifecycle)
   val messageConsumer = new MetadataSqsMessageConsumer(config, metrics, editsStore)

metadata-editor/app/lib/EditsConfig.scala

@@ -1,14 +1,9 @@
 package lib
 
-import com.amazonaws.regions.{Region, RegionUtils}
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
 
 
 class EditsConfig(resources: GridConfigResources) extends CommonConfig(resources) {
-  val dynamoRegion: Region = RegionUtils.getRegion(string("aws.region"))
-
-  val collectionsBucket: String = string("s3.collections.bucket")
-
   val editsTable = string("dynamo.table.edits")
   val editsTablePhotoshootIndex = string("dynamo.globalsecondaryindex.edits.photoshoots")
   val syndicationTable = string("dynamo.table.syndication")

usage/app/lib/UsageConfig.scala

@@ -1,6 +1,5 @@
 package lib
 
-import com.amazonaws.regions.{Region, RegionUtils}
 import com.amazonaws.services.identitymanagement._
 import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources}
 import com.gu.mediaservice.lib.logging.GridLogging
@@ -12,13 +11,9 @@ import scala.util.Try
 case class KinesisReaderConfig(streamName: String, arn: String, appName: String)
 
 class UsageConfig(resources: GridConfigResources) extends CommonConfig(resources) with GridLogging {
-  val rootUri: String = services.metadataBaseUri
-  val kahunaUri: String = services.kahunaBaseUri
   val usageUri: String = services.usageBaseUri
   val apiUri: String = services.apiBaseUri
-  val loginUriTemplate: String = services.loginUriTemplate
 
-  val defaultPageSize = 100
   val defaultMaxRetries = 4
   val defaultMaxPrintRequestSizeInKb = 500
   val defaultDateLimit = "2016-01-01T00:00:00+00:00"
@@ -29,7 +24,6 @@ class UsageConfig(resources: GridConfigResources) extends CommonConfig(resources
   val capiPreviewUrl = string("capi.preview.url")
   val capiPreviewRole = stringOpt("capi.preview.role")
   val capiApiKey = string("capi.apiKey")
-  val capiPageSize: Int = intDefault("capi.page.size", defaultPageSize)
   val capiMaxRetries: Int = intDefault("capi.maxRetries", defaultMaxRetries)
 
   val usageDateLimit: String = stringDefault("usage.dateLimit", defaultDateLimit)
@@ -41,7 +35,6 @@ class UsageConfig(resources: GridConfigResources) extends CommonConfig(resources
 
   val usageRecordTable = string("dynamo.tablename.usageRecordTable")
 
-  val dynamoRegion: Region = RegionUtils.getRegion(string("aws.region"))
   val awsRegionName = string("aws.region")
 
   private val iamClient: AmazonIdentityManagement = withAWSCredentials(AmazonIdentityManagementClientBuilder.standard()).build()