diff --git a/app/controllers/UploadController.scala b/app/controllers/UploadController.scala index 9f4b60b75..b77db8088 100644 --- a/app/controllers/UploadController.scala +++ b/app/controllers/UploadController.scala @@ -26,7 +26,7 @@ class UploadController(override val authActions: HMACAuthActions, awsConfig: AWS import authActions.APIAuthAction private val credsGenerator = new CredentialsGenerator(awsConfig) - private val uploadDecorator = new UploadDecorator(awsConfig, stepFunctions) + private val uploadDecorator = new UploadDecorator(awsConfig.scanamo, awsConfig.cacheTableName, stepFunctions) def list(atomId: String) = APIAuthAction { req => val atom = MediaAtom.fromThrift(getPreviewAtom(atomId)) diff --git a/app/data/DataStores.scala b/app/data/DataStores.scala index bac1e0818..95d3bbf5c 100644 --- a/app/data/DataStores.scala +++ b/app/data/DataStores.scala @@ -14,7 +14,7 @@ class DataStores(aws: AWSConfig with SNSAccess, capi: CapiAccess) { val preview = new PreviewDynamoDataStore(aws.dynamoDB, aws.dynamoTableName) val published = new PublishedDynamoDataStore(aws.dynamoDB, aws.publishedDynamoTableName) - val pluto: PlutoDataStore = new PlutoDataStore(aws.dynamoDB, aws.manualPlutoDynamo) + val pluto: PlutoDataStore = new PlutoDataStore(aws.scanamo, aws.manualPlutoDynamo) val livePublisher: AtomPublisher = aws.capiContentEventsTopicName match { case Some(capiContentEventsTopicName) => @@ -48,8 +48,8 @@ class DataStores(aws: AWSConfig with SNSAccess, capi: CapiAccess) { val reindexPublished: PublishedKinesisAtomReindexer = new PublishedKinesisAtomReindexer(aws.publishedKinesisReindexStreamName, aws.crossAccountKinesisClient) - val plutoCommissionStore: PlutoCommissionDataStore = new PlutoCommissionDataStore(aws) - val plutoProjectStore: PlutoProjectDataStore = new PlutoProjectDataStore(aws, plutoCommissionStore) + val plutoCommissionStore: PlutoCommissionDataStore = new PlutoCommissionDataStore(aws.scanamo, aws.plutoCommissionTableName) + val plutoProjectStore: PlutoProjectDataStore = new PlutoProjectDataStore(aws.scanamo, aws.plutoProjectTableName, plutoCommissionStore) val atomListStore = AtomListStore(aws.stage, capi, preview) diff --git a/app/di.scala b/app/di.scala index be414b635..c7c6f3556 100644 --- a/app/di.scala +++ b/app/di.scala @@ -73,7 +73,7 @@ class MediaAtomMaker(context: Context) private val capi = new Capi(config) private val stores = new DataStores(aws, capi) - private val permissions = new MediaAtomMakerPermissionsProvider(aws.stage, aws.region.getName, aws.credentials.instance) + private val permissions = new MediaAtomMakerPermissionsProvider(aws.stage, aws.region.getName, aws.credentials.instance.v1) private val reindexer = buildReindexer() diff --git a/app/util/AWS.scala b/app/util/AWS.scala index 1d9f36b41..06d461cbd 100644 --- a/app/util/AWS.scala +++ b/app/util/AWS.scala @@ -26,7 +26,7 @@ class AWSConfig(override val config: Config, override val credentials: AwsCreden lazy val ec2Client = AmazonEC2ClientBuilder .standard() .withRegion(region.getName) - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.v1) .build() lazy val pinboardLoaderUrl = getString("panda.domain").map(domain => s"https://pinboard.$domain/pinboard.loader.js") @@ -43,7 +43,7 @@ class AWSConfig(override val config: Config, override val credentials: AwsCreden lazy val expiryPollerName = "Expiry" lazy val expiryPollerLastName = "Poller" - final override def regionName = getString("aws.region") + final override def region = AwsAccess.regionFrom(this) final override def readTag(tagName: String) = { val tagsResult = ec2Client.describeTags( diff --git a/app/util/UploadDecorator.scala b/app/util/UploadDecorator.scala index b4ac81d3a..2ecb7d1eb 100644 --- a/app/util/UploadDecorator.scala +++ b/app/util/UploadDecorator.scala @@ -1,14 +1,13 @@ package util -import com.gu.media.aws.{DynamoAccess, UploadAccess} import com.gu.media.model.{ClientAsset, ClientAssetMetadata} import com.gu.media.upload.model.Upload -import org.scanamo.{Scanamo, Table} +import org.scanamo.generic.auto._ import org.scanamo.syntax._ -import org.scanamo.auto._ +import org.scanamo.{Scanamo, Table} -class UploadDecorator(aws: DynamoAccess with UploadAccess, stepFunctions: StepFunctions) { - private val table = Table[Upload](aws.cacheTableName) +class UploadDecorator(scanamo: Scanamo, cacheTableName: String, stepFunctions: StepFunctions) { + private val table = Table[Upload](cacheTableName) def addMetadata(atomId: String, video: ClientAsset): ClientAsset = { val id = s"$atomId-${video.id}" @@ -28,10 +27,6 @@ class UploadDecorator(aws: DynamoAccess with UploadAccess, stepFunctions: StepFu } } - private def getUpload(id: String): Option[Upload] = { - val op = table.get('id -> id) - val result = Scanamo.exec(aws.dynamoDB)(op).flatMap(_.right.toOption) - - result orElse { stepFunctions.getById(id) } - } + private def getUpload(id: String): Option[Upload] = + scanamo.exec(table.get("id" === id)).flatMap(_.toOption) orElse { stepFunctions.getById(id) } } diff --git a/build.sbt b/build.sbt index 84f92ce3b..a5abc9c15 100644 --- a/build.sbt +++ b/build.sbt @@ -4,11 +4,12 @@ import scala.sys.process._ val scroogeVersion = "4.12.0" val awsVersion = "1.11.678" +val awsV2Version = "2.21.17" val pandaVersion = "1.2.0" val pandaHmacVersion = "2.1.0" val atomMakerVersion = "1.3.4" val typesafeConfigVersion = "1.4.0" // to match what we get from Play transitively -val scanamoVersion = "1.0.0-M9" +val scanamoVersion = "1.0.0-M28" val playJsonExtensionsVersion = "0.40.2" val okHttpVersion = "2.4.0" @@ -47,6 +48,7 @@ lazy val commonSettings = Seq( ThisBuild / scalaVersion := "2.12.16", scalacOptions ++= Seq("-feature", "-deprecation"/*, "-Xfatal-warnings"*/), ThisBuild / organization := "com.gu", + libraryDependencySchemes += "org.scala-lang.modules" %% "scala-java8-compat" % VersionScheme.Always, resolvers ++= Seq( "Sonatype OSS" at "https://oss.sonatype.org/content/repositories/releases/", @@ -102,10 +104,12 @@ lazy val common = (project in file("common")) "com.amazonaws" % "aws-lambda-java-core" % awsLambdaCoreVersion, "com.amazonaws" % "aws-java-sdk-s3" % awsVersion, "com.amazonaws" % "aws-java-sdk-dynamodb" % awsVersion, + "software.amazon.awssdk" % "dynamodb" % awsV2Version, "com.amazonaws" % "aws-java-sdk-kinesis" % awsVersion, "ai.x" %% "play-json-extensions" % playJsonExtensionsVersion, "ch.qos.logback" % "logback-classic" % logbackClassicVersion, "com.amazonaws" % "aws-java-sdk-sts" % awsVersion, + "software.amazon.awssdk" % "sts" % awsV2Version, "com.amazonaws" % "aws-java-sdk-elastictranscoder" % awsVersion, "org.scanamo" %% "scanamo" % scanamoVersion, "com.squareup.okhttp" % "okhttp" % okHttpVersion, @@ -132,6 +136,7 @@ lazy val app = (project in file(".")) ehcache, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion, "com.amazonaws" % "aws-java-sdk-sts" % awsVersion, + "software.amazon.awssdk" % "sts" % awsV2Version, "com.amazonaws" % "aws-java-sdk-ec2" % awsVersion, "org.scalatestplus.play" %% "scalatestplus-play" % scalaTestPlusPlayVersion % "test", "org.mockito" % "mockito-core" % mockitoVersion % "test", diff --git a/common/src/main/scala/com/gu/media/PlutoDataStore.scala b/common/src/main/scala/com/gu/media/PlutoDataStore.scala index 61378d7d3..ee459953f 100644 --- a/common/src/main/scala/com/gu/media/PlutoDataStore.scala +++ b/common/src/main/scala/com/gu/media/PlutoDataStore.scala @@ -1,20 +1,17 @@ package com.gu.media -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB -import com.amazonaws.services.dynamodbv2.model.DeleteItemResult import com.gu.media.model.PlutoSyncMetadataMessage -import org.scanamo.error.DynamoReadError +import org.scanamo.generic.auto._ import org.scanamo.syntax._ import org.scanamo.{Scanamo, Table} -import org.scanamo.auto._ -class PlutoDataStore(client: AmazonDynamoDB, dynamoTableName: String) { +class PlutoDataStore(scanamo: Scanamo, dynamoTableName: String) { val table = Table[PlutoSyncMetadataMessage](dynamoTableName) def getUploadsWithAtomId(id: String): List[PlutoSyncMetadataMessage] = { val atomIdIndex = table.index("atom-id") - val results = Scanamo.exec(client)(atomIdIndex.query('atomId -> id)) + val results = scanamo.exec(atomIdIndex.query("atomId" === id)) val errors = results.collect { case Left(err) => err } if (errors.nonEmpty) { @@ -25,8 +22,8 @@ class PlutoDataStore(client: AmazonDynamoDB, dynamoTableName: String) { } def get(id: String): Option[PlutoSyncMetadataMessage] = { - val operation = table.get('id -> id) - val result = Scanamo.exec(client)(operation) + val operation = table.get("id" === id) + val result = scanamo.exec(operation) result.map { case Right(item) => item @@ -34,12 +31,12 @@ class PlutoDataStore(client: AmazonDynamoDB, dynamoTableName: String) { } } - def put(item: PlutoSyncMetadataMessage): Option[Either[DynamoReadError, PlutoSyncMetadataMessage]] = { - Scanamo.exec(client)(table.put(item)) + def put(item: PlutoSyncMetadataMessage): Unit = { + scanamo.exec(table.put(item)) } - def delete(id: String): DeleteItemResult = { - Scanamo.exec(client)(table.delete('id -> id)) + def delete(id: String): Unit = { + scanamo.exec(table.delete("id" === id)) } case class DynamoPlutoTableException(err: String) extends RuntimeException(err) diff --git a/common/src/main/scala/com/gu/media/aws/AwsAccess.scala b/common/src/main/scala/com/gu/media/aws/AwsAccess.scala index a6c46b168..4b3cc2013 100644 --- a/common/src/main/scala/com/gu/media/aws/AwsAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/AwsAccess.scala @@ -5,17 +5,17 @@ import com.amazonaws.regions.{Region, Regions} import com.gu.media.Settings trait AwsAccess { this: Settings => - def regionName: Option[String] def readTag(tag: String): Option[String] val credentials: AwsCredentials + // To avoid renaming references everywhere - def credsProvider: AWSCredentialsProvider = credentials.instance + def credsProvider: AWSCredentialsProvider = credentials.instance.v1 - final def defaultRegion: Region = Region.getRegion(Regions.EU_WEST_1) - final def region: Region = regionName - .map { name => Region.getRegion(Regions.fromName(name)) } - .getOrElse(defaultRegion) + def region: Region + + final def awsV2Region: software.amazon.awssdk.regions.Region = + software.amazon.awssdk.regions.Region.of(region.getName) // These are injected as environment variables when running in a Lambda (unfortunately they cannot be tagged) final val stage = sys.env.getOrElse("STAGE", readTag("Stage").getOrElse("DEV")) @@ -24,3 +24,11 @@ trait AwsAccess { this: Settings => final val stack: Option[String] = if (isDev) Some("media-atom-maker") else readTag("Stack") final val app: String = if (isDev) "media-atom-maker" else readTag("App").getOrElse("media-atom-maker") } + +object AwsAccess { + def regionFrom(maybeName: Option[String]): Region = maybeName + .map { name => Region.getRegion(Regions.fromName(name)) } + .getOrElse(Region.getRegion(Regions.EU_WEST_1)) + + def regionFrom(settings: Settings): Region = regionFrom(settings.getString("aws.region")) +} diff --git a/common/src/main/scala/com/gu/media/aws/AwsCredentials.scala b/common/src/main/scala/com/gu/media/aws/AwsCredentials.scala index e87dc2770..2a0fdb5ee 100644 --- a/common/src/main/scala/com/gu/media/aws/AwsCredentials.scala +++ b/common/src/main/scala/com/gu/media/aws/AwsCredentials.scala @@ -1,20 +1,20 @@ package com.gu.media.aws -import com.amazonaws.auth._ -import com.amazonaws.auth.profile.ProfileCredentialsProvider -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder import com.gu.media.Settings -case class AwsCredentials(instance: AWSCredentialsProvider, crossAccount: AWSCredentialsProvider, - upload: AWSCredentialsProvider) +case class AwsCredentials( + instance: AwsCredentialsProvidersForBothSdkVersions, + crossAccount: AwsCredentialsProvidersForBothSdkVersions, + upload: AwsCredentialsProvidersForBothSdkVersions +) object AwsCredentials { def dev(settings: Settings): AwsCredentials = { val profile = settings.getMandatoryString("aws.profile") - val instance = new ProfileCredentialsProvider(profile) + val instance = AwsCredentialsProvidersForBothSdkVersions.profile(profile) // To enable publishing to CAPI code from DEV, update the kinesis streams in config and uncomment below: - // val crossAccount = new ProfileCredentialsProvider("composer") + // val crossAccount = AwsCredentialsProvidersForBothSdkVersions.profile("composer") val crossAccount = instance val upload = devUpload(settings) @@ -23,7 +23,7 @@ object AwsCredentials { } def app(settings: Settings): AwsCredentials = { - val instance = InstanceProfileCredentialsProvider.getInstance() + val instance = AwsCredentialsProvidersForBothSdkVersions.instance() val crossAccount = assumeCrossAccountRole(instance, settings) @@ -31,30 +31,23 @@ object AwsCredentials { } def lambda(): AwsCredentials = { - val instance = new EnvironmentVariableCredentialsProvider() + val instance = AwsCredentialsProvidersForBothSdkVersions.environmentVariables() AwsCredentials(instance, crossAccount = instance, upload = instance) } - private def devUpload(settings: Settings): AWSCredentialsProvider = { + private def devUpload(settings: Settings): AwsCredentialsProvidersForBothSdkVersions = { // Only required in dev (because federated credentials such as those from Janus cannot do STS requests). // Instance profile credentials are sufficient when deployed. val accessKey = settings.getMandatoryString("aws.upload.accessKey", "This is the AwsId output of the dev cloudformation") val secretKey = settings.getMandatoryString("aws.upload.secretKey", "This is the AwsSecret output of the dev cloudformation") - new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)) + AwsCredentialsProvidersForBothSdkVersions.static(accessKey, secretKey) } - private def assumeCrossAccountRole(instance: AWSCredentialsProvider, settings: Settings) = { + private def assumeCrossAccountRole(instance: AwsCredentialsProvidersForBothSdkVersions, settings: Settings): AwsCredentialsProvidersForBothSdkVersions = { val crossAccountRoleArn = settings.getMandatoryString("aws.kinesis.stsCapiRoleToAssume", "Role to assume to access CAPI streams (in format arn:aws:iam:::role/)") - assumeAccountRole(instance, crossAccountRoleArn, "capi") - } - - private def assumeAccountRole(instance: AWSCredentialsProvider, roleArn: String, sessionNameSuffix: String): AWSCredentialsProvider = { - val securityTokens = AWSSecurityTokenServiceClientBuilder.standard().withCredentials(instance).build() - - new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, s"media-atom-maker-${sessionNameSuffix}") - .withStsClient(securityTokens).build() + instance.assumeAccountRole(crossAccountRoleArn, "capi", AwsAccess.regionFrom(settings).getName) } } diff --git a/common/src/main/scala/com/gu/media/aws/AwsCredentialsProvidersForBothSdkVersions.scala b/common/src/main/scala/com/gu/media/aws/AwsCredentialsProvidersForBothSdkVersions.scala new file mode 100644 index 000000000..1ae75b17b --- /dev/null +++ b/common/src/main/scala/com/gu/media/aws/AwsCredentialsProvidersForBothSdkVersions.scala @@ -0,0 +1,50 @@ +package com.gu.media.aws + +import com.amazonaws.auth._ +import com.amazonaws.auth.profile.ProfileCredentialsProvider +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder +import com.gu.media.Settings +import com.gu.media.aws.AwsV2Util +import software.amazon.awssdk.auth.{credentials => awsv2} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.sts.{StsClient, StsClientBuilder} +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest + +case class AwsCredentialsProvidersForBothSdkVersions( + v1: com.amazonaws.auth.AWSCredentialsProvider, + v2: software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +) { + def assumeAccountRole(roleArn: String, sessionNameSuffix: String, regionName: String): AwsCredentialsProvidersForBothSdkVersions = { + val roleSessionName = s"media-atom-maker-$sessionNameSuffix" + AwsCredentialsProvidersForBothSdkVersions( + new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, roleSessionName) + .withStsClient(AWSSecurityTokenServiceClientBuilder.standard().withCredentials(v1).build()).build(), + software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider.builder() + .stsClient(AwsV2Util.buildSync[StsClient, StsClientBuilder](StsClient.builder(), v2, Region.of(regionName))) + .refreshRequest(AssumeRoleRequest.builder.roleSessionName(roleSessionName).roleArn(roleArn).build) + .build() + ) + } +} + +object AwsCredentialsProvidersForBothSdkVersions { + def profile(name: String): AwsCredentialsProvidersForBothSdkVersions = AwsCredentialsProvidersForBothSdkVersions( + new ProfileCredentialsProvider(name), + awsv2.ProfileCredentialsProvider.create(name) + ) + + def instance(): AwsCredentialsProvidersForBothSdkVersions = AwsCredentialsProvidersForBothSdkVersions( + InstanceProfileCredentialsProvider.getInstance(), + awsv2.InstanceProfileCredentialsProvider.create() + ) + + def environmentVariables(): AwsCredentialsProvidersForBothSdkVersions = AwsCredentialsProvidersForBothSdkVersions( + new EnvironmentVariableCredentialsProvider(), + awsv2.EnvironmentVariableCredentialsProvider.create() + ) + + def static(accessKey: String, secretKey: String): AwsCredentialsProvidersForBothSdkVersions = AwsCredentialsProvidersForBothSdkVersions( + new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)), + awsv2.StaticCredentialsProvider.create(awsv2.AwsBasicCredentials.create(accessKey, secretKey)) + ) +} diff --git a/common/src/main/scala/com/gu/media/aws/AwsV2Util.scala b/common/src/main/scala/com/gu/media/aws/AwsV2Util.scala new file mode 100644 index 000000000..701425379 --- /dev/null +++ b/common/src/main/scala/com/gu/media/aws/AwsV2Util.scala @@ -0,0 +1,15 @@ +package com.gu.media.aws + +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.awscore.client.builder.{AwsClientBuilder, AwsSyncClientBuilder} +import software.amazon.awssdk.http.apache.ApacheHttpClient + +object AwsV2Util { + def buildSync[T, B <: AwsClientBuilder[B, T] with AwsSyncClientBuilder[B, T]]( + builder: B, creds: AwsCredentialsProvider, region: software.amazon.awssdk.regions.Region + ): T = builder + .httpClientBuilder(ApacheHttpClient.builder()) + .credentialsProvider(creds) + .region(region) + .build() +} diff --git a/common/src/main/scala/com/gu/media/aws/DynamoAccess.scala b/common/src/main/scala/com/gu/media/aws/DynamoAccess.scala index f6a972d7b..3569abd31 100644 --- a/common/src/main/scala/com/gu/media/aws/DynamoAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/DynamoAccess.scala @@ -2,6 +2,12 @@ package com.gu.media.aws import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder import com.gu.media.Settings +import com.gu.media.aws.AwsV2Util.buildSync +import org.scanamo.Scanamo +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.awscore.client.builder.{AwsClientBuilder, AwsSyncClientBuilder} +import software.amazon.awssdk.http.apache.ApacheHttpClient +import software.amazon.awssdk.services.dynamodb.{DynamoDbClient, DynamoDbClientBuilder} trait DynamoAccess { this: Settings with AwsAccess => lazy val dynamoTableName: String = sys.env.getOrElse("ATOM_TABLE_NAME", @@ -24,4 +30,9 @@ trait DynamoAccess { this: Settings with AwsAccess => .withCredentials(credsProvider) .withRegion(region.getName) .build() + + lazy val dynamoDbSdkV2: DynamoDbClient = + buildSync[DynamoDbClient, DynamoDbClientBuilder](DynamoDbClient.builder(), credentials.instance.v2, awsV2Region) + + lazy val scanamo: Scanamo = Scanamo(dynamoDbSdkV2) } diff --git a/common/src/main/scala/com/gu/media/aws/KinesisAccess.scala b/common/src/main/scala/com/gu/media/aws/KinesisAccess.scala index cca55b5c7..451461934 100644 --- a/common/src/main/scala/com/gu/media/aws/KinesisAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/KinesisAccess.scala @@ -21,12 +21,12 @@ trait KinesisAccess { this: Settings with AwsAccess with Logging => val syncWithPluto: Boolean = getBoolean("pluto.sync").getOrElse(false) lazy val crossAccountKinesisClient = AmazonKinesisClientBuilder.standard() - .withCredentials(credentials.crossAccount) + .withCredentials(credentials.crossAccount.v1) .withRegion(region.getName) .build() lazy val kinesisClient = AmazonKinesisClientBuilder.standard() - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.v1) .withRegion(region.getName) .build() diff --git a/common/src/main/scala/com/gu/media/aws/SNSAccess.scala b/common/src/main/scala/com/gu/media/aws/SNSAccess.scala index 36f8a191e..bf5f2f730 100644 --- a/common/src/main/scala/com/gu/media/aws/SNSAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/SNSAccess.scala @@ -11,6 +11,6 @@ trait SNSAccess { this: Settings with AwsAccess => lazy val snsClient = AmazonSNSClientBuilder.standard() .withRegion(Regions.fromName(region.getName)) - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.v1) .build() } diff --git a/common/src/main/scala/com/gu/media/aws/UploadAccess.scala b/common/src/main/scala/com/gu/media/aws/UploadAccess.scala index 8dcfd6310..753c67b60 100644 --- a/common/src/main/scala/com/gu/media/aws/UploadAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/UploadAccess.scala @@ -30,7 +30,7 @@ trait UploadAccess { this: Settings with AwsAccess => throw new IllegalArgumentException("aws.upload.role must be in ARN format: arn:aws:iam:::role/") } - AWSSecurityTokenServiceClientBuilder.standard().withCredentials(credentials.upload).withRegion(region.getName).build() + AWSSecurityTokenServiceClientBuilder.standard().withCredentials(credentials.upload.v1).withRegion(region.getName).build() } private def getPipelineArn() = { diff --git a/common/src/main/scala/com/gu/media/lambda/LambdaBase.scala b/common/src/main/scala/com/gu/media/lambda/LambdaBase.scala index 17538b065..bad5dfcda 100644 --- a/common/src/main/scala/com/gu/media/lambda/LambdaBase.scala +++ b/common/src/main/scala/com/gu/media/lambda/LambdaBase.scala @@ -9,10 +9,10 @@ import com.gu.media.aws.{AwsAccess, AwsCredentials, HMACSettings} import com.typesafe.config.{Config, ConfigFactory} trait LambdaBase extends Settings with AwsAccess with HMACSettings { - final override def regionName = sys.env.get("REGION") + final override def region = AwsAccess.regionFrom(sys.env.get("REGION")) final override def readTag(tag: String) = sys.env.get(tag.toUpperCase(Locale.ENGLISH)) - final override val credentials = AwsCredentials.lambda() + final override val credentials: AwsCredentials = AwsCredentials.lambda() private val remoteConfig = downloadConfig() private val mergedConfig = remoteConfig.withFallback(ConfigFactory.load()) @@ -24,7 +24,7 @@ trait LambdaBase extends Settings with AwsAccess with HMACSettings { case (Some(bucket), Some(key)) => val defaultRegionS3 = AmazonS3ClientBuilder .standard() - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.v1) .withRegion(region.getName) .build() diff --git a/common/src/main/scala/com/gu/media/lambda/LambdaYoutubeCredentials.scala b/common/src/main/scala/com/gu/media/lambda/LambdaYoutubeCredentials.scala index 7ecc48dd3..6d5410a53 100644 --- a/common/src/main/scala/com/gu/media/lambda/LambdaYoutubeCredentials.scala +++ b/common/src/main/scala/com/gu/media/lambda/LambdaYoutubeCredentials.scala @@ -10,7 +10,7 @@ trait LambdaYoutubeCredentials { self: AwsAccess => case (Some(bucket), Some(key)) => val defaultRegionS3 = AmazonS3ClientBuilder .standard() - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.v1) .withRegion(region.getName) .build() diff --git a/common/src/main/scala/com/gu/media/pluto/PlutoCommissionDataStore.scala b/common/src/main/scala/com/gu/media/pluto/PlutoCommissionDataStore.scala index 003576935..c3d69b209 100644 --- a/common/src/main/scala/com/gu/media/pluto/PlutoCommissionDataStore.scala +++ b/common/src/main/scala/com/gu/media/pluto/PlutoCommissionDataStore.scala @@ -1,29 +1,23 @@ package com.gu.media.pluto -import com.amazonaws.services.dynamodbv2.model.DeleteItemResult -import com.gu.media.aws.DynamoAccess import com.gu.media.logging.Logging import com.gu.media.pluto.PlutoItem.numericIdsOnlyFilter -import org.scanamo.error.DynamoReadError +import org.scanamo.generic.auto._ import org.scanamo.syntax._ -import org.scanamo.{Scanamo, Table} -import org.scanamo.auto._ +import org.scanamo.{DynamoReadError, Scanamo, Table} case class PlutoCommissionDataStoreException(err: String) extends Exception(err) -class PlutoCommissionDataStore(aws: DynamoAccess) extends Logging { - private val table = Table[PlutoCommission](aws.plutoCommissionTableName) +class PlutoCommissionDataStore(scanamo: Scanamo, plutoCommissionTableName: String) extends Logging { + private val table = Table[PlutoCommission](plutoCommissionTableName) def getById(commissionId: String): Option[Either[DynamoReadError, PlutoCommission]] = { log.info(s"getting commission $commissionId") - Scanamo.exec(aws.dynamoDB)(table.get('id -> commissionId)) + scanamo.exec(table.get("id" === commissionId)) } def list(): List[PlutoCommission] = { - val op = table.scan() - val results = Scanamo.exec(aws.dynamoDB)(op) - - results.collect { + scanamo.exec(table.scan()).collect { case Left(error) => log.error("failed to list pluto commissions") throw PlutoCommissionDataStoreException(error.toString) @@ -32,15 +26,14 @@ class PlutoCommissionDataStore(aws: DynamoAccess) extends Logging { }.filter(numericIdsOnlyFilter).sortBy(_.title) } - def upsert(plutoUpsertRequest: PlutoUpsertRequest): Option[Either[DynamoReadError, PlutoCommission]] = { + def upsert(plutoUpsertRequest: PlutoUpsertRequest): Unit = { val commission = PlutoCommission.build(plutoUpsertRequest) log.info(s"upserting pluto commission ${commission.id}") - val op = table.put(commission) - Scanamo.exec(aws.dynamoDB)(op) + scanamo.exec(table.put(commission)) } - def delete(commissionId: String): DeleteItemResult = { + def delete(commissionId: String): Unit = { log.info(s"deleting commission $commissionId") - Scanamo.exec(aws.dynamoDB)(table.delete('id -> commissionId)) + scanamo.exec(table.delete("id" === commissionId)) } } diff --git a/common/src/main/scala/com/gu/media/pluto/PlutoProjectDataStore.scala b/common/src/main/scala/com/gu/media/pluto/PlutoProjectDataStore.scala index d751b52ba..f1309c28c 100644 --- a/common/src/main/scala/com/gu/media/pluto/PlutoProjectDataStore.scala +++ b/common/src/main/scala/com/gu/media/pluto/PlutoProjectDataStore.scala @@ -1,36 +1,31 @@ package com.gu.media.pluto -import com.amazonaws.services.dynamodbv2.model.DeleteItemResult -import com.gu.media.aws.DynamoAccess import com.gu.media.logging.Logging import com.gu.media.pluto.PlutoItem.numericIdsOnlyFilter import org.joda.time.{DateTime, DateTimeZone} import org.scanamo.DynamoFormat._ +import org.scanamo.generic.auto._ import org.scanamo.syntax._ -import org.scanamo.{DynamoFormat, Scanamo, Table} -import org.scanamo.auto._ +import org.scanamo.{DynamoFormat, DynamoReadError, Scanamo, Table} case class PlutoProjectDataStoreException(err: String) extends Exception(err) -class PlutoProjectDataStore(aws: DynamoAccess, plutoCommissionDataStore: PlutoCommissionDataStore) extends Logging { - implicit val dateTimeFormat = DynamoFormat.coercedXmap[DateTime, String, IllegalArgumentException]( - DateTime.parse(_).withZone(DateTimeZone.UTC) - )(_.toString) +class PlutoProjectDataStore(scanamo: Scanamo, plutoProjectTableName: String, plutoCommissionDataStore: PlutoCommissionDataStore) extends Logging { + implicit val dateTimeFormat: DynamoFormat[DateTime] = DynamoFormat.coercedXmap[DateTime, String, IllegalArgumentException]( + DateTime.parse(_).withZone(DateTimeZone.UTC), + _.toString + ) - private val table = Table[PlutoProject](aws.plutoProjectTableName) + private val table = Table[PlutoProject](plutoProjectTableName) private val commmissionIndex = table.index("commission-index") - def getById(projectId: String) = { + def getById(projectId: String): Option[Either[DynamoReadError, PlutoProject]] = { log.info(s"getting project $projectId") - Scanamo.exec(aws.dynamoDB)(table.get('id -> projectId)) + scanamo.exec(table.get("id" === projectId)) } def getByCommissionId(commissionId: String): List[PlutoProject] = { - val op = commmissionIndex.query('commissionId -> commissionId) - - val results = Scanamo.exec(aws.dynamoDB)(op) - - results.collect { + scanamo.exec(commmissionIndex.query("commissionId" === commissionId)).collect { case Left(error) => log.error(s"failed to get pluto projects for commission $commissionId") throw PlutoProjectDataStoreException(error.toString) @@ -44,13 +39,12 @@ class PlutoProjectDataStore(aws: DynamoAccess, plutoCommissionDataStore: PlutoCo val project = PlutoProject.build(plutoUpsertRequest) log.info(s"upserting pluto project ${project.id}") - val op = table.put(project) - Scanamo.exec(aws.dynamoDB)(op) + scanamo.exec(table.put(project)) project } - def deleteByCommissionId(commissionId: String): List[DeleteItemResult] = { + def deleteByCommissionId(commissionId: String): Unit = { log.info(s"deleting all pluto projects for commission $commissionId") - getByCommissionId(commissionId).map(project => Scanamo.exec(aws.dynamoDB)(table.delete('id -> project.id))) + getByCommissionId(commissionId).foreach(project => scanamo.exec(table.delete("id" === project.id))) } } diff --git a/common/src/main/scala/com/gu/media/upload/PlutoUploadActions.scala b/common/src/main/scala/com/gu/media/upload/PlutoUploadActions.scala index 73b764b62..c7d1bbfd6 100644 --- a/common/src/main/scala/com/gu/media/upload/PlutoUploadActions.scala +++ b/common/src/main/scala/com/gu/media/upload/PlutoUploadActions.scala @@ -8,7 +8,7 @@ import com.gu.media.{PlutoDataStore, Settings} class PlutoUploadActions(config: Settings with DynamoAccess with KinesisAccess with SESSettings) extends Logging { private val mailer = new Mailer(config) - private val plutoStore = new PlutoDataStore(config.dynamoDB, config.manualPlutoDynamo) + private val plutoStore = new PlutoDataStore(config.scanamo, config.manualPlutoDynamo) def sendToPluto(plutoIntegrationMessage: PlutoIntegrationMessage): Unit = { diff --git a/common/src/test/scala/com/gu/media/util/MediaAtomHelpersTest.scala b/common/src/test/scala/com/gu/media/util/MediaAtomHelpersTest.scala index 6d81e633c..846c3a24e 100644 --- a/common/src/test/scala/com/gu/media/util/MediaAtomHelpersTest.scala +++ b/common/src/test/scala/com/gu/media/util/MediaAtomHelpersTest.scala @@ -42,7 +42,7 @@ class MediaAtomHelpersTest extends FunSuite with MustMatchers { } private def assets(atom: Atom): Seq[Asset] = { - atom.data.asInstanceOf[AtomData.Media].media.assets + atom.data.asInstanceOf[AtomData.Media].media.assets.toSeq } private def asset(): Asset = Asset( diff --git a/project/plugins.sbt b/project/plugins.sbt index 822a5da28..bb4c0c12a 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,3 @@ -// This early version of coursier needs to be removed, but right now doing so exposes dependency conflicts... -addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC12") - // Use the Play sbt plugin for Play projects addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.19") diff --git a/uploader/src/main/scala/com/gu/media/upload/AddUploadDataToCache.scala b/uploader/src/main/scala/com/gu/media/upload/AddUploadDataToCache.scala index 67a8d8a83..1bd453ce2 100644 --- a/uploader/src/main/scala/com/gu/media/upload/AddUploadDataToCache.scala +++ b/uploader/src/main/scala/com/gu/media/upload/AddUploadDataToCache.scala @@ -4,13 +4,13 @@ import com.gu.media.aws.{DynamoAccess, UploadAccess} import com.gu.media.lambda.LambdaWithParams import com.gu.media.upload.model.Upload import org.scanamo.{Scanamo, Table} -import org.scanamo.auto._ +import org.scanamo.generic.auto._ class AddUploadDataToCache extends LambdaWithParams[Upload, Upload] with DynamoAccess with UploadAccess { private val table = Table[Upload](this.cacheTableName) - override def handle(input: Upload) = { - Scanamo.exec(this.dynamoDB)(table.put(input)) + override def handle(input: Upload): Upload = { + scanamo.exec(table.put(input)) input } }