diff --git a/app/data/DataStores.scala b/app/data/DataStores.scala index bac1e0818..2ad4fae47 100644 --- a/app/data/DataStores.scala +++ b/app/data/DataStores.scala @@ -14,7 +14,7 @@ class DataStores(aws: AWSConfig with SNSAccess, capi: CapiAccess) { val preview = new PreviewDynamoDataStore(aws.dynamoDB, aws.dynamoTableName) val published = new PublishedDynamoDataStore(aws.dynamoDB, aws.publishedDynamoTableName) - val pluto: PlutoDataStore = new PlutoDataStore(aws.dynamoDB, aws.manualPlutoDynamo) + val pluto: PlutoDataStore = new PlutoDataStore(aws.scanamo, aws.manualPlutoDynamo) val livePublisher: AtomPublisher = aws.capiContentEventsTopicName match { case Some(capiContentEventsTopicName) => diff --git a/app/di.scala b/app/di.scala index be414b635..332d01b5a 100644 --- a/app/di.scala +++ b/app/di.scala @@ -73,7 +73,7 @@ class MediaAtomMaker(context: Context) private val capi = new Capi(config) private val stores = new DataStores(aws, capi) - private val permissions = new MediaAtomMakerPermissionsProvider(aws.stage, aws.region.getName, aws.credentials.instance) + private val permissions = new MediaAtomMakerPermissionsProvider(aws.stage, aws.region.getName, aws.credentials.instance.awsV1Creds) private val reindexer = buildReindexer() diff --git a/app/util/AWS.scala b/app/util/AWS.scala index 1d9f36b41..eb0dc694d 100644 --- a/app/util/AWS.scala +++ b/app/util/AWS.scala @@ -26,7 +26,7 @@ class AWSConfig(override val config: Config, override val credentials: AwsCreden lazy val ec2Client = AmazonEC2ClientBuilder .standard() .withRegion(region.getName) - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.awsV1Creds) .build() lazy val pinboardLoaderUrl = getString("panda.domain").map(domain => s"https://pinboard.$domain/pinboard.loader.js") @@ -43,7 +43,7 @@ class AWSConfig(override val config: Config, override val credentials: AwsCreden lazy val expiryPollerName = "Expiry" lazy val expiryPollerLastName = "Poller" - final override def regionName = getString("aws.region") + final override def region = AwsAccess.regionFrom(this) final override def readTag(tagName: String) = { val tagsResult = ec2Client.describeTags( diff --git a/app/util/UploadDecorator.scala b/app/util/UploadDecorator.scala index b4ac81d3a..18e4d49a9 100644 --- a/app/util/UploadDecorator.scala +++ b/app/util/UploadDecorator.scala @@ -3,9 +3,9 @@ package util import com.gu.media.aws.{DynamoAccess, UploadAccess} import com.gu.media.model.{ClientAsset, ClientAssetMetadata} import com.gu.media.upload.model.Upload -import org.scanamo.{Scanamo, Table} +import org.scanamo.Table import org.scanamo.syntax._ -import org.scanamo.auto._ +import org.scanamo.generic.auto._ class UploadDecorator(aws: DynamoAccess with UploadAccess, stepFunctions: StepFunctions) { private val table = Table[Upload](aws.cacheTableName) @@ -28,10 +28,6 @@ class UploadDecorator(aws: DynamoAccess with UploadAccess, stepFunctions: StepFu } } - private def getUpload(id: String): Option[Upload] = { - val op = table.get('id -> id) - val result = Scanamo.exec(aws.dynamoDB)(op).flatMap(_.right.toOption) - - result orElse { stepFunctions.getById(id) } - } + private def getUpload(id: String): Option[Upload] = + aws.scanamo.exec(table.get("id" === id)).flatMap(_.toOption) orElse { stepFunctions.getById(id) } } diff --git a/build.sbt b/build.sbt index 84f92ce3b..d16f5fe84 100644 --- a/build.sbt +++ b/build.sbt @@ -4,11 +4,12 @@ import scala.sys.process._ val scroogeVersion = "4.12.0" val awsVersion = "1.11.678" +val awsV2Version = "2.21.17" val pandaVersion = "1.2.0" val pandaHmacVersion = "2.1.0" val atomMakerVersion = "1.3.4" val typesafeConfigVersion = "1.4.0" // to match what we get from Play transitively -val scanamoVersion = "1.0.0-M9" +val scanamoVersion = "1.0.0-M28" val playJsonExtensionsVersion = "0.40.2" val okHttpVersion = "2.4.0" @@ -102,10 +103,12 @@ lazy val common = (project in file("common")) "com.amazonaws" % "aws-lambda-java-core" % awsLambdaCoreVersion, "com.amazonaws" % "aws-java-sdk-s3" % awsVersion, "com.amazonaws" % "aws-java-sdk-dynamodb" % awsVersion, + "software.amazon.awssdk" % "dynamodb" % awsV2Version, "com.amazonaws" % "aws-java-sdk-kinesis" % awsVersion, "ai.x" %% "play-json-extensions" % playJsonExtensionsVersion, "ch.qos.logback" % "logback-classic" % logbackClassicVersion, "com.amazonaws" % "aws-java-sdk-sts" % awsVersion, + "software.amazon.awssdk" % "sts" % awsV2Version, "com.amazonaws" % "aws-java-sdk-elastictranscoder" % awsVersion, "org.scanamo" %% "scanamo" % scanamoVersion, "com.squareup.okhttp" % "okhttp" % okHttpVersion, @@ -132,6 +135,7 @@ lazy val app = (project in file(".")) ehcache, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion, "com.amazonaws" % "aws-java-sdk-sts" % awsVersion, + "software.amazon.awssdk" % "sts" % awsV2Version, "com.amazonaws" % "aws-java-sdk-ec2" % awsVersion, "org.scalatestplus.play" %% "scalatestplus-play" % scalaTestPlusPlayVersion % "test", "org.mockito" % "mockito-core" % mockitoVersion % "test", diff --git a/common/src/main/scala/com/gu/media/PlutoDataStore.scala b/common/src/main/scala/com/gu/media/PlutoDataStore.scala index 61378d7d3..ee459953f 100644 --- a/common/src/main/scala/com/gu/media/PlutoDataStore.scala +++ b/common/src/main/scala/com/gu/media/PlutoDataStore.scala @@ -1,20 +1,17 @@ package com.gu.media -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB -import com.amazonaws.services.dynamodbv2.model.DeleteItemResult import com.gu.media.model.PlutoSyncMetadataMessage -import org.scanamo.error.DynamoReadError +import org.scanamo.generic.auto._ import org.scanamo.syntax._ import org.scanamo.{Scanamo, Table} -import org.scanamo.auto._ -class PlutoDataStore(client: AmazonDynamoDB, dynamoTableName: String) { +class PlutoDataStore(scanamo: Scanamo, dynamoTableName: String) { val table = Table[PlutoSyncMetadataMessage](dynamoTableName) def getUploadsWithAtomId(id: String): List[PlutoSyncMetadataMessage] = { val atomIdIndex = table.index("atom-id") - val results = Scanamo.exec(client)(atomIdIndex.query('atomId -> id)) + val results = scanamo.exec(atomIdIndex.query("atomId" === id)) val errors = results.collect { case Left(err) => err } if (errors.nonEmpty) { @@ -25,8 +22,8 @@ class PlutoDataStore(client: AmazonDynamoDB, dynamoTableName: String) { } def get(id: String): Option[PlutoSyncMetadataMessage] = { - val operation = table.get('id -> id) - val result = Scanamo.exec(client)(operation) + val operation = table.get("id" === id) + val result = scanamo.exec(operation) result.map { case Right(item) => item @@ -34,12 +31,12 @@ class PlutoDataStore(client: AmazonDynamoDB, dynamoTableName: String) { } } - def put(item: PlutoSyncMetadataMessage): Option[Either[DynamoReadError, PlutoSyncMetadataMessage]] = { - Scanamo.exec(client)(table.put(item)) + def put(item: PlutoSyncMetadataMessage): Unit = { + scanamo.exec(table.put(item)) } - def delete(id: String): DeleteItemResult = { - Scanamo.exec(client)(table.delete('id -> id)) + def delete(id: String): Unit = { + scanamo.exec(table.delete("id" === id)) } case class DynamoPlutoTableException(err: String) extends RuntimeException(err) diff --git a/common/src/main/scala/com/gu/media/aws/AwsAccess.scala b/common/src/main/scala/com/gu/media/aws/AwsAccess.scala index a6c46b168..48aa2fbec 100644 --- a/common/src/main/scala/com/gu/media/aws/AwsAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/AwsAccess.scala @@ -5,17 +5,17 @@ import com.amazonaws.regions.{Region, Regions} import com.gu.media.Settings trait AwsAccess { this: Settings => - def regionName: Option[String] def readTag(tag: String): Option[String] val credentials: AwsCredentials + // To avoid renaming references everywhere - def credsProvider: AWSCredentialsProvider = credentials.instance + def credsProvider: AWSCredentialsProvider = credentials.instance.awsV1Creds - final def defaultRegion: Region = Region.getRegion(Regions.EU_WEST_1) - final def region: Region = regionName - .map { name => Region.getRegion(Regions.fromName(name)) } - .getOrElse(defaultRegion) + def region: Region + + final def awsV2Region: software.amazon.awssdk.regions.Region = + software.amazon.awssdk.regions.Region.of(region.getName) // These are injected as environment variables when running in a Lambda (unfortunately they cannot be tagged) final val stage = sys.env.getOrElse("STAGE", readTag("Stage").getOrElse("DEV")) @@ -24,3 +24,11 @@ trait AwsAccess { this: Settings => final val stack: Option[String] = if (isDev) Some("media-atom-maker") else readTag("Stack") final val app: String = if (isDev) "media-atom-maker" else readTag("App").getOrElse("media-atom-maker") } + +object AwsAccess { + def regionFrom(maybeName: Option[String]): Region = maybeName + .map { name => Region.getRegion(Regions.fromName(name)) } + .getOrElse(Region.getRegion(Regions.EU_WEST_1)) + + def regionFrom(settings: Settings): Region = regionFrom(settings.getString("aws.region")) +} diff --git a/common/src/main/scala/com/gu/media/aws/AwsCredentials.scala b/common/src/main/scala/com/gu/media/aws/AwsCredentials.scala index e87dc2770..bb7e0fc23 100644 --- a/common/src/main/scala/com/gu/media/aws/AwsCredentials.scala +++ b/common/src/main/scala/com/gu/media/aws/AwsCredentials.scala @@ -1,20 +1,20 @@ package com.gu.media.aws -import com.amazonaws.auth._ -import com.amazonaws.auth.profile.ProfileCredentialsProvider -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder import com.gu.media.Settings -case class AwsCredentials(instance: AWSCredentialsProvider, crossAccount: AWSCredentialsProvider, - upload: AWSCredentialsProvider) +case class AwsCredentials( + instance: CredentialsForBothSdkVersions, + crossAccount: CredentialsForBothSdkVersions, + upload: CredentialsForBothSdkVersions +) object AwsCredentials { def dev(settings: Settings): AwsCredentials = { val profile = settings.getMandatoryString("aws.profile") - val instance = new ProfileCredentialsProvider(profile) + val instance = CredentialsForBothSdkVersions.profile(profile) // To enable publishing to CAPI code from DEV, update the kinesis streams in config and uncomment below: - // val crossAccount = new ProfileCredentialsProvider("composer") + // val crossAccount = AwsCredentialsProvidersForBothSdkVersions.profile("composer") val crossAccount = instance val upload = devUpload(settings) @@ -23,7 +23,7 @@ object AwsCredentials { } def app(settings: Settings): AwsCredentials = { - val instance = InstanceProfileCredentialsProvider.getInstance() + val instance = CredentialsForBothSdkVersions.instance() val crossAccount = assumeCrossAccountRole(instance, settings) @@ -31,30 +31,23 @@ object AwsCredentials { } def lambda(): AwsCredentials = { - val instance = new EnvironmentVariableCredentialsProvider() + val instance = CredentialsForBothSdkVersions.environmentVariables() AwsCredentials(instance, crossAccount = instance, upload = instance) } - private def devUpload(settings: Settings): AWSCredentialsProvider = { + private def devUpload(settings: Settings): CredentialsForBothSdkVersions = { // Only required in dev (because federated credentials such as those from Janus cannot do STS requests). // Instance profile credentials are sufficient when deployed. val accessKey = settings.getMandatoryString("aws.upload.accessKey", "This is the AwsId output of the dev cloudformation") val secretKey = settings.getMandatoryString("aws.upload.secretKey", "This is the AwsSecret output of the dev cloudformation") - new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)) + CredentialsForBothSdkVersions.static(accessKey, secretKey) } - private def assumeCrossAccountRole(instance: AWSCredentialsProvider, settings: Settings) = { + private def assumeCrossAccountRole(instance: CredentialsForBothSdkVersions, settings: Settings): CredentialsForBothSdkVersions = { val crossAccountRoleArn = settings.getMandatoryString("aws.kinesis.stsCapiRoleToAssume", "Role to assume to access CAPI streams (in format arn:aws:iam:::role/)") - assumeAccountRole(instance, crossAccountRoleArn, "capi") - } - - private def assumeAccountRole(instance: AWSCredentialsProvider, roleArn: String, sessionNameSuffix: String): AWSCredentialsProvider = { - val securityTokens = AWSSecurityTokenServiceClientBuilder.standard().withCredentials(instance).build() - - new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, s"media-atom-maker-${sessionNameSuffix}") - .withStsClient(securityTokens).build() + instance.assumeAccountRole(crossAccountRoleArn, "capi", AwsAccess.regionFrom(settings).getName) } } diff --git a/common/src/main/scala/com/gu/media/aws/AwsV2Util.scala b/common/src/main/scala/com/gu/media/aws/AwsV2Util.scala new file mode 100644 index 000000000..cf7cf8ca6 --- /dev/null +++ b/common/src/main/scala/com/gu/media/aws/AwsV2Util.scala @@ -0,0 +1,16 @@ +package com.gu.media.aws + +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.awscore.client.builder.{AwsClientBuilder, AwsSyncClientBuilder} +import software.amazon.awssdk.http.apache.ApacheHttpClient +import software.amazon.awssdk.regions.Region + +object AwsV2Util { + def buildSync[T, B <: AwsClientBuilder[B, T] with AwsSyncClientBuilder[B, T]]( + builder: B, creds: AwsCredentialsProvider, region: Region + ): T = builder + .httpClientBuilder(ApacheHttpClient.builder()) + .credentialsProvider(creds) + .region(region) + .build() +} diff --git a/common/src/main/scala/com/gu/media/aws/CredentialsForBothSdkVersions.scala b/common/src/main/scala/com/gu/media/aws/CredentialsForBothSdkVersions.scala new file mode 100644 index 000000000..b148741af --- /dev/null +++ b/common/src/main/scala/com/gu/media/aws/CredentialsForBothSdkVersions.scala @@ -0,0 +1,48 @@ +package com.gu.media.aws + +import com.amazonaws.auth._ +import com.amazonaws.auth.profile.ProfileCredentialsProvider +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder +import software.amazon.awssdk.auth.{credentials => awsv2} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.sts.{StsClient, StsClientBuilder} +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest + +case class CredentialsForBothSdkVersions( + awsV1Creds: com.amazonaws.auth.AWSCredentialsProvider, + awsV2Creds: software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +) { + def assumeAccountRole(roleArn: String, sessionNameSuffix: String, regionName: String): CredentialsForBothSdkVersions = { + val roleSessionName = s"media-atom-maker-$sessionNameSuffix" + CredentialsForBothSdkVersions( + new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, roleSessionName) + .withStsClient(AWSSecurityTokenServiceClientBuilder.standard().withCredentials(awsV1Creds).build()).build(), + software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider.builder() + .stsClient(AwsV2Util.buildSync[StsClient, StsClientBuilder](StsClient.builder(), awsV2Creds, Region.of(regionName))) + .refreshRequest(AssumeRoleRequest.builder.roleSessionName(roleSessionName).roleArn(roleArn).build) + .build() + ) + } +} + +object CredentialsForBothSdkVersions { + def profile(name: String): CredentialsForBothSdkVersions = CredentialsForBothSdkVersions( + new ProfileCredentialsProvider(name), + awsv2.ProfileCredentialsProvider.create(name) + ) + + def instance(): CredentialsForBothSdkVersions = CredentialsForBothSdkVersions( + InstanceProfileCredentialsProvider.getInstance(), + awsv2.InstanceProfileCredentialsProvider.create() + ) + + def environmentVariables(): CredentialsForBothSdkVersions = CredentialsForBothSdkVersions( + new EnvironmentVariableCredentialsProvider(), + awsv2.EnvironmentVariableCredentialsProvider.create() + ) + + def static(accessKey: String, secretKey: String): CredentialsForBothSdkVersions = CredentialsForBothSdkVersions( + new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)), + awsv2.StaticCredentialsProvider.create(awsv2.AwsBasicCredentials.create(accessKey, secretKey)) + ) +} diff --git a/common/src/main/scala/com/gu/media/aws/DynamoAccess.scala b/common/src/main/scala/com/gu/media/aws/DynamoAccess.scala index f6a972d7b..4711e4981 100644 --- a/common/src/main/scala/com/gu/media/aws/DynamoAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/DynamoAccess.scala @@ -2,6 +2,9 @@ package com.gu.media.aws import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder import com.gu.media.Settings +import com.gu.media.aws.AwsV2Util.buildSync +import org.scanamo.Scanamo +import software.amazon.awssdk.services.dynamodb.{DynamoDbClient, DynamoDbClientBuilder} trait DynamoAccess { this: Settings with AwsAccess => lazy val dynamoTableName: String = sys.env.getOrElse("ATOM_TABLE_NAME", @@ -24,4 +27,9 @@ trait DynamoAccess { this: Settings with AwsAccess => .withCredentials(credsProvider) .withRegion(region.getName) .build() + + lazy val dynamoDbSdkV2: DynamoDbClient = + buildSync[DynamoDbClient, DynamoDbClientBuilder](DynamoDbClient.builder(), credentials.instance.awsV2Creds, awsV2Region) + + lazy val scanamo: Scanamo = Scanamo(dynamoDbSdkV2) } diff --git a/common/src/main/scala/com/gu/media/aws/KinesisAccess.scala b/common/src/main/scala/com/gu/media/aws/KinesisAccess.scala index cca55b5c7..89536ded8 100644 --- a/common/src/main/scala/com/gu/media/aws/KinesisAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/KinesisAccess.scala @@ -21,12 +21,12 @@ trait KinesisAccess { this: Settings with AwsAccess with Logging => val syncWithPluto: Boolean = getBoolean("pluto.sync").getOrElse(false) lazy val crossAccountKinesisClient = AmazonKinesisClientBuilder.standard() - .withCredentials(credentials.crossAccount) + .withCredentials(credentials.crossAccount.awsV1Creds) .withRegion(region.getName) .build() lazy val kinesisClient = AmazonKinesisClientBuilder.standard() - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.awsV1Creds) .withRegion(region.getName) .build() diff --git a/common/src/main/scala/com/gu/media/aws/SNSAccess.scala b/common/src/main/scala/com/gu/media/aws/SNSAccess.scala index 36f8a191e..cbaf01a7f 100644 --- a/common/src/main/scala/com/gu/media/aws/SNSAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/SNSAccess.scala @@ -11,6 +11,6 @@ trait SNSAccess { this: Settings with AwsAccess => lazy val snsClient = AmazonSNSClientBuilder.standard() .withRegion(Regions.fromName(region.getName)) - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.awsV1Creds) .build() } diff --git a/common/src/main/scala/com/gu/media/aws/UploadAccess.scala b/common/src/main/scala/com/gu/media/aws/UploadAccess.scala index 8dcfd6310..a52895dc1 100644 --- a/common/src/main/scala/com/gu/media/aws/UploadAccess.scala +++ b/common/src/main/scala/com/gu/media/aws/UploadAccess.scala @@ -30,7 +30,7 @@ trait UploadAccess { this: Settings with AwsAccess => throw new IllegalArgumentException("aws.upload.role must be in ARN format: arn:aws:iam:::role/") } - AWSSecurityTokenServiceClientBuilder.standard().withCredentials(credentials.upload).withRegion(region.getName).build() + AWSSecurityTokenServiceClientBuilder.standard().withCredentials(credentials.upload.awsV1Creds).withRegion(region.getName).build() } private def getPipelineArn() = { diff --git a/common/src/main/scala/com/gu/media/lambda/LambdaBase.scala b/common/src/main/scala/com/gu/media/lambda/LambdaBase.scala index 17538b065..4d7d604ef 100644 --- a/common/src/main/scala/com/gu/media/lambda/LambdaBase.scala +++ b/common/src/main/scala/com/gu/media/lambda/LambdaBase.scala @@ -9,10 +9,10 @@ import com.gu.media.aws.{AwsAccess, AwsCredentials, HMACSettings} import com.typesafe.config.{Config, ConfigFactory} trait LambdaBase extends Settings with AwsAccess with HMACSettings { - final override def regionName = sys.env.get("REGION") + final override def region = AwsAccess.regionFrom(sys.env.get("REGION")) final override def readTag(tag: String) = sys.env.get(tag.toUpperCase(Locale.ENGLISH)) - final override val credentials = AwsCredentials.lambda() + final override val credentials: AwsCredentials = AwsCredentials.lambda() private val remoteConfig = downloadConfig() private val mergedConfig = remoteConfig.withFallback(ConfigFactory.load()) @@ -24,7 +24,7 @@ trait LambdaBase extends Settings with AwsAccess with HMACSettings { case (Some(bucket), Some(key)) => val defaultRegionS3 = AmazonS3ClientBuilder .standard() - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.awsV1Creds) .withRegion(region.getName) .build() diff --git a/common/src/main/scala/com/gu/media/lambda/LambdaYoutubeCredentials.scala b/common/src/main/scala/com/gu/media/lambda/LambdaYoutubeCredentials.scala index 7ecc48dd3..292554296 100644 --- a/common/src/main/scala/com/gu/media/lambda/LambdaYoutubeCredentials.scala +++ b/common/src/main/scala/com/gu/media/lambda/LambdaYoutubeCredentials.scala @@ -10,7 +10,7 @@ trait LambdaYoutubeCredentials { self: AwsAccess => case (Some(bucket), Some(key)) => val defaultRegionS3 = AmazonS3ClientBuilder .standard() - .withCredentials(credentials.instance) + .withCredentials(credentials.instance.awsV1Creds) .withRegion(region.getName) .build() diff --git a/common/src/main/scala/com/gu/media/pluto/PlutoCommissionDataStore.scala b/common/src/main/scala/com/gu/media/pluto/PlutoCommissionDataStore.scala index 003576935..efa5c6e78 100644 --- a/common/src/main/scala/com/gu/media/pluto/PlutoCommissionDataStore.scala +++ b/common/src/main/scala/com/gu/media/pluto/PlutoCommissionDataStore.scala @@ -1,29 +1,25 @@ package com.gu.media.pluto -import com.amazonaws.services.dynamodbv2.model.DeleteItemResult import com.gu.media.aws.DynamoAccess import com.gu.media.logging.Logging import com.gu.media.pluto.PlutoItem.numericIdsOnlyFilter -import org.scanamo.error.DynamoReadError +import org.scanamo.generic.auto._ import org.scanamo.syntax._ -import org.scanamo.{Scanamo, Table} -import org.scanamo.auto._ +import org.scanamo.{DynamoReadError, Scanamo, Table} case class PlutoCommissionDataStoreException(err: String) extends Exception(err) class PlutoCommissionDataStore(aws: DynamoAccess) extends Logging { + val scanamo: Scanamo = aws.scanamo private val table = Table[PlutoCommission](aws.plutoCommissionTableName) def getById(commissionId: String): Option[Either[DynamoReadError, PlutoCommission]] = { log.info(s"getting commission $commissionId") - Scanamo.exec(aws.dynamoDB)(table.get('id -> commissionId)) + scanamo.exec(table.get("id" === commissionId)) } def list(): List[PlutoCommission] = { - val op = table.scan() - val results = Scanamo.exec(aws.dynamoDB)(op) - - results.collect { + scanamo.exec(table.scan()).collect { case Left(error) => log.error("failed to list pluto commissions") throw PlutoCommissionDataStoreException(error.toString) @@ -32,15 +28,14 @@ class PlutoCommissionDataStore(aws: DynamoAccess) extends Logging { }.filter(numericIdsOnlyFilter).sortBy(_.title) } - def upsert(plutoUpsertRequest: PlutoUpsertRequest): Option[Either[DynamoReadError, PlutoCommission]] = { + def upsert(plutoUpsertRequest: PlutoUpsertRequest): Unit = { val commission = PlutoCommission.build(plutoUpsertRequest) log.info(s"upserting pluto commission ${commission.id}") - val op = table.put(commission) - Scanamo.exec(aws.dynamoDB)(op) + scanamo.exec(table.put(commission)) } - def delete(commissionId: String): DeleteItemResult = { + def delete(commissionId: String): Unit = { log.info(s"deleting commission $commissionId") - Scanamo.exec(aws.dynamoDB)(table.delete('id -> commissionId)) + scanamo.exec(table.delete("id" === commissionId)) } } diff --git a/common/src/main/scala/com/gu/media/pluto/PlutoProjectDataStore.scala b/common/src/main/scala/com/gu/media/pluto/PlutoProjectDataStore.scala index d751b52ba..8a4133c03 100644 --- a/common/src/main/scala/com/gu/media/pluto/PlutoProjectDataStore.scala +++ b/common/src/main/scala/com/gu/media/pluto/PlutoProjectDataStore.scala @@ -1,36 +1,33 @@ package com.gu.media.pluto -import com.amazonaws.services.dynamodbv2.model.DeleteItemResult import com.gu.media.aws.DynamoAccess import com.gu.media.logging.Logging import com.gu.media.pluto.PlutoItem.numericIdsOnlyFilter import org.joda.time.{DateTime, DateTimeZone} import org.scanamo.DynamoFormat._ +import org.scanamo.generic.auto._ import org.scanamo.syntax._ -import org.scanamo.{DynamoFormat, Scanamo, Table} -import org.scanamo.auto._ +import org.scanamo.{DynamoFormat, DynamoReadError, Scanamo, Table} case class PlutoProjectDataStoreException(err: String) extends Exception(err) class PlutoProjectDataStore(aws: DynamoAccess, plutoCommissionDataStore: PlutoCommissionDataStore) extends Logging { - implicit val dateTimeFormat = DynamoFormat.coercedXmap[DateTime, String, IllegalArgumentException]( - DateTime.parse(_).withZone(DateTimeZone.UTC) - )(_.toString) + implicit val dateTimeFormat: DynamoFormat[DateTime] = DynamoFormat.coercedXmap[DateTime, String, IllegalArgumentException]( + DateTime.parse(_).withZone(DateTimeZone.UTC), + _.toString + ) + val scanamo: Scanamo = aws.scanamo private val table = Table[PlutoProject](aws.plutoProjectTableName) private val commmissionIndex = table.index("commission-index") - def getById(projectId: String) = { + def getById(projectId: String): Option[Either[DynamoReadError, PlutoProject]] = { log.info(s"getting project $projectId") - Scanamo.exec(aws.dynamoDB)(table.get('id -> projectId)) + scanamo.exec(table.get("id" === projectId)) } def getByCommissionId(commissionId: String): List[PlutoProject] = { - val op = commmissionIndex.query('commissionId -> commissionId) - - val results = Scanamo.exec(aws.dynamoDB)(op) - - results.collect { + scanamo.exec(commmissionIndex.query("commissionId" === commissionId)).collect { case Left(error) => log.error(s"failed to get pluto projects for commission $commissionId") throw PlutoProjectDataStoreException(error.toString) @@ -44,13 +41,12 @@ class PlutoProjectDataStore(aws: DynamoAccess, plutoCommissionDataStore: PlutoCo val project = PlutoProject.build(plutoUpsertRequest) log.info(s"upserting pluto project ${project.id}") - val op = table.put(project) - Scanamo.exec(aws.dynamoDB)(op) + scanamo.exec(table.put(project)) project } - def deleteByCommissionId(commissionId: String): List[DeleteItemResult] = { + def deleteByCommissionId(commissionId: String): Unit = { log.info(s"deleting all pluto projects for commission $commissionId") - getByCommissionId(commissionId).map(project => Scanamo.exec(aws.dynamoDB)(table.delete('id -> project.id))) + getByCommissionId(commissionId).foreach(project => scanamo.exec(table.delete("id" === project.id))) } } diff --git a/common/src/main/scala/com/gu/media/upload/PlutoUploadActions.scala b/common/src/main/scala/com/gu/media/upload/PlutoUploadActions.scala index 73b764b62..c7d1bbfd6 100644 --- a/common/src/main/scala/com/gu/media/upload/PlutoUploadActions.scala +++ b/common/src/main/scala/com/gu/media/upload/PlutoUploadActions.scala @@ -8,7 +8,7 @@ import com.gu.media.{PlutoDataStore, Settings} class PlutoUploadActions(config: Settings with DynamoAccess with KinesisAccess with SESSettings) extends Logging { private val mailer = new Mailer(config) - private val plutoStore = new PlutoDataStore(config.dynamoDB, config.manualPlutoDynamo) + private val plutoStore = new PlutoDataStore(config.scanamo, config.manualPlutoDynamo) def sendToPluto(plutoIntegrationMessage: PlutoIntegrationMessage): Unit = { diff --git a/uploader/src/main/scala/com/gu/media/upload/AddUploadDataToCache.scala b/uploader/src/main/scala/com/gu/media/upload/AddUploadDataToCache.scala index 67a8d8a83..1bd453ce2 100644 --- a/uploader/src/main/scala/com/gu/media/upload/AddUploadDataToCache.scala +++ b/uploader/src/main/scala/com/gu/media/upload/AddUploadDataToCache.scala @@ -4,13 +4,13 @@ import com.gu.media.aws.{DynamoAccess, UploadAccess} import com.gu.media.lambda.LambdaWithParams import com.gu.media.upload.model.Upload import org.scanamo.{Scanamo, Table} -import org.scanamo.auto._ +import org.scanamo.generic.auto._ class AddUploadDataToCache extends LambdaWithParams[Upload, Upload] with DynamoAccess with UploadAccess { private val table = Table[Upload](this.cacheTableName) - override def handle(input: Upload) = { - Scanamo.exec(this.dynamoDB)(table.put(input)) + override def handle(input: Upload): Upload = { + scanamo.exec(table.put(input)) input } }