diff --git a/gradle.properties b/gradle.properties index 7fa3aaf9..8e426bde 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,4 +16,4 @@ group=com.nike artifactId=cerberus-lifecycle-cli -version=4.11.1 +version=4.12.0 diff --git a/src/main/java/com/nike/cerberus/cli/CerberusRunner.java b/src/main/java/com/nike/cerberus/cli/CerberusRunner.java index 4c0034dd..482efc8c 100644 --- a/src/main/java/com/nike/cerberus/cli/CerberusRunner.java +++ b/src/main/java/com/nike/cerberus/cli/CerberusRunner.java @@ -31,10 +31,7 @@ import com.nike.cerberus.ConfigConstants; import com.nike.cerberus.command.CerberusCommand; import com.nike.cerberus.command.Command; -import com.nike.cerberus.command.audit.CreateAuditAthenaDbAndTableCommand; -import com.nike.cerberus.command.audit.CreateAuditLoggingStackCommand; -import com.nike.cerberus.command.audit.DisableAuditLoggingCommand; -import com.nike.cerberus.command.audit.EnableAuditLoggingForExistingEnvironmentCommand; +import com.nike.cerberus.command.audit.*; import com.nike.cerberus.command.certificates.RotateAcmeAccountPrivateKeyCommand; import com.nike.cerberus.command.cms.CreateCmsClusterCommand; import com.nike.cerberus.command.cms.CreateCmsConfigCommand; @@ -234,6 +231,7 @@ private void registerAllCommands() { registerCommand(new CreateAuditAthenaDbAndTableCommand()); registerCommand(new DisableAuditLoggingCommand()); registerCommand(new EnableAuditLoggingForExistingEnvironmentCommand()); + registerCommand(new UpdateAuditAthenaTableCommand()); registerCommand(new UpdateStackTagsCommand()); registerCommand(new UpdateAllStackTagsCommand()); registerCommand(new SyncConfigCommand()); diff --git a/src/main/java/com/nike/cerberus/command/audit/UpdateAuditAthenaTableCommand.java b/src/main/java/com/nike/cerberus/command/audit/UpdateAuditAthenaTableCommand.java new file mode 100644 index 00000000..df58ae6a --- /dev/null +++ b/src/main/java/com/nike/cerberus/command/audit/UpdateAuditAthenaTableCommand.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 Nike, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nike.cerberus.command.audit; + +import com.beust.jcommander.Parameters; +import com.nike.cerberus.command.Command; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.operation.audit.UpdateAuditAthenaTableOperation; + +import static com.nike.cerberus.command.audit.UpdateAuditAthenaTableCommand.COMMAND_NAME; + +@Parameters( + commandNames = COMMAND_NAME, + commandDescription = "Updates the table by adding additional columns for new data" +) +public class UpdateAuditAthenaTableCommand implements Command { + + public static final String COMMAND_NAME = "update-audit-log-athena-table"; + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + @Override + public Class> getOperationClass() { + return UpdateAuditAthenaTableOperation.class; + } +} diff --git a/src/main/java/com/nike/cerberus/operation/audit/UpdateAuditAthenaTableOperation.java b/src/main/java/com/nike/cerberus/operation/audit/UpdateAuditAthenaTableOperation.java new file mode 100644 index 00000000..5d80e0b8 --- /dev/null +++ b/src/main/java/com/nike/cerberus/operation/audit/UpdateAuditAthenaTableOperation.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 Nike, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nike.cerberus.operation.audit; + +import com.nike.cerberus.ConfigConstants; +import com.nike.cerberus.command.audit.UpdateAuditAthenaTableCommand; +import com.nike.cerberus.domain.cloudformation.AuditOutputs; +import com.nike.cerberus.domain.environment.Stack; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.service.AthenaService; +import com.nike.cerberus.service.CloudFormationService; +import com.nike.cerberus.store.ConfigStore; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Named; +import java.io.IOException; + +import static com.nike.cerberus.module.CerberusModule.ENV_NAME; + +public class UpdateAuditAthenaTableOperation implements Operation { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final CloudFormationService cloudFormationService; + private final ConfigStore configStore; + private final AthenaService athenaService; + private final String databaseName; + private final String tableName; + private final String environmentName; + + @Inject + public UpdateAuditAthenaTableOperation(CloudFormationService cloudFormationService, + ConfigStore configStore, + @Named(ENV_NAME) String environmentName, + AthenaService athenaService) { + + this.cloudFormationService = cloudFormationService; + this.configStore = configStore; + this.athenaService = athenaService; + + databaseName = environmentName + "_audit_db"; + tableName = databaseName + ".audit_data"; + this.environmentName = environmentName; + } + + @Override + public void run(UpdateAuditAthenaTableCommand command) { + AuditOutputs outputs = + configStore.getStackOutputs(configStore.getPrimaryRegion(), + Stack.AUDIT.getFullName(environmentName), AuditOutputs.class); + + String bucketName = outputs.getAuditBucketName(); + + log.info("Dropping table to update"); + String dropTable = "DROP TABLE IF EXISTS " + tableName + ";"; + log.info(athenaService.executeAthenaQuery(dropTable, bucketName, configStore.getPrimaryRegion()).toString()); + + log.info("Creating new table with additional columns"); + String updateAuditTable; + try { + String template = "/com/nike/cerberus/operation/audit/create_audit_table.ddl"; + updateAuditTable = IOUtils.toString(getClass().getResourceAsStream(template), ConfigConstants.DEFAULT_ENCODING); + updateAuditTable = updateAuditTable.replace("@@TABLE_NAME@@", tableName); + updateAuditTable = updateAuditTable.replace("@@BUCKET_NAME@@", bucketName); + } catch (IOException e) { + throw new RuntimeException("failed to load update athena table template", e); + } + log.info(athenaService.executeAthenaQuery(updateAuditTable, bucketName, configStore.getPrimaryRegion()).toString()); + + log.info("Repairing table partitions"); + String repairTablePartitions = "MSCK REPAIR TABLE " + tableName + ";"; + log.info(athenaService.executeAthenaQuery(repairTablePartitions, bucketName, configStore.getPrimaryRegion()).toString()); + } + + @Override + public boolean isRunnable(UpdateAuditAthenaTableCommand command) { + boolean isRunnable = true; + + if (! cloudFormationService.isStackPresent(configStore.getPrimaryRegion(), Stack.AUDIT.getFullName(environmentName))) { + log.error("You must create the audit stack using create-audit-logging-stack command"); + isRunnable = false; + } + return isRunnable; + } +} diff --git a/src/main/resources/com/nike/cerberus/operation/audit/create_audit_table.ddl b/src/main/resources/com/nike/cerberus/operation/audit/create_audit_table.ddl index d5e5a359..3aac13b6 100644 --- a/src/main/resources/com/nike/cerberus/operation/audit/create_audit_table.ddl +++ b/src/main/resources/com/nike/cerberus/operation/audit/create_audit_table.ddl @@ -15,7 +15,9 @@ CREATE EXTERNAL TABLE IF NOT EXISTS @@TABLE_NAME@@ ( name string, sdb_name_slug string, originating_class string, - trace_id string + trace_id string, + status_code string, + cerberus_version string ) PARTITIONED BY (year INT, month INT, day INT, hour INT) ROW FORMAT serde 'org.apache.hive.hcatalog.data.JsonSerDe' with serdeproperties (