diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala index e363481af..084aac487 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala @@ -82,6 +82,12 @@ object EnrichmentManager { ) .leftMap(NonEmptyList.one) .possiblyExitingEarly(emitIncomplete) + // Next 2 lines remove the invalid contexts and the invalid unstructured event from the event. + // This should be done after the bad row was created and only if emitIncomplete is enabled. + _ = { + enriched.contexts = ME.formatContexts(extractResult.contexts).orNull + enriched.unstruct_event = ME.formatUnstructEvent(extractResult.unstructEvent).orNull + } enrichmentsContexts <- runEnrichments( registry, processor, @@ -109,10 +115,10 @@ object EnrichmentManager { .possiblyExitingEarly(emitIncomplete) } yield enriched - iorT.leftMap(_.last) + iorT.leftMap(_.head) } - def mapAndValidateInput[F[_]: Clock: Monad]( + private def mapAndValidateInput[F[_]: Clock: Monad]( raw: RawEvent, enrichedEvent: EnrichedEvent, etlTstamp: DateTime, @@ -123,10 +129,6 @@ object EnrichmentManager { val iorT = for { _ <- IorT.fromIor[F](setupEnrichedEvent(raw, enrichedEvent, etlTstamp, processor)) extract <- IgluUtils.extractAndValidateInputJsons(enrichedEvent, client, registryLookup) - _ = { - enrichedEvent.contexts = ME.formatContexts(extract.contexts).orNull - enrichedEvent.unstruct_event = ME.formatUnstructEvent(extract.unstructEvent).orNull - } } yield extract iorT.leftMap { violations => diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala index f75ff340b..6bf35e859 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala @@ -37,7 +37,7 @@ import com.snowplowanalytics.iglu.client.{IgluCirceClient, Resolver} import com.snowplowanalytics.iglu.client.resolver.registries.Registry import com.snowplowanalytics.iglu.client.resolver.registries.JavaNetRegistryLookup -import com.snowplowanalytics.iglu.core.SelfDescribingData +import com.snowplowanalytics.iglu.core.{SchemaKey, SelfDescribingData} import com.snowplowanalytics.iglu.core.circe.implicits._ import com.snowplowanalytics.lrumap.CreateLruMap._ @@ -129,6 +129,25 @@ object SpecHelpers extends CatsEffect { .flatMap(SelfDescribingData.parse[Json]) .leftMap(err => s"Can't parse Json [$rawJson] as as SelfDescribingData, error: [$err]") + def listContextsSchemas(rawContexts: String): List[SchemaKey] = + jsonStringToSDJ(rawContexts) + .map(_.data.asArray.get.toList) + .flatMap(contexts => contexts.traverse(c => SelfDescribingData.parse[Json](c).map(_.schema))) match { + case Left(err) => + throw new IllegalArgumentException(s"Couldn't list contexts schemas. Error: [$err]") + case Right(schemas) => schemas + } + + def getUnstructSchema(rawUnstruct: String): SchemaKey = + jsonStringToSDJ(rawUnstruct) + .map(_.data) + .flatMap(SelfDescribingData.parse[Json]) + .map(_.schema) match { + case Left(err) => + throw new IllegalArgumentException(s"Couldn't get unstruct event schema. Error: [$err]") + case Right(schema) => schema + } + implicit class MapOps[A, B](underlying: Map[A, B]) { def toOpt: Map[A, Option[B]] = underlying.map { case (a, b) => (a, Option(b)) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala index 43b23cc22..5ab783e28 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala @@ -137,6 +137,52 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE } } + "return a SchemaViolations bad row that contains both a mapping error and a ValidationError if the 2 problems are in the event" >> { + val parameters = Map( + "e" -> "ue", + "tv" -> "js-0.13.1", + "p" -> "web", + "tr_tt" -> "not_number", + "ue_pr" -> + """ + { + "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", + "data":{ + "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0", + "data": { + "emailAddress": "hello@world.com", + "emailAddress2": "foo@bar.org", + "unallowedAdditionalField": "foo@bar.org" + } + } + }""" + ).toOpt + val rawEvent = RawEvent(api, parameters, None, source, context) + val enriched = EnrichmentManager.enrichEvent[IO]( + enrichmentReg, + client, + processor, + timestamp, + rawEvent, + AcceptInvalid.featureFlags, + IO.unit, + SpecHelpers.registryLookup, + atomicFieldLimits, + emitIncomplete + ) + enriched.value map { + case Ior.Left(BadRow.SchemaViolations(_, Failure.SchemaViolations(_, messages), _)) => + messages match { + case NonEmptyList(one, List(two)) => + one.toString must contain("cannot be converted to java.math.BigDecimal") + two.toString must contain("unallowedAdditionalField: is not defined in the schema") + case _ => + ko(s"[$messages] doesn't contain 2 errors but ${messages.size}") + } + case other => ko(s"[$other] is not a SchemaViolations bad row") + } + } + "return an EnrichmentFailures bad row if one of the enrichment (JS enrichment here) fails" >> { val script = """ @@ -1167,6 +1213,372 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE case other => ko(s"[$other] is not an enriched event") } } + + "remove the invalid unstructured event and enrich the event if emitIncomplete is set to true" >> { + val script = + s""" + function process(event) { + return [ ${emailSent} ]; + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "javascript_script_config", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + val enrichmentReg = EnrichmentRegistry[IO]( + javascriptScript = List( + JavascriptScriptEnrichment(schemaKey, script) + ) + ) + + val parameters = Map( + "e" -> "pp", + "tv" -> "js-0.13.1", + "p" -> "web", + "co" -> + s""" + { + "schema": "iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0", + "data": [ + $clientSession + ] + } + """, + "ue_pr" -> + """ + { + "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", + "data":{ + "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0", + "data": { + "emailAddress": "hello@world.com", + "emailAddress2": "foo@bar.org", + "unallowedAdditionalField": "foo@bar.org" + } + } + }""" + ).toOpt + val rawEvent = RawEvent(api, parameters, None, source, context) + val enriched = EnrichmentManager.enrichEvent[IO]( + enrichmentReg, + client, + processor, + timestamp, + rawEvent, + AcceptInvalid.featureFlags, + IO.unit, + SpecHelpers.registryLookup, + atomicFieldLimits, + emitIncomplete = true + ) + enriched.value.map { + case Ior.Both(_: BadRow.SchemaViolations, enriched) + if Option(enriched.unstruct_event).isEmpty && + SpecHelpers.listContextsSchemas(enriched.contexts) == List(clientSessionSchema) && + SpecHelpers.listContextsSchemas(enriched.derived_contexts).contains(emailSentSchema) => + ok + case other => ko(s"[$other] is not a SchemaViolations bad row and an enriched event without the unstructured event") + } + } + + "remove the invalid context and enrich the event if emitIncomplete is set to true" >> { + val script = + s""" + function process(event) { + return [ ${emailSent} ]; + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "javascript_script_config", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + val enrichmentReg = EnrichmentRegistry[IO]( + javascriptScript = List( + JavascriptScriptEnrichment(schemaKey, script) + ) + ) + + val parameters = Map( + "e" -> "pp", + "tv" -> "js-0.13.1", + "p" -> "web", + "co" -> + """ + { + "schema": "iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0", + "data": [ + { + "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0", + "data": { + "foo": "hello@world.com", + "emailAddress2": "foo@bar.org" + } + } + ] + } + """, + "ue_pr" -> + s""" + { + "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", + "data": $clientSession + }""" + ).toOpt + val rawEvent = RawEvent(api, parameters, None, source, context) + val enriched = EnrichmentManager.enrichEvent[IO]( + enrichmentReg, + client, + processor, + timestamp, + rawEvent, + AcceptInvalid.featureFlags, + IO.unit, + SpecHelpers.registryLookup, + atomicFieldLimits, + emitIncomplete = true + ) + enriched.value.map { + case Ior.Both(_: BadRow.SchemaViolations, enriched) + if Option(enriched.contexts).isEmpty && + SpecHelpers.getUnstructSchema(enriched.unstruct_event) == clientSessionSchema && + SpecHelpers.listContextsSchemas(enriched.derived_contexts).contains(emailSentSchema) => + ok + case other => ko(s"[$other] is not a SchemaViolations bad row and an enriched event with no input contexts") + } + } + + "remove one invalid context (out of 2) and enrich the event if emitIncomplete is set to true" >> { + val script = + s""" + function process(event) { + return [ ${emailSent} ]; + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "javascript_script_config", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + val enrichmentReg = EnrichmentRegistry[IO]( + javascriptScript = List( + JavascriptScriptEnrichment(schemaKey, script) + ) + ) + + val parameters = Map( + "e" -> "pp", + "tv" -> "js-0.13.1", + "p" -> "web", + "co" -> + s""" + { + "schema": "iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0", + "data": [ + { + "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0", + "data": { + "foo": "hello@world.com", + "emailAddress2": "foo@bar.org" + } + }, + $clientSession + ] + } + """, + "ue_pr" -> + s""" + { + "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", + "data": $clientSession + }""" + ).toOpt + val rawEvent = RawEvent(api, parameters, None, source, context) + val enriched = EnrichmentManager.enrichEvent[IO]( + enrichmentReg, + client, + processor, + timestamp, + rawEvent, + AcceptInvalid.featureFlags, + IO.unit, + SpecHelpers.registryLookup, + atomicFieldLimits, + emitIncomplete = true + ) + enriched.value.map { + case Ior.Both(_: BadRow.SchemaViolations, enriched) + if SpecHelpers.getUnstructSchema(enriched.unstruct_event) == clientSessionSchema && + SpecHelpers.listContextsSchemas(enriched.contexts) == List(clientSessionSchema) && + SpecHelpers.listContextsSchemas(enriched.derived_contexts).contains(emailSentSchema) => + ok + case other => ko(s"[$other] is not a SchemaViolations bad row and an enriched event with 1 input context") + } + } + + "return the enriched event after an enrichment exception if emitIncomplete is set to true" >> { + val script = + s""" + function process(event) { + throw "Javascript exception"; + return [ $emailSent ]; + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "javascript_script_config", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + val enrichmentReg = EnrichmentRegistry[IO]( + yauaa = Some(YauaaEnrichment(None)), + javascriptScript = List( + JavascriptScriptEnrichment(schemaKey, script) + ) + ) + + val parameters = Map( + "e" -> "pp", + "tv" -> "js-0.13.1", + "p" -> "web", + "ue_pr" -> + s""" + { + "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", + "data": $clientSession + }""" + ).toOpt + val rawEvent = RawEvent(api, parameters, None, source, context) + val enriched = EnrichmentManager.enrichEvent[IO]( + enrichmentReg, + client, + processor, + timestamp, + rawEvent, + AcceptInvalid.featureFlags, + IO.unit, + SpecHelpers.registryLookup, + atomicFieldLimits, + emitIncomplete = true + ) + enriched.value.map { + case Ior.Both(_: BadRow.EnrichmentFailures, enriched) + if SpecHelpers.getUnstructSchema(enriched.unstruct_event) == clientSessionSchema && + !SpecHelpers.listContextsSchemas(enriched.derived_contexts).contains(emailSentSchema) => + ok + case other => ko(s"[$other] is not an EnrichmentFailures bad row and an enriched event") + } + } + + "return a SchemaViolations bad row in the Left in case of both a schema violation and an enrichment failure if emitIncomplete is set to true" >> { + val script = + s""" + function process(event) { + throw "Javascript exception"; + return [ $emailSent ]; + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "javascript_script_config", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + val enrichmentReg = EnrichmentRegistry[IO]( + javascriptScript = List( + JavascriptScriptEnrichment(schemaKey, script) + ) + ) + + val parameters = Map( + "e" -> "pp", + "tv" -> "js-0.13.1", + "p" -> "web", + "tr_tt" -> "foo", + "ue_pr" -> + s""" + { + "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", + "data": $clientSession + }""" + ).toOpt + val rawEvent = RawEvent(api, parameters, None, source, context) + val enriched = EnrichmentManager.enrichEvent[IO]( + enrichmentReg, + client, + processor, + timestamp, + rawEvent, + AcceptInvalid.featureFlags, + IO.unit, + SpecHelpers.registryLookup, + atomicFieldLimits, + emitIncomplete = true + ) + enriched.value.map { + case Ior.Both(_: BadRow.SchemaViolations, _) => ok + case other => ko(s"[$other] doesn't have a SchemaViolations bad row in the Left") + } + } + + "remove an invalid enrichment context and return the enriched event if emitIncomplete is set to true" >> { + val script = + s""" + function process(event) { + return [ + { + "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0", + "data": { + "foo": "hello@world.com", + "emailAddress2": "foo@bar.org" + } + } + ]; + }""" + val schemaKey = SchemaKey( + "com.snowplowanalytics.snowplow", + "javascript_script_config", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + val enrichmentReg = EnrichmentRegistry[IO]( + yauaa = Some(YauaaEnrichment(None)), + javascriptScript = List( + JavascriptScriptEnrichment(schemaKey, script) + ) + ) + + val parameters = Map( + "e" -> "pp", + "tv" -> "js-0.13.1", + "p" -> "web", + "ue_pr" -> + s""" + { + "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", + "data": $clientSession + }""" + ).toOpt + val rawEvent = RawEvent(api, parameters, None, source, context) + val enriched = EnrichmentManager.enrichEvent[IO]( + enrichmentReg, + client, + processor, + timestamp, + rawEvent, + AcceptInvalid.featureFlags, + IO.unit, + SpecHelpers.registryLookup, + atomicFieldLimits, + emitIncomplete = true + ) + enriched.value.map { + case Ior.Both(_: BadRow.EnrichmentFailures, enriched) + if SpecHelpers.getUnstructSchema(enriched.unstruct_event) == clientSessionSchema && + !SpecHelpers.listContextsSchemas(enriched.derived_contexts).contains(emailSentSchema) => + ok + case other => ko(s"[$other] is not an EnrichmentFailures bad row and an enriched event without the faulty enrichment context") + } + } } "getCrossDomain" should { @@ -1748,4 +2160,37 @@ object EnrichmentManagerSpec { .getOrElse(throw new RuntimeException("IAB enrichment couldn't be initialised")) // to make sure it's not none .enrichment[IO] + val emailSentSchema = + SchemaKey( + "com.acme", + "email_sent", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + + val emailSent = s"""{ + "schema": "${emailSentSchema.toSchemaUri}", + "data": { + "emailAddress": "hello@world.com", + "emailAddress2": "foo@bar.org" + } + }""" + val clientSessionSchema = + SchemaKey( + "com.snowplowanalytics.snowplow", + "client_session", + "jsonschema", + SchemaVer.Full(1, 0, 1) + ) + val clientSession = s"""{ + "schema": "${clientSessionSchema.toSchemaUri}", + "data": { + "sessionIndex": 1, + "storageMechanism": "LOCAL_STORAGE", + "firstEventId": "5c33fccf-6be5-4ce6-afb1-e34026a3ca75", + "sessionId": "21c2a0dd-892d-42d1-b156-3a9d4e147eef", + "previousSessionId": null, + "userId": "20d631b8-7837-49df-a73e-6da73154e6fd" + } + }""" } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala index 3572d8c06..e605e8dfd 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala @@ -415,8 +415,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect } } - // TODO: check good schema - "return an expected failure if one context is valid and the other invalid" >> { + "return an expected failure and an expected SDJ if one context is invalid and one is invalid" >> { val input = new EnrichedEvent input.setContexts(buildInputContexts(List(emailSent1, noSchema))) @@ -424,8 +423,10 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) .value .map { - case Ior.Both(NonEmptyList(_: FailureDetails.SchemaViolation.IgluError, Nil), List(_)) => ok - case other => ko(s"[$other] is not one IgluError and one valid SDJ") + case Ior.Both(NonEmptyList(_: FailureDetails.SchemaViolation.IgluError, Nil), List(extract)) + if extract.sdj.schema == emailSentSchema => + ok + case other => ko(s"[$other] is not one IgluError and one SDJ with schema $emailSentSchema") } } @@ -528,8 +529,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect } } - // TODO: check schema in the Right - "return a failure and a SDJ for one valid context and one invalid" >> { + "return a failure and an expectected SDJ for one valid context and one invalid" >> { val contexts = List( SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get, SpecHelpers.jsonStringToSDJ(emailSent1).right.get @@ -546,14 +546,13 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect ), Nil ), - List(_) - ) => + List(sdj) + ) if sdj.schema == emailSentSchema => ok - case other => ko(s"[$other] is not one error with a ValidationError and one valid SDJ") + case other => ko(s"[$other] is not one ValidationError and one SDJ with schema $emailSentSchema") } } - // TODO: check the schemas "return 2 valid contexts" >> { val contexts = List( SpecHelpers.jsonStringToSDJ(emailSent1).right.get, @@ -564,143 +563,193 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .validateEnrichmentsContexts(SpecHelpers.client, contexts, SpecHelpers.registryLookup) .value .map { - case Ior.Right(List(_, _)) => ok - case other => ko(s"[$other] doesn't contain the 2 valid contexts") - } - } - - "extractAndValidateInputJsons" should { - "return one SchemaViolation if the input event contains an invalid unstructured event" >> { - val input = new EnrichedEvent - input.setUnstruct_event(buildUnstruct(invalidEmailSent)) - - IgluUtils - .extractAndValidateInputJsons( - input, - SpecHelpers.client, - SpecHelpers.registryLookup - ) - .value - .map { - case Ior.Both( - NonEmptyList( - _: FailureDetails.SchemaViolation, - Nil - ), - IgluUtils.EventExtractResult(Nil, None, Nil) - ) => - ok - case other => ko(s"[$other] isn't an error with SchemaViolation") - } - } - - "return one SchemaViolation if the input event contains an invalid context" >> { - val input = new EnrichedEvent - input.setContexts(buildInputContexts(List(invalidEmailSent))) - - IgluUtils - .extractAndValidateInputJsons( - input, - SpecHelpers.client, - SpecHelpers.registryLookup - ) - .value - .map { - case Ior.Both( - NonEmptyList( - _: FailureDetails.SchemaViolation, - Nil - ), - IgluUtils.EventExtractResult(Nil, None, Nil) - ) => - ok - case other => ko(s"[$other] isn't an error with SchemaViolation") - } - } - - "return 2 SchemaViolation if the input event contains an invalid unstructured event and 1 invalid context" >> { - val input = new EnrichedEvent - input.setUnstruct_event(invalidEmailSent) - input.setContexts(buildInputContexts(List(invalidEmailSent))) - - IgluUtils - .extractAndValidateInputJsons( - input, - SpecHelpers.client, - SpecHelpers.registryLookup - ) - .value - .map { - case Ior.Both( - NonEmptyList( - _: FailureDetails.SchemaViolation, - List(_: FailureDetails.SchemaViolation) - ), - IgluUtils.EventExtractResult(Nil, None, Nil) - ) => - ok - case other => ko(s"[$other] isn't 2 errors with SchemaViolation") - } - } - - "return the extracted unstructured event and the extracted input contexts if they are all valid" >> { - val input = new EnrichedEvent - input.setUnstruct_event(buildUnstruct(emailSent1)) - input.setContexts(buildInputContexts(List(emailSent1, emailSent2))) - - IgluUtils - .extractAndValidateInputJsons( - input, - SpecHelpers.client, - SpecHelpers.registryLookup - ) - .value - .map { - case Ior.Right(IgluUtils.EventExtractResult(contexts, Some(unstructEvent), validationInfos)) - if contexts.size == 2 - && validationInfos.isEmpty - && (unstructEvent :: contexts).forall(_.schema == emailSentSchema) => - ok - case other => - ko( - s"[$other] doesn't contain the 2 contexts and the unstructured event" - ) - } - } - - "return the extracted unstructured event and the extracted input contexts when schema is superseded by another schema" >> { - val input = new EnrichedEvent - input.setUnstruct_event(buildUnstruct(supersedingExample1)) - input.setContexts(buildInputContexts(List(supersedingExample1, supersedingExample2))) - - val expectedValidationInfoContext = parse( - """ { + case Ior.Right(List(sdj1, sdj2)) if sdj1.schema == emailSentSchema && sdj2.schema == emailSentSchema => ok + case other => ko(s"[$other] doesn't contain 2 valid contexts with schema $emailSentSchema") + } + } + } + + "extractAndValidateInputJsons" should { + "return one SchemaViolation if the input event contains an invalid unstructured event" >> { + val input = new EnrichedEvent + input.setUnstruct_event(buildUnstruct(invalidEmailSent)) + + IgluUtils + .extractAndValidateInputJsons( + input, + SpecHelpers.client, + SpecHelpers.registryLookup + ) + .value + .map { + case Ior.Both( + NonEmptyList( + _: FailureDetails.SchemaViolation, + Nil + ), + IgluUtils.EventExtractResult(Nil, None, Nil) + ) => + ok + case other => ko(s"[$other] isn't an error with SchemaViolation") + } + } + + "return one SchemaViolation if the input event contains an invalid context" >> { + val input = new EnrichedEvent + input.setContexts(buildInputContexts(List(invalidEmailSent))) + + IgluUtils + .extractAndValidateInputJsons( + input, + SpecHelpers.client, + SpecHelpers.registryLookup + ) + .value + .map { + case Ior.Both( + NonEmptyList( + _: FailureDetails.SchemaViolation, + Nil + ), + IgluUtils.EventExtractResult(Nil, None, Nil) + ) => + ok + case other => ko(s"[$other] isn't an error with SchemaViolation") + } + } + + "return 2 SchemaViolation if the input event contains an invalid unstructured event and 1 invalid context" >> { + val input = new EnrichedEvent + input.setUnstruct_event(invalidEmailSent) + input.setContexts(buildInputContexts(List(invalidEmailSent))) + + IgluUtils + .extractAndValidateInputJsons( + input, + SpecHelpers.client, + SpecHelpers.registryLookup + ) + .value + .map { + case Ior.Both( + NonEmptyList( + _: FailureDetails.SchemaViolation, + List(_: FailureDetails.SchemaViolation) + ), + IgluUtils.EventExtractResult(Nil, None, Nil) + ) => + ok + case other => ko(s"[$other] isn't 2 errors with SchemaViolation") + } + } + + "return the extracted unstructured event and the extracted input contexts if they are all valid" >> { + val input = new EnrichedEvent + input.setUnstruct_event(buildUnstruct(emailSent1)) + input.setContexts(buildInputContexts(List(emailSent1, emailSent2))) + + IgluUtils + .extractAndValidateInputJsons( + input, + SpecHelpers.client, + SpecHelpers.registryLookup + ) + .value + .map { + case Ior.Right(IgluUtils.EventExtractResult(contexts, Some(unstructEvent), validationInfos)) + if contexts.size == 2 + && validationInfos.isEmpty + && (unstructEvent :: contexts).forall(_.schema == emailSentSchema) => + ok + case other => + ko( + s"[$other] doesn't contain the 2 contexts and the unstructured event" + ) + } + } + + "return the SchemaViolation of the invalid context in the Left and the extracted unstructured event in the Right" >> { + val input = new EnrichedEvent + input.setUnstruct_event(buildUnstruct(emailSent1)) + input.setContexts(buildInputContexts(List(invalidEmailSent))) + + IgluUtils + .extractAndValidateInputJsons( + input, + SpecHelpers.client, + SpecHelpers.registryLookup + ) + .value + .map { + case Ior.Both( + NonEmptyList(FailureDetails.SchemaViolation.IgluError(_, ValidationError(_, _)), _), + extract + ) if extract.contexts.isEmpty && extract.unstructEvent.isDefined && extract.unstructEvent.get.schema == emailSentSchema => + ok + case other => + ko( + s"[$other] isn't one ValidationError and an unstructured event with schema $emailSentSchema" + ) + } + } + + "return the SchemaViolation of the invalid unstructured event in the Left and the valid context in the Right" >> { + val input = new EnrichedEvent + input.setUnstruct_event(buildUnstruct(invalidEmailSent)) + input.setContexts(buildInputContexts(List(emailSent1))) + + IgluUtils + .extractAndValidateInputJsons( + input, + SpecHelpers.client, + SpecHelpers.registryLookup + ) + .value + .map { + case Ior.Both( + NonEmptyList(FailureDetails.SchemaViolation.IgluError(_, ValidationError(_, _)), _), + extract + ) if extract.contexts.size == 1 && extract.contexts.head.schema == emailSentSchema && extract.unstructEvent.isEmpty => + ok + case other => + ko( + s"[$other] isn't one ValidationError and one context with schema $emailSentSchema" + ) + } + } + + "return the extracted unstructured event and the extracted input contexts when schema is superseded by another schema" >> { + val input = new EnrichedEvent + input.setUnstruct_event(buildUnstruct(supersedingExample1)) + input.setContexts(buildInputContexts(List(supersedingExample1, supersedingExample2))) + + val expectedValidationInfoContext = parse( + """ { | "originalSchema" : "iglu:com.acme/superseding_example/jsonschema/1-0-0", | "validatedWith" : "1-0-1" |}""".stripMargin - ).toOption.get - - IgluUtils - .extractAndValidateInputJsons( - input, - SpecHelpers.client, - SpecHelpers.registryLookup - ) - .value - .map { - case Ior.Right(IgluUtils.EventExtractResult(contexts, Some(unstructEvent), List(validationInfo))) - if contexts.size == 2 - && unstructEvent.schema == supersedingExampleSchema101 - && contexts.count(_.schema == supersedingExampleSchema101) == 2 - && validationInfo.schema == IgluUtils.ValidationInfo.schemaKey - && validationInfo.data == expectedValidationInfoContext => - ok - case other => - ko( - s"[$other] doesn't contain the 2 contexts and the unstructured event with the superseded schema" - ) - } - } + ).toOption.get + + IgluUtils + .extractAndValidateInputJsons( + input, + SpecHelpers.client, + SpecHelpers.registryLookup + ) + .value + .map { + case Ior.Right(IgluUtils.EventExtractResult(contexts, Some(unstructEvent), List(validationInfo))) + if contexts.size == 2 + && unstructEvent.schema == supersedingExampleSchema101 + && contexts.count(_.schema == supersedingExampleSchema101) == 2 + && validationInfo.schema == IgluUtils.ValidationInfo.schemaKey + && validationInfo.data == expectedValidationInfoContext => + ok + case other => + ko( + s"[$other] doesn't contain the 2 contexts and the unstructured event with the superseded schema" + ) + } } }