diff --git a/schema-util/bigquery/src/test/resources/notaschema.json b/schema-util/bigquery/src/test/resources/notaschema.json index 0292c835e2..a73a32e4e5 100644 --- a/schema-util/bigquery/src/test/resources/notaschema.json +++ b/schema-util/bigquery/src/test/resources/notaschema.json @@ -1 +1 @@ -[{"type":"record","name":"publishedAuction","namespace":"connect.biddit.mongodb.auction","fields":[{"name":"auction","type":["null",{"type":"record","name":"auction","namespace":"connect.biddit.mongodb.auction.publishedAuction","fields":[{"name":"propertyId","type":["null","string"],"default":null},{"name":"interactiveSaleId","type":["null","string"],"default":null},{"name":"initialSellingPrice","type":["null","string"],"default":null},{"name":"biddingStartDate","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"biddingEndDate","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"biddingPace","type":["null","string"],"default":null},{"name":"sellingPrice","type":["null","string"],"default":null},{"name":"initalSellingPrice","type":["null","string"],"default":null},{"name":"preEmptiveRight","type":["null","boolean"],"default":null},{"name":"preEmptiveRightRUP","type":["null","boolean"],"default":null},{"name":"createdByPersonId","type":["null","string"],"default":null},{"name":"studyId","type":["null","string"],"default":null},{"name":"creationDate","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"highestBidTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"infoSessionDetailsMultilingual","type":["null",{"type":"record","name":"infoSessionDetailsMultilingual","namespace":"connect.biddit.mongodb.auction.publishedAuction.auction","fields":[{"name":"nl","type":["null","string"],"default":null},{"name":"fr","type":["null","string"],"default":null},{"name":"en","type":["null","string"],"default":null},{"name":"de","type":["null","string"],"default":null}]}],"default":null},{"name":"allocationDetailsMultilingual","type":["null",{"type":"record","name":"allocationDetailsMultilingual","namespace":"connect.biddit.mongodb.auction.publishedAuction.auction","fields":[{"name":"fr","type":["null","string"],"default":null},{"name":"nl","type":["null","string"],"default":null},{"name":"en","type":["null","string"],"default":null},{"name":"de","type":["null","string"],"default":null}]}],"default":null},{"name":"sellingPriceChanged","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"sellingPriceChangedReason","type":["null","string"],"default":null},{"name":"archived","type":["null","boolean"],"default":null}]}],"default":null},{"name":"auctionPremium","type":["null","double"],"default":null},{"name":"soldTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"extended","type":["null","boolean"],"default":null},{"name":"archived","type":["null","boolean"],"default":null},{"name":"assignedBid","type":["null",{"type":"record","name":"assignedBid","namespace":"connect.biddit.mongodb.auction.publishedAuction","fields":[{"name":"bid","type":["null",{"type":"record","name":"bid","namespace":"connect.biddit.mongodb.auction.publishedAuction.assignedBid","fields":[{"name":"_id","type":["null","string"],"default":null},{"name":"propertyId","type":["null","string"],"default":null},{"name":"accountId","type":["null","string"],"default":null},{"name":"amount","type":["null","double"],"default":null},{"name":"refused","type":["null","boolean"],"default":null},{"name":"removed","type":["null","boolean"],"default":null},{"name":"bidTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"bidType","type":["null","string"],"default":null},{"name":"proxyBidderId","type":["null","string"],"default":null}]}],"default":null},{"name":"auctionBidderProfile","type":["null",{"type":"record","name":"auctionBidderProfile","namespace":"connect.biddit.mongodb.auction.publishedAuction.assignedBid","fields":[{"name":"accountId","type":["null","string"],"default":null}]}],"default":null},{"name":"soldTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null}]}],"default":null},{"name":"currentBid","type":["null","double"],"default":null},{"name":"bids","type":["null",{"type":"array","items":["null",{"type":"record","name":"bids","namespace":"connect.biddit.mongodb.auction.publishedAuction","fields":[{"name":"_id","type":["null","string"],"default":null},{"name":"propertyId","type":["null","string"],"default":null},{"name":"accountId","type":["null","string"],"default":null},{"name":"amount","type":["null","double"],"default":null},{"name":"refused","type":["null","boolean"],"default":null},{"name":"removed","type":["null","boolean"],"default":null},{"name":"bidTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"bidType","type":["null","string"],"default":null},{"name":"proxyBidderId","type":["null","string"],"default":null}]}]}],"default":null},{"name":"bidderProfiles","type":["null",{"type":"map","values":["null",{"type":"record","name":"bidderProfiles","namespace":"connect.biddit.mongodb.auction.publishedAuction","fields":[{"name":"propertyId","type":["null","string"],"default":null},{"name":"accountId","type":["null","string"],"default":null},{"name":"maritalStatus","type":["null","string"],"default":null},{"name":"profileBidderType","type":["null","string"],"default":null},{"name":"acceptedTermsAndConditions","type":["null","boolean"],"default":null},{"name":"birthYear","type":["null","int"],"default":null},{"name":"gender","type":["null","string"],"default":null},{"name":"nationality","type":["null","string"],"default":null},{"name":"inNameOfCompany","type":["null","boolean"],"default":null}]}]}],"default":null},{"name":"_class","type":["null","string"],"default":null},{"name":"id","type":["null","string"],"default":null},{"name":"__deleted","type":["null","boolean"],"default":null},{"name":"archivedTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"auctionWithdrawnTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"withdrawal","type":["null",{"type":"record","name":"withdrawal","namespace":"connect.biddit.mongodb.auction.publishedAuction","fields":[{"name":"timestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"reason","type":["null","string"],"default":null},{"name":"extra","type":["null",{"type":"record","name":"extra","namespace":"connect.biddit.mongodb.auction.publishedAuction.withdrawal","fields":[{"name":"alreadyPutUpForSaleBeforeBiddit","type":["null","boolean"],"default":null},{"name":"initialSellingPriceHigherThan80PercentOfTheAskedPrice","type":["null","boolean"],"default":null},{"name":"askedPriceWasRealisticPrice","type":["null","boolean"],"default":null}]}],"default":null},{"name":"details","type":["null","string"],"default":null}]}],"default":null},{"name":"excludedBidders","type":["null",{"type":"array","items":["null","string"]}],"default":null},{"name":"_meta","type":["null",{"type":"record","name":"MetaRecord","namespace":"io.tree53.data.model.meta","fields":[{"name":"offset","type":{"type":"array","items":{"type":"record","name":"OffsetRecord","fields":[{"name":"topic","type":"string"},{"name":"partition","type":"int"},{"name":"offset","type":"long"}]}}},{"name":"ts","type":{"type":"long","logicalType":"timestamp-millis"}},{"name":"appId","type":"string"}]}],"default":null},{"name":"_src","type":["null",{"type":"record","name":"SourceRecord","namespace":"io.tree53.data.model.meta","fields":[{"name":"op","type":["null","string"],"default":null},{"name":"ts","type":["null",{"type":"long","logicalType":"timestamp-millis"}],"default":null},{"name":"partial","type":["null","boolean"],"default":null},{"name":"entity_ns","type":["null","string"],"default":null},{"name":"entity_name","type":["null","string"],"default":null},{"name":"version","type":["null","string"],"default":null},{"name":"connector","type":["null","string"],"default":null},{"name":"snapshot","type":["null","string"],"default":null},{"name":"ts_ms","type":["null",{"type":"long","logicalType":"timestamp-millis"}],"default":null},{"name":"db","type":["null","string"],"default":null},{"name":"rs","type":["null","string"],"default":null},{"name":"ord","type":["null","long"],"default":null}]}],"default":null}]}] \ No newline at end of file +[{"type":"record","name":"publishedAction","namespace":"connect.billing.mongodb.action","fields":[{"name":"action","type":["null",{"type":"record","name":"action","namespace":"connect.billing.mongodb.action.publishedAction","fields":[{"name":"propertyId","type":["null","string"],"default":null},{"name":"interactiveSaleId","type":["null","string"],"default":null},{"name":"initialSellingPrice","type":["null","string"],"default":null},{"name":"billingStartDate","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"billingEndDate","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"billingPace","type":["null","string"],"default":null},{"name":"sellingPrice","type":["null","string"],"default":null},{"name":"initalSellingPrice","type":["null","string"],"default":null},{"name":"preEmptiveRight","type":["null","boolean"],"default":null},{"name":"preEmptiveRightRUP","type":["null","boolean"],"default":null},{"name":"createdByPersonId","type":["null","string"],"default":null},{"name":"studyId","type":["null","string"],"default":null},{"name":"creationDate","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"highestBidTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"infoSessionDetailsMultilingual","type":["null",{"type":"record","name":"infoSessionDetailsMultilingual","namespace":"connect.billing.mongodb.action.publishedAction.action","fields":[{"name":"nl","type":["null","string"],"default":null},{"name":"fr","type":["null","string"],"default":null},{"name":"en","type":["null","string"],"default":null},{"name":"de","type":["null","string"],"default":null}]}],"default":null},{"name":"allocationDetailsMultilingual","type":["null",{"type":"record","name":"allocationDetailsMultilingual","namespace":"connect.billing.mongodb.action.publishedAction.action","fields":[{"name":"fr","type":["null","string"],"default":null},{"name":"nl","type":["null","string"],"default":null},{"name":"en","type":["null","string"],"default":null},{"name":"de","type":["null","string"],"default":null}]}],"default":null},{"name":"sellingPriceChanged","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"sellingPriceChangedReason","type":["null","string"],"default":null},{"name":"archived","type":["null","boolean"],"default":null}]}],"default":null},{"name":"actionPremium","type":["null","double"],"default":null},{"name":"soldTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"extended","type":["null","boolean"],"default":null},{"name":"archived","type":["null","boolean"],"default":null},{"name":"assignedBid","type":["null",{"type":"record","name":"assignedBid","namespace":"connect.billing.mongodb.action.publishedAction","fields":[{"name":"bid","type":["null",{"type":"record","name":"bid","namespace":"connect.billing.mongodb.action.publishedAction.assignedBid","fields":[{"name":"_id","type":["null","string"],"default":null},{"name":"propertyId","type":["null","string"],"default":null},{"name":"accountId","type":["null","string"],"default":null},{"name":"amount","type":["null","double"],"default":null},{"name":"refused","type":["null","boolean"],"default":null},{"name":"removed","type":["null","boolean"],"default":null},{"name":"bidTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"bidType","type":["null","string"],"default":null},{"name":"proxyBidderId","type":["null","string"],"default":null}]}],"default":null},{"name":"actionBidderProfile","type":["null",{"type":"record","name":"actionBidderProfile","namespace":"connect.billing.mongodb.action.publishedAction.assignedBid","fields":[{"name":"accountId","type":["null","string"],"default":null}]}],"default":null},{"name":"soldTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null}]}],"default":null},{"name":"currentBid","type":["null","double"],"default":null},{"name":"bids","type":["null",{"type":"array","items":["null",{"type":"record","name":"bids","namespace":"connect.billing.mongodb.action.publishedAction","fields":[{"name":"_id","type":["null","string"],"default":null},{"name":"propertyId","type":["null","string"],"default":null},{"name":"accountId","type":["null","string"],"default":null},{"name":"amount","type":["null","double"],"default":null},{"name":"refused","type":["null","boolean"],"default":null},{"name":"removed","type":["null","boolean"],"default":null},{"name":"bidTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"bidType","type":["null","string"],"default":null},{"name":"proxyBidderId","type":["null","string"],"default":null}]}]}],"default":null},{"name":"bidderProfiles","type":["null",{"type":"map","values":["null",{"type":"record","name":"bidderProfiles","namespace":"connect.billing.mongodb.action.publishedAction","fields":[{"name":"propertyId","type":["null","string"],"default":null},{"name":"accountId","type":["null","string"],"default":null},{"name":"maritalStatus","type":["null","string"],"default":null},{"name":"profileBidderType","type":["null","string"],"default":null},{"name":"acceptedTermsAndConditions","type":["null","boolean"],"default":null},{"name":"birthYear","type":["null","int"],"default":null},{"name":"gender","type":["null","string"],"default":null},{"name":"nationality","type":["null","string"],"default":null},{"name":"inNameOfCompany","type":["null","boolean"],"default":null}]}]}],"default":null},{"name":"_class","type":["null","string"],"default":null},{"name":"id","type":["null","string"],"default":null},{"name":"__deleted","type":["null","boolean"],"default":null},{"name":"archivedTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"actionWithdrawnTimestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"withdrawal","type":["null",{"type":"record","name":"withdrawal","namespace":"connect.billing.mongodb.action.publishedAction","fields":[{"name":"timestamp","type":["null",{"type":"long","connect.version":1,"connect.name":"org.apache.kafka.connect.data.Timestamp","logicalType":"timestamp-millis"}],"default":null},{"name":"reason","type":["null","string"],"default":null},{"name":"extra","type":["null",{"type":"record","name":"extra","namespace":"connect.billing.mongodb.action.publishedAction.withdrawal","fields":[{"name":"alreadyPutUpForSaleBeforeBiddit","type":["null","boolean"],"default":null},{"name":"initialSellingPriceHigherThan80PercentOfTheAskedPrice","type":["null","boolean"],"default":null},{"name":"askedPriceWasRealisticPrice","type":["null","boolean"],"default":null}]}],"default":null},{"name":"details","type":["null","string"],"default":null}]}],"default":null},{"name":"excludedBidders","type":["null",{"type":"array","items":["null","string"]}],"default":null},{"name":"_meta","type":["null",{"type":"record","name":"MetaRecord","namespace":"io.tree53.data.model.meta","fields":[{"name":"offset","type":{"type":"array","items":{"type":"record","name":"OffsetRecord","fields":[{"name":"topic","type":"string"},{"name":"partition","type":"int"},{"name":"offset","type":"long"}]}}},{"name":"ts","type":{"type":"long","logicalType":"timestamp-millis"}},{"name":"appId","type":"string"}]}],"default":null},{"name":"_src","type":["null",{"type":"record","name":"SourceRecord","namespace":"io.tree53.data.model.meta","fields":[{"name":"op","type":["null","string"],"default":null},{"name":"ts","type":["null",{"type":"long","logicalType":"timestamp-millis"}],"default":null},{"name":"partial","type":["null","boolean"],"default":null},{"name":"entity_ns","type":["null","string"],"default":null},{"name":"entity_name","type":["null","string"],"default":null},{"name":"version","type":["null","string"],"default":null},{"name":"connector","type":["null","string"],"default":null},{"name":"snapshot","type":["null","string"],"default":null},{"name":"ts_ms","type":["null",{"type":"long","logicalType":"timestamp-millis"}],"default":null},{"name":"db","type":["null","string"],"default":null},{"name":"rs","type":["null","string"],"default":null},{"name":"ord","type":["null","long"],"default":null}]}],"default":null}]}] \ No newline at end of file diff --git a/schema-util/bigquery/src/test/resources/reorderedschema.json b/schema-util/bigquery/src/test/resources/reorderedschema.json index e69de29bb2..f1c21c261a 100644 --- a/schema-util/bigquery/src/test/resources/reorderedschema.json +++ b/schema-util/bigquery/src/test/resources/reorderedschema.json @@ -0,0 +1,59 @@ +[ + { + "name": "id", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "first_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "last_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "addresses", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "address", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "city", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "state", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "zip", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "numberOfYears", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "status", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "dob", + "type": "DATE", + "mode": "NULLABLE" + } +] \ No newline at end of file diff --git a/schema-util/bigquery/src/test/resources/requiredschema.json b/schema-util/bigquery/src/test/resources/requiredschema.json index e69de29bb2..3600f609f2 100644 --- a/schema-util/bigquery/src/test/resources/requiredschema.json +++ b/schema-util/bigquery/src/test/resources/requiredschema.json @@ -0,0 +1,59 @@ +[ + { + "name": "id", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "first_name", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "last_name", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "dob", + "type": "DATE", + "mode": "NULLABLE" + }, + { + "name": "addresses", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "status", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "address", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "city", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "state", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "zip", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "numberOfYears", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } +] \ No newline at end of file diff --git a/schema-util/bigquery/src/test/resources/shortenedschema.json b/schema-util/bigquery/src/test/resources/shortenedschema.json index e69de29bb2..b62ab01237 100644 --- a/schema-util/bigquery/src/test/resources/shortenedschema.json +++ b/schema-util/bigquery/src/test/resources/shortenedschema.json @@ -0,0 +1,54 @@ +[ + { + "name": "id", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "first_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "last_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "dob", + "type": "DATE", + "mode": "NULLABLE" + }, + { + "name": "addresses", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "status", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "address", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "city", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "state", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "zip", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } +] \ No newline at end of file diff --git a/schema-util/bigquery/src/test/resources/typemodifiedschema.json b/schema-util/bigquery/src/test/resources/typemodifiedschema.json index e69de29bb2..84ee1fa584 100644 --- a/schema-util/bigquery/src/test/resources/typemodifiedschema.json +++ b/schema-util/bigquery/src/test/resources/typemodifiedschema.json @@ -0,0 +1,59 @@ +[ + { + "name": "id", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "first_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "last_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "dob", + "type": "DATE", + "mode": "NULLABLE" + }, + { + "name": "addresses", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "status", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "address", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "city", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "state", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "zip", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "numberOfYears", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + } +] \ No newline at end of file