Skip to content

Commit

Permalink
Populate remaining event fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@spenes
spenes committed Jan 1, 2024
1 parent b9000d1 commit a6113a3
Show file tree
Hide file tree
Showing 4 changed files with 346 additions and 80 deletions.
165 changes: 91 additions & 74 deletions core/src/main/scala/com/snowplowanalytics/snowplow/eventgen/enrich/SdkEvent.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import com.snowplowanalytics.snowplow.eventgen.protocol.event.{
LegacyEvent,
PagePing,
PageView,
RemainingFields,
StructEvent,
UnstructEventWrapper
}
import io.circe.Json
Expand Down Expand Up @@ -53,7 +55,7 @@ object SdkEvent {
private def eventFromColPayload(p: CollectorPayload, fallbackEid: UUID): List[Event] =
p.payload.map { el =>
val evnt = Some(el.e match {
case EventType.Struct => "struct"
case EventType.Struct | EventType.RemainingFields => "struct"
case EventType.Unstruct => "unstruct"
case EventType.PageView => "page_view"
case EventType.PagePing => "page_ping"
Expand All @@ -67,6 +69,21 @@ object SdkEvent {
(UnstructEvent(data = None), evnt, None, None, None)
}

val remainingFieldsOpt = el.event match {
case a: RemainingFields => Some(a)
case _ => None
}

val structEventOpt = el.event match {
case s: StructEvent => Some(s)
case _ => None
}

val pagePingOpt = el.event match {
case p: PagePing => Some(p)
case _ => None
}

Event(
app_id = el.app.aid,
platform = Some(el.app.p),
Expand All @@ -91,71 +108,71 @@ object SdkEvent {
.orElse(p.context.userId)
.orElse(p.context.headers.cookie)
.map(_.toString),
geo_country = None,
geo_region = None,
geo_city = None,
geo_zipcode = None,
geo_latitude = None,
geo_longitude = None,
geo_region_name = None,
ip_isp = None,
ip_organization = None,
ip_domain = None,
ip_netspeed = None,
geo_country = remainingFieldsOpt.flatMap(_.geo_country),
geo_region = remainingFieldsOpt.flatMap(_.geo_region),
geo_city = remainingFieldsOpt.flatMap(_.geo_city),
geo_zipcode = remainingFieldsOpt.flatMap(_.geo_zipcode),
geo_latitude = remainingFieldsOpt.flatMap(_.geo_latitude),
geo_longitude = remainingFieldsOpt.flatMap(_.geo_longitude),
geo_region_name = remainingFieldsOpt.flatMap(_.geo_region_name),
ip_isp = remainingFieldsOpt.flatMap(_.ip_isp),
ip_organization = remainingFieldsOpt.flatMap(_.ip_organization),
ip_domain = remainingFieldsOpt.flatMap(_.ip_domain),
ip_netspeed = remainingFieldsOpt.flatMap(_.ip_netspeed),
page_url = extractWeb(el, _.url.map(_.toString)),
page_title = extractWeb(el, _.page),
page_referrer = extractWeb(el, _.refr.map(_.toString)),
page_urlscheme = extractWeb(el, _.url.map(_.scheme)),
page_urlhost = extractWeb(el, _.url.map(_.host)),
page_urlport = extractWeb(el, _.url.map(_.sdkPort)),
page_urlpath = extractWeb(el, _.url.map(_.path)),
page_urlquery = None,
page_urlfragment = None,
page_urlquery = remainingFieldsOpt.flatMap(_.page_urlquery),
page_urlfragment = remainingFieldsOpt.flatMap(_.page_urlfragment),
refr_urlscheme = extractWeb(el, _.refr.map(_.scheme)),
refr_urlhost = extractWeb(el, _.refr.map(_.host)),
refr_urlport = extractWeb(el, _.refr.map(_.sdkPort)),
refr_urlpath = extractWeb(el, _.refr.map(_.path)),
refr_urlquery = None,
refr_urlfragment = None,
refr_medium = None,
refr_source = None,
refr_term = None,
mkt_medium = None,
mkt_source = None,
mkt_term = None,
mkt_content = None,
mkt_campaign = None,
refr_urlquery = remainingFieldsOpt.flatMap(_.refr_urlquery),
refr_urlfragment = remainingFieldsOpt.flatMap(_.refr_urlfragment),
refr_medium = remainingFieldsOpt.flatMap(_.refr_medium),
refr_source = remainingFieldsOpt.flatMap(_.refr_source),
refr_term = remainingFieldsOpt.flatMap(_.refr_term),
mkt_medium = remainingFieldsOpt.flatMap(_.mkt_medium),
mkt_source = remainingFieldsOpt.flatMap(_.mkt_source),
mkt_term = remainingFieldsOpt.flatMap(_.mkt_term),
mkt_content = remainingFieldsOpt.flatMap(_.mkt_content),
mkt_campaign = remainingFieldsOpt.flatMap(_.mkt_campaign),
contexts = el.context.map(_.contexts).getOrElse(Contexts(List.empty[SelfDescribingData[Json]])),
se_category = None,
se_action = None,
se_label = None,
se_property = None,
se_value = None,
se_category = structEventOpt.flatMap(_.se_ca),
se_action = structEventOpt.flatMap(_.se_ac),
se_label = structEventOpt.flatMap(_.se_la),
se_property = structEventOpt.flatMap(_.se_pr),
se_value = structEventOpt.flatMap(_.se_va),
unstruct_event = ue,
tr_orderid = None,
tr_affiliation = None,
tr_total = None,
tr_tax = None,
tr_shipping = None,
tr_city = None,
tr_state = None,
tr_country = None,
ti_orderid = None,
ti_sku = None,
ti_name = None,
ti_category = None,
ti_price = None,
ti_quantity = None,
pp_xoffset_min = None,
pp_xoffset_max = None,
pp_yoffset_min = None,
pp_yoffset_max = None,
tr_orderid = remainingFieldsOpt.flatMap(_.tr_orderid),
tr_affiliation = remainingFieldsOpt.flatMap(_.tr_affiliation),
tr_total = remainingFieldsOpt.flatMap(_.tr_total),
tr_tax = remainingFieldsOpt.flatMap(_.tr_tax),
tr_shipping = remainingFieldsOpt.flatMap(_.tr_shipping),
tr_city = remainingFieldsOpt.flatMap(_.tr_city),
tr_state = remainingFieldsOpt.flatMap(_.tr_state),
tr_country = remainingFieldsOpt.flatMap(_.tr_country),
ti_orderid = remainingFieldsOpt.flatMap(_.ti_orderid),
ti_sku = remainingFieldsOpt.flatMap(_.ti_sku),
ti_name = remainingFieldsOpt.flatMap(_.ti_name),
ti_category = remainingFieldsOpt.flatMap(_.ti_category),
ti_price = remainingFieldsOpt.flatMap(_.ti_price),
ti_quantity = remainingFieldsOpt.flatMap(_.ti_quantity),
pp_xoffset_min = pagePingOpt.flatMap(_.pp_mix),
pp_xoffset_max = pagePingOpt.flatMap(_.pp_max),
pp_yoffset_min = pagePingOpt.flatMap(_.pp_miy),
pp_yoffset_max = pagePingOpt.flatMap(_.pp_may),
useragent = extractWeb(el, _.ua).orElse(p.context.headers.ua),
br_name = None,
br_family = None,
br_version = None,
br_type = None,
br_renderengine = None,
br_name = remainingFieldsOpt.flatMap(_.br_name),
br_family = remainingFieldsOpt.flatMap(_.br_family),
br_version = remainingFieldsOpt.flatMap(_.br_version),
br_type = remainingFieldsOpt.flatMap(_.br_type),
br_renderengine = remainingFieldsOpt.flatMap(_.br_renderengine),
br_lang = extractWeb(el, _.lang),
br_features_pdf = extractWeb(el, _.f_pdf),
br_features_flash = extractWeb(el, _.f_fla),
Expand All @@ -170,41 +187,41 @@ object SdkEvent {
br_colordepth = extractWeb(el, _.cd).map(_.toString),
br_viewwidth = extractWeb(el, _.vp.map(_.x)),
br_viewheight = extractWeb(el, _.vp.map(_.y)),
os_name = None,
os_family = None,
os_manufacturer = None,
os_name = remainingFieldsOpt.flatMap(_.os_name),
os_family = remainingFieldsOpt.flatMap(_.os_family),
os_manufacturer = remainingFieldsOpt.flatMap(_.os_manufacturer),
os_timezone = el.dt.flatMap(_.tz),
dvce_type = None,
dvce_ismobile = None,
dvce_type = remainingFieldsOpt.flatMap(_.dvce_type),
dvce_ismobile = remainingFieldsOpt.flatMap(_.dvce_ismobile),
dvce_screenwidth = el.dev.flatMap(_.res.map(_.x)),
dvce_screenheight = el.dev.flatMap(_.res.map(_.y)),
doc_charset = extractWeb(el, _.cs),
doc_width = extractWeb(el, _.ds.map(_.x)),
doc_height = extractWeb(el, _.ds.map(_.y)),
tr_currency = None,
tr_total_base = None,
tr_tax_base = None,
tr_shipping_base = None,
ti_currency = None,
ti_price_base = None,
base_currency = None,
geo_timezone = None,
mkt_clickid = None,
mkt_network = None,
etl_tags = None,
tr_currency = remainingFieldsOpt.flatMap(_.tr_currency),
tr_total_base = remainingFieldsOpt.flatMap(_.tr_total_base),
tr_tax_base = remainingFieldsOpt.flatMap(_.tr_tax_base),
tr_shipping_base = remainingFieldsOpt.flatMap(_.tr_shipping_base),
ti_currency = remainingFieldsOpt.flatMap(_.ti_currency),
ti_price_base = remainingFieldsOpt.flatMap(_.ti_price_base),
base_currency = remainingFieldsOpt.flatMap(_.base_currency),
geo_timezone = remainingFieldsOpt.flatMap(_.geo_timezone),
mkt_clickid = remainingFieldsOpt.flatMap(_.mkt_clickid),
mkt_network = remainingFieldsOpt.flatMap(_.mkt_network),
etl_tags = remainingFieldsOpt.flatMap(_.etl_tags),
dvce_sent_tstamp = el.dt.flatMap(_.dtm),
refr_domain_userid = None,
refr_dvce_tstamp = None,
refr_domain_userid = remainingFieldsOpt.flatMap(_.refr_domain_userid),
refr_dvce_tstamp = remainingFieldsOpt.flatMap(_.refr_dvce_tstamp),
derived_contexts = Contexts(List.empty[SelfDescribingData[Json]]),
domain_sessionid = el.u.flatMap(_.sid.map(_.toString)),
derived_tstamp = None,
derived_tstamp = remainingFieldsOpt.flatMap(_.derived_tstamp),
event_vendor = ueVendor,
event_name = eName,
event_format = ueFormat,
event_version = ueVersion,
event_fingerprint = None,
true_tstamp = None,
etl_tstamp = None
event_fingerprint = remainingFieldsOpt.flatMap(_.event_fingerprint),
true_tstamp = el.dt.flatMap(_.ttm),
etl_tstamp = remainingFieldsOpt.flatMap(_.etl_tstamp)
)
}

Expand Down
9 changes: 5 additions & 4 deletions core/src/main/scala/com/snowplowanalytics/snowplow/eventgen/protocol/Body.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,11 @@ object Body {
tv <- TrackerVersion.gen
u <- User.genOpt
event <- e match {
case EventType.Struct => StructEvent.gen
case EventType.Unstruct => UnstructEventWrapper.gen(frequencies.unstructEventFrequencies)
case EventType.PageView => PageView.gen
case EventType.PagePing => PagePing.gen
case EventType.Struct => StructEvent.gen
case EventType.Unstruct => UnstructEventWrapper.gen(frequencies.unstructEventFrequencies)
case EventType.PageView => PageView.gen
case EventType.PagePing => PagePing.gen
case EventType.RemainingFields => RemainingFields.gen
}
context <- Context.ContextsWrapper.genOps
} yield Body(e, app, dt, dev, tv, et, u, event, context)
Expand Down
8 changes: 6 additions & 2 deletions core/src/main/scala/com/snowplowanalytics/snowplow/eventgen/protocol/event/EventType.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,13 @@ case class EventFrequencies(
unstruct: Int,
pageView: Int,
pagePing: Int,
remainingFields: Int,
unstructEventFrequencies: UnstructEventFrequencies
)

sealed trait EventType {
override def toString: String = this match {
case EventType.Struct => "se"
case EventType.Struct | EventType.RemainingFields => "se"
case EventType.Unstruct => "ue"
case EventType.PageView => "pv"
case EventType.PagePing => "pp"
Expand All @@ -45,6 +46,8 @@ object EventType {

case object PagePing extends EventType

case object RemainingFields extends EventType

// Commented ADT are not implemented in generator
// case object StructLegacy extends EventType
// case object AdImpression extends EventType
Expand All @@ -55,7 +58,8 @@ object EventType {
(frequencies.struct, Struct),
(frequencies.unstruct, Unstruct),
(frequencies.pageView, PageView),
(frequencies.pagePing, PagePing)
(frequencies.pagePing, PagePing),
(frequencies.remainingFields, RemainingFields)
// AdImpression, Transaction, TransactionItem,
)
}
Loading

0 comments on commit a6113a3

Please sign in to comment.