round_3_reliability.Rmd

---
title: "round_3_reliability"
author: "Drew Walker"
date: "10/24/2021"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
library(tidyverse)
library(irr)
library(janitor)
library(table1)
```


```{r read_data}
raw_annotation_data_dw_cl <- read_csv("round_3_codings_merged.csv") %>% 
  clean_names() 
colnames(raw_annotation_data_dw_cl)
ordered <- raw_annotation_data_dw_cl[,order(colnames(raw_annotation_data_dw_cl))] %>% 
  select(-text,-word,-n,-id,-name,-comments,-cl,-keyword) %>% 
  filter(duplicate != 1)
# FILTER duplicate == 0
# Change variables to reflect new sheet. 
str(ordered)
ordered$non_ingestible <- as.numeric(ordered$non_ingestible)
```

```{r kappas}

#cbdsmokeshop
cbd_smokeshop <- raw_annotation_data_dw_cl %>% 
  select(cbdsmokeshop,cbdsmokeshop_cl)

cbd_smokeshop_kappa <- irr::kappa2(cbd_smokeshop)
cbd_smokeshop_kappa
#retailer
retailer <- raw_annotation_data_dw_cl %>% 
  select(retailer,retailer_cl)

retailer_kappa <- irr::kappa2(retailer)
retailer_kappa

#duplicate
duplicate <- raw_annotation_data_dw_cl %>% 
  select(duplicate,duplicate_cl)

duplicate_kappa <- irr::kappa2(duplicate)
duplicate_kappa
#rt
rt <- raw_annotation_data_dw_cl %>% 
  select(rt,rt_cl)

rt_kappa <- irr::kappa2(rt)
rt_kappa

#keyword_mentioned_in_post

keyword_mentioned_in_post <- raw_annotation_data_dw_cl %>% 
  select(keyword_mentioned_in_post,keyword_mentioned_in_post_cl)

keyword_mentioned_in_post_kappa <- irr::kappa2(keyword_mentioned_in_post)
keyword_mentioned_in_post_kappa


#amount_of_drug

amount_of_drug <- raw_annotation_data_dw_cl %>% 
  select(amount_of_drug,amount_of_drug_cl)

amount_of_drug_kappa <- irr::kappa2(amount_of_drug)
amount_of_drug_kappa


#ingestible_edible
ingestible_edible <- raw_annotation_data_dw_cl %>% 
  select(ingestible_edible,ingestible_edible_cl)

ingestible_edible_kappa <- irr::kappa2(ingestible_edible)
ingestible_edible_kappa
#vape
vape <- raw_annotation_data_dw_cl %>% 
  select(vape,vape_cl)

vape_kappa <- irr::kappa2(vape)
vape_kappa
#flower_combustible
flower_combustible <- raw_annotation_data_dw_cl %>% 
  select(flower_combustible,flower_combustible_cl)

flower_kappa <- irr::kappa2(flower_combustible)
flower_kappa

#noningestible
noningestible <- raw_annotation_data_dw_cl %>% 
  select(non_ingestible,non_ingestible_cl)

noningestible_kappa <- irr::kappa2(noningestible)
noningestible_kappa

#delta_8_mentioned_in_post
delta_8_specific <- raw_annotation_data_dw_cl %>% 
  select(delta_8_specific,delta_8_specific_cl)

delta_8_specific_kappa <- irr::kappa2(delta_8_specific)
delta_8_specific_kappa

#flavoring_mentioned
flavoring_mentioned <- raw_annotation_data_dw_cl %>% 
  select(flavoring_mentioned,flavoring_mentioned_cl)

flavoring_mentioned_kappa <- irr::kappa2(flavoring_mentioned)
flavoring_mentioned_kappa

#tested
tested <- raw_annotation_data_dw_cl %>% 
  select(tested,tested_cl)

tested_kappa <- irr::kappa2(tested)
tested_kappa

#medical_claim
therapeutic_claim <- raw_annotation_data_dw_cl %>% 
  select(therapeutic_claim,therapeutic_claim_cl)

therapeutic_claim_kappa <- irr::kappa2(therapeutic_claim)
therapeutic_claim_kappa




#effects_mentioned
effects_mentioned <- raw_annotation_data_dw_cl %>% 
  select(effects_mentioned,effects_mentioned_cl)

effects_mentioned_kappa <- irr::kappa2(effects_mentioned)
effects_mentioned_kappa
#giveaway
sale_promotion_or_giveaway <- raw_annotation_data_dw_cl %>% 
  select(sale_promotion_or_giveaway,sale_promotion_or_giveaway_cl)

sale_promotion_or_giveaway_kappa <- irr::kappa2(sale_promotion_or_giveaway)
sale_promotion_or_giveaway_kappa

#legal_high
legal_high <- raw_annotation_data_dw_cl %>% 
  select(legal_high,legal_high_cl)

legal_high_kappa <- irr::kappa2(legal_high)
legal_high_kappa
#popularity
popularity <- raw_annotation_data_dw_cl %>% 
  select(popularity,popularity_cl)

popularity_kappa <- irr::kappa2(popularity)
popularity_kappa
#website
website <- raw_annotation_data_dw_cl %>% 
  select(website,website_cl)

website_kappa <- irr::kappa2(website)
website_kappa

#availability
availability <- raw_annotation_data_dw_cl %>% 
  select(availability,availability_cl)

availability_kappa <- irr::kappa2(availability)
availability_kappa

#naturalness
naturalness <- raw_annotation_data_dw_cl %>% 
  select(naturalness,naturalness_cl)

naturalness_kappa <- irr::kappa2(naturalness)
naturalness_kappa
```

```{r pooled-kappas}

ordered_20_codes <- ordered %>% 
  select(-duplicate, -duplicate_cl)

ordered_pivoted <- ordered_20_codes %>% 
  pivot_longer(
    cols= everything(),
    names_to = c("coder"),
    names_pattern = "(_cl$)",
    values_to = "score"
  )%>% 
  replace_na(list(coder="drew"))

ordered_pivot_back <- ordered_pivoted %>% 
  pivot_wider(
    names_from="coder",
    values_from = "score",
    ) %>% 
  unnest() 

pooled_kappa <- irr::kappa2(ordered_pivot_back)
pooled_kappa


```


```{r combine-labels}
combined <- ordered %>% 
  mutate(rt_comb = case_when(rt==1|rt_cl==1 ~1,TRUE~0),
         cbdsmoke_comb = case_when(cbdsmokeshop==1|cbdsmokeshop_cl==1 ~1,TRUE~0),
         non_ingestible_comb = case_when(non_ingestible==1|non_ingestible_cl==1 ~1,TRUE~0),
         keyword_comb = case_when(keyword_mentioned_in_post==1|keyword_mentioned_in_post_cl==1 ~1,TRUE~0),
         retailer_comb = case_when(retailer==1|retailer_cl==1 ~1,TRUE~0),
         flower_comb = case_when(flower_combustible==1|flower_combustible_cl==1 ~1,TRUE~0),
         sale_promotion_or_giveaway_comb = case_when(sale_promotion_or_giveaway==1|sale_promotion_or_giveaway_cl==1 ~1,TRUE~0),
         flavoring_mentioned_comb = case_when(flavoring_mentioned==1|flavoring_mentioned_cl==1 ~1,TRUE~0),
         amount_comb = case_when(amount_of_drug==1|amount_of_drug_cl==1 ~1,TRUE~0),
         ingestible_comb = case_when(ingestible_edible==1|ingestible_edible_cl==1 ~1,TRUE~0),
         vape_comb = case_when(vape==1|vape_cl==1 ~1,TRUE~0),
         website_comb = case_when(website==1|website_cl==1 ~1,TRUE~0),
         popularity_comb = case_when(popularity==1|popularity_cl==1 ~1,TRUE~0),
         delta_8_specific_comb = case_when(delta_8_specific==1|delta_8_specific_cl==1 ~1,TRUE~0),
         legal_high_comb = case_when(legal_high==1|legal_high_cl==1 ~1,TRUE~0),
         naturalness_comb = case_when(naturalness==1|naturalness_cl==1 ~1,TRUE~0),
         therapeutic_comb = case_when(therapeutic_claim==1|therapeutic_claim_cl==1 ~1,TRUE~0),
         availability_comb = case_when(availability==1|availability_cl==1 ~1,TRUE~0),
         duplicate_comb = case_when(duplicate==1|duplicate_cl==1 ~1,TRUE~0),
         tested_comb = case_when(tested==1|tested_cl==1 ~1,TRUE~0),
         effects_mentioned_comb = case_when(effects_mentioned==1|effects_mentioned_cl==1 ~1,TRUE~0)
         )
table(combined$rt_comb)
table(combined$cbdsmoke_comb)
table(combined$non_ingestible_comb)
table(combined$keyword_comb)
table(combined$retailer_comb)
table(combined$flower_comb)
table(combined$sale_promotion_or_giveaway_comb)
table(combined$flavoring_mentioned_comb)
table(combined$amount_comb)
table(combined$ingestible_comb)
table(combined$vape_comb)
table(combined$website_comb)
table(combined$popularity_comb)
table(combined$delta_8_specific_comb)
table(combined$legal_high_comb)
table(combined$naturalness_comb)
table(combined$therapeutic_comb)
table(combined$availability_comb)
table(combined$duplicate_comb)
table(combined$tested_comb)
table(combined$effects_mentioned_comb)
```



#Delta 8 ml dataset 
```{r delta8-ml}
#delta8_ml <- ordered %>% 
#mutate(delta8_combined = case_when(delta_8_mentioned_in_post == "1" #| delta_8_mentioned_in_post_c == "1" ~ "1", TRUE ~ "0")) 
#
#delta8_ml$text <- raw_annotation_data_dw_cl$text 
#
#delta8_ml_clean <- delta8_ml %>% 
#  select(text,delta8_combined)
```