googleonly_regression_mardown.Rmd

---
title: "Windbelt_JustGoogle_regressions"
author: "WINDBELT GP 2019"
date: "January 28, 2019"
output: ''
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```


## R Markdown


```{r packages}

### Packages 
library(pdftools)
library(tm)
# install.packages("rlang")
# remove.packages("devtools")
# install.packages("devtools")
library(devtools)
library(dplyr)
library(tidytext)
library(broom)
library(data.table)
# install.packages("colorspace")
library(tidyverse)
library(purrr)
library(googledrive)
library(knitr)
library(readr)
library(stringr)
library(gsubfn)
# install.packages("tidyverse")
# install.packages("tidytext")
# library(tm.plugin.lexisnexis)
# install.packages("gsubfn", "dplyr", "tidytext")

```

```{r ventyx_readcsv}

#### Ventyx ####

#For MAC and BREN computers:
ventyx_projects <- read_csv("~/Desktop/ventyx_converted_01_08_2019_PopDensity_Income_Viewshed_lowhighimpact_doc_names_IncomeMedian.csv")

#For Alex:
#ventyx_projects <- read_csv("C:/Users/airvi/Desktop/ventyx_converted_01_08_2019_PopDensity_Income_Viewshed_lowhighimpact_doc_names_IncomeMedian.csv")

#To view the dataset:
#View(ventyx_projects)

#To change the 0-1 definition to Low-High
ventyx_df <- mutate(ventyx_projects, H_L_W = ifelse(lr_tif>0, "Low", "High"))

#Remove whitespace for name matching later one
ventyx_df$ProjectName <- gsub(" ", "", ventyx_df$ProjectName, fixed = TRUE)
ventyx_df$document <- gsub(" ", "", ventyx_df$document, fixed = TRUE)


```


```{r google_readcsv}

#### Google News ######
#Before running this code, make sure you have the Google scraping dataset saved on your desktop. The file path that the "read_csv" function accesses should be where the file is located on your desktop. 

#For MAC and BREN computers:
google_df <- read_csv("~/Desktop/google_scraping_01-08-19.csv")

#For Alex:
#google_df <- read_csv("C:/Users/airvi/Desktop/google_scraping_01-08-19.csv")

#To view the dataset:
#View(google_df)


#### Google ####
#This creates a data frame of just the project name and full text for each Google News article
#Each row is the project name with the full article text
projects_google <- data_frame(document = google_df$ProjectName, text = google_df$FullText)


```


```{r google_unest}

#### Google #####
#Dataset with each word in a row associated with its project source 
projects_words_google <- projects_google %>% 
  tidytext::unnest_tokens(output = word, input = text, token = "words", to_lower = F)
head(projects_words_google)

```



```{r google_count}

#### Google ####
projects_words_count_google <- projects_words_google %>%
  group_by(document, word) %>% 
  summarise(count = n())
#View(projects_pdfwords_count)
#Counts the number of time a specific words is found in the article


```


#### Sentiment dictionaries
```{r sentiment_dictionaries}
# Using 'afinn' vs. 'nrc sentiment tests.

get_sentiments("afinn") # associates word with a sentiment score
#afinn scores/ranks from -5 to +5 for positive or negative sentiment.

get_sentiments("nrc") # associated word with another sentiment feeling word

# View(get_sentiments("afinn"))
# View(get_sentiments("nrc"))

# We want scores not categorized words, so we will use AFINN for now.
```


```{r sentiment_bind}

#### Google ####
projects_score_bind_google <- projects_words_count_google %>% 
  left_join(get_sentiments("afinn"), by = "word") %>% 
  filter(score != "NA")


```

```{r sentiment_scores}


#### Google #####
total_sentiment_with_stats_google <- projects_score_bind_google %>% 
  #filter(score !="NA") %>% 
  group_by(document) %>% 
  summarise(totals = weighted.mean(score, w = count),
            standard_dev = sd(score), 
            variance = var(score)
            )

total_sentiment_google <- projects_score_bind_google %>% 
  #filter(score !="NA") %>% 
  group_by(document) %>% 
  summarise(totals = mean(score))
  
#View(total_sentiment_with_stats_google)
#View(total_sentiment_google)


```

```{r}


ventyx_df[ventyx_df$ProjectName == "71RWind",]$document = "71RWind_71Ranch_1(1).PDF"
ventyx_df[ventyx_df$ProjectName == "CHillWindFarm",]$document = "CHill_ExergyIntegratedSystems_1.PDF"
ventyx_df[ventyx_df$ProjectName == "AdamsProject",]$document = "AdamsProject_MidAmerican_1(2).PDF"
ventyx_df[ventyx_df$ProjectName == "LJTrust",]$document = "LJTrust_LJTrust_1(1).PDF"
ventyx_df[ventyx_df$ProjectName == "MinonkWindFarm",]$document = "MinonkWindFarm_MinonkWindFarm_1.PDF"


```

```{r merge}

#### Google #####
#Convert to data frame
total_sentiment_google_df = as.data.frame(total_sentiment_google)
#Remove white space in order to perform merge (google data has extra white spaces)
total_sentiment_google_df$document <- gsub(" ", "", total_sentiment_google_df$document, fixed = TRUE)

# View(total_sentiment_google_df)


#Merge with original data
ventyx_df_google_Sen <- merge(ventyx_df, total_sentiment_google_df, by.x ="ProjectName", by.y = "document", all = TRUE)
#Rename "totals" columns
ventyx_df_google_Sen <- rename(ventyx_df_google_Sen, Google_Sentiment=totals)

# View(ventyx_df_google_Sen)
# names(total_sentiment_google_df)
# names(ventyx_df_NU_Sen)
# ventyx_df_NU_Sen$

names(ventyx_df_google_Sen)

ventyx_df_google_Sen <- ventyx_df_google_Sen %>% 
  # rename(Google_Sentiment=totals) %>% 
  mutate(Google_Sentiment = ifelse(is.na(Google_Sentiment),0,Google_Sentiment))

ventyx_df_google_Sen <- ventyx_df_NU_google_Sen %>% 
  filter(Capacity >= 0) %>% 
  filter(TimelineDays > 0)

ventyx_df_google_Sen <- mutate(ventyx_df_google_Sen, Sign_Google = ifelse(Google_Sentiment>0, "Positive", "Negative"))

```

####Add environmental memberships data
```{r env_members}
#For Alex
#zip_conversion <- read_csv("C:/Users/airvi/Documents/Bren/GroupProject/zip_code_database.csv")

#For Bren computers
zip_conversion <- read_csv("~/Desktop/zip_code_database.csv")

#For Alex
#memberships <- read_csv("C:/Users/airvi/Documents/Bren/GroupProject/membership_data_combined.csv")

#For Bren computers
memberships <- read_csv("~/Desktop/membership_data_combined.csv")

#Make sure all zip codes are 5 digits
memberships$zip <- sprintf("%05d", memberships$zip)

#Merge memberships data with county
zip_county_merge <- merge(zip_conversion, memberships, by = "zip")

#Remove the word county, parish, and municipality, then trim any trailing whitespace
zip_county_merge$county <- gsub("County","",zip_county_merge$county)
zip_county_merge$county <- gsub("Parish","",zip_county_merge$county)
zip_county_merge$county <- gsub("Municipality","",zip_county_merge$county)
zip_county_merge$county <- trimws(zip_county_merge$county, "r")

#Group by county and sum all of the members for environmental organizations
zip_county_merge_aggregate <- zip_county_merge %>% 
  group_by(county) %>% 
  summarise(members_env = sum(members_nrdc,members_nwf,members_tnc))

#Merge with ventyx dataset and set NA membership values to zero
ventyx_df_NU_google_Sen <- merge(ventyx_df_NU_google_Sen, zip_county_merge_aggregate, by.x = "County", by.y = "county", all.x = TRUE)
ventyx_df_NU_google_Sen$members_env[is.na(ventyx_df_NU_google_Sen$members_env)] <- 0
```

####Prepare the relevant columns for regression
```{r organize_for_regressions}
#na.omit(ventyx_df_NU_Sen$Household_MeanIncome)
#na.omit(ventyx_df_NU_Sen$PopDensity_mi)
#na.omit(ventyx_df_NU_Sen$lr_tif)
#na.omit(ventyx_df_NU_Sen$View_Score)


#### Google ####
ventyx_df_google_Sen$H_L_W <- as.factor(ventyx_df_google_Sen$H_L_W)

ventyx_df_google_Sen$PopDensity_mi <- as.numeric(ventyx_df_google_Sen$PopDensity_mi)

ventyx_df_google_Sen$Household_MedianIncome <- as.numeric(ventyx_df_google_Sen$Household_MedianIncome)

ventyx_df_google_Sen$View_Score <- as.numeric(ventyx_df_google_Sen$View_Score)

ventyx_df_google_Sen$H_L_W <- relevel(ventyx_df_google_Sen$H_L_W, ref = "High")

ventyx_df_google_Sen$Sign_Google <- as.factor(ventyx_df_google_Sen$Sign_Google)

#str(ventyx_df_google_Sen)

#summary(ventyx_df_google_Sen)

```

####Run Regressions
```{r R1}

require(broom)
# install.packages("dotwhisker")
library(dotwhisker)####Run Regressions

```

```{r R1}

require(broom)
# install.packages("dotwhisker")
library(dotwhisker)

ventyx_df_NU_Sen$Capacity_1 <- scale(ventyx_df_NU_Sen$Capacity)

View(ventyx_df_NU_Sen)

renamed_variables <- c(View_Score = "View Score",
                        Google_Sentiment = "Google News Publicity score",
                        NU_Sentiment = "Nexis Uni Publicity score",
                        PopDensity_mi = "Population Density per m2",
                        Household_MedianIncome = "Household Median Income",
                        Capacity = "Capacity",
                        H_L_WLow = "High vs. low impact area")

```


```{r regressions}
####1. All variables included -- using NU sentiment scores. N/A scores removed from dataset.
reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_Sen)
print(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State)




#Plotting residuals. This should be repeated for the other regressions.
r1 <- resid(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State)
plot(r1)

summary(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State)
require(ggplot2)

View(tidy(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State))

dwplot(tidy(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State))

#Results: significance found in household median income (0.00358) and capacity (0.00582). 
#NU sentiment p-value: 0.22164
#H_L_W p-value: 0.62724
####

```



```{r R2}

# ##2. All variables included -- using NU sentiment scores and an interaction between the high-low variable and NU sentiment. N/A scores removed from dataset.
# reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*NU_Sentiment, data=ventyx_df_NU_Sen)
# 
# summary(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State_Inter)
# dwplot(tidy(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State_Inter))+
#     theme_bw()+
#   labs(x = "timeline")
#  
# # Results: significance found in household median income (0.002891) and capacity (0.004197) and H_L_W (0.04786) and the interaction term (0.03391)
# #NU sentiment p-value: 0.82485
# ####

```

```{r R3}

####3. All variables included -- using Google sentiment scores. N/A scores removed from dataset.
reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + Google_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_google_Sen)

summary(reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State)
dwplot(tidy(reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State))+
  theme_bw()+
  labs(x = "timeline")
 
#Results: no significance found, although Google sentiment is close
#Google sentiment p-value: 0.1015
#H_L_W p-value: 0.4502
####
```


```{r R4}

####4. All variables included -- using Google sentiment scores and an interaction between the high-low variable and Google sentiment. N/A scores removed from dataset.
reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + Google_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*Google_Sentiment, data=ventyx_df_google_Sen)

summary(reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter)
dwplot(tidy(reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter))+
    theme_bw()+
  labs(x = "timeline")
 
#Results: no significance found, although Google sentiment is close
#Google sentiment p-value: 0.1021
#H_L_W p-value: 0.97
#Interaction: 0.48
#

```


### NA Sentiment scores coded as '0' - No interaction term

#### NU and Google 
```{r R5}
# 
# ####5. All variables included -- using both NU and Google scores. N/A scores coded as "0":
# 
# reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + Google_Sentiment + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_google_Sen)
# 
# summary(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State)
# 
# # reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State$terms
# 
# renamed_variables <- c(View_Score = "View Score",
#                         Google_Sentiment = "Google News Publicity score",
#                         NU_Sentiment = "Nexis Uni Publicity score",
#                         PopDensity_mi = "Population Density per m2",
#                         Household_MedianIncome = "Household Median Income",
#                         Capacity = "Capacity",
#                         H_L_WLow = "High vs. low impact area")
# 
# 
# # everything 
# dwplot(tidy(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State), dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick", lwd=0.5))+
#   ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
#     theme_classic()+
#   labs(x = "timeline")+
#     ggtitle("Regression output: all variables, including NU and Google scores. N/A \n scores coded as 0:")+
#   theme(legend.background = element_rect(colour="grey80"))
# 
# 
# regression1 <- tidy(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State)
# 
# dwplot(regression1[!grepl("^State*", regression1$term),] %>% 
#    relabel_predictors(renamed_variables), 
#    dot_args = list(colour = "firebrick"),
#    whisker_args = list(colour = "firebrick", lwd = 0.5)
#    )+
#   labs(x = "Timeline (days)", y = "")+
#   ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
#   theme_classic()+
#   ggtitle("Regression output: all variables, including NU and Google scores. N/A \n scores coded as 0:")+
#   theme(legend.background = element_rect(colour="grey80"))
#   
# 
# #Results: significance found in Google sentiment (0.02867) and capacity (0.09094)
# #H_L_W p-value: 0.91039
# #NU_Sentiment: 0.57313
# ####
# ## 01/20/18 - MS: with additional text, got more significance on some values it seems 

```

#### NU only
```{r R6}
# ####6. All variables included -- using only NU scores. N/A scores coded as "0".
# reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_google_Sen)
# 
# summary(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State)
# 
# dwplot(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State, dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick"))+
#   ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
#     theme_bw()+
#   labs(x = "timeline")
# 
# regression2 <- tidy(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State)
# 
# dwplot(regression2[!grepl("^State*", regression2$term),] %>% 
#    relabel_predictors(renamed_variables), 
#    dot_args = list(colour = "firebrick"),
#    whisker_args = list(colour = "firebrick")
#    )+
#   labs(x = "Timeline (days)", y = "")+
#   ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
#   theme_classic()+
#   ggtitle("Regression output: all variables, using only NU scores. N/A \n scores coded as 0:")+
#   theme(legend.background = element_rect(colour="grey80"))
#   

#Results: significance found in capacity (0.07004)
#NU_Sentiment: 0.986
#H_L_W: 0.856

```

```{r R7}

# ####7. All variables included -- using only NU scores and an interaction between high-low and NU sentiment. N/A scores coded as "0".
# reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*NU_Sentiment, data=ventyx_df_NU_google_Sen)

## 01/20/18 - MS: with additional text, got more significance on values 
#Results: significance found in capacity (0.04916)
#NU_Sentiment: 0.73846
#H_L_W: 0.73898

#>>>>>>> 234d9c8e20e0a8df12e4b5896502ba0366e05b08

```

#### Google only
```{r R8}
####8. All variables included -- using only Google scores. N/A scores coded as "0"
reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + Google_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_google_Sen)

summary(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State)

## everything
dwplot(tidy(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State), dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick"))+
    theme_bw()+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  labs(x = "Timeline (days)")

regression3 <- tidy(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State) %>% 
  mutate()

# Main variables
dwplot(regression3[!grepl("^State*", regression3$term),] %>%  
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick")
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables, using only Google scores. N/A \n scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))


#Results: significance found in Google sentiment (0.02972) and capacity (0.08989)
#H_L_W: 0.93917
#
####
## 01/20/18 - MS: with additional text, got more significance on values 
# Results: significance found in Google sentiment (0.02486) and capacity (0.06455)
# H_L_W: 0.84519


```


```{r COmparative dotwhiskerplot}

R1 <- reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State
R2 <- reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State
R3 <- reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State

# general_everything plot

# unique(regression1$term)
# 
# dwplot(list(tidy(R1),tidy(R2),tidy(R3), conf.int = FALSE)
#        # dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick")
#        )+
#     theme_bw()+
#   labs(x = "timeline")



### does not work unless subset by regression 
regression1_1 <- tidy(R1)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression1_G+NU")

regression2_1 <- tidy(R2)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression2_NU")

regression3_1 <- tidy(R3)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression3_G")

combined_reg <-  bind_rows(regression1_1, regression2_1, regression3_1)
View(combined_reg)


dwplot(combined_reg %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(aes(shape = model)),
   whisker_args = list(aes(colour = model))
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "grey80", linetype = 2)+
  theme_classic()+
  ggtitle("Combined regression outputs;\n N/A scores coded as 0:")+
  theme(plot.title = element_text(face="bold"),
          legend.position = c(0.65, 0.8),
          legend.justification = c(0, 0),
          legend.background = element_rect(colour = "grey60"),
          legend.title.align = .5)
  


# install.packages("jtools")
# library(jtools)
# plot_summs(tidy(R1), scale = TRUE, plot.distributions = TRUE, inner_ci_level = .9)
#  coefplot()

```



### NA Sentiment scores coded as '0' - w/ interaction term

##### Google only
```{r R9}

####9. All variables included -- using only Google scores and an interaction between high-low and Google sentiment. N/A scores coded as "0"
reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + Google_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*Google_Sentiment, data=ventyx_df_NU_google_Sen)

summary(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter)

# dwplot(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter)+
#   theme_bw()+
#   labs(x = "timeline")


## everything
dwplot(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter, dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick"))+
    theme_bw()+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  labs(x = "Timeline (days)")

regression3 <- tidy(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter)

# Main variables
dwplot(regression3[!grepl("^State*", regression3$term),] %>%  
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick")
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables, using only Google scores \n and interaction term. N/A scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))


#Results: significance found in Google sentiment (0.01327) and capacity (0.09445)
#H_L_W: 0.63589
#Interaction: 0.204
####
#  With added text and removal of capacity/timelinedays  01/21/18
#Results: significance found in Google sentiment (0.01023) and capacity (0.06793)
#H_L_W: 0.54475
#Interaction: 0.18378
####


```

#### Google and NU 


```{r combined_with_interaction term}


R1 <- reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State_Inter
R2 <- reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State_Inter
R3 <- reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter

# general_everything plot

# unique(regression1$term)

# dwplot(list(tidy(R1),tidy(R2),tidy(R3), conf.int = FALSE)
#        # dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick")
#        )+
#     theme_bw()+
#   labs(x = "timeline")


### does not work unless subset by regression 
regression1_1 <- tidy(R1)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression1_G+NU")

regression2_1 <- tidy(R2)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression2_NU")

regression3_1 <- tidy(R3)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression3_G")

combined_reg <-  bind_rows(regression1_1, regression2_1, regression3_1)
# View(combined_reg)


dwplot(combined_reg %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(aes(shape = model)),
   whisker_args = list(aes(colour = model))
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "grey80", linetype = 2)+
  theme_classic()+
  ggtitle("Combined regression outputs with interaction term:\n N/A scores coded as 0:\n")+
  theme(plot.title = element_text(face="bold"),
          legend.position = c(0.7, 0.77),
          legend.justification = c(0, 0),
          legend.background = element_rect(colour = "grey60"),
          legend.title.align = .5)
  



```

### Sentiment on left hand side - NAs still coded as '0'

# google sentiment
```{r R12}
####12. Google Sentiment on left-hand side. N/A scores coded as "0"
reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State <- lm(Google_Sentiment ~ View_Score + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_google_Sen)

summary(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State)

# dwplot(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State)+
#   theme_bw()+
#   labs(x = "timeline")
# 

regressionB <- tidy(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State)

dwplot(regressionB[!grepl("^State*", regressionB$term),] %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick", lwd = 0.5)
   )+
  labs(x = "Sentiment score", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables including NU and Google \n scores, interaction terms. N/A scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))



#Results no significance found
#H_L_W: 0.154
####
# 01/21/18 - difference in results with changes to ventyx df and text from NU:
#H_L_W: 0.16372
#capacity: 0.11

```

```{r R13}
####13. Timeline on left-hand side, using NU sentiment. Interaction term between NU and high-low impact. Environmental memberships variable added. N/A scores coded as 0
reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter <- lm(TimelineDays ~ View_Score + NU_Sentiment + members_env + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*NU_Sentiment, data=ventyx_df_NU_google_Sen)

summary(reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter)
dwplot(reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter)
```


```{r R14}
####14. Timeline on left-hand side, using Google sentiment. Interaction term between Google and high-low impact. Environmental memberships variable added. N/A scores coded as 0
reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter <- lm(TimelineDays ~ View_Score + Google_Sentiment + members_env + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*Google_Sentiment, data=ventyx_df_NU_google_Sen)

summary(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter)
dwplot(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter)
```


## OTHER 

#### Headlines
```{r Extract_headlines_id_duplicates}

#can ignore for regression

## extract headlines 
projects_pdf_titles <- project_pdfs_full_text %>% 
  mutate(title_extract = grepl("\r\n \\d.(.*?) Client/Matter:", text, ignore.case = FALSE)) 
# %>% 
  # filter(title_extract == TRUE) %>% 
  # mutate(title_match = strapply(text, "\r\n \\d.(.*?) Client/Matter:"))


  
## Identify duplicate headlines 
  
  

  
```


#### Populate with negative words 
```{r negative_words}
negative_words <- paste0(c('negative|postpone|against|delay|lawsuit|litigation|protest|^cost|^stop'))
# Function to replace `character(0)` with NAs as NULL values are dropped when flattening list
# inspired by: https://colinfay.me/purrr-set-na/
charnull_set <- function(x) {
  p <- purrr::as_mapper(~identical(., character(0)))
  x[p(x)] <- NA
  return(x)
}
projects_pdftext_3 <- projects_pdftext_NU %>%
  mutate(query_hits = str_extract_all(text, pattern = regex(negative_words, ignore_case=TRUE)) %>%  # Extract all the keywords
           map(~charnull_set(.x)) %>%   # Replace character(0) with NAs
           map_chr(~glue::glue_collapse(.x, sep = ";")) %>%   # collapse the multiple hits
           tolower) # all our keywords are lower case
projects_pdftext_grouped <- projects_pdftext_2 %>%
  group_by(document, query_hits)
# OR 
my_data1grouped <- my_data1 %>%
  group_by(document, query_hits)%>%
  summarise(word_list = glue::glue_collapse(query_hits, sep = ";"))
View(projects_pdftext_2)
```

ventyx_df_NU_Sen$Capacity_1 <- scale(ventyx_df_NU_Sen$Capacity)

View(ventyx_df_NU_Sen)
####1. All variables included -- using NU sentiment scores. N/A scores removed from dataset.
reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_Sen)
print(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State)




#Plotting residuals. This should be repeated for the other regressions.
r1 <- resid(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State)
plot(r1)

summary(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State)
require(ggplot2)

View(tidy(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State))

dwplot(tidy(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State))

#Results: significance found in household median income (0.00358) and capacity (0.00582). 
#NU sentiment p-value: 0.22164
#H_L_W p-value: 0.62724
####

```



```{r R2}
##2. All variables included -- using NU sentiment scores and an interaction between the high-low variable and NU sentiment. N/A scores removed from dataset.
reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*NU_Sentiment, data=ventyx_df_NU_Sen)

summary(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State_Inter)
dwplot(tidy(reg_TLD_VS_Sen_PopDen_MedianIncome_Cap_HL_State_Inter))+
    theme_bw()+
  labs(x = "timeline")
 
# Results: significance found in household median income (0.002891) and capacity (0.004197) and H_L_W (0.04786) and the interaction term (0.03391)
#NU sentiment p-value: 0.82485
####

```

```{r R3}

####3. All variables included -- using Google sentiment scores. N/A scores removed from dataset.
reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + Google_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_google_Sen)

summary(reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State)
dwplot(tidy(reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State))+
  theme_bw()+
  labs(x = "timeline")
 
#Results: no significance found, although Google sentiment is close
#Google sentiment p-value: 0.1015
#H_L_W p-value: 0.4502
####
```


```{r R4}

####4. All variables included -- using Google sentiment scores and an interaction between the high-low variable and Google sentiment. N/A scores removed from dataset.
reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + Google_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*Google_Sentiment, data=ventyx_df_google_Sen)

summary(reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter)
dwplot(tidy(reg_TLD_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter))+
    theme_bw()+
  labs(x = "timeline")
 
# Results: no significance found, although Google sentiment is close
#  Google sentiment p-value: 0.1021
# H_L_W p-value: 0.97
# Interaction: 0.48
#

```


### NA Sentiment scores coded as '0' - No interaction term

#### NU and Google 
```{r R5}

####5. All variables included -- using both NU and Google scores. N/A scores coded as "0":

reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + Google_Sentiment + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_google_Sen)

summary(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State)

# reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State$terms

renamed_variables <- c(View_Score = "View Score",
                        Google_Sentiment = "Google News Publicity score",
                        NU_Sentiment = "Nexis Uni Publicity score",
                        PopDensity_mi = "Population Density per m2",
                        Household_MedianIncome = "Household Median Income",
                        Capacity = "Capacity",
                        H_L_WLow = "High vs. low impact area")


# everything 
dwplot(tidy(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State), dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick", lwd=0.5))+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
    theme_classic()+
  labs(x = "timeline")+
    ggtitle("Regression output: all variables, including NU and Google scores. N/A \n scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))


regression1 <- tidy(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State)

dwplot(regression1[!grepl("^State*", regression1$term),] %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick", lwd = 0.5)
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables, including NU and Google scores. N/A \n scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))
  

#Results: significance found in Google sentiment (0.02867) and capacity (0.09094)
#H_L_W p-value: 0.91039
#NU_Sentiment: 0.57313
####
## 01/20/18 - MS: with additional text, got more significance on some values it seems 

```

#### NU only
```{r R6}
####6. All variables included -- using only NU scores. N/A scores coded as "0".
reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_google_Sen)

summary(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State)

dwplot(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State, dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick"))+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
    theme_bw()+
  labs(x = "timeline")

regression2 <- tidy(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State)

dwplot(regression2[!grepl("^State*", regression2$term),] %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick")
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables, using only NU scores. N/A \n scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))
  

#Results: significance found in capacity (0.07004)
#NU_Sentiment: 0.986
#H_L_W: 0.856

```

```{r R7}
####7. All variables included -- using only NU scores and an interaction between high-low and NU sentiment. N/A scores coded as "0".
reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*NU_Sentiment, data=ventyx_df_NU_google_Sen)

## 01/20/18 - MS: with additional text, got more significance on values 
#Results: significance found in capacity (0.04916)
#NU_Sentiment: 0.73846
#H_L_W: 0.73898

#>>>>>>> 234d9c8e20e0a8df12e4b5896502ba0366e05b08

```

#### Google only
```{r R8}
####8. All variables included -- using only Google scores. N/A scores coded as "0"
reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State <- lm(TimelineDays ~ View_Score + Google_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_google_Sen)

summary(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State)

## everything
dwplot(tidy(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State), dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick"))+
    theme_bw()+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  labs(x = "Timeline (days)")

regression3 <- tidy(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State) %>% 
  mutate()

# Main variables
dwplot(regression3[!grepl("^State*", regression3$term),] %>%  
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick")
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables, using only Google scores. N/A \n scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))


#Results: significance found in Google sentiment (0.02972) and capacity (0.08989)
#H_L_W: 0.93917
#
####
## 01/20/18 - MS: with additional text, got more significance on values 
# Results: significance found in Google sentiment (0.02486) and capacity (0.06455)
# H_L_W: 0.84519


```


```{r COmparative dotwhiskerplot}

R1 <- reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State
R2 <- reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State
R3 <- reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State

# general_everything plot

# unique(regression1$term)
# 
# dwplot(list(tidy(R1),tidy(R2),tidy(R3), conf.int = FALSE)
#        # dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick")
#        )+
#     theme_bw()+
#   labs(x = "timeline")



### does not work unless subset by regression 
regression1_1 <- tidy(R1)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression1_G+NU")

regression2_1 <- tidy(R2)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression2_NU")

regression3_1 <- tidy(R3)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression3_G")

combined_reg <-  bind_rows(regression1_1, regression2_1, regression3_1)
View(combined_reg)


dwplot(combined_reg %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(aes(shape = model)),
   whisker_args = list(aes(colour = model))
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "grey80", linetype = 2)+
  theme_classic()+
  ggtitle("Combined regression outputs;\n N/A scores coded as 0:")+
  theme(plot.title = element_text(face="bold"),
          legend.position = c(0.65, 0.8),
          legend.justification = c(0, 0),
          legend.background = element_rect(colour = "grey60"),
          legend.title.align = .5)
  


# install.packages("jtools")
# library(jtools)
# plot_summs(tidy(R1), scale = TRUE, plot.distributions = TRUE, inner_ci_level = .9)
#  coefplot()

```



### NA Sentiment scores coded as '0' - w/ interaction term

#### NU only 
```{r R7}

####7. All variables included -- using only NU scores and an interaction between high-low and NU sentiment. N/A scores coded as "0".
reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*NU_Sentiment, data=ventyx_df_NU_google_Sen)

summary(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State_Inter)
# dwplot(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State_Inter)+
#   theme_bw()+
#   labs(x = "timeline")+
  

dwplot(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State_Inter, dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick"))+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
    theme_bw()+
  labs(x = "timeline (days)")

regression2 <- tidy(reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State_Inter)

dwplot(regression2[!grepl("^State*", regression2$term),] %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick")
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables, using only NU scores\n and interaction term. N/A scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))
  
#Results: significance found in capacity (0.07323)
#NU_Sentiment: 0.505
#H_L_W: 0.643
#Interaction: 0.26455

# 01/20/18 - With added text from NU
#Results: significance found in capacity (0.05121)
#NU_Sentiment: 0.539
#H_L_W: 0.954
#Interaction: 0.56229



```

##### Google only
```{r R9}

####9. All variables included -- using only Google scores and an interaction between high-low and Google sentiment. N/A scores coded as "0"
reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + Google_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*Google_Sentiment, data=ventyx_df_NU_google_Sen)

summary(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter)

# dwplot(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter)+
#   theme_bw()+
#   labs(x = "timeline")


## everything
dwplot(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter, dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick"))+
    theme_bw()+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  labs(x = "Timeline (days)")

regression3 <- tidy(reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter)

# Main variables
dwplot(regression3[!grepl("^State*", regression3$term),] %>%  
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick")
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables, using only Google scores \n and interaction term. N/A scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))


#Results: significance found in Google sentiment (0.01327) and capacity (0.09445)
#H_L_W: 0.63589
#Interaction: 0.204
####
#  With added text and removal of capacity/timelinedays  01/21/18
#Results: significance found in Google sentiment (0.01023) and capacity (0.06793)
#H_L_W: 0.54475
#Interaction: 0.18378
####


```

#### Google and NU 
```{r R10}
####10. All variables included -- using both NU and Google scores and an interaction between the high-low variable and both NU and Google sentiment. N/A scores coded as "0"
reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State_Inter <- lm(TimelineDays ~ View_Score + Google_Sentiment + NU_Sentiment + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*Google_Sentiment + H_L_W*NU_Sentiment, data=ventyx_df_NU_google_Sen)

summary(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State_Inter)

dwplot(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State_Inter)+
  theme_bw()+
  labs(x = "timeline")


# reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State$terms

renamed_variables <- c(View_Score = "View Score",
                        Google_Sentiment = "Google News Publicity score",
                        NU_Sentiment = "Nexis Uni Publicity score",
                        PopDensity_mi = "Population Density per m2",
                        Household_MedianIncome = "Household Median Income",
                        Capacity = "Capacity",
                        H_L_WLow = "High vs. low impact area")


# everything 
dwplot(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State_Inter, dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick", lwd=0.5))+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
    theme_classic()+
  labs(x = "timeline(days)")+
    ggtitle("Regression output: all variables, including NU and Google \n scores. N/A scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))


regression1 <- tidy(reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State_Inter)

dwplot(regression1[!grepl("^State*", regression1$term),] %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick", lwd = 0.5)
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables including NU and Google \n scores, interaction terms. N/A scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))



#Results: significance found in Google sentiment (0.0067)
#NU_Sentiment: 0.196
#H_L_W: 0.804
#Interaction Google: 0.10846
#Interaction NU: 0.15778
####
# 01/21/18 - difference in results with changes to ventyx df and text from NU:
#Results: significance found in Google sentiment (0.00522)
#NU_Sentiment: 0.207
#H_L_W: 0.88696
#Interaction Google: 0.12861
#Interaction NU: 0.38246
####

```

```{r combined_with_interaction term}


R1 <- reg_TLD_ZEROES_VS_GNSen_PopDen_MedianIncome_Cap_HL_State_Inter
R2 <- reg_TLD_ZEROES_VS_NSen_PopDen_MedianIncome_Cap_HL_State_Inter
R3 <- reg_TLD_ZEROES_VS_GSen_PopDen_MedianIncome_Cap_HL_State_Inter

# general_everything plot

# unique(regression1$term)

# dwplot(list(tidy(R1),tidy(R2),tidy(R3), conf.int = FALSE)
#        # dot_args = list(colour = "firebrick"), whisker_args = list(colour = "firebrick")
#        )+
#     theme_bw()+
#   labs(x = "timeline")


### does not work unless subset by regression 
regression1_1 <- tidy(R1)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression1_G+NU")

regression2_1 <- tidy(R2)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression2_NU")

regression3_1 <- tidy(R3)  %>% 
  filter(!grepl('^State*', term)) %>% 
  mutate(model = "regression3_G")

combined_reg <-  bind_rows(regression1_1, regression2_1, regression3_1)
# View(combined_reg)


dwplot(combined_reg %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(aes(shape = model)),
   whisker_args = list(aes(colour = model))
   )+
  labs(x = "Timeline (days)", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "grey80", linetype = 2)+
  theme_classic()+
  ggtitle("Combined regression outputs with interaction term:\n N/A scores coded as 0:\n")+
  theme(plot.title = element_text(face="bold"),
          legend.position = c(0.7, 0.77),
          legend.justification = c(0, 0),
          legend.background = element_rect(colour = "grey60"),
          legend.title.align = .5)
  



```

### Sentiment on left hand side - NAs still coded as '0'

# NU sentiment
```{r R11}
####11. NU Sentiment on left-hand side. N/A scores coded as "0"
reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State <- lm(NU_Sentiment ~ View_Score + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_google_Sen)

summary(reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State)

dwplot(reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State)+
  theme_bw()+
  labs(x = "sentiment")

regressionA <- tidy(reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State)

dwplot(regressionA[!grepl("^State*", regressionA$term),] %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick", lwd = 0.5)
   )+
  labs(x = "Sentiment score", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables including NU and Google \n scores, interaction terms. N/A scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))




#Results: significance found in H_L_W (0.0273)
####
# 01/21/18 - difference in results with changes to ventyx df and text from NU:
#Results: significance found in H_L_W (0.0275)


```
# google sentiment
```{r R12}
####12. Google Sentiment on left-hand side. N/A scores coded as "0"
reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State <- lm(Google_Sentiment ~ View_Score + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State, data=ventyx_df_NU_google_Sen)

summary(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State)

# dwplot(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State)+
#   theme_bw()+
#   labs(x = "timeline")
# 

regressionB <- tidy(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State)

dwplot(regressionB[!grepl("^State*", regressionB$term),] %>% 
   relabel_predictors(renamed_variables), 
   dot_args = list(colour = "firebrick"),
   whisker_args = list(colour = "firebrick", lwd = 0.5)
   )+
  labs(x = "Sentiment score", y = "")+
  ggplot2::geom_vline(xintercept = 0, colour = "black", linetype = 2)+
  theme_classic()+
  ggtitle("Regression output: all variables including NU and Google \n scores, interaction terms. N/A scores coded as 0:")+
  theme(legend.background = element_rect(colour="grey80"))



#Results no significance found
#H_L_W: 0.154
####
# 01/21/18 - difference in results with changes to ventyx df and text from NU:
#H_L_W: 0.16372
#capacity: 0.11

```

```{r R13}
####13. Timeline on left-hand side, using NU sentiment. Interaction term between NU and high-low impact. Environmental memberships variable added. N/A scores coded as 0
reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter <- lm(TimelineDays ~ View_Score + NU_Sentiment + members_env + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*NU_Sentiment, data=ventyx_df_NU_google_Sen)

summary(reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter)
dwplot(reg_NSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter)
```


```{r R14}
####14. Timeline on left-hand side, using Google sentiment. Interaction term between Google and high-low impact. Environmental memberships variable added. N/A scores coded as 0
reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter <- lm(TimelineDays ~ View_Score + Google_Sentiment + members_env + PopDensity_mi + Household_MedianIncome + Capacity + H_L_W + State + H_L_W*Google_Sentiment, data=ventyx_df_NU_google_Sen)

summary(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter)
dwplot(reg_GSen_VS_PopDen_MedianIncome_Cap_HL_State_Memberships_Inter)
```


## OTHER 

#### Headlines
```{r Extract_headlines_id_duplicates}

#can ignore for regression

## extract headlines 
projects_pdf_titles <- project_pdfs_full_text %>% 
  mutate(title_extract = grepl("\r\n \\d.(.*?) Client/Matter:", text, ignore.case = FALSE)) 
# %>% 
  # filter(title_extract == TRUE) %>% 
  # mutate(title_match = strapply(text, "\r\n \\d.(.*?) Client/Matter:"))


  
## Identify duplicate headlines 
  
  

  
```


#### Populate with negative words 
```{r negative_words}
negative_words <- paste0(c('negative|postpone|against|delay|lawsuit|litigation|protest|^cost|^stop'))
# Function to replace `character(0)` with NAs as NULL values are dropped when flattening list
# inspired by: https://colinfay.me/purrr-set-na/
charnull_set <- function(x) {
  p <- purrr::as_mapper(~identical(., character(0)))
  x[p(x)] <- NA
  return(x)
}
projects_pdftext_3 <- projects_pdftext_NU %>%
  mutate(query_hits = str_extract_all(text, pattern = regex(negative_words, ignore_case=TRUE)) %>%  # Extract all the keywords
           map(~charnull_set(.x)) %>%   # Replace character(0) with NAs
           map_chr(~glue::glue_collapse(.x, sep = ";")) %>%   # collapse the multiple hits
           tolower) # all our keywords are lower case
projects_pdftext_grouped <- projects_pdftext_2 %>%
  group_by(document, query_hits)
# OR 
my_data1grouped <- my_data1 %>%
  group_by(document, query_hits)%>%
  summarise(word_list = glue::glue_collapse(query_hits, sep = ";"))
View(projects_pdftext_2)
```